deVision v0.4 (github model-v0.4, 0df9a73b091e0e8167f554da83a4034140354443)

MANIFEST.json

@@ -1,71 +1,71 @@
 {
  ".gitattributes": {
-  "size": 92,
-  "sha256": "127386e47ae53f224aaf02383189e1b11e6d52289775b426655aa5d87d7e953c"
+  "size": 144,
+  "sha256": "0f1bfba803b20cd477b98e2185c8ffdd6059059d02413a196f234e3b1d5b930e"
  },
  "LICENSE": {
   "size": 11358,
   "sha256": "cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30"
  },
  "README.md": {
-  "size": 12369,
-  "sha256": "9e5087ee27134943c32a5a3a3bd3bf90bbceffc942304dc00d7a5638a743a2e8"
+  "size": 12869,
+  "sha256": "1f4ea72cc502f98430ba3ec55c0df23445e6ec48aa0da97ffe7b6b7a6bb67e5d"
  },
  "assets/logo.png": {
   "size": 450887,
   "sha256": "4c1123544e9b08d79eb84ef399b244907249e5353bcbd5167667400e3a62a0f7"
  },
  "config.json": {
-  "size": 420,
-  "sha256": "f51a34393cc41ea56012bfd29eb35fbe9197934d897f24085eefb5e9a626452a"
+  "size": 510,
+  "sha256": "55d9bcdd737b57a1779f4b931232e36ea308e96ae3175c8365f214d32ea79365"
  },
  "encoder/config.json": {
   "size": 2084,
   "sha256": "5268d24ad3b77c8151de5dcb0762ba4391619aad9ab0bda33e36fb083cfeae6d"
  },
- "evaluation/accept_vs_v0.2.json": {
-  "size": 13821,
-  "sha256": "8e63d73d13f985160e787b741c41895c029ef00eefff8895ad94b2a9d026184c"
+ "evaluation/accept_vs_v0.3.json": {
+  "size": 14928,
+  "sha256": "91649ca42013f55fda7177b05f3c0dd6110bf829817e80e7165f83d2108c5d21"
+ },
+ "evaluation/accept_vs_v0.3.md": {
+  "size": 2867,
+  "sha256": "e3cd62bc1bd9f7bf5cab0ac322dd739e168529ea0163235536a890f6f5a73b71"
  },
  "evaluation/adoption.md": {
-  "size": 2269,
-  "sha256": "0b044443cf371dbe8e017cb2fc705b1b841f58e46059c1cb272c161bfecfe988"
+  "size": 1494,
+  "sha256": "7d4cd1f41d6c669c696a3c61f143d12dd4be5570f30ba96f6bc8359b61c43c35"
  },
  "evaluation/cpu_latency.json": {
-  "size": 8940,
-  "sha256": "1740cad45a9547c74c21e678f5ac51c906bb46890348472a1be9bebfa83328d8"
+  "size": 5885,
+  "sha256": "d1863034969220c6013a83bba01e26c7ed9bb3d8595e1603cb447788c184e481"
  },
  "evaluation/results.json": {
-  "size": 3567,
-  "sha256": "b9733f4f573a2652b079f7b899c71ca0290c0b199a85f34c0607bb0e577d6b22"
+  "size": 3243,
+  "sha256": "3cd6fd7e67c2557760db0915505ece93ae306018998707e7378d745e81c9f779"
  },
  "evaluation/results.md": {
-  "size": 2077,
-  "sha256": "a133c35428d625adde778a9e0ff2898aa227ea854156a980eef7b5561aa6edad"
- },
- "evaluation/seeded_from.json": {
-  "size": 143,
-  "sha256": "c167f7139c72fb0c70c200e60448af1a5302ed4cc218e609c53e10a0c3d993e4"
+  "size": 1901,
+  "sha256": "6538bfa316bf7f8b4f0d1458b3d5e44ff77b8c5e738982898cbdf171ab463d17"
  },
  "evaluation/training.yaml": {
-  "size": 3979,
-  "sha256": "a7733d9bc41db37b133868839c3080683167374eaf919c3cdd29d0575a894401"
+  "size": 9103,
+  "sha256": "50f3c8ca7428d1f25c4db6de014e8f58e15864d426db8a5d46e466ebbeb57bc3"
  },
  "model.safetensors": {
   "size": 2073754784,
-  "sha256": "110fea95889947c373258c88a396ad3eec574971815ee437b9c7cc3df8a5d2d5"
+  "sha256": "fda0f969a79cf77e6bb9288ddd38e4bb377f3f677018e67275ef927bf76b0397"
  },
  "provenance.json": {
-  "size": 4546,
-  "sha256": "d135fc16bd5e92fb5556972cd47ae0d783ed07d4715a8dde5b255c7f5e1d2f55"
+  "size": 3160,
+  "sha256": "ed0f1165ac62178d99b08bd5a0aea830fcf433257c5f0d5436c35497980f9288"
  },
  "tokenizer/tokenizer.json": {
   "size": 3583228,
   "sha256": "6c8aaa9a542084f2457eab775d4eeb51f92a70c0fd9de28d5edb0ddec3c08d30"
  },
  "tokenizer/tokenizer_config.json": {
-  "size": 379,
-  "sha256": "5926e6ec4294f80294bd98d9176defa9fbae7f140525d06a1da987488e74a973"
+  "size": 378,
+  "sha256": "68acf857b869285d8c3ac2757ddd26e6f8030fb275d6800131521eb736084608"
  },
  "vision/config.json": {
   "size": 361,

README.md

@@ -29,7 +29,7 @@ tags:
 - rlcd
 - cpu
 model-index:
-- name: deVision v0.3
+- name: deVision v0.4
   results:
   - task:
       type: visual-question-answering
@@ -38,9 +38,9 @@ model-index:
       type: test_exist
     metrics:
     - type: accuracy
-      value: 0.9473
+      value: 0.9554
     - type: ece
-      value: 0.0341
+      value: 0.0924
   - task:
       type: visual-question-answering
     dataset:
@@ -48,9 +48,9 @@ model-index:
       type: test_vqa_choice
     metrics:
     - type: accuracy
-      value: 0.9092
+      value: 0.905
     - type: ece
-      value: 0.0384
+      value: 0.0534
   - task:
       type: visual-question-answering
     dataset:
@@ -58,9 +58,9 @@ model-index:
       type: test_size
     metrics:
     - type: accuracy
-      value: 0.8718
+      value: 0.8698
     - type: ece
-      value: 0.0295
+      value: 0.2345
   - task:
       type: visual-question-answering
     dataset:
@@ -68,9 +68,9 @@ model-index:
       type: bench_pope
     metrics:
     - type: accuracy
-      value: 0.8496
+      value: 0.8653
     - type: ece
-      value: 0.0393
+      value: 0.0483
   - task:
       type: visual-question-answering
     dataset:
@@ -78,9 +78,9 @@ model-index:
       type: test_gqa
     metrics:
     - type: accuracy
-      value: 0.7903
+      value: 0.8428
     - type: ece
-      value: 0.0396
+      value: 0.0857
   - task:
       type: visual-question-answering
     dataset:
@@ -88,9 +88,9 @@ model-index:
       type: test_position
     metrics:
     - type: accuracy
-      value: 0.8846
+      value: 0.9381
     - type: ece
-      value: 0.029
+      value: 0.0468
   - task:
       type: visual-question-answering
     dataset:
@@ -98,9 +98,9 @@ model-index:
       type: test_vqa_yesno
     metrics:
     - type: accuracy
-      value: 0.704
+      value: 0.722
     - type: ece
-      value: 0.0265
+      value: 0.0552
   - task:
       type: visual-question-answering
     dataset:
@@ -108,9 +108,9 @@ model-index:
       type: test_relation
     metrics:
     - type: accuracy
-      value: 0.7549
+      value: 0.9167
     - type: ece
-      value: 0.0384
+      value: 0.0367
   - task:
       type: visual-question-answering
     dataset:
@@ -118,9 +118,9 @@ model-index:
       type: test_vsr
     metrics:
     - type: accuracy
-      value: 0.6615
+      value: 0.7306
     - type: ece
-      value: 0.0465
+      value: 0.0734
   - task:
       type: visual-question-answering
     dataset:
@@ -128,19 +128,9 @@ model-index:
       type: test_v7w
     metrics:
     - type: accuracy
-      value: 0.721
+      value: 0.783
     - type: ece
-      value: 0.0411
-  - task:
-      type: visual-question-answering
-    dataset:
-      name: Fresh counting test (unseen pictures)
-      type: test_count_fresh
-    metrics:
-    - type: accuracy
-      value: 0.7633
-    - type: ece
-      value: 0.054
+      value: 0.073
   - task:
       type: visual-question-answering
     dataset:
@@ -148,9 +138,9 @@ model-index:
       type: bench_countbench
     metrics:
     - type: accuracy
-      value: 0.3564
+      value: 0.501
     - type: ece
-      value: 0.0523
+      value: 0.1655
   - task:
       type: visual-question-answering
     dataset:
@@ -158,9 +148,9 @@ model-index:
       type: test_position_flip
     metrics:
     - type: accuracy
-      value: 0.8469
+      value: 0.9328
     - type: ece
-      value: 0.0444
+      value: 0.0532
   - task:
       type: visual-question-answering
     dataset:
@@ -168,19 +158,9 @@ model-index:
       type: test_relation_flip
     metrics:
     - type: accuracy
-      value: 0.6977
+      value: 0.9143
     - type: ece
-      value: 0.03
-  - task:
-      type: visual-question-answering
-    dataset:
-      name: GQA binding test
-      type: test_bind_gqa
-    metrics:
-    - type: accuracy
-      value: 0.6765
-    - type: ece
-      value: 0.0277
+      value: 0.0485
   - task:
       type: visual-question-answering
     dataset:
@@ -188,12 +168,12 @@ model-index:
       type: bench_whatsup
     metrics:
     - type: accuracy
-      value: 0.2866
+      value: 0.4671
     - type: ece
-      value: 0.2447
+      value: 0.0926
 ---
 
-# deVision v0.3
+# deVision v0.4
 
 <p align="center"><img src="assets/logo.png" alt="deVision" width="200"></p>
 
@@ -240,35 +220,33 @@ Accuracy through `decide` with the fitted temperatures. *Mismatched*: the same q
 
 | Set | Questions | Accuracy | Mismatched | NLL | ECE |
 |---|---:|---:|---:|---:|---:|
-| COCO object presence | 1,120 | 0.947 | 0.504 | 0.163 | 0.034 |
-| VQAv2 multiple choice | 1,420 | 0.909 | 0.380 | 0.271 | 0.038 |
-| COCO size | 1,100 | 0.872 | 0.511 | 0.336 | 0.029 |
-| POPE, project filtered set | 8,676 | 0.850 | 0.527 | 0.368 | 0.039 |
-| GQA val subset | 992 | 0.790 | 0.550 | 0.410 | 0.040 |
-| COCO position | 1,274 | 0.885 | 0.493 | 0.288 | 0.029 |
-| VQAv2 yes/no subset | 1,000 | 0.704 | 0.512 | 0.574 | 0.026 |
-| COCO relative position | 1,950 | 0.755 | 0.496 | 0.490 | 0.038 |
-| VSR, project held-out split | 904 | 0.661 | 0.487 | 0.620 | 0.046 |
-| Visual7W, project held-out split | 1,000 | 0.721 | 0.419 | 0.673 | 0.041 |
-| Fresh counting test (unseen pictures) | 600 | 0.763 | 0.405 | 0.574 | 0.054 |
-| CountBench, 0–10 choices | 491 | 0.356 | 0.104 | 1.602 | 0.052 |
-| COCO mirrored position | 640 | 0.847 | 0.477 | 0.359 | 0.044 |
-| COCO mirrored relative position | 1,492 | 0.698 | 0.507 | 0.545 | 0.030 |
-| GQA binding test | 1,484 | 0.676 | 0.498 | 0.559 | 0.028 |
-| What'sUp | 820 | 0.287 | 0.245 | 1.491 | 0.245 |
+| COCO object presence | 1,120 | 0.955 | 0.497 | 0.196 | 0.092 |
+| VQAv2 multiple choice | 1,453 | 0.905 | 0.370 | 0.280 | 0.053 |
+| COCO size | 1,198 | 0.870 | 0.516 | 0.506 | 0.234 |
+| POPE, project filtered set | 9,000 | 0.865 | 0.520 | 0.363 | 0.048 |
+| GQA val subset | 986 | 0.843 | 0.546 | 0.383 | 0.086 |
+| COCO position | 1,276 | 0.938 | 0.509 | 0.238 | 0.047 |
+| VQAv2 yes/no subset | 1,000 | 0.722 | 0.500 | 0.571 | 0.055 |
+| COCO relative position | 2,366 | 0.917 | 0.480 | 0.281 | 0.037 |
+| VSR, project held-out split | 902 | 0.731 | 0.460 | 0.591 | 0.073 |
+| Visual7W, project held-out split | 1,000 | 0.783 | 0.442 | 0.597 | 0.073 |
+| CountBench, 0–10 choices | 491 | 0.501 | 0.102 | 1.387 | 0.166 |
+| COCO mirrored position | 640 | 0.933 | 0.503 | 0.283 | 0.053 |
+| COCO mirrored relative position | 1,786 | 0.914 | 0.494 | 0.300 | 0.049 |
+| What'sUp | 820 | 0.467 | 0.233 | 1.214 | 0.093 |
 
 Against Laya Vision 201M on the same questions (its published per-question predictions; difference in points, 95% interval from paired resampling by picture):
 
 | Set | Questions | Laya Vision | deVision | Difference |
 |---|---:|---:|---:|---|
-| VQAv2 yes/no | 4,887 | 0.717 | 0.725 | +0.8 [-0.8, +2.4] |
-| A-OKVQA | 1,138 | 0.598 | 0.613 | +1.4 [-1.8, +4.7] |
-| ScienceQA with images | 2,097 | 0.824 | 0.789 | -3.5 [-5.6, -1.4] |
-| ScienceQA natural science, needs the picture | 323 | 0.700 | 0.495 | -20.4 [-27.2, -13.3] |
+| VQAv2 yes/no | 4,887 | 0.717 | 0.756 | +3.9 [+2.4, +5.3] |
+| A-OKVQA | 1,138 | 0.598 | 0.634 | +3.5 [+0.3, +6.6] |
+| ScienceQA with images | 2,097 | 0.824 | 0.821 | -0.3 [-2.2, +1.7] |
+| ScienceQA natural science, needs the picture | 323 | 0.700 | 0.638 | -6.2 [-12.7, +0.0] |
 
-On the full POPE random / popular / adversarial sets (3,000 questions each) deVision scores **0.880 / 0.859 / 0.805**, against Laya Vision's published **0.836 / 0.819 / 0.777** (aggregate scores only, not paired).
+On the full POPE random / popular / adversarial sets (3,000 questions each) deVision scores **0.899 / 0.873 / 0.824**, against Laya Vision's published **0.836 / 0.819 / 0.777** (aggregate scores only, not paired).
 
-CPU latency: P50 175 ms, P95 190 ms (Darwin arm64, 4 threads, FP32, one question per request, warm-up excluded).
+CPU latency: P50 289 ms, P95 301 ms (Linux x86_64, 4 threads, FP32, one question per request, warm-up excluded).
 
 ## Model
 
@@ -278,28 +256,54 @@ CPU latency: P50 175 ms, P95 190 ms (Darwin arm64, 4 threads, FP32, one question
 | Decision | Laya-initialised ModernBERT-large and Laya's decision head; each option is scored at its `[MASK]`, then a temperature-scaled softmax |
 | Size | About 518M parameters, FP32, 2.07 GB (LoRA merged) |
 
-This release continues v0.2 with 81,875 questions of counting, object presence, mirrored spatial questions, VSR and broad replay, for one 10,235-step schedule (seed 1). The first 2,000 steps were run as exp19; r15 resumes that exact training state. SigLIP2's top four encoder layers and final LayerNorm were fine-tuned at 5e-6 alongside the projector, decision head and ModernBERT LoRA. LoRA is merged into the released weights. The input remains 256×256 with 64 visual tokens; S2 question conditioning is **not** enabled.
+This release continues v0.3 in three steps on one RTX 5070 (bf16), with SigLIP2's top four encoder layers and final LayerNorm fine-tuned at 5e-6 alongside the projector, decision head and ModernBERT LoRA (merged into the released weights). The input remains 256×256 with 64 visual tokens; the architecture is unchanged.
+
+1. **742,238 questions, one pass from v0.3.** VQAv2 (annotators' answers as soft targets) and GQA (balanced train, all shards, open questions turned into choices), about 190,000 each, plus targeted data for the abilities v0.3 lacked:
+   - all of Visual7W telling (52,390 after filtering);
+   - caption-choice spatial relations (on / under / in front of / behind / left / right) from Visual Genome scene graphs, with every distractor checked against the boxes;
+   - counting on COCO and TallyQA, always with the eleven options 0–10;
+   - CLEVR, IconQA;
+   - programmatically drawn magnet and solution diagrams in ScienceQA's style;
+   - replay of v0.3's other abilities.
 
-Temperatures were fitted on the designated calibration set, by question type and option-count bucket. Missing buckets fall back to the question-type temperature:
+   Every picture of every evaluation set was excluded by id and image hash. Choice questions v0.3 still answered confidently with the wrong picture were dropped.
+2. **A short science-heavy continuation (123,084 questions):** all ScienceQA / AI2D / TQA training questions ×3, IconQA, the magnet diagrams and 40,000 replay. This also repaired an exclusion bug that had dropped about half of the ScienceQA training questions in step 1.
+3. **A short continuation (104,331 questions) with presence and size labels softened from 0 / 1 to 0.9 / 0.1,** to stop over-confident presence answers (POPE).
+
+Temperatures were fitted on a development mix with balanced sources, by question type and option-count bucket. Missing buckets fall back to the question-type temperature:
 
 | Bucket | Temperature |
 |---|---:|
-| Choice, 11+ options | 1.2012 |
-| Choice, 2 options | 1.9488 |
-| Choice, 3–5 options | 1.7575 |
-| Noul (yes / no) | 1.8258 |
-| Choice, any other count | 1.5922 |
+| Choice, 2 options | 3.5631 |
+| Choice, 3–5 options | 2.1146 |
+| Noul (yes / no) | 1.0895 |
+| Choice, any other count | 2.3642 |
+
+The owner adopted this model although one prewritten acceptance rule (calibration) is not met. See the [adoption decision](evaluation/adoption.md), the [full comparison with v0.3](evaluation/accept_vs_v0.3.md) and the [training configurations](evaluation/training.yaml).
 
-The owner adopted v0.3 as a trade-off: counting, relations and ScienceQA improved, while four old-capability protection conditions were not met. The original automated verdict remains `candidate=false`. See the [adoption decision and v0.2 comparison](evaluation/adoption.md), [original acceptance report](evaluation/accept_vs_v0.2.json) and [training configuration](evaluation/training.yaml).
+Against v0.3 on the same questions (accuracy points, 95% intervals), 22 of 23 judged sets improve and none is significantly worse:
+
+| Set | Change |
+|---|---|
+| mirrored relations | +21.2 |
+| What'sUp | +18.0 |
+| CountBench | +14.5 |
+| Visual7W | +7.2 |
+| VSR | +7.0 |
+| GQA | +6.1 |
+| CLEVR-CoGenT B | +4.0 |
+| ScienceQA | +3.1 [+1.6, +4.8] |
+| MMBench-en dev | +2.3 |
+| POPE | +1.7 |
 
 ## Limitations
 
-- English, one image, yes/no and choice questions only. `score` is not implemented. At 256×256, small text and fine diagram details are hard to read.
-- CountBench improved by 16.7 points over v0.2, but absolute accuracy is only 35.6%. Answer-order sensitivity increased (39.5% change their answer when options are reversed, versus 16.9%); this is not a general-purpose accurate counter.
-- VSR accuracy is 1.8 points lower and A-OKVQA 1.3 points lower than v0.2; their overall intervals contain zero, but non-inferiority was not established. VSR positive recall fell by 5.3 points. A-OKVQA and What'sUp have worse probability quality than v0.2. No prewritten NLL warning was triggered; that does not mean all probabilities improved.
-- Diagram, chart and map comparisons and question-to-region binding remain weak. On natural-science ScienceQA questions that require the picture, accuracy is 49.5% versus Laya Vision's 70.0%.
-- Calibration reflects the calibration distribution, not every deployment domain. Validate thresholds on your own examples.
-- Test sets have repeatedly informed experiment and release decisions. This is one training seed, not a fresh independent confirmation. Gains cannot be attributed solely to unfreezing the vision encoder.
+- **Scope.** English, one image, yes/no and choice questions only; `score` is not implemented. At 256×256, small text and fine diagram details are hard to read.
+- **Counting.** It improved (CountBench 35.6% → 50.1%, COCO counts 30.6% → 34.5%), but this is still not an accurate counter, especially above five.
+- **Calibration.** Probabilities are slightly under-confident on easy presence and size questions (size: mean confidence 0.64 at 0.87 accuracy) and slightly over-confident on MME. POPE calibration is on par with v0.3. Validate thresholds on your own examples.
+- **External benchmarks.** MME (−0.6), NaturalBench (+0.3), CV-Bench (+0.7) and A-OKVQA (+2.1) did not change significantly.
+- **Diagrams.** Diagram reading that depends on small printed numbers (particle speeds, magnet distances) remains weak. The drawn solution diagrams did not transfer to ScienceQA's beakers.
+- **Test-set reuse.** Test sets informed several decisions in this cycle. This is one training seed, not an independent confirmation.
 
 ## Licence
 
@@ -307,4 +311,4 @@ Weights and code: Apache-2.0, like the three base models ([Laya](https://hugging
 
 ## Links
 
-[Code and training records](https://github.com/byebyebruce/devision) (this release: tag `model-v0.3`) · [evaluation details](evaluation/results.md) · [provenance](provenance.json) · [Laya Vision](https://github.com/r33drichards/laya-vision)
+[Code and training records](https://github.com/byebyebruce/devision) (this release: tag `model-v0.4`) · [evaluation details](evaluation/results.md) · [provenance](provenance.json) · [Laya Vision](https://github.com/r33drichards/laya-vision)

config.json

@@ -1,5 +1,5 @@
 {
- "model_name": "devision-v0.3",
+ "model_name": "devision-v0.4",
  "image_size": 256,
  "visual_shuffle": 2,
  "max_len": 512,
@@ -7,15 +7,19 @@
  "head_layers": 2,
  "n_act": 2,
  "temperature": [
-  1.5922062397003174,
+  2.3641862869262695,
   1.0,
-  1.825764536857605
+  1.0895380973815918
  ],
  "temperature_by_options": {
-  "choice:11+": 1.201171636581421,
-  "choice:2": 1.9487814903259277,
-  "choice:3-5": 1.7575252056121826,
-  "noul:2": 1.825764536857605
+  "choice:2": 3.563119649887085,
+  "choice:3-5": 2.1145524978637695,
+  "noul:2": 1.0895380973815918
  },
- "question_conditioning": false
+ "question_conditioning": false,
+ "interpolate_pos_encoding": false,
+ "local_views": 0,
+ "local_overlap": 0.0,
+ "local_source": "",
+ "number_reader": false
 }

evaluation/accept_vs_v0.3.json

@@ -0,0 +1,734 @@
+{
+ "rows": {
+  "test_vsr": {
+   "acc": {
+    "metric": "accuracy",
+    "questions": 902,
+    "pictures": 547,
+    "a": 0.6607538802660754,
+    "b": 0.7305986696230599,
+    "difference": 0.06984478935698446,
+    "ci95": [
+     0.03618421052631582,
+     0.10647639956092214
+    ],
+    "resamples": 2000
+   },
+   "nll": {
+    "metric": "nll",
+    "questions": 902,
+    "pictures": 547,
+    "a": 0.6155837500135749,
+    "b": 0.5904692457609072,
+    "difference": -0.025114504252667702,
+    "ci95": [
+     -0.06730200801082653,
+     0.01835022339389425
+    ],
+    "resamples": 2000
+   }
+  },
+  "test_v7w": {
+   "acc": {
+    "metric": "accuracy",
+    "questions": 1000,
+    "pictures": 1000,
+    "a": 0.711,
+    "b": 0.783,
+    "difference": 0.07200000000000006,
+    "ci95": [
+     0.04800000000000004,
+     0.09600000000000009
+    ],
+    "resamples": 2000
+   },
+   "nll": {
+    "metric": "nll",
+    "questions": 1000,
+    "pictures": 1000,
+    "a": 0.7061062017078975,
+    "b": 0.5966842790244825,
+    "difference": -0.109421922683415,
+    "ci95": [
+     -0.14233793828593067,
+     -0.07693998792887025
+    ],
+    "resamples": 2000
+   }
+  },
+  "bench_pope": {
+   "acc": {
+    "metric": "accuracy",
+    "questions": 9000,
+    "pictures": 500,
+    "a": 0.8481111111111111,
+    "b": 0.8653333333333333,
+    "difference": 0.01722222222222214,
+    "ci95": [
+     0.00855555555555565,
+     0.02577777777777779
+    ],
+    "resamples": 2000
+   },
+   "nll": {
+    "metric": "nll",
+    "questions": 9000,
+    "pictures": 500,
+    "a": 0.37213111549634703,
+    "b": 0.36302066313255416,
+    "difference": -0.009110452363792876,
+    "ci95": [
+     -0.02169886047608327,
+     0.003435790059213739
+    ],
+    "resamples": 2000
+   }
+  },
+  "test_vqa_yesno": {
+   "acc": {
+    "metric": "accuracy",
+    "questions": 1000,
+    "pictures": 977,
+    "a": 0.683,
+    "b": 0.722,
+    "difference": 0.038999999999999924,
+    "ci95": [
+     0.015090543259557276,
+     0.06480558325024932
+    ],
+    "resamples": 2000
+   },
+   "nll": {
+    "metric": "nll",
+    "questions": 1000,
+    "pictures": 977,
+    "a": 0.5941348191891218,
+    "b": 0.571303429093119,
+    "difference": -0.022831390096002768,
+    "ci95": [
+     -0.04409385128971266,
+     -0.0019899478295204442
+    ],
+    "resamples": 2000
+   }
+  },
+  "test_vqa_choice": {
+   "acc": {
+    "metric": "accuracy",
+    "questions": 1453,
+    "pictures": 1394,
+    "a": 0.8823124569855472,
+    "b": 0.9050240880935995,
+    "difference": 0.0227116311080523,
+    "ci95": [
+     0.007686932215234088,
+     0.03782668500687758
+    ],
+    "resamples": 2000
+   },
+   "nll": {
+    "metric": "nll",
+    "questions": 1453,
+    "pictures": 1394,
+    "a": 0.2975529214462992,
+    "b": 0.28024055404603077,
+    "difference": -0.017312367400268425,
+    "ci95": [
+     -0.04801307597139293,
+     0.017763019358089716
+    ],
+    "resamples": 2000
+   }
+  },
+  "test_gqa": {
+   "acc": {
+    "metric": "accuracy",
+    "questions": 986,
+    "pictures": 880,
+    "a": 0.781947261663286,
+    "b": 0.8427991886409736,
+    "difference": 0.0608519269776876,
+    "ci95": [
+     0.03567787971457692,
+     0.08611955420466055
+    ],
+    "resamples": 2000
+   },
+   "nll": {
+    "metric": "nll",
+    "questions": 986,
+    "pictures": 880,
+    "a": 0.4363967446492391,
+    "b": 0.38284475493705167,
+    "difference": -0.05355198971218744,
+    "ci95": [
+     -0.08716986207647759,
+     -0.02041387915707188
+    ],
+    "resamples": 2000
+   }
+  },
+  "test_exist": {
+   "acc": {
+    "metric": "accuracy",
+    "questions": 1120,
+    "pictures": 1104,
+    "a": 0.9392857142857143,
+    "b": 0.9553571428571429,
+    "difference": 0.016071428571428625,
+    "ci95": [
+     0.0026929982046678402,
+     0.02930728241563063
+    ],
+    "resamples": 2000
+   },
+   "nll": {
+    "metric": "nll",
+    "questions": 1120,
+    "pictures": 1104,
+    "a": 0.16983095379080976,
+    "b": 0.19567675111207672,
+    "difference": 0.02584579732126696,
+    "ci95": [
+     0.007774706565661066,
+     0.04337850870878224
+    ],
+    "resamples": 2000
+   }
+  },
+  "test_position": {
+   "acc": {
+    "metric": "accuracy",
+    "questions": 1276,
+    "pictures": 1232,
+    "a": 0.8863636363636364,
+    "b": 0.9380877742946708,
+    "difference": 0.051724137931034475,
+    "ci95": [
+     0.03299293008641002,
+     0.07092198581560283
+    ],
+    "resamples": 2000
+   },
+   "nll": {
+    "metric": "nll",
+    "questions": 1276,
+    "pictures": 1232,
+    "a": 0.27279601591009106,
+    "b": 0.23840018591039677,
+    "difference": -0.03439582999969429,
+    "ci95": [
+     -0.08219989842410494,
+     0.015482905705434202
+    ],
+    "resamples": 2000
+   }
+  },
+  "test_relation": {
+   "acc": {
+    "metric": "accuracy",
+    "questions": 2366,
+    "pictures": 1768,
+    "a": 0.7489433643279797,
+    "b": 0.9167371090448013,
+    "difference": 0.16779374471682162,
+    "ci95": [
+     0.1491782553729456,
+     0.18608549874266556
+    ],
+    "resamples": 2000
+   },
+   "nll": {
+    "metric": "nll",
+    "questions": 2366,
+    "pictures": 1768,
+    "a": 0.4909602253543323,
+    "b": 0.2805283450848128,
+    "difference": -0.21043188026951953,
+    "ci95": [
+     -0.25335914427956835,
+     -0.16455708571446082
+    ],
+    "resamples": 2000
+   }
+  },
+  "test_size": {
+   "acc": {
+    "metric": "accuracy",
+    "questions": 1198,
+    "pictures": 1081,
+    "a": 0.8564273789649416,
+    "b": 0.8697829716193656,
+    "difference": 0.013355592654423987,
+    "ci95": [
+     -0.002523128679562725,
+     0.030252100840336138
+    ],
+    "resamples": 2000
+   },
+   "nll": {
+    "metric": "nll",
+    "questions": 1198,
+    "pictures": 1081,
+    "a": 0.35047560484582363,
+    "b": 0.5065374303761236,
+    "difference": 0.1560618255303,
+    "ci95": [
+     0.12203261708552204,
+     0.18841077280205148
+    ],
+    "resamples": 2000
+   }
+  },
+  "test_position_flip": {
+   "acc": {
+    "metric": "accuracy",
+    "questions": 640,
+    "pictures": 636,
+    "a": 0.828125,
+    "b": 0.9328125,
+    "difference": 0.10468750000000004,
+    "ci95": [
+     0.07931570762052875,
+     0.1316614420062696
+    ],
+    "resamples": 2000
+   },
+   "nll": {
+    "metric": "nll",
+    "questions": 640,
+    "pictures": 636,
+    "a": 0.3792944504698811,
+    "b": 0.28297517066909245,
+    "difference": -0.09631927980078864,
+    "ci95": [
+     -0.16729839354099188,
+     -0.019257792204828483
+    ],
+    "resamples": 2000
+   }
+  },
+  "test_relation_flip": {
+   "acc": {
+    "metric": "accuracy",
+    "questions": 1786,
+    "pictures": 1460,
+    "a": 0.7021276595744681,
+    "b": 0.9143337066069429,
+    "difference": 0.2122060470324748,
+    "ci95": [
+     0.18990246701090074,
+     0.2346089850249583
+    ],
+    "resamples": 2000
+   },
+   "nll": {
+    "metric": "nll",
+    "questions": 1786,
+    "pictures": 1460,
+    "a": 0.5234402751432231,
+    "b": 0.3004006765189471,
+    "difference": -0.22303959862427603,
+    "ci95": [
+     -0.27563722667679236,
+     -0.16170787803919912
+    ],
+    "resamples": 2000
+   }
+  },
+  "bench_lv_vqav2_yesno": {
+   "acc": {
+    "metric": "accuracy",
+    "questions": 4887,
+    "pictures": 2069,
+    "a": 0.7249846531614488,
+    "b": 0.7558829547779824,
+    "difference": 0.03089830161653362,
+    "ci95": [
+     0.01905156347069792,
+     0.04241915161696774
+    ],
+    "resamples": 2000
+   },
+   "nll": {
+    "metric": "nll",
+    "questions": 4887,
+    "pictures": 2069,
+    "a": 0.551730515857613,
+    "b": 0.5256937856914787,
+    "difference": -0.02603673016613428,
+    "ci95": [
+     -0.03484135061656102,
+     -0.01729670636981273
+    ],
+    "resamples": 2000
+   }
+  },
+  "bench_lv_aokvqa": {
+   "acc": {
+    "metric": "accuracy",
+    "questions": 1138,
+    "pictures": 1138,
+    "a": 0.6124780316344464,
+    "b": 0.6335676625659051,
+    "difference": 0.02108963093145877,
+    "ci95": [
+     -0.0035149384885764245,
+     0.0448154657293498
+    ],
+    "resamples": 2000
+   },
+   "nll": {
+    "metric": "nll",
+    "questions": 1138,
+    "pictures": 1138,
+    "a": 0.9357384620349536,
+    "b": 0.9235836608133525,
+    "difference": -0.012154801221601064,
+    "ci95": [
+     -0.050025256755278025,
+     0.02747038629963383
+    ],
+    "resamples": 2000
+   }
+  },
+  "bench_lv_scienceqa": {
+   "acc": {
+    "metric": "accuracy",
+    "questions": 2097,
+    "pictures": 2097,
+    "a": 0.7892226990939437,
+    "b": 0.8206962327134001,
+    "difference": 0.03147353361945637,
+    "ci95": [
+     0.015736766809728242,
+     0.047687172150691515
+    ],
+    "resamples": 2000
+   },
+   "nll": {
+    "metric": "nll",
+    "questions": 2097,
+    "pictures": 2097,
+    "a": 0.47098928326742956,
+    "b": 0.46387570466731803,
+    "difference": -0.007113578600111525,
+    "ci95": [
+     -0.036043017961029544,
+     0.02201631592471548
+    ],
+    "resamples": 2000
+   }
+  },
+  "bench_whatsup": {
+   "acc": {
+    "metric": "accuracy",
+    "questions": 820,
+    "pictures": 820,
+    "a": 0.2865853658536585,
+    "b": 0.46707317073170734,
+    "difference": 0.18048780487804883,
+    "ci95": [
+     0.1353658536585366,
+     0.22073170731707314
+    ],
+    "resamples": 2000
+   },
+   "nll": {
+    "metric": "nll",
+    "questions": 820,
+    "pictures": 820,
+    "a": 1.490915758604817,
+    "b": 1.2141925113554008,
+    "difference": -0.2767232472494161,
+    "ci95": [
+     -0.335244532597462,
+     -0.219079539978984
+    ],
+    "resamples": 2000
+   }
+  },
+  "bench_countbench": {
+   "acc": {
+    "metric": "accuracy",
+    "questions": 491,
+    "pictures": 491,
+    "a": 0.3564154786150713,
+    "b": 0.5010183299389002,
+    "difference": 0.14460285132382888,
+    "ci95": [
+     0.09368635437881867,
+     0.1955193482688391
+    ],
+    "resamples": 2000
+   },
+   "nll": {
+    "metric": "nll",
+    "questions": 491,
+    "pictures": 491,
+    "a": 1.6021549698788784,
+    "b": 1.3872406640068242,
+    "difference": -0.21491430587205418,
+    "ci95": [
+     -0.2562705789097375,
+     -0.17616865330760545
+    ],
+    "resamples": 2000
+   }
+  },
+  "bench_cogent_b": {
+   "acc": {
+    "metric": "accuracy",
+    "questions": 2000,
+    "pictures": 2000,
+    "a": 0.5775,
+    "b": 0.618,
+    "difference": 0.04049999999999998,
+    "ci95": [
+     0.01649999999999996,
+     0.0655
+    ],
+    "resamples": 2000
+   },
+   "nll": {
+    "metric": "nll",
+    "questions": 2000,
+    "pictures": 2000,
+    "a": 0.8426200623459391,
+    "b": 0.8368894876234722,
+    "difference": -0.005730574722466963,
+    "ci95": [
+     -0.02600691985620429,
+     0.013703787042339766
+    ],
+    "resamples": 2000
+   }
+  },
+  "test_count_coco": {
+   "acc": {
+    "metric": "accuracy",
+    "questions": 1100,
+    "pictures": 1100,
+    "a": 0.30636363636363634,
+    "b": 0.34454545454545454,
+    "difference": 0.038181818181818206,
+    "ci95": [
+     0.011818181818181783,
+     0.06545454545454549
+    ],
+    "resamples": 2000
+   },
+   "nll": {
+    "metric": "nll",
+    "questions": 1100,
+    "pictures": 1100,
+    "a": 1.9893882985025666,
+    "b": 1.779380167228201,
+    "difference": -0.21000813127436557,
+    "ci95": [
+     -0.25964636612728054,
+     -0.1622590152983625
+    ],
+    "resamples": 2000
+   }
+  },
+  "bench_mmbench": {
+   "acc": {
+    "metric": "accuracy",
+    "questions": 4294,
+    "pictures": 4294,
+    "a": 0.6040987424312995,
+    "b": 0.6266884024219842,
+    "difference": 0.02258965999068474,
+    "ci95": [
+     0.011178388448998544,
+     0.03446669771774569
+    ],
+    "resamples": 2000
+   },
+   "nll": {
+    "metric": "nll",
+    "questions": 4294,
+    "pictures": 4294,
+    "a": 0.9621266293403985,
+    "b": 0.9076105848749486,
+    "difference": -0.05451604446544989,
+    "ci95": [
+     -0.07615708256293019,
+     -0.03108140813878635
+    ],
+    "resamples": 2000
+   }
+  },
+  "bench_cvbench": {
+   "acc": {
+    "metric": "accuracy",
+    "questions": 2638,
+    "pictures": 2638,
+    "a": 0.5443517816527672,
+    "b": 0.5511751326762699,
+    "difference": 0.006823351023502733,
+    "ci95": [
+     -0.009855951478392688,
+     0.02501895375284313
+    ],
+    "resamples": 2000
+   },
+   "nll": {
+    "metric": "nll",
+    "questions": 2638,
+    "pictures": 2638,
+    "a": 0.9016688344057614,
+    "b": 0.8083227674466051,
+    "difference": -0.09334606695915626,
+    "ci95": [
+     -0.11244997195903339,
+     -0.07520308890416327
+    ],
+    "resamples": 2000
+   }
+  },
+  "bench_mme": {
+   "acc": {
+    "metric": "accuracy",
+    "questions": 2374,
+    "pictures": 1187,
+    "a": 0.6533277169334457,
+    "b": 0.6474304970513901,
+    "difference": -0.005897219882055604,
+    "ci95": [
+     -0.02064026958719467,
+     0.008845829823083351
+    ],
+    "resamples": 2000
+   },
+   "nll": {
+    "metric": "nll",
+    "questions": 2374,
+    "pictures": 1187,
+    "a": 0.6057342796352463,
+    "b": 0.6429894737804899,
+    "difference": 0.037255194145243586,
+    "ci95": [
+     0.023705705678477296,
+     0.05017788218628971
+    ],
+    "resamples": 2000
+   }
+  },
+  "bench_naturalbench": {
+   "acc": {
+    "metric": "accuracy",
+    "questions": 7570,
+    "pictures": 3800,
+    "a": 0.6206076618229854,
+    "b": 0.6240422721268164,
+    "difference": 0.003434610303830965,
+    "ci95": [
+     -0.007000396248844298,
+     0.013199577613516311
+    ],
+    "resamples": 2000
+   },
+   "nll": {
+    "metric": "nll",
+    "questions": 7570,
+    "pictures": 3800,
+    "a": 0.6743440115208639,
+    "b": 0.6925552586321518,
+    "difference": 0.018211247111287965,
+    "ci95": [
+     0.008188317325104721,
+     0.029288391972986494
+    ],
+    "resamples": 2000
+   }
+  },
+  "dev_mix": {
+   "acc": {
+    "metric": "accuracy",
+    "questions": 5178,
+    "pictures": 3042,
+    "a": 0.7576284279644651,
+    "b": 0.790845886442642,
+    "difference": 0.03321745847817692,
+    "ci95": [
+     0.022335794056407288,
+     0.04369682907965966
+    ],
+    "resamples": 2000
+   },
+   "nll": {
+    "metric": "nll",
+    "questions": 5178,
+    "pictures": 3042,
+    "a": 0.5432908524336165,
+    "b": 0.5260876670578333,
+    "difference": -0.017203185375783248,
+    "ci95": [
+     -0.034758412199066724,
+     0.0017373233636809315
+    ],
+    "resamples": 2000
+   }
+  },
+  "dev_sqa": {
+   "acc": {
+    "metric": "accuracy",
+    "questions": 2017,
+    "pictures": 1799,
+    "a": 0.8041646008924145,
+    "b": 0.8463063956370848,
+    "difference": 0.04214179474467028,
+    "ci95": [
+     0.024938875305623442,
+     0.05867970660146704
+    ],
+    "resamples": 2000
+   },
+   "nll": {
+    "metric": "nll",
+    "questions": 2017,
+    "pictures": 1799,
+    "a": 0.45615675456359894,
+    "b": 0.416725354374512,
+    "difference": -0.03943140018908692,
+    "ci95": [
+     -0.06878795828969625,
+     -0.007295041475122277
+    ],
+    "resamples": 2000
+   }
+  }
+ },
+ "verdict": {
+  "missing": [],
+  "1_no_set_worse": {
+   "pass": true,
+   "worse": []
+  },
+  "2_at_least_15_up": {
+   "pass": true,
+   "up": 22,
+   "of": 23
+  },
+  "3_weak_4_of_5": {
+   "pass": true,
+   "by_ability": {
+    "ScienceQA": true,
+    "VSR": true,
+    "counting": true,
+    "A-OKVQA": false,
+    "V7W": true
+   }
+  },
+  "4_nll_not_worse": {
+   "pass": false,
+   "worse": [
+    "test_exist",
+    "test_size",
+    "bench_mme"
+   ]
+  }
+ }
+}

evaluation/accept_vs_v0.3.md

@@ -0,0 +1,60 @@
+| 集合 | 判定 | 题数 | v0.3 | 候选 | 准确率差 [95%] | NLL 差 [95%] |
+|---|---|---|---|---|---|---|
+| test_vsr | 是 | 902 | 0.661 | 0.731 | +7.0 [+3.6, +10.6] | -0.025 [-0.067, +0.018] |
+| test_v7w | 是 | 1000 | 0.711 | 0.783 | +7.2 [+4.8, +9.6] | -0.109 [-0.142, -0.077] |
+| bench_pope | 是 | 9000 | 0.848 | 0.865 | +1.7 [+0.9, +2.6] | -0.009 [-0.022, +0.003] |
+| test_vqa_yesno | 是 | 1000 | 0.683 | 0.722 | +3.9 [+1.5, +6.5] | -0.023 [-0.044, -0.002] |
+| test_vqa_choice | 是 | 1453 | 0.882 | 0.905 | +2.3 [+0.8, +3.8] | -0.017 [-0.048, +0.018] |
+| test_gqa | 是 | 986 | 0.782 | 0.843 | +6.1 [+3.6, +8.6] | -0.054 [-0.087, -0.020] |
+| test_exist | 是 | 1120 | 0.939 | 0.955 | +1.6 [+0.3, +2.9] | +0.026 [+0.008, +0.043] |
+| test_position | 是 | 1276 | 0.886 | 0.938 | +5.2 [+3.3, +7.1] | -0.034 [-0.082, +0.015] |
+| test_relation | 是 | 2366 | 0.749 | 0.917 | +16.8 [+14.9, +18.6] | -0.210 [-0.253, -0.165] |
+| test_size | 是 | 1198 | 0.856 | 0.870 | +1.3 [-0.3, +3.0] | +0.156 [+0.122, +0.188] |
+| test_position_flip | 是 | 640 | 0.828 | 0.933 | +10.5 [+7.9, +13.2] | -0.096 [-0.167, -0.019] |
+| test_relation_flip | 是 | 1786 | 0.702 | 0.914 | +21.2 [+19.0, +23.5] | -0.223 [-0.276, -0.162] |
+| bench_lv_vqav2_yesno | 是 | 4887 | 0.725 | 0.756 | +3.1 [+1.9, +4.2] | -0.026 [-0.035, -0.017] |
+| bench_lv_aokvqa | 是 | 1138 | 0.612 | 0.634 | +2.1 [-0.4, +4.5] | -0.012 [-0.050, +0.027] |
+| bench_lv_scienceqa | 是 | 2097 | 0.789 | 0.821 | +3.1 [+1.6, +4.8] | -0.007 [-0.036, +0.022] |
+| bench_whatsup | 是 | 820 | 0.287 | 0.467 | +18.0 [+13.5, +22.1] | -0.277 [-0.335, -0.219] |
+| bench_countbench | 是 | 491 | 0.356 | 0.501 | +14.5 [+9.4, +19.6] | -0.215 [-0.256, -0.176] |
+| bench_cogent_b | 是 | 2000 | 0.578 | 0.618 | +4.0 [+1.6, +6.6] | -0.006 [-0.026, +0.014] |
+| test_count_coco | 是 | 1100 | 0.306 | 0.345 | +3.8 [+1.2, +6.5] | -0.210 [-0.260, -0.162] |
+| bench_mmbench | 是 | 4294 | 0.604 | 0.627 | +2.3 [+1.1, +3.4] | -0.055 [-0.076, -0.031] |
+| bench_cvbench | 是 | 2638 | 0.544 | 0.551 | +0.7 [-1.0, +2.5] | -0.093 [-0.112, -0.075] |
+| bench_mme | 是 | 2374 | 0.653 | 0.647 | -0.6 [-2.1, +0.9] | +0.037 [+0.024, +0.050] |
+| bench_naturalbench | 是 | 7570 | 0.621 | 0.624 | +0.3 [-0.7, +1.3] | +0.018 [+0.008, +0.029] |
+| dev_mix | 报告 | 5178 | 0.758 | 0.791 | +3.3 [+2.2, +4.4] | -0.017 [-0.035, +0.002] |
+| dev_sqa | 报告 | 2017 | 0.804 | 0.846 | +4.2 [+2.5, +5.9] | -0.039 [-0.069, -0.007] |
+
+```json
+{
+ "missing": [],
+ "1_no_set_worse": {
+  "pass": true,
+  "worse": []
+ },
+ "2_at_least_15_up": {
+  "pass": true,
+  "up": 22,
+  "of": 23
+ },
+ "3_weak_4_of_5": {
+  "pass": true,
+  "by_ability": {
+   "ScienceQA": true,
+   "VSR": true,
+   "counting": true,
+   "A-OKVQA": false,
+   "V7W": true
+  }
+ },
+ "4_nll_not_worse": {
+  "pass": false,
+  "worse": [
+   "test_exist",
+   "test_size",
+   "bench_mme"
+  ]
+ }
+}
+```

evaluation/adoption.md

@@ -1,27 +1,14 @@
-# deVision v0.3 adoption decision
+# v0.4 adoption
 
-Recorded: 2026-10-14 22:49:39 +0800
+On 2026-10-22 the project owner decided to publish `r21-soft` as v0.4 ("发布这一版"), choosing it over the alternative candidate (r20 with temperatures refitted on the balanced dev set), although the prewritten acceptance rule 4 (calibration) is not met. The rules were frozen before training in `docs/experiments/2026-10-21-v04-eval-freeze.md`; the full verdict is in [accept_vs_v0.3.md](accept_vs_v0.3.md).
 
-The project owner explicitly chose r15-vision to replace v12-ext and authorized publication as model v0.3. This is an owner-approved trade-off. The prewritten acceptance rule was not passed: the original `candidate=false` is preserved in [accept_vs_v0.2.json](accept_vs_v0.2.json). The counting condition passed; four old-capability protection conditions did not pass; no prewritten NLL warning was triggered.
+| Rule (vs v0.3, paired by question, pictures resampled) | Result |
+|---|---|
+| 1. no judged set significantly worse (accuracy) | met (0 of 23) |
+| 2. at least 15 of 23 sets improve | met (22 of 23) |
+| 3. at least 4 of 5 weak abilities improve with lower bound > 0 | met (ScienceQA, VSR, counting, V7W; A-OKVQA +2.1 with an interval containing 0) |
+| 4. no set's NLL rises by > 0.02 with lower bound > 0 | **not met**: size +0.156, MME +0.037, COCO object presence +0.026 |
 
-| Set | v0.2 | v0.3 | Difference, points [95% CI] |
-|---|---:|---:|---:|
-| bench_countbench | 18.9% | 35.6% | +16.7 [+12.8, +21.0] |
-| test_relation | 71.1% | 75.5% | +4.4 [+2.8, +5.9] |
-| test_relation_flip | 66.9% | 69.8% | +2.9 [+1.0, +4.8] |
-| bench_lv_scienceqa | 76.6% | 78.9% | +2.3 [+1.1, +3.5] |
-| test_vsr | 67.9% | 66.2% | -1.8 [-3.9, +0.3] |
-| bench_lv_aokvqa | 62.6% | 61.2% | -1.3 [-2.9, +0.3] |
-| test_vqa_yesno | 71.0% | 70.4% | -0.6 [-2.2, +1.1] |
-| test_position_flip | 85.0% | 84.7% | -0.3 [-2.2, +1.4] |
+The rule-4 failures come mostly from under-confidence: size answers average 0.64 confidence at 0.87 accuracy, presence 0.88 at 0.955 (the presence / size training labels were softened to 0.9 / 0.1 to fix POPE over-confidence, which worked: POPE NLL 0.363 vs v0.3's 0.372). MME is slightly over-confident. One temperature per question type and option count cannot fit all of these at once.
 
-Intervals use paired resampling by picture. These evaluation sets have informed multiple experiments; the intervals do not correct for adaptive selection or multiple comparisons. A confidence interval containing zero does not establish equivalence.
-
-Accepted limitations:
-
-- VSR positive recall fell from 72.6% to 67.3% (post-hoc diagnostic); A-OKVQA lost 15 net correct answers, with worse NLL and ECE.
-- CountBench accuracy is still only 35.6%. Reversing answer order changes the selected answer on 39.5% of questions, versus 16.9% for v0.2; reversed-order accuracy also improved, so the gain is not confined to the original ordering.
-- What'sUp NLL increased by 0.0186 [0.0051, 0.0314], below the prewritten 0.02 warning threshold but still a deterioration in probability quality.
-- Diagram reasoning, small text and general spatial binding remain limited. There is no completed frozen-vision control isolating the effect of unfreezing.
-
-The final checkpoint is `runs/r15-vision/stage2`, step 10,235, seed 1. It continues exp19 at step 2,000 with the same optimizer/schedule state; see [seeded_from.json](seeded_from.json) and [training.yaml](training.yaml). Calibration is unchanged from the completed run; publication does not train or refit temperatures.
+Test sets informed several decisions in this release cycle (a stop at step 22,000, the science continuation, the label softening and the choice between two candidates). This is one training seed.

evaluation/cpu_latency.json

@@ -1,14 +1,14 @@
 {
   "n": 200,
-  "images": 73,
-  "accuracy_all": 0.81,
+  "images": 199,
+  "accuracy_all": 0.63,
   "accuracy": {
-    "noul": 0.8202247191011236,
-    "choice": 0.8018018018018018
+    "noul": null,
+    "choice": 0.63
   },
-  "nll": 0.4400090990936913,
-  "brier": 0.25543111423553433,
-  "ece": 0.06582706868648526,
+  "nll": 0.9023860797262202,
+  "brier": 0.48241933227960376,
+  "ece": 0.05323468044400214,
   "ece_bins": 15,
   "reliability": [
     {
@@ -16,369 +16,136 @@
         0.2,
         0.3
       ],
-      "n": 1,
-      "confidence": 0.28821372985839844,
-      "accuracy": 1.0
+      "n": 8,
+      "confidence": 0.28236541524529457,
+      "accuracy": 0.25
     },
     {
       "bin": [
         0.3,
         0.4
       ],
-      "n": 5,
-      "confidence": 0.3615827262401581,
-      "accuracy": 0.4
+      "n": 41,
+      "confidence": 0.3474449958743119,
+      "accuracy": 0.36585365853658536
     },
     {
       "bin": [
         0.4,
         0.5
       ],
-      "n": 7,
-      "confidence": 0.4423674430165972,
-      "accuracy": 0.42857142857142855
+      "n": 31,
+      "confidence": 0.44290193915367126,
+      "accuracy": 0.5161290322580645
     },
     {
       "bin": [
         0.5,
         0.6
       ],
-      "n": 33,
-      "confidence": 0.5360189441478613,
-      "accuracy": 0.6363636363636364
+      "n": 20,
+      "confidence": 0.5382369369268417,
+      "accuracy": 0.6
     },
     {
       "bin": [
         0.6,
         0.7
       ],
-      "n": 26,
-      "confidence": 0.6526116694395359,
-      "accuracy": 0.6923076923076923
+      "n": 29,
+      "confidence": 0.6475593776538454,
+      "accuracy": 0.6206896551724138
     },
     {
       "bin": [
         0.7,
         0.8
       ],
-      "n": 15,
-      "confidence": 0.7433490335941315,
-      "accuracy": 0.8
+      "n": 18,
+      "confidence": 0.7407801515526242,
+      "accuracy": 0.7777777777777778
     },
     {
       "bin": [
         0.8,
         0.9
       ],
-      "n": 38,
-      "confidence": 0.8569445978654059,
-      "accuracy": 0.8421052631578947
+      "n": 17,
+      "confidence": 0.8483305994202109,
+      "accuracy": 0.9411764705882353
     },
     {
       "bin": [
         0.9,
         1.0
       ],
-      "n": 75,
-      "confidence": 0.953959968884786,
-      "accuracy": 0.9733333333333334
+      "n": 36,
+      "confidence": 0.9695057719945908,
+      "accuracy": 0.9166666666666666
     }
   ],
   "thresholds": {
     "0.6": {
-      "coverage": 0.77,
-      "accuracy": 0.8766233766233766
+      "coverage": 0.5,
+      "accuracy": 0.81
     },
     "0.7": {
-      "coverage": 0.64,
-      "accuracy": 0.9140625
+      "coverage": 0.355,
+      "accuracy": 0.8873239436619719
     },
     "0.8": {
-      "coverage": 0.565,
-      "accuracy": 0.9292035398230089
+      "coverage": 0.265,
+      "accuracy": 0.9245283018867925
     },
     "0.9": {
-      "coverage": 0.375,
-      "accuracy": 0.9733333333333334
+      "coverage": 0.18,
+      "accuracy": 0.9166666666666666
     }
   },
   "by_source": {
-    "ai2d": {
-      "n": 2,
-      "images": 1,
-      "accuracy": 0.5,
-      "nll": 1.2898404388867466,
-      "ece": 0.3732716739177704
-    },
     "aokvqa": {
-      "n": 4,
-      "images": 2,
-      "accuracy": 0.75,
-      "nll": 0.611274152606737,
-      "ece": 0.2411895990371704
-    },
-    "coco-exist": {
-      "n": 39,
-      "images": 25,
-      "accuracy": 0.9230769230769231,
-      "nll": 0.23563569869662415,
-      "ece": 0.0540550612868407
-    },
-    "coco-position": {
-      "n": 26,
-      "images": 18,
-      "accuracy": 0.9230769230769231,
-      "nll": 0.20050254680859508,
-      "ece": 0.12922574006594145
-    },
-    "coco-relation": {
-      "n": 10,
-      "images": 5,
-      "accuracy": 0.8,
-      "nll": 0.4535431995357432,
-      "ece": 0.29919506013393404
-    },
-    "coco-size": {
-      "n": 2,
-      "images": 2,
-      "accuracy": 1.0,
-      "nll": 0.3333616534201184,
-      "ece": 0.2508668899536133
-    },
-    "gqa-v2": {
-      "n": 25,
-      "images": 7,
-      "accuracy": 0.68,
-      "nll": 0.5026796189257893,
-      "ece": 0.21928748846054075
-    },
-    "tqa": {
-      "n": 10,
-      "images": 2,
-      "accuracy": 0.5,
-      "nll": 1.2075195747510135,
-      "ece": 0.35332188010215765
-    },
-    "vqav2-choice": {
-      "n": 35,
-      "images": 21,
-      "accuracy": 0.8857142857142857,
-      "nll": 0.3503283097869317,
-      "ece": 0.15751930900982447
-    },
-    "vqav2-yesno": {
-      "n": 38,
-      "images": 24,
-      "accuracy": 0.7368421052631579,
-      "nll": 0.5690740238980908,
-      "ece": 0.11526736793549436
-    },
-    "vsr": {
-      "n": 9,
-      "images": 4,
-      "accuracy": 0.7777777777777778,
-      "nll": 0.5381709545359337,
-      "ece": 0.2373169246647093
+      "n": 200,
+      "images": 199,
+      "accuracy": 0.63,
+      "nll": 0.9023860797262202,
+      "ece": 0.05323468044400214
     }
   },
   "by_kind": {
-    "ai2d": {
-      "n": 2,
-      "images": 1,
-      "accuracy": 0.5,
-      "nll": 1.2898404388867466,
-      "ece": 0.3732716739177704
-    },
-    "animal": {
-      "n": 2,
-      "images": 2,
-      "accuracy": 1.0,
-      "nll": 0.08400760063551806,
-      "ece": 0.07780307531356812
-    },
     "aokvqa": {
-      "n": 4,
-      "images": 2,
-      "accuracy": 0.75,
-      "nll": 0.611274152606737,
-      "ece": 0.2411895990371704
-    },
-    "color": {
-      "n": 14,
-      "images": 11,
-      "accuracy": 0.8571428571428571,
-      "nll": 0.3618174527077554,
-      "ece": 0.09759957662650516
-    },
-    "count": {
-      "n": 10,
-      "images": 7,
-      "accuracy": 0.8,
-      "nll": 0.6097179990869517,
-      "ece": 0.3152927696704865
-    },
-    "exist": {
-      "n": 39,
-      "images": 25,
-      "accuracy": 0.9230769230769231,
-      "nll": 0.23563569869662415,
-      "ece": 0.0540550612868407
-    },
-    "food": {
-      "n": 1,
-      "images": 1,
-      "accuracy": 1.0,
-      "nll": 0.0042057737329474765,
-      "ece": 0.00419694185256958
-    },
-    "gqa": {
-      "n": 11,
-      "images": 6,
-      "accuracy": 0.7272727272727273,
-      "nll": 0.5031112251493984,
-      "ece": 0.2824250134554776
-    },
-    "material": {
-      "n": 2,
-      "images": 2,
-      "accuracy": 1.0,
-      "nll": 0.22872908175288753,
-      "ece": 0.19662755727767944
-    },
-    "meat": {
-      "n": 1,
-      "images": 1,
-      "accuracy": 1.0,
-      "nll": 0.1798803908020786,
-      "ece": 0.16462987661361694
-    },
-    "position": {
-      "n": 26,
-      "images": 18,
-      "accuracy": 0.9230769230769231,
-      "nll": 0.20050254680859508,
-      "ece": 0.12922574006594145
-    },
-    "relation": {
-      "n": 10,
-      "images": 5,
-      "accuracy": 0.8,
-      "nll": 0.4535431995357432,
-      "ece": 0.29919506013393404
-    },
-    "room": {
-      "n": 1,
-      "images": 1,
-      "accuracy": 1.0,
-      "nll": 0.005263986423792628,
-      "ece": 0.005250155925750732
-    },
-    "size": {
-      "n": 2,
-      "images": 2,
-      "accuracy": 1.0,
-      "nll": 0.3333616534201184,
-      "ece": 0.2508668899536133
-    },
-    "spatial": {
-      "n": 14,
-      "images": 5,
-      "accuracy": 0.6428571428571429,
-      "nll": 0.5023404997500963,
-      "ece": 0.1696794331073761
-    },
-    "tqa": {
-      "n": 10,
-      "images": 2,
-      "accuracy": 0.5,
-      "nll": 1.2075195747510135,
-      "ece": 0.35332188010215765
-    },
-    "vegetable": {
-      "n": 4,
-      "images": 4,
-      "accuracy": 1.0,
-      "nll": 0.07101074950722219,
-      "ece": 0.06772895157337189
-    },
-    "vsr": {
-      "n": 9,
-      "images": 4,
-      "accuracy": 0.7777777777777778,
-      "nll": 0.5381709545359337,
-      "ece": 0.2373169246647093
-    },
-    "yesno": {
-      "n": 38,
-      "images": 24,
-      "accuracy": 0.7368421052631579,
-      "nll": 0.5690740238980908,
-      "ece": 0.11526736793549436
+      "n": 200,
+      "images": 199,
+      "accuracy": 0.63,
+      "nll": 0.9023860797262202,
+      "ece": 0.05323468044400214
     }
   },
   "by_axis": {
     "-": {
-      "n": 162,
-      "images": 67,
-      "accuracy": 0.7901234567901234,
-      "nll": 0.47554891340501837,
-      "ece": 0.05595421993438109
-    },
-    "lr": {
-      "n": 27,
-      "images": 18,
-      "accuracy": 0.8888888888888888,
-      "nll": 0.3277449296076739,
-      "ece": 0.19794687407988087
-    },
-    "tb": {
-      "n": 11,
-      "images": 11,
-      "accuracy": 0.9090909090909091,
-      "nll": 0.1921620679743721,
-      "ece": 0.09886410019614479
+      "n": 200,
+      "images": 199,
+      "accuracy": 0.63,
+      "nll": 0.9023860797262202,
+      "ece": 0.05323468044400214
     }
   },
   "by_options": {
-    "2": {
-      "n": 157,
-      "images": 64,
-      "accuracy": 0.8280254777070064,
-      "nll": 0.38058168910086837,
-      "ece": 0.06867314277181197
-    },
-    "3": {
-      "n": 10,
-      "images": 9,
-      "accuracy": 0.9,
-      "nll": 0.4393064845098579,
-      "ece": 0.22365574836730956
-    },
     "4": {
-      "n": 31,
-      "images": 18,
-      "accuracy": 0.6774193548387096,
-      "nll": 0.7616504939115978,
-      "ece": 0.10811415699220474
-    },
-    "5": {
-      "n": 2,
-      "images": 2,
-      "accuracy": 1.0,
-      "nll": 0.12313223677190567,
-      "ece": 0.1151973307132721
+      "n": 200,
+      "images": 199,
+      "accuracy": 0.63,
+      "nll": 0.9023860797262202,
+      "ece": 0.05323468044400214
     }
   },
-  "noul_confusion": {
-    "tp": 42,
-    "fp": 10,
-    "fn": 6,
-    "tn": 31,
-    "yes_ratio": 0.5842696629213483
+  "ms_per_question": {
+    "p50": 289.0499159984756,
+    "p95": 301.4491552057734,
+    "overall": 289.6095040804357
   },
   "request_latency_ms": {
-    "p50": 174.91745902225375,
-    "p95": 189.9587146937847,
+    "p50": 289.0499159984756,
+    "p95": 301.4491552057734,
     "requests": 200,
     "warmup_excluded": 3
   },
@@ -387,47 +154,94 @@
     "p50": 1.0,
     "max": 1
   },
-  "ms_per_question": {
-    "p50": 174.91745902225375,
-    "p95": 189.9587146937847,
-    "overall": 172.57170106749982
-  },
   "latency_ms": {
-    "p50": 174.91745902225375,
-    "p95": 189.9587146937847,
+    "p50": 289.0499159984756,
+    "p95": 301.4491552057734,
     "requests": 200,
     "warmup_excluded": 3
   },
   "control": "none",
-  "model": "devision-0.1",
+  "model": "devision-v0.3",
+  "reading": {
+    "mode": "none",
+    "routed_questions": 0,
+    "read_questions": 0
+  },
   "temperature": {
-    "choice": 1.5922062397003174,
-    "noul": 1.825764536857605,
+    "choice": 2.3641862869262695,
+    "noul": 1.0895380973815918,
     "by_options": {
-      "choice:11+": 1.201171636581421,
-      "choice:2": 1.9487814903259277,
-      "choice:3-5": 1.7575252056121826,
-      "noul:2": 1.825764536857605
+      "choice:2": 3.563119649887085,
+      "choice:3-5": 2.1145524978637695,
+      "noul:2": 1.0895380973815918
     }
   },
   "grouping": {
     "max_questions_per_request": 1,
-    "by": "none (one question per request)"
+    "by": "none (one question per request)",
+    "max_questions_per_call": "one request per call"
   },
   "environment": {
     "threads": 4,
-    "machine": "arm64",
-    "system": "Darwin"
+    "machine": "x86_64",
+    "system": "Linux"
   },
   "run": {
-    "checkpoint": "runs/r15-vision/stage2",
-    "data": "data/r12/cpu_latency.jsonl",
-    "data_sha256": "352f0b4676aeb7389b45a93086169096d6b8e3962568543cfa373877c3e84020",
-    "role": "unspecified",
+    "checkpoint": "runs/r21-soft/train",
+    "data": "runs/v04-cpu/latency200.jsonl",
+    "data_sha256": "e0f4d5b0f82ff59ba91d141f49833ad9a097ac36842b98c32e2c4a834d0b9691",
+    "role": "monitoring",
     "device": "cpu",
     "temperature": "fitted",
     "control": "none",
     "seed": 0,
-    "code": "6feb0b3+dirty"
+    "code": "a6b4e8a+dirty",
+    "reading": {
+      "mode": "none",
+      "routed_questions": 0,
+      "read_questions": 0
+    },
+    "code_identity": {
+      "modules": {
+        "devision": {
+          "path": "/home/bruce/proj/devision/src/devision/__init__.py",
+          "sha256": "aa00e0d329f33444a51bc4cab7691911b6b226664fdc2fbd438103de8d0c27a1"
+        },
+        "devision.image": {
+          "path": "/home/bruce/proj/devision/src/devision/image.py",
+          "sha256": "a139a63f985b8a210de4b9a5327dd56cc078a986cb36ebcc7fd3c4aba4bef312"
+        },
+        "devision.infer": {
+          "path": "/home/bruce/proj/devision/src/devision/infer.py",
+          "sha256": "d7e3e5f8bc54278cbdba99f77da567344cf40e9748c6b1a55ecb48e48d593893"
+        },
+        "devision.model": {
+          "path": "/home/bruce/proj/devision/src/devision/model.py",
+          "sha256": "079e9c9c60545242146a99ec66ba09a5cd1435545967b37b2649e071b3245209"
+        },
+        "devision.reading": {
+          "path": "/home/bruce/proj/devision/src/devision/reading.py",
+          "sha256": "dc2f3cc2151ec4761dcfc404b28d609a932aac70ff0ff02b0dd6e170b6c1f8ba"
+        },
+        "devision.train": {
+          "path": "/home/bruce/proj/devision/src/devision/train/__init__.py",
+          "sha256": "16c32ba2bae66029fb0afd4946827c475729aa45c7dde335864d002c564df289"
+        },
+        "devision.train.cli": {
+          "path": "/home/bruce/proj/devision/src/devision/train/cli.py",
+          "sha256": "aab8811a63c89726fd78dd7ea88ab930c6d115b529a011173ccc8637f081d342"
+        },
+        "devision.train.evaluate": {
+          "path": "/home/bruce/proj/devision/src/devision/train/evaluate.py",
+          "sha256": "4d5df39b37b0854ead1b8c7d0a04933e24ab9b05e0e341f0bdbfbc6911af2d90"
+        },
+        "devision.train.samples": {
+          "path": "/home/bruce/proj/devision/src/devision/train/samples.py",
+          "sha256": "5a4d5a75ec31380f5470a22c0ba4e370e170ba0499e5d4a77ed573450f9c3e31"
+        }
+      },
+      "version": "0.3.0.dev1",
+      "python": "3.12.3"
+    }
   }
 }

evaluation/results.json

@@ -1,189 +1,174 @@
 {
  "temperature": {
-  "choice": 1.5922062397003174,
-  "noul": 1.825764536857605,
+  "choice": 2.3641862869262695,
+  "noul": 1.0895380973815918,
   "by_options": {
-   "choice:11+": 1.201171636581421,
-   "choice:2": 1.9487814903259277,
-   "choice:3-5": 1.7575252056121826,
-   "noul:2": 1.825764536857605
+   "choice:2": 3.563119649887085,
+   "choice:3-5": 2.1145524978637695,
+   "noul:2": 1.0895380973815918
   }
  },
  "test_sets": {
   "test_exist": {
    "questions": 1120,
-   "accuracy": 0.9473,
-   "nll": 0.1631,
-   "ece": 0.0341,
-   "accuracy_mismatched": 0.5036
+   "accuracy": 0.9554,
+   "nll": 0.1957,
+   "ece": 0.0924,
+   "accuracy_mismatched": 0.4973
   },
   "test_vqa_choice": {
-   "questions": 1420,
-   "accuracy": 0.9092,
-   "nll": 0.2705,
-   "ece": 0.0384,
-   "accuracy_mismatched": 0.3803
+   "questions": 1453,
+   "accuracy": 0.905,
+   "nll": 0.2802,
+   "ece": 0.0534,
+   "accuracy_mismatched": 0.3696
   },
   "test_size": {
-   "questions": 1100,
-   "accuracy": 0.8718,
-   "nll": 0.3361,
-   "ece": 0.0295,
-   "accuracy_mismatched": 0.5109
+   "questions": 1198,
+   "accuracy": 0.8698,
+   "nll": 0.5065,
+   "ece": 0.2345,
+   "accuracy_mismatched": 0.5159
   },
   "bench_pope": {
-   "questions": 8676,
-   "accuracy": 0.8496,
-   "nll": 0.3679,
-   "ece": 0.0393,
-   "accuracy_mismatched": 0.5269
+   "questions": 9000,
+   "accuracy": 0.8653,
+   "nll": 0.363,
+   "ece": 0.0483,
+   "accuracy_mismatched": 0.5201
   },
   "test_gqa": {
-   "questions": 992,
-   "accuracy": 0.7903,
-   "nll": 0.4105,
-   "ece": 0.0396,
-   "accuracy_mismatched": 0.5504
+   "questions": 986,
+   "accuracy": 0.8428,
+   "nll": 0.3828,
+   "ece": 0.0857,
+   "accuracy_mismatched": 0.5456
   },
   "test_position": {
-   "questions": 1274,
-   "accuracy": 0.8846,
-   "nll": 0.2882,
-   "ece": 0.029,
-   "accuracy_mismatched": 0.4929
+   "questions": 1276,
+   "accuracy": 0.9381,
+   "nll": 0.2384,
+   "ece": 0.0468,
+   "accuracy_mismatched": 0.5086
   },
   "test_vqa_yesno": {
    "questions": 1000,
-   "accuracy": 0.704,
-   "nll": 0.5741,
-   "ece": 0.0265,
-   "accuracy_mismatched": 0.512
+   "accuracy": 0.722,
+   "nll": 0.5713,
+   "ece": 0.0552,
+   "accuracy_mismatched": 0.5
   },
   "test_relation": {
-   "questions": 1950,
-   "accuracy": 0.7549,
-   "nll": 0.4896,
-   "ece": 0.0384,
-   "accuracy_mismatched": 0.4959
+   "questions": 2366,
+   "accuracy": 0.9167,
+   "nll": 0.2805,
+   "ece": 0.0367,
+   "accuracy_mismatched": 0.4801
   },
   "test_vsr": {
-   "questions": 904,
-   "accuracy": 0.6615,
-   "nll": 0.6198,
-   "ece": 0.0465,
-   "accuracy_mismatched": 0.4867
+   "questions": 902,
+   "accuracy": 0.7306,
+   "nll": 0.5905,
+   "ece": 0.0734,
+   "accuracy_mismatched": 0.4601
   },
   "test_v7w": {
    "questions": 1000,
-   "accuracy": 0.721,
-   "nll": 0.6726,
-   "ece": 0.0411,
-   "accuracy_mismatched": 0.419
-  },
-  "test_count_fresh": {
-   "questions": 600,
-   "accuracy": 0.7633,
-   "nll": 0.5738,
-   "ece": 0.054,
-   "accuracy_mismatched": 0.405
+   "accuracy": 0.783,
+   "nll": 0.5967,
+   "ece": 0.073,
+   "accuracy_mismatched": 0.442
   },
   "bench_countbench": {
    "questions": 491,
-   "accuracy": 0.3564,
-   "nll": 1.6022,
-   "ece": 0.0523,
-   "accuracy_mismatched": 0.1039
+   "accuracy": 0.501,
+   "nll": 1.3872,
+   "ece": 0.1655,
+   "accuracy_mismatched": 0.1018
   },
   "test_position_flip": {
    "questions": 640,
-   "accuracy": 0.8469,
-   "nll": 0.3592,
-   "ece": 0.0444,
-   "accuracy_mismatched": 0.4766
+   "accuracy": 0.9328,
+   "nll": 0.283,
+   "ece": 0.0532,
+   "accuracy_mismatched": 0.5031
   },
   "test_relation_flip": {
-   "questions": 1492,
-   "accuracy": 0.6977,
-   "nll": 0.545,
-   "ece": 0.03,
-   "accuracy_mismatched": 0.5074
-  },
-  "test_bind_gqa": {
-   "questions": 1484,
-   "accuracy": 0.6765,
-   "nll": 0.5593,
-   "ece": 0.0277,
-   "accuracy_mismatched": 0.498
+   "questions": 1786,
+   "accuracy": 0.9143,
+   "nll": 0.3004,
+   "ece": 0.0485,
+   "accuracy_mismatched": 0.4938
   },
   "bench_whatsup": {
    "questions": 820,
-   "accuracy": 0.2866,
-   "nll": 1.4909,
-   "ece": 0.2447,
-   "accuracy_mismatched": 0.2451
+   "accuracy": 0.4671,
+   "nll": 1.2142,
+   "ece": 0.0926,
+   "accuracy_mismatched": 0.2329
   }
  },
  "laya_vision": {
   "vqav2_yesno": {
    "questions": 4887,
    "laya_vision": 0.7172,
-   "devision": 0.725,
-   "difference": 0.0078,
+   "devision": 0.7559,
+   "difference": 0.0387,
    "ci95": [
-    -0.0075,
-    0.0241
+    0.0244,
+    0.0533
    ]
   },
   "aokvqa": {
    "questions": 1138,
    "laya_vision": 0.5984,
-   "devision": 0.6125,
-   "difference": 0.0141,
+   "devision": 0.6336,
+   "difference": 0.0351,
    "ci95": [
-    -0.0185,
-    0.0466
+    0.0026,
+    0.0659
    ]
   },
   "scienceqa": {
    "questions": 2097,
    "laya_vision": 0.824,
-   "devision": 0.7892,
-   "difference": -0.0348,
+   "devision": 0.8207,
+   "difference": -0.0033,
    "ci95": [
-    -0.0558,
-    -0.0143
+    -0.0224,
+    0.0172
    ]
   },
   "scienceqa_natural_needs_picture": {
    "questions": 323,
    "laya_vision": 0.6997,
-   "devision": 0.4954,
-   "difference": -0.2043,
+   "devision": 0.6378,
+   "difference": -0.0619,
    "ci95": [
-    -0.2724,
-    -0.1331
+    -0.1269,
+    0.0
    ]
   },
   "pope": {
    "random": {
     "laya_vision_published": 0.836,
-    "devision": 0.8797
+    "devision": 0.8987
    },
    "popular": {
     "laya_vision_published": 0.819,
-    "devision": 0.8593
+    "devision": 0.8733
    },
    "adversarial": {
     "laya_vision_published": 0.777,
-    "devision": 0.8053
+    "devision": 0.824
    }
   }
  },
  "cpu_latency": {
-  "p50": 175,
-  "p95": 190,
+  "p50": 289,
+  "p95": 301,
   "threads": 4,
-  "machine": "Darwin arm64",
-  "source": "release/v0.3/cpu_latency.json"
+  "machine": "Linux x86_64",
+  "source": "release/v0.4/cpu_latency.json"
  }
 }

evaluation/results.md

@@ -1,43 +1,40 @@
-# deVision v0.3 evaluation
+# deVision v0.4 evaluation
 
 Generated by `scripts/release/hf.py` from the evaluation of this release.
 
 | Set | Questions | Accuracy | Mismatched | NLL | ECE |
 |---|---:|---:|---:|---:|---:|
-| COCO object presence | 1,120 | 0.947 | 0.504 | 0.163 | 0.034 |
-| VQAv2 multiple choice | 1,420 | 0.909 | 0.380 | 0.271 | 0.038 |
-| COCO size | 1,100 | 0.872 | 0.511 | 0.336 | 0.029 |
-| POPE, project filtered set | 8,676 | 0.850 | 0.527 | 0.368 | 0.039 |
-| GQA val subset | 992 | 0.790 | 0.550 | 0.410 | 0.040 |
-| COCO position | 1,274 | 0.885 | 0.493 | 0.288 | 0.029 |
-| VQAv2 yes/no subset | 1,000 | 0.704 | 0.512 | 0.574 | 0.026 |
-| COCO relative position | 1,950 | 0.755 | 0.496 | 0.490 | 0.038 |
-| VSR, project held-out split | 904 | 0.661 | 0.487 | 0.620 | 0.046 |
-| Visual7W, project held-out split | 1,000 | 0.721 | 0.419 | 0.673 | 0.041 |
-| Fresh counting test (unseen pictures) | 600 | 0.763 | 0.405 | 0.574 | 0.054 |
-| CountBench, 0–10 choices | 491 | 0.356 | 0.104 | 1.602 | 0.052 |
-| COCO mirrored position | 640 | 0.847 | 0.477 | 0.359 | 0.044 |
-| COCO mirrored relative position | 1,492 | 0.698 | 0.507 | 0.545 | 0.030 |
-| GQA binding test | 1,484 | 0.676 | 0.498 | 0.559 | 0.028 |
-| What'sUp | 820 | 0.287 | 0.245 | 1.491 | 0.245 |
+| COCO object presence | 1,120 | 0.955 | 0.497 | 0.196 | 0.092 |
+| VQAv2 multiple choice | 1,453 | 0.905 | 0.370 | 0.280 | 0.053 |
+| COCO size | 1,198 | 0.870 | 0.516 | 0.506 | 0.234 |
+| POPE, project filtered set | 9,000 | 0.865 | 0.520 | 0.363 | 0.048 |
+| GQA val subset | 986 | 0.843 | 0.546 | 0.383 | 0.086 |
+| COCO position | 1,276 | 0.938 | 0.509 | 0.238 | 0.047 |
+| VQAv2 yes/no subset | 1,000 | 0.722 | 0.500 | 0.571 | 0.055 |
+| COCO relative position | 2,366 | 0.917 | 0.480 | 0.281 | 0.037 |
+| VSR, project held-out split | 902 | 0.731 | 0.460 | 0.591 | 0.073 |
+| Visual7W, project held-out split | 1,000 | 0.783 | 0.442 | 0.597 | 0.073 |
+| CountBench, 0–10 choices | 491 | 0.501 | 0.102 | 1.387 | 0.166 |
+| COCO mirrored position | 640 | 0.933 | 0.503 | 0.283 | 0.053 |
+| COCO mirrored relative position | 1,786 | 0.914 | 0.494 | 0.300 | 0.049 |
+| What'sUp | 820 | 0.467 | 0.233 | 1.214 | 0.093 |
 
 ## Laya Vision 201M, same questions
 
 | Set | Questions | Laya Vision | deVision | Difference |
 |---|---:|---:|---:|---|
-| VQAv2 yes/no | 4,887 | 0.717 | 0.725 | +0.8 [-0.8, +2.4] |
-| A-OKVQA | 1,138 | 0.598 | 0.613 | +1.4 [-1.8, +4.7] |
-| ScienceQA with images | 2,097 | 0.824 | 0.789 | -3.5 [-5.6, -1.4] |
-| ScienceQA natural science, needs the picture | 323 | 0.700 | 0.495 | -20.4 [-27.2, -13.3] |
+| VQAv2 yes/no | 4,887 | 0.717 | 0.756 | +3.9 [+2.4, +5.3] |
+| A-OKVQA | 1,138 | 0.598 | 0.634 | +3.5 [+0.3, +6.6] |
+| ScienceQA with images | 2,097 | 0.824 | 0.821 | -0.3 [-2.2, +1.7] |
+| ScienceQA natural science, needs the picture | 323 | 0.700 | 0.638 | -6.2 [-12.7, +0.0] |
 
-On the full POPE random / popular / adversarial sets (3,000 questions each) deVision scores **0.880 / 0.859 / 0.805**, against Laya Vision's published **0.836 / 0.819 / 0.777** (aggregate scores only, not paired).
+On the full POPE random / popular / adversarial sets (3,000 questions each) deVision scores **0.899 / 0.873 / 0.824**, against Laya Vision's published **0.836 / 0.819 / 0.777** (aggregate scores only, not paired).
 
 ## Temperatures
 
 | Bucket | Temperature |
 |---|---:|
-| Choice, 11+ options | 1.2012 |
-| Choice, 2 options | 1.9488 |
-| Choice, 3–5 options | 1.7575 |
-| Noul (yes / no) | 1.8258 |
-| Choice, any other count | 1.5922 |
+| Choice, 2 options | 3.5631 |
+| Choice, 3–5 options | 2.1146 |
+| Noul (yes / no) | 1.0895 |
+| Choice, any other count | 2.3642 |

evaluation/training.yaml

@@ -1,47 +1,35 @@
-# Round 15 (r15-vision; owner 2026-10-14: option 1 after exp19, "1"): r14b's mix (r14 + v6 mirror pairs + VSR) with
-# the SigLIP tower's top 4 layers and final norm trained at 5e-6, the whole 10,235-step schedule, seed 1 -- exp19's
-# configuration exactly, continued from its step-2000 training state (scripts/r15/chain.sh seeds the stage directory;
-# the resume check refuses any setting that differs). r14b's frozen stop rules every 2,000 steps; temperatures fitted
-# on data/r13b/val.jsonl; full evaluation = r14b's sets; acceptance = r14's rules against v12, written before training
-# in docs/experiments/2026-10-14-r15-vision.md.
-name: r15-vision
-device: mps
+# r19-n2 (from v0.3)
+# Round 19 -- v0.3 trained on data-n2 (docs/experiments/2026-10-21-v04-plan.md; owner 2026-10-21: fill the weak
+# abilities with good training data, no teacher). Starts from v0.3 (runs/v03/model), one pass over
+# data/n2/train_mix.jsonl: data-n1 with VQAv2 / GQA cut to 200,000 each, r17's existence questions, Visual7W telling in
+# full, caption-choice spatial relations from scene graphs, COCO counting 0..10, IconQA / TallyQA / MapQA / CLEVR, with
+# counting options always 0..10, held-out pictures dropped by id and dHash, and choices answerable without the picture
+# dropped (scripts/data/n2/). r16's settings (bf16, top 4 SigLIP layers unfrozen). Monitored every 10,000 steps on
+# the new blocks' dev sets and the ScienceQA test split (dev_sqa). Evaluated on every set of configs/v03-baseline.yaml
+# and configs/v03-n2bench.yaml, so each pairs question by question with runs/v03/eval/.
+#   scripts/data/n2/mix.py builds the data
+#   UV_NO_SYNC=1 HF_HUB_OFFLINE=1 nohup uv run devision-pipeline configs/r19-n2.yaml > runs/r19-n2.out 2>&1 &
+name: r19-n2
+device: cuda
 common:
   log_every: 100
 
 stages:
-  - name: stage2
+  - name: train
     kind: train
-    init: runs/v12-ext/stage2
-    data: data/r14b/train.jsonl
-    val: data/r13b/val.jsonl
-    eval: {exist: data/r13b/panel_exist.jsonl, science: data/r13b/panel_science.jsonl, ext: data/ext/dev_ext.jsonl,
-           bind_gqa: data/r13/gqa/dev_bind_gqa.jsonl, count_narrow: data/v11/dev_count_transfer.jsonl,
-           count_wide: data/r13b/count_wide.jsonl, px_wide: data/exp14/pixmo_count_wide.jsonl,
-           px_narrow: data/exp14/pixmo_count_narrow.jsonl, px_exist: data/exp14/pixmo_exist.jsonl,
-           flip: data/v6/dev_lr_pairs.jsonl, vsr: data/v5/dev_vsr.jsonl}
-    params: {epochs: 1, lr_new: 2.0e-5, lr_head: 1.0e-5, lr_lora: 5.0e-5, warmup: 300, eval_every: 2000,
-             save_every: 500, amp: "off", seed: 1, monitor_details: true,
-             vision_unfreeze_top: 4, lr_vision: 5.0e-6,
-             start_check: runs/exp16-r14b-s1/stage2/monitor/step0, start_tolerance: 1.0e-5}
+    init: runs/v03/model
+    data: data/n2/train_mix.jsonl
+    val: data/n1/dev_mix.jsonl
+    eval: {sqa: data/n2/dev_sqa.jsonl, v7w: data/n2/dev_v7w.jsonl, spatial: data/n2/dev_spatial.jsonl,
+           count: data/n2/dev_count.jsonl, iconqa: data/n2/dev_iconqa.jsonl, tallyqa: data/n2/dev_tallyqa.jsonl,
+           sqa_synth: data/n2/dev_sqa_synth.jsonl, clevr: data/n2/dev_clevr.jsonl}
+    params: {epochs: 1, lr_new: 2.0e-5, lr_head: 1.0e-5, lr_lora: 5.0e-5, warmup: 300, eval_every: 10000,
+             vision_unfreeze_top: 4, lr_vision: 5.0e-6, seed: 0}
 
 evaluate:
-  calibration_fit: data/r13b/val.jsonl
-  history: [data/ext/dev_ext.jsonl, data/r13/gqa/dev_bind_gqa.jsonl, data/r13b/panel_exist.jsonl, data/r13b/panel_science.jsonl,
-            data/r13b/count_wide.jsonl, data/v11/dev_count_transfer.jsonl, data/exp14/pixmo_count_wide.jsonl,
-            data/exp14/pixmo_count_narrow.jsonl, data/exp14/pixmo_exist.jsonl,
-            data/v6/dev_lr_pairs.jsonl, data/v5/dev_vsr.jsonl]
   image_identity: data/v5/image_identity.json
   controls: true
   sets:
-    exist: {path: data/r13b/panel_exist.jsonl, role: monitoring}
-    science: {path: data/r13b/panel_science.jsonl, role: monitoring}
-    ext: {path: data/ext/dev_ext.jsonl, role: monitoring}
-    bind_gqa: {path: data/r13/gqa/dev_bind_gqa.jsonl, role: monitoring}
-    count_narrow: {path: data/v11/dev_count_transfer.jsonl, role: monitoring}
-    count_wide: {path: data/r13b/count_wide.jsonl, role: monitoring}
-    flip: {path: data/v6/dev_lr_pairs.jsonl, role: monitoring}
-    vsr: {path: data/v5/dev_vsr.jsonl, role: monitoring}
     dev_mix: data/v5/dev_mix.jsonl
     test_vsr: data/v5/test_vsr.jsonl
     test_v7w: data/v5/test_v7w.jsonl
@@ -59,10 +47,124 @@ evaluate:
     bench_lv_aokvqa: data/lv_bench/aokvqa.jsonl
     bench_lv_scienceqa: data/lv_bench/scienceqa.jsonl
     bench_lv_pope: data/lv_bench/pope.jsonl
-    test_count_fresh: {path: data/v11/test_count_fresh.jsonl, role: heldout}
-    test_bind_gqa: {path: data/r13/gqa/test_bind_gqa.jsonl, role: heldout}
+    bench_whatsup: {path: data/r13/bench/whatsup.jsonl, role: heldout}
+    bench_countbench: {path: data/r13/bench/countbench.jsonl, role: heldout}
+    bench_cogent_b: {path: data/r13/bench/cogent_b.jsonl, role: heldout}
+    test_count_coco: {path: data/n2/test_count_coco.jsonl, role: heldout}
+    bench_mmbench: {path: data/n2/bench/mmbench.jsonl, role: heldout}
+    bench_cvbench: {path: data/n2/bench/cvbench.jsonl, role: heldout}
+    bench_mme: {path: data/n2/bench/mme.jsonl, role: heldout}
+    bench_naturalbench: {path: data/n2/bench/naturalbench.jsonl, role: heldout}
+    dev_sqa: data/n2/dev_sqa.jsonl
+
+# r20-science (from r19)
+# Round 20 -- a short science-heavy continuation of r19 (docs/experiments/2026-10-21-v04-plan.md, stage 4). r19's
+# ScienceQA dev fell while it trained; the main cause was the data: its held-out rule matched drawn maps by dHash and
+# dropped 2,291 of 4,948 ScienceQA questions (state capitals, colonies, oceans). Fixed in scripts/data/n2/held.py
+# (drawn pictures by identical pixels only); data/n2/science_mix.jsonl (scripts/data/n2/science_mix.py) draws from the
+# corrected mix: all ScienceQA / AI2D / TQA x3, IconQA, the synthetic magnet questions (not the solution-concentration
+# ones), 40,000 replay of the rest. One pass from runs/r19-n2/train, r19's settings; the same evaluation sets.
+#   UV_NO_SYNC=1 HF_HUB_OFFLINE=1 nohup uv run devision-pipeline configs/r20-science.yaml > runs/r20-science.out 2>&1 &
+name: r20-science
+device: cuda
+common:
+  log_every: 100
+
+stages:
+  - name: train
+    kind: train
+    init: runs/r19-n2/train
+    data: data/n2/science_mix.jsonl
+    val: data/n1/dev_mix.jsonl
+    eval: {sqa: data/n2/dev_sqa.jsonl, v7w: data/n2/dev_v7w.jsonl, spatial: data/n2/dev_spatial.jsonl,
+           count: data/n2/dev_count.jsonl, iconqa: data/n2/dev_iconqa.jsonl, tallyqa: data/n2/dev_tallyqa.jsonl,
+           sqa_synth: data/n2/dev_sqa_synth.jsonl, clevr: data/n2/dev_clevr.jsonl}
+    params: {epochs: 1, lr_new: 2.0e-5, lr_head: 1.0e-5, lr_lora: 5.0e-5, warmup: 300, eval_every: 5000,
+             vision_unfreeze_top: 4, lr_vision: 5.0e-6, seed: 0}
+
+evaluate:
+  image_identity: data/v5/image_identity.json
+  controls: true
+  sets:
+    dev_mix: data/v5/dev_mix.jsonl
+    test_vsr: data/v5/test_vsr.jsonl
+    test_v7w: data/v5/test_v7w.jsonl
+    bench_pope: data/v2/bench_pope.jsonl
+    test_vqa_yesno: data/v2/test_vqa_yesno.jsonl
+    test_vqa_choice: data/v2/test_vqa_choice.jsonl
+    test_gqa: data/v2/test_gqa.jsonl
+    test_exist: data/v2/test_exist.jsonl
+    test_position: data/v2/test_position.jsonl
+    test_relation: data/v2/test_relation.jsonl
+    test_size: data/v2/test_size.jsonl
+    test_position_flip: data/v6/test_position_flip.jsonl
+    test_relation_flip: data/v6/test_relation_flip.jsonl
+    bench_lv_vqav2_yesno: data/lv_bench/vqav2_yesno.jsonl
+    bench_lv_aokvqa: data/lv_bench/aokvqa.jsonl
+    bench_lv_scienceqa: data/lv_bench/scienceqa.jsonl
+    bench_lv_pope: data/lv_bench/pope.jsonl
+    bench_whatsup: {path: data/r13/bench/whatsup.jsonl, role: heldout}
     bench_countbench: {path: data/r13/bench/countbench.jsonl, role: heldout}
+    bench_cogent_b: {path: data/r13/bench/cogent_b.jsonl, role: heldout}
+    test_count_coco: {path: data/n2/test_count_coco.jsonl, role: heldout}
+    bench_mmbench: {path: data/n2/bench/mmbench.jsonl, role: heldout}
+    bench_cvbench: {path: data/n2/bench/cvbench.jsonl, role: heldout}
+    bench_mme: {path: data/n2/bench/mme.jsonl, role: heldout}
+    bench_naturalbench: {path: data/n2/bench/naturalbench.jsonl, role: heldout}
+    dev_sqa: data/n2/dev_sqa.jsonl
+
+# r21-soft (from r20) -- the released weights
+# Round 21 -- r20 with the hard 0 / 1 labels of existence and size questions softened (docs/experiments/
+# 2026-10-21-v04-plan.md, stage 4). r20 with temperatures refitted on data/v5/dev_mix passes the frozen rules 1-3 but
+# not 4 (POPE NLL 0.372 -> 0.454, size 0.350 -> 0.440; runs/v04-interp/a1.0/compare/accept_v04.md). One pass from
+# runs/r20-science/train over data/n2/soft_mix.jsonl (scripts/data/n2/soft_mix.py: 40,000 existence + 4,331 size
+# questions with golds 0.9 / 0.1, 60,000 unchanged replay); --val is data/v5/dev_mix.jsonl (balanced sources), so the
+# final temperatures are fitted there. r20's settings and evaluation sets.
+#   UV_NO_SYNC=1 HF_HUB_OFFLINE=1 nohup uv run devision-pipeline configs/r21-soft.yaml > runs/r21-soft.out 2>&1 &
+name: r21-soft
+device: cuda
+common:
+  log_every: 100
+
+stages:
+  - name: train
+    kind: train
+    init: runs/r20-science/train
+    data: data/n2/soft_mix.jsonl
+    val: data/v5/dev_mix.jsonl
+    eval: {sqa: data/n2/dev_sqa.jsonl, v7w: data/n2/dev_v7w.jsonl, spatial: data/n2/dev_spatial.jsonl,
+           count: data/n2/dev_count.jsonl, iconqa: data/n2/dev_iconqa.jsonl, tallyqa: data/n2/dev_tallyqa.jsonl,
+           sqa_synth: data/n2/dev_sqa_synth.jsonl, clevr: data/n2/dev_clevr.jsonl}
+    params: {epochs: 1, lr_new: 2.0e-5, lr_head: 1.0e-5, lr_lora: 5.0e-5, warmup: 300, eval_every: 5000,
+             vision_unfreeze_top: 4, lr_vision: 5.0e-6, seed: 0}
+
+evaluate:
+  image_identity: data/v5/image_identity.json
+  controls: true
+  sets:
+    dev_mix: data/v5/dev_mix.jsonl
+    test_vsr: data/v5/test_vsr.jsonl
+    test_v7w: data/v5/test_v7w.jsonl
+    bench_pope: data/v2/bench_pope.jsonl
+    test_vqa_yesno: data/v2/test_vqa_yesno.jsonl
+    test_vqa_choice: data/v2/test_vqa_choice.jsonl
+    test_gqa: data/v2/test_gqa.jsonl
+    test_exist: data/v2/test_exist.jsonl
+    test_position: data/v2/test_position.jsonl
+    test_relation: data/v2/test_relation.jsonl
+    test_size: data/v2/test_size.jsonl
+    test_position_flip: data/v6/test_position_flip.jsonl
+    test_relation_flip: data/v6/test_relation_flip.jsonl
+    bench_lv_vqav2_yesno: data/lv_bench/vqav2_yesno.jsonl
+    bench_lv_aokvqa: data/lv_bench/aokvqa.jsonl
+    bench_lv_scienceqa: data/lv_bench/scienceqa.jsonl
+    bench_lv_pope: data/lv_bench/pope.jsonl
     bench_whatsup: {path: data/r13/bench/whatsup.jsonl, role: heldout}
-  commands:
-    - "uv run python scripts/r13b/judge.py --rules scripts/r14b/rules.json final {eval_dir} runs/r13b-v12-base/stage2-recal || true"
-    - "uv run python scripts/lv_report.py {eval_dir} --prev runs/v12-ext/eval || true"
+    bench_countbench: {path: data/r13/bench/countbench.jsonl, role: heldout}
+    bench_cogent_b: {path: data/r13/bench/cogent_b.jsonl, role: heldout}
+    test_count_coco: {path: data/n2/test_count_coco.jsonl, role: heldout}
+    bench_mmbench: {path: data/n2/bench/mmbench.jsonl, role: heldout}
+    bench_cvbench: {path: data/n2/bench/cvbench.jsonl, role: heldout}
+    bench_mme: {path: data/n2/bench/mme.jsonl, role: heldout}
+    bench_naturalbench: {path: data/n2/bench/naturalbench.jsonl, role: heldout}
+    dev_sqa: data/n2/dev_sqa.jsonl

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:110fea95889947c373258c88a396ad3eec574971815ee437b9c7cc3df8a5d2d5
+oid sha256:fda0f969a79cf77e6bb9288ddd38e4bb377f3f677018e67275ef927bf76b0397
 size 2073754784

provenance.json

@@ -1,43 +1,40 @@
 {
- "version": "v0.3",
- "git_tag": "model-v0.3",
- "built_from_commit": "85956234d5e6638618b1d5c8c0907709575df3d5",
- "round": "r15-vision",
- "stage": "stage2",
- "training_commit": "aea46fc3c4bb753c13cde0a81e8261815399d9e9+dirty",
- "training_started": "2026-10-14 12:57:49",
+ "version": "v0.4",
+ "git_tag": "model-v0.4",
+ "built_from_commit": "a6b4e8af3290255ce861e63397a0ec399b9f49c8",
+ "round": "r21-soft",
+ "stage": "train",
+ "training_commit": "d228b648f8a36628e46fd0e1e212f84af17522e2+dirty",
+ "training_started": "2026-10-22 03:03:02",
  "stages": [
   {
-   "name": "stage2",
-   "out": "runs/r15-vision/stage2",
-   "init": "runs/v12-ext/stage2"
+   "name": "train",
+   "out": "runs/r21-soft/train",
+   "init": "runs/r20-science/train"
   }
  ],
  "training_config": {
-  "name": "r15-vision",
-  "device": "mps",
+  "name": "r21-soft",
+  "device": "cuda",
   "common": {
    "log_every": 100
   },
   "stages": [
    {
-    "name": "stage2",
+    "name": "train",
     "kind": "train",
-    "init": "runs/v12-ext/stage2",
-    "data": "data/r14b/train.jsonl",
-    "val": "data/r13b/val.jsonl",
+    "init": "runs/r20-science/train",
+    "data": "data/n2/soft_mix.jsonl",
+    "val": "data/v5/dev_mix.jsonl",
     "eval": {
-     "exist": "data/r13b/panel_exist.jsonl",
-     "science": "data/r13b/panel_science.jsonl",
-     "ext": "data/ext/dev_ext.jsonl",
-     "bind_gqa": "data/r13/gqa/dev_bind_gqa.jsonl",
-     "count_narrow": "data/v11/dev_count_transfer.jsonl",
-     "count_wide": "data/r13b/count_wide.jsonl",
-     "px_wide": "data/exp14/pixmo_count_wide.jsonl",
-     "px_narrow": "data/exp14/pixmo_count_narrow.jsonl",
-     "px_exist": "data/exp14/pixmo_exist.jsonl",
-     "flip": "data/v6/dev_lr_pairs.jsonl",
-     "vsr": "data/v5/dev_vsr.jsonl"
+     "sqa": "data/n2/dev_sqa.jsonl",
+     "v7w": "data/n2/dev_v7w.jsonl",
+     "spatial": "data/n2/dev_spatial.jsonl",
+     "count": "data/n2/dev_count.jsonl",
+     "iconqa": "data/n2/dev_iconqa.jsonl",
+     "tallyqa": "data/n2/dev_tallyqa.jsonl",
+     "sqa_synth": "data/n2/dev_sqa_synth.jsonl",
+     "clevr": "data/n2/dev_clevr.jsonl"
     },
     "params": {
      "epochs": 1,
@@ -45,68 +42,17 @@
      "lr_head": 1e-05,
      "lr_lora": 5e-05,
      "warmup": 300,
-     "eval_every": 2000,
-     "save_every": 500,
-     "amp": "off",
-     "seed": 1,
-     "monitor_details": true,
+     "eval_every": 5000,
      "vision_unfreeze_top": 4,
      "lr_vision": 5e-06,
-     "start_check": "runs/exp16-r14b-s1/stage2/monitor/step0",
-     "start_tolerance": 1e-05
+     "seed": 0
     }
    }
   ],
   "evaluate": {
-   "calibration_fit": "data/r13b/val.jsonl",
-   "history": [
-    "data/ext/dev_ext.jsonl",
-    "data/r13/gqa/dev_bind_gqa.jsonl",
-    "data/r13b/panel_exist.jsonl",
-    "data/r13b/panel_science.jsonl",
-    "data/r13b/count_wide.jsonl",
-    "data/v11/dev_count_transfer.jsonl",
-    "data/exp14/pixmo_count_wide.jsonl",
-    "data/exp14/pixmo_count_narrow.jsonl",
-    "data/exp14/pixmo_exist.jsonl",
-    "data/v6/dev_lr_pairs.jsonl",
-    "data/v5/dev_vsr.jsonl"
-   ],
    "image_identity": "data/v5/image_identity.json",
    "controls": true,
    "sets": {
-    "exist": {
-     "path": "data/r13b/panel_exist.jsonl",
-     "role": "monitoring"
-    },
-    "science": {
-     "path": "data/r13b/panel_science.jsonl",
-     "role": "monitoring"
-    },
-    "ext": {
-     "path": "data/ext/dev_ext.jsonl",
-     "role": "monitoring"
-    },
-    "bind_gqa": {
-     "path": "data/r13/gqa/dev_bind_gqa.jsonl",
-     "role": "monitoring"
-    },
-    "count_narrow": {
-     "path": "data/v11/dev_count_transfer.jsonl",
-     "role": "monitoring"
-    },
-    "count_wide": {
-     "path": "data/r13b/count_wide.jsonl",
-     "role": "monitoring"
-    },
-    "flip": {
-     "path": "data/v6/dev_lr_pairs.jsonl",
-     "role": "monitoring"
-    },
-    "vsr": {
-     "path": "data/v5/dev_vsr.jsonl",
-     "role": "monitoring"
-    },
     "dev_mix": "data/v5/dev_mix.jsonl",
     "test_vsr": "data/v5/test_vsr.jsonl",
     "test_v7w": "data/v5/test_v7w.jsonl",
@@ -124,28 +70,41 @@
     "bench_lv_aokvqa": "data/lv_bench/aokvqa.jsonl",
     "bench_lv_scienceqa": "data/lv_bench/scienceqa.jsonl",
     "bench_lv_pope": "data/lv_bench/pope.jsonl",
-    "test_count_fresh": {
-     "path": "data/v11/test_count_fresh.jsonl",
-     "role": "heldout"
-    },
-    "test_bind_gqa": {
-     "path": "data/r13/gqa/test_bind_gqa.jsonl",
+    "bench_whatsup": {
+     "path": "data/r13/bench/whatsup.jsonl",
      "role": "heldout"
     },
     "bench_countbench": {
      "path": "data/r13/bench/countbench.jsonl",
      "role": "heldout"
     },
-    "bench_whatsup": {
-     "path": "data/r13/bench/whatsup.jsonl",
+    "bench_cogent_b": {
+     "path": "data/r13/bench/cogent_b.jsonl",
      "role": "heldout"
-    }
-   },
-   "commands": [
-    "uv run python scripts/r13b/judge.py --rules scripts/r14b/rules.json final {eval_dir} runs/r13b-v12-base/stage2-recal || true",
-    "uv run python scripts/lv_report.py {eval_dir} --prev runs/v12-ext/eval || true"
-   ]
+    },
+    "test_count_coco": {
+     "path": "data/n2/test_count_coco.jsonl",
+     "role": "heldout"
+    },
+    "bench_mmbench": {
+     "path": "data/n2/bench/mmbench.jsonl",
+     "role": "heldout"
+    },
+    "bench_cvbench": {
+     "path": "data/n2/bench/cvbench.jsonl",
+     "role": "heldout"
+    },
+    "bench_mme": {
+     "path": "data/n2/bench/mme.jsonl",
+     "role": "heldout"
+    },
+    "bench_naturalbench": {
+     "path": "data/n2/bench/naturalbench.jsonl",
+     "role": "heldout"
+    },
+    "dev_sqa": "data/n2/dev_sqa.jsonl"
+   }
   }
  },
- "source_checkpoint_sha256": "110fea95889947c373258c88a396ad3eec574971815ee437b9c7cc3df8a5d2d5"
+ "source_checkpoint_sha256": "fda0f969a79cf77e6bb9288ddd38e4bb377f3f677018e67275ef927bf76b0397"
 }

tokenizer/tokenizer_config.json

@@ -3,7 +3,7 @@
   "clean_up_tokenization_spaces": true,
   "cls_token": "[CLS]",
   "is_local": true,
-  "local_files_only": false,
+  "local_files_only": true,
   "mask_token": "[MASK]",
   "model_input_names": [
     "input_ids",