| { |
| "schema_version": "1.0", |
| "model_name": "UHR-BAT2", |
| "checkpoint_logical_id": "qwen3vl-8b-geollava-ft3-selected-10k", |
| "audit_date": "2026-10-13", |
| "accuracy_unit": "percent", |
| "results": [ |
| { |
| "dataset_id": "rshr", |
| "dataset_name": "RSHR-Bench", |
| "reported_scope": "All 1,932 QA records; single-turn Overall is reported separately.", |
| "protocol_id": "20260913_strict_sets_matched_schedule_v1", |
| "overall": { |
| "n": 1932, |
| "correct": 980, |
| "accuracy_pct": 50.72463768115942 |
| }, |
| "subsets": { |
| "single_turn_1590": { |
| "n": 1590, |
| "correct": 726, |
| "accuracy_pct": 45.660377358490564 |
| }, |
| "multi_turn_342": { |
| "n": 342, |
| "correct": 254, |
| "accuracy_pct": 74.26900584795321 |
| } |
| }, |
| "categories": { |
| "anomaly_detection": { |
| "n": 50, |
| "correct": 37, |
| "accuracy_pct": 74.0 |
| }, |
| "color": { |
| "n": 200, |
| "correct": 135, |
| "accuracy_pct": 67.5 |
| }, |
| "detection": { |
| "n": 100, |
| "correct": 32, |
| "accuracy_pct": 32.0 |
| }, |
| "future_prediction_two_image": { |
| "n": 50, |
| "correct": 27, |
| "accuracy_pct": 54.0 |
| }, |
| "multi_region_join_contrast": { |
| "n": 20, |
| "correct": 13, |
| "accuracy_pct": 65.0 |
| }, |
| "multi_region_join_contrast_singal": { |
| "n": 50, |
| "correct": 22, |
| "accuracy_pct": 44.0 |
| }, |
| "multitrun-future_prediction": { |
| "n": 150, |
| "correct": 100, |
| "accuracy_pct": 66.66666666666667 |
| }, |
| "multiturn_object_state_judgement": { |
| "n": 132, |
| "correct": 112, |
| "accuracy_pct": 84.84848484848484 |
| }, |
| "multturn_anomaly_detection": { |
| "n": 60, |
| "correct": 42, |
| "accuracy_pct": 70.0 |
| }, |
| "object_clasfication": { |
| "n": 200, |
| "correct": 112, |
| "accuracy_pct": 56.0 |
| }, |
| "object_counting": { |
| "n": 200, |
| "correct": 33, |
| "accuracy_pct": 16.5 |
| }, |
| "object_grounding": { |
| "n": 200, |
| "correct": 74, |
| "accuracy_pct": 37.0 |
| }, |
| "object_state_judgement": { |
| "n": 50, |
| "correct": 30, |
| "accuracy_pct": 60.0 |
| }, |
| "regional_grouding": { |
| "n": 70, |
| "correct": 33, |
| "accuracy_pct": 47.142857142857146 |
| }, |
| "regionla_counting": { |
| "n": 100, |
| "correct": 43, |
| "accuracy_pct": 43.0 |
| }, |
| "relation": { |
| "n": 200, |
| "correct": 108, |
| "accuracy_pct": 54.0 |
| }, |
| "shape": { |
| "n": 100, |
| "correct": 27, |
| "accuracy_pct": 27.0 |
| } |
| }, |
| "observed_visual_tokens_per_qa": { |
| "mean": 1288.5150103519668, |
| "min": 1196, |
| "max": 3767 |
| }, |
| "source": { |
| "run_id": "rshr_iclr_required_20260913", |
| "condition_id": "rshr_dense_trace_r1_l28_s2171_normal", |
| "records_sha256": "fcd5b95230b836a73c2382f62523c56a3cc11bcceaef8794d80897da08124ec3", |
| "prediction_shards": [ |
| { |
| "logical_id": "rshr_dense_part0", |
| "n": 483, |
| "sha256": "39ef82fc64927463eee2ab57849b20d1494dd0580d2a8b194239a5a2426fc2d1" |
| }, |
| { |
| "logical_id": "rshr_dense_part1", |
| "n": 483, |
| "sha256": "287d33430fa9ee247f5814722fc61b1ee58e6c9b11cf58a1bcf38af10af29174" |
| }, |
| { |
| "logical_id": "rshr_dense_part2", |
| "n": 483, |
| "sha256": "5259fcf7867ddcbb9269906bc316c8029830b7586d9fef69f3badc84e5a058ca" |
| }, |
| { |
| "logical_id": "rshr_dense_part3", |
| "n": 483, |
| "sha256": "4dcbd1e22bed383fbfa04de32e7482828d9850d2edc9c1f0fdd3ee4da1f9f4ce" |
| } |
| ] |
| }, |
| "verification": { |
| "unique_indices": 1932, |
| "all_expected_indices_present": true, |
| "correctness_recomputed_from_answer_prediction": true, |
| "protocol_and_completion_verified": true |
| } |
| }, |
| { |
| "dataset_id": "xlrs", |
| "dataset_name": "XLRS-Bench-lite", |
| "reported_scope": "All 3,080 benchmark records; stored split label is train.", |
| "protocol_id": "20260913_strict_sets_matched_schedule_v1", |
| "overall": { |
| "n": 3080, |
| "correct": 1610, |
| "accuracy_pct": 52.27272727272727 |
| }, |
| "subsets": { |
| "single_choice_2980": { |
| "n": 2980, |
| "correct": 1563, |
| "accuracy_pct": 52.4496644295302 |
| }, |
| "multiple_choice_100": { |
| "n": 100, |
| "correct": 47, |
| "accuracy_pct": 47.0 |
| } |
| }, |
| "categories": { |
| "Complex reasoning/Anomaly Detection and Interpretation": { |
| "n": 100, |
| "correct": 73, |
| "accuracy_pct": 73.0 |
| }, |
| "Complex reasoning/Environmental condition reasoning": { |
| "n": 100, |
| "correct": 80, |
| "accuracy_pct": 80.0 |
| }, |
| "Complex reasoning/Route planning": { |
| "n": 100, |
| "correct": 50, |
| "accuracy_pct": 50.0 |
| }, |
| "Counting/Counting with changing detection": { |
| "n": 60, |
| "correct": 30, |
| "accuracy_pct": 50.0 |
| }, |
| "Counting/Counting with complex reasoning": { |
| "n": 100, |
| "correct": 55, |
| "accuracy_pct": 55.0 |
| }, |
| "Counting/Overall counting": { |
| "n": 60, |
| "correct": 22, |
| "accuracy_pct": 36.666666666666664 |
| }, |
| "Counting/Regional counting": { |
| "n": 100, |
| "correct": 47, |
| "accuracy_pct": 47.0 |
| }, |
| "Land use classification/Overall Land use classification": { |
| "n": 100, |
| "correct": 47, |
| "accuracy_pct": 47.0 |
| }, |
| "Land use classification/Regional Land use classification": { |
| "n": 200, |
| "correct": 169, |
| "accuracy_pct": 84.5 |
| }, |
| "Object properties/Object classification": { |
| "n": 800, |
| "correct": 422, |
| "accuracy_pct": 52.75 |
| }, |
| "Object properties/Object color": { |
| "n": 800, |
| "correct": 374, |
| "accuracy_pct": 46.75 |
| }, |
| "Object properties/Object motion state": { |
| "n": 60, |
| "correct": 39, |
| "accuracy_pct": 65.0 |
| }, |
| "Object spatial relationship/Object spatial relationship": { |
| "n": 500, |
| "correct": 202, |
| "accuracy_pct": 40.4 |
| } |
| }, |
| "observed_visual_tokens_per_qa": { |
| "mean": 1252.3522727272727, |
| "min": 1224, |
| "max": 2450 |
| }, |
| "source": { |
| "run_id": "rshr_iclr_required_20260913", |
| "condition_id": "xlrs_dense_trace_r1_l28_s2171_normal", |
| "records_sha256": "7307b5303fc72f1aeec29aee3cd442193b30912c1c28a3bb410d84e8d278c4a6", |
| "prediction_shards": [ |
| { |
| "logical_id": "xlrs_dense_part0", |
| "n": 770, |
| "sha256": "f8029b0a9877f6b7b1efbbd1614a2342f51adb44e3d4d03acd16cf729b626fdf" |
| }, |
| { |
| "logical_id": "xlrs_dense_part1", |
| "n": 770, |
| "sha256": "f19b5453e834e7db2bb1b6626be7ade53478fe5f57accd536c56c8a5edf11cd7" |
| }, |
| { |
| "logical_id": "xlrs_dense_part2", |
| "n": 770, |
| "sha256": "ff5496c422f23964f96bd7ce1cbb2069d3ddb83ad276578276a2c5590f3fa16b" |
| }, |
| { |
| "logical_id": "xlrs_dense_part3", |
| "n": 770, |
| "sha256": "085bdcb9c4175a96a15a96d3dc525eef8ccf85769e98cf2c31ecd0fb18387ec7" |
| } |
| ] |
| }, |
| "verification": { |
| "unique_indices": 3080, |
| "all_expected_indices_present": true, |
| "correctness_recomputed_from_answer_prediction": true, |
| "protocol_and_completion_verified": true |
| } |
| }, |
| { |
| "dataset_id": "mme", |
| "dataset_name": "MME-RealWorld Remote Sensing", |
| "reported_scope": "All 3,738 Remote Sensing records; not the Lite subset.", |
| "protocol_id": "20260913_strict_sets_matched_schedule_v1", |
| "overall": { |
| "n": 3738, |
| "correct": 2123, |
| "accuracy_pct": 56.795077581594434 |
| }, |
| "subsets": {}, |
| "categories": { |
| "color": { |
| "n": 1255, |
| "correct": 855, |
| "accuracy_pct": 68.12749003984064 |
| }, |
| "count": { |
| "n": 1226, |
| "correct": 337, |
| "accuracy_pct": 27.487765089722675 |
| }, |
| "position": { |
| "n": 1257, |
| "correct": 931, |
| "accuracy_pct": 74.06523468575975 |
| } |
| }, |
| "observed_visual_tokens_per_qa": { |
| "mean": 1235.5414660246122, |
| "min": 726, |
| "max": 1278 |
| }, |
| "source": { |
| "run_id": "rshr_iclr_required_20260913", |
| "condition_id": "mme_dense_trace_r1_l28_s2171_normal", |
| "records_sha256": "edd85dc4f5fbe3ed86145d83fc4c928998287c75c3f32bbdc56581b0cf9a6380", |
| "prediction_shards": [ |
| { |
| "logical_id": "mme_dense_part0", |
| "n": 935, |
| "sha256": "918f3d43161bcbdf4a1a61164aa19ef5d1949a9d8fb54980f22c1d2459971dc5" |
| }, |
| { |
| "logical_id": "mme_dense_part1", |
| "n": 935, |
| "sha256": "aa474f1284217febafee6e6931cb31dcacf92ea8e69aea3dc3d89c8b36508320" |
| }, |
| { |
| "logical_id": "mme_dense_part2", |
| "n": 934, |
| "sha256": "aa91f231aed2e805964e219d08a940913632813a024793c64ef60360562ab9ef" |
| }, |
| { |
| "logical_id": "mme_dense_part3", |
| "n": 934, |
| "sha256": "99c4968a8723a3090f8f2b4abc248c3c0b57eebaa25f9c5fba2e2a4483039106" |
| } |
| ] |
| }, |
| "verification": { |
| "unique_indices": 3738, |
| "all_expected_indices_present": true, |
| "correctness_recomputed_from_answer_prediction": true, |
| "protocol_and_completion_verified": true |
| } |
| } |
| ], |
| "protocols": { |
| "20260913_strict_sets_matched_schedule_v1": { |
| "model_dtype": "bf16", |
| "attention_implementation": "eager", |
| "min_pixels": 784, |
| "max_pixels": 1310720, |
| "pruning": "none", |
| "single_choice_scoring": "Argmax of candidate option logits computed using the final hidden state and FP32 option rows of the language-model head.", |
| "xlrs_multiple_choice_scoring": "Constrained greedy token-trie decoding with FP32 candidate grammar rows, followed by exact canonical option-set matching.", |
| "xlrs_multiple_choice_task": "Land use classification/Overall Land use classification", |
| "source_code_sha256": { |
| "worker.py": "bdff71b17de6f4b20b5d747ba173ad32563249af20fb035a5db2abadc5676573", |
| "protocol.py": "e328cb2d29a8e309442f0fdbf7c8465ac823fa37962db17f6a9e9f233b64bf3f", |
| "vendor/pruning_core.py": "1207c4c3f819b472bef5e47103015399e7d193b1975814c3b5e5ba5aefb01321", |
| "vendor/qwen3vl_pruning_eval.py": "471b68fe5f519b144cbf6daaf452cdd838e286dce0574cde5743bc9fac018b98" |
| }, |
| "verification": "All three datasets passed complete unique-index, answer/prediction correctness, protocol-sidecar and completion-marker checks. Recorded checkpoint/code/protocol identities match across their shards and conditions." |
| } |
| }, |
| "limitations": [ |
| "A standard model.generate quickstart demonstrates inference only. It does not reproduce the FP32 option-logit or constrained option-set scoring used for the three corrected dense benchmarks.", |
| "Dense means no token pruning within the stated image preprocessing budget. These scores do not establish native 4K/8K or full-resolution ultra-high-resolution perception.", |
| "RSHR all-QA accuracy and its 1,590-record single-turn Overall metric use different denominators and must not be interchanged.", |
| "Historical XLRS results from the single-letter prompt/normalizer are excluded because multi-option ground truths were truncated. They are not valid strict option-set accuracy.", |
| "No same-protocol gain over the original base model is claimed by this artifact.", |
| "This audit rechecked existing evaluation artifacts; no new model inference was run on the audit date.", |
| "Raw prediction files, benchmark images and questions are not included in this aggregate artifact. Hashes identify the audited source artifacts but do not constitute a complete public reproduction package." |
| ] |
| } |
|
|
