mis2 commited on
Commit
78ac017
·
verified ·
1 Parent(s): 6ce30e3

Release ViSS-8B weights, processors and source code

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .gitattributes +2 -0
  2. CITATION.cff +29 -0
  3. LICENSE +202 -0
  4. README.md +113 -0
  5. added_tokens.json +28 -0
  6. chat_template.jinja +120 -0
  7. code/.gitignore +20 -0
  8. code/LICENSE +202 -0
  9. code/NOTICE +16 -0
  10. code/README.md +135 -0
  11. code/configs/evaluation/recent4.example.json +74 -0
  12. code/configs/training/deepspeed_zero3_bf16.json +16 -0
  13. code/data_pipeline/__init__.py +1 -0
  14. code/data_pipeline/temporal_scope/__init__.py +1 -0
  15. code/data_pipeline/temporal_scope/build_size_matched_generic_control.py +107 -0
  16. code/data_pipeline/temporal_scope/build_source_aligned_sft_release.py +722 -0
  17. code/data_pipeline/temporal_scope/build_vst_prefix32_semantic.py +525 -0
  18. code/data_pipeline/temporal_scope/filter_semantic_release_by_blind_audit.py +329 -0
  19. code/data_pipeline/temporal_scope/filter_semantic_release_by_option_consensus.py +298 -0
  20. code/data_pipeline/temporal_scope/select_quality_capability_rows.py +420 -0
  21. code/data_pipeline/temporal_scope/semantic_mcq.py +90 -0
  22. code/docs/DATA.md +63 -0
  23. code/docs/EVALUATION.md +114 -0
  24. code/docs/INSTALL.md +59 -0
  25. code/docs/TRAINING.md +50 -0
  26. code/experiments/analyze_uncertainty_temporal_escalation.py +451 -0
  27. code/experiments/evaluate_qwen3vl_recent4_full.py +0 -0
  28. code/experiments/materialize_confidence_gated_third_view_agent.py +259 -0
  29. code/experiments/materialize_third_view_disagreement_agent.py +203 -0
  30. code/experiments/materialize_tvs_view_oracle.py +118 -0
  31. code/experiments/materialize_uncertainty_temporal_agent.py +465 -0
  32. code/experiments/select_uncertainty_escalation_uids.py +192 -0
  33. code/experiments/train_qwen3vl_sft.py +1091 -0
  34. code/experiments/tvs_correctness.py +80 -0
  35. code/pyproject.toml +32 -0
  36. code/scripts/__init__.py +1 -0
  37. code/scripts/configure_evaluation_protocol.py +62 -0
  38. code/scripts/merge_eval_shards.py +483 -0
  39. code/scripts/run_recent4_single_benchmark.sh +380 -0
  40. code/scripts/score_official_benchmark_predictions.py +348 -0
  41. code/scripts/validate_evaluation_protocol.py +28 -0
  42. code/tests/test_confidence_gated_third_view_agent.py +77 -0
  43. code/tests/test_filter_semantic_release_by_option_consensus.py +84 -0
  44. code/tests/test_materialize_uncertainty_temporal_agent.py +551 -0
  45. code/tests/test_public_protocol.py +47 -0
  46. code/tests/test_recent4_runner.py +893 -0
  47. code/tests/test_semantic_answer_loss_mask.py +108 -0
  48. code/tests/test_size_matched_generic_control.py +26 -0
  49. code/tests/test_source_aligned_sft_release.py +125 -0
  50. code/tests/test_third_view_disagreement_agent.py +78 -0
.gitattributes CHANGED
@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ locked_processor/tokenizer.json filter=lfs diff=lfs merge=lfs -text
37
+ tokenizer.json filter=lfs diff=lfs merge=lfs -text
CITATION.cff ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ cff-version: 1.2.0
2
+ message: "Please cite the ViSS manuscript when using this model and code."
3
+ title: "Supervising the Visible State: Separating Representation from View Selection in Streaming Video"
4
+ type: software
5
+ authors:
6
+ - family-names: Hu
7
+ given-names: Yihao
8
+ - family-names: Wang
9
+ given-names: Pan
10
+ - family-names: Lin
11
+ given-names: Yitong
12
+ - family-names: Wen
13
+ given-names: Zhihao
14
+ - family-names: Liu
15
+ given-names: Deyuan
16
+ - family-names: Wang
17
+ given-names: Hang
18
+ - family-names: Wan
19
+ given-names: Mingda
20
+ - family-names: Zhao
21
+ given-names: Yiwen
22
+ - family-names: Li
23
+ given-names: Yutong
24
+ - family-names: Mao
25
+ given-names: Yuchen
26
+ - family-names: Lin
27
+ given-names: Tao
28
+ license: Apache-2.0
29
+ url: "https://huggingface.co/mis2/ViSS-8B"
LICENSE ADDED
@@ -0,0 +1,202 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+ Apache License
3
+ Version 2.0, January 2004
4
+ http://www.apache.org/licenses/
5
+
6
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
7
+
8
+ 1. Definitions.
9
+
10
+ "License" shall mean the terms and conditions for use, reproduction,
11
+ and distribution as defined by Sections 1 through 9 of this document.
12
+
13
+ "Licensor" shall mean the copyright owner or entity authorized by
14
+ the copyright owner that is granting the License.
15
+
16
+ "Legal Entity" shall mean the union of the acting entity and all
17
+ other entities that control, are controlled by, or are under common
18
+ control with that entity. For the purposes of this definition,
19
+ "control" means (i) the power, direct or indirect, to cause the
20
+ direction or management of such entity, whether by contract or
21
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
22
+ outstanding shares, or (iii) beneficial ownership of such entity.
23
+
24
+ "You" (or "Your") shall mean an individual or Legal Entity
25
+ exercising permissions granted by this License.
26
+
27
+ "Source" form shall mean the preferred form for making modifications,
28
+ including but not limited to software source code, documentation
29
+ source, and configuration files.
30
+
31
+ "Object" form shall mean any form resulting from mechanical
32
+ transformation or translation of a Source form, including but
33
+ not limited to compiled object code, generated documentation,
34
+ and conversions to other media types.
35
+
36
+ "Work" shall mean the work of authorship, whether in Source or
37
+ Object form, made available under the License, as indicated by a
38
+ copyright notice that is included in or attached to the work
39
+ (an example is provided in the Appendix below).
40
+
41
+ "Derivative Works" shall mean any work, whether in Source or Object
42
+ form, that is based on (or derived from) the Work and for which the
43
+ editorial revisions, annotations, elaborations, or other modifications
44
+ represent, as a whole, an original work of authorship. For the purposes
45
+ of this License, Derivative Works shall not include works that remain
46
+ separable from, or merely link (or bind by name) to the interfaces of,
47
+ the Work and Derivative Works thereof.
48
+
49
+ "Contribution" shall mean any work of authorship, including
50
+ the original version of the Work and any modifications or additions
51
+ to that Work or Derivative Works thereof, that is intentionally
52
+ submitted to Licensor for inclusion in the Work by the copyright owner
53
+ or by an individual or Legal Entity authorized to submit on behalf of
54
+ the copyright owner. For the purposes of this definition, "submitted"
55
+ means any form of electronic, verbal, or written communication sent
56
+ to the Licensor or its representatives, including but not limited to
57
+ communication on electronic mailing lists, source code control systems,
58
+ and issue tracking systems that are managed by, or on behalf of, the
59
+ Licensor for the purpose of discussing and improving the Work, but
60
+ excluding communication that is conspicuously marked or otherwise
61
+ designated in writing by the copyright owner as "Not a Contribution."
62
+
63
+ "Contributor" shall mean Licensor and any individual or Legal Entity
64
+ on behalf of whom a Contribution has been received by Licensor and
65
+ subsequently incorporated within the Work.
66
+
67
+ 2. Grant of Copyright License. Subject to the terms and conditions of
68
+ this License, each Contributor hereby grants to You a perpetual,
69
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
70
+ copyright license to reproduce, prepare Derivative Works of,
71
+ publicly display, publicly perform, sublicense, and distribute the
72
+ Work and such Derivative Works in Source or Object form.
73
+
74
+ 3. Grant of Patent License. Subject to the terms and conditions of
75
+ this License, each Contributor hereby grants to You a perpetual,
76
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
77
+ (except as stated in this section) patent license to make, have made,
78
+ use, offer to sell, sell, import, and otherwise transfer the Work,
79
+ where such license applies only to those patent claims licensable
80
+ by such Contributor that are necessarily infringed by their
81
+ Contribution(s) alone or by combination of their Contribution(s)
82
+ with the Work to which such Contribution(s) was submitted. If You
83
+ institute patent litigation against any entity (including a
84
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
85
+ or a Contribution incorporated within the Work constitutes direct
86
+ or contributory patent infringement, then any patent licenses
87
+ granted to You under this License for that Work shall terminate
88
+ as of the date such litigation is filed.
89
+
90
+ 4. Redistribution. You may reproduce and distribute copies of the
91
+ Work or Derivative Works thereof in any medium, with or without
92
+ modifications, and in Source or Object form, provided that You
93
+ meet the following conditions:
94
+
95
+ (a) You must give any other recipients of the Work or
96
+ Derivative Works a copy of this License; and
97
+
98
+ (b) You must cause any modified files to carry prominent notices
99
+ stating that You changed the files; and
100
+
101
+ (c) You must retain, in the Source form of any Derivative Works
102
+ that You distribute, all copyright, patent, trademark, and
103
+ attribution notices from the Source form of the Work,
104
+ excluding those notices that do not pertain to any part of
105
+ the Derivative Works; and
106
+
107
+ (d) If the Work includes a "NOTICE" text file as part of its
108
+ distribution, then any Derivative Works that You distribute must
109
+ include a readable copy of the attribution notices contained
110
+ within such NOTICE file, excluding those notices that do not
111
+ pertain to any part of the Derivative Works, in at least one
112
+ of the following places: within a NOTICE text file distributed
113
+ as part of the Derivative Works; within the Source form or
114
+ documentation, if provided along with the Derivative Works; or,
115
+ within a display generated by the Derivative Works, if and
116
+ wherever such third-party notices normally appear. The contents
117
+ of the NOTICE file are for informational purposes only and
118
+ do not modify the License. You may add Your own attribution
119
+ notices within Derivative Works that You distribute, alongside
120
+ or as an addendum to the NOTICE text from the Work, provided
121
+ that such additional attribution notices cannot be construed
122
+ as modifying the License.
123
+
124
+ You may add Your own copyright statement to Your modifications and
125
+ may provide additional or different license terms and conditions
126
+ for use, reproduction, or distribution of Your modifications, or
127
+ for any such Derivative Works as a whole, provided Your use,
128
+ reproduction, and distribution of the Work otherwise complies with
129
+ the conditions stated in this License.
130
+
131
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
132
+ any Contribution intentionally submitted for inclusion in the Work
133
+ by You to the Licensor shall be under the terms and conditions of
134
+ this License, without any additional terms or conditions.
135
+ Notwithstanding the above, nothing herein shall supersede or modify
136
+ the terms of any separate license agreement you may have executed
137
+ with Licensor regarding such Contributions.
138
+
139
+ 6. Trademarks. This License does not grant permission to use the trade
140
+ names, trademarks, service marks, or product names of the Licensor,
141
+ except as required for reasonable and customary use in describing the
142
+ origin of the Work and reproducing the content of the NOTICE file.
143
+
144
+ 7. Disclaimer of Warranty. Unless required by applicable law or
145
+ agreed to in writing, Licensor provides the Work (and each
146
+ Contributor provides its Contributions) on an "AS IS" BASIS,
147
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
148
+ implied, including, without limitation, any warranties or conditions
149
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
150
+ PARTICULAR PURPOSE. You are solely responsible for determining the
151
+ appropriateness of using or redistributing the Work and assume any
152
+ risks associated with Your exercise of permissions under this License.
153
+
154
+ 8. Limitation of Liability. In no event and under no legal theory,
155
+ whether in tort (including negligence), contract, or otherwise,
156
+ unless required by applicable law (such as deliberate and grossly
157
+ negligent acts) or agreed to in writing, shall any Contributor be
158
+ liable to You for damages, including any direct, indirect, special,
159
+ incidental, or consequential damages of any character arising as a
160
+ result of this License or out of the use or inability to use the
161
+ Work (including but not limited to damages for loss of goodwill,
162
+ work stoppage, computer failure or malfunction, or any and all
163
+ other commercial damages or losses), even if such Contributor
164
+ has been advised of the possibility of such damages.
165
+
166
+ 9. Accepting Warranty or Additional Liability. While redistributing
167
+ the Work or Derivative Works thereof, You may choose to offer,
168
+ and charge a fee for, acceptance of support, warranty, indemnity,
169
+ or other liability obligations and/or rights consistent with this
170
+ License. However, in accepting such obligations, You may act only
171
+ on Your own behalf and on Your sole responsibility, not on behalf
172
+ of any other Contributor, and only if You agree to indemnify,
173
+ defend, and hold each Contributor harmless for any liability
174
+ incurred by, or claims asserted against, such Contributor by reason
175
+ of your accepting any such warranty or additional liability.
176
+
177
+ END OF TERMS AND CONDITIONS
178
+
179
+ APPENDIX: How to apply the Apache License to your work.
180
+
181
+ To apply the Apache License to your work, attach the following
182
+ boilerplate notice, with the fields enclosed by brackets "[]"
183
+ replaced with your own identifying information. (Don't include
184
+ the brackets!) The text should be enclosed in the appropriate
185
+ comment syntax for the file format. We also recommend that a
186
+ file or class name and description of purpose be included on the
187
+ same "printed page" as the copyright notice for easier
188
+ identification within third-party archives.
189
+
190
+ Copyright [yyyy] [name of copyright owner]
191
+
192
+ Licensed under the Apache License, Version 2.0 (the "License");
193
+ you may not use this file except in compliance with the License.
194
+ You may obtain a copy of the License at
195
+
196
+ http://www.apache.org/licenses/LICENSE-2.0
197
+
198
+ Unless required by applicable law or agreed to in writing, software
199
+ distributed under the License is distributed on an "AS IS" BASIS,
200
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
201
+ See the License for the specific language governing permissions and
202
+ limitations under the License.
README.md ADDED
@@ -0,0 +1,113 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ license: apache-2.0
3
+ base_model: Qwen/Qwen3-VL-8B-Instruct
4
+ library_name: transformers
5
+ pipeline_tag: image-text-to-text
6
+ tags:
7
+ - qwen3-vl
8
+ - viss
9
+ - streaming-video
10
+ - visual-question-answering
11
+ - causal-video
12
+ ---
13
+
14
+ # ViSS-8B
15
+
16
+ **Supervising the Visible State: Separating Representation from View Selection in Streaming Video**
17
+
18
+ Yihao Hu*, Pan Wang*, Yitong Lin*, Zhihao Wen†, Deyuan Liu, Hang Wang,
19
+ Mingda Wan, Yiwen Zhao, Yutong Li, Yuchen Mao, Tao Lin†
20
+
21
+ *Equal contribution. †Corresponding author.*
22
+
23
+ [Collection](https://huggingface.co/collections/mis2/viss-6adc8caf041ae0c501c9a11b) ·
24
+ [Source code](https://huggingface.co/mis2/ViSS-8B/tree/main/code) ·
25
+ [Installation](code/docs/INSTALL.md) ·
26
+ [Training](code/docs/TRAINING.md) ·
27
+ [Evaluation](code/docs/EVALUATION.md)
28
+
29
+ ## Overview
30
+
31
+ Visible-State Supervision (ViSS) trains a vision–language model on
32
+ source-grounded questions about event identity, order, accumulation,
33
+ relation changes, and whether the available evidence suffices to answer.
34
+ Test-Time Causal View Search (TVS) keeps the model fixed and acquires
35
+ complementary recent, event, and compact-history views.
36
+
37
+ The model is initialized from Qwen3-VL-8B-Instruct and trained for one epoch
38
+ on 7,677 examples, with learning rate 2e-6 and global batch size 12.
39
+ The vision tower is frozen; the language model and multimodal projector
40
+ are trainable. Model-only ViSS and TVS use the same checkpoint.
41
+
42
+ ## Released files
43
+
44
+ - Four original BF16 safetensors shards (17.53 GB in total).
45
+ - Model configuration, tokenizer, chat template, and native processor.
46
+ - `locked_processor/`: the processor used for the recent-view evaluation.
47
+ - `code/`: training, inference, TVS composition, evaluation adapters,
48
+ public configuration templates, documentation, and synthetic tests.
49
+
50
+ The release contains no training annotations, raw media, company experiment
51
+ records, prediction files, evaluation summaries, optimizer states, or internal
52
+ training manifests. Training data are not being released in this version.
53
+
54
+ ## Download
55
+
56
+ ```bash
57
+ hf download mis2/ViSS-8B --local-dir checkpoints/ViSS-8B
58
+ ```
59
+
60
+ ## Load the model
61
+
62
+ Use a Transformers version supporting Qwen3-VL; the retained model
63
+ configuration specifies Transformers 4.57.3.
64
+
65
+ ```python
66
+ import torch
67
+ from transformers import AutoProcessor, Qwen3VLForConditionalGeneration
68
+
69
+ model_id = "mis2/ViSS-8B"
70
+ processor = AutoProcessor.from_pretrained(
71
+ model_id, subfolder="locked_processor"
72
+ )
73
+ model = Qwen3VLForConditionalGeneration.from_pretrained(
74
+ model_id,
75
+ dtype=torch.bfloat16,
76
+ device_map="auto",
77
+ )
78
+ ```
79
+
80
+ The root processor is the native training processor. The
81
+ `locked_processor/` subdirectory preserves the recent-view evaluation
82
+ processor. The supplied evaluator implements causal sampling and benchmark
83
+ prompt conventions; loading the processor alone does not implement TVS.
84
+
85
+ ## Use the code
86
+
87
+ ```bash
88
+ cd checkpoints/ViSS-8B/code
89
+ python -m pip install -e ".[inference,test]"
90
+ python -m pytest -q tests
91
+ ```
92
+
93
+ See the evaluation guide to bind the public configuration template to your
94
+ own processor, benchmark query exports, and model. Benchmark media and
95
+ official scoring installations must be obtained separately. TVS additionally
96
+ requires calibration inputs prepared independently of benchmark evaluation.
97
+
98
+ ## License
99
+
100
+ The source code follows the project's Apache-2.0 license. The model is
101
+ derived from Qwen3-VL-8B-Instruct, which is also distributed under Apache-2.0.
102
+ External benchmark and media terms remain with their respective providers.
103
+
104
+ ## Citation
105
+
106
+ ```bibtex
107
+ @misc{hu2026viss,
108
+ title={Supervising the Visible State: Separating Representation from View Selection in Streaming Video},
109
+ author={Yihao Hu and Pan Wang and Yitong Lin and Zhihao Wen and Deyuan Liu and Hang Wang and Mingda Wan and Yiwen Zhao and Yutong Li and Yuchen Mao and Tao Lin},
110
+ year={2026},
111
+ note={Manuscript}
112
+ }
113
+ ```
added_tokens.json ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "</think>": 151668,
3
+ "</tool_call>": 151658,
4
+ "</tool_response>": 151666,
5
+ "<think>": 151667,
6
+ "<tool_call>": 151657,
7
+ "<tool_response>": 151665,
8
+ "<|box_end|>": 151649,
9
+ "<|box_start|>": 151648,
10
+ "<|endoftext|>": 151643,
11
+ "<|file_sep|>": 151664,
12
+ "<|fim_middle|>": 151660,
13
+ "<|fim_pad|>": 151662,
14
+ "<|fim_prefix|>": 151659,
15
+ "<|fim_suffix|>": 151661,
16
+ "<|im_end|>": 151645,
17
+ "<|im_start|>": 151644,
18
+ "<|image_pad|>": 151655,
19
+ "<|object_ref_end|>": 151647,
20
+ "<|object_ref_start|>": 151646,
21
+ "<|quad_end|>": 151651,
22
+ "<|quad_start|>": 151650,
23
+ "<|repo_name|>": 151663,
24
+ "<|video_pad|>": 151656,
25
+ "<|vision_end|>": 151653,
26
+ "<|vision_pad|>": 151654,
27
+ "<|vision_start|>": 151652
28
+ }
chat_template.jinja ADDED
@@ -0,0 +1,120 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {%- if tools %}
2
+ {{- '<|im_start|>system\n' }}
3
+ {%- if messages[0].role == 'system' %}
4
+ {%- if messages[0].content is string %}
5
+ {{- messages[0].content }}
6
+ {%- else %}
7
+ {%- for content in messages[0].content %}
8
+ {%- if 'text' in content %}
9
+ {{- content.text }}
10
+ {%- endif %}
11
+ {%- endfor %}
12
+ {%- endif %}
13
+ {{- '\n\n' }}
14
+ {%- endif %}
15
+ {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
16
+ {%- for tool in tools %}
17
+ {{- "\n" }}
18
+ {{- tool | tojson }}
19
+ {%- endfor %}
20
+ {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
21
+ {%- else %}
22
+ {%- if messages[0].role == 'system' %}
23
+ {{- '<|im_start|>system\n' }}
24
+ {%- if messages[0].content is string %}
25
+ {{- messages[0].content }}
26
+ {%- else %}
27
+ {%- for content in messages[0].content %}
28
+ {%- if 'text' in content %}
29
+ {{- content.text }}
30
+ {%- endif %}
31
+ {%- endfor %}
32
+ {%- endif %}
33
+ {{- '<|im_end|>\n' }}
34
+ {%- endif %}
35
+ {%- endif %}
36
+ {%- set image_count = namespace(value=0) %}
37
+ {%- set video_count = namespace(value=0) %}
38
+ {%- for message in messages %}
39
+ {%- if message.role == "user" %}
40
+ {{- '<|im_start|>' + message.role + '\n' }}
41
+ {%- if message.content is string %}
42
+ {{- message.content }}
43
+ {%- else %}
44
+ {%- for content in message.content %}
45
+ {%- if content.type == 'image' or 'image' in content or 'image_url' in content %}
46
+ {%- set image_count.value = image_count.value + 1 %}
47
+ {%- if add_vision_id %}Picture {{ image_count.value }}: {% endif -%}
48
+ <|vision_start|><|image_pad|><|vision_end|>
49
+ {%- elif content.type == 'video' or 'video' in content %}
50
+ {%- set video_count.value = video_count.value + 1 %}
51
+ {%- if add_vision_id %}Video {{ video_count.value }}: {% endif -%}
52
+ <|vision_start|><|video_pad|><|vision_end|>
53
+ {%- elif 'text' in content %}
54
+ {{- content.text }}
55
+ {%- endif %}
56
+ {%- endfor %}
57
+ {%- endif %}
58
+ {{- '<|im_end|>\n' }}
59
+ {%- elif message.role == "assistant" %}
60
+ {{- '<|im_start|>' + message.role + '\n' }}
61
+ {%- if message.content is string %}
62
+ {{- message.content }}
63
+ {%- else %}
64
+ {%- for content_item in message.content %}
65
+ {%- if 'text' in content_item %}
66
+ {{- content_item.text }}
67
+ {%- endif %}
68
+ {%- endfor %}
69
+ {%- endif %}
70
+ {%- if message.tool_calls %}
71
+ {%- for tool_call in message.tool_calls %}
72
+ {%- if (loop.first and message.content) or (not loop.first) %}
73
+ {{- '\n' }}
74
+ {%- endif %}
75
+ {%- if tool_call.function %}
76
+ {%- set tool_call = tool_call.function %}
77
+ {%- endif %}
78
+ {{- '<tool_call>\n{"name": "' }}
79
+ {{- tool_call.name }}
80
+ {{- '", "arguments": ' }}
81
+ {%- if tool_call.arguments is string %}
82
+ {{- tool_call.arguments }}
83
+ {%- else %}
84
+ {{- tool_call.arguments | tojson }}
85
+ {%- endif %}
86
+ {{- '}\n</tool_call>' }}
87
+ {%- endfor %}
88
+ {%- endif %}
89
+ {{- '<|im_end|>\n' }}
90
+ {%- elif message.role == "tool" %}
91
+ {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
92
+ {{- '<|im_start|>user' }}
93
+ {%- endif %}
94
+ {{- '\n<tool_response>\n' }}
95
+ {%- if message.content is string %}
96
+ {{- message.content }}
97
+ {%- else %}
98
+ {%- for content in message.content %}
99
+ {%- if content.type == 'image' or 'image' in content or 'image_url' in content %}
100
+ {%- set image_count.value = image_count.value + 1 %}
101
+ {%- if add_vision_id %}Picture {{ image_count.value }}: {% endif -%}
102
+ <|vision_start|><|image_pad|><|vision_end|>
103
+ {%- elif content.type == 'video' or 'video' in content %}
104
+ {%- set video_count.value = video_count.value + 1 %}
105
+ {%- if add_vision_id %}Video {{ video_count.value }}: {% endif -%}
106
+ <|vision_start|><|video_pad|><|vision_end|>
107
+ {%- elif 'text' in content %}
108
+ {{- content.text }}
109
+ {%- endif %}
110
+ {%- endfor %}
111
+ {%- endif %}
112
+ {{- '\n</tool_response>' }}
113
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
114
+ {{- '<|im_end|>\n' }}
115
+ {%- endif %}
116
+ {%- endif %}
117
+ {%- endfor %}
118
+ {%- if add_generation_prompt %}
119
+ {{- '<|im_start|>assistant\n' }}
120
+ {%- endif %}
code/.gitignore ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ __pycache__/
2
+ *.py[cod]
3
+ .pytest_cache/
4
+ *.egg-info/
5
+ build/
6
+ dist/
7
+ .venv/
8
+ outputs/
9
+ results/
10
+ logs/
11
+ checkpoints/
12
+ data/
13
+ records/
14
+ runs/
15
+ manifests/
16
+ configs/evaluation/recent4_full_v1.json
17
+ release_manifest.json
18
+ *.log
19
+ .env
20
+ .env.*
code/LICENSE ADDED
@@ -0,0 +1,202 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+ Apache License
3
+ Version 2.0, January 2004
4
+ http://www.apache.org/licenses/
5
+
6
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
7
+
8
+ 1. Definitions.
9
+
10
+ "License" shall mean the terms and conditions for use, reproduction,
11
+ and distribution as defined by Sections 1 through 9 of this document.
12
+
13
+ "Licensor" shall mean the copyright owner or entity authorized by
14
+ the copyright owner that is granting the License.
15
+
16
+ "Legal Entity" shall mean the union of the acting entity and all
17
+ other entities that control, are controlled by, or are under common
18
+ control with that entity. For the purposes of this definition,
19
+ "control" means (i) the power, direct or indirect, to cause the
20
+ direction or management of such entity, whether by contract or
21
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
22
+ outstanding shares, or (iii) beneficial ownership of such entity.
23
+
24
+ "You" (or "Your") shall mean an individual or Legal Entity
25
+ exercising permissions granted by this License.
26
+
27
+ "Source" form shall mean the preferred form for making modifications,
28
+ including but not limited to software source code, documentation
29
+ source, and configuration files.
30
+
31
+ "Object" form shall mean any form resulting from mechanical
32
+ transformation or translation of a Source form, including but
33
+ not limited to compiled object code, generated documentation,
34
+ and conversions to other media types.
35
+
36
+ "Work" shall mean the work of authorship, whether in Source or
37
+ Object form, made available under the License, as indicated by a
38
+ copyright notice that is included in or attached to the work
39
+ (an example is provided in the Appendix below).
40
+
41
+ "Derivative Works" shall mean any work, whether in Source or Object
42
+ form, that is based on (or derived from) the Work and for which the
43
+ editorial revisions, annotations, elaborations, or other modifications
44
+ represent, as a whole, an original work of authorship. For the purposes
45
+ of this License, Derivative Works shall not include works that remain
46
+ separable from, or merely link (or bind by name) to the interfaces of,
47
+ the Work and Derivative Works thereof.
48
+
49
+ "Contribution" shall mean any work of authorship, including
50
+ the original version of the Work and any modifications or additions
51
+ to that Work or Derivative Works thereof, that is intentionally
52
+ submitted to Licensor for inclusion in the Work by the copyright owner
53
+ or by an individual or Legal Entity authorized to submit on behalf of
54
+ the copyright owner. For the purposes of this definition, "submitted"
55
+ means any form of electronic, verbal, or written communication sent
56
+ to the Licensor or its representatives, including but not limited to
57
+ communication on electronic mailing lists, source code control systems,
58
+ and issue tracking systems that are managed by, or on behalf of, the
59
+ Licensor for the purpose of discussing and improving the Work, but
60
+ excluding communication that is conspicuously marked or otherwise
61
+ designated in writing by the copyright owner as "Not a Contribution."
62
+
63
+ "Contributor" shall mean Licensor and any individual or Legal Entity
64
+ on behalf of whom a Contribution has been received by Licensor and
65
+ subsequently incorporated within the Work.
66
+
67
+ 2. Grant of Copyright License. Subject to the terms and conditions of
68
+ this License, each Contributor hereby grants to You a perpetual,
69
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
70
+ copyright license to reproduce, prepare Derivative Works of,
71
+ publicly display, publicly perform, sublicense, and distribute the
72
+ Work and such Derivative Works in Source or Object form.
73
+
74
+ 3. Grant of Patent License. Subject to the terms and conditions of
75
+ this License, each Contributor hereby grants to You a perpetual,
76
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
77
+ (except as stated in this section) patent license to make, have made,
78
+ use, offer to sell, sell, import, and otherwise transfer the Work,
79
+ where such license applies only to those patent claims licensable
80
+ by such Contributor that are necessarily infringed by their
81
+ Contribution(s) alone or by combination of their Contribution(s)
82
+ with the Work to which such Contribution(s) was submitted. If You
83
+ institute patent litigation against any entity (including a
84
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
85
+ or a Contribution incorporated within the Work constitutes direct
86
+ or contributory patent infringement, then any patent licenses
87
+ granted to You under this License for that Work shall terminate
88
+ as of the date such litigation is filed.
89
+
90
+ 4. Redistribution. You may reproduce and distribute copies of the
91
+ Work or Derivative Works thereof in any medium, with or without
92
+ modifications, and in Source or Object form, provided that You
93
+ meet the following conditions:
94
+
95
+ (a) You must give any other recipients of the Work or
96
+ Derivative Works a copy of this License; and
97
+
98
+ (b) You must cause any modified files to carry prominent notices
99
+ stating that You changed the files; and
100
+
101
+ (c) You must retain, in the Source form of any Derivative Works
102
+ that You distribute, all copyright, patent, trademark, and
103
+ attribution notices from the Source form of the Work,
104
+ excluding those notices that do not pertain to any part of
105
+ the Derivative Works; and
106
+
107
+ (d) If the Work includes a "NOTICE" text file as part of its
108
+ distribution, then any Derivative Works that You distribute must
109
+ include a readable copy of the attribution notices contained
110
+ within such NOTICE file, excluding those notices that do not
111
+ pertain to any part of the Derivative Works, in at least one
112
+ of the following places: within a NOTICE text file distributed
113
+ as part of the Derivative Works; within the Source form or
114
+ documentation, if provided along with the Derivative Works; or,
115
+ within a display generated by the Derivative Works, if and
116
+ wherever such third-party notices normally appear. The contents
117
+ of the NOTICE file are for informational purposes only and
118
+ do not modify the License. You may add Your own attribution
119
+ notices within Derivative Works that You distribute, alongside
120
+ or as an addendum to the NOTICE text from the Work, provided
121
+ that such additional attribution notices cannot be construed
122
+ as modifying the License.
123
+
124
+ You may add Your own copyright statement to Your modifications and
125
+ may provide additional or different license terms and conditions
126
+ for use, reproduction, or distribution of Your modifications, or
127
+ for any such Derivative Works as a whole, provided Your use,
128
+ reproduction, and distribution of the Work otherwise complies with
129
+ the conditions stated in this License.
130
+
131
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
132
+ any Contribution intentionally submitted for inclusion in the Work
133
+ by You to the Licensor shall be under the terms and conditions of
134
+ this License, without any additional terms or conditions.
135
+ Notwithstanding the above, nothing herein shall supersede or modify
136
+ the terms of any separate license agreement you may have executed
137
+ with Licensor regarding such Contributions.
138
+
139
+ 6. Trademarks. This License does not grant permission to use the trade
140
+ names, trademarks, service marks, or product names of the Licensor,
141
+ except as required for reasonable and customary use in describing the
142
+ origin of the Work and reproducing the content of the NOTICE file.
143
+
144
+ 7. Disclaimer of Warranty. Unless required by applicable law or
145
+ agreed to in writing, Licensor provides the Work (and each
146
+ Contributor provides its Contributions) on an "AS IS" BASIS,
147
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
148
+ implied, including, without limitation, any warranties or conditions
149
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
150
+ PARTICULAR PURPOSE. You are solely responsible for determining the
151
+ appropriateness of using or redistributing the Work and assume any
152
+ risks associated with Your exercise of permissions under this License.
153
+
154
+ 8. Limitation of Liability. In no event and under no legal theory,
155
+ whether in tort (including negligence), contract, or otherwise,
156
+ unless required by applicable law (such as deliberate and grossly
157
+ negligent acts) or agreed to in writing, shall any Contributor be
158
+ liable to You for damages, including any direct, indirect, special,
159
+ incidental, or consequential damages of any character arising as a
160
+ result of this License or out of the use or inability to use the
161
+ Work (including but not limited to damages for loss of goodwill,
162
+ work stoppage, computer failure or malfunction, or any and all
163
+ other commercial damages or losses), even if such Contributor
164
+ has been advised of the possibility of such damages.
165
+
166
+ 9. Accepting Warranty or Additional Liability. While redistributing
167
+ the Work or Derivative Works thereof, You may choose to offer,
168
+ and charge a fee for, acceptance of support, warranty, indemnity,
169
+ or other liability obligations and/or rights consistent with this
170
+ License. However, in accepting such obligations, You may act only
171
+ on Your own behalf and on Your sole responsibility, not on behalf
172
+ of any other Contributor, and only if You agree to indemnify,
173
+ defend, and hold each Contributor harmless for any liability
174
+ incurred by, or claims asserted against, such Contributor by reason
175
+ of your accepting any such warranty or additional liability.
176
+
177
+ END OF TERMS AND CONDITIONS
178
+
179
+ APPENDIX: How to apply the Apache License to your work.
180
+
181
+ To apply the Apache License to your work, attach the following
182
+ boilerplate notice, with the fields enclosed by brackets "[]"
183
+ replaced with your own identifying information. (Don't include
184
+ the brackets!) The text should be enclosed in the appropriate
185
+ comment syntax for the file format. We also recommend that a
186
+ file or class name and description of purpose be included on the
187
+ same "printed page" as the copyright notice for easier
188
+ identification within third-party archives.
189
+
190
+ Copyright [yyyy] [name of copyright owner]
191
+
192
+ Licensed under the Apache License, Version 2.0 (the "License");
193
+ you may not use this file except in compliance with the License.
194
+ You may obtain a copy of the License at
195
+
196
+ http://www.apache.org/licenses/LICENSE-2.0
197
+
198
+ Unless required by applicable law or agreed to in writing, software
199
+ distributed under the License is distributed on an "AS IS" BASIS,
200
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
201
+ See the License for the specific language governing permissions and
202
+ limitations under the License.
code/NOTICE ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ViSS: Visible-State Supervision and Test-Time Causal View Search
2
+
3
+ This source release is derived from the ViSS research repository and carries
4
+ that repository's Apache License 2.0, supplied in LICENSE.
5
+
6
+ The code uses external libraries including PyTorch, Transformers, Accelerate,
7
+ DeepSpeed, qwen-vl-utils, Decord, NumPy, and Pillow. Those projects retain
8
+ their own licenses. Qwen3-VL model weights, training media, benchmark data,
9
+ and official benchmark scorers are not redistributed in this archive.
10
+
11
+ Evaluation adapters preserve the StreamingBench/SimpleStream prompt interface
12
+ and OVO-Bench task interfaces described in the evaluation protocol.
13
+ Users must obtain the corresponding resources under their original terms.
14
+
15
+ The README organization takes inspiration from the OneStreamer repository.
16
+ No OneStreamer implementation, model weights, data, or figures are included.
code/README.md ADDED
@@ -0,0 +1,135 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ <div align="center">
2
+
3
+ # ViSS
4
+ ### Supervising the Visible State: Separating Representation from View Selection in Streaming Video
5
+
6
+ Yihao Hu*, Pan Wang*, Yitong Lin*, Zhihao Wen†, Deyuan Liu, Hang Wang,
7
+ Mingda WAN, Yiwen Zhao, Yutong Li, Yuchen Mao, Tao Lin†
8
+
9
+ <sup>* Equal contribution. † Corresponding author.</sup>
10
+
11
+ [Overview](#overview) · [Codebase](#codebase) · [Quick start](#quick-start) ·
12
+ [Models and data](#models-and-data) · [Training](docs/TRAINING.md) ·
13
+ [Evaluation](docs/EVALUATION.md) · [Citation](#citation)
14
+
15
+ </div>
16
+
17
+ ## Overview
18
+
19
+ **Visible-State Supervision (ViSS)** trains a vision–language model on
20
+ source-grounded questions about event identity, order, accumulation,
21
+ relation changes, and whether the visible evidence is sufficient.
22
+
23
+ **Test-Time Causal View Search (TVS)** keeps the trained model fixed and
24
+ acquires complementary causal views. It starts with recent frames, requests
25
+ an event view when needed, and consults compact history on disagreement.
26
+ Acquisition and answer commitment use separately calibrated thresholds.
27
+
28
+ This release provides data construction, training, inference, policy replay,
29
+ and scoring code. It contains no saved benchmark scores, prediction outputs,
30
+ or training logs. Scoring utilities and synthetic unit tests are included.
31
+
32
+ ## Codebase
33
+
34
+ | Directory | Purpose | Guide |
35
+ | --- | --- | --- |
36
+ | `data_pipeline/` | Build source-grounded probes and matched generic controls | [Training](docs/TRAINING.md) |
37
+ | `experiments/` | Train the model, generate causal-view proposals, and apply TVS | [Evaluation](docs/EVALUATION.md) |
38
+ | `viss_core/` | Record validation, media handling, and benchmark adapters | [Data preparation](docs/DATA.md) |
39
+ | `scripts/` | Evaluation launchers, shard merging, and official scoring | [Evaluation](docs/EVALUATION.md) |
40
+ | `configs/` | Evaluation protocol and DeepSpeed configuration | [Installation](docs/INSTALL.md) |
41
+ | `tests/` | CPU checks of data contracts, sampling, loss masks, and view policies | [Quick start](#quick-start) |
42
+
43
+ The repository contains no internal training manifests, run records, data
44
+ inventories, calibration outputs, or file identities from company experiments.
45
+ The evaluation configuration is a public template; each user binds it to
46
+ their own inputs. See [Data preparation](docs/DATA.md).
47
+
48
+ ## Quick start
49
+
50
+ Run these commands from the extracted repository root.
51
+
52
+ ### Install and check the source
53
+
54
+ ```bash
55
+ python -m venv .venv
56
+ source .venv/bin/activate
57
+ python -m pip install --upgrade pip
58
+ python -m pip install -e ".[test]"
59
+ python -m pytest -q tests
60
+ ```
61
+
62
+ The unit tests use synthetic inputs and do not download model weights or
63
+ benchmark media. GPU inference and training require the additional setup in
64
+ [Installation](docs/INSTALL.md).
65
+
66
+ ### Generate predictions and score separately
67
+
68
+ After preparing your model, annotations, query exports, media, and
69
+ [local evaluation protocol](docs/EVALUATION.md#1-generate-recent-view-proposals):
70
+
71
+ ```bash
72
+ python -m experiments.evaluate_qwen3vl_recent4_full \
73
+ --benchmark streamingbench \
74
+ --annotation data/streamingbench/annotations.json \
75
+ --samples data/streamingbench/queries.jsonl \
76
+ --video-root data/streamingbench/videos \
77
+ --model checkpoints/viss \
78
+ --processor checkpoints/viss/locked_processor \
79
+ --protocol configs/evaluation/recent4_full_v1.json \
80
+ --frame-policy recent4 \
81
+ --visual-input-mode image-list \
82
+ --record-choice-confidence \
83
+ --output outputs/streamingbench_recent.json
84
+ ```
85
+
86
+ The paths above are placeholders for separately prepared assets. Bind the
87
+ processor and query exports used in your evaluation before running inference.
88
+ The [evaluation guide](docs/EVALUATION.md) explains model
89
+ identity, TVS inputs, official scorers, and full-population checks.
90
+
91
+ ## Models and data
92
+
93
+ | Resource | Included in this archive? | Preparation |
94
+ | --- | --- | --- |
95
+ | Source code, tests, and protocol | Yes | This repository |
96
+ | Qwen3-VL-8B-Instruct initialization | No | Obtain the official model |
97
+ | Trained ViSS weights and locked processor | No | Obtain separately or train ViSS |
98
+ | Accepted training records and media | No | Follow [data preparation](docs/DATA.md) |
99
+ | Benchmark annotations, videos, and official scorers | No | Obtain from the benchmark owners |
100
+ | TVS calibration and model-identity artifacts | No | Supply artifacts matching your evaluation |
101
+
102
+ No public download endpoint for ViSS-specific weights or training records is
103
+ asserted by this source package.
104
+
105
+ ## Training
106
+
107
+ The paper recipe is a fresh one-epoch run from Qwen3-VL-8B-Instruct.
108
+ It freezes the vision tower and updates the language model and multimodal
109
+ projector. [Training](docs/TRAINING.md) gives the two-GPU command and explains
110
+ the required sealed training release.
111
+
112
+ Some serialized records retain `streameva.*` schema identifiers. These are
113
+ record-format identifiers; the corresponding implementation in this release
114
+ is under `viss_core/`.
115
+
116
+ ## License and acknowledgements
117
+
118
+ The project code uses the repository's [Apache-2.0 license](LICENSE).
119
+ Third-party software, media, models, and benchmark assets retain their
120
+ respective terms; see [NOTICE](NOTICE).
121
+
122
+ We acknowledge Qwen3-VL, StreamingBench, OVO-Bench, and SimpleStream.
123
+ The documentation organization follows the separation of setup, inference,
124
+ and evaluation used in [OneStreamer](https://github.com/MCG-NJU/OneStreamer).
125
+
126
+ ## Citation
127
+
128
+ ```bibtex
129
+ @misc{hu2026viss,
130
+ title={Supervising the Visible State: Separating Representation from View Selection in Streaming Video},
131
+ author={Yihao Hu and Pan Wang and Yitong Lin and Zhihao Wen and Deyuan Liu and Hang Wang and Mingda Wan and Yiwen Zhao and Yutong Li and Yuchen Mao and Tao Lin},
132
+ year={2026},
133
+ note={Manuscript}
134
+ }
135
+ ```
code/configs/evaluation/recent4.example.json ADDED
@@ -0,0 +1,74 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "format": "iclr_video.locked_evaluation_protocol.v1",
3
+ "protocol_id": "recent4_full_v1",
4
+ "status": "template",
5
+ "applies_to": "all baseline and candidate checkpoints",
6
+ "inference": {
7
+ "causal_prefix_only": true,
8
+ "sampling_fps": 1.0,
9
+ "context_policy": "recent_n",
10
+ "recent_frames": 4,
11
+ "timestamp_rule": "match the SimpleStream release: decode a four-second half-open causal prefix window [max(0,t-4),t) at nominal 1 FPS and retain exactly four independently encoded image frames; only prefixes shorter than two raw frames repeat frame zero",
12
+ "allow_future_frames": false,
13
+ "allow_global_frames": false,
14
+ "allow_extra_frame_inspection": false,
15
+ "allow_persistent_visual_or_text_memory": false,
16
+ "visual_input_mode": "four independent image frames using qwen-vl-utils fetch_video (image_patch_size=14), never Qwen video tokens",
17
+ "processor_contract": {
18
+ "preprocessor_config_sha256": "",
19
+ "image_processor_type": "Qwen2VLImageProcessorFast",
20
+ "patch_size": 16,
21
+ "merge_size": 2,
22
+ "temporal_patch_size": 2,
23
+ "shortest_edge": 65536,
24
+ "longest_edge": 16777216,
25
+ "reference_visual_tokens_per_frame": 576,
26
+ "reason": "Bind the processor used by your evaluation before inference."
27
+ },
28
+ "prompt_source": "StreamingBench SimpleStream release MCQ template; OVO backward/realtime SimpleStream release one-letter template; OVO forward frozen official task wrapper; no system message",
29
+ "decode": {
30
+ "do_sample": false,
31
+ "temperature": 0.0
32
+ }
33
+ },
34
+ "benchmarks": {
35
+ "streamingbench": {
36
+ "scope": "Real-Time full",
37
+ "export": "data/streamingbench/queries.jsonl",
38
+ "export_sha256": "",
39
+ "expected_queries": 2495,
40
+ "expected_groups": {
41
+ "real_time_visual": 2495
42
+ },
43
+ "primary_metric": "official Real-Time accuracy"
44
+ },
45
+ "ovo": {
46
+ "scope": "all 12 tasks: Real-Time, Backward, and Forward",
47
+ "export": "data/ovo/queries.jsonl",
48
+ "export_sha256": "",
49
+ "expected_queries": 3035,
50
+ "expected_tasks": {
51
+ "ACR": 109,
52
+ "ASI": 148,
53
+ "ATR": 116,
54
+ "CRR": 240,
55
+ "EPM": 297,
56
+ "FPD": 101,
57
+ "HLD": 186,
58
+ "OCR": 149,
59
+ "OJR": 184,
60
+ "REC": 698,
61
+ "SSR": 629,
62
+ "STU": 178
63
+ },
64
+ "primary_metric": "official Total Avg",
65
+ "secondary_metrics": [
66
+ "micro Overall",
67
+ "Real-Time Avg",
68
+ "Backward Avg",
69
+ "Forward Avg",
70
+ "RT/BT Avg"
71
+ ]
72
+ }
73
+ }
74
+ }
code/configs/training/deepspeed_zero3_bf16.json ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bf16": {"enabled": "auto"},
3
+ "train_micro_batch_size_per_gpu": "auto",
4
+ "train_batch_size": "auto",
5
+ "gradient_accumulation_steps": "auto",
6
+ "gradient_clipping": "auto",
7
+ "zero_optimization": {
8
+ "stage": 3,
9
+ "overlap_comm": true,
10
+ "contiguous_gradients": true,
11
+ "reduce_bucket_size": "auto",
12
+ "stage3_prefetch_bucket_size": "auto",
13
+ "stage3_param_persistence_threshold": "auto",
14
+ "stage3_gather_16bit_weights_on_model_save": true
15
+ }
16
+ }
code/data_pipeline/__init__.py ADDED
@@ -0,0 +1 @@
 
 
1
+ """Stable, user-facing entry points for reproducing StreamEVA data."""
code/data_pipeline/temporal_scope/__init__.py ADDED
@@ -0,0 +1 @@
 
 
1
+ """Deterministic semantic-SFT and public typed-contract compilers."""
code/data_pipeline/temporal_scope/build_size_matched_generic_control.py ADDED
@@ -0,0 +1,107 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """Select a deterministic, source-stratified generic temporal SFT control."""
3
+
4
+ from __future__ import annotations
5
+
6
+ import argparse
7
+ import hashlib
8
+ import json
9
+ from collections import defaultdict
10
+ from pathlib import Path
11
+ from typing import Any
12
+
13
+
14
+ SUPPORTED_PARENT_KINDS = (
15
+ "source_semantic_row",
16
+ "cited_nonbenchmark_refresh_projection",
17
+ )
18
+
19
+
20
+ def _rank(seed: str, parent_id: str) -> str:
21
+ return hashlib.sha256(f"{seed}\0{parent_id}".encode()).hexdigest()
22
+
23
+
24
+ def select_parent_ids(
25
+ rows: list[dict[str, Any]], *, seed: str, quotas: dict[str, int]
26
+ ) -> tuple[list[str], dict[str, int]]:
27
+ grouped: dict[str, list[str]] = defaultdict(list)
28
+ seen: set[str] = set()
29
+ for row in rows:
30
+ parents = row.get("parents") or []
31
+ if len(parents) != 1:
32
+ raise ValueError("generic control requires exactly one parent per row")
33
+ parent_id = str(parents[0].get("id") or "")
34
+ parent_kind = str(parents[0].get("kind") or "")
35
+ if not parent_id or parent_kind not in SUPPORTED_PARENT_KINDS:
36
+ raise ValueError("generic control parent provenance is malformed")
37
+ if parent_id in seen:
38
+ raise ValueError("generic control parent IDs must be unique")
39
+ seen.add(parent_id)
40
+ grouped[parent_kind].append(parent_id)
41
+
42
+ selected: list[str] = []
43
+ counts: dict[str, int] = {}
44
+ for kind in SUPPORTED_PARENT_KINDS:
45
+ quota = int(quotas.get(kind, 0))
46
+ candidates = grouped[kind]
47
+ if quota <= 0 or quota > len(candidates):
48
+ raise ValueError(f"invalid quota for {kind}: {quota}/{len(candidates)}")
49
+ ranked = sorted(candidates, key=lambda value: (_rank(seed, value), value))
50
+ chosen = ranked[:quota]
51
+ selected.extend(chosen)
52
+ counts[kind] = len(chosen)
53
+ return sorted(selected), counts
54
+
55
+
56
+ def main() -> None:
57
+ parser = argparse.ArgumentParser()
58
+ parser.add_argument("--source", type=Path, required=True)
59
+ parser.add_argument("--output", type=Path, required=True)
60
+ parser.add_argument("--seed", default="viss-generic-control-7677-v1")
61
+ parser.add_argument("--source-semantic-rows", type=int, default=5903)
62
+ parser.add_argument("--cited-refresh-rows", type=int, default=1774)
63
+ args = parser.parse_args()
64
+
65
+ source = args.source.resolve(strict=True)
66
+ trainer = source / "trainer/sft.jsonl"
67
+ manifest = source / "manifest.json"
68
+ if args.output.exists():
69
+ raise FileExistsError(args.output)
70
+ rows = [json.loads(line) for line in trainer.read_text().splitlines() if line]
71
+ selected, counts = select_parent_ids(
72
+ rows,
73
+ seed=args.seed,
74
+ quotas={
75
+ "source_semantic_row": args.source_semantic_rows,
76
+ "cited_nonbenchmark_refresh_projection": args.cited_refresh_rows,
77
+ },
78
+ )
79
+ if len(selected) != args.source_semantic_rows + args.cited_refresh_rows:
80
+ raise AssertionError("generic control selection size diverged")
81
+
82
+ payload = {
83
+ "format": "iclr_video.capability_parent_selection.v1",
84
+ "status": "sealed_deduplicated_union",
85
+ "purpose": "size_matched_generic_temporal_sft_control",
86
+ "seed": args.seed,
87
+ "source_release": str(source),
88
+ "source_release_id": json.loads(manifest.read_text()).get("release_id"),
89
+ "parent_ids": selected,
90
+ "profiles": list(SUPPORTED_PARENT_KINDS),
91
+ "counts_by_profile": counts,
92
+ "total_rows": len(selected),
93
+ "contracts": {
94
+ "benchmark_content_used_for_selection": False,
95
+ "source_release_membership_verified": True,
96
+ "model_predictions_used_for_selection": False,
97
+ "task_labels_used_for_selection": False,
98
+ "selection_is_sha256_ranked_within_source_kind": True,
99
+ },
100
+ }
101
+ args.output.parent.mkdir(parents=True, exist_ok=True)
102
+ args.output.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n")
103
+ print(json.dumps({"output": str(args.output), "rows": len(selected), "counts": counts}))
104
+
105
+
106
+ if __name__ == "__main__":
107
+ main()
code/data_pipeline/temporal_scope/build_source_aligned_sft_release.py ADDED
@@ -0,0 +1,722 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """Compose one source-aligned, benchmark-firewalled pure-VLM SFT release.
3
+
4
+ The inputs are already sealed semantic releases built from non-benchmark
5
+ training videos. This compiler keeps the original semantic labels, removes
6
+ overlap by cited source identity and exact pixel/question/answer identity,
7
+ canonicalizes media paths by SHA256, and emits the only JSONL consumed by one
8
+ joint SFT run. Benchmark examples and auditor predictions are never labels.
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ import argparse
14
+ import collections
15
+ import copy
16
+ import hashlib
17
+ import json
18
+ import math
19
+ import os
20
+ import random
21
+ import shutil
22
+ import tempfile
23
+ from pathlib import Path
24
+ from typing import Any
25
+
26
+ from viss_core.data_v0.canonical import (
27
+ attach_content_sha256,
28
+ canonical_json_bytes,
29
+ canonical_json_line,
30
+ file_sha256,
31
+ read_jsonl,
32
+ stable_id,
33
+ write_jsonl,
34
+ )
35
+ from viss_core.data_v0.schema import validate_row
36
+ from viss_core.data_v0.semantic_release import (
37
+ EXPECTED_PUBLIC_UPSTREAM_MANIFEST_SHA256,
38
+ FORMAT,
39
+ validate_semantic_release,
40
+ )
41
+
42
+
43
+ class SourceAlignedSFTError(ValueError):
44
+ """Raised when source releases cannot be composed without leakage."""
45
+
46
+
47
+ def _target(row: dict[str, Any]) -> str:
48
+ values = [
49
+ str(item.get("text") or "")
50
+ for message in row.get("messages") or []
51
+ if message.get("role") == "assistant" and message.get("loss") is True
52
+ for item in message.get("content") or []
53
+ if item.get("type") == "text"
54
+ ]
55
+ if len(values) != 1:
56
+ raise SourceAlignedSFTError("semantic row has no unique target")
57
+ return values[0]
58
+
59
+
60
+ def _question(row: dict[str, Any]) -> str:
61
+ values = [
62
+ str(item.get("text") or "")
63
+ for message in row.get("messages") or []
64
+ if message.get("role") == "user"
65
+ for item in message.get("content") or []
66
+ if item.get("type") == "text"
67
+ ]
68
+ if len(values) != 1:
69
+ raise SourceAlignedSFTError("semantic row has no unique public prompt")
70
+ return " ".join(values[0].split())
71
+
72
+
73
+ def _images(row: dict[str, Any]) -> list[str]:
74
+ return [
75
+ str(item["image"])
76
+ for message in row.get("messages") or []
77
+ for item in message.get("content") or []
78
+ if item.get("type") == "image"
79
+ ]
80
+
81
+
82
+ def _cited_parents(row: dict[str, Any]) -> set[str]:
83
+ return {
84
+ str(parent["id"])
85
+ for parent in row.get("parents") or []
86
+ if parent.get("kind") == "cited_nonbenchmark_refresh_projection"
87
+ and parent.get("id")
88
+ }
89
+
90
+
91
+ def _semantic_identity(
92
+ row: dict[str, Any], inventory: dict[str, dict[str, Any]]
93
+ ) -> str:
94
+ try:
95
+ image_sha256s = [str(inventory[path]["sha256"]) for path in _images(row)]
96
+ except KeyError as exc:
97
+ raise SourceAlignedSFTError("row references undeclared source media") from exc
98
+ payload = {
99
+ "answer": _target(row),
100
+ "image_sha256s": image_sha256s,
101
+ "question": _question(row),
102
+ }
103
+ return hashlib.sha256(canonical_json_bytes(payload)).hexdigest()
104
+
105
+
106
+ def _canonical_media_path(relative: str, digest: str) -> str:
107
+ suffix = Path(relative).suffix.casefold()
108
+ if not suffix or len(suffix) > 10:
109
+ raise SourceAlignedSFTError("source media suffix is invalid")
110
+ return (Path("media/sha256") / digest[:2] / f"{digest}{suffix}").as_posix()
111
+
112
+
113
+ def _materialize(source: Path, destination: Path, *, mode: str) -> None:
114
+ if source.is_symlink() or not source.is_file():
115
+ raise SourceAlignedSFTError(f"unsafe source media: {source}")
116
+ destination.parent.mkdir(parents=True, exist_ok=True)
117
+ if mode in {"hardlink", "auto"}:
118
+ try:
119
+ os.link(source, destination)
120
+ return
121
+ except OSError:
122
+ if mode == "hardlink":
123
+ raise
124
+ shutil.copy2(source, destination)
125
+
126
+
127
+ def _lineage_key(manifest: dict[str, Any]) -> tuple[str, str, str]:
128
+ lineage = manifest.get("model_lineage") or {}
129
+ return (
130
+ str(lineage.get("required_sha256s_sha256") or ""),
131
+ str(lineage.get("upstream_repo") or ""),
132
+ str(lineage.get("upstream_revision") or ""),
133
+ )
134
+
135
+
136
+ def _teacher_used_for_labels(manifest: dict[str, Any]) -> bool:
137
+ value = (manifest.get("contracts") or {}).get("teacher_used_for_labels")
138
+ if not isinstance(value, bool):
139
+ raise SourceAlignedSFTError(
140
+ "source teacher-label provenance is absent or malformed"
141
+ )
142
+ return value
143
+
144
+
145
+ def _load_legacy_v62di_source(root: Path, *, mode: str = "v62di_only"):
146
+ raise SourceAlignedSFTError("Private legacy adapters are not distributed; supply a standard source release")
147
+
148
+
149
+ def _load_legacy_repcount_source(root: Path):
150
+ raise SourceAlignedSFTError("Private legacy adapters are not distributed; supply a standard source release")
151
+
152
+
153
+ def build(args: argparse.Namespace) -> dict[str, Any]:
154
+ sealed_source_count = (
155
+ len(args.source)
156
+ + int(getattr(args, "legacy_v62di_source", None) is not None)
157
+ + int(getattr(args, "legacy_repcount_source", None) is not None)
158
+ )
159
+ # A hash-bound parent selection is also a legitimate one-source filtering
160
+ # operation. Unfiltered releases remain compositions and therefore still
161
+ # require at least two sealed inputs.
162
+ if sealed_source_count < 2 and getattr(args, "include_parent_selection", None) is None:
163
+ raise SourceAlignedSFTError(
164
+ "at least two sealed sources are required unless a parent selection is supplied"
165
+ )
166
+ if args.copy_mode not in {"copy", "hardlink", "auto"}:
167
+ raise SourceAlignedSFTError("unsupported copy mode")
168
+ sample_loss_weight_cap = float(args.sample_loss_weight_cap)
169
+ if not math.isfinite(sample_loss_weight_cap) or sample_loss_weight_cap < 0:
170
+ raise SourceAlignedSFTError(
171
+ "sample loss weight cap must be finite and non-negative"
172
+ )
173
+ excluded_parent_kinds = {
174
+ str(kind).strip()
175
+ for kind in getattr(args, "exclude_parent_kind", [])
176
+ if str(kind).strip()
177
+ }
178
+ included_source_stage_release_ids = {
179
+ str(release_id).strip()
180
+ for release_id in getattr(args, "include_source_stage_release_id", [])
181
+ if str(release_id).strip()
182
+ }
183
+ parent_selection_path = getattr(args, "include_parent_selection", None)
184
+ selected_parent_ids: set[str] = set()
185
+ parent_selection_record: dict[str, Any] | None = None
186
+ if parent_selection_path is not None:
187
+ parent_selection_path = parent_selection_path.resolve(strict=True)
188
+ selection = json.loads(parent_selection_path.read_text(encoding="utf-8"))
189
+ if selection.get("format") not in {
190
+ "iclr_video.capability_parent_selection.v0",
191
+ "iclr_video.capability_parent_selection.v1",
192
+ }:
193
+ raise SourceAlignedSFTError("parent selection format diverged")
194
+ if selection.get("format").endswith(".v1") and (
195
+ selection.get("status") != "sealed_deduplicated_union"
196
+ or (selection.get("contracts") or {}).get(
197
+ "benchmark_content_used_for_selection"
198
+ )
199
+ is not False
200
+ or (selection.get("contracts") or {}).get(
201
+ "source_release_membership_verified"
202
+ )
203
+ is not True
204
+ ):
205
+ raise SourceAlignedSFTError("v1 parent selection contract diverged")
206
+ parent_ids = selection.get("parent_ids") or []
207
+ if (
208
+ not isinstance(parent_ids, list)
209
+ or not parent_ids
210
+ or any(not isinstance(value, str) or not value for value in parent_ids)
211
+ or len(set(parent_ids)) != len(parent_ids)
212
+ ):
213
+ raise SourceAlignedSFTError("parent selection IDs are malformed")
214
+ selected_parent_ids = set(parent_ids)
215
+ parent_selection_record = {
216
+ "path": str(parent_selection_path),
217
+ "sha256": file_sha256(parent_selection_path),
218
+ "parent_ids": len(selected_parent_ids),
219
+ "profiles": list(
220
+ selection.get("profiles")
221
+ or sorted((selection.get("counts_by_profile") or {}).keys())
222
+ ),
223
+ }
224
+ output = args.output.resolve()
225
+ if output.exists():
226
+ raise FileExistsError(output)
227
+
228
+ ordered_curriculum = bool(getattr(args, "ordered_curriculum", False))
229
+ retain_semantic_exposures = bool(
230
+ getattr(args, "retain_semantic_exposures", False)
231
+ )
232
+ legacy_v62di_source = getattr(args, "legacy_v62di_source", None)
233
+ legacy_v62di_mode = getattr(args, "legacy_v62di_mode", "v62di_only")
234
+ legacy_repcount_source = getattr(args, "legacy_repcount_source", None)
235
+ if legacy_v62di_source is not None and not ordered_curriculum:
236
+ raise SourceAlignedSFTError(
237
+ "legacy V62DI input is allowed only for ordered curriculum"
238
+ )
239
+ if legacy_repcount_source is not None and not ordered_curriculum:
240
+ raise SourceAlignedSFTError(
241
+ "legacy RepCount input is allowed only for ordered curriculum"
242
+ )
243
+ expected_lineage: tuple[str, str, str] | None = None
244
+ seen_identity: set[str] = set()
245
+ seen_cited_parent: dict[str, str] = {}
246
+ accepted: list[tuple[dict[str, Any], Path, dict[str, dict[str, Any]], str]] = []
247
+ source_records = []
248
+ rejected: collections.Counter[str] = collections.Counter()
249
+ observed_selected_parent_ids: set[str] = set()
250
+
251
+ if legacy_v62di_source is not None:
252
+ legacy_root = legacy_v62di_source.resolve(strict=True)
253
+ legacy_rows, legacy_inventory, legacy_record = _load_legacy_v62di_source(
254
+ legacy_root, mode=legacy_v62di_mode
255
+ )
256
+ for row in legacy_rows:
257
+ identity = _semantic_identity(row, legacy_inventory)
258
+ accepted.append((row, legacy_root, legacy_inventory, identity))
259
+ seen_identity.add(identity)
260
+ source_records.append(legacy_record)
261
+
262
+ source_specs: list[tuple[str, Path]] = []
263
+ for source_index, raw_root in enumerate(args.source):
264
+ source_specs.append(("semantic", raw_root))
265
+ if source_index == 0 and legacy_repcount_source is not None:
266
+ source_specs.append(("legacy_repcount", legacy_repcount_source))
267
+
268
+ for source_kind, raw_root in source_specs:
269
+ root = raw_root.resolve(strict=True)
270
+ if source_kind == "legacy_repcount":
271
+ rows, inventory, source_record = _load_legacy_repcount_source(root)
272
+ manifest = json.loads((root / "manifest.json").read_text(encoding="utf-8"))
273
+ else:
274
+ manifest = validate_semantic_release(root, verify_media_hashes=False)
275
+ lineage_key = _lineage_key(manifest)
276
+ if expected_lineage is None:
277
+ expected_lineage = lineage_key
278
+ elif lineage_key != expected_lineage:
279
+ raise SourceAlignedSFTError("source model lineages diverged")
280
+ if lineage_key[0] != EXPECTED_PUBLIC_UPSTREAM_MANIFEST_SHA256:
281
+ raise SourceAlignedSFTError("source does not descend from pinned Instruct")
282
+ if (manifest.get("contracts") or {}).get("benchmark_rows_or_labels_admitted", 0) != 0:
283
+ raise SourceAlignedSFTError("source admits benchmark rows or labels")
284
+ teacher_used_for_labels = _teacher_used_for_labels(manifest)
285
+
286
+ if source_kind == "semantic":
287
+ inventory_rows = read_jsonl(
288
+ root / "manifests/media_inventory.jsonl", require_canonical=True
289
+ )
290
+ inventory = {str(item["path"]): item for item in inventory_rows}
291
+ if len(inventory) != len(inventory_rows):
292
+ raise SourceAlignedSFTError("source media inventory paths collide")
293
+ rows = read_jsonl(
294
+ root / "trainer/sft.jsonl",
295
+ verify_content_hashes=True,
296
+ require_canonical=True,
297
+ )
298
+ if included_source_stage_release_ids:
299
+ stages = (manifest.get("inputs") or {}).get("curriculum_stages") or []
300
+ selected_stages = [
301
+ stage
302
+ for stage in stages
303
+ if str(stage.get("release_id"))
304
+ in included_source_stage_release_ids
305
+ ]
306
+ selected_ids = {
307
+ str(stage["release_id"]) for stage in selected_stages
308
+ }
309
+ missing_ids = included_source_stage_release_ids - selected_ids
310
+ if missing_ids:
311
+ raise SourceAlignedSFTError(
312
+ "requested source stages are absent: "
313
+ + ",".join(sorted(missing_ids))
314
+ )
315
+ selected_rows = []
316
+ for stage in sorted(
317
+ selected_stages, key=lambda item: int(item["start_row"])
318
+ ):
319
+ start = int(stage["start_row"])
320
+ end = int(stage["end_row_exclusive"])
321
+ if start < 0 or end <= start or end > len(rows):
322
+ raise SourceAlignedSFTError(
323
+ "source curriculum stage range is invalid"
324
+ )
325
+ if end - start != int(stage["rows"]):
326
+ raise SourceAlignedSFTError(
327
+ "source curriculum stage row count diverged"
328
+ )
329
+ selected_rows.extend(rows[start:end])
330
+ rows = selected_rows
331
+ accepted_before = len(accepted)
332
+ for row in rows:
333
+ row_parent_ids = {
334
+ str(parent.get("id") or "") for parent in row.get("parents") or []
335
+ }
336
+ selected_matches = row_parent_ids & selected_parent_ids
337
+ if selected_parent_ids and not selected_matches:
338
+ rejected["outside_parent_selection"] += 1
339
+ continue
340
+ observed_selected_parent_ids.update(selected_matches)
341
+ row_parent_kinds = {
342
+ str(parent.get("kind") or "") for parent in row.get("parents") or []
343
+ }
344
+ excluded = sorted(row_parent_kinds & excluded_parent_kinds)
345
+ if excluded:
346
+ for kind in excluded:
347
+ rejected[f"excluded_parent_kind:{kind}"] += 1
348
+ continue
349
+ identity = _semantic_identity(row, inventory)
350
+ cited = _cited_parents(row)
351
+ conflicts = {
352
+ seen_cited_parent[parent]
353
+ for parent in cited
354
+ if parent in seen_cited_parent
355
+ }
356
+ if conflicts and conflicts != {identity}:
357
+ raise SourceAlignedSFTError(
358
+ "one cited source parent maps to conflicting semantic rows"
359
+ )
360
+ retain_repeated_rows = ordered_curriculum or retain_semantic_exposures
361
+ if identity in seen_identity and not retain_repeated_rows:
362
+ rejected["duplicate_semantic_identity"] += 1
363
+ for parent in cited:
364
+ seen_cited_parent.setdefault(parent, identity)
365
+ continue
366
+ if conflicts and not retain_repeated_rows:
367
+ rejected["duplicate_cited_source_parent"] += 1
368
+ continue
369
+ seen_identity.add(identity)
370
+ for parent in cited:
371
+ seen_cited_parent[parent] = identity
372
+ accepted.append((row, root, inventory, identity))
373
+ record = (
374
+ source_record
375
+ if source_kind == "legacy_repcount"
376
+ else {
377
+ "accepted_rows": len(accepted) - accepted_before,
378
+ "manifest_sha256": file_sha256(root / "manifest.json"),
379
+ "release_id": manifest["release_id"],
380
+ "root": str(root),
381
+ "source_rows": len(rows),
382
+ "teacher_used_for_labels": teacher_used_for_labels,
383
+ "trainer_sha256": file_sha256(root / "trainer/sft.jsonl"),
384
+ }
385
+ )
386
+ record["accepted_rows"] = len(accepted) - accepted_before
387
+ source_records.append(record)
388
+
389
+ if selected_parent_ids:
390
+ missing_selected_parent_ids = selected_parent_ids - observed_selected_parent_ids
391
+ if missing_selected_parent_ids:
392
+ raise SourceAlignedSFTError(
393
+ f"{len(missing_selected_parent_ids)} selected parent IDs were absent"
394
+ )
395
+
396
+ within_stage_shuffle_seed = int(
397
+ getattr(args, "shuffle_within_stage_seed", -1)
398
+ )
399
+ if within_stage_shuffle_seed >= 0:
400
+ shuffled: list[
401
+ tuple[dict[str, Any], Path, dict[str, dict[str, Any]], str]
402
+ ] = []
403
+ cursor = 0
404
+ for stage_index, record in enumerate(source_records):
405
+ count = int(record["accepted_rows"])
406
+ stage = accepted[cursor : cursor + count]
407
+ random.Random(within_stage_shuffle_seed + stage_index).shuffle(stage)
408
+ shuffled.extend(stage)
409
+ cursor += count
410
+ if cursor != len(accepted):
411
+ raise SourceAlignedSFTError("curriculum stage row accounting diverged")
412
+ accepted = shuffled
413
+
414
+ assert expected_lineage is not None
415
+ output.parent.mkdir(parents=True, exist_ok=True)
416
+ scratch = Path(tempfile.mkdtemp(prefix=f".{output.name}.partial.", dir=output.parent))
417
+ try:
418
+ output_inventory: dict[str, dict[str, Any]] = {}
419
+ media_sources: dict[str, Path] = {}
420
+ output_rows = []
421
+ by_source: collections.Counter[str] = collections.Counter()
422
+ weights_before: collections.Counter[float] = collections.Counter()
423
+ weights_after: collections.Counter[float] = collections.Counter()
424
+ reweighted_rows = 0
425
+ source_stage = {
426
+ item["root"]: index for index, item in enumerate(source_records)
427
+ }
428
+ source_row_index: collections.Counter[str] = collections.Counter()
429
+ repeated_semantic_exposures = len(accepted) - len(seen_identity)
430
+ stage_cursor = 0
431
+ curriculum_stages = []
432
+ for stage_index, record in enumerate(source_records):
433
+ stage_rows = int(record["accepted_rows"])
434
+ curriculum_stages.append(
435
+ {
436
+ "end_row_exclusive": stage_cursor + stage_rows,
437
+ "index": stage_index,
438
+ "release_id": record["release_id"],
439
+ "rows": stage_rows,
440
+ "start_row": stage_cursor,
441
+ }
442
+ )
443
+ stage_cursor += stage_rows
444
+ if stage_cursor != len(accepted):
445
+ raise SourceAlignedSFTError("curriculum manifest row accounting diverged")
446
+ for row, root, inventory, identity in accepted:
447
+ messages = copy.deepcopy(row["messages"])
448
+ for message in messages:
449
+ for item in message.get("content") or []:
450
+ if item.get("type") != "image":
451
+ continue
452
+ source_relative = str(item["image"])
453
+ declaration = inventory[source_relative]
454
+ digest = str(declaration["sha256"])
455
+ canonical_relative = _canonical_media_path(source_relative, digest)
456
+ previous = output_inventory.get(canonical_relative)
457
+ normalized = {
458
+ "bytes": int(declaration["bytes"]),
459
+ "path": canonical_relative,
460
+ "sha256": digest,
461
+ }
462
+ if previous is not None and previous != normalized:
463
+ raise SourceAlignedSFTError("canonical media declarations conflict")
464
+ output_inventory.setdefault(canonical_relative, normalized)
465
+ media_sources.setdefault(canonical_relative, root / source_relative)
466
+ item["image"] = canonical_relative
467
+ source_weight = float(row.get("sample_loss_weight", 1.0))
468
+ output_weight = (
469
+ min(source_weight, sample_loss_weight_cap)
470
+ if sample_loss_weight_cap > 0
471
+ else source_weight
472
+ )
473
+ weights_before[source_weight] += 1
474
+ weights_after[output_weight] += 1
475
+ reweighted_rows += int(output_weight != source_weight)
476
+ stage_index = source_stage[str(root)]
477
+ row_index = source_row_index[str(root)]
478
+ source_row_index[str(root)] += 1
479
+ uid_parts: tuple[Any, ...] = (identity,)
480
+ if ordered_curriculum or retain_semantic_exposures:
481
+ uid_parts = (stage_index, row_index, identity)
482
+ clean = attach_content_sha256(
483
+ {
484
+ "schema": "streameva.sft_row.v1",
485
+ "uid": stable_id(
486
+ "sourcealignedsft", *uid_parts, digest_length=24
487
+ ),
488
+ "component": "preservation",
489
+ "group_id": stable_id(
490
+ "sourcealignedsftgroup", *uid_parts, digest_length=24
491
+ ),
492
+ "split": "train",
493
+ "messages": messages,
494
+ "sample_loss_weight": output_weight,
495
+ "parents": copy.deepcopy(row.get("parents") or []),
496
+ }
497
+ )
498
+ validate_row(clean)
499
+ output_rows.append(clean)
500
+ source_release = next(
501
+ item["release_id"]
502
+ for item in source_records
503
+ if item["root"] == str(root)
504
+ )
505
+ by_source[source_release] += 1
506
+
507
+ if not ordered_curriculum:
508
+ output_rows.sort(key=lambda row: str(row["uid"]))
509
+ if len({str(row["uid"]) for row in output_rows}) != len(output_rows):
510
+ raise SourceAlignedSFTError("output UIDs collide")
511
+ write_jsonl(output_rows, scratch / "trainer/sft.jsonl")
512
+ inventory_rows = [output_inventory[path] for path in sorted(output_inventory)]
513
+ write_jsonl(inventory_rows, scratch / "manifests/media_inventory.jsonl")
514
+ for item in inventory_rows:
515
+ relative = str(item["path"])
516
+ destination = scratch / relative
517
+ _materialize(media_sources[relative], destination, mode=args.copy_mode)
518
+ if (
519
+ destination.stat().st_size != int(item["bytes"])
520
+ or file_sha256(destination) != item["sha256"]
521
+ ):
522
+ raise SourceAlignedSFTError("materialized media hash diverged")
523
+
524
+ answers = collections.Counter(_target(row) for row in output_rows)
525
+ frames = collections.Counter(str(len(_images(row))) for row in output_rows)
526
+ manifest = {
527
+ "artifacts": {
528
+ "manifests/media_inventory.jsonl": file_sha256(
529
+ scratch / "manifests/media_inventory.jsonl"
530
+ ),
531
+ "trainer/sft.jsonl": file_sha256(scratch / "trainer/sft.jsonl"),
532
+ },
533
+ "contracts": {
534
+ "agent_action_targets": 0,
535
+ "assistant_targets_plain_semantic_only": True,
536
+ "benchmark_examples_admitted": 0,
537
+ "benchmark_firewall_inherited": True,
538
+ "counterfactual_rows_admitted": 0,
539
+ "environment_state_actor_targets": 0,
540
+ "legacy_stage_metadata_admitted": 0,
541
+ "loss_on_assistant_only": True,
542
+ "policy_holdout_admitted": 0,
543
+ "private_verifier_actor_targets": 0,
544
+ "all_sample_loss_weights_one": set(weights_after) == {1.0},
545
+ "sample_loss_weight_cap": (
546
+ sample_loss_weight_cap if sample_loss_weight_cap > 0 else None
547
+ ),
548
+ "sample_loss_weight_rows_changed": reweighted_rows,
549
+ "self_contained_media": True,
550
+ "teacher_used_for_labels": any(
551
+ bool(item["teacher_used_for_labels"])
552
+ for item in source_records
553
+ ),
554
+ "training_runs_required": 1,
555
+ "ordered_curriculum": ordered_curriculum,
556
+ "retain_semantic_exposures": retain_semantic_exposures,
557
+ "repeated_semantic_exposures": repeated_semantic_exposures,
558
+ "shuffle_within_stage_seed": (
559
+ within_stage_shuffle_seed
560
+ if within_stage_shuffle_seed >= 0
561
+ else None
562
+ ),
563
+ },
564
+ "counts": {
565
+ "media_bytes": sum(int(item["bytes"]) for item in inventory_rows),
566
+ "media_files": len(inventory_rows),
567
+ "trainer_rows": len(output_rows),
568
+ },
569
+ "distributions": {
570
+ "answer": dict(sorted(answers.items())),
571
+ "frames_per_row": dict(sorted(frames.items())),
572
+ "sample_loss_weight": {
573
+ str(weight): count
574
+ for weight, count in sorted(weights_after.items())
575
+ },
576
+ "source_release": dict(sorted(by_source.items())),
577
+ },
578
+ "format": FORMAT,
579
+ "inputs": {
580
+ "benchmark_rows_or_labels_admitted": 0,
581
+ "builder_sha256": file_sha256(Path(__file__).resolve(strict=True)),
582
+ "deduplication": {
583
+ "cited_source_parent": True,
584
+ "exact_pixel_question_answer_identity": True,
585
+ "retain_semantic_exposures": retain_semantic_exposures,
586
+ "rejected": dict(sorted(rejected.items())),
587
+ },
588
+ "excluded_parent_kinds": sorted(excluded_parent_kinds),
589
+ "included_source_stage_release_ids": sorted(
590
+ included_source_stage_release_ids
591
+ ),
592
+ "parent_selection": parent_selection_record,
593
+ "final_release_must_restart_from": "Qwen/Qwen3-VL-8B-Instruct",
594
+ "rows_relabelled": 0,
595
+ "rows_reweighted": reweighted_rows,
596
+ "sample_loss_weight_policy": {
597
+ "cap": (
598
+ sample_loss_weight_cap
599
+ if sample_loss_weight_cap > 0
600
+ else None
601
+ ),
602
+ "source_distribution": {
603
+ str(weight): count
604
+ for weight, count in sorted(weights_before.items())
605
+ },
606
+ "output_distribution": {
607
+ str(weight): count
608
+ for weight, count in sorted(weights_after.items())
609
+ },
610
+ },
611
+ "sources": source_records,
612
+ "curriculum_stages": curriculum_stages,
613
+ "trainer_row_order": (
614
+ "source_stage_then_source_row"
615
+ if ordered_curriculum
616
+ else "canonical_uid"
617
+ ),
618
+ },
619
+ "license": {
620
+ "public_media_redistribution_claimed": False,
621
+ "release_plan": "code_recipe_selection_and_source_checksums",
622
+ "scope": "internal_noncommercial_research_only",
623
+ },
624
+ "model_lineage": {
625
+ "required_initialization": "Qwen/Qwen3-VL-8B-Instruct",
626
+ "required_sha256s_sha256": expected_lineage[0],
627
+ "upstream_repo": expected_lineage[1],
628
+ "upstream_revision": expected_lineage[2],
629
+ },
630
+ "public_release_allowed": False,
631
+ "release_id": args.release_id,
632
+ "status": "sealed_internal_training_release",
633
+ "training_allowed": True,
634
+ "training_scope": "pure_vlm_semantic_sft_only",
635
+ }
636
+ (scratch / "manifest.json").write_bytes(canonical_json_line(manifest))
637
+ validate_semantic_release(scratch, verify_media_hashes=True)
638
+ os.replace(scratch, output)
639
+ return manifest | {"output": str(output)}
640
+ except BaseException:
641
+ shutil.rmtree(scratch, ignore_errors=True)
642
+ raise
643
+
644
+
645
+ def main() -> None:
646
+ parser = argparse.ArgumentParser(description=__doc__)
647
+ parser.add_argument("--source", type=Path, action="append", required=True)
648
+ parser.add_argument("--output", type=Path, required=True)
649
+ parser.add_argument("--release-id", required=True)
650
+ parser.add_argument(
651
+ "--copy-mode", choices=("copy", "hardlink", "auto"), default="hardlink"
652
+ )
653
+ parser.add_argument(
654
+ "--sample-loss-weight-cap",
655
+ type=float,
656
+ default=0.0,
657
+ help="Cap inherited sample weights; zero preserves source weights.",
658
+ )
659
+ parser.add_argument(
660
+ "--exclude-parent-kind",
661
+ action="append",
662
+ default=[],
663
+ help="Exclude rows carrying this exact parent kind; may be repeated.",
664
+ )
665
+ parser.add_argument(
666
+ "--include-source-stage-release-id",
667
+ action="append",
668
+ default=[],
669
+ help=(
670
+ "For a composite semantic source, admit only rows inside the "
671
+ "named hash-bound curriculum stages; may be repeated."
672
+ ),
673
+ )
674
+ parser.add_argument(
675
+ "--include-parent-selection",
676
+ type=Path,
677
+ help=(
678
+ "Hash-bind a capability selection and admit only semantic rows "
679
+ "whose cited parent ID is listed in it."
680
+ ),
681
+ )
682
+ parser.add_argument(
683
+ "--ordered-curriculum",
684
+ action="store_true",
685
+ help="Preserve source-stage order and retain repeated stage exposures.",
686
+ )
687
+ parser.add_argument(
688
+ "--retain-semantic-exposures",
689
+ action="store_true",
690
+ help=(
691
+ "Retain every source row as an intentional training exposure while "
692
+ "keeping the normal globally canonicalized row order."
693
+ ),
694
+ )
695
+ parser.add_argument(
696
+ "--legacy-v62di-source",
697
+ type=Path,
698
+ help="Pinned legacy release; admits only its 2,320 V62DI rows.",
699
+ )
700
+ parser.add_argument(
701
+ "--legacy-v62di-mode",
702
+ choices=("v62di_only", "full"),
703
+ default="v62di_only",
704
+ help="Use only V62DI rows or the complete committed 3,594-row pool.",
705
+ )
706
+ parser.add_argument(
707
+ "--legacy-repcount-source",
708
+ type=Path,
709
+ help="Pinned 2,172-row legacy RepCount source, inserted after source 1.",
710
+ )
711
+ parser.add_argument(
712
+ "--shuffle-within-stage-seed",
713
+ type=int,
714
+ default=-1,
715
+ help="Deterministically shuffle each stage while preserving stage order.",
716
+ )
717
+ args = parser.parse_args()
718
+ print(json.dumps(build(args), indent=2, sort_keys=True))
719
+
720
+
721
+ if __name__ == "__main__":
722
+ main()
code/data_pipeline/temporal_scope/build_vst_prefix32_semantic.py ADDED
@@ -0,0 +1,525 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """Build a source-disjoint, pixel-grounded VST prefix-32 semantic pack.
3
+
4
+ The upstream VST metadata provides cumulative count states. A recent-only
5
+ window cannot in general support those labels, so this compiler exposes a
6
+ uniform causal prefix ending strictly before the query time. It emits only
7
+ plain semantic answers (open integer and balanced MCQ); no agent action or
8
+ private state is ever an actor target.
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ import argparse
14
+ import collections
15
+ import concurrent.futures
16
+ import hashlib
17
+ import io
18
+ import json
19
+ import math
20
+ import os
21
+ import shutil
22
+ import tempfile
23
+ from collections.abc import Mapping, Sequence
24
+ from pathlib import Path
25
+ from typing import Any
26
+
27
+ from PIL import Image
28
+
29
+ from data_pipeline.temporal_scope.semantic_mcq import (
30
+ DIRECT_PROMPT_PREFIX,
31
+ DIRECT_PROMPT_SUFFIX,
32
+ rebalance_answer_slot,
33
+ )
34
+ from viss_core.data_v0.canonical import (
35
+ attach_content_sha256,
36
+ canonical_json_line,
37
+ file_sha256,
38
+ stable_id,
39
+ write_jsonl,
40
+ )
41
+ from viss_core.data_v0.schema import validate_row
42
+
43
+
44
+ FORMAT = "streameva.vst_prefix32_semantic.v1"
45
+ PREFLIGHT_FORMAT = "streameva.vst_count_metadata_preflight.v1"
46
+ OPTION_LETTERS = "ABCD"
47
+ TARGET_COUNTS = tuple(range(11))
48
+ PREFIX_FRAME_MAX_PIXELS = 128 * 28 * 28
49
+ PREFIX_FRAME_MAX_SIDE = 504
50
+
51
+
52
+ class VSTPrefixSemanticError(ValueError):
53
+ """Raised when a prefix-32 semantic candidate violates its contract."""
54
+
55
+
56
+ def main() -> None:
57
+ parser = argparse.ArgumentParser(description=__doc__)
58
+ parser.add_argument("--selection", type=Path, required=True)
59
+ parser.add_argument("--preflight", type=Path, required=True)
60
+ parser.add_argument("--media-root", type=Path, required=True)
61
+ parser.add_argument("--output", type=Path, required=True)
62
+ parser.add_argument("--frames", type=int, default=32)
63
+ parser.add_argument("--train-per-count", type=int, default=92)
64
+ parser.add_argument("--holdout-per-count", type=int, default=5)
65
+ parser.add_argument("--workers", type=int, default=16)
66
+ parser.add_argument("--seed", type=int, default=173)
67
+ args = parser.parse_args()
68
+ print(json.dumps(materialize(args), indent=2, sort_keys=True))
69
+
70
+
71
+ def select_available_states(
72
+ rows: Sequence[Mapping[str, Any]],
73
+ *,
74
+ media_root: Path,
75
+ train_per_count: int,
76
+ holdout_per_count: int,
77
+ seed: int,
78
+ ) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
79
+ """Select balanced, source-disjoint train and holdout states."""
80
+
81
+ if train_per_count <= 0 or holdout_per_count <= 0:
82
+ raise VSTPrefixSemanticError("train/holdout quotas must be positive")
83
+ by_count: dict[int, list[dict[str, Any]]] = {
84
+ value: [] for value in TARGET_COUNTS
85
+ }
86
+ seen_videos: set[str] = set()
87
+ seen_ids: set[str] = set()
88
+ for raw in rows:
89
+ row = dict(raw)
90
+ answer = int(row.get("answer", -1))
91
+ relative = Path(str(row.get("source_video_relative") or ""))
92
+ candidate_id = str(row.get("candidate_id") or "")
93
+ if (
94
+ answer not in by_count
95
+ or not candidate_id
96
+ or relative.is_absolute()
97
+ or ".." in relative.parts
98
+ or relative.suffix.casefold() != ".mp4"
99
+ ):
100
+ raise VSTPrefixSemanticError("selection row is malformed")
101
+ if candidate_id in seen_ids or relative.as_posix() in seen_videos:
102
+ raise VSTPrefixSemanticError("selection IDs/videos must be unique")
103
+ seen_ids.add(candidate_id)
104
+ seen_videos.add(relative.as_posix())
105
+ source = (media_root / relative).resolve()
106
+ if not source.is_file() or source.is_symlink():
107
+ continue
108
+ row["source_video"] = str(source)
109
+ by_count[answer].append(row)
110
+
111
+ train: list[dict[str, Any]] = []
112
+ holdout: list[dict[str, Any]] = []
113
+ required = train_per_count + holdout_per_count
114
+ for answer in TARGET_COUNTS:
115
+ values = by_count[answer]
116
+ values.sort(
117
+ key=lambda row: hashlib.sha256(
118
+ f"{seed}|{answer}|{row['candidate_id']}".encode()
119
+ ).digest()
120
+ )
121
+ if len(values) < required:
122
+ raise VSTPrefixSemanticError(
123
+ f"count {answer} has {len(values)} available videos; need {required}"
124
+ )
125
+ for row in values[:holdout_per_count]:
126
+ value = dict(row)
127
+ value["split"] = "policy_holdout"
128
+ holdout.append(value)
129
+ for row in values[holdout_per_count:required]:
130
+ value = dict(row)
131
+ value["split"] = "train"
132
+ train.append(value)
133
+ return (
134
+ sorted(train, key=lambda row: str(row["candidate_id"])),
135
+ sorted(holdout, key=lambda row: str(row["candidate_id"])),
136
+ )
137
+
138
+
139
+ def uniform_causal_indices(
140
+ *, source_total: int, source_fps: float, query_time: float, frames: int
141
+ ) -> tuple[list[int], list[float]]:
142
+ """Return exactly ``frames`` uniform indices whose timestamps are causal."""
143
+
144
+ if (
145
+ source_total <= 0
146
+ or not math.isfinite(source_fps)
147
+ or source_fps <= 0
148
+ or not math.isfinite(query_time)
149
+ or query_time <= 0
150
+ or frames <= 1
151
+ ):
152
+ raise VSTPrefixSemanticError("invalid video sampling metadata")
153
+ # ceil(t * fps) - 1 is strictly earlier than t, including exact boundaries.
154
+ last = min(source_total - 1, max(0, math.ceil(query_time * source_fps) - 1))
155
+ if last == 0:
156
+ indices = [0] * frames
157
+ else:
158
+ indices = [
159
+ int(round(position * last / (frames - 1))) for position in range(frames)
160
+ ]
161
+ timestamps = [index / source_fps for index in indices]
162
+ if any(timestamp >= query_time + 1e-9 for timestamp in timestamps):
163
+ raise VSTPrefixSemanticError("prefix sampler exposed a future frame")
164
+ return indices, timestamps
165
+
166
+
167
+ def project_semantic_rows(
168
+ row: Mapping[str, Any],
169
+ *,
170
+ relative_paths: Sequence[str],
171
+ frame_timestamps: Sequence[float],
172
+ source_sha256: str,
173
+ selection_sha256: str,
174
+ ) -> list[dict[str, Any]]:
175
+ """Project one grounded count state into open and balanced-MCQ rows."""
176
+
177
+ if len(relative_paths) != 32 or len(frame_timestamps) != 32:
178
+ raise VSTPrefixSemanticError("prefix projection requires exactly 32 frames")
179
+ answer = int(row["answer"])
180
+ if answer not in TARGET_COUNTS:
181
+ raise VSTPrefixSemanticError("count answer is outside 0--10")
182
+ candidate_id = str(row["candidate_id"])
183
+ split = str(row["split"])
184
+ if split not in {"train", "policy_holdout"}:
185
+ raise VSTPrefixSemanticError("invalid semantic split")
186
+ images = [
187
+ {
188
+ "type": "image",
189
+ "image": path,
190
+ "frame_id": stable_id(
191
+ "vstprefixframe", candidate_id, index, timestamp, digest_length=20
192
+ ),
193
+ }
194
+ for index, (path, timestamp) in enumerate(
195
+ zip(relative_paths, frame_timestamps, strict=True)
196
+ )
197
+ ]
198
+ parent = {
199
+ "kind": "vst_count_metadata_state",
200
+ "id": stable_id("vstprefixsource", candidate_id, digest_length=20),
201
+ "sha256": selection_sha256,
202
+ }
203
+ group_id = stable_id("vstprefixgroup", candidate_id, digest_length=20)
204
+ question = " ".join(str(row["question"]).split())
205
+ answer_text = str(answer)
206
+ open_row = attach_content_sha256(
207
+ {
208
+ "schema": "streameva.sft_row.v1",
209
+ "uid": stable_id("sftvstprefixopen", candidate_id, digest_length=20),
210
+ "component": "preservation",
211
+ "group_id": group_id,
212
+ "split": split,
213
+ "decision_loss_profile": "full",
214
+ "messages": [
215
+ {
216
+ "role": "user",
217
+ "content": [
218
+ *images,
219
+ {
220
+ "type": "text",
221
+ "text": (
222
+ f"{question}\nAnswer with one integer from 0 to 10 "
223
+ "and no other text."
224
+ ),
225
+ },
226
+ ],
227
+ "loss": False,
228
+ },
229
+ {
230
+ "role": "assistant",
231
+ "content": [{"type": "text", "text": answer_text}],
232
+ "loss": True,
233
+ },
234
+ ],
235
+ "sample_loss_weight": 1.0,
236
+ "parents": [parent],
237
+ }
238
+ )
239
+
240
+ distractors = [value for value in TARGET_COUNTS if value != answer]
241
+ distractors.sort(
242
+ key=lambda value: (
243
+ abs(value - answer),
244
+ hashlib.sha256(f"{candidate_id}|{value}".encode()).digest(),
245
+ )
246
+ )
247
+ options, mcq_answer = rebalance_answer_slot(
248
+ [answer_text, *[str(value) for value in distractors[:3]]],
249
+ answer="A",
250
+ answer_slot=int(hashlib.sha256(candidate_id.encode()).hexdigest(), 16) % 4,
251
+ seed=f"{candidate_id}|prefix32-mcq-v1",
252
+ )
253
+ option_text = "\n".join(
254
+ f"{letter}. {value}"
255
+ for letter, value in zip(OPTION_LETTERS, options, strict=True)
256
+ )
257
+ mcq_prompt = (
258
+ f"{DIRECT_PROMPT_PREFIX}\n\nQuestion: {question}\n\n"
259
+ f"Options:\n{option_text}\n\n{DIRECT_PROMPT_SUFFIX}"
260
+ )
261
+ mcq_row = attach_content_sha256(
262
+ {
263
+ "schema": "streameva.sft_row.v1",
264
+ "uid": stable_id("sftvstprefixmcq", candidate_id, digest_length=20),
265
+ "component": "preservation",
266
+ "group_id": group_id,
267
+ "split": split,
268
+ "decision_loss_profile": "full",
269
+ "messages": [
270
+ {
271
+ "role": "user",
272
+ "content": [*images, {"type": "text", "text": mcq_prompt}],
273
+ "loss": False,
274
+ },
275
+ {
276
+ "role": "assistant",
277
+ "content": [{"type": "text", "text": mcq_answer}],
278
+ "loss": True,
279
+ },
280
+ ],
281
+ "sample_loss_weight": 1.0,
282
+ "parents": [parent],
283
+ }
284
+ )
285
+ for output in (open_row, mcq_row):
286
+ validate_row(output)
287
+ # Bind source pixels in the manifest-facing auxiliary state without leaking
288
+ # an absolute path into the strict trainer row.
289
+ del source_sha256
290
+ return [open_row, mcq_row]
291
+
292
+
293
+ def materialize(args: argparse.Namespace) -> dict[str, Any]:
294
+ selection = args.selection.resolve()
295
+ preflight_path = args.preflight.resolve()
296
+ media_root = args.media_root.resolve()
297
+ output = args.output.resolve()
298
+ if output.exists():
299
+ raise FileExistsError(output)
300
+ if args.frames != 32:
301
+ raise VSTPrefixSemanticError("this release is frozen to exactly 32 frames")
302
+ if not 1 <= args.workers <= 32:
303
+ raise VSTPrefixSemanticError("workers must lie in [1, 32]")
304
+ preflight = json.loads(preflight_path.read_text(encoding="utf-8"))
305
+ declaration = preflight.get("selection_artifact", {})
306
+ if (
307
+ preflight.get("format") != PREFLIGHT_FORMAT
308
+ or preflight.get("status") != "pass_metadata_firewall_and_selection"
309
+ or preflight.get("contracts", {}).get("basename_hits_admitted") != 0
310
+ or preflight.get("contracts", {}).get("question_hits_admitted") != 0
311
+ or declaration.get("sha256") != file_sha256(selection)
312
+ ):
313
+ raise VSTPrefixSemanticError("metadata preflight is not valid or hash-bound")
314
+ rows = [
315
+ json.loads(line)
316
+ for line in selection.read_text(encoding="utf-8").splitlines()
317
+ if line.strip()
318
+ ]
319
+ train, holdout = select_available_states(
320
+ rows,
321
+ media_root=media_root,
322
+ train_per_count=args.train_per_count,
323
+ holdout_per_count=args.holdout_per_count,
324
+ seed=args.seed,
325
+ )
326
+ selected = [*train, *holdout]
327
+ output.parent.mkdir(parents=True, exist_ok=True)
328
+ scratch = Path(tempfile.mkdtemp(prefix=f".{output.name}.partial.", dir=output.parent))
329
+ try:
330
+ media_out = scratch / "media"
331
+
332
+ def decode(row: Mapping[str, Any]) -> tuple[str, dict[str, Any]]:
333
+ return str(row["candidate_id"]), _decode_prefix(row, media_out, args.frames)
334
+
335
+ decoded: dict[str, dict[str, Any]] = {}
336
+ errors: list[dict[str, str]] = []
337
+ with concurrent.futures.ThreadPoolExecutor(max_workers=args.workers) as pool:
338
+ futures = {pool.submit(decode, row): row for row in selected}
339
+ for future in concurrent.futures.as_completed(futures):
340
+ row = futures[future]
341
+ try:
342
+ candidate_id, state = future.result()
343
+ decoded[candidate_id] = state
344
+ except Exception as exc: # pragma: no cover - remote media failures
345
+ errors.append(
346
+ {
347
+ "candidate_id": str(row["candidate_id"]),
348
+ "reason": f"{type(exc).__name__}: {exc}",
349
+ }
350
+ )
351
+ write_jsonl(errors, scratch / "decode_errors.jsonl")
352
+ if errors or len(decoded) != len(selected):
353
+ raise VSTPrefixSemanticError(f"prefix decode failed for {len(errors)} rows")
354
+
355
+ train_rows: list[dict[str, Any]] = []
356
+ holdout_rows: list[dict[str, Any]] = []
357
+ source_inventory = []
358
+ selection_sha = file_sha256(selection)
359
+ for row in selected:
360
+ state = decoded[str(row["candidate_id"])]
361
+ source_sha = file_sha256(str(row["source_video"]))
362
+ source_inventory.append(
363
+ {
364
+ "candidate_id": row["candidate_id"],
365
+ "source_video_relative": row["source_video_relative"],
366
+ "source_sha256": source_sha,
367
+ "split": row["split"],
368
+ "query_time": row["query_time"],
369
+ "answer": row["answer"],
370
+ "frame_indices": state["frame_indices"],
371
+ "frame_timestamps": state["frame_timestamps"],
372
+ }
373
+ )
374
+ projected = project_semantic_rows(
375
+ row,
376
+ relative_paths=state["relative_paths"],
377
+ frame_timestamps=state["frame_timestamps"],
378
+ source_sha256=source_sha,
379
+ selection_sha256=selection_sha,
380
+ )
381
+ (train_rows if row["split"] == "train" else holdout_rows).extend(projected)
382
+ write_jsonl(train_rows, scratch / "trainer/sft.jsonl", sort_key="uid")
383
+ write_jsonl(
384
+ holdout_rows, scratch / "holdout/policy_holdout.jsonl", sort_key="uid"
385
+ )
386
+ write_jsonl(
387
+ source_inventory,
388
+ scratch / "source_inventory.jsonl",
389
+ sort_key="candidate_id",
390
+ )
391
+ media_files = sorted(path for path in media_out.rglob("*.jpg") if path.is_file())
392
+ manifest = {
393
+ "format": FORMAT,
394
+ "status": "materialized_internal_research_candidate",
395
+ "training_allowed": True,
396
+ "public_release_allowed": False,
397
+ "protocol": {
398
+ "causal_prefix_only": True,
399
+ "frames": 32,
400
+ "per_frame_max_pixels": PREFIX_FRAME_MAX_PIXELS,
401
+ "sampling": "exact_uniform_indices_over_[0,query_time)",
402
+ "visual_input_mode": "independent_image_frames",
403
+ },
404
+ "counts": {
405
+ "train_source_states": len(train),
406
+ "holdout_source_states": len(holdout),
407
+ "train_rows": len(train_rows),
408
+ "holdout_rows": len(holdout_rows),
409
+ "media_files": len(media_files),
410
+ "train_by_answer": dict(
411
+ sorted(collections.Counter(str(row["answer"]) for row in train).items())
412
+ ),
413
+ "holdout_by_answer": dict(
414
+ sorted(
415
+ collections.Counter(str(row["answer"]) for row in holdout).items()
416
+ )
417
+ ),
418
+ },
419
+ "inputs": {
420
+ "selection_sha256": selection_sha,
421
+ "preflight_sha256": file_sha256(preflight_path),
422
+ "builder_sha256": file_sha256(Path(__file__)),
423
+ },
424
+ "artifacts": {
425
+ "trainer/sft.jsonl": file_sha256(scratch / "trainer/sft.jsonl"),
426
+ "holdout/policy_holdout.jsonl": file_sha256(
427
+ scratch / "holdout/policy_holdout.jsonl"
428
+ ),
429
+ "source_inventory.jsonl": file_sha256(
430
+ scratch / "source_inventory.jsonl"
431
+ ),
432
+ "decode_errors.jsonl": file_sha256(scratch / "decode_errors.jsonl"),
433
+ },
434
+ "contracts": {
435
+ "all_labels_metadata_derived": True,
436
+ "all_pixels_strictly_before_query": True,
437
+ "benchmark_firewall_inherited": True,
438
+ "holdout_source_disjoint": True,
439
+ "no_agent_actions": True,
440
+ "no_private_state_actor_targets": True,
441
+ "source_videos_globally_unique": True,
442
+ "teacher_used_for_labels": False,
443
+ },
444
+ "license": {
445
+ "public_media_redistribution_claimed": False,
446
+ "release_mode": "recipe_plus_source_checksums",
447
+ },
448
+ }
449
+ (scratch / "manifest.json").write_bytes(canonical_json_line(manifest))
450
+ os.replace(scratch, output)
451
+ return {**manifest, "output": str(output)}
452
+ except Exception:
453
+ shutil.rmtree(scratch, ignore_errors=True)
454
+ raise
455
+
456
+
457
+ def _decode_prefix(
458
+ row: Mapping[str, Any], media_root: Path, frames: int
459
+ ) -> dict[str, Any]:
460
+ from decord import VideoReader
461
+
462
+ source = Path(str(row["source_video"]))
463
+ reader = VideoReader(str(source))
464
+ try:
465
+ fps = float(reader.get_avg_fps())
466
+ total = len(reader)
467
+ indices, timestamps = uniform_causal_indices(
468
+ source_total=total,
469
+ source_fps=fps,
470
+ query_time=float(row["query_time"]),
471
+ frames=frames,
472
+ )
473
+ arrays = reader.get_batch(indices).asnumpy()
474
+ finally:
475
+ del reader
476
+ relative_paths = []
477
+ for array in arrays:
478
+ image = Image.fromarray(array)
479
+ height, width = image.height, image.width
480
+ resized_height, resized_width = _rounded_size(height, width)
481
+ if (resized_height, resized_width) != (height, width):
482
+ image = image.resize((resized_width, resized_height), Image.Resampling.LANCZOS)
483
+ buffer = io.BytesIO()
484
+ image.save(buffer, format="JPEG", quality=95, subsampling=0, optimize=False)
485
+ payload = buffer.getvalue()
486
+ digest = hashlib.sha256(payload).hexdigest()
487
+ relative = Path("media") / digest[:2] / f"{digest}.jpg"
488
+ destination = media_root.parent / relative
489
+ destination.parent.mkdir(parents=True, exist_ok=True)
490
+ try:
491
+ with destination.open("xb") as handle:
492
+ handle.write(payload)
493
+ except FileExistsError:
494
+ pass
495
+ if file_sha256(destination) != digest:
496
+ raise VSTPrefixSemanticError("decoded frame hash mismatch")
497
+ relative_paths.append(relative.as_posix())
498
+ return {
499
+ "relative_paths": relative_paths,
500
+ "frame_indices": indices,
501
+ "frame_timestamps": timestamps,
502
+ }
503
+
504
+
505
+ def _rounded_size(height: int, width: int) -> tuple[int, int]:
506
+ factor = 28
507
+ max_pixels = PREFIX_FRAME_MAX_PIXELS
508
+ max_side = PREFIX_FRAME_MAX_SIDE
509
+ scale = min(1.0, max_side / max(height, width))
510
+ if height * width * scale * scale > max_pixels:
511
+ scale = min(scale, math.sqrt(max_pixels / (height * width)))
512
+ new_height = max(factor, int(round(height * scale / factor)) * factor)
513
+ new_width = max(factor, int(round(width * scale / factor)) * factor)
514
+ while new_height * new_width > max_pixels:
515
+ if new_height >= new_width and new_height > factor:
516
+ new_height -= factor
517
+ elif new_width > factor:
518
+ new_width -= factor
519
+ else:
520
+ break
521
+ return new_height, new_width
522
+
523
+
524
+ if __name__ == "__main__":
525
+ main()
code/data_pipeline/temporal_scope/filter_semantic_release_by_blind_audit.py ADDED
@@ -0,0 +1,329 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """Seal the high-confidence agreement subset of a blind-audited SFT release.
3
+
4
+ The visual auditor never receives the source answer. This builder also never
5
+ uses the auditor prediction as a new label: it only admits an existing source
6
+ row when a complete, hash-bound audit independently recovered that row's
7
+ answer. Partial audits are deliberately ineligible for materialization.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ import argparse
13
+ import collections
14
+ import hashlib
15
+ import json
16
+ import os
17
+ import shutil
18
+ import tempfile
19
+ from pathlib import Path
20
+ from typing import Any
21
+
22
+ from viss_core.data_v0.canonical import (
23
+ canonical_json_line,
24
+ file_sha256,
25
+ read_jsonl,
26
+ write_jsonl,
27
+ )
28
+ from viss_core.data_v0.semantic_release import (
29
+ FORMAT,
30
+ validate_semantic_release,
31
+ )
32
+
33
+
34
+ AUDIT_ROW_FORMAT = "streameva.semantic_release.blind_visual_audit_row.v0"
35
+ AUDIT_SUMMARY_FORMAT = "streameva.semantic_release.blind_visual_audit.v0"
36
+ FILTER_FORMAT = "streameva.semantic_release.blind_visual_filter.v0"
37
+
38
+
39
+ class BlindAuditFilterError(ValueError):
40
+ """Raised when an audit is incomplete or not independent of its labels."""
41
+
42
+
43
+ def _target(row: dict[str, Any]) -> str:
44
+ values = [
45
+ str(item.get("text") or "")
46
+ for message in row.get("messages") or []
47
+ if message.get("role") == "assistant" and message.get("loss") is True
48
+ for item in message.get("content") or []
49
+ if item.get("type") == "text"
50
+ ]
51
+ if len(values) != 1:
52
+ raise BlindAuditFilterError("semantic row has no unique target")
53
+ return values[0]
54
+
55
+
56
+ def _images(rows: list[dict[str, Any]]) -> set[str]:
57
+ return {
58
+ str(item["image"])
59
+ for row in rows
60
+ for message in row["messages"]
61
+ for item in message["content"]
62
+ if item.get("type") == "image"
63
+ }
64
+
65
+
66
+ def _answer_balanced_subset(
67
+ rows: list[dict[str, Any]], *, count: int, seed: str
68
+ ) -> list[dict[str, Any]]:
69
+ if count <= 0 or count > len(rows):
70
+ raise BlindAuditFilterError("subset row count exceeds audited agreements")
71
+ by_answer: dict[str, list[dict[str, Any]]] = collections.defaultdict(list)
72
+ for row in rows:
73
+ by_answer[_target(row)].append(row)
74
+ if set(by_answer) != set("ABCD"):
75
+ raise BlindAuditFilterError("answer-balanced subset requires A/B/C/D rows")
76
+ base, remainder = divmod(count, 4)
77
+ selected: list[dict[str, Any]] = []
78
+ for index, answer in enumerate("ABCD"):
79
+ quota = base + int(index < remainder)
80
+ ranked = sorted(
81
+ by_answer[answer],
82
+ key=lambda row: hashlib.sha256(
83
+ f"blind-filter-subset:{seed}:{row['uid']}".encode()
84
+ ).hexdigest(),
85
+ )
86
+ if len(ranked) < quota:
87
+ raise BlindAuditFilterError(
88
+ f"answer {answer} has {len(ranked)} rows but needs {quota}"
89
+ )
90
+ selected.extend(ranked[:quota])
91
+ return sorted(selected, key=lambda row: str(row["uid"]))
92
+
93
+
94
+ def _materialize(source: Path, destination: Path, *, mode: str) -> None:
95
+ destination.parent.mkdir(parents=True, exist_ok=True)
96
+ if source.is_symlink() or not source.is_file():
97
+ raise BlindAuditFilterError(f"unsafe source media: {source}")
98
+ if mode in {"hardlink", "auto"}:
99
+ try:
100
+ os.link(source, destination)
101
+ return
102
+ except OSError:
103
+ if mode == "hardlink":
104
+ raise
105
+ shutil.copy2(source, destination)
106
+
107
+
108
+ def build(args: argparse.Namespace) -> dict[str, Any]:
109
+ source = args.source.resolve(strict=True)
110
+ audit_path = args.audit.resolve(strict=True)
111
+ summary_path = args.audit_summary.resolve(strict=True)
112
+ output = args.output.resolve()
113
+ if output.exists():
114
+ raise FileExistsError(output)
115
+ if args.copy_mode not in {"copy", "hardlink", "auto"}:
116
+ raise BlindAuditFilterError("unsupported copy mode")
117
+ allowed_confidence = set(args.confidence)
118
+ if not allowed_confidence or not allowed_confidence <= {"high", "medium"}:
119
+ raise BlindAuditFilterError("confidence must be high and/or medium")
120
+ if not 0.0 <= args.min_source_accuracy <= 1.0:
121
+ raise BlindAuditFilterError("minimum source accuracy must lie in [0, 1]")
122
+ if args.min_rows <= 0:
123
+ raise BlindAuditFilterError("minimum retained rows must be positive")
124
+
125
+ source_manifest = validate_semantic_release(source, verify_media_hashes=False)
126
+ source_trainer = source / "trainer/sft.jsonl"
127
+ source_rows = read_jsonl(
128
+ source_trainer, verify_content_hashes=True, require_canonical=True
129
+ )
130
+ rows_by_uid = {str(row["uid"]): row for row in source_rows}
131
+ if len(rows_by_uid) != len(source_rows):
132
+ raise BlindAuditFilterError("source UIDs are not unique")
133
+
134
+ summary = json.loads(summary_path.read_text(encoding="utf-8"))
135
+ required_summary = {
136
+ "format": AUDIT_SUMMARY_FORMAT,
137
+ "status": "admit_semantic_release",
138
+ "admitted": True,
139
+ "release_id": source_manifest["release_id"],
140
+ "release_manifest_sha256": file_sha256(source / "manifest.json"),
141
+ "trainer_sha256": file_sha256(source_trainer),
142
+ "sample_rows": len(source_rows),
143
+ "valid_rows": len(source_rows),
144
+ "errors": 0,
145
+ "expected_answers_sent_to_auditor": False,
146
+ "auditor_outputs_admitted_to_training": False,
147
+ "benchmark_rows_used": 0,
148
+ "result_rows_sha256": file_sha256(audit_path),
149
+ }
150
+ if any(summary.get(key) != value for key, value in required_summary.items()):
151
+ raise BlindAuditFilterError("audit summary is partial, rejected, or unbound")
152
+ accuracy = float(summary.get("accuracy", -1.0))
153
+ if accuracy < args.min_source_accuracy:
154
+ raise BlindAuditFilterError(
155
+ f"blind source agreement {accuracy:.4f} is below "
156
+ f"{args.min_source_accuracy:.4f}"
157
+ )
158
+
159
+ audit_rows = read_jsonl(audit_path, require_canonical=True)
160
+ audit_by_uid: dict[str, dict[str, Any]] = {}
161
+ for item in audit_rows:
162
+ uid = str(item.get("uid") or "")
163
+ if (
164
+ item.get("format") != AUDIT_ROW_FORMAT
165
+ or not uid
166
+ or uid in audit_by_uid
167
+ or uid not in rows_by_uid
168
+ ):
169
+ raise BlindAuditFilterError("audit row identity diverged")
170
+ expected = _target(rows_by_uid[uid])
171
+ prediction = item.get("prediction")
172
+ if (
173
+ item.get("expected") != expected
174
+ or item.get("error") is not None
175
+ or item.get("confidence") not in {"high", "medium", "low"}
176
+ or item.get("correct") is not (prediction == expected)
177
+ ):
178
+ raise BlindAuditFilterError("audit decision is malformed")
179
+ audit_by_uid[uid] = item
180
+ if set(audit_by_uid) != set(rows_by_uid):
181
+ raise BlindAuditFilterError("blind audit does not cover every source row")
182
+
183
+ selected = [
184
+ row
185
+ for uid, row in rows_by_uid.items()
186
+ if audit_by_uid[uid]["correct"] is True
187
+ and audit_by_uid[uid]["confidence"] in allowed_confidence
188
+ ]
189
+ selected.sort(key=lambda row: str(row["uid"]))
190
+ eligible_rows = len(selected)
191
+ max_rows = getattr(args, "max_rows", None)
192
+ selection_seed = str(getattr(args, "selection_seed", "blind-filter-subset-v0"))
193
+ if max_rows is not None:
194
+ selected = _answer_balanced_subset(
195
+ selected, count=int(max_rows), seed=selection_seed
196
+ )
197
+ if len(selected) < args.min_rows:
198
+ raise BlindAuditFilterError(
199
+ f"only {len(selected)} high-confidence rows survived; "
200
+ f"requires {args.min_rows}"
201
+ )
202
+
203
+ source_inventory_path = source / "manifests/media_inventory.jsonl"
204
+ source_inventory = {
205
+ str(item["path"]): item
206
+ for item in read_jsonl(source_inventory_path, require_canonical=True)
207
+ }
208
+ referenced = _images(selected)
209
+ if not referenced <= set(source_inventory):
210
+ raise BlindAuditFilterError("selected media is absent from source inventory")
211
+
212
+ output.parent.mkdir(parents=True, exist_ok=True)
213
+ scratch = Path(tempfile.mkdtemp(prefix=f".{output.name}.partial.", dir=output.parent))
214
+ try:
215
+ write_jsonl(selected, scratch / "trainer/sft.jsonl")
216
+ selected_inventory = [source_inventory[path] for path in sorted(referenced)]
217
+ write_jsonl(selected_inventory, scratch / "manifests/media_inventory.jsonl")
218
+ for item in selected_inventory:
219
+ relative = Path(str(item["path"]))
220
+ source_file = source / relative
221
+ destination = scratch / relative
222
+ _materialize(source_file, destination, mode=args.copy_mode)
223
+ if (
224
+ destination.stat().st_size != int(item["bytes"])
225
+ or file_sha256(destination) != item["sha256"]
226
+ ):
227
+ raise BlindAuditFilterError("materialized media hash diverged")
228
+
229
+ answers = collections.Counter(_target(row) for row in selected)
230
+ frames = collections.Counter(
231
+ sum(
232
+ item.get("type") == "image"
233
+ for message in row["messages"]
234
+ for item in message["content"]
235
+ )
236
+ for row in selected
237
+ )
238
+ confidence = collections.Counter(
239
+ str(audit_by_uid[str(row["uid"])]["confidence"]) for row in selected
240
+ )
241
+ manifest = {
242
+ "artifacts": {
243
+ "manifests/media_inventory.jsonl": file_sha256(
244
+ scratch / "manifests/media_inventory.jsonl"
245
+ ),
246
+ "trainer/sft.jsonl": file_sha256(scratch / "trainer/sft.jsonl"),
247
+ },
248
+ "contracts": source_manifest["contracts"],
249
+ "counts": {
250
+ "media_bytes": sum(int(item["bytes"]) for item in selected_inventory),
251
+ "media_files": len(selected_inventory),
252
+ "trainer_rows": len(selected),
253
+ },
254
+ "distributions": {
255
+ "answer": dict(sorted(answers.items())),
256
+ "blind_auditor_confidence": dict(sorted(confidence.items())),
257
+ "frames_per_row": {
258
+ str(key): value for key, value in sorted(frames.items())
259
+ },
260
+ },
261
+ "format": FORMAT,
262
+ "inputs": {
263
+ **source_manifest["inputs"],
264
+ "blind_visual_filter": {
265
+ "format": FILTER_FORMAT,
266
+ "source_release_id": source_manifest["release_id"],
267
+ "source_manifest_sha256": file_sha256(source / "manifest.json"),
268
+ "source_trainer_sha256": file_sha256(source_trainer),
269
+ "audit_sha256": file_sha256(audit_path),
270
+ "audit_summary_sha256": file_sha256(summary_path),
271
+ "auditor_model": str(summary.get("auditor_model") or ""),
272
+ "auditor_received_expected_answers": False,
273
+ "auditor_predictions_used_as_labels": False,
274
+ "complete_source_coverage_required": True,
275
+ "eligible_rows_after_audit_filter": eligible_rows,
276
+ "filter_builder_sha256": file_sha256(Path(__file__).resolve(strict=True)),
277
+ "max_rows": max_rows,
278
+ "source_agreement_accuracy": accuracy,
279
+ "subset_selection_seed": selection_seed if max_rows is not None else None,
280
+ "subset_selection_strategy": (
281
+ "answer_balanced_stable_hash" if max_rows is not None else "all"
282
+ ),
283
+ "allowed_confidence": sorted(allowed_confidence),
284
+ "source_rows": len(source_rows),
285
+ "selected_rows": len(selected),
286
+ },
287
+ },
288
+ "license": source_manifest["license"],
289
+ "model_lineage": source_manifest["model_lineage"],
290
+ "public_release_allowed": False,
291
+ "release_id": args.release_id,
292
+ "status": "sealed_internal_training_release",
293
+ "training_allowed": True,
294
+ "training_scope": "pure_vlm_semantic_sft_only",
295
+ }
296
+ (scratch / "manifest.json").write_bytes(canonical_json_line(manifest))
297
+ validate_semantic_release(scratch, verify_media_hashes=True)
298
+ os.replace(scratch, output)
299
+ return manifest
300
+ except Exception:
301
+ shutil.rmtree(scratch, ignore_errors=True)
302
+ raise
303
+
304
+
305
+ def main() -> None:
306
+ parser = argparse.ArgumentParser(description=__doc__)
307
+ parser.add_argument("--source", type=Path, required=True)
308
+ parser.add_argument("--audit", type=Path, required=True)
309
+ parser.add_argument("--audit-summary", type=Path, required=True)
310
+ parser.add_argument("--output", type=Path, required=True)
311
+ parser.add_argument("--release-id", required=True)
312
+ parser.add_argument(
313
+ "--confidence", action="append", choices=("high", "medium"), default=[]
314
+ )
315
+ parser.add_argument("--min-source-accuracy", type=float, default=0.90)
316
+ parser.add_argument("--min-rows", type=int, default=1)
317
+ parser.add_argument("--max-rows", type=int)
318
+ parser.add_argument("--selection-seed", default="blind-filter-subset-v0")
319
+ parser.add_argument(
320
+ "--copy-mode", choices=("copy", "hardlink", "auto"), default="hardlink"
321
+ )
322
+ args = parser.parse_args()
323
+ if not args.confidence:
324
+ args.confidence = ["high"]
325
+ print(json.dumps(build(args), indent=2, sort_keys=True))
326
+
327
+
328
+ if __name__ == "__main__":
329
+ main()
code/data_pipeline/temporal_scope/filter_semantic_release_by_option_consensus.py ADDED
@@ -0,0 +1,298 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """Seal source-labelled rows that survive an option-invariant visual audit.
3
+
4
+ The two-presentation auditor never receives the source answer. This builder
5
+ also never treats an auditor prediction as a replacement label: a source row
6
+ is eligible only when both option orders independently recover the unchanged
7
+ source answer with an allowed confidence level.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ import argparse
13
+ import collections
14
+ import json
15
+ import os
16
+ import shutil
17
+ import tempfile
18
+ from pathlib import Path
19
+ from typing import Any
20
+
21
+ from data_pipeline.temporal_scope.filter_semantic_release_by_blind_audit import (
22
+ BlindAuditFilterError,
23
+ _answer_balanced_subset,
24
+ _images,
25
+ _materialize,
26
+ _target,
27
+ )
28
+ from viss_core.data_v0.canonical import (
29
+ canonical_json_line,
30
+ file_sha256,
31
+ read_jsonl,
32
+ write_jsonl,
33
+ )
34
+ from viss_core.data_v0.semantic_release import FORMAT, validate_semantic_release
35
+
36
+
37
+ AUDIT_ROW_FORMAT = "streameva.semantic_release.option_consensus_audit_row.v0"
38
+ AUDIT_SUMMARY_FORMAT = "streameva.semantic_release.option_consensus_audit.v0"
39
+ FILTER_FORMAT = "streameva.semantic_release.option_consensus_filter.v0"
40
+
41
+
42
+ def _validate_presentations(
43
+ *,
44
+ source_rows: list[dict[str, Any]],
45
+ audit_rows: list[dict[str, Any]],
46
+ allowed_confidence: set[str],
47
+ ) -> tuple[list[dict[str, Any]], int]:
48
+ rows_by_uid = {str(row["uid"]): row for row in source_rows}
49
+ if len(rows_by_uid) != len(source_rows):
50
+ raise BlindAuditFilterError("source UIDs are not unique")
51
+ presentations: dict[str, dict[int, dict[str, Any]]] = collections.defaultdict(dict)
52
+ for item in audit_rows:
53
+ uid = str(item.get("uid") or "")
54
+ presentation = item.get("presentation")
55
+ if (
56
+ item.get("format") != AUDIT_ROW_FORMAT
57
+ or uid not in rows_by_uid
58
+ or presentation not in {0, 1}
59
+ or presentation in presentations[uid]
60
+ ):
61
+ raise BlindAuditFilterError("option-consensus audit identity diverged")
62
+ expected = _target(rows_by_uid[uid])
63
+ prediction = item.get("prediction_source")
64
+ mapping = item.get("displayed_to_source")
65
+ if (
66
+ item.get("expected_source") != expected
67
+ or item.get("error") is not None
68
+ or item.get("confidence") not in {"high", "medium", "low"}
69
+ or item.get("correct_source") is not (prediction == expected)
70
+ or not isinstance(mapping, list)
71
+ or sorted(mapping) != list("ABCD")
72
+ or (presentation == 0 and mapping != list("ABCD"))
73
+ or (presentation == 1 and mapping == list("ABCD"))
74
+ ):
75
+ raise BlindAuditFilterError("option-consensus audit decision is malformed")
76
+ presentations[uid][int(presentation)] = item
77
+ if set(presentations) != set(rows_by_uid) or any(
78
+ set(values) != {0, 1} for values in presentations.values()
79
+ ):
80
+ raise BlindAuditFilterError("option-consensus audit lacks complete two-order coverage")
81
+
82
+ stable_rows = 0
83
+ selected: list[dict[str, Any]] = []
84
+ for uid, source_row in rows_by_uid.items():
85
+ pair = [presentations[uid][0], presentations[uid][1]]
86
+ predictions = {str(item["prediction_source"]) for item in pair}
87
+ if (
88
+ len(predictions) == 1
89
+ and all(str(item["confidence"]) in allowed_confidence for item in pair)
90
+ ):
91
+ stable_rows += 1
92
+ if next(iter(predictions)) == _target(source_row):
93
+ selected.append(source_row)
94
+ selected.sort(key=lambda row: str(row["uid"]))
95
+ return selected, stable_rows
96
+
97
+
98
+ def build(args: argparse.Namespace) -> dict[str, Any]:
99
+ source = args.source.resolve(strict=True)
100
+ audit_path = args.audit.resolve(strict=True)
101
+ summary_path = args.audit_summary.resolve(strict=True)
102
+ output = args.output.resolve()
103
+ if output.exists():
104
+ raise FileExistsError(output)
105
+ if args.copy_mode not in {"copy", "hardlink", "auto"}:
106
+ raise BlindAuditFilterError("unsupported copy mode")
107
+ allowed_confidence = set(args.confidence)
108
+ if not allowed_confidence or not allowed_confidence <= {"high", "medium"}:
109
+ raise BlindAuditFilterError("confidence must be high and/or medium")
110
+ if not 0.0 <= args.min_consensus_source_accuracy <= 1.0:
111
+ raise BlindAuditFilterError("minimum consensus accuracy must lie in [0, 1]")
112
+ if args.min_rows <= 0:
113
+ raise BlindAuditFilterError("minimum retained rows must be positive")
114
+
115
+ source_manifest = validate_semantic_release(source, verify_media_hashes=False)
116
+ source_trainer = source / "trainer/sft.jsonl"
117
+ source_rows = read_jsonl(
118
+ source_trainer, verify_content_hashes=True, require_canonical=True
119
+ )
120
+ summary = json.loads(summary_path.read_text(encoding="utf-8"))
121
+ required_summary = {
122
+ "format": AUDIT_SUMMARY_FORMAT,
123
+ "status": "complete_option_consensus_audit",
124
+ "release_id": source_manifest["release_id"],
125
+ "release_manifest_sha256": file_sha256(source / "manifest.json"),
126
+ "trainer_sha256": file_sha256(source_trainer),
127
+ "sample_rows": len(source_rows),
128
+ "presentations": 2 * len(source_rows),
129
+ "errors": 0,
130
+ "expected_answers_sent_to_auditor": False,
131
+ "auditor_outputs_admitted_to_training": False,
132
+ "benchmark_rows_used": 0,
133
+ "result_rows_sha256": file_sha256(audit_path),
134
+ }
135
+ if any(summary.get(key) != value for key, value in required_summary.items()):
136
+ raise BlindAuditFilterError("consensus audit is partial, unsafe, or unbound")
137
+ request_contract = summary.get("request_contract") or {}
138
+ if (
139
+ request_contract.get("presentations_per_row") != 2
140
+ or request_contract.get("second_option_order_non_identity") is not True
141
+ or request_contract.get("temperature") != 0
142
+ ):
143
+ raise BlindAuditFilterError("consensus request contract diverged")
144
+
145
+ audit_rows = read_jsonl(audit_path, require_canonical=True)
146
+ selected, stable_rows = _validate_presentations(
147
+ source_rows=source_rows,
148
+ audit_rows=audit_rows,
149
+ allowed_confidence=allowed_confidence,
150
+ )
151
+ summary_stable = int(summary.get("high_confidence_option_consensus_rows", -1))
152
+ if allowed_confidence == {"high"} and stable_rows != summary_stable:
153
+ raise BlindAuditFilterError("auditor summary stable-row count diverged")
154
+ source_agreements = len(selected)
155
+ consensus_accuracy = source_agreements / stable_rows if stable_rows else 0.0
156
+ if consensus_accuracy < args.min_consensus_source_accuracy:
157
+ raise BlindAuditFilterError(
158
+ f"stable consensus source agreement {consensus_accuracy:.4f} is below "
159
+ f"{args.min_consensus_source_accuracy:.4f}"
160
+ )
161
+ eligible_rows = len(selected)
162
+ if args.max_rows is not None:
163
+ selected = _answer_balanced_subset(
164
+ selected, count=int(args.max_rows), seed=args.selection_seed
165
+ )
166
+ if len(selected) < args.min_rows:
167
+ raise BlindAuditFilterError(
168
+ f"only {len(selected)} option-invariant source agreements survived; "
169
+ f"requires {args.min_rows}"
170
+ )
171
+
172
+ source_inventory_path = source / "manifests/media_inventory.jsonl"
173
+ source_inventory = {
174
+ str(item["path"]): item
175
+ for item in read_jsonl(source_inventory_path, require_canonical=True)
176
+ }
177
+ referenced = _images(selected)
178
+ if not referenced <= set(source_inventory):
179
+ raise BlindAuditFilterError("selected media is absent from source inventory")
180
+
181
+ output.parent.mkdir(parents=True, exist_ok=True)
182
+ scratch = Path(tempfile.mkdtemp(prefix=f".{output.name}.partial.", dir=output.parent))
183
+ try:
184
+ write_jsonl(selected, scratch / "trainer/sft.jsonl")
185
+ selected_inventory = [source_inventory[path] for path in sorted(referenced)]
186
+ write_jsonl(selected_inventory, scratch / "manifests/media_inventory.jsonl")
187
+ for item in selected_inventory:
188
+ relative = Path(str(item["path"]))
189
+ destination = scratch / relative
190
+ _materialize(source / relative, destination, mode=args.copy_mode)
191
+ if (
192
+ destination.stat().st_size != int(item["bytes"])
193
+ or file_sha256(destination) != item["sha256"]
194
+ ):
195
+ raise BlindAuditFilterError("materialized media hash diverged")
196
+
197
+ answers = collections.Counter(_target(row) for row in selected)
198
+ frames = collections.Counter(
199
+ sum(
200
+ item.get("type") == "image"
201
+ for message in row["messages"]
202
+ for item in message["content"]
203
+ )
204
+ for row in selected
205
+ )
206
+ manifest = {
207
+ "artifacts": {
208
+ "manifests/media_inventory.jsonl": file_sha256(
209
+ scratch / "manifests/media_inventory.jsonl"
210
+ ),
211
+ "trainer/sft.jsonl": file_sha256(scratch / "trainer/sft.jsonl"),
212
+ },
213
+ "contracts": source_manifest["contracts"],
214
+ "counts": {
215
+ "media_bytes": sum(int(item["bytes"]) for item in selected_inventory),
216
+ "media_files": len(selected_inventory),
217
+ "trainer_rows": len(selected),
218
+ },
219
+ "distributions": {
220
+ "answer": dict(sorted(answers.items())),
221
+ "frames_per_row": {
222
+ str(key): value for key, value in sorted(frames.items())
223
+ },
224
+ },
225
+ "format": FORMAT,
226
+ "inputs": {
227
+ **source_manifest["inputs"],
228
+ "option_consensus_filter": {
229
+ "format": FILTER_FORMAT,
230
+ "source_release_id": source_manifest["release_id"],
231
+ "source_manifest_sha256": file_sha256(source / "manifest.json"),
232
+ "source_trainer_sha256": file_sha256(source_trainer),
233
+ "audit_sha256": file_sha256(audit_path),
234
+ "audit_summary_sha256": file_sha256(summary_path),
235
+ "auditor_model": str(summary.get("auditor_model") or ""),
236
+ "auditor_received_expected_answers": False,
237
+ "auditor_predictions_used_as_labels": False,
238
+ "presentations_per_row": 2,
239
+ "option_invariance_required": True,
240
+ "source_agreement_required": True,
241
+ "stable_consensus_rows": stable_rows,
242
+ "stable_consensus_source_agreements": source_agreements,
243
+ "stable_consensus_source_accuracy": consensus_accuracy,
244
+ "eligible_rows_after_filter": eligible_rows,
245
+ "allowed_confidence": sorted(allowed_confidence),
246
+ "max_rows": args.max_rows,
247
+ "subset_selection_seed": (
248
+ args.selection_seed if args.max_rows is not None else None
249
+ ),
250
+ "filter_builder_sha256": file_sha256(Path(__file__).resolve(strict=True)),
251
+ "shared_filter_helpers_sha256": file_sha256(
252
+ Path(__file__)
253
+ .with_name("filter_semantic_release_by_blind_audit.py")
254
+ .resolve(strict=True)
255
+ ),
256
+ },
257
+ },
258
+ "license": source_manifest["license"],
259
+ "model_lineage": source_manifest["model_lineage"],
260
+ "public_release_allowed": False,
261
+ "release_id": args.release_id,
262
+ "status": "sealed_internal_training_release",
263
+ "training_allowed": True,
264
+ "training_scope": "pure_vlm_semantic_sft_only",
265
+ }
266
+ (scratch / "manifest.json").write_bytes(canonical_json_line(manifest))
267
+ validate_semantic_release(scratch, verify_media_hashes=True)
268
+ os.replace(scratch, output)
269
+ return manifest
270
+ except Exception:
271
+ shutil.rmtree(scratch, ignore_errors=True)
272
+ raise
273
+
274
+
275
+ def main() -> None:
276
+ parser = argparse.ArgumentParser(description=__doc__)
277
+ parser.add_argument("--source", type=Path, required=True)
278
+ parser.add_argument("--audit", type=Path, required=True)
279
+ parser.add_argument("--audit-summary", type=Path, required=True)
280
+ parser.add_argument("--output", type=Path, required=True)
281
+ parser.add_argument("--release-id", required=True)
282
+ parser.add_argument(
283
+ "--confidence", action="append", choices=("high", "medium"), default=[]
284
+ )
285
+ parser.add_argument("--min-consensus-source-accuracy", type=float, default=0.80)
286
+ parser.add_argument("--min-rows", type=int, default=1)
287
+ parser.add_argument("--max-rows", type=int)
288
+ parser.add_argument("--selection-seed", default="option-consensus-filter-v0")
289
+ parser.add_argument(
290
+ "--copy-mode", choices=("copy", "hardlink", "auto"), default="hardlink"
291
+ )
292
+ args = parser.parse_args()
293
+ manifest = build(args)
294
+ print(json.dumps(manifest, ensure_ascii=False, indent=2, sort_keys=True))
295
+
296
+
297
+ if __name__ == "__main__":
298
+ main()
code/data_pipeline/temporal_scope/select_quality_capability_rows.py ADDED
@@ -0,0 +1,420 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """Freeze a source-disjoint holdout and a quality-ranked capability selection.
3
+
4
+ This selector never reads benchmark rows, labels, predictions, task IDs, or
5
+ question templates. It joins an already sealed non-benchmark teacher release
6
+ back to its generation evidence, removes nominal capability rows whose answer
7
+ was available too early, and reserves a policy holdout before selecting SFT
8
+ rows. A final training selection additionally requires a complete student
9
+ screen over the structural candidate set.
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ import argparse
15
+ import collections
16
+ import hashlib
17
+ import json
18
+ from pathlib import Path
19
+ from typing import Any, Iterable
20
+
21
+ from viss_core.data_v0.canonical import (
22
+ canonical_json_line,
23
+ file_sha256,
24
+ read_jsonl,
25
+ stable_id,
26
+ )
27
+ from viss_core.data_v0.semantic_release import validate_semantic_release
28
+
29
+
30
+ SELECTION_FORMAT = "iclr_video.quality_capability_selection.v1"
31
+ HOLDOUT_FORMAT = "iclr_video.nonbenchmark_policy_holdout.v1"
32
+ STUDENT_SCREEN_FORMAT = "iclr_video.nonbenchmark_student_screen.v1"
33
+ PROFILE_KIND = "nonbenchmark_capability_profile"
34
+ PARENT_KIND = "teacher_generated_nonbenchmark_visual_mcq"
35
+ PROFILES = (
36
+ "backward_epm",
37
+ "backward_asi",
38
+ "rec_counting",
39
+ "sequence_crr",
40
+ "sequence_ssr",
41
+ )
42
+ DEFAULT_HOLDOUT = {profile: 60 for profile in PROFILES}
43
+ DEFAULT_TRAIN = {"backward_epm": 120, "backward_asi": 180}
44
+
45
+
46
+ class QualityCapabilitySelectionError(ValueError):
47
+ pass
48
+
49
+
50
+ def _target(row: dict[str, Any]) -> str:
51
+ values = [
52
+ str(item.get("text") or "").strip()
53
+ for message in row.get("messages") or []
54
+ if message.get("role") == "assistant" and message.get("loss") is True
55
+ for item in message.get("content") or []
56
+ if item.get("type") == "text"
57
+ ]
58
+ if len(values) != 1 or values[0] not in set("ABCD"):
59
+ raise QualityCapabilitySelectionError("row lacks one A-D target")
60
+ return values[0]
61
+
62
+
63
+ def _single_parent(row: dict[str, Any], kind: str) -> str:
64
+ values = [
65
+ str(parent.get("id") or "")
66
+ for parent in row.get("parents") or []
67
+ if parent.get("kind") == kind
68
+ ]
69
+ if len(values) != 1 or not values[0]:
70
+ raise QualityCapabilitySelectionError(f"row lacks one {kind} parent")
71
+ return values[0]
72
+
73
+
74
+ def _rank(seed: str, profile: str, answer: str, uid: str) -> str:
75
+ return hashlib.sha256(
76
+ f"quality-capability-v1:{seed}:{profile}:{answer}:{uid}".encode()
77
+ ).hexdigest()
78
+
79
+
80
+ def _balanced_select(
81
+ rows: Iterable[dict[str, Any]],
82
+ *,
83
+ quota: int,
84
+ seed: str,
85
+ profile: str,
86
+ ) -> list[dict[str, Any]]:
87
+ if quota <= 0 or quota % 4:
88
+ raise QualityCapabilitySelectionError("balanced quota must be positive / 4")
89
+ by_answer: dict[str, list[dict[str, Any]]] = collections.defaultdict(list)
90
+ for row in rows:
91
+ by_answer[str(row["answer"])].append(row)
92
+ per_answer = quota // 4
93
+ selected = []
94
+ for answer in "ABCD":
95
+ values = sorted(
96
+ by_answer.get(answer, []),
97
+ key=lambda row: (_rank(seed, profile, answer, str(row["uid"])), str(row["uid"])),
98
+ )
99
+ if len(values) < per_answer:
100
+ raise QualityCapabilitySelectionError(
101
+ f"{profile}/{answer} has {len(values)} rows; requires {per_answer}"
102
+ )
103
+ selected.extend(values[:per_answer])
104
+ return sorted(selected, key=lambda row: str(row["uid"]))
105
+
106
+
107
+ def _image_paths(request: dict[str, Any]) -> list[str]:
108
+ return [
109
+ str(image)
110
+ for group in request.get("groups") or []
111
+ for image in group.get("images") or []
112
+ ]
113
+
114
+
115
+ def _structural_candidate(
116
+ request: dict[str, Any], generation: dict[str, Any]
117
+ ) -> tuple[bool, dict[str, Any]]:
118
+ profile = str(request.get("target_capability") or "")
119
+ groups = [
120
+ tuple(str(image) for image in group.get("images") or [])
121
+ for group in request.get("groups") or []
122
+ ]
123
+ flattened = [image for group in groups for image in group]
124
+ evidence_groups = generation.get("evidence_groups") or []
125
+ first_sufficient = generation.get("first_sufficient_group")
126
+ minimum_sufficient = {
127
+ "backward_epm": 1,
128
+ "backward_asi": 2,
129
+ "rec_counting": 1,
130
+ "sequence_crr": 1,
131
+ "sequence_ssr": 1,
132
+ }.get(profile)
133
+ diagnostics = {
134
+ "first_sufficient_group": first_sufficient,
135
+ "evidence_groups": len(evidence_groups),
136
+ "observation_groups": len(groups),
137
+ "distinct_observation_groups": len(set(groups)),
138
+ "image_positions": len(flattened),
139
+ "unique_images": len(set(flattened)),
140
+ "duplicate_image_positions": len(flattened) - len(set(flattened)),
141
+ }
142
+ eligible = (
143
+ minimum_sufficient is not None
144
+ and generation.get("decision") == "accept"
145
+ and generation.get("confidence") == "high"
146
+ and isinstance(first_sufficient, int)
147
+ and first_sufficient >= minimum_sufficient
148
+ and len(evidence_groups) >= 2
149
+ and len(set(groups)) >= 2
150
+ and len(set(flattened)) >= 6
151
+ )
152
+ return bool(eligible), diagnostics
153
+
154
+
155
+ def _student_keep(row: dict[str, Any], audit: dict[str, Any]) -> bool:
156
+ if audit.get("error") is not None:
157
+ return False
158
+ target = str(row["answer"])
159
+ if str(audit.get("target") or "") != target:
160
+ raise QualityCapabilitySelectionError("student audit target diverges")
161
+ prediction = str(audit.get("prediction") or "")
162
+ margin = float(audit.get("visible_option_margin"))
163
+ if not 0.0 <= margin <= 1.0:
164
+ raise QualityCapabilitySelectionError("student audit margin is outside [0,1]")
165
+ return prediction != target or margin <= 0.25
166
+
167
+
168
+ def _parse_quota(value: str) -> tuple[str, int]:
169
+ try:
170
+ profile, raw = value.rsplit("=", 1)
171
+ count = int(raw)
172
+ except (TypeError, ValueError) as exc:
173
+ raise argparse.ArgumentTypeError("quota must be PROFILE=COUNT") from exc
174
+ if profile not in PROFILES or count <= 0 or count % 4:
175
+ raise argparse.ArgumentTypeError("quota must use a known profile and be / 4")
176
+ return profile, count
177
+
178
+
179
+ def _excluded_parent_ids(releases: list[Path]) -> set[str]:
180
+ excluded: set[str] = set()
181
+ for raw in releases:
182
+ root = raw.resolve(strict=True)
183
+ validate_semantic_release(root, verify_media_hashes=False)
184
+ for row in read_jsonl(
185
+ root / "trainer/sft.jsonl",
186
+ verify_content_hashes=True,
187
+ require_canonical=True,
188
+ ):
189
+ excluded.update(
190
+ str(parent["id"])
191
+ for parent in row.get("parents") or []
192
+ if parent.get("kind") == PARENT_KIND and parent.get("id")
193
+ )
194
+ return excluded
195
+
196
+
197
+ def build(args: argparse.Namespace) -> tuple[dict[str, Any], dict[str, Any]]:
198
+ release = args.release.resolve(strict=True)
199
+ pack = args.pack.resolve(strict=True)
200
+ generation_path = args.generation.resolve(strict=True)
201
+ release_manifest = validate_semantic_release(release, verify_media_hashes=False)
202
+ pack_manifest_path = pack / "manifest.json"
203
+ pack_manifest = json.loads(pack_manifest_path.read_text(encoding="utf-8"))
204
+ if (
205
+ (release_manifest.get("inputs") or {}).get("pack_manifest_sha256")
206
+ != file_sha256(pack_manifest_path)
207
+ or (release_manifest.get("inputs") or {}).get("generation_sha256")
208
+ != file_sha256(generation_path)
209
+ ):
210
+ raise QualityCapabilitySelectionError("release is not bound to pack/generation")
211
+ requests = read_jsonl(pack / "requests.jsonl", require_canonical=True)
212
+ generation_rows = read_jsonl(generation_path, require_canonical=True)
213
+ trainer_rows = read_jsonl(
214
+ release / "trainer/sft.jsonl",
215
+ verify_content_hashes=True,
216
+ require_canonical=True,
217
+ )
218
+ request_by_uid = {str(row["uid"]): row for row in requests}
219
+ generation_by_uid = {str(row["uid"]): row for row in generation_rows}
220
+ trainer_by_uid = {str(row["uid"]): row for row in trainer_rows}
221
+ if len(request_by_uid) != len(requests) or len(generation_by_uid) != len(generation_rows):
222
+ raise QualityCapabilitySelectionError("request/generation UIDs collide")
223
+
224
+ excluded = _excluded_parent_ids(list(args.exclude_release or []))
225
+ joined = []
226
+ for request_uid, request in request_by_uid.items():
227
+ trainer_uid = stable_id("benchmarkdistillsft", request_uid, digest_length=24)
228
+ trainer = trainer_by_uid.get(trainer_uid)
229
+ generation = generation_by_uid.get(request_uid)
230
+ if trainer is None:
231
+ continue
232
+ if generation is None:
233
+ raise QualityCapabilitySelectionError("filtered row lacks generation record")
234
+ parent_id = _single_parent(trainer, PARENT_KIND)
235
+ profile = _single_parent(trainer, PROFILE_KIND)
236
+ if profile != str(request.get("target_capability") or ""):
237
+ raise QualityCapabilitySelectionError("request/trainer profile diverges")
238
+ if parent_id in excluded:
239
+ continue
240
+ joined.append(
241
+ {
242
+ "uid": trainer_uid,
243
+ "request_uid": request_uid,
244
+ "parent_id": parent_id,
245
+ "profile": profile,
246
+ "answer": _target(trainer),
247
+ "source_video_basename": str(request["source_video_basename"]),
248
+ "request": request,
249
+ "generation": generation,
250
+ }
251
+ )
252
+
253
+ holdout_quotas = dict(args.holdout_quota or DEFAULT_HOLDOUT)
254
+ train_quotas = dict(args.train_quota or DEFAULT_TRAIN)
255
+ holdout_rows = []
256
+ for profile, quota in sorted(holdout_quotas.items()):
257
+ holdout_rows.extend(
258
+ _balanced_select(
259
+ (row for row in joined if row["profile"] == profile),
260
+ quota=quota,
261
+ seed=args.seed + ":holdout",
262
+ profile=profile,
263
+ )
264
+ )
265
+ holdout_uids = {str(row["uid"]) for row in holdout_rows}
266
+ holdout_videos = {str(row["source_video_basename"]) for row in holdout_rows}
267
+ if len(holdout_uids) != len(holdout_rows) or len(holdout_videos) != len(holdout_rows):
268
+ raise QualityCapabilitySelectionError("policy holdout is not source-video unique")
269
+
270
+ structural = []
271
+ for row in joined:
272
+ if row["uid"] in holdout_uids:
273
+ continue
274
+ eligible, diagnostics = _structural_candidate(
275
+ row["request"], row["generation"]
276
+ )
277
+ if eligible:
278
+ structural.append({**row, "diagnostics": diagnostics})
279
+
280
+ audit_by_uid: dict[str, dict[str, Any]] | None = None
281
+ audit_record = None
282
+ if args.student_audit is not None:
283
+ audit_path = args.student_audit.resolve(strict=True)
284
+ audit_value = json.loads(audit_path.read_text(encoding="utf-8"))
285
+ if (
286
+ audit_value.get("format") != STUDENT_SCREEN_FORMAT
287
+ or audit_value.get("benchmark_rows_or_labels_used") is not False
288
+ or audit_value.get("source_release_manifest_sha256")
289
+ != file_sha256(release / "manifest.json")
290
+ ):
291
+ raise QualityCapabilitySelectionError("student screen contract diverges")
292
+ audit_rows = list(audit_value.get("rows") or [])
293
+ audit_by_uid = {str(row.get("uid") or ""): row for row in audit_rows}
294
+ if set(audit_by_uid) != {str(row["uid"]) for row in structural}:
295
+ raise QualityCapabilitySelectionError("student screen coverage diverges")
296
+ audit_record = {"path": str(audit_path), "sha256": file_sha256(audit_path)}
297
+
298
+ selected_rows = []
299
+ if audit_by_uid is not None:
300
+ hard = [row for row in structural if _student_keep(row, audit_by_uid[row["uid"]])]
301
+ for profile, quota in sorted(train_quotas.items()):
302
+ selected_rows.extend(
303
+ _balanced_select(
304
+ (row for row in hard if row["profile"] == profile),
305
+ quota=quota,
306
+ seed=args.seed + ":train",
307
+ profile=profile,
308
+ )
309
+ )
310
+
311
+ def public_row(row: dict[str, Any]) -> dict[str, Any]:
312
+ return {
313
+ key: row[key]
314
+ for key in (
315
+ "uid",
316
+ "request_uid",
317
+ "parent_id",
318
+ "profile",
319
+ "answer",
320
+ "source_video_basename",
321
+ )
322
+ } | ({"diagnostics": row["diagnostics"]} if "diagnostics" in row else {})
323
+
324
+ holdout = {
325
+ "format": HOLDOUT_FORMAT,
326
+ "status": "sealed_before_training_selection",
327
+ "source_release": str(release),
328
+ "source_release_manifest_sha256": file_sha256(release / "manifest.json"),
329
+ "selection_seed": args.seed + ":holdout",
330
+ "quotas": holdout_quotas,
331
+ "rows": [public_row(row) for row in sorted(holdout_rows, key=lambda row: row["uid"])],
332
+ "parent_ids": sorted(row["parent_id"] for row in holdout_rows),
333
+ "contracts": {
334
+ "source_video_disjoint_from_sft_selection": True,
335
+ "may_enter_sft_or_rl": False,
336
+ "benchmark_rows_or_labels_used": False,
337
+ },
338
+ }
339
+ selection = {
340
+ "format": SELECTION_FORMAT,
341
+ "status": (
342
+ "sealed_nonbenchmark_quality_selection"
343
+ if audit_by_uid is not None
344
+ else "awaiting_complete_student_screen"
345
+ ),
346
+ "training_allowed": audit_by_uid is not None,
347
+ "source_release": str(release),
348
+ "source_release_manifest_sha256": file_sha256(release / "manifest.json"),
349
+ "pack_manifest_sha256": file_sha256(pack_manifest_path),
350
+ "generation_sha256": file_sha256(generation_path),
351
+ "excluded_parent_ids": len(excluded),
352
+ "holdout_parent_ids_sha256": hashlib.sha256(
353
+ canonical_json_line(sorted(holdout["parent_ids"]))
354
+ ).hexdigest(),
355
+ "structural_candidates": [
356
+ public_row(row) for row in sorted(structural, key=lambda row: row["uid"])
357
+ ],
358
+ "structural_counts_by_profile": dict(
359
+ sorted(collections.Counter(row["profile"] for row in structural).items())
360
+ ),
361
+ "student_screen": audit_record,
362
+ "train_quotas": train_quotas,
363
+ "selected_rows": [
364
+ public_row(row) for row in sorted(selected_rows, key=lambda row: row["uid"])
365
+ ],
366
+ "parent_ids": sorted(row["parent_id"] for row in selected_rows),
367
+ "transformation": "deduplicate_identical_images_keep_first_chronological_occurrence",
368
+ "contracts": {
369
+ "benchmark_rows_labels_predictions_or_tasks_used": False,
370
+ "counterfactual_rows": 0,
371
+ "holdout_overlap": 0,
372
+ "already_trained_parent_overlap": 0,
373
+ "teacher_confidence_high": True,
374
+ "two_order_option_consensus_inherited": True,
375
+ },
376
+ }
377
+ return holdout, selection
378
+
379
+
380
+ def main() -> None:
381
+ parser = argparse.ArgumentParser(description=__doc__)
382
+ parser.add_argument("--release", type=Path, required=True)
383
+ parser.add_argument("--pack", type=Path, required=True)
384
+ parser.add_argument("--generation", type=Path, required=True)
385
+ parser.add_argument("--exclude-release", type=Path, action="append", default=[])
386
+ parser.add_argument("--student-audit", type=Path)
387
+ parser.add_argument("--holdout-quota", type=_parse_quota, action="append")
388
+ parser.add_argument("--train-quota", type=_parse_quota, action="append")
389
+ parser.add_argument("--seed", default="quality-capability-v1")
390
+ parser.add_argument("--holdout-output", type=Path, required=True)
391
+ parser.add_argument("--selection-output", type=Path, required=True)
392
+ args = parser.parse_args()
393
+ if args.holdout_quota:
394
+ args.holdout_quota = dict(args.holdout_quota)
395
+ if args.train_quota:
396
+ args.train_quota = dict(args.train_quota)
397
+ for path in (args.holdout_output, args.selection_output):
398
+ if path.exists():
399
+ raise FileExistsError(path)
400
+ holdout, selection = build(args)
401
+ args.holdout_output.parent.mkdir(parents=True, exist_ok=True)
402
+ args.selection_output.parent.mkdir(parents=True, exist_ok=True)
403
+ args.holdout_output.write_bytes(canonical_json_line(holdout))
404
+ args.selection_output.write_bytes(canonical_json_line(selection))
405
+ print(
406
+ json.dumps(
407
+ {
408
+ "holdout_rows": len(holdout["rows"]),
409
+ "structural_counts_by_profile": selection["structural_counts_by_profile"],
410
+ "selected_rows": len(selection["selected_rows"]),
411
+ "training_allowed": selection["training_allowed"],
412
+ },
413
+ indent=2,
414
+ sort_keys=True,
415
+ )
416
+ )
417
+
418
+
419
+ if __name__ == "__main__":
420
+ main()
code/data_pipeline/temporal_scope/semantic_mcq.py ADDED
@@ -0,0 +1,90 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Shared pure-VLM prompt and deterministic MCQ utilities."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import hashlib
6
+ import re
7
+ from collections.abc import Sequence
8
+
9
+
10
+ DIRECT_PROMPT_PREFIX = (
11
+ "You are an advanced video question-answering AI assistant. You have been "
12
+ "provided with causal frames from the video and a multiple-choice question. "
13
+ "Analyze only the visible frames and provide the best answer."
14
+ )
15
+ DIRECT_PROMPT_SUFFIX = "Only give the best option's letter (A, B, C, or D) directly."
16
+ OPTION_LETTERS = "ABCD"
17
+ DIRECT_PROMPT_RE = re.compile(
18
+ r"(?:^|\n)Question:\s*(?P<question>.*?)\n+\s*Options:\s*\n"
19
+ r"(?P<options>.*?)\n+\s*Only give the best option's letter",
20
+ re.DOTALL,
21
+ )
22
+ OPTION_LINE_RE = re.compile(r"^\s*([A-D])[.):]\s*(.+?)\s*$")
23
+
24
+
25
+ class SemanticMCQError(ValueError):
26
+ """Raised when a deterministic semantic MCQ cannot be constructed."""
27
+
28
+
29
+ def parse_direct_prompt(value: str) -> tuple[str, list[str]]:
30
+ """Recover one canonical public question and its four visible options."""
31
+
32
+ match = DIRECT_PROMPT_RE.search(str(value))
33
+ if match is None:
34
+ raise SemanticMCQError("canonical direct prompt is malformed")
35
+ question = " ".join(match.group("question").split())
36
+ parsed: dict[str, str] = {}
37
+ for line in match.group("options").splitlines():
38
+ option = OPTION_LINE_RE.fullmatch(line)
39
+ if option is None:
40
+ if line.strip():
41
+ raise SemanticMCQError("canonical option line is malformed")
42
+ continue
43
+ letter, text = option.groups()
44
+ if letter in parsed:
45
+ raise SemanticMCQError("canonical option letter is duplicated")
46
+ parsed[letter] = " ".join(text.split())
47
+ if not question or list(parsed) != list(OPTION_LETTERS):
48
+ raise SemanticMCQError("canonical prompt lacks one question and A--D options")
49
+ options = [parsed[letter] for letter in OPTION_LETTERS]
50
+ if any(not option for option in options) or len(set(options)) != len(options):
51
+ raise SemanticMCQError("canonical prompt options are empty or duplicated")
52
+ return question, options
53
+
54
+
55
+ def rebalance_answer_slot(
56
+ options: Sequence[str],
57
+ *,
58
+ answer: str,
59
+ answer_slot: int,
60
+ seed: str,
61
+ ) -> tuple[list[str], str]:
62
+ """Move the target to a fixed slot and hash-order the three distractors."""
63
+
64
+ if (
65
+ len(options) != len(OPTION_LETTERS)
66
+ or answer not in OPTION_LETTERS
67
+ or answer_slot not in range(len(OPTION_LETTERS))
68
+ or len(set(options)) != len(OPTION_LETTERS)
69
+ ):
70
+ raise SemanticMCQError("invalid four-option answer-space contract")
71
+ target_index = OPTION_LETTERS.index(answer)
72
+ target = str(options[target_index])
73
+ distractors = [
74
+ (index, str(option))
75
+ for index, option in enumerate(options)
76
+ if index != target_index
77
+ ]
78
+ distractors.sort(
79
+ key=lambda item: hashlib.sha256(
80
+ f"{seed}|{item[0]}|{item[1]}".encode("utf-8")
81
+ ).digest()
82
+ )
83
+ balanced: list[str | None] = [None] * len(OPTION_LETTERS)
84
+ balanced[answer_slot] = target
85
+ remaining = [index for index, value in enumerate(balanced) if value is None]
86
+ for slot, (_, option) in zip(remaining, distractors, strict=True):
87
+ balanced[slot] = option
88
+ if any(value is None for value in balanced):
89
+ raise SemanticMCQError("balanced options are incomplete")
90
+ return [str(value) for value in balanced], OPTION_LETTERS[answer_slot]
code/docs/DATA.md ADDED
@@ -0,0 +1,63 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Data preparation
2
+
3
+ The code package distributes construction and validation code. Training
4
+ records, source media, benchmark exports, and calibration artifacts are
5
+ external inputs.
6
+
7
+ ## Training records
8
+
9
+ The trainer consumes a sealed semantic release. Its validation checks the
10
+ manifest, source inventory, user/assistant message schema, media identity,
11
+ and permitted targets. See:
12
+
13
+ - `viss_core/data_v0/schema.py`
14
+ - `viss_core/data_v0/semantic_release.py`
15
+ - `viss_core/data_v0/trainer_adapter.py`
16
+
17
+ The source-aligned builder combines already prepared source releases:
18
+
19
+ ```bash
20
+ python -m data_pipeline.temporal_scope.build_source_aligned_sft_release \
21
+ --source data/prepared_source_a \
22
+ --source data/prepared_source_b \
23
+ --output data/viss_release \
24
+ --release-id my_viss_release \
25
+ --copy-mode copy
26
+ ```
27
+
28
+ `prepared_source_a` and `prepared_source_b` are placeholders for valid
29
+ source releases, not raw-video directories. This generic example does not
30
+ reconstruct the paper's accepted training mixture by itself.
31
+
32
+ The paper describes the training mixture. Company training records and their
33
+ manifests are not distributed. The training guide assumes users have prepared
34
+ their own valid source releases, media, example weights, and ordering.
35
+
36
+ Set `VISS_UPSTREAM_MANIFEST_SHA256` to the trusted upstream identity for your
37
+ training inputs before starting the builder or trainer. The validator retains
38
+ the identity check, but the code package embeds no company-specific digest.
39
+ Private legacy source adapters are excluded; supply the standard semantic
40
+ release format instead.
41
+
42
+ ## Evaluation assets
43
+
44
+ For each benchmark, prepare:
45
+
46
+ 1. Official annotations and authorized video files.
47
+ 2. A query export using the fields read by `viss_core/benchmarks.py`.
48
+ 3. The benchmark's official scoring installation.
49
+ 4. Model and processor files compatible with the locked evaluation protocol.
50
+
51
+ `configs/evaluation/recent4.example.json` describes the public sampling,
52
+ query-count, prompt, and processor conventions. It contains no run dates,
53
+ company file identities, or query records. Follow the evaluation guide to
54
+ create a local configuration bound to your own query exports and processor.
55
+ The generated configuration is excluded from publication.
56
+
57
+ ## TVS calibration
58
+
59
+ The TVS policy consumes acquisition/commit calibration artifacts and aligned
60
+ view predictions from one model. Calibrate on source-disjoint data, then
61
+ freeze the thresholds for benchmark evaluation. The archive includes the
62
+ selection and analysis implementation, but not the paper's calibration
63
+ records or precomputed predictions.
code/docs/EVALUATION.md ADDED
@@ -0,0 +1,114 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Inference, TVS, and scoring
2
+
3
+ Run from the repository root after [installation](INSTALL.md) and
4
+ [data preparation](DATA.md). The code separates model inference, view-policy
5
+ composition, and official scoring.
6
+
7
+ ## 1. Generate recent-view proposals
8
+
9
+ Create a local configuration from the public template and your own assets:
10
+
11
+ ```bash
12
+ python -m scripts.configure_evaluation_protocol \
13
+ --streaming-samples data/streamingbench/queries.jsonl \
14
+ --ovo-samples data/ovo/queries.jsonl \
15
+ --processor-config checkpoints/viss/locked_processor/preprocessor_config.json \
16
+ --output configs/evaluation/recent4_full_v1.json
17
+ ```
18
+
19
+ The generated file contains local paths and file identities. It remains
20
+ private and is excluded from the source archive. Then validate coverage:
21
+
22
+ ```bash
23
+ python -m scripts.validate_evaluation_protocol \
24
+ --protocol configs/evaluation/recent4_full_v1.json
25
+ ```
26
+
27
+ This check requires the external benchmark exports. It is not a data-free
28
+ installation test.
29
+
30
+ Use the command in the main README with the prepared checkpoint and locked
31
+ processor. The runner accepts `--benchmark streamingbench` or
32
+ `--benchmark ovo`; annotations, query exports, and media must match.
33
+
34
+ Use `--record-choice-confidence` when the proposals will be passed to TVS.
35
+ For a reproducibility run, supply `--model-manifest` with the matching model
36
+ identity artifact. The same identity must be retained across all views.
37
+ The runner also supports `--start-index` and `--end-index` for sharding.
38
+
39
+ The optional multi-GPU launcher is:
40
+
41
+ ```text
42
+ bash scripts/run_recent4_single_benchmark.sh \
43
+ REPO MODEL MODEL_MANIFEST OUTPUT PYTHON BENCHMARK GPU_CSV MODEL_ID
44
+ ```
45
+
46
+ It additionally requires the benchmark paths documented by its environment
47
+ variable checks. Start with the Python entry point when preparing new assets.
48
+
49
+ ## 2. Apply the TVS acquisition and commitment policy
50
+
51
+ The event policy is `causalbudget20`; the compact-history policy is
52
+ `causalhybrid32`. Use the same checkpoint and the view encodings prescribed
53
+ by your protocol. Selecting a policy name alone does not fix image resolution
54
+ or the processor configuration.
55
+
56
+ Given recent-view predictions and a frozen calibration artifact:
57
+
58
+ ```bash
59
+ python -m experiments.select_uncertainty_escalation_uids \
60
+ --recent outputs/recent.json \
61
+ --threshold data/calibration/thresholds.json \
62
+ --samples data/benchmark/queries.jsonl \
63
+ --output-uids outputs/event_uids.json \
64
+ --output-samples outputs/event_queries.jsonl \
65
+ --output-audit outputs/selection.json
66
+ ```
67
+
68
+ Run event-view inference on the selected queries. Then compose the first
69
+ two stages:
70
+
71
+ ```bash
72
+ python -m experiments.materialize_uncertainty_temporal_agent \
73
+ --recent outputs/recent.json \
74
+ --temporal outputs/event.json \
75
+ --selection-audit outputs/selection.json \
76
+ --threshold-audit data/calibration/thresholds.json \
77
+ --output outputs/two_view.json
78
+ ```
79
+
80
+ Once the required compact-history proposals are available:
81
+
82
+ ```bash
83
+ python -m experiments.materialize_confidence_gated_third_view_agent \
84
+ --recent outputs/recent.json \
85
+ --event outputs/event.json \
86
+ --base-agent outputs/two_view.json \
87
+ --third-view outputs/history.json \
88
+ --threshold-audit data/calibration/thresholds.json \
89
+ --output outputs/tvs.json
90
+ ```
91
+
92
+ These commands name user-generated files; no prediction or score artifacts
93
+ are shipped. Materialization combines existing proposals and does not call
94
+ the VLM. `materialize_tvs_view_oracle.py` uses reference correctness for
95
+ analysis and is not the deployed selection policy.
96
+
97
+ ## 3. Score saved predictions
98
+
99
+ ```bash
100
+ python -m scripts.score_official_benchmark_predictions \
101
+ --benchmark streamingbench \
102
+ --annotation data/streamingbench/annotations.json \
103
+ --predictions outputs/tvs.json \
104
+ --official-root external/StreamingBench \
105
+ --output-json outputs/streamingbench_score.json
106
+ ```
107
+
108
+ Use `--benchmark ovo` and the corresponding official assets for OVO-Bench.
109
+ The adapter checks query coverage and invokes the original scorer. For OVO,
110
+ the primary metric is the task-then-group macro average. The official scorer
111
+ may have additional dependencies; install those from its own repository.
112
+
113
+ Generated predictions and scores are your run outputs. They are excluded
114
+ from version control and from the source-release archive.
code/docs/INSTALL.md ADDED
@@ -0,0 +1,59 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Installation
2
+
3
+ Use Python 3.10 or later. Run commands from the source root; editable
4
+ installation keeps the configuration and script paths available.
5
+
6
+ ## CPU source checks
7
+
8
+ ```bash
9
+ python -m venv .venv
10
+ source .venv/bin/activate
11
+ python -m pip install --upgrade pip
12
+ python -m pip install -e ".[test]"
13
+ python -m pytest -q tests
14
+ ```
15
+
16
+ These tests exercise code contracts with synthetic inputs. They do not
17
+ validate a trained checkpoint or execute a full benchmark.
18
+
19
+ ## GPU inference
20
+
21
+ Install a CUDA-enabled PyTorch build appropriate for the machine first,
22
+ then install the inference extras:
23
+
24
+ ```bash
25
+ python -m pip install -e ".[inference]"
26
+ ```
27
+
28
+ The inference extras pin PyTorch 2.7.0 and Transformers 4.57.3.
29
+ The PyTorch version follows the repository runtime requirements; the
30
+ Transformers version follows the retained ViSS model configuration.
31
+ The extras declare Accelerate, qwen-vl-utils, and Decord as well.
32
+ They specify an installation recipe, not a claim that every GPU/driver
33
+ combination has been validated.
34
+
35
+ Install FFmpeg and ensure `ffprobe` is on `PATH`. Multi-GPU shell launchers
36
+ also use Bash 4 or newer, `jq`, `sha256sum`, and `nvidia-smi`.
37
+
38
+ ## Training and attention
39
+
40
+ ```bash
41
+ python -m pip install -e ".[inference,training]"
42
+ python -m pip install flash-attn==2.7.4.post1 --no-build-isolation
43
+ ```
44
+
45
+ FlashAttention installation needs a compatible CUDA toolkit and compiler.
46
+ The trainer defaults to `flash_attention_2`. Use `--attn-implementation sdpa`
47
+ for a development run when FlashAttention is unavailable; this changes the
48
+ runtime configuration.
49
+
50
+ The package includes no model weights. Before a GPU run, confirm that the
51
+ selected checkpoint loads with `AutoModelForImageTextToText` and that the
52
+ processor matches the intended evaluation contract.
53
+
54
+ ## Environment records
55
+
56
+ Save the installed package versions and launch command alongside your own
57
+ runs. Generated outputs belong under `outputs/` or `checkpoints/` and are
58
+ excluded from the source archive. Do not commit credentials, benchmark media,
59
+ or local machine paths.
code/docs/TRAINING.md ADDED
@@ -0,0 +1,50 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Training ViSS
2
+
3
+ Install the training environment described in [Installation](INSTALL.md) and
4
+ prepare a sealed training release as described in [Data preparation](DATA.md).
5
+
6
+ ## Paper recipe
7
+
8
+ The manuscript specifies one epoch, learning rate `2e-6`, cosine scheduling,
9
+ warmup ratio `0.03`, seed `59`, maximum length `8192`, maximum image pixels
10
+ `200704`, and a global batch of `12`. The vision tower is frozen; the language
11
+ model and multimodal projector are trainable.
12
+
13
+ For two GPUs, a microbatch of one and six accumulation steps give the stated
14
+ global batch:
15
+
16
+ ```bash
17
+ torchrun --nproc_per_node=2 -m experiments.train_qwen3vl_sft \
18
+ --model checkpoints/Qwen3-VL-8B-Instruct \
19
+ --train data/viss_release/train.jsonl \
20
+ --output checkpoints/viss \
21
+ --epochs 1 \
22
+ --lr 2e-6 \
23
+ --lr-scheduler-type cosine \
24
+ --warmup-ratio 0.03 \
25
+ --per-device-batch-size 1 \
26
+ --gradient-accumulation-steps 6 \
27
+ --max-length 8192 \
28
+ --max-pixels 200704 \
29
+ --seed 59 \
30
+ --train-order sequential \
31
+ --freeze-vision-tower \
32
+ --train-vision-projector \
33
+ --attn-implementation flash_attention_2 \
34
+ --deepspeed configs/training/deepspeed_zero3_bf16.json
35
+ ```
36
+
37
+ Replace `train.jsonl` with the actual row file inside the sealed release.
38
+ The sequential setting preserves prepared row order. The paper preparation
39
+ uses a parent stage followed by capability rows, with stage shuffle seed 59;
40
+ that ordering must already be reflected in the input release.
41
+
42
+ Changing the GPU count requires adjusting accumulation if the global batch
43
+ is to remain fixed. Check the trainer's complete options with:
44
+
45
+ ```bash
46
+ python -m experiments.train_qwen3vl_sft --help
47
+ ```
48
+
49
+ The training entry point requires Transformers even for `--help`. Training
50
+ outputs and logs are generated at runtime and are not part of this archive.
code/experiments/analyze_uncertainty_temporal_escalation.py ADDED
@@ -0,0 +1,451 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """Audit a task-agnostic recent-to-temporal escalation policy.
3
+
4
+ The online policy sees only the recent-view first-token generation margin. Gold labels are
5
+ used offline to fit one global threshold on a deterministic calibration split
6
+ and to score the disjoint exploratory split. Task, group, benchmark identity,
7
+ and answer-space semantics are never policy inputs.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ import argparse
13
+ import hashlib
14
+ import json
15
+ from collections import defaultdict
16
+ from pathlib import Path
17
+ from typing import Any
18
+
19
+
20
+ def parse_args() -> argparse.Namespace:
21
+ parser = argparse.ArgumentParser(description=__doc__)
22
+ parser.add_argument("--recent", type=Path, required=True)
23
+ parser.add_argument("--temporal", type=Path, required=True)
24
+ parser.add_argument("--output", type=Path, required=True)
25
+ parser.add_argument(
26
+ "--calibration-modulus",
27
+ type=int,
28
+ default=2,
29
+ help="SHA256(uid) modulo this value; bucket zero fits the threshold.",
30
+ )
31
+ return parser.parse_args()
32
+
33
+
34
+ def load_result(path: Path) -> dict[str, Any]:
35
+ value = json.loads(path.read_text(encoding="utf-8"))
36
+ if not isinstance(value, dict) or not isinstance(value.get("rows"), list):
37
+ raise ValueError(f"{path}: expected benchmark result with rows")
38
+ if int(value.get("num_errors", 0)) != 0:
39
+ raise ValueError(f"{path}: result contains inference errors")
40
+ return value
41
+
42
+
43
+ def index_rows(result: dict[str, Any]) -> dict[str, dict[str, Any]]:
44
+ indexed = {str(row.get("uid")): row for row in result["rows"]}
45
+ if "" in indexed or len(indexed) != len(result["rows"]):
46
+ raise ValueError("result contains missing or duplicate UIDs")
47
+ return indexed
48
+
49
+
50
+ def index_scores(result: dict[str, Any]) -> dict[str, dict[str, Any]]:
51
+ rows = (result.get("score") or {}).get("rows")
52
+ if not isinstance(rows, list):
53
+ raise ValueError("result is missing score.rows")
54
+ indexed = {str(row.get("query_id")): row for row in rows}
55
+ if "" in indexed or len(indexed) != len(rows):
56
+ raise ValueError("score rows contain missing or duplicate query IDs")
57
+ return indexed
58
+
59
+
60
+ def index_temporal_scores(result: dict[str, Any]) -> dict[str, dict[str, Any]]:
61
+ score_rows = (result.get("score") or {}).get("rows")
62
+ if isinstance(score_rows, list):
63
+ return index_scores(result)
64
+ rows = result.get("rows")
65
+ if not isinstance(rows, list) or any("correct" not in row for row in rows):
66
+ raise ValueError("temporal result lacks official or inherited correctness")
67
+ indexed = {str(row.get("uid")): row for row in rows}
68
+ if "" in indexed or len(indexed) != len(rows):
69
+ raise ValueError("temporal score rows contain missing or duplicate UIDs")
70
+ return indexed
71
+
72
+
73
+ def calibration_bucket(uid: str, modulus: int) -> int:
74
+ if modulus < 2:
75
+ raise ValueError("calibration modulus must be at least two")
76
+ return int(hashlib.sha256(uid.encode("utf-8")).hexdigest(), 16) % modulus
77
+
78
+
79
+ def choice_margin(row: dict[str, Any]) -> float:
80
+ # New confirmatory runs use the same full-vocabulary first-token margin for
81
+ # MCQ and open-ended questions. Older exploratory artifacts contain only
82
+ # the option-letter diagnostic and remain readable for historical audits.
83
+ confidence = row.get("generation_confidence") or row.get("choice_confidence")
84
+ if not isinstance(confidence, dict):
85
+ raise ValueError(f"{row.get('uid')}: missing generation confidence")
86
+ if confidence.get("task_or_group_used") is not False:
87
+ raise ValueError(f"{row.get('uid')}: confidence provenance is not task blind")
88
+ value = float(confidence.get("top2_margin"))
89
+ if not 0.0 <= value <= 1.0:
90
+ raise ValueError(f"{row.get('uid')}: invalid generation margin")
91
+ return value
92
+
93
+
94
+ def select_prediction(
95
+ recent_row: dict[str, Any], temporal_row: dict[str, Any], threshold: float
96
+ ) -> tuple[str, bool]:
97
+ escalate = choice_margin(recent_row) <= threshold
98
+ source = temporal_row if escalate else recent_row
99
+ return str(
100
+ source.get("prediction") or source.get("selected_prediction") or ""
101
+ ), escalate
102
+
103
+
104
+ def confidence_gain(
105
+ recent_row: dict[str, Any], temporal_row: dict[str, Any]
106
+ ) -> float:
107
+ """Return the task-blind confidence change after acquiring the temporal view."""
108
+
109
+ return choice_margin(temporal_row) - choice_margin(recent_row)
110
+
111
+
112
+ def select_acquire_commit_prediction(
113
+ recent_row: dict[str, Any],
114
+ temporal_row: dict[str, Any],
115
+ acquire_threshold: float,
116
+ commit_threshold: float,
117
+ ) -> tuple[str, bool, bool, float]:
118
+ """Apply one shared acquire threshold and one shared safe-commit threshold."""
119
+
120
+ acquired = choice_margin(recent_row) <= acquire_threshold
121
+ recent_prediction = str(
122
+ recent_row.get("prediction") or recent_row.get("selected_prediction") or ""
123
+ )
124
+ temporal_prediction = str(
125
+ temporal_row.get("prediction")
126
+ or temporal_row.get("selected_prediction")
127
+ or ""
128
+ )
129
+ gain = confidence_gain(recent_row, temporal_row)
130
+ committed = bool(
131
+ acquired
132
+ and temporal_prediction != recent_prediction
133
+ and gain > commit_threshold
134
+ )
135
+ return (
136
+ temporal_prediction if committed else recent_prediction,
137
+ acquired,
138
+ committed,
139
+ gain,
140
+ )
141
+
142
+
143
+ def evaluate_threshold(
144
+ uids: list[str],
145
+ *,
146
+ recent: dict[str, dict[str, Any]],
147
+ temporal: dict[str, dict[str, Any]],
148
+ recent_scores: dict[str, dict[str, Any]],
149
+ temporal_scores: dict[str, dict[str, Any]],
150
+ threshold: float,
151
+ ) -> tuple[int, int]:
152
+ correct = 0
153
+ escalated = 0
154
+ for uid in uids:
155
+ _, did_escalate = select_prediction(
156
+ recent[uid], temporal[uid], threshold
157
+ )
158
+ official = temporal_scores[uid] if did_escalate else recent_scores[uid]
159
+ correct += int(bool(official["correct"]))
160
+ escalated += int(did_escalate)
161
+ return correct, escalated
162
+
163
+
164
+ def fit_threshold(
165
+ calibration_uids: list[str],
166
+ *,
167
+ recent: dict[str, dict[str, Any]],
168
+ temporal: dict[str, dict[str, Any]],
169
+ recent_scores: dict[str, dict[str, Any]],
170
+ temporal_scores: dict[str, dict[str, Any]],
171
+ ) -> tuple[float, list[dict[str, Any]]]:
172
+ margins = sorted({choice_margin(recent[uid]) for uid in calibration_uids})
173
+ candidates = [-1.0, *margins, 1.0]
174
+ sweep = []
175
+ for threshold in candidates:
176
+ correct, escalated = evaluate_threshold(
177
+ calibration_uids,
178
+ recent=recent,
179
+ temporal=temporal,
180
+ recent_scores=recent_scores,
181
+ temporal_scores=temporal_scores,
182
+ threshold=threshold,
183
+ )
184
+ sweep.append(
185
+ {
186
+ "threshold": threshold,
187
+ "correct": correct,
188
+ "total": len(calibration_uids),
189
+ "accuracy": correct / len(calibration_uids),
190
+ "escalated": escalated,
191
+ }
192
+ )
193
+ best = max(
194
+ sweep,
195
+ key=lambda row: (
196
+ int(row["correct"]),
197
+ -int(row["escalated"]),
198
+ -float(row["threshold"]),
199
+ ),
200
+ )
201
+ return float(best["threshold"]), sweep
202
+
203
+
204
+ def evaluate_acquire_commit_thresholds(
205
+ uids: list[str],
206
+ *,
207
+ recent: dict[str, dict[str, Any]],
208
+ temporal: dict[str, dict[str, Any]],
209
+ recent_scores: dict[str, dict[str, Any]],
210
+ temporal_scores: dict[str, dict[str, Any]],
211
+ acquire_threshold: float,
212
+ commit_threshold: float,
213
+ ) -> tuple[int, int, int]:
214
+ correct = 0
215
+ acquired = 0
216
+ committed = 0
217
+ for uid in uids:
218
+ _, did_acquire, did_commit, _ = select_acquire_commit_prediction(
219
+ recent[uid],
220
+ temporal[uid],
221
+ acquire_threshold,
222
+ commit_threshold,
223
+ )
224
+ official = temporal_scores[uid] if did_commit else recent_scores[uid]
225
+ correct += int(bool(official["correct"]))
226
+ acquired += int(did_acquire)
227
+ committed += int(did_commit)
228
+ return correct, acquired, committed
229
+
230
+
231
+ def fit_acquire_commit_thresholds(
232
+ calibration_uids: list[str],
233
+ *,
234
+ recent: dict[str, dict[str, Any]],
235
+ temporal: dict[str, dict[str, Any]],
236
+ recent_scores: dict[str, dict[str, Any]],
237
+ temporal_scores: dict[str, dict[str, Any]],
238
+ ) -> tuple[float, float, list[dict[str, Any]]]:
239
+ acquire_candidates = [
240
+ -1.000001,
241
+ *sorted({choice_margin(recent[uid]) for uid in calibration_uids}),
242
+ 1.000001,
243
+ ]
244
+ commit_candidates = [
245
+ -1.000001,
246
+ *sorted(
247
+ {confidence_gain(recent[uid], temporal[uid]) for uid in calibration_uids}
248
+ ),
249
+ 1.000001,
250
+ ]
251
+ sweep = []
252
+ for acquire_threshold in acquire_candidates:
253
+ for commit_threshold in commit_candidates:
254
+ correct, acquired, committed = evaluate_acquire_commit_thresholds(
255
+ calibration_uids,
256
+ recent=recent,
257
+ temporal=temporal,
258
+ recent_scores=recent_scores,
259
+ temporal_scores=temporal_scores,
260
+ acquire_threshold=acquire_threshold,
261
+ commit_threshold=commit_threshold,
262
+ )
263
+ sweep.append(
264
+ {
265
+ "acquire_threshold": acquire_threshold,
266
+ "commit_threshold": commit_threshold,
267
+ "correct": correct,
268
+ "total": len(calibration_uids),
269
+ "accuracy": correct / len(calibration_uids),
270
+ "acquired": acquired,
271
+ "committed": committed,
272
+ }
273
+ )
274
+ best = max(
275
+ sweep,
276
+ key=lambda row: (
277
+ int(row["correct"]),
278
+ -int(row["acquired"]),
279
+ -int(row["committed"]),
280
+ -float(row["acquire_threshold"]),
281
+ float(row["commit_threshold"]),
282
+ ),
283
+ )
284
+ return (
285
+ float(best["acquire_threshold"]),
286
+ float(best["commit_threshold"]),
287
+ sweep,
288
+ )
289
+
290
+
291
+ def main() -> None:
292
+ args = parse_args()
293
+ recent_result = load_result(args.recent)
294
+ temporal_result = load_result(args.temporal)
295
+ for field in ("benchmark", "model_manifest_sha256", "samples_sha256"):
296
+ if recent_result.get(field) != temporal_result.get(field):
297
+ raise ValueError(f"recent/temporal identity mismatch: {field}")
298
+
299
+ recent = index_rows(recent_result)
300
+ temporal = index_rows(temporal_result)
301
+ recent_scores = index_scores(recent_result)
302
+ temporal_scores = index_temporal_scores(temporal_result)
303
+ if (
304
+ set(recent) != set(temporal)
305
+ or set(recent) != set(recent_scores)
306
+ or set(recent) != set(temporal_scores)
307
+ ):
308
+ raise ValueError("recent, temporal, and score UID sets must match exactly")
309
+ if any(row.get("task") is None for row in recent_scores.values()):
310
+ raise ValueError("score rows require task labels for reporting only")
311
+
312
+ ordered_uids = [str(row["uid"]) for row in recent_result["rows"]]
313
+ calibration_uids = [
314
+ uid
315
+ for uid in ordered_uids
316
+ if calibration_bucket(uid, args.calibration_modulus) == 0
317
+ ]
318
+ evaluation_uids = [uid for uid in ordered_uids if uid not in calibration_uids]
319
+ if not calibration_uids or not evaluation_uids:
320
+ raise ValueError("deterministic split produced an empty partition")
321
+
322
+ acquire_threshold, commit_threshold, sweep = fit_acquire_commit_thresholds(
323
+ calibration_uids,
324
+ recent=recent,
325
+ temporal=temporal,
326
+ recent_scores=recent_scores,
327
+ temporal_scores=temporal_scores,
328
+ )
329
+ rows = []
330
+ calibration_set = set(calibration_uids)
331
+ by_task: dict[str, list[int]] = defaultdict(lambda: [0, 0])
332
+ for uid in ordered_uids:
333
+ prediction, escalated, committed, gain = select_acquire_commit_prediction(
334
+ recent[uid], temporal[uid], acquire_threshold, commit_threshold
335
+ )
336
+ official = temporal_scores[uid] if committed else recent_scores[uid]
337
+ correct = bool(official["correct"])
338
+ partition = "calibration" if uid in calibration_set else "evaluation"
339
+ task = str(recent_scores[uid]["task"])
340
+ if partition == "evaluation":
341
+ by_task[task][0] += int(correct)
342
+ by_task[task][1] += 1
343
+ rows.append(
344
+ {
345
+ "uid": uid,
346
+ "partition": partition,
347
+ "margin": choice_margin(recent[uid]),
348
+ "temporal_margin": choice_margin(temporal[uid]),
349
+ "confidence_gain": gain,
350
+ "acquire_threshold": acquire_threshold,
351
+ "commit_threshold": commit_threshold,
352
+ "escalated": escalated,
353
+ "committed": committed,
354
+ "joint_required_after_escalation": bool(
355
+ escalated and temporal[uid].get("joint_required", False)
356
+ ),
357
+ "prediction": prediction,
358
+ "correct": correct,
359
+ "task": task,
360
+ "task_used_by_policy": False,
361
+ }
362
+ )
363
+
364
+ def partition_summary(partition: str) -> dict[str, Any]:
365
+ chosen = [row for row in rows if row["partition"] == partition]
366
+ correct = sum(int(row["correct"]) for row in chosen)
367
+ escalated = sum(int(row["escalated"]) for row in chosen)
368
+ committed = sum(int(row["committed"]) for row in chosen)
369
+ joint_calls = sum(
370
+ int(row["joint_required_after_escalation"]) for row in chosen
371
+ )
372
+ return {
373
+ "correct": correct,
374
+ "total": len(chosen),
375
+ "accuracy": correct / len(chosen),
376
+ "escalated": escalated,
377
+ "escalation_rate": escalated / len(chosen),
378
+ "committed": committed,
379
+ "commit_rate": committed / len(chosen),
380
+ "joint_calls": joint_calls,
381
+ "mean_model_calls": (len(chosen) + escalated + joint_calls)
382
+ / len(chosen),
383
+ }
384
+
385
+ internal_calibration = (
386
+ recent_result.get("benchmark") == "internal_policy_holdout"
387
+ and recent_result.get("status")
388
+ == "source_disjoint_internal_controller_calibration"
389
+ and temporal_result.get("status")
390
+ == "source_disjoint_internal_controller_calibration"
391
+ )
392
+ output = {
393
+ "format": "streameva.uncertainty_temporal_escalation_audit.v0",
394
+ "status": (
395
+ "source_disjoint_internal_threshold_calibration"
396
+ if internal_calibration
397
+ else "benchmark_selected_exploratory_not_confirmatory"
398
+ ),
399
+ "eligible_as_frozen_benchmark_controller_config": internal_calibration,
400
+ "benchmark": recent_result.get("benchmark"),
401
+ "model_manifest_sha256": recent_result.get("model_manifest_sha256"),
402
+ "policy": {
403
+ "first_view": (
404
+ "recorded_initial_causal_view" if internal_calibration else "recent"
405
+ ),
406
+ "second_view": (
407
+ "all_recorded_causal_views"
408
+ if internal_calibration
409
+ else temporal_result.get("inference_policy")
410
+ ),
411
+ "trigger": "recent_first_step_generation_top2_margin_leq_global_threshold",
412
+ "threshold": acquire_threshold,
413
+ "acquire_threshold": acquire_threshold,
414
+ "commit_threshold": commit_threshold,
415
+ "commit_signal": "temporal_minus_recent_full_vocabulary_top2_margin",
416
+ "commit_rule": "changed_answer_and_confidence_gain_gt_global_threshold",
417
+ "task_group_or_question_router": False,
418
+ "gold_available_at_runtime": False,
419
+ },
420
+ "split": {
421
+ "method": "sha256(uid) modulo calibration_modulus",
422
+ "calibration_modulus": args.calibration_modulus,
423
+ "calibration_bucket": 0,
424
+ },
425
+ "calibration": partition_summary("calibration"),
426
+ "evaluation": partition_summary("evaluation"),
427
+ "evaluation_by_task_reporting_only": {
428
+ task: {"correct": values[0], "total": values[1], "accuracy": values[0] / values[1]}
429
+ for task, values in sorted(by_task.items())
430
+ },
431
+ "calibration_sweep": sweep,
432
+ "rows": rows,
433
+ }
434
+ args.output.parent.mkdir(parents=True, exist_ok=True)
435
+ args.output.write_text(
436
+ json.dumps(output, indent=2, sort_keys=True) + "\n", encoding="utf-8"
437
+ )
438
+ print(
439
+ json.dumps(
440
+ {
441
+ "acquire_threshold": acquire_threshold,
442
+ "commit_threshold": commit_threshold,
443
+ **output["evaluation"],
444
+ },
445
+ sort_keys=True,
446
+ )
447
+ )
448
+
449
+
450
+ if __name__ == "__main__":
451
+ main()
code/experiments/evaluate_qwen3vl_recent4_full.py ADDED
The diff for this file is too large to render. See raw diff
 
code/experiments/materialize_confidence_gated_third_view_agent.py ADDED
@@ -0,0 +1,259 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """Materialize a task-blind confidence-gated third-view temporal agent.
3
+
4
+ The third view is acquired on the same answer-disagreement cohort as the direct
5
+ third-view agent. Its proposal is committed only when its first-token margin
6
+ improves over the margin of the view that supplied the current answer by more
7
+ than one source-disjoint, globally frozen threshold. Runtime policy inputs do
8
+ not include task, benchmark identity, or gold labels.
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ import argparse
14
+ import copy
15
+ import json
16
+ from pathlib import Path
17
+ from statistics import fmean
18
+ from typing import Any
19
+
20
+ from experiments.analyze_uncertainty_temporal_escalation import choice_margin
21
+ from experiments.materialize_third_view_disagreement_agent import (
22
+ IDENTITY_FIELDS,
23
+ _frame_indices,
24
+ _index,
25
+ )
26
+
27
+
28
+ def materialize(
29
+ recent: dict[str, Any],
30
+ event: dict[str, Any],
31
+ base_agent: dict[str, Any],
32
+ third: dict[str, Any],
33
+ threshold: dict[str, Any],
34
+ ) -> dict[str, Any]:
35
+ for field in IDENTITY_FIELDS:
36
+ values = {value.get(field) for value in (recent, event, base_agent, third)}
37
+ if len(values) != 1:
38
+ raise ValueError(f"view identity mismatch: {field}")
39
+ if (
40
+ threshold.get("status") != "source_disjoint_internal_threshold_calibration"
41
+ or threshold.get("eligible_as_frozen_benchmark_controller_config") is not True
42
+ or threshold.get("model_manifest_sha256")
43
+ != recent.get("model_manifest_sha256")
44
+ ):
45
+ raise ValueError("threshold is not an identity-aligned internal calibration")
46
+ commit_threshold = float(threshold["policy"]["commit_threshold"])
47
+
48
+ recent_rows, event_rows, base_rows, third_rows = map(
49
+ _index, (recent, event, base_agent, third)
50
+ )
51
+ if set(base_rows) != set(recent_rows) or not set(event_rows) <= set(recent_rows):
52
+ raise ValueError("base/recent/event UID coverage diverged")
53
+ disagreements = {
54
+ uid
55
+ for uid, row in event_rows.items()
56
+ if str(row.get("prediction") or "")
57
+ != str(recent_rows[uid].get("prediction") or "")
58
+ }
59
+ if not disagreements <= set(third_rows):
60
+ raise ValueError("third-view coverage must include the event disagreement cohort")
61
+
62
+ output_rows: list[dict[str, Any]] = []
63
+ committed_count = 0
64
+ for recent_row in recent.get("rows") or []:
65
+ uid = str(recent_row["uid"])
66
+ base = copy.deepcopy(base_rows[uid])
67
+ if uid not in disagreements:
68
+ base["third_view_policy"] = {
69
+ "triggered": False,
70
+ "reason": "no_acquired_event_disagreement",
71
+ "question_task_group_or_benchmark_router": False,
72
+ }
73
+ output_rows.append(base)
74
+ continue
75
+
76
+ event_row = event_rows[uid]
77
+ third_row = third_rows[uid]
78
+ if event_row.get("error") or third_row.get("error"):
79
+ raise ValueError(f"{uid}: acquired view contains an error")
80
+ if not bool((base.get("agent_policy") or {}).get("escalated")):
81
+ raise ValueError(f"{uid}: third view was triggered outside acquired cohort")
82
+ timestamps = (third_row.get("sampling") or {}).get("frame_timestamps") or []
83
+ if timestamps and max(map(float, timestamps)) > float(third_row["query_time"]) + 1e-3:
84
+ raise ValueError(f"{uid}: third view crosses the causal frontier")
85
+
86
+ recent_prediction = str(recent_row.get("prediction") or "")
87
+ event_prediction = str(event_row.get("prediction") or "")
88
+ current_prediction = str(base.get("prediction") or "")
89
+ third_prediction = str(third_row.get("prediction") or "")
90
+ if not all((recent_prediction, event_prediction, current_prediction, third_prediction)):
91
+ raise ValueError(f"{uid}: an agent proposal is empty")
92
+ if recent_prediction == event_prediction:
93
+ raise ValueError(f"{uid}: third view lacks a disagreement trigger")
94
+ if current_prediction == recent_prediction:
95
+ current_source = recent_row
96
+ current_view = "recent4"
97
+ elif current_prediction == event_prediction:
98
+ current_source = event_row
99
+ current_view = "causalbudget20"
100
+ else:
101
+ raise ValueError(f"{uid}: current answer is not supplied by recent or event view")
102
+
103
+ confidence_gain = choice_margin(third_row) - choice_margin(current_source)
104
+ committed = bool(
105
+ third_prediction != current_prediction
106
+ and confidence_gain > commit_threshold
107
+ )
108
+ if committed:
109
+ committed_count += 1
110
+ base["prediction"] = third_prediction
111
+ base["raw_response"] = third_row.get("raw_response")
112
+ unique_indices = _frame_indices(event_row) | _frame_indices(third_row)
113
+ base.update(
114
+ {
115
+ "current_prediction": current_prediction,
116
+ "third_view_prediction": third_prediction,
117
+ "third_view_policy": {
118
+ "triggered": True,
119
+ "trigger": "acquired_event_prediction_differs_from_recent_prediction",
120
+ "commit": "global_source_disjoint_confidence_gain",
121
+ "current_view": current_view,
122
+ "current_margin": choice_margin(current_source),
123
+ "third_margin": choice_margin(third_row),
124
+ "confidence_gain": confidence_gain,
125
+ "commit_threshold": commit_threshold,
126
+ "committed": committed,
127
+ "question_task_group_or_benchmark_router": False,
128
+ "gold_available_to_policy": False,
129
+ },
130
+ "media_calls": int(base.get("media_calls") or 2) + 1,
131
+ "num_turns": int(base.get("num_turns") or 2) + 1,
132
+ "model_processed_frames": int(base.get("model_processed_frames") or 24)
133
+ + int(third_row.get("frames_observed") or 32),
134
+ "unique_frames": len(unique_indices) if unique_indices else 32,
135
+ "latency_seconds": float(base.get("latency_seconds") or 0.0)
136
+ + float(third_row.get("latency_seconds") or 0.0),
137
+ "visual_tokens": int(recent_row.get("visual_tokens") or 0)
138
+ + int(event_row.get("visual_tokens") or 0)
139
+ + int(third_row.get("visual_tokens") or 0),
140
+ }
141
+ )
142
+ output_rows.append(base)
143
+
144
+ if any(not str(row.get("prediction") or "") for row in output_rows):
145
+ raise ValueError("materialized agent contains an empty prediction")
146
+ triggered = len(disagreements)
147
+ return {
148
+ **{
149
+ key: base_agent.get(key)
150
+ for key in (
151
+ "annotation",
152
+ "annotation_sha256",
153
+ "benchmark",
154
+ "evaluation_protocol",
155
+ "evaluation_protocol_sha256",
156
+ "model",
157
+ "model_manifest_sha256",
158
+ "processor",
159
+ )
160
+ },
161
+ "format": "streameva.confidence_gated_third_view_agent.v0",
162
+ "status": "benchmark_selected_router_free_development_agent",
163
+ "classification": "benchmark_selected_development_not_confirmatory",
164
+ "comparable_under_locked_recent4_protocol": False,
165
+ "policy": {
166
+ "first_view": "recent4",
167
+ "second_view": "frozen_uncertainty_triggered_causalbudget20",
168
+ "third_view": "causalbudget32r8_on_acquired_answer_disagreement",
169
+ "third_view_commit": "global_source_disjoint_confidence_gain",
170
+ "commit_threshold": commit_threshold,
171
+ "question_task_group_or_benchmark_router": False,
172
+ "benchmark_identity_available_to_policy": False,
173
+ "gold_available_to_policy": False,
174
+ },
175
+ "num_samples": len(output_rows),
176
+ "num_errors": 0,
177
+ "third_view_triggered": triggered,
178
+ "third_view_input_rows": len(third_rows),
179
+ "third_view_unused_precomputed_rows": len(set(third_rows) - disagreements),
180
+ "third_view_committed": committed_count,
181
+ "third_view_trigger_rate": triggered / len(output_rows),
182
+ "third_view_commit_rate": committed_count / triggered if triggered else 0.0,
183
+ "mean_model_calls": fmean(int(row.get("media_calls") or 1) for row in output_rows),
184
+ "mean_unique_frames": fmean(int(row.get("unique_frames") or 4) for row in output_rows),
185
+ "mean_model_processed_frames": fmean(
186
+ int(row.get("model_processed_frames") or 4) for row in output_rows
187
+ ),
188
+ "rows": output_rows,
189
+ }
190
+
191
+
192
+ def main() -> None:
193
+ parser = argparse.ArgumentParser(description=__doc__)
194
+ parser.add_argument("--recent", type=Path, required=True)
195
+ parser.add_argument("--event", type=Path, required=True)
196
+ parser.add_argument("--base-agent", type=Path, required=True)
197
+ parser.add_argument("--third-view", type=Path, required=True)
198
+ parser.add_argument("--threshold-audit", type=Path, required=True)
199
+ parser.add_argument("--third-view-model-identity-sha256")
200
+ parser.add_argument("--output", type=Path, required=True)
201
+ args = parser.parse_args()
202
+ if args.output.exists():
203
+ raise FileExistsError(args.output)
204
+ values = [
205
+ json.loads(path.read_text(encoding="utf-8"))
206
+ for path in (
207
+ args.recent,
208
+ args.event,
209
+ args.base_agent,
210
+ args.third_view,
211
+ args.threshold_audit,
212
+ )
213
+ ]
214
+ identity_override = None
215
+ if args.third_view_model_identity_sha256:
216
+ if len(args.third_view_model_identity_sha256) != 64 or any(
217
+ value not in "0123456789abcdef"
218
+ for value in args.third_view_model_identity_sha256
219
+ ):
220
+ raise ValueError("third-view model identity is malformed")
221
+ third = values[3]
222
+ if third.get("model_manifest_sha256") not in {
223
+ None,
224
+ args.third_view_model_identity_sha256,
225
+ }:
226
+ raise ValueError("third-view model identity override diverges")
227
+ if third.get("model_manifest_sha256") is None:
228
+ third["model_manifest_sha256"] = args.third_view_model_identity_sha256
229
+ identity_override = {
230
+ "field": "model_manifest_sha256",
231
+ "source": "previously_sealed_checkpoint_identity",
232
+ "value": args.third_view_model_identity_sha256,
233
+ }
234
+ result = materialize(*values)
235
+ result["third_view_identity_override"] = identity_override
236
+ result["artifacts"] = {
237
+ "recent": str(args.recent.resolve()),
238
+ "event": str(args.event.resolve()),
239
+ "base_agent": str(args.base_agent.resolve()),
240
+ "third_view": str(args.third_view.resolve()),
241
+ "threshold_audit": str(args.threshold_audit.resolve()),
242
+ }
243
+ args.output.parent.mkdir(parents=True, exist_ok=True)
244
+ args.output.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n")
245
+ print(
246
+ json.dumps(
247
+ {
248
+ "output": str(args.output),
249
+ "num_samples": result["num_samples"],
250
+ "triggered": result["third_view_triggered"],
251
+ "committed": result["third_view_committed"],
252
+ },
253
+ sort_keys=True,
254
+ )
255
+ )
256
+
257
+
258
+ if __name__ == "__main__":
259
+ main()
code/experiments/materialize_third_view_disagreement_agent.py ADDED
@@ -0,0 +1,203 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """Materialize a router-free third-view agent on event-view disagreements.
3
+
4
+ The frozen first stage decides whether to acquire the event view. A third
5
+ 32-frame view is acquired only when that event proposal disagrees with the
6
+ recent proposal, and its answer is committed directly. Runtime decisions use
7
+ predictions and acquisition state only; task names, benchmark identity, and
8
+ gold labels are never inputs to the policy.
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ import argparse
14
+ import copy
15
+ import json
16
+ from pathlib import Path
17
+ from statistics import fmean
18
+ from typing import Any
19
+
20
+
21
+ IDENTITY_FIELDS = (
22
+ "benchmark",
23
+ "model_manifest_sha256",
24
+ "annotation_sha256",
25
+ "evaluation_protocol_sha256",
26
+ )
27
+
28
+
29
+ def _index(value: dict[str, Any]) -> dict[str, dict[str, Any]]:
30
+ rows = value.get("rows") or []
31
+ indexed = {str(row.get("uid") or ""): row for row in rows}
32
+ if "" in indexed or len(indexed) != len(rows):
33
+ raise ValueError("result rows contain empty or duplicate UIDs")
34
+ return indexed
35
+
36
+
37
+ def _frame_indices(row: dict[str, Any]) -> set[int]:
38
+ return {int(value) for value in (row.get("sampling") or {}).get("frame_indices") or []}
39
+
40
+
41
+ def materialize(
42
+ recent: dict[str, Any],
43
+ event: dict[str, Any],
44
+ base_agent: dict[str, Any],
45
+ third: dict[str, Any],
46
+ ) -> dict[str, Any]:
47
+ for field in IDENTITY_FIELDS:
48
+ values = {value.get(field) for value in (recent, event, base_agent, third)}
49
+ if len(values) != 1:
50
+ raise ValueError(f"view identity mismatch: {field}")
51
+ recent_rows, event_rows, base_rows, third_rows = map(
52
+ _index, (recent, event, base_agent, third)
53
+ )
54
+ if set(base_rows) != set(recent_rows) or not set(event_rows) <= set(recent_rows):
55
+ raise ValueError("base/recent/event UID coverage diverged")
56
+ disagreements = {
57
+ uid
58
+ for uid, row in event_rows.items()
59
+ if str(row.get("prediction") or "")
60
+ != str(recent_rows[uid].get("prediction") or "")
61
+ }
62
+ if set(third_rows) != disagreements:
63
+ raise ValueError("third-view coverage must equal the event disagreement cohort")
64
+
65
+ output_rows = []
66
+ for recent_row in recent.get("rows") or []:
67
+ uid = str(recent_row["uid"])
68
+ base = copy.deepcopy(base_rows[uid])
69
+ if uid not in third_rows:
70
+ base["third_view_policy"] = {
71
+ "triggered": False,
72
+ "reason": "no_acquired_event_disagreement",
73
+ "question_task_group_or_benchmark_router": False,
74
+ }
75
+ output_rows.append(base)
76
+ continue
77
+
78
+ proposal = event_rows[uid]
79
+ adjudicator = third_rows[uid]
80
+ if proposal.get("error") or adjudicator.get("error"):
81
+ raise ValueError(f"{uid}: acquired view contains an error")
82
+ if not bool((base.get("agent_policy") or {}).get("escalated")):
83
+ raise ValueError(f"{uid}: third view was triggered outside the acquired cohort")
84
+ timestamps = (adjudicator.get("sampling") or {}).get("frame_timestamps") or []
85
+ if timestamps and max(map(float, timestamps)) > float(adjudicator["query_time"]) + 1e-3:
86
+ raise ValueError(f"{uid}: third view crosses the causal frontier")
87
+ recent_prediction = str(recent_row.get("prediction") or "")
88
+ event_prediction = str(proposal.get("prediction") or "")
89
+ third_prediction = str(adjudicator.get("prediction") or "")
90
+ if not recent_prediction or not event_prediction or not third_prediction:
91
+ raise ValueError(f"{uid}: an agent proposal is empty")
92
+ if recent_prediction == event_prediction:
93
+ raise ValueError(f"{uid}: third view lacks a disagreement trigger")
94
+
95
+ unique_indices = _frame_indices(proposal) | _frame_indices(adjudicator)
96
+ base.update(
97
+ {
98
+ "prediction": third_prediction,
99
+ "raw_response": adjudicator.get("raw_response"),
100
+ "third_view_prediction": third_prediction,
101
+ "third_view_policy": {
102
+ "triggered": True,
103
+ "trigger": "acquired_event_prediction_differs_from_recent_prediction",
104
+ "commit": "direct_third_view_prediction",
105
+ "recent_prediction": recent_prediction,
106
+ "event_prediction": event_prediction,
107
+ "question_task_group_or_benchmark_router": False,
108
+ "gold_available_to_policy": False,
109
+ },
110
+ "media_calls": int(base.get("media_calls") or 2) + 1,
111
+ "num_turns": int(base.get("num_turns") or 2) + 1,
112
+ "model_processed_frames": int(base.get("model_processed_frames") or 24)
113
+ + int(adjudicator.get("frames_observed") or 32),
114
+ "unique_frames": len(unique_indices) if unique_indices else 32,
115
+ "latency_seconds": float(base.get("latency_seconds") or 0.0)
116
+ + float(adjudicator.get("latency_seconds") or 0.0),
117
+ "visual_tokens": int(recent_row.get("visual_tokens") or 0)
118
+ + int(proposal.get("visual_tokens") or 0)
119
+ + int(adjudicator.get("visual_tokens") or 0),
120
+ }
121
+ )
122
+ output_rows.append(base)
123
+ if any(not str(row.get("prediction") or "") for row in output_rows):
124
+ raise ValueError("materialized agent contains an empty prediction")
125
+
126
+ triggered = len(third_rows)
127
+ return {
128
+ **{
129
+ key: base_agent.get(key)
130
+ for key in (
131
+ "annotation",
132
+ "annotation_sha256",
133
+ "benchmark",
134
+ "evaluation_protocol",
135
+ "evaluation_protocol_sha256",
136
+ "model",
137
+ "model_manifest_sha256",
138
+ "processor",
139
+ )
140
+ },
141
+ "format": "streameva.third_view_disagreement_agent.v0",
142
+ "status": "benchmark_selected_router_free_development_agent",
143
+ "classification": "benchmark_selected_development_not_confirmatory",
144
+ "comparable_under_locked_recent4_protocol": False,
145
+ "policy": {
146
+ "first_view": "recent4",
147
+ "second_view": "frozen_uncertainty_triggered_causalbudget20",
148
+ "third_view": "causalbudget32r8_on_acquired_answer_disagreement",
149
+ "third_view_commit": "direct_prediction",
150
+ "question_task_group_or_benchmark_router": False,
151
+ "benchmark_identity_available_to_policy": False,
152
+ "gold_available_to_policy": False,
153
+ },
154
+ "num_samples": len(output_rows),
155
+ "num_errors": 0,
156
+ "third_view_triggered": triggered,
157
+ "third_view_trigger_rate": triggered / len(output_rows),
158
+ "mean_model_calls": fmean(int(row.get("media_calls") or 1) for row in output_rows),
159
+ "mean_unique_frames": fmean(int(row.get("unique_frames") or 4) for row in output_rows),
160
+ "mean_model_processed_frames": fmean(
161
+ int(row.get("model_processed_frames") or 4) for row in output_rows
162
+ ),
163
+ "rows": output_rows,
164
+ }
165
+
166
+
167
+ def main() -> None:
168
+ parser = argparse.ArgumentParser(description=__doc__)
169
+ parser.add_argument("--recent", type=Path, required=True)
170
+ parser.add_argument("--event", type=Path, required=True)
171
+ parser.add_argument("--base-agent", type=Path, required=True)
172
+ parser.add_argument("--third-view", type=Path, required=True)
173
+ parser.add_argument("--output", type=Path, required=True)
174
+ args = parser.parse_args()
175
+ if args.output.exists():
176
+ raise FileExistsError(args.output)
177
+ values = [
178
+ json.loads(path.read_text(encoding="utf-8"))
179
+ for path in (args.recent, args.event, args.base_agent, args.third_view)
180
+ ]
181
+ result = materialize(*values)
182
+ result["artifacts"] = {
183
+ "recent": str(args.recent.resolve()),
184
+ "event": str(args.event.resolve()),
185
+ "base_agent": str(args.base_agent.resolve()),
186
+ "third_view": str(args.third_view.resolve()),
187
+ }
188
+ args.output.parent.mkdir(parents=True, exist_ok=True)
189
+ args.output.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n")
190
+ print(
191
+ json.dumps(
192
+ {
193
+ "output": str(args.output),
194
+ "num_samples": result["num_samples"],
195
+ "third_view_triggered": result["third_view_triggered"],
196
+ },
197
+ sort_keys=True,
198
+ )
199
+ )
200
+
201
+
202
+ if __name__ == "__main__":
203
+ main()
code/experiments/materialize_tvs_view_oracle.py ADDED
@@ -0,0 +1,118 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """Materialize a strict post-hoc oracle over three complete causal views."""
3
+
4
+ from __future__ import annotations
5
+
6
+ import argparse
7
+ import copy
8
+ import json
9
+ from pathlib import Path
10
+ from statistics import fmean
11
+ from typing import Any
12
+
13
+ try:
14
+ from experiments.tvs_correctness import is_correct, load_gold
15
+ except ModuleNotFoundError: # Direct script execution without repository PYTHONPATH.
16
+ from tvs_correctness import is_correct, load_gold
17
+
18
+
19
+ IDENTITY_FIELDS = (
20
+ "benchmark",
21
+ "model_manifest_sha256",
22
+ "annotation_sha256",
23
+ "evaluation_protocol_sha256",
24
+ )
25
+
26
+
27
+ def _index(value: dict[str, Any]) -> dict[str, dict[str, Any]]:
28
+ rows = value.get("rows") or []
29
+ result = {str(row["uid"]): row for row in rows}
30
+ if len(result) != len(rows):
31
+ raise ValueError("duplicate view UID")
32
+ return result
33
+
34
+
35
+ def materialize(
36
+ recent: dict[str, Any],
37
+ event: dict[str, Any],
38
+ history: dict[str, Any],
39
+ gold: dict[str, dict[str, Any]],
40
+ ) -> dict[str, Any]:
41
+ views = [_index(value) for value in (recent, event, history)]
42
+ uid_set = set(views[0])
43
+ if any(set(view) != uid_set for view in views[1:]) or set(gold) != uid_set:
44
+ raise ValueError("strict oracle requires exact three-view and gold coverage")
45
+ for field in IDENTITY_FIELDS:
46
+ expected = recent.get(field)
47
+ if expected in (None, ""):
48
+ raise ValueError(f"recent view lacks identity: {field}")
49
+ for value in (event, history):
50
+ observed = value.get(field)
51
+ if observed not in (None, "", expected):
52
+ raise ValueError(f"view identity mismatch: {field}")
53
+
54
+ rows = []
55
+ chosen_counts = [0, 0, 0]
56
+ for source in recent.get("rows") or []:
57
+ uid = str(source["uid"])
58
+ candidates = [view[uid] for view in views]
59
+ correct = [is_correct(row.get("prediction"), gold[uid]) for row in candidates]
60
+ chosen_index = next((i for i, value in enumerate(correct) if value), 0)
61
+ chosen_counts[chosen_index] += 1
62
+ output = copy.deepcopy(candidates[chosen_index])
63
+ output.update(
64
+ {
65
+ "uid": uid,
66
+ "prediction": str(candidates[chosen_index].get("prediction") or ""),
67
+ "media_calls": 3,
68
+ "model_processed_frames": sum(
69
+ int(row.get("frames_observed") or fallback)
70
+ for row, fallback in zip(candidates, (4, 20, 32), strict=True)
71
+ ),
72
+ "tvs_oracle": {
73
+ "analysis_only": True,
74
+ "gold_available_to_selector": True,
75
+ "correct_proposals": correct,
76
+ "chosen_view": ("recent", "event", "history")[chosen_index],
77
+ },
78
+ }
79
+ )
80
+ rows.append(output)
81
+ return {
82
+ **{field: recent.get(field) for field in IDENTITY_FIELDS},
83
+ "format": "iclr_video.strict_three_view_oracle.v1",
84
+ "status": "complete",
85
+ "analysis_only": True,
86
+ "num_samples": len(rows),
87
+ "num_errors": 0,
88
+ "mean_model_calls": 3.0,
89
+ "mean_processed_frames": fmean(row["model_processed_frames"] for row in rows),
90
+ "chosen_view_counts": dict(zip(("recent", "event", "history"), chosen_counts, strict=True)),
91
+ "rows": rows,
92
+ }
93
+
94
+
95
+ def main() -> None:
96
+ parser = argparse.ArgumentParser()
97
+ parser.add_argument("--benchmark", choices=("streamingbench", "ovo"), required=True)
98
+ parser.add_argument("--annotation", type=Path, required=True)
99
+ parser.add_argument("--recent", type=Path, required=True)
100
+ parser.add_argument("--event", type=Path, required=True)
101
+ parser.add_argument("--history", type=Path, required=True)
102
+ parser.add_argument("--output", type=Path, required=True)
103
+ args = parser.parse_args()
104
+ if args.output.exists():
105
+ raise FileExistsError(args.output)
106
+ values = [json.loads(path.read_text()) for path in (args.recent, args.event, args.history)]
107
+ result = materialize(*values, load_gold(args.benchmark, args.annotation))
108
+ args.output.parent.mkdir(parents=True, exist_ok=True)
109
+ args.output.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n")
110
+ print(json.dumps({
111
+ "rows": result["num_samples"],
112
+ "mean_model_calls": result["mean_model_calls"],
113
+ "mean_processed_frames": result["mean_processed_frames"],
114
+ }, sort_keys=True))
115
+
116
+
117
+ if __name__ == "__main__":
118
+ main()
code/experiments/materialize_uncertainty_temporal_agent.py ADDED
@@ -0,0 +1,465 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """Materialize one full prediction set from recent and triggered temporal calls."""
3
+
4
+ from __future__ import annotations
5
+
6
+ import argparse
7
+ import json
8
+ from pathlib import Path
9
+ from statistics import fmean
10
+ from typing import Any
11
+
12
+ from experiments.evaluate_qwen3vl_recent4_full import atomic_write
13
+ from experiments.analyze_uncertainty_temporal_escalation import choice_margin
14
+
15
+
16
+ def parse_args() -> argparse.Namespace:
17
+ parser = argparse.ArgumentParser(description=__doc__)
18
+ parser.add_argument("--recent", type=Path, required=True)
19
+ parser.add_argument("--temporal", type=Path, required=True)
20
+ parser.add_argument("--selection-audit", type=Path, required=True)
21
+ parser.add_argument("--threshold-audit", type=Path)
22
+ parser.add_argument(
23
+ "--temporal-model-identity-sha256",
24
+ help=(
25
+ "Previously sealed model identity to attach when the temporal result "
26
+ "was launched without embedding it; avoids rehashing the model."
27
+ ),
28
+ )
29
+ parser.add_argument(
30
+ "--commit-threshold",
31
+ type=float,
32
+ help=(
33
+ "Optional fixed proposal-minus-current margin threshold. This is "
34
+ "required for an unlabeled fixed-budget selection and overrides "
35
+ "the commit threshold stored in an internal calibration."
36
+ ),
37
+ )
38
+ parser.add_argument(
39
+ "--fallback-recent-on-temporal-error",
40
+ action="store_true",
41
+ help=(
42
+ "Fail closed to the aligned recent4 prediction when a preselected "
43
+ "temporal call has a recorded runtime error. The fallback is fixed "
44
+ "before scoring and never reads benchmark labels."
45
+ ),
46
+ )
47
+ parser.add_argument("--output", type=Path, required=True)
48
+ return parser.parse_args()
49
+
50
+
51
+ def selected_view_name(policy: str, mode: str, *, escalated: bool) -> str:
52
+ if not escalated:
53
+ return "recent4"
54
+ if policy in {"causalbudget20", "causalbudget32r8"}:
55
+ suffix = "32_recent8" if policy == "causalbudget32r8" else "20"
56
+ mode_name = {
57
+ "native-video": "native_video",
58
+ "image-list": "image_list",
59
+ "hybrid-history-video-recent-images": (
60
+ "hybrid_history_video_recent_images"
61
+ ),
62
+ "lowres-history-recent-images": "lowres_history_recent_images",
63
+ }[mode]
64
+ return f"causal_temporal{suffix}_{mode_name}"
65
+ if policy in {"causalhybrid32", "causalhybrid32r8"}:
66
+ recent_frames = 8 if policy == "causalhybrid32r8" else 4
67
+ history_frames = 32 - recent_frames
68
+ if mode == "image-list":
69
+ return (
70
+ f"causal_uniform_history{history_frames}_plus_recent"
71
+ f"{recent_frames}_image_list"
72
+ )
73
+ return f"causal_hybrid_history{history_frames}_plus_recent{recent_frames}"
74
+ if policy == "prefix32":
75
+ return "causal_prefix32"
76
+ raise ValueError(f"unsupported second-view policy: {policy}")
77
+
78
+
79
+ def second_view_policy_name(policy: str, mode: str) -> str:
80
+ if policy in {"causalbudget20", "causalbudget32r8"}:
81
+ suffix = "32_recent8" if policy == "causalbudget32r8" else "20"
82
+ prefix = {
83
+ "native-video": "native_video",
84
+ "image-list": "image_list",
85
+ "hybrid-history-video-recent-images": (
86
+ "hybrid_history_video_recent_images"
87
+ ),
88
+ "lowres-history-recent-images": "lowres_history_recent_images",
89
+ }[mode]
90
+ return f"{prefix}_causal_temporal{suffix}"
91
+ if policy in {"causalhybrid32", "causalhybrid32r8"}:
92
+ if mode == "hybrid-history-video-recent-images":
93
+ return (
94
+ "hybrid_history_video_recent_images_causal_hybrid32r8"
95
+ if policy == "causalhybrid32r8"
96
+ else "hybrid_history_video_recent_images_causal_hybrid32"
97
+ )
98
+ return (
99
+ "image_list_uniform_history24_plus_recent8"
100
+ if policy == "causalhybrid32r8"
101
+ else "image_list_uniform_history28_plus_recent4"
102
+ )
103
+ if policy == "prefix32":
104
+ return "image_list_causal_prefix32"
105
+ raise ValueError(f"unsupported second-view policy: {policy}")
106
+
107
+
108
+ def materialize(
109
+ recent: dict[str, Any],
110
+ temporal: dict[str, Any],
111
+ selection: dict[str, Any],
112
+ threshold: dict[str, Any] | None,
113
+ *,
114
+ commit_threshold_override: float | None = None,
115
+ fallback_recent_on_temporal_error: bool = False,
116
+ ) -> dict[str, Any]:
117
+ for field in (
118
+ "benchmark",
119
+ "model_manifest_sha256",
120
+ "annotation_sha256",
121
+ "evaluation_protocol_sha256",
122
+ ):
123
+ if recent.get(field) != temporal.get(field):
124
+ raise ValueError(f"recent/temporal identity mismatch: {field}")
125
+ selection_format = selection.get("format")
126
+ fixed_quantile = None
127
+ frozen_threshold = None
128
+ if threshold is not None:
129
+ if (
130
+ threshold.get("status")
131
+ != "source_disjoint_internal_threshold_calibration"
132
+ or threshold.get("eligible_as_frozen_benchmark_controller_config")
133
+ is not True
134
+ or threshold.get("model_manifest_sha256")
135
+ != recent.get("model_manifest_sha256")
136
+ ):
137
+ raise ValueError("threshold is not an eligible internal calibration")
138
+ frozen_threshold = float(threshold["policy"]["threshold"])
139
+ commit_threshold = threshold["policy"].get("commit_threshold")
140
+ if selection_format not in (None, "streameva.internal_threshold_uid_selection.v0"):
141
+ raise ValueError("threshold calibration and selection mode diverge")
142
+ else:
143
+ if selection_format != "streameva.unlabeled_budget_uid_selection.v0":
144
+ raise ValueError("selection is not an unlabeled fixed-budget audit")
145
+ fixed_quantile = float(selection.get("quantile"))
146
+ if not 0.0 < fixed_quantile <= 1.0:
147
+ raise ValueError("fixed uncertainty quantile is invalid")
148
+ if selection.get("benchmark_identity_used_for_policy_parameters") is not False:
149
+ raise ValueError("fixed-budget acquisition used benchmark identity")
150
+ if selection.get("cohort_labels_read_for_selection") is not False:
151
+ raise ValueError("fixed-budget acquisition used cohort labels")
152
+ if commit_threshold_override is None:
153
+ raise ValueError("fixed-budget selection requires a commit threshold")
154
+ commit_threshold = commit_threshold_override
155
+ if commit_threshold_override is not None:
156
+ commit_threshold = commit_threshold_override
157
+ if commit_threshold is not None:
158
+ commit_threshold = float(commit_threshold)
159
+ if (
160
+ selection.get("status")
161
+ != "frozen_before_benchmark_temporal_inference"
162
+ or selection.get("benchmark") != recent.get("benchmark")
163
+ or selection.get("model_manifest_sha256")
164
+ != recent.get("model_manifest_sha256")
165
+ or selection.get("score_or_gold_fields_read_for_selection") is not False
166
+ ):
167
+ raise ValueError("selection audit is not frozen or identity-aligned")
168
+ if threshold is not None and float(selection.get("threshold")) != frozen_threshold:
169
+ raise ValueError("selection threshold diverges from internal calibration")
170
+
171
+ recent_rows = {str(row["uid"]): row for row in recent.get("rows") or []}
172
+ temporal_rows = {str(row["uid"]): row for row in temporal.get("rows") or []}
173
+ decisions = {str(row["uid"]): row for row in selection.get("rows") or []}
174
+ if (
175
+ len(recent_rows) != len(recent.get("rows") or [])
176
+ or len(temporal_rows) != len(temporal.get("rows") or [])
177
+ or len(decisions) != len(selection.get("rows") or [])
178
+ ):
179
+ raise ValueError("duplicate UID in input artifacts")
180
+ if set(decisions) != set(recent_rows):
181
+ raise ValueError("selection audit does not cover the recent result")
182
+ selected = {uid for uid, row in decisions.items() if bool(row["escalated"])}
183
+ temporal_input_count = len(temporal_rows)
184
+ if not selected.issubset(temporal_rows):
185
+ missing = sorted(selected - set(temporal_rows))
186
+ raise ValueError(
187
+ f"temporal inference is missing {len(missing)} selected UIDs"
188
+ )
189
+ temporal_rows = {uid: temporal_rows[uid] for uid in selected}
190
+ temporal_error_uids = {
191
+ uid for uid, row in temporal_rows.items() if bool(row.get("error"))
192
+ }
193
+ if temporal_error_uids and not fallback_recent_on_temporal_error:
194
+ raise ValueError(
195
+ f"{len(temporal_error_uids)} selected temporal rows contain errors"
196
+ )
197
+ valid_temporal_rows = {
198
+ uid: row for uid, row in temporal_rows.items() if uid not in temporal_error_uids
199
+ }
200
+ if not valid_temporal_rows:
201
+ raise ValueError("no valid temporal row remains to establish the second-view policy")
202
+ second_view_policies = {
203
+ str((row.get("sampling") or {}).get("frame_policy") or "")
204
+ for row in valid_temporal_rows.values()
205
+ }
206
+ if second_view_policies not in (
207
+ {"causalbudget20"},
208
+ {"causalbudget32r8"},
209
+ {"prefix32"},
210
+ {"causalhybrid32"},
211
+ {"causalhybrid32r8"},
212
+ ):
213
+ raise ValueError(
214
+ "second-view rows must share causalbudget20, causalbudget32r8, prefix32, or "
215
+ "causalhybrid32*"
216
+ )
217
+ second_view_policy = next(iter(second_view_policies))
218
+ second_view_modes = {
219
+ str(row.get("visual_input_mode") or "")
220
+ for row in valid_temporal_rows.values()
221
+ }
222
+ if len(second_view_modes) != 1:
223
+ raise ValueError("second-view rows do not share one visual input mode")
224
+ second_view_mode = next(iter(second_view_modes))
225
+ if second_view_mode not in {
226
+ "native-video",
227
+ "hybrid-history-video-recent-images",
228
+ "lowres-history-recent-images",
229
+ "image-list",
230
+ }:
231
+ raise ValueError("second-view visual input mode is missing or unsupported")
232
+
233
+ rows = []
234
+ for recent_row in recent.get("rows") or []:
235
+ uid = str(recent_row["uid"])
236
+ decision = decisions[uid]
237
+ escalated = bool(decision["escalated"])
238
+ temporal_error_fallback = escalated and uid in temporal_error_uids
239
+ proposal = (
240
+ recent_row
241
+ if temporal_error_fallback or not escalated
242
+ else temporal_rows[uid]
243
+ )
244
+ sampling = proposal.get("sampling") or {}
245
+ if sampling.get("frame_timestamps") and max(sampling["frame_timestamps"]) > float(
246
+ proposal["query_time"]
247
+ ) + 1e-3:
248
+ raise ValueError(f"{uid}: selected inference crosses the causal frontier")
249
+ recent_frames = int(recent_row.get("frames_observed") or 4)
250
+ chosen_frames = int(proposal.get("frames_observed") or recent_frames)
251
+ probe_frames = 0
252
+ extra_probe_frames = 0
253
+ if escalated and not temporal_error_fallback:
254
+ if sampling.get("frame_policy") != second_view_policy:
255
+ raise ValueError(f"{uid}: second-view policy diverged")
256
+ if second_view_policy in {"causalbudget20", "causalbudget32r8"}:
257
+ candidate_indices = sampling.get("candidate_frame_indices")
258
+ if not isinstance(candidate_indices, list) or not candidate_indices:
259
+ raise ValueError(f"{uid}: temporal row lacks probe-frame audit")
260
+ probe_frames = len(candidate_indices)
261
+ extra_probe_frames = int(sampling.get("extra_frames_inspected", -1))
262
+ if extra_probe_frames != max(0, probe_frames - chosen_frames):
263
+ raise ValueError(f"{uid}: temporal probe accounting diverged")
264
+ elif second_view_policy == "prefix32" and chosen_frames != 32:
265
+ raise ValueError(f"{uid}: prefix32 row does not expose 32 frames")
266
+ elif second_view_policy in {
267
+ "causalhybrid32",
268
+ "causalhybrid32r8",
269
+ } and not (
270
+ 1 <= chosen_frames <= 32
271
+ ):
272
+ raise ValueError(f"{uid}: causalhybrid32* frame budget is invalid")
273
+ recent_prediction = str(recent_row.get("prediction") or "")
274
+ proposal_prediction = str(proposal.get("prediction") or "")
275
+ confidence_gain = None
276
+ committed = escalated and not temporal_error_fallback
277
+ if escalated and not temporal_error_fallback and commit_threshold is not None:
278
+ confidence_gain = choice_margin(proposal) - choice_margin(recent_row)
279
+ committed = bool(
280
+ proposal_prediction != recent_prediction
281
+ and confidence_gain > commit_threshold
282
+ )
283
+ final_prediction = proposal_prediction if committed else recent_prediction
284
+ rows.append(
285
+ {
286
+ **proposal,
287
+ "uid": uid,
288
+ "prediction": final_prediction,
289
+ "current_prediction": recent_prediction,
290
+ "proposal_prediction": proposal_prediction if escalated else None,
291
+ "agent_policy": {
292
+ "margin": float(decision["margin"]),
293
+ "threshold": frozen_threshold,
294
+ "uncertainty_quantile": fixed_quantile,
295
+ "uncertainty_rank": decision.get("uncertainty_rank"),
296
+ "escalated": escalated,
297
+ "temporal_error_fallback": temporal_error_fallback,
298
+ "temporal_error": (
299
+ str(temporal_rows[uid].get("error"))
300
+ if temporal_error_fallback
301
+ else None
302
+ ),
303
+ "commit_threshold": commit_threshold,
304
+ "confidence_gain": confidence_gain,
305
+ "committed": committed,
306
+ "selected_view": (
307
+ "recent4_temporal_error_fallback"
308
+ if temporal_error_fallback
309
+ else selected_view_name(
310
+ second_view_policy, second_view_mode, escalated=escalated
311
+ )
312
+ ),
313
+ "question_task_group_or_benchmark_router": False,
314
+ },
315
+ "unique_frames": chosen_frames,
316
+ "model_processed_frames": (
317
+ recent_frames + chosen_frames
318
+ if escalated and not temporal_error_fallback
319
+ else recent_frames
320
+ ),
321
+ "low_resolution_probe_frames": probe_frames,
322
+ "extra_probe_frames_not_visible_to_vlm": extra_probe_frames,
323
+ "total_decoded_frame_instances": (
324
+ recent_frames + probe_frames + chosen_frames
325
+ if escalated and not temporal_error_fallback
326
+ else recent_frames
327
+ ),
328
+ "media_calls": 2 if escalated else 1,
329
+ "num_turns": 2 if escalated else 1,
330
+ "latency_seconds": float(recent_row.get("latency_seconds") or 0.0)
331
+ + (
332
+ float(temporal_rows[uid].get("latency_seconds") or 0.0)
333
+ if escalated
334
+ else 0.0
335
+ ),
336
+ }
337
+ )
338
+ if any(not row["prediction"] for row in rows):
339
+ raise ValueError("materialized agent contains an empty prediction")
340
+ escalated_count = len(selected)
341
+ return {
342
+ "format": "streameva.uncertainty_temporal_agent_eval.v0",
343
+ "status": (
344
+ "internally_calibrated_router_free_agent"
345
+ if threshold is not None
346
+ else "fixed_budget_router_free_agent"
347
+ ),
348
+ "benchmark": recent.get("benchmark"),
349
+ "model": recent.get("model"),
350
+ "model_manifest_sha256": recent.get("model_manifest_sha256"),
351
+ "processor": recent.get("processor"),
352
+ "annotation": recent.get("annotation"),
353
+ "annotation_sha256": recent.get("annotation_sha256"),
354
+ "evaluation_protocol": recent.get("evaluation_protocol"),
355
+ "evaluation_protocol_sha256": recent.get("evaluation_protocol_sha256"),
356
+ "comparable_under_locked_recent4_protocol": False,
357
+ "policy": {
358
+ "first_view": "recent4",
359
+ "uncertainty": "first_step_full_vocabulary_top2_margin",
360
+ "threshold": frozen_threshold,
361
+ "uncertainty_quantile": fixed_quantile,
362
+ "commit_threshold": commit_threshold,
363
+ "second_view": second_view_policy_name(
364
+ second_view_policy, second_view_mode
365
+ ),
366
+ "commit_rule": (
367
+ "changed_answer_and_confidence_gain_gt_global_threshold"
368
+ if commit_threshold is not None
369
+ else "legacy_always_accept_temporal_proposal"
370
+ ),
371
+ "question_task_group_or_benchmark_router": False,
372
+ "benchmark_identity_available_to_policy": False,
373
+ "gold_available_to_policy": False,
374
+ "temporal_error_policy": (
375
+ "fail_closed_to_recent4"
376
+ if fallback_recent_on_temporal_error
377
+ else "reject_materialization"
378
+ ),
379
+ },
380
+ "num_samples": len(rows),
381
+ "num_errors": 0,
382
+ "escalated": escalated_count,
383
+ "temporal_input_num_rows": temporal_input_count,
384
+ "temporal_unused_num_rows": temporal_input_count - escalated_count,
385
+ "temporal_error_fallback_count": len(temporal_error_uids),
386
+ "escalation_rate": escalated_count / len(rows),
387
+ "mean_model_calls": 1.0 + escalated_count / len(rows),
388
+ "mean_unique_frames": fmean(row["unique_frames"] for row in rows),
389
+ "mean_model_processed_frames": fmean(
390
+ row["model_processed_frames"] for row in rows
391
+ ),
392
+ "mean_low_resolution_probe_frames": fmean(
393
+ row["low_resolution_probe_frames"] for row in rows
394
+ ),
395
+ "mean_extra_probe_frames_not_visible_to_vlm": fmean(
396
+ row["extra_probe_frames_not_visible_to_vlm"] for row in rows
397
+ ),
398
+ "mean_total_decoded_frame_instances": fmean(
399
+ row["total_decoded_frame_instances"] for row in rows
400
+ ),
401
+ "rows": rows,
402
+ }
403
+
404
+
405
+ def main() -> None:
406
+ args = parse_args()
407
+ recent = json.loads(args.recent.read_text(encoding="utf-8"))
408
+ temporal = json.loads(args.temporal.read_text(encoding="utf-8"))
409
+ identity_override_applied = False
410
+ if args.temporal_model_identity_sha256 is not None:
411
+ identity = args.temporal_model_identity_sha256.strip().lower()
412
+ if len(identity) != 64 or any(
413
+ character not in "0123456789abcdef" for character in identity
414
+ ):
415
+ raise ValueError("temporal model identity must be a 64-character SHA-256")
416
+ embedded_identity = temporal.get("model_manifest_sha256")
417
+ if embedded_identity not in (None, "", identity):
418
+ raise ValueError("temporal model identity override conflicts with artifact")
419
+ if embedded_identity in (None, ""):
420
+ temporal["model_manifest_sha256"] = identity
421
+ identity_override_applied = True
422
+ selection = json.loads(args.selection_audit.read_text(encoding="utf-8"))
423
+ threshold = (
424
+ json.loads(args.threshold_audit.read_text(encoding="utf-8"))
425
+ if args.threshold_audit is not None
426
+ else None
427
+ )
428
+ output = materialize(
429
+ recent,
430
+ temporal,
431
+ selection,
432
+ threshold,
433
+ commit_threshold_override=args.commit_threshold,
434
+ fallback_recent_on_temporal_error=args.fallback_recent_on_temporal_error,
435
+ )
436
+ output["artifacts"] = {
437
+ "recent": {"path": str(args.recent.resolve())},
438
+ "temporal": {"path": str(args.temporal.resolve())},
439
+ "selection_audit": {"path": str(args.selection_audit.resolve())},
440
+ }
441
+ if args.threshold_audit is not None:
442
+ output["artifacts"]["threshold_audit"] = {
443
+ "path": str(args.threshold_audit.resolve()),
444
+ }
445
+ output["artifact_checksums_skipped"] = True
446
+ output["artifact_checksum_skip_reason"] = (
447
+ "native artifacts succeeded and identities are embedded in their manifests"
448
+ )
449
+ output["temporal_model_identity_override_applied"] = identity_override_applied
450
+ atomic_write(args.output, output)
451
+ print(
452
+ json.dumps(
453
+ {
454
+ "benchmark": output["benchmark"],
455
+ "num_samples": output["num_samples"],
456
+ "escalated": output["escalated"],
457
+ "mean_model_calls": output["mean_model_calls"],
458
+ },
459
+ sort_keys=True,
460
+ )
461
+ )
462
+
463
+
464
+ if __name__ == "__main__":
465
+ main()
code/experiments/select_uncertainty_escalation_uids.py ADDED
@@ -0,0 +1,192 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """Select temporal-view calls using a threshold or fixed uncertainty budget."""
3
+
4
+ from __future__ import annotations
5
+
6
+ import argparse
7
+ import json
8
+ import math
9
+ from pathlib import Path
10
+ from typing import Any
11
+
12
+ from experiments.analyze_uncertainty_temporal_escalation import choice_margin
13
+ from experiments.evaluate_qwen3vl_recent4_full import atomic_write, sha256_file
14
+
15
+
16
+ def parse_args() -> argparse.Namespace:
17
+ parser = argparse.ArgumentParser(description=__doc__)
18
+ parser.add_argument("--recent", type=Path, required=True)
19
+ policy = parser.add_mutually_exclusive_group(required=True)
20
+ policy.add_argument("--threshold", type=Path)
21
+ policy.add_argument(
22
+ "--quantile",
23
+ type=float,
24
+ help="Acquire the lowest-margin fraction without reading cohort labels.",
25
+ )
26
+ parser.add_argument("--samples", type=Path, required=True)
27
+ parser.add_argument("--output-uids", type=Path, required=True)
28
+ parser.add_argument("--output-samples", type=Path, required=True)
29
+ parser.add_argument("--output-audit", type=Path, required=True)
30
+ return parser.parse_args()
31
+
32
+
33
+ def load_frozen_threshold(path: Path, model_manifest_sha256: str | None) -> float:
34
+ value = json.loads(path.read_text(encoding="utf-8"))
35
+ if (
36
+ value.get("status") != "source_disjoint_internal_threshold_calibration"
37
+ or value.get("eligible_as_frozen_benchmark_controller_config") is not True
38
+ or value.get("benchmark") != "internal_policy_holdout"
39
+ or value.get("model_manifest_sha256") != model_manifest_sha256
40
+ ):
41
+ raise ValueError("threshold is not an eligible source-disjoint calibration")
42
+ threshold = float(value["policy"]["threshold"])
43
+ if not 0.0 <= threshold <= 1.0:
44
+ raise ValueError("invalid calibrated threshold")
45
+ return threshold
46
+
47
+
48
+ def select_rows(
49
+ recent: dict[str, Any], *, threshold: float
50
+ ) -> tuple[list[str], list[dict[str, Any]]]:
51
+ selected = []
52
+ audit_rows = []
53
+ for row in recent.get("rows") or []:
54
+ uid = str(row.get("uid") or "")
55
+ if not uid:
56
+ raise ValueError("recent result has a row without uid")
57
+ margin = choice_margin(row)
58
+ escalated = margin <= threshold
59
+ if escalated:
60
+ selected.append(uid)
61
+ audit_rows.append(
62
+ {
63
+ "uid": uid,
64
+ "margin": margin,
65
+ "threshold": threshold,
66
+ "escalated": escalated,
67
+ }
68
+ )
69
+ if len(audit_rows) != int(recent.get("num_samples", -1)):
70
+ raise ValueError("recent result sample count diverged")
71
+ if int(recent.get("num_errors", -1)) != 0:
72
+ raise ValueError("recent result contains inference errors")
73
+ if len({row["uid"] for row in audit_rows}) != len(audit_rows):
74
+ raise ValueError("recent result contains duplicate uids")
75
+ return selected, audit_rows
76
+
77
+
78
+ def select_rows_by_quantile(
79
+ recent: dict[str, Any], *, quantile: float
80
+ ) -> tuple[list[str], list[dict[str, Any]], float]:
81
+ """Select an exact bottom-margin budget with deterministic UID tie-breaking."""
82
+
83
+ if not 0.0 < quantile <= 1.0:
84
+ raise ValueError("quantile must be in (0, 1]")
85
+ rows = list(recent.get("rows") or [])
86
+ if len(rows) != int(recent.get("num_samples", -1)):
87
+ raise ValueError("recent result sample count diverged")
88
+ if int(recent.get("num_errors", -1)) != 0:
89
+ raise ValueError("recent result contains inference errors")
90
+ values = []
91
+ for row in rows:
92
+ uid = str(row.get("uid") or "")
93
+ if not uid:
94
+ raise ValueError("recent result has a row without uid")
95
+ values.append((choice_margin(row), uid))
96
+ if len({uid for _, uid in values}) != len(values):
97
+ raise ValueError("recent result contains duplicate uids")
98
+ budget = max(1, math.ceil(len(values) * quantile))
99
+ ranked = sorted(values, key=lambda item: (item[0], item[1]))
100
+ selected_set = {uid for _, uid in ranked[:budget]}
101
+ cutoff = float(ranked[budget - 1][0])
102
+ selected = [str(row["uid"]) for row in rows if str(row["uid"]) in selected_set]
103
+ rank_by_uid = {uid: rank for rank, (_, uid) in enumerate(ranked, 1)}
104
+ audit_rows = [
105
+ {
106
+ "uid": str(row["uid"]),
107
+ "margin": choice_margin(row),
108
+ "uncertainty_rank": rank_by_uid[str(row["uid"])],
109
+ "quantile": quantile,
110
+ "escalated": str(row["uid"]) in selected_set,
111
+ }
112
+ for row in rows
113
+ ]
114
+ return selected, audit_rows, cutoff
115
+
116
+
117
+ def main() -> None:
118
+ args = parse_args()
119
+ recent = json.loads(args.recent.read_text(encoding="utf-8"))
120
+ threshold = None
121
+ quantile = None
122
+ cutoff = None
123
+ if args.threshold is not None:
124
+ threshold = load_frozen_threshold(
125
+ args.threshold, recent.get("model_manifest_sha256")
126
+ )
127
+ selected, audit_rows = select_rows(recent, threshold=threshold)
128
+ else:
129
+ quantile = float(args.quantile)
130
+ selected, audit_rows, cutoff = select_rows_by_quantile(
131
+ recent, quantile=quantile
132
+ )
133
+ selected_set = set(selected)
134
+ sample_rows = [
135
+ json.loads(line)
136
+ for line in args.samples.read_text(encoding="utf-8").splitlines()
137
+ if line.strip()
138
+ ]
139
+ ordered_uids = [str(row.get("uid") or "") for row in sample_rows]
140
+ recent_uids = [str(row["uid"]) for row in recent["rows"]]
141
+ if ordered_uids != recent_uids:
142
+ raise ValueError("recent result does not follow the frozen sample order")
143
+ selected_samples = [row for row in sample_rows if str(row["uid"]) in selected_set]
144
+ if [str(row["uid"]) for row in selected_samples] != selected:
145
+ raise ValueError("selected UID ordering diverged")
146
+ args.output_uids.parent.mkdir(parents=True, exist_ok=True)
147
+ args.output_uids.write_text("\n".join(selected) + "\n", encoding="utf-8")
148
+ args.output_samples.write_text(
149
+ "".join(
150
+ json.dumps(row, sort_keys=True, ensure_ascii=False) + "\n"
151
+ for row in selected_samples
152
+ ),
153
+ encoding="utf-8",
154
+ )
155
+ atomic_write(
156
+ args.output_audit,
157
+ {
158
+ "format": (
159
+ "streameva.internal_threshold_uid_selection.v0"
160
+ if threshold is not None
161
+ else "streameva.unlabeled_budget_uid_selection.v0"
162
+ ),
163
+ "status": "frozen_before_benchmark_temporal_inference",
164
+ "benchmark": recent.get("benchmark"),
165
+ "model_manifest_sha256": recent.get("model_manifest_sha256"),
166
+ "threshold": threshold,
167
+ "quantile": quantile,
168
+ "empirical_margin_cutoff": cutoff,
169
+ "threshold_audit": (
170
+ str(args.threshold.resolve()) if args.threshold is not None else None
171
+ ),
172
+ "threshold_audit_sha256": (
173
+ sha256_file(args.threshold) if args.threshold is not None else None
174
+ ),
175
+ "recent_result": str(args.recent.resolve()),
176
+ "recent_result_sha256": sha256_file(args.recent),
177
+ "samples_sha256": sha256_file(args.samples),
178
+ "question_task_group_or_benchmark_router": False,
179
+ "score_or_gold_fields_read_for_selection": False,
180
+ "benchmark_identity_used_for_policy_parameters": False,
181
+ "cohort_labels_read_for_selection": False,
182
+ "total": len(audit_rows),
183
+ "selected": len(selected),
184
+ "selection_rate": len(selected) / len(audit_rows),
185
+ "rows": audit_rows,
186
+ },
187
+ )
188
+ print(json.dumps({"total": len(audit_rows), "selected": len(selected)}))
189
+
190
+
191
+ if __name__ == "__main__":
192
+ main()
code/experiments/train_qwen3vl_sft.py ADDED
@@ -0,0 +1,1091 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """Audited full-parameter Qwen3-VL trainer for the pure-VLM backbone."""
3
+
4
+ from __future__ import annotations
5
+
6
+ import argparse
7
+ import inspect
8
+ import json
9
+ import math
10
+ import os
11
+ import shutil
12
+ from dataclasses import dataclass
13
+ from pathlib import Path
14
+ from typing import Any, Dict, List
15
+
16
+ import torch
17
+ from PIL import Image
18
+ from torch.utils.data import Dataset, SequentialSampler
19
+
20
+ try:
21
+ from transformers import (
22
+ AutoModelForImageTextToText,
23
+ AutoProcessor,
24
+ Trainer,
25
+ TrainingArguments,
26
+ set_seed,
27
+ )
28
+ except ModuleNotFoundError:
29
+ AutoModelForImageTextToText = None
30
+ AutoProcessor = None
31
+ TrainingArguments = None
32
+ set_seed = None
33
+
34
+ class Trainer: # type: ignore[no-redef]
35
+ """Import-only fallback for dependency-light contract tests."""
36
+
37
+ pass
38
+
39
+ from viss_core.data_v0.trainer_adapter import (
40
+ infer_release_root,
41
+ load_training_rows,
42
+ normalize_messages,
43
+ )
44
+ class MediaSemanticDataset(Dataset):
45
+ """Load only a sealed pure-semantic JSONL release."""
46
+
47
+ def __init__(
48
+ self,
49
+ path: Path,
50
+ processor: Any,
51
+ *,
52
+ max_samples: int = 0,
53
+ media_root: Path | None = None,
54
+ development_unsealed_overlay: bool = False,
55
+ ) -> None:
56
+ self.development_unsealed_overlay = development_unsealed_overlay
57
+ if development_unsealed_overlay:
58
+ self.rows = [
59
+ json.loads(line)
60
+ for line in path.read_text(encoding="utf-8").splitlines()
61
+ if line.strip()
62
+ ]
63
+ else:
64
+ self.rows = _load_training_rows(path)
65
+ if max_samples > 0:
66
+ self.rows = self.rows[:max_samples]
67
+ self.processor = processor
68
+ inferred_root = _infer_release_root(path)
69
+ if development_unsealed_overlay:
70
+ if media_root is None:
71
+ raise RuntimeError("development overlay requires --media-root")
72
+ self.media_root = media_root.resolve(strict=True)
73
+ else:
74
+ if inferred_root is None:
75
+ raise RuntimeError("semantic release root could not be inferred")
76
+ if media_root is not None and media_root.resolve() != inferred_root:
77
+ raise RuntimeError("--media-root may not override sealed release media")
78
+ self.media_root = inferred_root
79
+
80
+ def __len__(self) -> int:
81
+ return len(self.rows)
82
+
83
+ def __getitem__(self, index: int) -> Dict[str, Any]:
84
+ row = self.rows[index]
85
+ raw_weight = row.get("sample_loss_weight")
86
+ sample_loss_weight = float(1.0 if raw_weight is None else raw_weight)
87
+ if not math.isfinite(sample_loss_weight) or sample_loss_weight <= 0:
88
+ raise RuntimeError(
89
+ "train-ready contract violated: "
90
+ f"uid={row.get('uid') or f'row_{index}'} "
91
+ f"sample_loss_weight={sample_loss_weight}"
92
+ )
93
+ return {
94
+ "uid": str(row.get("uid") or f"row_{index}"),
95
+ "messages": (
96
+ _normalize_development_messages(row["messages"])
97
+ if self.development_unsealed_overlay
98
+ else _normalize_messages(row["messages"], media_root=self.media_root)
99
+ ),
100
+ "sample_loss_weight": sample_loss_weight,
101
+ }
102
+
103
+
104
+ @dataclass
105
+ class MediaSemanticCollator:
106
+ processor: Any
107
+ max_length: int
108
+
109
+ def __call__(self, features: List[Dict[str, Any]]) -> Dict[str, torch.Tensor]:
110
+ encoded = []
111
+ labels_list = []
112
+ weights_list = []
113
+ for feature in features:
114
+ messages = feature["messages"]
115
+ item = _encode_processor_item(self.processor, messages)
116
+ if item["input_ids"].numel() > self.max_length:
117
+ raise RuntimeError(
118
+ f"train-ready contract violated: uid={feature['uid']} "
119
+ f"tokens={item['input_ids'].numel()} max_length={self.max_length}"
120
+ )
121
+ labels, loss_weights = _assistant_only_labels_and_weights(
122
+ getattr(self.processor, "tokenizer", self.processor),
123
+ item["input_ids"],
124
+ messages,
125
+ )
126
+ if not torch.any(labels != -100):
127
+ raise RuntimeError(
128
+ f"train-ready contract violated: uid={feature['uid']} "
129
+ "has no assistant labels"
130
+ )
131
+ encoded.append(item)
132
+ labels_list.append(labels)
133
+ weights_list.append(loss_weights)
134
+
135
+ tokenizer = getattr(self.processor, "tokenizer", self.processor)
136
+ if tokenizer.pad_token_id is None:
137
+ raise RuntimeError("tokenizer has no pad token")
138
+ maximum = max(item["input_ids"].numel() for item in encoded)
139
+ batch: Dict[str, list[torch.Tensor]] = {
140
+ "input_ids": [],
141
+ "attention_mask": [],
142
+ "labels": [],
143
+ "loss_weights": [],
144
+ "sample_loss_weight": [],
145
+ }
146
+ pixel_values = []
147
+ image_grid_thw = []
148
+ for item, labels, loss_weights, feature in zip(
149
+ encoded, labels_list, weights_list, features, strict=True
150
+ ):
151
+ pad = maximum - item["input_ids"].numel()
152
+ batch["input_ids"].append(
153
+ torch.nn.functional.pad(
154
+ item["input_ids"], (0, pad), value=tokenizer.pad_token_id
155
+ )
156
+ )
157
+ batch["attention_mask"].append(
158
+ torch.nn.functional.pad(item["attention_mask"], (0, pad), value=0)
159
+ )
160
+ batch["labels"].append(
161
+ torch.nn.functional.pad(labels, (0, pad), value=-100)
162
+ )
163
+ batch["loss_weights"].append(
164
+ torch.nn.functional.pad(loss_weights, (0, pad), value=0.0)
165
+ )
166
+ batch["sample_loss_weight"].append(
167
+ torch.tensor(
168
+ float(feature["sample_loss_weight"]), dtype=torch.float32
169
+ )
170
+ )
171
+ if "pixel_values" in item:
172
+ pixel_values.append(item["pixel_values"])
173
+ if "image_grid_thw" in item:
174
+ image_grid_thw.append(item["image_grid_thw"])
175
+ output = {key: torch.stack(value) for key, value in batch.items()}
176
+ if pixel_values:
177
+ output["pixel_values"] = torch.cat(pixel_values, dim=0)
178
+ if image_grid_thw:
179
+ output["image_grid_thw"] = torch.cat(image_grid_thw, dim=0)
180
+ return output
181
+
182
+
183
+ def main() -> None:
184
+ if any(
185
+ value is None
186
+ for value in (
187
+ AutoModelForImageTextToText,
188
+ AutoProcessor,
189
+ TrainingArguments,
190
+ set_seed,
191
+ )
192
+ ):
193
+ raise RuntimeError(
194
+ "training requires transformers; the dependency-light import path "
195
+ "supports only data and loss-mask contract tests"
196
+ )
197
+ parser = argparse.ArgumentParser(
198
+ description="Full-parameter Qwen3-VL pure-semantic SFT."
199
+ )
200
+ parser.add_argument("--model", type=Path, required=True)
201
+ parser.add_argument("--train", type=Path, required=True)
202
+ parser.add_argument("--val", type=Path)
203
+ parser.add_argument("--media-root", type=Path)
204
+ parser.add_argument("--val-media-root", type=Path)
205
+ parser.add_argument(
206
+ "--development-unsealed-overlay",
207
+ action="store_true",
208
+ help="Development only: read an unsealed JSONL overlay with absolute media paths.",
209
+ )
210
+ parser.add_argument("--output", type=Path, required=True)
211
+ parser.add_argument("--max-length", type=int, default=8192)
212
+ parser.add_argument("--max-pixels", type=int, default=0)
213
+ parser.add_argument("--per-device-batch-size", type=int, default=1)
214
+ parser.add_argument("--gradient-accumulation-steps", type=int, default=8)
215
+ parser.add_argument("--epochs", type=float, default=1.0)
216
+ parser.add_argument("--lr", type=float, default=1e-4)
217
+ parser.add_argument(
218
+ "--vision-projector-lr-scale",
219
+ type=float,
220
+ default=1.0,
221
+ help=(
222
+ "Multiply the optimizer learning rate for trainable visual "
223
+ "merger/deepstack projector parameters."
224
+ ),
225
+ )
226
+ parser.add_argument(
227
+ "--lr-scheduler-type",
228
+ choices=("cosine", "constant_with_warmup"),
229
+ default="cosine",
230
+ )
231
+ parser.add_argument(
232
+ "--train-order", choices=("shuffled", "sequential"), default="shuffled"
233
+ )
234
+ parser.add_argument(
235
+ "--stage-lr-plan",
236
+ type=Path,
237
+ help="Audited row-boundary learning-rate plan for one ordered run.",
238
+ )
239
+ parser.add_argument("--warmup-ratio", type=float, default=0.03)
240
+ parser.add_argument("--deepspeed", type=Path, required=True)
241
+ parser.add_argument("--freeze-vision-tower", action="store_true")
242
+ parser.add_argument("--train-vision-projector", action="store_true")
243
+ parser.add_argument("--logging-steps", type=int, default=10)
244
+ parser.add_argument("--save-steps", type=int, default=250)
245
+ parser.add_argument("--save-total-limit", type=int, default=2)
246
+ parser.add_argument("--skip-final-save", action="store_true")
247
+ parser.add_argument("--max-train-samples", type=int, default=0)
248
+ parser.add_argument("--max-val-samples", type=int, default=0)
249
+ parser.add_argument("--max-steps", type=int, default=-1)
250
+ parser.add_argument("--seed", type=int, default=42)
251
+ parser.add_argument("--attn-implementation", default="flash_attention_2")
252
+ parser.add_argument("--dataloader-num-workers", type=int, default=0)
253
+ args = parser.parse_args()
254
+
255
+ if args.max_length <= 0 or args.max_pixels < 0:
256
+ raise ValueError("length must be positive and max-pixels non-negative")
257
+ if (
258
+ not math.isfinite(args.vision_projector_lr_scale)
259
+ or args.vision_projector_lr_scale <= 0
260
+ ):
261
+ raise ValueError("vision-projector-lr-scale must be finite and positive")
262
+ if args.train_vision_projector and not args.freeze_vision_tower:
263
+ raise ValueError("--train-vision-projector requires --freeze-vision-tower")
264
+ if args.vision_projector_lr_scale != 1.0 and not args.train_vision_projector:
265
+ raise ValueError(
266
+ "a non-default vision-projector LR scale requires a trainable projector"
267
+ )
268
+ if not args.deepspeed.is_file():
269
+ raise FileNotFoundError(args.deepspeed)
270
+
271
+ set_seed(args.seed)
272
+ has_eval = args.val is not None and args.val.exists()
273
+ training_args = _training_args(args, has_eval)
274
+ processor = AutoProcessor.from_pretrained(args.model, trust_remote_code=True)
275
+ _configure_processor_pixel_budget(processor, args.max_pixels)
276
+ tokenizer = getattr(processor, "tokenizer", processor)
277
+ if tokenizer.pad_token_id is None:
278
+ tokenizer.pad_token = tokenizer.eos_token
279
+ model = AutoModelForImageTextToText.from_pretrained(
280
+ args.model,
281
+ dtype=torch.bfloat16,
282
+ trust_remote_code=True,
283
+ attn_implementation=args.attn_implementation,
284
+ )
285
+ model.config.use_cache = False
286
+ model.gradient_checkpointing_enable()
287
+ if args.freeze_vision_tower:
288
+ model.visual.requires_grad_(False)
289
+ if args.train_vision_projector:
290
+ for name in ("merger", "deepstack_merger_list"):
291
+ module = getattr(model.visual, name, None)
292
+ if module is None:
293
+ raise RuntimeError(
294
+ f"Qwen3-VL visual module lacks projector component {name!r}"
295
+ )
296
+ module.requires_grad_(True)
297
+ _audit_trainable_parameters(model, args)
298
+
299
+ if _is_main_process():
300
+ trainable = sum(
301
+ _logical_parameter_numel(parameter)
302
+ for parameter in model.parameters()
303
+ if parameter.requires_grad
304
+ )
305
+ total = sum(
306
+ _logical_parameter_numel(parameter) for parameter in model.parameters()
307
+ )
308
+ if total <= 0:
309
+ raise RuntimeError("could not determine logical parameter count")
310
+ print(
311
+ {
312
+ "training_mode": "full_parameter_no_adapter",
313
+ "vision_tower_frozen": args.freeze_vision_tower,
314
+ "trainable_parameters": trainable,
315
+ "total_parameters": total,
316
+ "trainable_fraction": trainable / total,
317
+ },
318
+ flush=True,
319
+ )
320
+
321
+ train_dataset = MediaSemanticDataset(
322
+ args.train,
323
+ processor,
324
+ max_samples=args.max_train_samples,
325
+ media_root=args.media_root,
326
+ development_unsealed_overlay=args.development_unsealed_overlay,
327
+ )
328
+ global_batch = (
329
+ args.per_device_batch_size
330
+ * args.gradient_accumulation_steps
331
+ * int(os.environ.get("WORLD_SIZE", "1"))
332
+ )
333
+ stage_lr_plan = (
334
+ _load_stage_lr_plan(
335
+ args.stage_lr_plan,
336
+ release_manifest=inferred_root_manifest(train_dataset.media_root),
337
+ dataset_rows=len(train_dataset),
338
+ global_batch=global_batch,
339
+ optimizer_learning_rate=args.lr,
340
+ warmup_ratio=args.warmup_ratio,
341
+ )
342
+ if args.stage_lr_plan is not None
343
+ else None
344
+ )
345
+ eval_dataset = (
346
+ MediaSemanticDataset(
347
+ args.val,
348
+ processor,
349
+ max_samples=args.max_val_samples,
350
+ media_root=args.val_media_root,
351
+ )
352
+ if has_eval
353
+ else None
354
+ )
355
+ trainer = SemanticTokenTrainer(
356
+ model=model,
357
+ args=training_args,
358
+ train_dataset=train_dataset,
359
+ eval_dataset=eval_dataset,
360
+ data_collator=MediaSemanticCollator(processor, args.max_length),
361
+ train_order=args.train_order,
362
+ stage_lr_plan=stage_lr_plan,
363
+ vision_projector_lr_scale=args.vision_projector_lr_scale,
364
+ vision_projector_parameter_ids={
365
+ id(parameter)
366
+ for name in ("merger", "deepstack_merger_list")
367
+ for parameter in getattr(model.visual, name).parameters()
368
+ }
369
+ if args.train_vision_projector
370
+ else set(),
371
+ )
372
+ trainer.train()
373
+ if args.skip_final_save:
374
+ if _is_main_process():
375
+ print(
376
+ {
377
+ "output": str(args.output),
378
+ "training_mode": "full_parameter_no_adapter",
379
+ "final_model": None,
380
+ "systems_smoke_skipped_final_save": True,
381
+ },
382
+ flush=True,
383
+ )
384
+ return
385
+ _release_save_reserve()
386
+ final_dir = args.output / "final_model"
387
+ trainer.save_model(final_dir)
388
+ processor.save_pretrained(final_dir)
389
+ if _is_main_process():
390
+ print(
391
+ {
392
+ "output": str(args.output),
393
+ "training_mode": "full_parameter_no_adapter",
394
+ "final_model": str(final_dir),
395
+ },
396
+ flush=True,
397
+ )
398
+
399
+
400
+ def _release_save_reserve() -> None:
401
+ raw = os.environ.get("STREAMEVA_SAVE_RESERVE_PATH")
402
+ if raw and _is_main_process():
403
+ path = Path(raw).resolve()
404
+ if (
405
+ path.name != "save_reserve.bin"
406
+ or path.parent.parent != Path("/tmp")
407
+ or not path.parent.name.startswith("streameva_")
408
+ ):
409
+ raise RuntimeError("save reserve path is outside the short runtime root")
410
+ path.unlink(missing_ok=True)
411
+ os.sync()
412
+ print(
413
+ {
414
+ "save_reserve_released": str(path),
415
+ "free_bytes": shutil.disk_usage(path.parent).free,
416
+ },
417
+ flush=True,
418
+ )
419
+ if torch.distributed.is_available() and torch.distributed.is_initialized():
420
+ torch.distributed.barrier()
421
+
422
+
423
+ def _audit_trainable_parameters(model: Any, args: argparse.Namespace) -> None:
424
+ if not all(
425
+ parameter.requires_grad for parameter in model.language_model.parameters()
426
+ ):
427
+ raise RuntimeError("full-parameter contract froze a language parameter")
428
+ if not all(parameter.requires_grad for parameter in model.lm_head.parameters()):
429
+ raise RuntimeError("full-parameter contract froze an LM-head parameter")
430
+ if not args.freeze_vision_tower:
431
+ return
432
+ projector_ids: set[int] = set()
433
+ if args.train_vision_projector:
434
+ for name in ("merger", "deepstack_merger_list"):
435
+ projector_ids.update(
436
+ id(parameter)
437
+ for parameter in getattr(model.visual, name).parameters()
438
+ )
439
+ for parameter in model.visual.parameters():
440
+ if parameter.requires_grad != (id(parameter) in projector_ids):
441
+ raise RuntimeError("frozen-vision/trainable-projector contract violated")
442
+
443
+
444
+ def _load_training_rows(
445
+ path: Path, *, training_schema: str = "semantic"
446
+ ) -> List[Dict[str, Any]]:
447
+ if training_schema != "semantic":
448
+ raise ValueError(f"unsupported training schema: {training_schema}")
449
+ return load_training_rows(path)
450
+
451
+
452
+ def _normalize_development_messages(messages: Any) -> List[Dict[str, Any]]:
453
+ """Normalize an explicit development overlay without weakening release loading."""
454
+ if not isinstance(messages, list):
455
+ raise TypeError("development messages must be a list")
456
+ roles = [message.get("role") for message in messages]
457
+ if roles != ["user", "assistant"]:
458
+ raise ValueError("development overlay accepts pure-VLM rows only")
459
+ output: List[Dict[str, Any]] = []
460
+ for message in messages:
461
+ content: List[Dict[str, Any]] = []
462
+ for item in message.get("content") or []:
463
+ if item.get("type") == "text":
464
+ content.append({"type": "text", "text": str(item.get("text") or "")})
465
+ continue
466
+ if item.get("type") != "image" or message["role"] != "user":
467
+ raise ValueError("development overlay contains unsupported content")
468
+ image_path = Path(str(item.get("image") or ""))
469
+ if not image_path.is_absolute() or not image_path.is_file():
470
+ raise RuntimeError("development overlay image must be an existing absolute path")
471
+ content.append(
472
+ {"type": "image", "image": Image.open(image_path).convert("RGB")}
473
+ )
474
+ if not content:
475
+ raise ValueError("development overlay message content is empty")
476
+ output.append(
477
+ {
478
+ "role": str(message["role"]),
479
+ "content": content,
480
+ "loss": bool(message.get("loss")),
481
+ }
482
+ )
483
+ return output
484
+
485
+
486
+ def _infer_release_root(
487
+ path: Path, *, training_schema: str = "semantic"
488
+ ) -> Path | None:
489
+ if training_schema != "semantic":
490
+ raise ValueError(f"unsupported training schema: {training_schema}")
491
+ return infer_release_root(path)
492
+
493
+
494
+ def _normalize_messages(
495
+ messages: Any,
496
+ *,
497
+ media_root: Path | None = None,
498
+ training_schema: str = "semantic",
499
+ ) -> List[Dict[str, Any]]:
500
+ if training_schema != "semantic":
501
+ raise ValueError(f"unsupported training schema: {training_schema}")
502
+ return normalize_messages(messages, media_root=media_root)
503
+
504
+
505
+ def _load_images(messages: List[Dict[str, Any]]) -> List[Image.Image]:
506
+ images = []
507
+ for message in messages:
508
+ for item in message.get("content", []):
509
+ if not isinstance(item, dict) or item.get("type") != "image":
510
+ continue
511
+ image = item.get("image")
512
+ if isinstance(image, Image.Image):
513
+ images.append(image)
514
+ elif image:
515
+ images.append(Image.open(image).convert("RGB"))
516
+ return images
517
+
518
+
519
+ def _configure_processor_pixel_budget(
520
+ processor: Any, max_pixels: int
521
+ ) -> Dict[str, int]:
522
+ image_processor = getattr(processor, "image_processor", None)
523
+ size = getattr(image_processor, "size", None)
524
+ if image_processor is None or not isinstance(size, dict):
525
+ raise RuntimeError("processor has no configurable image size contract")
526
+ shortest = int(size.get("shortest_edge", 0))
527
+ longest = int(size.get("longest_edge", 0))
528
+ if shortest <= 0 or longest <= 0 or shortest > longest:
529
+ raise RuntimeError("processor image size contract is invalid")
530
+ requested = longest if max_pixels == 0 else int(max_pixels)
531
+ if requested < shortest:
532
+ raise ValueError(f"max_pixels={requested} is below shortest_edge={shortest}")
533
+ image_processor.size = {
534
+ "shortest_edge": shortest,
535
+ "longest_edge": requested,
536
+ }
537
+ return dict(image_processor.size)
538
+
539
+
540
+ def _encode_processor_item(
541
+ processor: Any, messages: List[Dict[str, Any]]
542
+ ) -> Dict[str, torch.Tensor]:
543
+ rendered = processor.apply_chat_template(
544
+ messages,
545
+ tokenize=False,
546
+ add_generation_prompt=False,
547
+ )
548
+ images = _load_images(messages)
549
+ if rendered.count("<|image_pad|>") != len(images):
550
+ raise RuntimeError("chat-template image slots and message images differ")
551
+ kwargs: Dict[str, Any] = {
552
+ "text": rendered,
553
+ "return_dict": True,
554
+ "return_tensors": "pt",
555
+ }
556
+ if images:
557
+ kwargs["images"] = images
558
+ item = processor(**kwargs)
559
+ return {
560
+ key: (
561
+ value.squeeze(0)
562
+ if hasattr(value, "dim")
563
+ and value.dim() > 1
564
+ and key in {"input_ids", "attention_mask"}
565
+ else value
566
+ )
567
+ for key, value in item.items()
568
+ }
569
+
570
+
571
+ def _locate_rendered_text_token_positions(
572
+ tokenizer: Any,
573
+ input_ids: torch.Tensor,
574
+ messages: List[Dict[str, Any]],
575
+ ) -> Dict[tuple[int, int], List[int]] | None:
576
+ """Align message text spans through chat rendering and visual expansion."""
577
+
578
+ if not hasattr(tokenizer, "apply_chat_template"):
579
+ return None
580
+ try:
581
+ rendered = tokenizer.apply_chat_template(
582
+ messages,
583
+ tokenize=False,
584
+ add_generation_prompt=False,
585
+ )
586
+ encoded = tokenizer(
587
+ rendered,
588
+ add_special_tokens=False,
589
+ return_offsets_mapping=True,
590
+ )
591
+ except (KeyError, NotImplementedError, TypeError, ValueError):
592
+ return None
593
+ template_ids = encoded.get("input_ids")
594
+ offsets = encoded.get("offset_mapping")
595
+ if template_ids is None or offsets is None:
596
+ return None
597
+ if hasattr(template_ids, "tolist"):
598
+ template_ids = template_ids.tolist()
599
+ if hasattr(offsets, "tolist"):
600
+ offsets = offsets.tolist()
601
+ if template_ids and isinstance(template_ids[0], list):
602
+ template_ids = template_ids[0]
603
+ if offsets and isinstance(offsets[0][0], list):
604
+ offsets = offsets[0]
605
+
606
+ actual_ids = input_ids.tolist()
607
+ visual_pad_ids = {
608
+ token_id
609
+ for token in ("<|image_pad|>", "<|video_pad|>")
610
+ for token_id in [tokenizer.convert_tokens_to_ids(token)]
611
+ if isinstance(token_id, int) and token_id >= 0
612
+ }
613
+ template_to_actual: Dict[int, int] = {}
614
+ actual_index = 0
615
+ for template_index, token_id in enumerate(template_ids):
616
+ if actual_index >= len(actual_ids) or actual_ids[actual_index] != token_id:
617
+ raise RuntimeError("rendered template and processor input IDs diverged")
618
+ template_to_actual[template_index] = actual_index
619
+ actual_index += 1
620
+ if token_id in visual_pad_ids:
621
+ while actual_index < len(actual_ids) and actual_ids[actual_index] == token_id:
622
+ actual_index += 1
623
+ if actual_index != len(actual_ids):
624
+ raise RuntimeError("processor input IDs contain non-template tokens")
625
+
626
+ output: Dict[tuple[int, int], List[int]] = {}
627
+ character_cursor = 0
628
+ for message_index, message in enumerate(messages):
629
+ text_items = [
630
+ item
631
+ for item in message.get("content", [])
632
+ if isinstance(item, dict) and item.get("type") == "text"
633
+ ]
634
+ for text_index, item in enumerate(text_items):
635
+ text = str(item.get("text", ""))
636
+ start = rendered.find(text, character_cursor)
637
+ if start < 0:
638
+ raise RuntimeError("message text is absent from the rendered template")
639
+ end = start + len(text)
640
+ positions = [
641
+ template_to_actual[index]
642
+ for index, pair in enumerate(offsets)
643
+ if int(pair[0]) < end and int(pair[1]) > start
644
+ ]
645
+ if text and not positions:
646
+ raise RuntimeError("rendered message text has no token positions")
647
+ output[(message_index, text_index)] = positions
648
+ character_cursor = end
649
+ return output
650
+
651
+
652
+ def _assistant_only_labels(
653
+ tokenizer: Any,
654
+ input_ids: torch.Tensor,
655
+ messages: List[Dict[str, Any]],
656
+ ) -> torch.Tensor:
657
+ labels, _ = _assistant_only_labels_and_weights(tokenizer, input_ids, messages)
658
+ return labels
659
+
660
+
661
+ def _assistant_only_labels_and_weights(
662
+ tokenizer: Any,
663
+ input_ids: torch.Tensor,
664
+ messages: List[Dict[str, Any]],
665
+ ) -> tuple[torch.Tensor, torch.Tensor]:
666
+ """Supervise assistant semantic text and its turn terminator only."""
667
+
668
+ labels = torch.full_like(input_ids, -100)
669
+ weights = torch.zeros_like(input_ids, dtype=torch.float32)
670
+ ids = input_ids.tolist()
671
+ cursor = 0
672
+ rendered_positions = _locate_rendered_text_token_positions(
673
+ tokenizer, input_ids, messages
674
+ )
675
+ for message_index, message in enumerate(messages):
676
+ is_assistant = message.get("role") == "assistant"
677
+ supervise = is_assistant and message.get("loss", True) is not False
678
+ text_items = [
679
+ item
680
+ for item in message.get("content", [])
681
+ if isinstance(item, dict) and item.get("type") == "text"
682
+ ]
683
+ for text_index, item in enumerate(text_items):
684
+ text = str(item.get("text", ""))
685
+ if rendered_positions is not None:
686
+ positions = rendered_positions[(message_index, text_index)]
687
+ start = positions[0] if positions else cursor
688
+ semantic_end = positions[-1] + 1 if positions else cursor
689
+ else:
690
+ target = tokenizer(text, add_special_tokens=False)["input_ids"]
691
+ start = _find_subsequence(ids, target, cursor)
692
+ if start < 0:
693
+ raise RuntimeError("could not locate message text in role order")
694
+ semantic_end = min(start + len(target), labels.numel())
695
+ end = semantic_end
696
+ im_end_id = tokenizer.convert_tokens_to_ids("<|im_end|>")
697
+ if (
698
+ is_assistant
699
+ and text_index == len(text_items) - 1
700
+ and im_end_id is not None
701
+ and end < labels.numel()
702
+ and ids[end] == im_end_id
703
+ ):
704
+ end += 1
705
+ cursor = end
706
+ if supervise:
707
+ labels[start:end] = input_ids[start:end]
708
+ weights[start:end] = 1.0
709
+ return labels, weights
710
+
711
+
712
+ def _find_subsequence(haystack: List[int], needle: List[int], start: int) -> int:
713
+ if not needle:
714
+ return -1
715
+ for index in range(max(start, 0), len(haystack) - len(needle) + 1):
716
+ if haystack[index : index + len(needle)] == needle:
717
+ return index
718
+ return -1
719
+
720
+
721
+ def _split_optimizer_groups_for_projector(
722
+ param_groups: list[dict[str, Any]],
723
+ *,
724
+ projector_parameter_ids: set[int],
725
+ learning_rate_scale: float,
726
+ ) -> tuple[list[dict[str, Any]], int]:
727
+ """Preserve group options while isolating/scaling projector parameters."""
728
+
729
+ replacement_groups: list[dict[str, Any]] = []
730
+ projector_parameters = 0
731
+ for original in param_groups:
732
+ projector = [
733
+ parameter
734
+ for parameter in original["params"]
735
+ if id(parameter) in projector_parameter_ids
736
+ ]
737
+ backbone = [
738
+ parameter
739
+ for parameter in original["params"]
740
+ if id(parameter) not in projector_parameter_ids
741
+ ]
742
+ if backbone:
743
+ group = dict(original)
744
+ group["params"] = backbone
745
+ replacement_groups.append(group)
746
+ if projector:
747
+ group = dict(original)
748
+ group["params"] = projector
749
+ group["lr"] = float(group["lr"]) * learning_rate_scale
750
+ if "initial_lr" in group:
751
+ group["initial_lr"] = (
752
+ float(group["initial_lr"]) * learning_rate_scale
753
+ )
754
+ replacement_groups.append(group)
755
+ projector_parameters += len(projector)
756
+ return replacement_groups, projector_parameters
757
+
758
+
759
+ class SemanticTokenTrainer(Trainer):
760
+ def __init__(
761
+ self,
762
+ *args: Any,
763
+ train_order: str = "shuffled",
764
+ stage_lr_plan: dict[str, Any] | None = None,
765
+ vision_projector_lr_scale: float = 1.0,
766
+ vision_projector_parameter_ids: set[int] | None = None,
767
+ **kwargs: Any,
768
+ ) -> None:
769
+ if train_order not in {"shuffled", "sequential"}:
770
+ raise ValueError("unsupported train order")
771
+ if stage_lr_plan is not None and train_order != "sequential":
772
+ raise ValueError("stage learning-rate plan requires sequential stage order")
773
+ self.train_order = train_order
774
+ self.stage_lr_plan = stage_lr_plan
775
+ self.vision_projector_lr_scale = float(vision_projector_lr_scale)
776
+ self.vision_projector_parameter_ids = set(
777
+ vision_projector_parameter_ids or set()
778
+ )
779
+ self._projector_lr_groups_applied = False
780
+ if (
781
+ self.vision_projector_lr_scale != 1.0
782
+ and not self.vision_projector_parameter_ids
783
+ ):
784
+ raise ValueError("projector LR scaling requires projector parameters")
785
+ super().__init__(*args, **kwargs)
786
+
787
+ def create_optimizer(self) -> Any:
788
+ """Split Trainer's established decay groups and scale projector LR only."""
789
+
790
+ optimizer = super().create_optimizer()
791
+ if self.vision_projector_lr_scale == 1.0:
792
+ return optimizer
793
+ if self._projector_lr_groups_applied:
794
+ return optimizer
795
+ if optimizer is None:
796
+ raise RuntimeError("optimizer is absent while applying projector LR")
797
+
798
+ replacement_groups, projector_parameters = (
799
+ _split_optimizer_groups_for_projector(
800
+ list(optimizer.param_groups),
801
+ projector_parameter_ids=self.vision_projector_parameter_ids,
802
+ learning_rate_scale=self.vision_projector_lr_scale,
803
+ )
804
+ )
805
+ if projector_parameters != len(self.vision_projector_parameter_ids):
806
+ raise RuntimeError(
807
+ "optimizer projector parameter coverage diverged: "
808
+ f"{projector_parameters} != "
809
+ f"{len(self.vision_projector_parameter_ids)}"
810
+ )
811
+ optimizer.param_groups[:] = replacement_groups
812
+ self._projector_lr_groups_applied = True
813
+ if _is_main_process():
814
+ print(
815
+ {
816
+ "optimizer_lr_policy": "split_visual_projector",
817
+ "base_learning_rate": self.args.learning_rate,
818
+ "vision_projector_lr_scale": self.vision_projector_lr_scale,
819
+ "vision_projector_learning_rate": (
820
+ self.args.learning_rate
821
+ * self.vision_projector_lr_scale
822
+ ),
823
+ "projector_parameter_tensors": projector_parameters,
824
+ "optimizer_parameter_groups": len(replacement_groups),
825
+ },
826
+ flush=True,
827
+ )
828
+ return optimizer
829
+
830
+ def create_scheduler(
831
+ self, num_training_steps: int, optimizer: Any = None
832
+ ) -> Any:
833
+ if self.stage_lr_plan is None:
834
+ try:
835
+ return super().create_scheduler(num_training_steps, optimizer)
836
+ except TypeError:
837
+ return super().create_scheduler(num_training_steps)
838
+ if num_training_steps != int(self.stage_lr_plan["total_steps"]):
839
+ raise RuntimeError(
840
+ "stage LR plan and Trainer optimization steps diverged: "
841
+ f"{self.stage_lr_plan['total_steps']} != {num_training_steps}"
842
+ )
843
+ if self.lr_scheduler is None:
844
+ active_optimizer = self.optimizer if optimizer is None else optimizer
845
+ if active_optimizer is None:
846
+ raise RuntimeError("optimizer is absent while creating stage scheduler")
847
+ self.lr_scheduler = torch.optim.lr_scheduler.LambdaLR(
848
+ active_optimizer,
849
+ lr_lambda=lambda step: _stage_lr_multiplier(
850
+ self.stage_lr_plan, int(step)
851
+ ),
852
+ )
853
+ return self.lr_scheduler
854
+
855
+ def _get_train_sampler(self, train_dataset: Any = None) -> Any:
856
+ dataset = self.train_dataset if train_dataset is None else train_dataset
857
+ if self.train_order == "sequential":
858
+ return SequentialSampler(dataset)
859
+ try:
860
+ return super()._get_train_sampler(train_dataset)
861
+ except TypeError:
862
+ return super()._get_train_sampler()
863
+
864
+ def compute_loss(
865
+ self,
866
+ model: Any,
867
+ inputs: Dict[str, Any],
868
+ return_outputs: bool = False,
869
+ num_items_in_batch: Any = None,
870
+ ) -> Any:
871
+ del num_items_in_batch
872
+ labels = inputs.pop("labels")
873
+ loss_weights = inputs.pop("loss_weights")
874
+ sample_loss_weight = inputs.pop("sample_loss_weight").float()
875
+ outputs = model(**inputs)
876
+ shift_logits = outputs.logits[..., :-1, :]
877
+ shift_labels = labels[..., 1:].contiguous()
878
+ shift_weights = loss_weights[..., 1:].contiguous().float()
879
+ loss = _sparse_weighted_token_loss(
880
+ shift_logits, shift_labels, shift_weights, sample_loss_weight
881
+ )
882
+ return (loss, outputs) if return_outputs else loss
883
+
884
+
885
+ def _sparse_weighted_token_loss(
886
+ logits: torch.Tensor,
887
+ labels: torch.Tensor,
888
+ loss_weights: torch.Tensor,
889
+ sample_loss_weight: torch.Tensor,
890
+ ) -> torch.Tensor:
891
+ """Compute the same weighted CE without densifying every prompt token.
892
+
893
+ Qwen emits logits for the visual tokens, prompt and assistant answer, while
894
+ the semantic SFT contract supervises only assistant answer positions. The
895
+ previous implementation cast the complete ``batch x sequence x vocab``
896
+ tensor to fp32 and then discarded all ignored positions. Selecting active
897
+ positions first is mathematically equivalent and avoids the transient
898
+ 13--20 GiB allocation seen on long examples.
899
+ """
900
+
901
+ if logits.ndim != 3 or labels.shape != logits.shape[:2]:
902
+ raise RuntimeError("logits and labels do not align")
903
+ if loss_weights.shape != labels.shape:
904
+ raise RuntimeError("token loss weights do not align with labels")
905
+ if sample_loss_weight.shape != (logits.shape[0],):
906
+ raise RuntimeError("sample loss weights do not align with the batch")
907
+
908
+ active = labels.ne(-100) & loss_weights.gt(0)
909
+ active_batch, _ = torch.nonzero(active, as_tuple=True)
910
+ if active_batch.numel() == 0:
911
+ raise RuntimeError("batch has no positively weighted assistant labels")
912
+ active_logits = logits[active].float()
913
+ active_labels = labels[active]
914
+ active_weights = loss_weights[active].float()
915
+ token_loss = torch.nn.functional.cross_entropy(
916
+ active_logits, active_labels, reduction="none"
917
+ )
918
+ numerator = torch.zeros(
919
+ logits.shape[0], device=logits.device, dtype=torch.float32
920
+ ).scatter_add_(0, active_batch, token_loss * active_weights)
921
+ denominator = torch.zeros_like(numerator).scatter_add_(
922
+ 0, active_batch, active_weights
923
+ )
924
+ per_sample_loss = numerator / denominator.clamp_min(1e-8)
925
+ return (per_sample_loss * sample_loss_weight.float()).mean()
926
+
927
+
928
+ def inferred_root_manifest(release_root: Path) -> dict[str, Any]:
929
+ path = release_root / "manifest.json"
930
+ if not path.is_file():
931
+ raise RuntimeError("sealed release manifest is absent")
932
+ return json.loads(path.read_text(encoding="utf-8"))
933
+
934
+
935
+ def _load_stage_lr_plan(
936
+ path: Path,
937
+ *,
938
+ release_manifest: dict[str, Any],
939
+ dataset_rows: int,
940
+ global_batch: int,
941
+ optimizer_learning_rate: float,
942
+ warmup_ratio: float,
943
+ ) -> dict[str, Any]:
944
+ """Validate and compile row boundaries into optimizer-step boundaries."""
945
+
946
+ raw = json.loads(path.read_text(encoding="utf-8"))
947
+ if raw.get("format") != "iclr_video.stage_lr_plan.v1":
948
+ raise RuntimeError("unsupported stage LR plan format")
949
+ if int(raw.get("dataset_rows", -1)) != dataset_rows or global_batch <= 0:
950
+ raise RuntimeError("stage LR plan dataset/global-batch contract diverged")
951
+ if int(raw.get("global_batch", -1)) != global_batch:
952
+ raise RuntimeError("stage LR plan global batch diverged")
953
+ manifest_stages = (release_manifest.get("inputs") or {}).get(
954
+ "curriculum_stages"
955
+ )
956
+ stages = raw.get("stages")
957
+ if not isinstance(stages, list) or not stages or not isinstance(manifest_stages, list):
958
+ raise RuntimeError("stage LR plan or release stages are absent")
959
+ if len(stages) != len(manifest_stages):
960
+ raise RuntimeError("stage LR plan count diverged from release")
961
+ compiled = []
962
+ previous_row = 0
963
+ previous_step = 0
964
+ maximum_lr = 0.0
965
+ for index, (stage, manifest_stage) in enumerate(
966
+ zip(stages, manifest_stages, strict=True)
967
+ ):
968
+ end_row = int(stage.get("end_row_exclusive", -1))
969
+ learning_rate = float(stage.get("learning_rate", 0.0))
970
+ if (
971
+ int(stage.get("index", -1)) != index
972
+ or int(manifest_stage.get("index", -1)) != index
973
+ or str(stage.get("release_id")) != str(manifest_stage.get("release_id"))
974
+ or end_row != int(manifest_stage.get("end_row_exclusive", -1))
975
+ or end_row <= previous_row
976
+ or not math.isfinite(learning_rate)
977
+ or learning_rate <= 0
978
+ ):
979
+ raise RuntimeError("stage LR entry diverged from release manifest")
980
+ end_step = math.ceil(end_row / global_batch)
981
+ if end_step <= previous_step:
982
+ raise RuntimeError("stage has no optimizer steps")
983
+ stage_steps = end_step - previous_step
984
+ warmup_steps = math.ceil(stage_steps * warmup_ratio)
985
+ compiled.append(
986
+ {
987
+ "end_step_exclusive": end_step,
988
+ "index": index,
989
+ "learning_rate": learning_rate,
990
+ "release_id": str(stage["release_id"]),
991
+ "start_step": previous_step,
992
+ "warmup_steps": warmup_steps,
993
+ }
994
+ )
995
+ maximum_lr = max(maximum_lr, learning_rate)
996
+ previous_row = end_row
997
+ previous_step = end_step
998
+ if previous_row != dataset_rows:
999
+ raise RuntimeError("stage LR plan does not cover the complete dataset")
1000
+ if not math.isclose(
1001
+ maximum_lr, optimizer_learning_rate, rel_tol=1e-9, abs_tol=0.0
1002
+ ):
1003
+ raise RuntimeError("optimizer LR must equal the maximum stage LR")
1004
+ return {
1005
+ "base_learning_rate": optimizer_learning_rate,
1006
+ "format": raw["format"],
1007
+ "stages": compiled,
1008
+ "total_steps": previous_step,
1009
+ }
1010
+
1011
+
1012
+ def _stage_lr_multiplier(plan: dict[str, Any], step: int) -> float:
1013
+ """Return a per-stage warmup/cosine multiplier for one optimizer step."""
1014
+
1015
+ base_lr = float(plan["base_learning_rate"])
1016
+ for stage in plan["stages"]:
1017
+ start = int(stage["start_step"])
1018
+ end = int(stage["end_step_exclusive"])
1019
+ if step >= end:
1020
+ continue
1021
+ offset = max(step - start, 0)
1022
+ duration = end - start
1023
+ warmup = int(stage["warmup_steps"])
1024
+ scale = float(stage["learning_rate"]) / base_lr
1025
+ if warmup > 0 and offset < warmup:
1026
+ return scale * (offset / warmup)
1027
+ decay_steps = max(duration - warmup, 1)
1028
+ progress = min(max((offset - warmup) / decay_steps, 0.0), 1.0)
1029
+ return scale * 0.5 * (1.0 + math.cos(math.pi * progress))
1030
+ return 0.0
1031
+
1032
+
1033
+ def _training_args(args: argparse.Namespace, has_eval: bool) -> Any:
1034
+ kwargs: Dict[str, Any] = {
1035
+ "output_dir": str(args.output),
1036
+ "num_train_epochs": args.epochs,
1037
+ "per_device_train_batch_size": args.per_device_batch_size,
1038
+ "gradient_accumulation_steps": args.gradient_accumulation_steps,
1039
+ "learning_rate": args.lr,
1040
+ "warmup_ratio": args.warmup_ratio,
1041
+ "bf16": True,
1042
+ "logging_steps": args.logging_steps,
1043
+ "save_steps": args.save_steps,
1044
+ "save_total_limit": args.save_total_limit,
1045
+ "max_steps": args.max_steps,
1046
+ "gradient_checkpointing": True,
1047
+ "lr_scheduler_type": args.lr_scheduler_type,
1048
+ "optim": "adamw_torch",
1049
+ "weight_decay": 0.0,
1050
+ "max_grad_norm": 1.0,
1051
+ "tf32": True,
1052
+ "save_only_model": True,
1053
+ "report_to": "none",
1054
+ "remove_unused_columns": False,
1055
+ "ddp_find_unused_parameters": False,
1056
+ "dataloader_num_workers": args.dataloader_num_workers,
1057
+ "dataloader_pin_memory": True,
1058
+ "seed": args.seed,
1059
+ "deepspeed": str(args.deepspeed),
1060
+ }
1061
+ signature = inspect.signature(TrainingArguments.__init__)
1062
+ if args.save_steps >= 999999 and "save_strategy" in signature.parameters:
1063
+ kwargs["save_strategy"] = "no"
1064
+ if args.dataloader_num_workers > 0:
1065
+ if "dataloader_persistent_workers" in signature.parameters:
1066
+ kwargs["dataloader_persistent_workers"] = True
1067
+ if "dataloader_prefetch_factor" in signature.parameters:
1068
+ kwargs["dataloader_prefetch_factor"] = 2
1069
+ if "data_seed" in signature.parameters:
1070
+ kwargs["data_seed"] = args.seed
1071
+ evaluation_key = (
1072
+ "eval_strategy"
1073
+ if "eval_strategy" in signature.parameters
1074
+ else "evaluation_strategy"
1075
+ )
1076
+ kwargs[evaluation_key] = "steps" if has_eval else "no"
1077
+ if has_eval:
1078
+ kwargs["eval_steps"] = max(args.save_steps, args.logging_steps)
1079
+ return TrainingArguments(**kwargs)
1080
+
1081
+
1082
+ def _is_main_process() -> bool:
1083
+ return int(os.environ.get("RANK", "0")) == 0
1084
+
1085
+
1086
+ def _logical_parameter_numel(parameter: torch.nn.Parameter) -> int:
1087
+ return int(getattr(parameter, "ds_numel", parameter.numel()))
1088
+
1089
+
1090
+ if __name__ == "__main__":
1091
+ main()
code/experiments/tvs_correctness.py ADDED
@@ -0,0 +1,80 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Official-protocol per-query correctness helpers for TVS analyses."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ import re
7
+ from pathlib import Path
8
+ from typing import Any
9
+
10
+
11
+ OVO_FORWARD_TASKS = {"REC", "SSR", "CRR"}
12
+
13
+
14
+ def _mcq_letter(value: Any) -> str | None:
15
+ text = str(value or "").strip()
16
+ match = re.match(r"^[\(\[]?([A-D])(?:[\)\].:\s-]|$)", text, re.IGNORECASE)
17
+ return match.group(1).upper() if match else None
18
+
19
+
20
+ def load_gold(benchmark: str, annotation: Path) -> dict[str, dict[str, Any]]:
21
+ data = json.loads(annotation.read_text())
22
+ gold: dict[str, dict[str, Any]] = {}
23
+ if benchmark == "streamingbench":
24
+ for video_index, entry in enumerate(data):
25
+ for question_index, question in enumerate(entry.get("questions") or []):
26
+ uid = f"streaming_{video_index}_{question_index}"
27
+ gold[uid] = {
28
+ "kind": "streaming_mcq",
29
+ "answer": str(question["answer"]),
30
+ "task": str(question.get("task_type") or ""),
31
+ }
32
+ return gold
33
+ if benchmark != "ovo":
34
+ raise ValueError(f"unsupported benchmark: {benchmark}")
35
+ for item in data:
36
+ task = str(item.get("task") or "").upper()
37
+ if task not in OVO_FORWARD_TASKS:
38
+ uid = f"ovo_{item['id']}"
39
+ gold[uid] = {
40
+ "kind": "ovo_mcq",
41
+ "answer": chr(ord("A") + int(item["gt"])),
42
+ "task": task,
43
+ }
44
+ continue
45
+ for index, test_info in enumerate(item.get("test_info") or []):
46
+ uid = f"ovo_{item['id']}_{index}"
47
+ if task == "REC":
48
+ gold[uid] = {
49
+ "kind": "ovo_rec",
50
+ "answer": str(test_info["count"]),
51
+ "task": task,
52
+ }
53
+ else:
54
+ gold[uid] = {
55
+ "kind": "ovo_binary",
56
+ "answer": int(test_info["type"]),
57
+ "task": task,
58
+ }
59
+ return gold
60
+
61
+
62
+ def is_correct(prediction: Any, gold: dict[str, Any]) -> bool:
63
+ text = str(prediction or "")
64
+ kind = gold["kind"]
65
+ if kind == "streaming_mcq":
66
+ letter = _mcq_letter(text)
67
+ response = letter if letter is not None else text
68
+ return response == str(gold["answer"])
69
+ if kind == "ovo_mcq":
70
+ letter = _mcq_letter(text)
71
+ response = letter if letter is not None else text
72
+ return str(gold["answer"]) in response
73
+ if kind == "ovo_rec":
74
+ return "".join(re.findall(r"\d+", text)) == str(gold["answer"])
75
+ if kind == "ovo_binary":
76
+ positive = int(gold["answer"]) == 1
77
+ if (text == "Y" and positive) or (text == "N" and not positive):
78
+ return True
79
+ return ("Yes" if positive else "No") in text
80
+ raise ValueError(f"unknown gold kind: {kind}")
code/pyproject.toml ADDED
@@ -0,0 +1,32 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [build-system]
2
+ requires = ["setuptools>=61", "wheel"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "viss-code"
7
+ version = "0.3.1"
8
+ description = "Visible-State Supervision and Test-Time Causal View Search"
9
+ readme = "README.md"
10
+ requires-python = ">=3.10"
11
+ license = {file = "LICENSE"}
12
+ dependencies = ["numpy>=1.24,<3", "Pillow>=10,<12"]
13
+
14
+ [project.optional-dependencies]
15
+ test = ["pytest>=8,<10", "torch>=2.7,<3"]
16
+ inference = [
17
+ "torch==2.7.0",
18
+ "transformers==4.57.3",
19
+ "accelerate==1.8.1",
20
+ "qwen-vl-utils>=0.0.11,<0.1",
21
+ "decord==0.6.0",
22
+ ]
23
+ training = ["deepspeed>=0.16,<0.18"]
24
+
25
+ [tool.setuptools.packages.find]
26
+ where = ["."]
27
+ include = ["viss_core*", "experiments*", "data_pipeline*", "scripts*"]
28
+ namespaces = true
29
+
30
+ [tool.pytest.ini_options]
31
+ testpaths = ["tests"]
32
+ pythonpath = ["."]
code/scripts/__init__.py ADDED
@@ -0,0 +1 @@
 
 
1
+ """Version-neutral command-line helpers for the Data V0 release."""
code/scripts/configure_evaluation_protocol.py ADDED
@@ -0,0 +1,62 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Bind the public protocol template to the user's own evaluation assets."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import argparse
6
+ import json
7
+ from pathlib import Path
8
+
9
+ from experiments.evaluate_qwen3vl_recent4_full import (
10
+ sha256_file,
11
+ validate_processor_contract,
12
+ )
13
+
14
+
15
+ def configure(
16
+ *,
17
+ template: Path,
18
+ streaming_samples: Path,
19
+ ovo_samples: Path,
20
+ processor_config: Path,
21
+ output: Path,
22
+ ) -> None:
23
+ protocol = json.loads(template.read_text(encoding="utf-8"))
24
+ for benchmark, path in (
25
+ ("streamingbench", streaming_samples),
26
+ ("ovo", ovo_samples),
27
+ ):
28
+ path = path.resolve(strict=True)
29
+ protocol["benchmarks"][benchmark]["export"] = str(path)
30
+ protocol["benchmarks"][benchmark]["export_sha256"] = sha256_file(path)
31
+ processor_config = processor_config.resolve(strict=True)
32
+ if processor_config.name != "preprocessor_config.json":
33
+ raise ValueError("Pass the processor's preprocessor_config.json")
34
+ protocol["inference"]["processor_contract"]["preprocessor_config_sha256"] = (
35
+ sha256_file(processor_config)
36
+ )
37
+ validate_processor_contract(processor_config.parent, protocol)
38
+ protocol["status"] = "locked"
39
+ output.parent.mkdir(parents=True, exist_ok=True)
40
+ with output.open("x", encoding="utf-8") as handle:
41
+ json.dump(protocol, handle, indent=2)
42
+ handle.write("\n")
43
+
44
+
45
+ def main() -> None:
46
+ parser = argparse.ArgumentParser(description=__doc__)
47
+ parser.add_argument(
48
+ "--template",
49
+ type=Path,
50
+ default=Path("configs/evaluation/recent4.example.json"),
51
+ )
52
+ parser.add_argument("--streaming-samples", type=Path, required=True)
53
+ parser.add_argument("--ovo-samples", type=Path, required=True)
54
+ parser.add_argument("--processor-config", type=Path, required=True)
55
+ parser.add_argument("--output", type=Path, required=True)
56
+ args = parser.parse_args()
57
+ configure(**vars(args))
58
+ print(f"Created local evaluation protocol: {args.output}")
59
+
60
+
61
+ if __name__ == "__main__":
62
+ main()
code/scripts/merge_eval_shards.py ADDED
@@ -0,0 +1,483 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """Merge disjoint agent-evaluation shards into canonical official UID order."""
3
+
4
+ from __future__ import annotations
5
+
6
+ import argparse
7
+ import hashlib
8
+ import json
9
+ import os
10
+ from collections import defaultdict
11
+ from pathlib import Path
12
+ from statistics import fmean
13
+ from typing import Any
14
+
15
+
16
+ IDENTITY_FIELDS = (
17
+ "format",
18
+ "classification",
19
+ "benchmark_selected",
20
+ "source_policy_gate_admitted",
21
+ "schema_version",
22
+ "benchmark",
23
+ "model",
24
+ "model_config_sha256",
25
+ "model_manifest_sha256",
26
+ "processor",
27
+ "processor_sha256",
28
+ "processor_contract",
29
+ "controller_processor",
30
+ "controller_processor_sha256",
31
+ "answer_processor",
32
+ "answer_processor_contract",
33
+ "processor_geometry",
34
+ "adapter",
35
+ "evaluator",
36
+ "evaluation_protocol",
37
+ "evaluation_protocol_sha256",
38
+ "protocol_validation",
39
+ "policy_hash",
40
+ "policy_head",
41
+ "policy_head_sha256",
42
+ "policy_report_sha256",
43
+ "policy_calibration_sha256",
44
+ "global_action_threshold",
45
+ "policy_inputs",
46
+ "budget",
47
+ "policy_training_report",
48
+ "one_shared_commit_policy",
49
+ "benchmark_task_or_group_router",
50
+ "question_keyword_router",
51
+ "answer_space_router",
52
+ "gold_answer_available_to_commit_policy",
53
+ "commit_policy",
54
+ "unparseable_or_error_policy",
55
+ "maximum_unique_frames",
56
+ "direct_prediction_cache",
57
+ "direct_prediction_cache_sha256",
58
+ "source_gate",
59
+ "source_gate_sha256",
60
+ "global_action_thresholds",
61
+ "acquisition_identity",
62
+ "action_abi_schema_sha256",
63
+ "inference_policy_hash",
64
+ "inference_policy",
65
+ "policy_config",
66
+ "comparable_under_locked_recent4_protocol",
67
+ "generation_config",
68
+ "execution_optimization",
69
+ "system_prompt_sha256",
70
+ "prompt_contract_sha256",
71
+ "launch_marker",
72
+ "launch_marker_sha256",
73
+ "annotation",
74
+ "annotation_sha256",
75
+ "samples",
76
+ "samples_sha256",
77
+ "runner_sha256",
78
+ "software",
79
+ "policy",
80
+ "identity_hash",
81
+ "protocol_sha256",
82
+ "evaluator_sha256",
83
+ "generation",
84
+ )
85
+
86
+
87
+ def sha256(path: Path) -> str:
88
+ digest = hashlib.sha256()
89
+ with path.open("rb") as handle:
90
+ for block in iter(lambda: handle.read(8 << 20), b""):
91
+ digest.update(block)
92
+ return digest.hexdigest()
93
+
94
+
95
+ def load_expected_uids(path: Path) -> list[str]:
96
+ uids: list[str] = []
97
+ for line_number, line in enumerate(
98
+ path.read_text(encoding="utf-8").splitlines(), start=1
99
+ ):
100
+ if not line.strip():
101
+ continue
102
+ uid = str(json.loads(line).get("uid") or "")
103
+ if not uid:
104
+ raise ValueError(f"{path}:{line_number} has no uid")
105
+ uids.append(uid)
106
+ if len(uids) != len(set(uids)):
107
+ raise ValueError(f"{path} contains duplicate uids")
108
+ return uids
109
+
110
+
111
+ def grouped_accuracy(rows: list[dict[str, Any]], field: str) -> dict[str, Any]:
112
+ grouped: dict[str, list[dict[str, Any]]] = defaultdict(list)
113
+ for row in rows:
114
+ grouped[str(row.get(field) or "unknown")].append(row)
115
+ result: dict[str, Any] = {}
116
+ for name in sorted(grouped):
117
+ group_rows = grouped[name]
118
+ correct = sum(int(row.get("correct") or 0) for row in group_rows)
119
+ total = len(group_rows)
120
+ result[name] = {
121
+ "accuracy": correct / total if total else 0.0,
122
+ "correct": correct,
123
+ "total": total,
124
+ }
125
+ return result
126
+
127
+
128
+ def aggregate_score(
129
+ score_rows: list[dict[str, Any]], benchmark: str
130
+ ) -> dict[str, Any]:
131
+ correct = sum(int(row.get("correct") or 0) for row in score_rows)
132
+ total = len(score_rows)
133
+ by_task = grouped_accuracy(score_rows, "task")
134
+ by_group = grouped_accuracy(score_rows, "group")
135
+ score: dict[str, Any] = {
136
+ "accuracy": correct / total if total else 0.0,
137
+ "by_group": by_group,
138
+ "by_task": by_task,
139
+ "num_queries": total,
140
+ "rows": score_rows,
141
+ }
142
+ if benchmark == "ovo":
143
+ tasks_by_group: dict[str, set[str]] = defaultdict(set)
144
+ for row in score_rows:
145
+ tasks_by_group[str(row.get("group") or "unknown")].add(
146
+ str(row.get("task") or "unknown")
147
+ )
148
+ official_groups: dict[str, Any] = {}
149
+ for group in sorted(tasks_by_group):
150
+ task_names = sorted(tasks_by_group[group])
151
+ official_groups[group] = {
152
+ "accuracy": fmean(by_task[name]["accuracy"] for name in task_names),
153
+ "num_tasks": len(task_names),
154
+ }
155
+ score["ovo_official"] = {
156
+ "accuracy": fmean(
157
+ value["accuracy"] for value in official_groups.values()
158
+ ),
159
+ "by_group": official_groups,
160
+ }
161
+ return score
162
+
163
+
164
+ def aggregate_efficiency(rows: list[dict[str, Any]]) -> dict[str, float]:
165
+ def mean(field: str, *fallbacks: str) -> float:
166
+ def value(row: dict[str, Any]) -> float:
167
+ for name in (field, *fallbacks):
168
+ if row.get(name) is not None:
169
+ return float(row[name])
170
+ return 0.0
171
+
172
+ return fmean(value(row) for row in rows) if rows else 0.0
173
+
174
+ result = {
175
+ "avg_frames_observed": mean(
176
+ "frames_observed", "unique_frames_acquired", "unique_frames"
177
+ ),
178
+ "avg_model_processed_frames": mean(
179
+ "model_processed_frames", "processed_frames"
180
+ ),
181
+ "avg_media_calls": mean("media_calls"),
182
+ "avg_total_tokens": (
183
+ mean("total_tokens")
184
+ if any(row.get("total_tokens") is not None for row in rows)
185
+ else mean("input_tokens") + mean("output_tokens")
186
+ ),
187
+ "avg_turns": mean("num_turns"),
188
+ }
189
+ if any(row.get("commit_latency_seconds") is not None for row in rows):
190
+ result["avg_commit_latency_seconds"] = mean("commit_latency_seconds")
191
+ if any(row.get("commit_visual_tokens") is not None for row in rows):
192
+ result["avg_commit_visual_tokens"] = mean("commit_visual_tokens")
193
+ if any(row.get("latency_seconds") is not None for row in rows):
194
+ result["avg_latency_seconds"] = mean("latency_seconds")
195
+ return result
196
+
197
+
198
+ def aggregate_agent_behavior(rows: list[dict[str, Any]]) -> dict[str, Any]:
199
+ terminal_counts: dict[str, int] = defaultdict(int)
200
+ for row in rows:
201
+ terminal_counts[str(row.get("policy_terminal_event") or "unknown")] += 1
202
+ total = len(rows)
203
+ failures = sum(
204
+ terminal_counts[event]
205
+ for event in ("malformed_action", "illegal_observe", "missing_answer")
206
+ )
207
+ return {
208
+ "terminal_counts": dict(sorted(terminal_counts.items())),
209
+ "num_policy_failures": failures,
210
+ "strict_terminal_answer_rate": (
211
+ terminal_counts["answered"] / total if total else 0.0
212
+ ),
213
+ "malformed_action_rate": (
214
+ terminal_counts["malformed_action"] / total if total else 0.0
215
+ ),
216
+ "illegal_observe_rate": (
217
+ terminal_counts["illegal_observe"] / total if total else 0.0
218
+ ),
219
+ "missing_answer_rate": (
220
+ terminal_counts["missing_answer"] / total if total else 0.0
221
+ ),
222
+ }
223
+
224
+
225
+ def aggregate_execution_statistics(
226
+ shard_statistics: list[dict[str, Any]],
227
+ ) -> dict[str, Any]:
228
+ """Sum disjoint-shard cache counters and per-row stage timings."""
229
+
230
+ counter_fields = (
231
+ "decode_window_cache_hits",
232
+ "decode_window_cache_misses",
233
+ "decode_window_singleflight_hits",
234
+ "temporal_budget_cache_hits",
235
+ "temporal_budget_cache_misses",
236
+ "video_reader_cache_hits",
237
+ "video_reader_cache_misses",
238
+ "stage_timing_samples",
239
+ )
240
+ stage_totals: dict[str, float] = defaultdict(float)
241
+ result = {
242
+ field: sum(int(stats.get(field) or 0) for stats in shard_statistics)
243
+ for field in counter_fields
244
+ }
245
+ for stats in shard_statistics:
246
+ for name, seconds in (stats.get("stage_seconds_total") or {}).items():
247
+ stage_totals[str(name)] += float(seconds or 0.0)
248
+ result["stage_seconds_total"] = {
249
+ name: round(seconds, 6) for name, seconds in sorted(stage_totals.items())
250
+ }
251
+ return result
252
+
253
+
254
+ def main() -> None:
255
+ parser = argparse.ArgumentParser(description=__doc__)
256
+ parser.add_argument("--inputs", nargs="+", type=Path, required=True)
257
+ parser.add_argument("--output", type=Path, required=True)
258
+ parser.add_argument("--expected-count", type=int, default=-1)
259
+ parser.add_argument(
260
+ "--expected-uids",
261
+ type=Path,
262
+ help="Official JSONL export whose UID order becomes the canonical order.",
263
+ )
264
+ args = parser.parse_args()
265
+
266
+ rows_by_uid: dict[str, dict[str, Any]] = {}
267
+ score_by_uid: dict[str, dict[str, Any]] = {}
268
+ seen_in: dict[str, str] = {}
269
+ metadata: list[dict[str, Any]] = []
270
+ execution_statistics: list[dict[str, Any]] = []
271
+ row_repairs: list[dict[str, Any]] = []
272
+ identity: dict[str, Any] | None = None
273
+ benchmark: str | None = None
274
+
275
+ for path in args.inputs:
276
+ data = json.loads(path.read_text(encoding="utf-8"))
277
+ # A sharded run intentionally records a different ``samples`` JSONL
278
+ # path (and therefore a different samples hash) in every shard. When
279
+ # a frozen full-export is supplied via --expected-uids, that export is
280
+ # the canonical merged sample manifest; all other identity fields
281
+ # must still match exactly.
282
+ comparable_identity_fields = tuple(
283
+ field
284
+ for field in IDENTITY_FIELDS
285
+ if args.expected_uids is None
286
+ or field not in {"samples", "samples_sha256"}
287
+ )
288
+ current_identity = {
289
+ field: data.get(field) for field in comparable_identity_fields
290
+ }
291
+ if identity is None:
292
+ identity = current_identity
293
+ benchmark = str(data.get("benchmark") or "") or None
294
+ elif current_identity != identity:
295
+ changed = [
296
+ field
297
+ for field in comparable_identity_fields
298
+ if current_identity.get(field) != identity.get(field)
299
+ ]
300
+ # Annotation files are frequently remounted from a volatile
301
+ # location (for example /dev/shm) to persistent storage when an
302
+ # evaluation is resumed. The absolute path is not part of the
303
+ # benchmark identity when both shards carry the same non-empty
304
+ # content hash. Keep the hash itself in the frozen identity so
305
+ # different annotation contents remain a hard failure.
306
+ if (
307
+ "annotation" in changed
308
+ and current_identity.get("annotation_sha256")
309
+ and current_identity.get("annotation_sha256")
310
+ == identity.get("annotation_sha256")
311
+ ):
312
+ changed.remove("annotation")
313
+ if not changed:
314
+ current_identity = identity
315
+ else:
316
+ raise ValueError(
317
+ f"{path} does not match the frozen shard identity: {changed}"
318
+ )
319
+ shard_rows = data.get("rows")
320
+ if not isinstance(shard_rows, list):
321
+ raise ValueError(f"{path} has no rows list")
322
+ for row in shard_rows:
323
+ uid = str(row.get("uid") or "")
324
+ if not uid:
325
+ raise ValueError(f"{path} contains a row without uid")
326
+ if uid in rows_by_uid:
327
+ raise ValueError(f"duplicate uid {uid!r} in {seen_in[uid]} and {path}")
328
+ rows_by_uid[uid] = row
329
+ seen_in[uid] = str(path)
330
+ row_benchmark = str(row.get("benchmark") or "")
331
+ if row_benchmark:
332
+ if benchmark is None:
333
+ benchmark = row_benchmark
334
+ elif row_benchmark != benchmark:
335
+ raise ValueError(
336
+ f"mixed benchmarks in shards: {benchmark!r} and {row_benchmark!r}"
337
+ )
338
+ shard_score_rows = (data.get("score") or {}).get("rows") or []
339
+ if not isinstance(shard_score_rows, list):
340
+ raise ValueError(f"{path} score.rows is not a list")
341
+ for score_row in shard_score_rows:
342
+ uid = str(score_row.get("query_id") or "")
343
+ if not uid:
344
+ raise ValueError(f"{path} contains a score row without query_id")
345
+ if uid in score_by_uid:
346
+ raise ValueError(f"duplicate score query_id {uid!r}")
347
+ score_by_uid[uid] = score_row
348
+ metadata.append(
349
+ {
350
+ "path": str(path),
351
+ "sha256": sha256(path),
352
+ "num_samples": len(shard_rows),
353
+ "sample_slice": data.get("sample_slice"),
354
+ "samples": data.get("samples"),
355
+ "samples_sha256": data.get("samples_sha256"),
356
+ "annotation": data.get("annotation"),
357
+ "annotation_sha256": data.get("annotation_sha256"),
358
+ }
359
+ )
360
+ if isinstance(data.get("execution_statistics"), dict):
361
+ execution_statistics.append(data["execution_statistics"])
362
+ row_repairs.extend(data.get("row_repairs") or [])
363
+
364
+ if args.expected_count >= 0 and len(rows_by_uid) != args.expected_count:
365
+ raise ValueError(
366
+ f"merged {len(rows_by_uid)} rows, expected {args.expected_count}"
367
+ )
368
+
369
+ expected_uids = (
370
+ load_expected_uids(args.expected_uids)
371
+ if args.expected_uids is not None
372
+ else list(rows_by_uid)
373
+ )
374
+ expected_set = set(expected_uids)
375
+ actual_set = set(rows_by_uid)
376
+ if actual_set != expected_set:
377
+ missing = [uid for uid in expected_uids if uid not in actual_set]
378
+ extra = sorted(actual_set - expected_set)
379
+ raise ValueError(
380
+ f"official UID mismatch: missing={missing[:20]} extra={extra[:20]}"
381
+ )
382
+ if set(score_by_uid) != expected_set:
383
+ missing = [uid for uid in expected_uids if uid not in score_by_uid]
384
+ extra = sorted(set(score_by_uid) - expected_set)
385
+ raise ValueError(
386
+ f"score UID mismatch: missing={missing[:20]} extra={extra[:20]}"
387
+ )
388
+
389
+ rows = [rows_by_uid[uid] for uid in expected_uids]
390
+ score_rows = [score_by_uid[uid] for uid in expected_uids]
391
+ empty_predictions = [
392
+ str(row.get("uid"))
393
+ for row in rows
394
+ if not str(row.get("prediction") or "").strip()
395
+ ]
396
+ errors = [
397
+ row for row in rows if row.get("error") or row.get("commit_error")
398
+ ]
399
+ merged_identity = dict(identity or {})
400
+ if args.expected_uids is not None:
401
+ merged_identity["samples"] = str(args.expected_uids)
402
+ merged_identity["samples_sha256"] = sha256(args.expected_uids)
403
+ result = {
404
+ "benchmark": benchmark,
405
+ **merged_identity,
406
+ "canonical_official_uid_order": args.expected_uids is not None,
407
+ "efficiency": aggregate_efficiency(rows),
408
+ "empty_prediction_uids": empty_predictions,
409
+ "num_empty_predictions": len(empty_predictions),
410
+ "num_errors": len(errors),
411
+ "num_future_frame_violations": sum(
412
+ bool(row.get("future_frame_violation")) for row in rows
413
+ ),
414
+ "num_samples": len(rows),
415
+ "strict_terminal_answer_rate": (
416
+ sum(bool(row.get("strict_terminal_answer")) for row in rows) / len(rows)
417
+ if rows else 0.0
418
+ ),
419
+ "score": aggregate_score(score_rows, benchmark or ""),
420
+ "row_repairs": row_repairs,
421
+ "source_shards": metadata,
422
+ "expected_uid_source": (
423
+ {
424
+ "path": str(args.expected_uids),
425
+ "sha256": sha256(args.expected_uids),
426
+ }
427
+ if args.expected_uids is not None
428
+ else None
429
+ ),
430
+ "rows": rows,
431
+ }
432
+ if execution_statistics:
433
+ result["execution_statistics"] = aggregate_execution_statistics(
434
+ execution_statistics
435
+ )
436
+ if any("policy_terminal_event" in row for row in rows):
437
+ result["agent_behavior"] = aggregate_agent_behavior(rows)
438
+ result["strict_terminal_answer_rate"] = result["agent_behavior"][
439
+ "strict_terminal_answer_rate"
440
+ ]
441
+ if any("commit_action" in row for row in rows):
442
+ result.update(
443
+ {
444
+ "num_commit_errors": sum(
445
+ bool(row.get("commit_error")) for row in rows
446
+ ),
447
+ "num_unparsed": sum(
448
+ row.get("commit_parsed") is False for row in rows
449
+ ),
450
+ "decision_counts": {
451
+ action: sum(row.get("commit_action") == action for row in rows)
452
+ for action in (
453
+ "not_applicable",
454
+ "unchanged",
455
+ "retain",
456
+ "commit",
457
+ )
458
+ },
459
+ }
460
+ )
461
+ args.output.parent.mkdir(parents=True, exist_ok=True)
462
+ temporary = args.output.with_suffix(args.output.suffix + f".tmp.{os.getpid()}")
463
+ temporary.write_text(
464
+ json.dumps(result, indent=2, sort_keys=True, ensure_ascii=False) + "\n",
465
+ encoding="utf-8",
466
+ )
467
+ temporary.replace(args.output)
468
+ print(
469
+ json.dumps(
470
+ {
471
+ "output": str(args.output),
472
+ "num_samples": len(rows),
473
+ "num_errors": len(errors),
474
+ "num_empty_predictions": len(empty_predictions),
475
+ "canonical_official_uid_order": args.expected_uids is not None,
476
+ },
477
+ sort_keys=True,
478
+ )
479
+ )
480
+
481
+
482
+ if __name__ == "__main__":
483
+ main()
code/scripts/run_recent4_single_benchmark.sh ADDED
@@ -0,0 +1,380 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env bash
2
+ set -euo pipefail
3
+
4
+ # Run one half of the locked recent4_full_v1 evaluation. Two independent
5
+ # invocations may therefore evaluate StreamingBench and OVO concurrently on
6
+ # disjoint GPU sets while retaining the exact common runner and official
7
+ # scorers used by the serial baseline launcher.
8
+
9
+ if [[ $# -ne 8 ]]; then
10
+ echo "usage: $0 REPO MODEL MODEL_MANIFEST OUTPUT PYTHON BENCHMARK GPU_CSV MODEL_ID" >&2
11
+ exit 2
12
+ fi
13
+
14
+ REPO=$1
15
+ MODEL=$2
16
+ MODEL_MANIFEST=$3
17
+ OUTPUT=$4
18
+ PYTHON=$5
19
+ BENCHMARK=$6
20
+ GPU_CSV=$7
21
+ MODEL_ID=$8
22
+ PROCESSOR=${STREAMEVA_PROCESSOR:-$MODEL}
23
+ EVAL_BATCH_SIZE=${STREAMEVA_EVAL_BATCH_SIZE:-16}
24
+ DECODE_PREFETCH_BATCHES=${STREAMEVA_DECODE_PREFETCH_BATCHES:-2}
25
+ FLUSH_EVERY_BATCHES=${STREAMEVA_EVAL_FLUSH_EVERY_BATCHES:-16}
26
+ RECORD_CHOICE_CONFIDENCE=${STREAMEVA_RECORD_CHOICE_CONFIDENCE:-1}
27
+ FRAME_POLICY=${STREAMEVA_FRAME_POLICY:-recent4}
28
+ VISUAL_INPUT_MODE=${STREAMEVA_VISUAL_INPUT_MODE:-image-list}
29
+ ALLOW_SHARED_GPU=${STREAMEVA_ALLOW_SHARED_GPU:-0}
30
+ MIN_FREE_MIB=${STREAMEVA_MIN_FREE_MIB:-35840}
31
+ PROTOCOL=$REPO/configs/evaluation/recent4_full_v1.json
32
+ RUNNER=${STREAMEVA_EVALUATOR_ENTRYPOINT:-$REPO/experiments/evaluate_qwen3vl_recent4_full.py}
33
+ MODEL_IDENTITY_SHA256=${STREAMEVA_MODEL_IDENTITY_SHA256:-}
34
+
35
+ # Keep non-login tmux jobs aligned with the CUDA wheel runtime used by
36
+ # training. In particular, libcusparse from the wheel requires the matching
37
+ # bundled nvJitLink library to precede system CUDA libraries.
38
+ PYTHON_SITE=$($PYTHON -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')
39
+ NVIDIA_LIB_DIRS=$(find "$PYTHON_SITE/nvidia" -mindepth 2 -maxdepth 2 \
40
+ -type d -name lib -print 2>/dev/null | LC_ALL=C sort | paste -sd: -)
41
+ if [[ -n $NVIDIA_LIB_DIRS ]]; then
42
+ export LD_LIBRARY_PATH="$NVIDIA_LIB_DIRS${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
43
+ fi
44
+ [[ $EVAL_BATCH_SIZE =~ ^[1-9][0-9]*$ ]]
45
+ [[ $DECODE_PREFETCH_BATCHES =~ ^[0-9]+$ ]]
46
+ [[ $FLUSH_EVERY_BATCHES =~ ^[1-9][0-9]*$ ]]
47
+ [[ $RECORD_CHOICE_CONFIDENCE =~ ^[01]$ ]]
48
+ [[ $FRAME_POLICY =~ ^(recent4|wide8|wide12|wide16|prefix8|prefix16|prefix32|prefix64|causalbudget20)$ ]]
49
+ [[ $VISUAL_INPUT_MODE =~ ^(image-list|native-video|hybrid-history-video-recent-images)$ ]]
50
+ [[ $ALLOW_SHARED_GPU =~ ^[01]$ ]]
51
+ [[ $MIN_FREE_MIB =~ ^[1-9][0-9]*$ ]]
52
+
53
+ case "$BENCHMARK" in
54
+ streamingbench)
55
+ ANNOTATION=${STREAM_ANNOTATION:?Set STREAM_ANNOTATION to the StreamingBench annotation file}
56
+ SAMPLES=${STREAM_SAMPLES:?Set STREAM_SAMPLES to the frozen StreamingBench query export}
57
+ VIDEO_ROOT=${STREAM_VIDEO_ROOT:?Set STREAM_VIDEO_ROOT to authorized benchmark media}
58
+ OFFICIAL_ROOT=${STREAM_OFFICIAL_ROOT:?Set STREAM_OFFICIAL_ROOT to the official scorer}
59
+ EXPECTED=2495
60
+ SMOKE_UIDS=(streaming_0_0 streaming_8_0 streaming_61_4)
61
+ ;;
62
+ ovo)
63
+ OVO_ROOT=${OVO_ROOT:?Set OVO_ROOT to authorized OVO runtime assets}
64
+ ANNOTATION=${OVO_ANNOTATION:-$OVO_ROOT/data/ovo_bench_new.json}
65
+ OVO_RUNTIME_REPORT=${OVO_RUNTIME_REPORT:-$OVO_ROOT/data/runtime_assets.json}
66
+ SAMPLES=${OVO_SAMPLES:?Set OVO_SAMPLES to the frozen OVO query export}
67
+ VIDEO_ROOT=${OVO_VIDEO_ROOT:-$OVO_ROOT}
68
+ OFFICIAL_ROOT=${OVO_OFFICIAL_ROOT:-$OVO_ROOT}
69
+ EXPECTED=3035
70
+ SMOKE_UIDS=(ovo_0 ovo_1117 ovo_1468_0 ovo_1516_0 ovo_1558_0)
71
+ ;;
72
+ *) echo "BENCHMARK must be streamingbench or ovo" >&2; exit 2 ;;
73
+ esac
74
+
75
+ IFS=, read -r -a GPUS <<<"$GPU_CSV"
76
+ [[ ${#GPUS[@]} -gt 0 ]] || exit 2
77
+ declare -A SEEN=()
78
+ for gpu in "${GPUS[@]}"; do
79
+ [[ $gpu =~ ^[0-7]$ || $gpu =~ ^GPU-[0-9A-Fa-f-]{36}$ ]] || {
80
+ echo "GPU_CSV must contain local indices or GPU UUIDs" >&2
81
+ exit 2
82
+ }
83
+ if [[ -n ${SEEN[$gpu]:-} && $ALLOW_SHARED_GPU -eq 0 ]]; then
84
+ echo "GPU_CSV may repeat an index only with STREAMEVA_ALLOW_SHARED_GPU=1" >&2
85
+ exit 2
86
+ fi
87
+ SEEN[$gpu]=1
88
+ done
89
+ [[ $MODEL_ID =~ ^[A-Za-z0-9._-]+$ ]] || exit 2
90
+ for path in "$MODEL/config.json" \
91
+ "$PROCESSOR/preprocessor_config.json" "$PROTOCOL" "$RUNNER" \
92
+ "$ANNOTATION" "$SAMPLES" "$VIDEO_ROOT" "$OFFICIAL_ROOT"; do
93
+ test -e "$path"
94
+ done
95
+ if [[ $MODEL_MANIFEST == none ]]; then
96
+ [[ $MODEL_IDENTITY_SHA256 =~ ^[0-9a-f]{64}$ ]] || {
97
+ echo "manifest-free evaluation requires STREAMEVA_MODEL_IDENTITY_SHA256" >&2
98
+ exit 2
99
+ }
100
+ MODEL_IDENTITY_ARGS=(--model-identity-sha256 "$MODEL_IDENTITY_SHA256")
101
+ else
102
+ test -s "$MODEL_MANIFEST"
103
+ MODEL_IDENTITY_ARGS=(--model-manifest "$MODEL_MANIFEST")
104
+ fi
105
+ if [[ $BENCHMARK == ovo ]]; then
106
+ test -s "$OVO_RUNTIME_REPORT"
107
+ ANNOTATION_SHA256=$(sha256sum "$ANNOTATION" | cut -d' ' -f1)
108
+ FROZEN_EXPORT_SHA256=$(sha256sum "$SAMPLES" | cut -d' ' -f1)
109
+ jq -e --arg annotation_sha256 "$ANNOTATION_SHA256" \
110
+ --arg frozen_export_sha256 "$FROZEN_EXPORT_SHA256" '
111
+ .status == "pass"
112
+ and .expanded_queries == 3035
113
+ and .covered_queries == 3035
114
+ and .coverage == 1
115
+ and .missing_videos == 0
116
+ and .annotation_sha256 == $annotation_sha256
117
+ and .frozen_export_sha256 == $frozen_export_sha256
118
+ ' "$OVO_RUNTIME_REPORT" >/dev/null
119
+ fi
120
+ test -x "$PYTHON"
121
+ export PYTHONPATH=$REPO${PYTHONPATH:+:$PYTHONPATH}
122
+
123
+ if [[ ${PREFLIGHT_ONLY:-0} == 1 ]]; then
124
+ "$PYTHON" -c 'import PIL, qwen_vl_utils, torch, transformers'
125
+ printf '{"status":"cpu_preflight_pass","benchmark":"%s","protocol":"recent4_full_v1","batch_size":%s,"decode_prefetch_batches":%s}\n' \
126
+ "$BENCHMARK" "$EVAL_BATCH_SIZE" "$DECODE_PREFETCH_BATCHES"
127
+ exit 0
128
+ fi
129
+
130
+ mkdir -p "$OUTPUT/logs" "$OUTPUT/$BENCHMARK"
131
+ if [[ -e $OUTPUT/$BENCHMARK/complete ]]; then
132
+ echo "$BENCHMARK is already complete"
133
+ exit 0
134
+ fi
135
+ if [[ $MODEL_MANIFEST == none ]]; then
136
+ :
137
+ elif [[ -n ${PREVERIFIED_MODEL_MANIFEST_SHA256:-} ]]; then
138
+ [[ $PREVERIFIED_MODEL_MANIFEST_SHA256 =~ ^[0-9a-f]{64}$ ]] || exit 2
139
+ [[ $(sha256sum "$MODEL_MANIFEST" | cut -d' ' -f1) == \
140
+ "$PREVERIFIED_MODEL_MANIFEST_SHA256" ]] || {
141
+ echo "preverified model manifest hash mismatch" >&2
142
+ exit 3
143
+ }
144
+ else
145
+ (cd "$MODEL" && sha256sum --quiet -c SHA256SUMS)
146
+ fi
147
+ if [[ $MODEL_MANIFEST != none ]]; then
148
+ MODEL_IDENTITY_SHA256=$(sha256sum "$MODEL_MANIFEST" | cut -d' ' -f1)
149
+ fi
150
+ "$PYTHON" "$REPO/scripts/validate_evaluation_protocol.py" --protocol "$PROTOCOL" \
151
+ >"$OUTPUT/$BENCHMARK/protocol_validation.json"
152
+
153
+ wait_gpu_free() {
154
+ local gpu=$1 pids used free
155
+ while true; do
156
+ pids=$(nvidia-smi -i "$gpu" --query-compute-apps=pid \
157
+ --format=csv,noheader,nounits 2>/dev/null | tr -d '[:space:]')
158
+ used=$(nvidia-smi -i "$gpu" --query-gpu=memory.used \
159
+ --format=csv,noheader,nounits 2>/dev/null | tr -d '[:space:]')
160
+ free=$(nvidia-smi -i "$gpu" --query-gpu=memory.free \
161
+ --format=csv,noheader,nounits 2>/dev/null | tr -d '[:space:]')
162
+ if [[ $ALLOW_SHARED_GPU -eq 0 ]]; then
163
+ if [[ -z $pids && ${used:-999999} -le 16 ]]; then return 0; fi
164
+ elif [[ ${free:-0} -ge $MIN_FREE_MIB ]]; then
165
+ return 0
166
+ fi
167
+ echo "$(date -u +%FT%TZ) waiting for GPU $gpu (used=${used:-unknown} MiB free=${free:-unknown} MiB shared=$ALLOW_SHARED_GPU minimum_free=$MIN_FREE_MIB MiB)"
168
+ sleep 20
169
+ done
170
+ }
171
+
172
+ validate_result() {
173
+ "$PYTHON" - "$1" "$2" "$FRAME_POLICY" "$VISUAL_INPUT_MODE" "$RECORD_CHOICE_CONFIDENCE" <<'PY'
174
+ import json, sys
175
+ x = json.load(open(sys.argv[1], encoding="utf-8"))
176
+ assert x["evaluation_protocol"] == "recent4_full_v1"
177
+ assert x["num_samples"] == int(sys.argv[2])
178
+ assert x["num_errors"] == 0
179
+ assert all(str(row.get("prediction") or "").strip() for row in x["rows"])
180
+ requested_policy = sys.argv[3]
181
+ visual_input_mode = sys.argv[4]
182
+ record_choice_confidence = bool(int(sys.argv[5]))
183
+ assert x["processor_geometry"]["visual_input_mode"] == visual_input_mode
184
+ assert x["comparable_under_locked_recent4_protocol"] == (
185
+ requested_policy == "recent4" and visual_input_mode == "image-list"
186
+ )
187
+ for row in x["rows"]:
188
+ sampling = row["sampling"]
189
+ if requested_policy == "prefix8":
190
+ assert sampling["num_sampled_frames"] == 8
191
+ elif requested_policy == "prefix16":
192
+ assert sampling["num_sampled_frames"] == 16
193
+ elif requested_policy == "prefix32":
194
+ assert sampling["num_sampled_frames"] == 32
195
+ elif requested_policy == "prefix64":
196
+ assert sampling["num_sampled_frames"] == 64
197
+ elif requested_policy == "causalbudget20":
198
+ assert 1 <= sampling["num_sampled_frames"] <= 20
199
+ assert sampling["controller"]["benchmark_task_or_group_router"] is False
200
+ assert sampling["controller"]["future_frame_violations"] == 0
201
+ else:
202
+ assert sampling["num_sampled_frames"] == 4
203
+ assert max(sampling["frame_timestamps"]) <= row["query_time"] + 1e-3
204
+ assert row["visual_input_mode"] == visual_input_mode
205
+ assert bool(row.get("generation_confidence")) == record_choice_confidence
206
+ assert bool(row["video_grid_thw"]) == (
207
+ visual_input_mode in {"native-video", "hybrid-history-video-recent-images"}
208
+ and bool(row.get("native_video_metadata"))
209
+ )
210
+ PY
211
+ }
212
+
213
+ CONFIDENCE_ARGS=()
214
+ if [[ $RECORD_CHOICE_CONFIDENCE -eq 1 ]]; then
215
+ CONFIDENCE_ARGS+=(--record-choice-confidence)
216
+ fi
217
+
218
+ SMOKE_OUTPUT=$OUTPUT/$BENCHMARK/smoke.json
219
+ if ! test -s "$SMOKE_OUTPUT" \
220
+ || ! validate_result "$SMOKE_OUTPUT" "${#SMOKE_UIDS[@]}"; then
221
+ wait_gpu_free "${GPUS[0]}"
222
+ smoke_batch=$EVAL_BATCH_SIZE
223
+ smoke_rc=1
224
+ for smoke_attempt in 1 2 3; do
225
+ smoke_rc=0
226
+ CUDA_VISIBLE_DEVICES=${GPUS[0]} PYTHONPATH=$REPO PYTHONUNBUFFERED=1 \
227
+ TOKENIZERS_PARALLELISM=false OMP_NUM_THREADS=1 \
228
+ PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \
229
+ "$PYTHON" "$RUNNER" --benchmark "$BENCHMARK" \
230
+ --annotation "$ANNOTATION" --samples "$SAMPLES" \
231
+ --video-root "$VIDEO_ROOT" --model "$MODEL" --processor "$PROCESSOR" \
232
+ "${MODEL_IDENTITY_ARGS[@]}" --protocol "$PROTOCOL" \
233
+ --output "$SMOKE_OUTPUT" --uids "${SMOKE_UIDS[@]}" \
234
+ --frame-policy "$FRAME_POLICY" \
235
+ --visual-input-mode "$VISUAL_INPUT_MODE" \
236
+ --batch-size "$smoke_batch" \
237
+ --decode-prefetch-batches "$DECODE_PREFETCH_BATCHES" \
238
+ --flush-every-batches "$FLUSH_EVERY_BATCHES" \
239
+ "${CONFIDENCE_ARGS[@]}" \
240
+ --resume --retry-errors >>"$OUTPUT/logs/${BENCHMARK}_smoke.log" 2>&1 || smoke_rc=$?
241
+ if validate_result "$SMOKE_OUTPUT" "${#SMOKE_UIDS[@]}"; then break; fi
242
+ echo "warning: $BENCHMARK smoke attempt=$smoke_attempt batch=$smoke_batch failed rc=$smoke_rc" >&2
243
+ if (( smoke_batch > 1 )); then smoke_batch=$(( (smoke_batch + 1) / 2 )); fi
244
+ done
245
+ if validate_result "$SMOKE_OUTPUT" "${#SMOKE_UIDS[@]}"; then
246
+ if [[ $smoke_rc -ne 0 ]]; then
247
+ echo "warning: $BENCHMARK smoke exited rc=$smoke_rc after writing a complete validated result" >&2
248
+ fi
249
+ else
250
+ echo "$BENCHMARK smoke failed validation after runner rc=$smoke_rc" >&2
251
+ if [[ $smoke_rc -eq 0 ]]; then smoke_rc=1; fi
252
+ exit "$smoke_rc"
253
+ fi
254
+ fi
255
+
256
+ SHARD_UID_ROOT=$OUTPUT/$BENCHMARK/shard_uids
257
+ mkdir -p "$SHARD_UID_ROOT"
258
+ "$PYTHON" - "$SAMPLES" "$SHARD_UID_ROOT" "${#GPUS[@]}" <<'PY'
259
+ import json, sys
260
+ from pathlib import Path
261
+ source, output, count = Path(sys.argv[1]), Path(sys.argv[2]), int(sys.argv[3])
262
+ uids = [str(json.loads(line)["uid"]) for line in source.read_text().splitlines() if line]
263
+ assert len(uids) == len(set(uids))
264
+ for index in range(count):
265
+ shard = uids[index::count]
266
+ (output / f"shard{index}.uids").write_text("\n".join(shard) + "\n")
267
+ assert sum(1 for path in output.glob("shard*.uids") for line in path.read_text().splitlines() if line) == len(uids)
268
+ PY
269
+
270
+ run_shard() {
271
+ local index=$1 gpu=$2 uid_file=$3 expected=$4 output log attempt runner_rc attempt_batch
272
+ output=$OUTPUT/$BENCHMARK/shard${index}.json
273
+ log=$OUTPUT/logs/${BENCHMARK}_shard${index}.log
274
+ wait_gpu_free "$gpu"
275
+ attempt_batch=$EVAL_BATCH_SIZE
276
+ for attempt in 1 2 3; do
277
+ runner_rc=0
278
+ CUDA_VISIBLE_DEVICES=$gpu PYTHONPATH=$REPO PYTHONUNBUFFERED=1 \
279
+ TOKENIZERS_PARALLELISM=false OMP_NUM_THREADS=1 \
280
+ PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \
281
+ "$PYTHON" "$RUNNER" --benchmark "$BENCHMARK" \
282
+ --annotation "$ANNOTATION" --samples "$SAMPLES" \
283
+ --video-root "$VIDEO_ROOT" --model "$MODEL" --processor "$PROCESSOR" \
284
+ "${MODEL_IDENTITY_ARGS[@]}" --protocol "$PROTOCOL" \
285
+ --output "$output" --uids-file "$uid_file" \
286
+ --frame-policy "$FRAME_POLICY" \
287
+ --visual-input-mode "$VISUAL_INPUT_MODE" \
288
+ --batch-size "$attempt_batch" \
289
+ --decode-prefetch-batches "$DECODE_PREFETCH_BATCHES" \
290
+ --flush-every-batches "$FLUSH_EVERY_BATCHES" \
291
+ "${CONFIDENCE_ARGS[@]}" \
292
+ --resume --retry-errors >>"$log" 2>&1 || runner_rc=$?
293
+ if validate_result "$output" "$expected"; then
294
+ if [[ $runner_rc -ne 0 ]]; then
295
+ echo "warning: $BENCHMARK shard$index exited rc=$runner_rc after writing a complete validated result" >&2
296
+ fi
297
+ return 0
298
+ fi
299
+ echo "warning: $BENCHMARK shard$index attempt=$attempt batch=$attempt_batch failed validation after runner rc=$runner_rc" >&2
300
+ if (( attempt_batch > 1 )); then attempt_batch=$(( (attempt_batch + 1) / 2 )); fi
301
+ done
302
+ return 1
303
+ }
304
+
305
+ pids=()
306
+ shards=()
307
+ for index in "${!GPUS[@]}"; do
308
+ uid_file=$SHARD_UID_ROOT/shard${index}.uids
309
+ expected=$(awk 'NF {count++} END {print count+0}' "$uid_file")
310
+ shards+=("$OUTPUT/$BENCHMARK/shard${index}.json")
311
+ run_shard "$index" "${GPUS[$index]}" "$uid_file" "$expected" &
312
+ pids+=("$!")
313
+ done
314
+ failed=0
315
+ for index in "${!pids[@]}"; do
316
+ pid=${pids[$index]}
317
+ if wait "$pid"; then
318
+ echo "$(date -u +%FT%TZ) $BENCHMARK shard$index exited rc=0"
319
+ else
320
+ rc=$?
321
+ echo "$(date -u +%FT%TZ) $BENCHMARK shard$index exited rc=$rc" >&2
322
+ failed=1
323
+ fi
324
+ done
325
+ [[ $failed -eq 0 ]]
326
+
327
+ "$PYTHON" "$REPO/scripts/merge_eval_shards.py" --inputs "${shards[@]}" \
328
+ --output "$OUTPUT/$BENCHMARK/full.json" --expected-count "$EXPECTED" \
329
+ --expected-uids "$SAMPLES"
330
+ validate_result "$OUTPUT/$BENCHMARK/full.json" "$EXPECTED"
331
+ "$PYTHON" "$REPO/scripts/score_official_benchmark_predictions.py" \
332
+ --benchmark "$BENCHMARK" --official-root "$OFFICIAL_ROOT" \
333
+ --annotation "$ANNOTATION" --predictions "$OUTPUT/$BENCHMARK/full.json" \
334
+ --output-json "$OUTPUT/$BENCHMARK/official_score.json" \
335
+ --official-input-json "$OUTPUT/$BENCHMARK/official_input.json" \
336
+ >"$OUTPUT/logs/${BENCHMARK}_official_score.log" 2>&1
337
+
338
+ RUNTIME_REPORT_ARG=${OVO_RUNTIME_REPORT:-}
339
+ "$PYTHON" - "$OUTPUT/$BENCHMARK" "$MODEL_IDENTITY_SHA256" "$PROTOCOL" \
340
+ "$MODEL_ID" "$BENCHMARK" "$PROCESSOR" "$RUNTIME_REPORT_ARG" <<'PY'
341
+ import hashlib, json, sys
342
+ from datetime import datetime, timezone
343
+ from pathlib import Path
344
+ out, model_identity, protocol = Path(sys.argv[1]), sys.argv[2], Path(sys.argv[3])
345
+ model_id, benchmark, processor = sys.argv[4:7]
346
+ processor = Path(processor)
347
+ runtime_report = Path(sys.argv[7]) if sys.argv[7] else None
348
+ score = json.loads((out / "official_score.json").read_text())
349
+ full = json.loads((out / "full.json").read_text())
350
+ sha = lambda p: hashlib.sha256(Path(p).read_bytes()).hexdigest()
351
+ summary = {
352
+ "format": "iclr_video.single_benchmark.v2",
353
+ "benchmark": benchmark,
354
+ "model_id": model_id,
355
+ "protocol_id": "recent4_full_v1",
356
+ "inference_policy": full["inference_policy"],
357
+ "inference_policy_hash": full["inference_policy_hash"],
358
+ "policy_config": full["policy_config"],
359
+ "comparable_under_locked_recent4_protocol": full.get(
360
+ "comparable_under_locked_recent4_protocol"
361
+ ),
362
+ "model_identity_sha256": model_identity,
363
+ "processor_path": str(processor),
364
+ "processor_config_sha256": sha(processor / "preprocessor_config.json"),
365
+ "processor_manifest_sha256": (
366
+ sha(processor / "SHA256SUMS")
367
+ if (processor / "SHA256SUMS").is_file()
368
+ else None
369
+ ),
370
+ "protocol_sha256": sha(protocol),
371
+ "benchmark_runtime_assets_sha256": (
372
+ sha(runtime_report) if runtime_report is not None else None
373
+ ),
374
+ "completed_at": datetime.now(timezone.utc).isoformat(),
375
+ "official_score": score,
376
+ }
377
+ (out / "summary.json").write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n")
378
+ PY
379
+ touch "$OUTPUT/$BENCHMARK/complete"
380
+ echo "$(date -u +%FT%TZ) completed $BENCHMARK for $MODEL_ID"
code/scripts/score_official_benchmark_predictions.py ADDED
@@ -0,0 +1,348 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ from __future__ import annotations
3
+
4
+ import argparse
5
+ import copy
6
+ import hashlib
7
+ import importlib.util
8
+ import io
9
+ import json
10
+ import re
11
+ import subprocess
12
+ import sys
13
+ import tempfile
14
+ from contextlib import redirect_stdout
15
+ from pathlib import Path
16
+ from statistics import mean
17
+ from types import SimpleNamespace
18
+ from typing import Any
19
+
20
+ PROJECT_ROOT = Path(__file__).resolve().parents[1]
21
+
22
+ from viss_core.benchmarks import load_prediction_file
23
+
24
+
25
+ OVO_GROUP_TASKS = {
26
+ "backward": ("EPM", "ASI", "HLD"),
27
+ "realtime": ("OCR", "ACR", "ATR", "STU", "FPD", "OJR"),
28
+ "forward": ("REC", "SSR", "CRR"),
29
+ }
30
+
31
+ INVALID_EMPTY_MODEL_RESPONSE = "__INVALID_EMPTY_MODEL_RESPONSE__"
32
+
33
+
34
+ def main() -> None:
35
+ parser = argparse.ArgumentParser(
36
+ description="Adapt agent predictions to the original benchmark formats and invoke their scorers."
37
+ )
38
+ parser.add_argument("--benchmark", required=True, choices=("streamingbench", "ovo"))
39
+ parser.add_argument("--official-root", type=Path, required=True)
40
+ parser.add_argument("--annotation", type=Path, required=True)
41
+ parser.add_argument("--predictions", type=Path, required=True)
42
+ parser.add_argument("--output-json", type=Path, required=True)
43
+ parser.add_argument("--official-input-json", type=Path, default=None)
44
+ args = parser.parse_args()
45
+
46
+ if args.benchmark == "streamingbench":
47
+ report, official_input = score_streamingbench(
48
+ args.official_root, args.annotation, args.predictions
49
+ )
50
+ else:
51
+ report, official_input = score_ovo(args.official_root, args.annotation, args.predictions)
52
+
53
+ args.output_json.parent.mkdir(parents=True, exist_ok=True)
54
+ args.output_json.write_text(
55
+ json.dumps(report, indent=2, ensure_ascii=False, sort_keys=True) + "\n",
56
+ encoding="utf-8",
57
+ )
58
+ if args.official_input_json:
59
+ args.official_input_json.parent.mkdir(parents=True, exist_ok=True)
60
+ args.official_input_json.write_text(
61
+ json.dumps(official_input, indent=2, ensure_ascii=False, sort_keys=True) + "\n",
62
+ encoding="utf-8",
63
+ )
64
+ print(json.dumps(report, indent=2, ensure_ascii=False, sort_keys=True))
65
+
66
+
67
+ def score_streamingbench(
68
+ official_root: Path, annotation: Path, predictions_path: Path
69
+ ) -> tuple[dict[str, Any], Any]:
70
+ predictions = load_prediction_file(predictions_path)
71
+ data = json.loads(annotation.read_text(encoding="utf-8"))
72
+ model_key = "AgenticSFT"
73
+ expected_ids: list[str] = []
74
+ empty_model_response_ids: list[str] = []
75
+ for video_index, entry in enumerate(data):
76
+ for question_index, question in enumerate(entry.get("questions") or []):
77
+ query_id = f"streaming_{video_index}_{question_index}"
78
+ expected_ids.append(query_id)
79
+ raw = predictions.get(query_id)
80
+ letter = _mcq_letter(raw, question.get("options") or [])
81
+ response = letter if letter is not None else str(raw or "")
82
+ if not response:
83
+ # The original scorer skips false-y responses instead of counting
84
+ # them as incorrect. Preserve the model failure without imputing an
85
+ # answer by passing an explicit invalid value through the scorer.
86
+ empty_model_response_ids.append(query_id)
87
+ response = INVALID_EMPTY_MODEL_RESPONSE
88
+ question[model_key] = [response]
89
+ _require_exact_coverage(predictions, expected_ids, "StreamingBench")
90
+
91
+ count_script = official_root / "src" / "data" / "count.py"
92
+ if not count_script.is_file():
93
+ raise FileNotFoundError(count_script)
94
+ with tempfile.TemporaryDirectory(prefix="streamingbench_official_score_") as tmp:
95
+ tmp_path = Path(tmp)
96
+ official_input = tmp_path / "predictions.json"
97
+ official_input.write_text(json.dumps(data, ensure_ascii=False), encoding="utf-8")
98
+ completed = subprocess.run(
99
+ [
100
+ sys.executable,
101
+ str(count_script),
102
+ "--model",
103
+ model_key,
104
+ "--task",
105
+ "real",
106
+ "--src",
107
+ str(official_input),
108
+ ],
109
+ cwd=tmp_path,
110
+ check=True,
111
+ text=True,
112
+ capture_output=True,
113
+ )
114
+ stats = json.loads((tmp_path / f"{model_key}_stats.json").read_text(encoding="utf-8"))
115
+
116
+ total = stats.get("total") or {}
117
+ by_task = {
118
+ task: {
119
+ "accuracy": float(values["accuracy"]),
120
+ "correct": float(values["correct"]),
121
+ "total": int(values["total"]),
122
+ }
123
+ for task, values in stats.items()
124
+ if task != "total"
125
+ }
126
+ if int(total.get("total", 0)) != len(expected_ids):
127
+ raise RuntimeError(
128
+ f"official StreamingBench scorer counted {total.get('total')} of {len(expected_ids)} queries"
129
+ )
130
+ report = {
131
+ "benchmark": "streamingbench",
132
+ "official_scorer": str(count_script),
133
+ "official_scorer_sha256": _sha256(count_script),
134
+ "official_repository_commit": _git_commit(official_root),
135
+ "official_stdout": completed.stdout.strip(),
136
+ "num_queries": len(expected_ids),
137
+ "num_predictions": len(expected_ids),
138
+ "coverage": 1.0,
139
+ "num_empty_model_responses": len(empty_model_response_ids),
140
+ "empty_model_response_ids": empty_model_response_ids,
141
+ "empty_response_scoring": "counted_as_incorrect_without_answer_imputation",
142
+ "accuracy": float(total["accuracy"]),
143
+ "correct": float(total["correct"]),
144
+ "total": int(total["total"]),
145
+ "by_task": by_task,
146
+ }
147
+ return report, data
148
+
149
+
150
+ def score_ovo(
151
+ official_root: Path, annotation: Path, predictions_path: Path
152
+ ) -> tuple[dict[str, Any], Any]:
153
+ predictions = load_prediction_file(predictions_path)
154
+ annotations = json.loads(annotation.read_text(encoding="utf-8"))
155
+ results: dict[str, list[dict[str, Any]]] = {key: [] for key in OVO_GROUP_TASKS}
156
+ expected_ids: list[str] = []
157
+ empty_model_response_ids: list[str] = []
158
+ for source in annotations:
159
+ item = copy.deepcopy(source)
160
+ task = str(item.get("task") or "").upper()
161
+ group = next((name for name, tasks in OVO_GROUP_TASKS.items() if task in tasks), None)
162
+ if group is None:
163
+ raise ValueError(f"unknown OVO task: {task}")
164
+ if group == "forward":
165
+ for index, test_info in enumerate(item.get("test_info") or []):
166
+ query_id = f"ovo_{item['id']}_{index}"
167
+ expected_ids.append(query_id)
168
+ response = str(predictions.get(query_id) or "")
169
+ if not response:
170
+ empty_model_response_ids.append(query_id)
171
+ test_info["response"] = response
172
+ else:
173
+ query_id = f"ovo_{item['id']}"
174
+ expected_ids.append(query_id)
175
+ letter = _mcq_letter(predictions.get(query_id), item.get("options") or [])
176
+ response = letter if letter is not None else str(predictions.get(query_id) or "")
177
+ if not response:
178
+ empty_model_response_ids.append(query_id)
179
+ item["response"] = response
180
+ item["ground_truth"] = chr(ord("A") + int(item["gt"]))
181
+ results[group].append(item)
182
+ _require_exact_coverage(predictions, expected_ids, "OVO-Bench")
183
+
184
+ score_module_path = official_root / "utils" / "OVOBenchScore.py"
185
+ if not score_module_path.is_file():
186
+ raise FileNotFoundError(score_module_path)
187
+ spec = importlib.util.spec_from_file_location("official_ovo_score", score_module_path)
188
+ if spec is None or spec.loader is None:
189
+ raise ImportError(score_module_path)
190
+ module = importlib.util.module_from_spec(spec)
191
+ spec.loader.exec_module(module)
192
+ scorer = module.OVOBenchOfflineScore(SimpleNamespace(model="AgenticSFT"), copy.deepcopy(results))
193
+
194
+ _, backward_scores = scorer.calculate_score_backward_realtime(copy.deepcopy(results["backward"]))
195
+ _, realtime_scores = scorer.calculate_score_backward_realtime(copy.deepcopy(results["realtime"]))
196
+ _, forward_scores = scorer.calculate_score_forward(copy.deepcopy(results["forward"]))
197
+ score_maps = {
198
+ "backward": backward_scores,
199
+ "realtime": realtime_scores,
200
+ "forward": forward_scores,
201
+ }
202
+ by_task: dict[str, dict[str, Any]] = {}
203
+ official_by_group: dict[str, dict[str, Any]] = {}
204
+ for group, expected_tasks in OVO_GROUP_TASKS.items():
205
+ scores = score_maps[group]
206
+ if set(scores) != set(expected_tasks):
207
+ raise RuntimeError(f"official OVO scorer tasks for {group}: {sorted(scores)}")
208
+ task_accuracies = []
209
+ for task in expected_tasks:
210
+ values = scores[task]
211
+ accuracy = sum(values) / len(values)
212
+ by_task[task] = {
213
+ "accuracy": accuracy,
214
+ "correct": float(sum(values)),
215
+ "total": len(values),
216
+ }
217
+ task_accuracies.append(accuracy)
218
+ official_by_group[group] = {
219
+ "accuracy": mean(task_accuracies),
220
+ "num_tasks": len(task_accuracies),
221
+ }
222
+ official_overall = mean(value["accuracy"] for value in official_by_group.values())
223
+ derived_metrics = _ovo_derived_metrics(by_task, official_by_group)
224
+
225
+ stdout_buffer = io.StringIO()
226
+ with redirect_stdout(stdout_buffer):
227
+ module.OVOBenchOfflineScore(
228
+ SimpleNamespace(model="AgenticSFT"), copy.deepcopy(results)
229
+ ).score()
230
+ official_stdout = stdout_buffer.getvalue().strip()
231
+ _verify_ovo_stdout(official_stdout, by_task, official_by_group, official_overall)
232
+
233
+ report = {
234
+ "benchmark": "ovo",
235
+ "official_scorer": str(score_module_path),
236
+ "official_scorer_sha256": _sha256(score_module_path),
237
+ "official_repository_commit": _git_commit(official_root),
238
+ "official_stdout": official_stdout,
239
+ "num_queries": len(expected_ids),
240
+ "num_predictions": len(expected_ids),
241
+ "coverage": 1.0,
242
+ "num_empty_model_responses": len(empty_model_response_ids),
243
+ "empty_model_response_ids": empty_model_response_ids,
244
+ "empty_response_scoring": "passed_unchanged_to_original_scorer_without_answer_imputation",
245
+ "by_task": by_task,
246
+ "ovo_official": {
247
+ "name": "Total Avg",
248
+ "aggregation": "mean of Real-Time, Backward, and Forward task-macro averages",
249
+ "accuracy": official_overall,
250
+ "by_group": official_by_group,
251
+ },
252
+ **derived_metrics,
253
+ }
254
+ return report, results
255
+
256
+
257
+ def _ovo_derived_metrics(
258
+ by_task: dict[str, dict[str, Any]],
259
+ by_group: dict[str, dict[str, Any]],
260
+ ) -> dict[str, dict[str, Any]]:
261
+ micro_correct = sum(float(value["correct"]) for value in by_task.values())
262
+ micro_total = sum(int(value["total"]) for value in by_task.values())
263
+ if micro_total <= 0:
264
+ raise RuntimeError("OVO-Bench has no scored predictions")
265
+ rt_bt_accuracy = mean(
266
+ float(by_group[group]["accuracy"]) for group in ("realtime", "backward")
267
+ )
268
+ return {
269
+ "ovo_micro_overall": {
270
+ "name": "Overall",
271
+ "aggregation": "micro average over all expanded OVO queries",
272
+ "accuracy": micro_correct / micro_total,
273
+ "correct": micro_correct,
274
+ "total": micro_total,
275
+ },
276
+ "ovo_rt_bt_avg": {
277
+ "name": "RT/BT Avg",
278
+ "aggregation": "mean of Real-Time and Backward task-macro averages",
279
+ "accuracy": rt_bt_accuracy,
280
+ },
281
+ }
282
+
283
+
284
+ def _require_exact_coverage(
285
+ predictions: dict[str, Any], expected_ids: list[str], benchmark: str
286
+ ) -> None:
287
+ expected = set(expected_ids)
288
+ missing = sorted(expected - set(predictions))
289
+ if missing:
290
+ raise RuntimeError(f"{benchmark} missing {len(missing)} predictions: {missing[:10]}")
291
+ if len(expected_ids) != len(expected):
292
+ raise RuntimeError(f"{benchmark} contains duplicate query IDs")
293
+
294
+
295
+ def _mcq_letter(prediction: Any, _options: list[Any]) -> str | None:
296
+ text = str(prediction or "").strip()
297
+ match = re.match(r"^[\(\[]?([A-D])(?:[\)\].:\s-]|$)", text, flags=re.IGNORECASE)
298
+ if match:
299
+ return match.group(1).upper()
300
+ return None
301
+
302
+
303
+ def _verify_ovo_stdout(
304
+ stdout: str,
305
+ by_task: dict[str, dict[str, Any]],
306
+ by_group: dict[str, dict[str, Any]],
307
+ overall: float,
308
+ ) -> None:
309
+ printed_tasks = {
310
+ task: float(value)
311
+ for task, value in re.findall(r"Task:\s*(\w+),\s*Acc:\s*([0-9.]+)", stdout)
312
+ }
313
+ for task, values in by_task.items():
314
+ if printed_tasks.get(task) != round(values["accuracy"] * 100, 2):
315
+ raise RuntimeError(f"official OVO stdout mismatch for {task}")
316
+ labels = {"backward": "Backward", "realtime": "Realtime", "forward": "Forward"}
317
+ for group, label in labels.items():
318
+ match = re.search(rf"{label} Avg\.:\s*([0-9.]+)", stdout)
319
+ if match is None or float(match.group(1)) != round(by_group[group]["accuracy"] * 100, 2):
320
+ raise RuntimeError(f"official OVO stdout mismatch for {group} Avg")
321
+ match = re.search(r"Total Avg\.:\s*([0-9.]+)", stdout)
322
+ if match is None or float(match.group(1)) != round(overall * 100, 2):
323
+ raise RuntimeError("official OVO stdout mismatch for Total Avg")
324
+
325
+
326
+ def _sha256(path: Path) -> str:
327
+ digest = hashlib.sha256()
328
+ with path.open("rb") as handle:
329
+ for block in iter(lambda: handle.read(8 << 20), b""):
330
+ digest.update(block)
331
+ return digest.hexdigest()
332
+
333
+
334
+ def _git_commit(root: Path) -> str | None:
335
+ completed = subprocess.run(
336
+ ["git", "-C", str(root), "rev-parse", "HEAD"],
337
+ text=True,
338
+ capture_output=True,
339
+ check=False,
340
+ )
341
+ if completed.returncode != 0:
342
+ return None
343
+ commit = completed.stdout.strip()
344
+ return commit or None
345
+
346
+
347
+ if __name__ == "__main__":
348
+ main()
code/scripts/validate_evaluation_protocol.py ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ from __future__ import annotations
3
+
4
+ import argparse
5
+ import json
6
+ from pathlib import Path
7
+
8
+ from viss_core.evaluation_protocol import LOCKED_PROTOCOL, validate_locked_protocol
9
+
10
+
11
+ def main() -> None:
12
+ parser = argparse.ArgumentParser(
13
+ description="Validate the immutable exports and rules of the locked evaluation protocol."
14
+ )
15
+ parser.add_argument("--protocol", type=Path, default=LOCKED_PROTOCOL)
16
+ args = parser.parse_args()
17
+ print(
18
+ json.dumps(
19
+ validate_locked_protocol(args.protocol),
20
+ indent=2,
21
+ ensure_ascii=False,
22
+ sort_keys=True,
23
+ )
24
+ )
25
+
26
+
27
+ if __name__ == "__main__":
28
+ main()
code/tests/test_confidence_gated_third_view_agent.py ADDED
@@ -0,0 +1,77 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from experiments.materialize_confidence_gated_third_view_agent import materialize
2
+
3
+
4
+ def _confidence(margin):
5
+ return {"top2_margin": margin, "task_or_group_used": False}
6
+
7
+
8
+ def _row(uid, prediction, margin, *, escalated=True):
9
+ return {
10
+ "uid": uid,
11
+ "prediction": prediction,
12
+ "generation_confidence": _confidence(margin),
13
+ "sampling": {"frame_indices": [1], "frame_timestamps": [0.0]},
14
+ "query_time": 1.0,
15
+ "frames_observed": 1,
16
+ "agent_policy": {"escalated": escalated},
17
+ }
18
+
19
+
20
+ def _value(rows):
21
+ return {
22
+ "benchmark": "streamingbench",
23
+ "model_manifest_sha256": "model",
24
+ "annotation_sha256": "annotation",
25
+ "evaluation_protocol_sha256": "protocol",
26
+ "rows": rows,
27
+ }
28
+
29
+
30
+ def _threshold(commit=-0.1):
31
+ return {
32
+ "status": "source_disjoint_internal_threshold_calibration",
33
+ "eligible_as_frozen_benchmark_controller_config": True,
34
+ "model_manifest_sha256": "model",
35
+ "policy": {"commit_threshold": commit},
36
+ }
37
+
38
+
39
+ def test_third_view_uses_current_answer_source_margin() -> None:
40
+ recent = _value([_row("x", "A", 0.8), _row("y", "A", 0.2)])
41
+ event = _value([_row("x", "B", 0.5), _row("y", "B", 0.8)])
42
+ base = _value([_row("x", "A", 0.5), _row("y", "B", 0.8)])
43
+ third = _value([_row("x", "C", 0.4), _row("y", "C", 0.6)])
44
+ result = materialize(recent, event, base, third, _threshold(-0.3))
45
+ rows = {row["uid"]: row for row in result["rows"]}
46
+ assert rows["x"]["prediction"] == "A"
47
+ assert rows["x"]["third_view_policy"]["current_view"] == "recent4"
48
+ assert rows["x"]["third_view_policy"]["committed"] is False
49
+ assert rows["y"]["prediction"] == "C"
50
+ assert rows["y"]["third_view_policy"]["current_view"] == "causalbudget20"
51
+ assert rows["y"]["third_view_policy"]["committed"] is True
52
+ assert result["third_view_committed"] == 1
53
+
54
+
55
+ def test_unchanged_third_answer_never_counts_as_commit() -> None:
56
+ recent = _value([_row("x", "A", 0.2)])
57
+ event = _value([_row("x", "B", 0.3)])
58
+ base = _value([_row("x", "B", 0.3)])
59
+ third = _value([_row("x", "B", 0.9)])
60
+ result = materialize(recent, event, base, third, _threshold())
61
+ assert result["rows"][0]["prediction"] == "B"
62
+ assert result["rows"][0]["third_view_policy"]["committed"] is False
63
+
64
+
65
+ def test_full_precomputed_third_view_only_triggers_on_event_disagreement() -> None:
66
+ recent = _value([_row("x", "A", 0.2), _row("z", "A", 0.4)])
67
+ event = _value([_row("x", "B", 0.3), _row("z", "A", 0.5)])
68
+ base = _value([_row("x", "B", 0.3), _row("z", "A", 0.5)])
69
+ third = _value([_row("x", "C", 0.9), _row("z", "C", 0.9)])
70
+ result = materialize(recent, event, base, third, _threshold())
71
+ rows = {row["uid"]: row for row in result["rows"]}
72
+ assert rows["x"]["prediction"] == "C"
73
+ assert rows["z"]["prediction"] == "A"
74
+ assert rows["z"]["third_view_policy"]["triggered"] is False
75
+ assert result["third_view_triggered"] == 1
76
+ assert result["third_view_input_rows"] == 2
77
+ assert result["third_view_unused_precomputed_rows"] == 1
code/tests/test_filter_semantic_release_by_option_consensus.py ADDED
@@ -0,0 +1,84 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from copy import deepcopy
2
+
3
+ import pytest
4
+
5
+ from data_pipeline.temporal_scope.filter_semantic_release_by_blind_audit import (
6
+ BlindAuditFilterError,
7
+ )
8
+ from data_pipeline.temporal_scope.filter_semantic_release_by_option_consensus import (
9
+ _validate_presentations,
10
+ )
11
+
12
+
13
+ def _source(uid: str, answer: str) -> dict:
14
+ return {
15
+ "uid": uid,
16
+ "messages": [
17
+ {"role": "user", "content": [{"type": "text", "text": "q"}]},
18
+ {
19
+ "role": "assistant",
20
+ "loss": True,
21
+ "content": [{"type": "text", "text": answer}],
22
+ },
23
+ ],
24
+ }
25
+
26
+
27
+ def _audit(uid: str, presentation: int, answer: str, *, confidence: str = "high") -> dict:
28
+ mapping = list("ABCD") if presentation == 0 else list("BCDA")
29
+ return {
30
+ "format": "streameva.semantic_release.option_consensus_audit_row.v0",
31
+ "uid": uid,
32
+ "presentation": presentation,
33
+ "displayed_to_source": mapping,
34
+ "expected_source": "B",
35
+ "prediction_source": answer,
36
+ "confidence": confidence,
37
+ "error": None,
38
+ "correct_source": answer == "B",
39
+ }
40
+
41
+
42
+ def test_option_consensus_selects_only_stable_source_agreements() -> None:
43
+ sources = [_source("kept", "B"), _source("wrong", "B"), _source("unstable", "B")]
44
+ audits = [
45
+ _audit("kept", 0, "B"),
46
+ _audit("kept", 1, "B"),
47
+ _audit("wrong", 0, "C"),
48
+ _audit("wrong", 1, "C"),
49
+ _audit("unstable", 0, "B"),
50
+ _audit("unstable", 1, "C"),
51
+ ]
52
+ selected, stable = _validate_presentations(
53
+ source_rows=sources, audit_rows=audits, allowed_confidence={"high"}
54
+ )
55
+ assert [row["uid"] for row in selected] == ["kept"]
56
+ assert stable == 2
57
+
58
+
59
+ def test_option_consensus_rejects_missing_or_identity_second_order() -> None:
60
+ source = _source("u", "B")
61
+ first = _audit("u", 0, "B")
62
+ with pytest.raises(BlindAuditFilterError, match="complete two-order"):
63
+ _validate_presentations(
64
+ source_rows=[source], audit_rows=[first], allowed_confidence={"high"}
65
+ )
66
+ second = _audit("u", 1, "B")
67
+ second["displayed_to_source"] = list("ABCD")
68
+ with pytest.raises(BlindAuditFilterError, match="malformed"):
69
+ _validate_presentations(
70
+ source_rows=[source],
71
+ audit_rows=[first, second],
72
+ allowed_confidence={"high"},
73
+ )
74
+
75
+
76
+ def test_option_consensus_rejects_tampered_correctness_flag() -> None:
77
+ source = _source("u", "B")
78
+ rows = [_audit("u", 0, "B"), _audit("u", 1, "B")]
79
+ tampered = deepcopy(rows)
80
+ tampered[1]["correct_source"] = False
81
+ with pytest.raises(BlindAuditFilterError, match="malformed"):
82
+ _validate_presentations(
83
+ source_rows=[source], audit_rows=tampered, allowed_confidence={"high"}
84
+ )
code/tests/test_materialize_uncertainty_temporal_agent.py ADDED
@@ -0,0 +1,551 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from experiments.materialize_uncertainty_temporal_agent import materialize
2
+
3
+
4
+ def _result(rows: list[dict]) -> dict:
5
+ return {
6
+ "benchmark": "x",
7
+ "model_manifest_sha256": "m",
8
+ "annotation_sha256": "a",
9
+ "evaluation_protocol_sha256": "p",
10
+ "rows": rows,
11
+ }
12
+
13
+
14
+ def test_materializer_uses_temporal_only_for_preselected_uids() -> None:
15
+ recent = _result(
16
+ [
17
+ {"uid": "a", "prediction": "A", "query_time": 10, "frames_observed": 4},
18
+ {"uid": "b", "prediction": "B", "query_time": 10, "frames_observed": 4},
19
+ ]
20
+ )
21
+ temporal = _result(
22
+ [
23
+ {
24
+ "uid": "a",
25
+ "prediction": "C",
26
+ "query_time": 10,
27
+ "frames_observed": 20,
28
+ "sampling": {
29
+ "frame_policy": "causalbudget20",
30
+ "frame_timestamps": [0, 10],
31
+ "candidate_frame_indices": list(range(64)),
32
+ "extra_frames_inspected": 44,
33
+ },
34
+ "visual_input_mode": "native-video",
35
+ }
36
+ ]
37
+ )
38
+ selection = {
39
+ "status": "frozen_before_benchmark_temporal_inference",
40
+ "threshold": 0.3,
41
+ "benchmark": "x",
42
+ "model_manifest_sha256": "m",
43
+ "score_or_gold_fields_read_for_selection": False,
44
+ "rows": [
45
+ {"uid": "a", "margin": 0.2, "escalated": True},
46
+ {"uid": "b", "margin": 0.8, "escalated": False},
47
+ ],
48
+ }
49
+ threshold = {
50
+ "status": "source_disjoint_internal_threshold_calibration",
51
+ "eligible_as_frozen_benchmark_controller_config": True,
52
+ "model_manifest_sha256": "m",
53
+ "policy": {"threshold": 0.3},
54
+ }
55
+ output = materialize(recent, temporal, selection, threshold)
56
+ assert [row["prediction"] for row in output["rows"]] == ["C", "B"]
57
+ assert output["escalated"] == 1
58
+ assert output["mean_model_calls"] == 1.5
59
+ assert output["mean_low_resolution_probe_frames"] == 32
60
+ assert output["mean_extra_probe_frames_not_visible_to_vlm"] == 22
61
+ assert output["mean_model_processed_frames"] == 14
62
+ assert output["mean_total_decoded_frame_instances"] == 46
63
+ assert output["rows"][0]["total_decoded_frame_instances"] == 88
64
+ assert output["rows"][1]["total_decoded_frame_instances"] == 4
65
+
66
+
67
+ def test_materializer_rejects_lower_confidence_changed_proposal() -> None:
68
+ recent = _result(
69
+ [
70
+ {
71
+ "uid": "a",
72
+ "prediction": "A",
73
+ "query_time": 10,
74
+ "frames_observed": 4,
75
+ "generation_confidence": {
76
+ "top2_margin": 0.3,
77
+ "task_or_group_used": False,
78
+ },
79
+ }
80
+ ]
81
+ )
82
+ temporal = _result(
83
+ [
84
+ {
85
+ "uid": "a",
86
+ "prediction": "C",
87
+ "query_time": 10,
88
+ "frames_observed": 20,
89
+ "generation_confidence": {
90
+ "top2_margin": 0.2,
91
+ "task_or_group_used": False,
92
+ },
93
+ "sampling": {
94
+ "frame_policy": "causalbudget20",
95
+ "frame_timestamps": [0, 10],
96
+ "candidate_frame_indices": list(range(20)),
97
+ "extra_frames_inspected": 0,
98
+ },
99
+ "visual_input_mode": "native-video",
100
+ }
101
+ ]
102
+ )
103
+ selection = {
104
+ "status": "frozen_before_benchmark_temporal_inference",
105
+ "threshold": 0.4,
106
+ "benchmark": "x",
107
+ "model_manifest_sha256": "m",
108
+ "score_or_gold_fields_read_for_selection": False,
109
+ "rows": [{"uid": "a", "margin": 0.3, "escalated": True}],
110
+ }
111
+ threshold = {
112
+ "status": "source_disjoint_internal_threshold_calibration",
113
+ "eligible_as_frozen_benchmark_controller_config": True,
114
+ "model_manifest_sha256": "m",
115
+ "policy": {"threshold": 0.4, "commit_threshold": 0.0},
116
+ }
117
+ output = materialize(recent, temporal, selection, threshold)
118
+ assert output["rows"][0]["prediction"] == "A"
119
+ assert output["rows"][0]["proposal_prediction"] == "C"
120
+ assert output["rows"][0]["agent_policy"]["committed"] is False
121
+
122
+
123
+ def test_materializer_names_image_list_event20_interface() -> None:
124
+ recent = _result(
125
+ [{
126
+ "uid": "a", "prediction": "A", "query_time": 10,
127
+ "frames_observed": 4,
128
+ "generation_confidence": {"top2_margin": 0.1, "task_or_group_used": False},
129
+ }]
130
+ )
131
+ temporal = _result(
132
+ [{
133
+ "uid": "a", "prediction": "C", "query_time": 10,
134
+ "frames_observed": 20,
135
+ "generation_confidence": {"top2_margin": 0.4, "task_or_group_used": False},
136
+ "sampling": {
137
+ "frame_policy": "causalbudget20", "frame_timestamps": [0, 10],
138
+ "candidate_frame_indices": list(range(20)), "extra_frames_inspected": 0,
139
+ },
140
+ "visual_input_mode": "image-list",
141
+ }]
142
+ )
143
+ selection = {
144
+ "status": "frozen_before_benchmark_temporal_inference",
145
+ "threshold": 0.5, "benchmark": "x", "model_manifest_sha256": "m",
146
+ "score_or_gold_fields_read_for_selection": False,
147
+ "rows": [{"uid": "a", "margin": 0.1, "escalated": True}],
148
+ }
149
+ threshold = {
150
+ "status": "source_disjoint_internal_threshold_calibration",
151
+ "eligible_as_frozen_benchmark_controller_config": True,
152
+ "model_manifest_sha256": "m",
153
+ "policy": {"threshold": 0.5, "commit_threshold": 0.0},
154
+ }
155
+ output = materialize(recent, temporal, selection, threshold)
156
+ assert output["policy"]["second_view"] == "image_list_causal_temporal20"
157
+ assert output["rows"][0]["agent_policy"]["selected_view"] == (
158
+ "causal_temporal20_image_list"
159
+ )
160
+
161
+
162
+ def test_materializer_names_image_list_event32_recent8_interface() -> None:
163
+ recent = _result(
164
+ [{
165
+ "uid": "a", "prediction": "A", "query_time": 10,
166
+ "frames_observed": 4,
167
+ "generation_confidence": {"top2_margin": 0.1, "task_or_group_used": False},
168
+ }]
169
+ )
170
+ temporal = _result(
171
+ [{
172
+ "uid": "a", "prediction": "C", "query_time": 10,
173
+ "frames_observed": 32,
174
+ "generation_confidence": {"top2_margin": 0.4, "task_or_group_used": False},
175
+ "sampling": {
176
+ "frame_policy": "causalbudget32r8", "frame_timestamps": [0, 10],
177
+ "candidate_frame_indices": list(range(64)),
178
+ "extra_frames_inspected": 32,
179
+ },
180
+ "visual_input_mode": "image-list",
181
+ }]
182
+ )
183
+ selection = {
184
+ "status": "frozen_before_benchmark_temporal_inference",
185
+ "threshold": 0.5, "benchmark": "x", "model_manifest_sha256": "m",
186
+ "score_or_gold_fields_read_for_selection": False,
187
+ "rows": [{"uid": "a", "margin": 0.1, "escalated": True}],
188
+ }
189
+ threshold = {
190
+ "status": "source_disjoint_internal_threshold_calibration",
191
+ "eligible_as_frozen_benchmark_controller_config": True,
192
+ "model_manifest_sha256": "m",
193
+ "policy": {"threshold": 0.5, "commit_threshold": 0.0},
194
+ }
195
+ output = materialize(recent, temporal, selection, threshold)
196
+ assert output["policy"]["second_view"] == (
197
+ "image_list_causal_temporal32_recent8"
198
+ )
199
+ assert output["rows"][0]["agent_policy"]["selected_view"] == (
200
+ "causal_temporal32_recent8_image_list"
201
+ )
202
+
203
+
204
+ def test_materializer_accepts_full_lowres_temporal_superset() -> None:
205
+ recent = _result(
206
+ [
207
+ {"uid": "a", "prediction": "A", "query_time": 10, "frames_observed": 4},
208
+ {"uid": "b", "prediction": "B", "query_time": 10, "frames_observed": 4},
209
+ ]
210
+ )
211
+ temporal_rows = []
212
+ for uid, prediction in (("a", "C"), ("b", "D")):
213
+ temporal_rows.append(
214
+ {
215
+ "uid": uid,
216
+ "prediction": prediction,
217
+ "query_time": 10,
218
+ "frames_observed": 32,
219
+ "sampling": {
220
+ "frame_policy": "causalbudget32r8",
221
+ "frame_timestamps": [0, 10],
222
+ "candidate_frame_indices": list(range(64)),
223
+ "extra_frames_inspected": 32,
224
+ },
225
+ "visual_input_mode": "lowres-history-recent-images",
226
+ }
227
+ )
228
+ temporal = _result(temporal_rows)
229
+ selection = {
230
+ "status": "frozen_before_benchmark_temporal_inference",
231
+ "threshold": 0.3,
232
+ "benchmark": "x",
233
+ "model_manifest_sha256": "m",
234
+ "score_or_gold_fields_read_for_selection": False,
235
+ "rows": [
236
+ {"uid": "a", "margin": 0.2, "escalated": True},
237
+ {"uid": "b", "margin": 0.8, "escalated": False},
238
+ ],
239
+ }
240
+ threshold = {
241
+ "status": "source_disjoint_internal_threshold_calibration",
242
+ "eligible_as_frozen_benchmark_controller_config": True,
243
+ "model_manifest_sha256": "m",
244
+ "policy": {"threshold": 0.3},
245
+ }
246
+ output = materialize(recent, temporal, selection, threshold)
247
+ assert [row["prediction"] for row in output["rows"]] == ["C", "B"]
248
+ assert output["temporal_input_num_rows"] == 2
249
+ assert output["temporal_unused_num_rows"] == 1
250
+ assert output["policy"]["second_view"] == (
251
+ "lowres_history_recent_images_causal_temporal32_recent8"
252
+ )
253
+ assert output["rows"][0]["agent_policy"]["selected_view"] == (
254
+ "causal_temporal32_recent8_lowres_history_recent_images"
255
+ )
256
+
257
+
258
+ def test_materializer_names_uniform32_image_list_interface() -> None:
259
+ recent = _result(
260
+ [{
261
+ "uid": "a", "prediction": "A", "query_time": 10,
262
+ "frames_observed": 4,
263
+ "generation_confidence": {"top2_margin": 0.1, "task_or_group_used": False},
264
+ }]
265
+ )
266
+ temporal = _result(
267
+ [{
268
+ "uid": "a", "prediction": "C", "query_time": 10,
269
+ "frames_observed": 32,
270
+ "generation_confidence": {"top2_margin": 0.4, "task_or_group_used": False},
271
+ "sampling": {
272
+ "frame_policy": "causalhybrid32", "frame_timestamps": [0, 10],
273
+ },
274
+ "visual_input_mode": "image-list",
275
+ }]
276
+ )
277
+ selection = {
278
+ "status": "frozen_before_benchmark_temporal_inference",
279
+ "threshold": 0.5, "benchmark": "x", "model_manifest_sha256": "m",
280
+ "score_or_gold_fields_read_for_selection": False,
281
+ "rows": [{"uid": "a", "margin": 0.1, "escalated": True}],
282
+ }
283
+ threshold = {
284
+ "status": "source_disjoint_internal_threshold_calibration",
285
+ "eligible_as_frozen_benchmark_controller_config": True,
286
+ "model_manifest_sha256": "m",
287
+ "policy": {"threshold": 0.5, "commit_threshold": 0.0},
288
+ }
289
+ output = materialize(recent, temporal, selection, threshold)
290
+ assert output["policy"]["second_view"] == (
291
+ "image_list_uniform_history28_plus_recent4"
292
+ )
293
+ assert output["rows"][0]["agent_policy"]["selected_view"] == (
294
+ "causal_uniform_history28_plus_recent4_image_list"
295
+ )
296
+
297
+
298
+ def test_materializer_names_hybrid32_recent8_interface() -> None:
299
+ recent = _result(
300
+ [{
301
+ "uid": "a", "prediction": "A", "query_time": 10,
302
+ "frames_observed": 4,
303
+ "generation_confidence": {"top2_margin": 0.1, "task_or_group_used": False},
304
+ }]
305
+ )
306
+ temporal = _result(
307
+ [{
308
+ "uid": "a", "prediction": "C", "query_time": 10,
309
+ "frames_observed": 32,
310
+ "generation_confidence": {"top2_margin": 0.4, "task_or_group_used": False},
311
+ "sampling": {
312
+ "frame_policy": "causalhybrid32r8", "frame_timestamps": [0, 10],
313
+ },
314
+ "visual_input_mode": "hybrid-history-video-recent-images",
315
+ }]
316
+ )
317
+ selection = {
318
+ "status": "frozen_before_benchmark_temporal_inference",
319
+ "threshold": 0.5, "benchmark": "x", "model_manifest_sha256": "m",
320
+ "score_or_gold_fields_read_for_selection": False,
321
+ "rows": [{"uid": "a", "margin": 0.1, "escalated": True}],
322
+ }
323
+ threshold = {
324
+ "status": "source_disjoint_internal_threshold_calibration",
325
+ "eligible_as_frozen_benchmark_controller_config": True,
326
+ "model_manifest_sha256": "m",
327
+ "policy": {"threshold": 0.5, "commit_threshold": 0.0},
328
+ }
329
+ output = materialize(recent, temporal, selection, threshold)
330
+ assert output["policy"]["second_view"] == (
331
+ "hybrid_history_video_recent_images_causal_hybrid32r8"
332
+ )
333
+ assert output["rows"][0]["agent_policy"]["selected_view"] == (
334
+ "causal_hybrid_history24_plus_recent8"
335
+ )
336
+
337
+
338
+ def test_materializer_accepts_task_blind_fixed_budget_selection() -> None:
339
+ recent = _result(
340
+ [
341
+ {
342
+ "uid": "a",
343
+ "prediction": "A",
344
+ "query_time": 10,
345
+ "frames_observed": 4,
346
+ "generation_confidence": {
347
+ "top2_margin": 0.1,
348
+ "task_or_group_used": False,
349
+ },
350
+ },
351
+ {
352
+ "uid": "b",
353
+ "prediction": "B",
354
+ "query_time": 10,
355
+ "frames_observed": 4,
356
+ },
357
+ ]
358
+ )
359
+ temporal = _result(
360
+ [
361
+ {
362
+ "uid": "a",
363
+ "prediction": "C",
364
+ "query_time": 10,
365
+ "frames_observed": 20,
366
+ "generation_confidence": {
367
+ "top2_margin": 0.3,
368
+ "task_or_group_used": False,
369
+ },
370
+ "sampling": {
371
+ "frame_policy": "causalbudget20",
372
+ "frame_timestamps": [0, 10],
373
+ "candidate_frame_indices": list(range(64)),
374
+ "extra_frames_inspected": 44,
375
+ },
376
+ "visual_input_mode": "native-video",
377
+ }
378
+ ]
379
+ )
380
+ selection = {
381
+ "format": "streameva.unlabeled_budget_uid_selection.v0",
382
+ "status": "frozen_before_benchmark_temporal_inference",
383
+ "benchmark": "x",
384
+ "model_manifest_sha256": "m",
385
+ "quantile": 0.5,
386
+ "score_or_gold_fields_read_for_selection": False,
387
+ "benchmark_identity_used_for_policy_parameters": False,
388
+ "cohort_labels_read_for_selection": False,
389
+ "rows": [
390
+ {
391
+ "uid": "a",
392
+ "margin": 0.1,
393
+ "uncertainty_rank": 1,
394
+ "escalated": True,
395
+ },
396
+ {
397
+ "uid": "b",
398
+ "margin": 0.7,
399
+ "uncertainty_rank": 2,
400
+ "escalated": False,
401
+ },
402
+ ],
403
+ }
404
+ output = materialize(
405
+ recent,
406
+ temporal,
407
+ selection,
408
+ None,
409
+ commit_threshold_override=0.0,
410
+ )
411
+ assert [row["prediction"] for row in output["rows"]] == ["C", "B"]
412
+ assert output["status"] == "fixed_budget_router_free_agent"
413
+ assert output["policy"]["uncertainty_quantile"] == 0.5
414
+ assert output["rows"][0]["agent_policy"]["uncertainty_rank"] == 1
415
+
416
+
417
+ def test_materializer_accounts_for_prefix32_without_hidden_probe() -> None:
418
+ recent = _result(
419
+ [
420
+ {
421
+ "uid": "a",
422
+ "prediction": "A",
423
+ "query_time": 10,
424
+ "frames_observed": 4,
425
+ "generation_confidence": {
426
+ "top2_margin": 0.1,
427
+ "task_or_group_used": False,
428
+ },
429
+ }
430
+ ]
431
+ )
432
+ temporal = _result(
433
+ [
434
+ {
435
+ "uid": "a",
436
+ "prediction": "B",
437
+ "query_time": 10,
438
+ "frames_observed": 32,
439
+ "generation_confidence": {
440
+ "top2_margin": 0.5,
441
+ "task_or_group_used": False,
442
+ },
443
+ "sampling": {
444
+ "frame_policy": "prefix32",
445
+ "frame_timestamps": [0, 10],
446
+ },
447
+ "visual_input_mode": "image-list",
448
+ }
449
+ ]
450
+ )
451
+ selection = {
452
+ "format": "streameva.unlabeled_budget_uid_selection.v0",
453
+ "status": "frozen_before_benchmark_temporal_inference",
454
+ "benchmark": "x",
455
+ "model_manifest_sha256": "m",
456
+ "quantile": 1.0,
457
+ "score_or_gold_fields_read_for_selection": False,
458
+ "benchmark_identity_used_for_policy_parameters": False,
459
+ "cohort_labels_read_for_selection": False,
460
+ "rows": [
461
+ {"uid": "a", "margin": 0.1, "uncertainty_rank": 1, "escalated": True}
462
+ ],
463
+ }
464
+ output = materialize(
465
+ recent,
466
+ temporal,
467
+ selection,
468
+ None,
469
+ commit_threshold_override=-1.0,
470
+ )
471
+ row = output["rows"][0]
472
+ assert row["prediction"] == "B"
473
+ assert row["agent_policy"]["selected_view"] == "causal_prefix32"
474
+ assert row["low_resolution_probe_frames"] == 0
475
+ assert row["model_processed_frames"] == 36
476
+ assert row["total_decoded_frame_instances"] == 36
477
+
478
+
479
+ def test_materializer_can_fail_closed_to_recent_on_temporal_error() -> None:
480
+ recent = _result(
481
+ [
482
+ {
483
+ "uid": "a",
484
+ "prediction": "A",
485
+ "query_time": 10,
486
+ "frames_observed": 4,
487
+ },
488
+ {
489
+ "uid": "b",
490
+ "prediction": "B",
491
+ "query_time": 10,
492
+ "frames_observed": 4,
493
+ },
494
+ ]
495
+ )
496
+ temporal = _result(
497
+ [
498
+ {
499
+ "uid": "a",
500
+ "prediction": "",
501
+ "error": "OutOfMemoryError",
502
+ "query_time": 10,
503
+ "frames_observed": 0,
504
+ },
505
+ {
506
+ "uid": "b",
507
+ "prediction": "C",
508
+ "query_time": 10,
509
+ "frames_observed": 20,
510
+ "sampling": {
511
+ "frame_policy": "causalbudget20",
512
+ "frame_timestamps": [0, 10],
513
+ "candidate_frame_indices": list(range(20)),
514
+ "extra_frames_inspected": 0,
515
+ },
516
+ "visual_input_mode": "image-list",
517
+ },
518
+ ]
519
+ )
520
+ selection = {
521
+ "status": "frozen_before_benchmark_temporal_inference",
522
+ "threshold": 0.3,
523
+ "benchmark": "x",
524
+ "model_manifest_sha256": "m",
525
+ "score_or_gold_fields_read_for_selection": False,
526
+ "rows": [
527
+ {"uid": "a", "margin": 0.1, "escalated": True},
528
+ {"uid": "b", "margin": 0.2, "escalated": True},
529
+ ],
530
+ }
531
+ threshold = {
532
+ "status": "source_disjoint_internal_threshold_calibration",
533
+ "eligible_as_frozen_benchmark_controller_config": True,
534
+ "model_manifest_sha256": "m",
535
+ "policy": {"threshold": 0.3},
536
+ }
537
+ output = materialize(
538
+ recent,
539
+ temporal,
540
+ selection,
541
+ threshold,
542
+ fallback_recent_on_temporal_error=True,
543
+ )
544
+ assert [row["prediction"] for row in output["rows"]] == ["A", "C"]
545
+ assert output["temporal_error_fallback_count"] == 1
546
+ assert output["policy"]["temporal_error_policy"] == "fail_closed_to_recent4"
547
+ assert output["rows"][0]["agent_policy"]["temporal_error_fallback"] is True
548
+ assert output["rows"][0]["agent_policy"]["committed"] is False
549
+ assert output["rows"][0]["agent_policy"]["selected_view"] == (
550
+ "recent4_temporal_error_fallback"
551
+ )
code/tests/test_public_protocol.py ADDED
@@ -0,0 +1,47 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import json
2
+ from pathlib import Path
3
+
4
+ import pytest
5
+
6
+ from experiments.evaluate_qwen3vl_recent4_full import validate_processor_contract
7
+ from scripts.configure_evaluation_protocol import configure
8
+ from viss_core.evaluation_protocol import ProtocolValidationError, load_locked_protocol
9
+
10
+
11
+ TEMPLATE = Path(__file__).resolve().parents[1] / "configs/evaluation/recent4.example.json"
12
+
13
+
14
+ def test_unconfigured_template_cannot_be_used_as_a_locked_protocol():
15
+ with pytest.raises(ProtocolValidationError, match="not locked"):
16
+ load_locked_protocol(TEMPLATE)
17
+
18
+
19
+ def test_user_configured_protocol_preserves_processor_identity_check(tmp_path):
20
+ template = json.loads(TEMPLATE.read_text())
21
+ contract = template["inference"]["processor_contract"]
22
+ processor = tmp_path / "processor"
23
+ processor.mkdir()
24
+ config = processor / "preprocessor_config.json"
25
+ config.write_text(json.dumps({
26
+ key: contract[key]
27
+ for key in ["image_processor_type", "patch_size", "merge_size", "temporal_patch_size"]
28
+ } | {"size": {
29
+ key: contract[key] for key in ["shortest_edge", "longest_edge"]
30
+ }}))
31
+ streaming = tmp_path / "streaming.jsonl"
32
+ ovo = tmp_path / "ovo.jsonl"
33
+ for path in (streaming, ovo):
34
+ path.write_text('{"uid": "synthetic-query"}\n')
35
+ output = tmp_path / "configured.json"
36
+ configure(
37
+ template=TEMPLATE,
38
+ streaming_samples=streaming,
39
+ ovo_samples=ovo,
40
+ processor_config=config,
41
+ output=output,
42
+ )
43
+ protocol = load_locked_protocol(output)
44
+ validate_processor_contract(processor, protocol)
45
+ config.write_text(config.read_text() + " ")
46
+ with pytest.raises(ValueError, match="SHA-256 diverged"):
47
+ validate_processor_contract(processor, protocol)
code/tests/test_recent4_runner.py ADDED
@@ -0,0 +1,893 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import concurrent.futures
2
+ import time
3
+ import unittest
4
+ from pathlib import Path
5
+ from types import SimpleNamespace
6
+ from unittest.mock import patch
7
+
8
+ from experiments.evaluate_qwen3vl_recent4_full import (
9
+ CausalTemporalBudgetCache,
10
+ DecordReaderPool,
11
+ RecentWindowCache,
12
+ _qwen_video_resize_limits,
13
+ frame_policy_sampling_fps,
14
+ resolve_frame_policy,
15
+ clamp_visible_until_to_source,
16
+ normalize_answer,
17
+ probe_source_video,
18
+ recent_window_bounds,
19
+ render_question,
20
+ render_semantic_revisit_prompt,
21
+ selected_uids,
22
+ build_visual_messages,
23
+ compose_temporal_storyboard,
24
+ downscale_temporal_history,
25
+ parse_semantic_revisit_positions,
26
+ native_video_metadata_payload,
27
+ partition_hybrid_frames,
28
+ render_anonymous_candidate_prompt,
29
+ render_candidate_binary_verification_prompt,
30
+ render_candidate_rejection_verification_prompt,
31
+ render_candidate_pairwise_verification_prompt,
32
+ render_candidate_revisit_prompt,
33
+ choose_binary_verified_candidate,
34
+ render_temporal_review_prompt,
35
+ render_temporal_deliberation_prompt,
36
+ render_candidate_evidence_deliberation_prompt,
37
+ decode_causal_temporal_budget,
38
+ decode_causal_hybrid_prefix,
39
+ load_causalbudget_index_plan,
40
+ load_anonymous_candidate_proposals,
41
+ uniform_prefix_indices,
42
+ )
43
+
44
+
45
+ class Recent4RunnerTest(unittest.TestCase):
46
+ def test_anonymous_candidate_sources_may_be_sparse(self) -> None:
47
+ import json
48
+ import tempfile
49
+
50
+ with tempfile.TemporaryDirectory() as directory:
51
+ path = Path(directory) / "partial.json"
52
+ path.write_text(
53
+ json.dumps(
54
+ {
55
+ "benchmark": "streamingbench",
56
+ "annotation_sha256": "annotation",
57
+ "model_manifest_sha256": "model",
58
+ "num_errors": 0,
59
+ "rows": [{"uid": "q0", "prediction": "A"}],
60
+ }
61
+ )
62
+ )
63
+ proposals, inputs = load_anonymous_candidate_proposals(
64
+ paths=[path],
65
+ wanted=["q0", "q1"],
66
+ benchmark="streamingbench",
67
+ annotation_sha256="annotation",
68
+ model_manifest_sha256="model",
69
+ )
70
+ self.assertEqual(proposals, [{"q0": "A"}])
71
+ self.assertEqual(inputs[0]["covered_rows"], 1)
72
+ self.assertEqual(inputs[0]["requested_rows"], 2)
73
+ self.assertTrue(inputs[0]["sparse_candidate_source"])
74
+
75
+ def test_semantic_revisit_parser_is_bounded_and_fills_deterministically(self):
76
+ self.assertEqual(
77
+ parse_semantic_revisit_positions(
78
+ "Frames: 7, 2, 7, 99",
79
+ history_count=8,
80
+ revisit_count=4,
81
+ fallback_positions=[8, 6, 4, 2],
82
+ ),
83
+ [2, 6, 7, 8],
84
+ )
85
+
86
+ def test_semantic_revisit_instruction_overrides_answer_format(self):
87
+ prompt = render_semantic_revisit_prompt(
88
+ "Question\nOnly give the best option's letter directly.",
89
+ history_count=24,
90
+ revisit_count=8,
91
+ )
92
+ self.assertTrue(prompt.endswith("`Frames: i,j,...` and nothing else."))
93
+
94
+ def test_semantic_revisit_storyboard_instruction_has_global_labels(self):
95
+ prompt = render_semantic_revisit_prompt(
96
+ "Question",
97
+ history_count=24,
98
+ revisit_count=8,
99
+ planner_mode="storyboard16",
100
+ )
101
+ self.assertIn("global frame number", prompt)
102
+ self.assertIn("numbered 1 through 24", prompt)
103
+
104
+ def test_storyboard_preserves_chronology_and_fixed_geometry(self) -> None:
105
+ from PIL import Image
106
+
107
+ frames = [Image.new("RGB", (32, 16), color) for color in ("red", "green", "blue")]
108
+ sheet = compose_temporal_storyboard(
109
+ frames, [0.0, 1.0, 2.0], columns=2, tile_width=40, tile_height=20
110
+ )
111
+ self.assertEqual(sheet.size, (80, 80))
112
+ self.assertEqual(sheet.getpixel((20, 10)), (255, 0, 0))
113
+ self.assertEqual(sheet.getpixel((60, 10)), (0, 128, 0))
114
+ self.assertEqual(sheet.getpixel((20, 50)), (0, 0, 255))
115
+
116
+ offset_sheet = compose_temporal_storyboard(
117
+ frames[:2], [3.0, 4.0], columns=2, label_start=17
118
+ )
119
+ self.assertEqual(offset_sheet.size[0], 448)
120
+
121
+ def test_lowres_history_keeps_recent_frames_exact(self) -> None:
122
+ from PIL import Image
123
+
124
+ frames = [Image.new("RGB", (640, 360), color) for color in ("red", "blue")]
125
+ output = downscale_temporal_history(frames, [3, 4], {4}, max_edge=224)
126
+ self.assertEqual(output[0].size, (224, 126))
127
+ self.assertIs(output[1], frames[1])
128
+ self.assertEqual(frames[0].size, (640, 360))
129
+
130
+ def test_deliberation_replaces_conflicting_direct_only_instruction(self) -> None:
131
+ prompt = render_temporal_deliberation_prompt(
132
+ "Question\nOnly give the best option's letter (A, B, C, or D) directly."
133
+ )
134
+ self.assertNotIn("Only give", prompt)
135
+ self.assertIn("Analyze the visible causal sequence", prompt)
136
+ self.assertIn("Answer: X", prompt)
137
+
138
+ def test_anonymous_candidates_can_be_verified_with_one_deliberation_contract(
139
+ self,
140
+ ) -> None:
141
+ prompt, ordered = render_anonymous_candidate_prompt(
142
+ "Question\nOnly give the best option's letter directly.",
143
+ predictions=["A", "C", "A"],
144
+ uid="q0",
145
+ )
146
+ prompt = render_temporal_deliberation_prompt(prompt)
147
+ self.assertEqual(sorted(ordered), ["A", "C"])
148
+ self.assertIn("Candidate 1", prompt)
149
+ self.assertIn("Answer: X", prompt)
150
+
151
+ def test_candidate_evidence_deliberation_requires_pixel_cited_elimination(
152
+ self,
153
+ ) -> None:
154
+ prompt, ordered = render_anonymous_candidate_prompt(
155
+ "Question\nOnly give the best option's letter directly.",
156
+ predictions=["B", "D", "B"],
157
+ uid="q1",
158
+ )
159
+ prompt = render_candidate_evidence_deliberation_prompt(prompt)
160
+ self.assertEqual(sorted(ordered), ["B", "D"])
161
+ self.assertNotIn("Only give", prompt)
162
+ self.assertIn("untrusted hypothesis", prompt)
163
+ self.assertIn("at most ten words per candidate", prompt)
164
+ self.assertIn("do not narrate", prompt)
165
+ self.assertIn("Answer: X", prompt)
166
+
167
+ def test_binary_candidate_verifier_is_isolated_and_removes_direct_contract(self):
168
+ prompt = render_candidate_binary_verification_prompt(
169
+ "Question: Where is it?\nOptions:\nA. left\nB. right\n"
170
+ "Only give the best option's letter directly.",
171
+ candidate="B",
172
+ options=["A. left", "B. right"],
173
+ )
174
+ self.assertNotIn("Only give", prompt)
175
+ self.assertIn("Candidate answer under isolated verification: B. right", prompt)
176
+ self.assertTrue(prompt.endswith("Answer exactly Yes or No."))
177
+
178
+ def test_binary_candidate_commit_is_source_order_invariant(self):
179
+ first = choose_binary_verified_candidate(
180
+ uid="q0", candidate_probabilities={"A": 0.3, "C": 0.8}
181
+ )
182
+ second = choose_binary_verified_candidate(
183
+ uid="q0", candidate_probabilities={"C": 0.8, "A": 0.3}
184
+ )
185
+ self.assertEqual(first, "C")
186
+ self.assertEqual(first, second)
187
+
188
+ def test_antipodal_candidate_prompt_reverses_only_the_decision_polarity(self):
189
+ prompt = render_candidate_rejection_verification_prompt(
190
+ "Question: Where is it?\nOptions:\nA. left\nB. right\n"
191
+ "Only give the best option's letter directly.",
192
+ candidate="B",
193
+ options=["A. left", "B. right"],
194
+ )
195
+ self.assertNotIn("Only give", prompt)
196
+ self.assertIn("isolated rejection check: B. right", prompt)
197
+ self.assertIn("should be rejected as the best answer", prompt)
198
+ self.assertTrue(prompt.endswith("Answer exactly Yes or No."))
199
+
200
+ def test_pairwise_candidate_prompt_is_source_anonymous_and_order_explicit(self):
201
+ prompt = render_candidate_pairwise_verification_prompt(
202
+ "Question: Where is it?\nOptions:\nA. left\nB. right\n"
203
+ "Only give the best option's letter directly.",
204
+ first_candidate="B",
205
+ second_candidate="A",
206
+ options=["A. left", "B. right"],
207
+ )
208
+ self.assertNotIn("Only give", prompt)
209
+ self.assertIn("Choice A: B. right", prompt)
210
+ self.assertIn("Choice B: A. left", prompt)
211
+ self.assertTrue(prompt.endswith("Answer exactly A or B."))
212
+
213
+ def test_candidate_conditioned_revisit_is_bounded_and_answer_free(self):
214
+ prompt = render_candidate_revisit_prompt(
215
+ "Question: What happened?\nOptions:\nA. left\nB. right\n"
216
+ "Only give the best option's letter directly.",
217
+ candidate="A",
218
+ options=["A. left", "B. right"],
219
+ history_count=24,
220
+ revisit_count=4,
221
+ )
222
+ self.assertNotIn("Only give", prompt)
223
+ self.assertIn("Candidate answer under isolated verification: A. left", prompt)
224
+ self.assertIn("Return exactly 4 distinct frame numbers", prompt)
225
+ self.assertTrue(prompt.endswith("and nothing else."))
226
+
227
+ def test_deliberative_explicit_answer_has_priority_for_open_outputs(self) -> None:
228
+ self.assertEqual(
229
+ normalize_answer(
230
+ "There are 2 early events, then one more. Answer: 3",
231
+ "Answer with a single number indicating the total count.",
232
+ [],
233
+ ),
234
+ "3",
235
+ )
236
+ self.assertEqual(
237
+ normalize_answer(
238
+ "The first condition suggests no, but the final state does. Answer: Yes",
239
+ 'Please answer only with "Yes" or "No".',
240
+ [],
241
+ ),
242
+ "Yes",
243
+ )
244
+
245
+ def test_causalbudget_index_plan_requires_question_blind_complete_coverage(
246
+ self,
247
+ ) -> None:
248
+ import json
249
+ import tempfile
250
+
251
+ with tempfile.TemporaryDirectory() as directory:
252
+ path = Path(directory) / "plan.json"
253
+ path.write_text(
254
+ json.dumps(
255
+ {
256
+ "format": "iclr_video.causalbudget20_index_plan.v0",
257
+ "status": "complete_query_independent_execution_cache",
258
+ "benchmark": "ovo",
259
+ "contracts": {
260
+ "question_text_read_for_plan": False,
261
+ "task_or_group_read_for_plan": False,
262
+ "model_prediction_read_for_plan": False,
263
+ "gold_answer_read": False,
264
+ "future_frames_read": False,
265
+ },
266
+ "rows": [{"uid": "u1", "sampling": {}}],
267
+ }
268
+ )
269
+ )
270
+ rows, artifact = load_causalbudget_index_plan(
271
+ path, benchmark="ovo", wanted=["u1"]
272
+ )
273
+ self.assertEqual(set(rows), {"u1"})
274
+ self.assertEqual(artifact["path"], str(path.resolve()))
275
+
276
+ value = json.loads(path.read_text())
277
+ value.update(
278
+ {
279
+ "format": "iclr_video.causalbudget_index_plan.v1",
280
+ "model_frames": 32,
281
+ "recent_frames": 8,
282
+ }
283
+ )
284
+ path.write_text(json.dumps(value))
285
+ rows, _ = load_causalbudget_index_plan(
286
+ path,
287
+ benchmark="ovo",
288
+ wanted=["u1"],
289
+ model_frames=32,
290
+ recent_frames=8,
291
+ )
292
+ self.assertEqual(set(rows), {"u1"})
293
+
294
+ def test_temporal_budget_cache_reuses_exact_query_independent_prefix(self) -> None:
295
+ decoder_path = (
296
+ "experiments.evaluate_qwen3vl_recent4_full."
297
+ "decode_causal_temporal_budget"
298
+ )
299
+ with patch(
300
+ decoder_path,
301
+ return_value=([{"pixels": [1]}], {"frame_indices": [3]}),
302
+ ) as decoder:
303
+ cache = CausalTemporalBudgetCache(2)
304
+ frames, sampling, hit = cache.get(Path("v.mp4"), visible_until=4.0)
305
+ self.assertFalse(hit)
306
+ frames[0]["pixels"] = [2]
307
+ sampling["frame_indices"].append(4)
308
+ frames2, sampling2, hit2 = cache.get(
309
+ Path("v.mp4"), visible_until=4.0
310
+ )
311
+ self.assertTrue(hit2)
312
+ self.assertEqual(frames2, [{"pixels": [1]}])
313
+ self.assertEqual(sampling2, {"frame_indices": [3]})
314
+ cache.get(Path("v.mp4"), visible_until=5.0)
315
+ self.assertEqual(decoder.call_count, 2)
316
+
317
+ def test_native_video_turn_has_one_video_placeholder_and_no_route(self) -> None:
318
+ frames = [object(), object(), object(), object()]
319
+ messages = build_visual_messages(
320
+ frames=frames,
321
+ prompt="question",
322
+ visual_input_mode="native-video",
323
+ )
324
+ self.assertEqual(messages[0]["content"][0], {"type": "video"})
325
+ self.assertEqual(messages[0]["content"][1]["text"], "question")
326
+
327
+ def test_hybrid_turn_keeps_history_as_video_and_recent_as_images(self) -> None:
328
+ history = [object(), object()]
329
+ recent = [object(), object(), object(), object()]
330
+ messages = build_visual_messages(
331
+ frames=[*history, *recent],
332
+ history_frames=history,
333
+ recent_frames=recent,
334
+ prompt="question",
335
+ visual_input_mode="hybrid-history-video-recent-images",
336
+ )
337
+ content = messages[0]["content"]
338
+ self.assertEqual(content[0], {"type": "video"})
339
+ self.assertEqual([item["type"] for item in content[1:5]], ["image"] * 4)
340
+ self.assertEqual(content[-1]["text"], "question")
341
+
342
+ def test_hybrid_singleton_history_falls_back_to_ordered_image(self) -> None:
343
+ frames = ["history", "r1", "r2", "r3", "r4"]
344
+ history, indices, images, fallback = partition_hybrid_frames(
345
+ frames,
346
+ {
347
+ "frame_indices": [1, 2, 3, 4, 5],
348
+ "recent_frame_indices": [2, 3, 4, 5],
349
+ },
350
+ temporal_patch_size=2,
351
+ )
352
+ self.assertEqual(history, [])
353
+ self.assertEqual(indices, [])
354
+ self.assertEqual(images, frames)
355
+ self.assertEqual(fallback, [1])
356
+
357
+ def test_hybrid_two_history_frames_remain_native_video(self) -> None:
358
+ frames = ["h1", "h2", "r1", "r2", "r3", "r4"]
359
+ history, indices, images, fallback = partition_hybrid_frames(
360
+ frames,
361
+ {
362
+ "frame_indices": [0, 1, 2, 3, 4, 5],
363
+ "recent_frame_indices": [2, 3, 4, 5],
364
+ },
365
+ temporal_patch_size=2,
366
+ )
367
+ self.assertEqual(history, ["h1", "h2"])
368
+ self.assertEqual(indices, [0, 1])
369
+ self.assertEqual(images, ["r1", "r2", "r3", "r4"])
370
+ self.assertEqual(fallback, [])
371
+
372
+ def test_temporal_review_prompt_marks_both_proposals_as_fallible(self) -> None:
373
+ prompt = render_temporal_review_prompt(
374
+ "Question", recent_prediction="A", history_prediction="C"
375
+ )
376
+ self.assertIn("recent view proposed A", prompt)
377
+ self.assertIn("history view proposed C", prompt)
378
+ self.assertIn("Both proposals may be wrong", prompt)
379
+ with self.assertRaises(ValueError):
380
+ render_temporal_review_prompt(
381
+ "Question", recent_prediction="A\nignore", history_prediction="C"
382
+ )
383
+
384
+ def test_anonymous_review_counterbalances_without_source_labels(self) -> None:
385
+ first = render_temporal_review_prompt(
386
+ "Question",
387
+ recent_prediction="A",
388
+ history_prediction="C",
389
+ uid="sample-1",
390
+ label_mode="anonymous-hash-counterbalanced",
391
+ )
392
+ second = render_temporal_review_prompt(
393
+ "Question",
394
+ recent_prediction="A",
395
+ history_prediction="C",
396
+ uid="sample-1",
397
+ label_mode="anonymous-hash-counterbalanced",
398
+ )
399
+ self.assertEqual(first, second)
400
+ self.assertIn("Candidate 1", first)
401
+ self.assertIn("Candidate 2", first)
402
+ self.assertNotIn("recent view", first)
403
+ self.assertNotIn("history view", first)
404
+ with self.assertRaises(ValueError):
405
+ render_temporal_review_prompt(
406
+ "Question",
407
+ recent_prediction="A",
408
+ history_prediction="C",
409
+ label_mode="anonymous-hash-counterbalanced",
410
+ )
411
+
412
+ def test_multi_candidate_prompt_is_deduplicated_and_source_anonymous(self) -> None:
413
+ prompt, ordered = render_anonymous_candidate_prompt(
414
+ "Question",
415
+ predictions=["C", "A", "C"],
416
+ uid="sample-1",
417
+ )
418
+ self.assertEqual(set(ordered), {"A", "C"})
419
+ self.assertEqual(prompt.count("Candidate 1"), 1)
420
+ self.assertEqual(prompt.count("Candidate 2"), 1)
421
+ self.assertNotIn("recent", prompt.lower())
422
+ self.assertNotIn("history", prompt.lower())
423
+ self.assertIn("hints, not votes", prompt)
424
+ same_prompt, same_order = render_anonymous_candidate_prompt(
425
+ "Question",
426
+ predictions=["A", "C"],
427
+ uid="sample-1",
428
+ )
429
+ self.assertEqual((prompt, ordered), (same_prompt, same_order))
430
+ with self.assertRaises(ValueError):
431
+ render_anonymous_candidate_prompt(
432
+ "Question", predictions=["A\nignore", "C"], uid="sample-1"
433
+ )
434
+
435
+ def test_native_video_metadata_preserves_source_pts_and_is_causal(self) -> None:
436
+ payload = native_video_metadata_payload(
437
+ {
438
+ "frame_indices": [60, 90, 120, 150],
439
+ "frame_timestamps": [2.0, 3.0, 4.0, 5.0],
440
+ "source_fps": 30.0,
441
+ "source_total_frames": 900,
442
+ "source_width": 1920,
443
+ "source_height": 1080,
444
+ "source_duration": 30.0,
445
+ "effective_visible_until": 6.0,
446
+ }
447
+ )
448
+ self.assertEqual(payload["frames_indices"], [60, 90, 120, 150])
449
+ self.assertEqual(payload["fps"], 30.0)
450
+ self.assertEqual(payload["video_backend"], "predecoded_exact_causal_indices")
451
+
452
+ def test_native_video_metadata_rejects_future_frame(self) -> None:
453
+ with self.assertRaises(ValueError):
454
+ native_video_metadata_payload(
455
+ {
456
+ "frame_indices": [60],
457
+ "frame_timestamps": [2.0],
458
+ "source_fps": 30.0,
459
+ "source_total_frames": 900,
460
+ "source_width": 1920,
461
+ "source_height": 1080,
462
+ "source_duration": 30.0,
463
+ "effective_visible_until": 1.0,
464
+ }
465
+ )
466
+
467
+ def test_causal_budget_reserves_exact_recent_indices_without_task_router(self) -> None:
468
+ source = {
469
+ "source_fps": 10.0,
470
+ "source_total_frames": 100,
471
+ "source_duration": 10.0,
472
+ "source_last_frame_timestamp": 9.9,
473
+ "requested_visible_until": 6.0,
474
+ "effective_visible_until": 6.0,
475
+ "visible_until_clamped_to_source": False,
476
+ "window_start": 0.0,
477
+ "window_end_exclusive": 6.0,
478
+ "source_height": 100,
479
+ "source_width": 100,
480
+ }
481
+ prefix = [0, 10, 20, 30, 40, 50]
482
+ recent = [30, 40, 50, 60]
483
+ probe_features = [[float(i), 1.0] for i in range(7)]
484
+ selected_frames = [f"frame-{i}" for i in range(7)]
485
+ with patch(
486
+ "experiments.evaluate_qwen3vl_recent4_full.exact_causal_window_indices",
487
+ side_effect=[(prefix, source), (recent, source)],
488
+ ), patch(
489
+ "experiments.evaluate_qwen3vl_recent4_full.decode_exact_indices_ffmpeg",
490
+ side_effect=[probe_features, selected_frames],
491
+ ):
492
+ frames, sampling = decode_causal_temporal_budget(
493
+ Path("video.mp4"), visible_until=6.0
494
+ )
495
+ self.assertEqual(frames, selected_frames)
496
+ self.assertTrue({30, 40, 50, 60}.issubset(sampling["frame_indices"]))
497
+ self.assertFalse(
498
+ sampling["controller"]["benchmark_task_or_group_router"]
499
+ )
500
+ self.assertEqual(
501
+ sampling["controller"]["policy_config"]["query_weight"], 0.0
502
+ )
503
+ self.assertEqual(sampling["controller"]["future_frame_violations"], 0)
504
+
505
+ def test_uniform_history_has_unique_indices_and_no_padding(self) -> None:
506
+ self.assertEqual(uniform_prefix_indices(-1, 28), [])
507
+ self.assertEqual(uniform_prefix_indices(2, 28), [0, 1, 2])
508
+ values = uniform_prefix_indices(99, 28)
509
+ self.assertEqual(len(values), 28)
510
+ self.assertEqual(values, sorted(set(values)))
511
+ self.assertEqual((values[0], values[-1]), (0, 99))
512
+
513
+ def test_hybrid32_preserves_recent4_as_separate_exact_indices(self) -> None:
514
+ source = {
515
+ "source_fps": 10.0,
516
+ "source_total_frames": 100,
517
+ "source_duration": 10.0,
518
+ "source_last_frame_timestamp": 9.9,
519
+ "requested_visible_until": 6.0,
520
+ "effective_visible_until": 6.0,
521
+ "visible_until_clamped_to_source": False,
522
+ "window_start": 2.0,
523
+ "window_end_exclusive": 6.0,
524
+ "source_height": 100,
525
+ "source_width": 100,
526
+ }
527
+ recent = [30, 40, 50, 60]
528
+ with patch(
529
+ "experiments.evaluate_qwen3vl_recent4_full.exact_causal_window_indices",
530
+ return_value=(recent, source),
531
+ ), patch(
532
+ "experiments.evaluate_qwen3vl_recent4_full.decode_exact_indices_ffmpeg",
533
+ side_effect=lambda _path, indices: [f"frame-{value}" for value in indices],
534
+ ):
535
+ frames, sampling = decode_causal_hybrid_prefix(
536
+ Path("video.mp4"), visible_until=6.0
537
+ )
538
+ self.assertEqual(sampling["recent_frame_indices"], recent)
539
+ self.assertEqual(len(sampling["history_frame_indices"]), 28)
540
+ self.assertEqual(len(sampling["frame_indices"]), 32)
541
+ self.assertEqual(sampling["frame_indices"], sorted(set(sampling["frame_indices"])))
542
+ self.assertEqual(len(frames), 32)
543
+ self.assertFalse(sampling["controller"]["benchmark_task_or_group_router"])
544
+ self.assertFalse(sampling["controller"]["question_keyword_router"])
545
+
546
+ def test_qwen_video_resize_limits_supports_pixel_api(self) -> None:
547
+ module = SimpleNamespace(
548
+ IMAGE_FACTOR=28,
549
+ VIDEO_MIN_PIXELS=100_352,
550
+ VIDEO_MAX_PIXELS=602_112,
551
+ VIDEO_TOTAL_PIXELS=90_316_800,
552
+ FRAME_FACTOR=2,
553
+ )
554
+ self.assertEqual(
555
+ _qwen_video_resize_limits(module, 32),
556
+ (28, 100_352, 602_112),
557
+ )
558
+
559
+ def test_qwen_video_resize_limits_supports_token_api(self) -> None:
560
+ module = SimpleNamespace(
561
+ SPATIAL_MERGE_SIZE=2,
562
+ VIDEO_MIN_TOKEN_NUM=128,
563
+ VIDEO_MAX_TOKEN_NUM=768,
564
+ MODEL_SEQ_LEN=128_000,
565
+ FRAME_FACTOR=2,
566
+ )
567
+ factor, minimum, maximum = _qwen_video_resize_limits(module, 32)
568
+ self.assertEqual((factor, minimum), (28, 100_352))
569
+ self.assertEqual(maximum, 602_112)
570
+
571
+ def test_training_free_active4_policies_only_change_official_sampling_fps(self) -> None:
572
+ self.assertEqual(frame_policy_sampling_fps("recent4"), 1.0)
573
+ self.assertEqual(frame_policy_sampling_fps("wide8"), 0.5)
574
+ self.assertAlmostEqual(frame_policy_sampling_fps("wide12"), 1.0 / 3.0)
575
+ self.assertEqual(frame_policy_sampling_fps("wide16"), 0.25)
576
+
577
+ def test_modelonly_frame_policy_is_question_independent(self) -> None:
578
+ self.assertEqual(resolve_frame_policy("recent4"), "recent4")
579
+ self.assertEqual(resolve_frame_policy("wide12"), "wide12")
580
+ self.assertEqual(resolve_frame_policy("prefix8"), "prefix8")
581
+ self.assertEqual(resolve_frame_policy("prefix32"), "prefix32")
582
+ self.assertEqual(resolve_frame_policy("causalbudget20"), "causalbudget20")
583
+ self.assertEqual(
584
+ resolve_frame_policy("causalbudget32r8"), "causalbudget32r8"
585
+ )
586
+ self.assertEqual(resolve_frame_policy("causalhybrid32"), "causalhybrid32")
587
+ self.assertEqual(
588
+ resolve_frame_policy("causalhybrid32r8"), "causalhybrid32r8"
589
+ )
590
+ with self.assertRaises(ValueError):
591
+ resolve_frame_policy("adaptive_query_budget32")
592
+
593
+ def test_source_probe_has_small_bounded_cache(self) -> None:
594
+ self.assertEqual(probe_source_video.cache_parameters()["maxsize"], 8)
595
+
596
+ def test_decord_reader_pool_bounds_native_decoder_threads(self) -> None:
597
+ calls = []
598
+
599
+ class Reader:
600
+ pass
601
+
602
+ def video_reader(path, *, num_threads):
603
+ calls.append((path, num_threads))
604
+ return Reader()
605
+
606
+ pool = DecordReaderPool(2, num_threads=3)
607
+ with patch.dict(
608
+ "sys.modules", {"decord": SimpleNamespace(VideoReader=video_reader)}
609
+ ):
610
+ first, first_hit = pool.reader("a.mp4")
611
+ second, second_hit = pool.reader("a.mp4")
612
+
613
+ self.assertIs(first, second)
614
+ self.assertFalse(first_hit)
615
+ self.assertTrue(second_hit)
616
+ self.assertEqual(calls, [("a.mp4", 3)])
617
+
618
+ def test_decord_reader_pool_serializes_same_video_prefetch(self) -> None:
619
+ state_lock = __import__("threading").Lock()
620
+ active = 0
621
+ peak = 0
622
+ instances = 0
623
+
624
+ class Frame:
625
+ shape = (16, 32, 3)
626
+
627
+ class Reader:
628
+ def get_avg_fps(self):
629
+ nonlocal active, peak
630
+ with state_lock:
631
+ active += 1
632
+ peak = max(peak, active)
633
+ time.sleep(0.01)
634
+ with state_lock:
635
+ active -= 1
636
+ return 30.0
637
+
638
+ def __len__(self):
639
+ return 10
640
+
641
+ def __getitem__(self, index):
642
+ return Frame()
643
+
644
+ def video_reader(path, *, num_threads):
645
+ nonlocal instances
646
+ instances += 1
647
+ return Reader()
648
+
649
+ pool = DecordReaderPool(2, num_threads=3)
650
+ with patch.dict(
651
+ "sys.modules", {"decord": SimpleNamespace(VideoReader=video_reader)}
652
+ ):
653
+ with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor:
654
+ values = list(executor.map(pool.metadata, ["a.mp4"] * 8))
655
+
656
+ self.assertEqual(values, [(30.0, 10, 16, 32)] * 8)
657
+ self.assertEqual(instances, 1)
658
+ self.assertEqual(peak, 1)
659
+
660
+ def test_exact_decode_cache_reuses_only_matching_sampling_request(self) -> None:
661
+ from pathlib import Path
662
+
663
+ class Frame:
664
+ def __init__(self, value: int) -> None:
665
+ self.value = value
666
+
667
+ def copy(self):
668
+ return Frame(self.value)
669
+
670
+ calls = []
671
+
672
+ def decoder(path, **kwargs):
673
+ calls.append((path, kwargs))
674
+ return [Frame(len(calls))], {"frame_indices": [len(calls)]}
675
+
676
+ cache = RecentWindowCache(2, decoder=decoder)
677
+ first_frames, first_sampling, first_hit = cache.get(
678
+ Path("video.mp4"),
679
+ visible_until=10.0,
680
+ recent_frames=4,
681
+ sampling_fps=1.0,
682
+ )
683
+ first_frames[0].value = 99
684
+ first_sampling["frame_indices"][0] = 99
685
+ second_frames, second_sampling, second_hit = cache.get(
686
+ Path("video.mp4"),
687
+ visible_until=10.0,
688
+ recent_frames=4,
689
+ sampling_fps=1.0,
690
+ )
691
+ _, _, third_hit = cache.get(
692
+ Path("video.mp4"),
693
+ visible_until=11.0,
694
+ recent_frames=4,
695
+ sampling_fps=1.0,
696
+ )
697
+
698
+ self.assertFalse(first_hit)
699
+ self.assertTrue(second_hit)
700
+ self.assertFalse(third_hit)
701
+ self.assertEqual(len(calls), 2)
702
+ self.assertEqual(second_frames[0].value, 1)
703
+ self.assertEqual(second_sampling["frame_indices"], [1])
704
+ self.assertEqual(second_sampling["frame_policy"], "recent4")
705
+ self.assertEqual(second_sampling["policy_sampling_fps"], 1.0)
706
+ self.assertEqual((cache.hits, cache.misses), (1, 2))
707
+
708
+ def test_prefix_policy_uses_dedicated_sequential_decoder(self) -> None:
709
+ cache = RecentWindowCache(0)
710
+ decoded = ([object()], {"frame_indices": [0]})
711
+ with patch(
712
+ "experiments.evaluate_qwen3vl_recent4_full.decode_prefix_window_ffmpeg",
713
+ return_value=decoded,
714
+ ) as decoder:
715
+ frames, sampling, hit = cache.get(
716
+ Path("video.mp4"),
717
+ visible_until=10.0,
718
+ recent_frames=4,
719
+ sampling_fps=1.0,
720
+ frame_policy="prefix32",
721
+ )
722
+
723
+ self.assertIs(frames, decoded[0])
724
+ self.assertFalse(hit)
725
+ self.assertEqual(sampling["frame_policy"], "prefix32")
726
+ decoder.assert_called_once_with(
727
+ Path("video.mp4"),
728
+ visible_until=10.0,
729
+ recent_frames=32,
730
+ sampling_fps=3.2,
731
+ )
732
+
733
+ def test_exact_decode_cache_is_safe_for_parallel_prefix_misses(self) -> None:
734
+ class Frame:
735
+ def __init__(self, value: str) -> None:
736
+ self.value = value
737
+
738
+ def copy(self):
739
+ return Frame(self.value)
740
+
741
+ def decoder(path, **kwargs):
742
+ time.sleep(0.01)
743
+ return [Frame(str(path))], {"frame_indices": [str(path)]}
744
+
745
+ cache = RecentWindowCache(8, decoder=decoder)
746
+ paths = [Path(f"video-{index}.mp4") for index in range(8)]
747
+ with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor:
748
+ values = list(
749
+ executor.map(
750
+ lambda path: cache.get(
751
+ path,
752
+ visible_until=10.0,
753
+ recent_frames=32,
754
+ sampling_fps=3.2,
755
+ frame_policy="prefix32",
756
+ ),
757
+ paths,
758
+ )
759
+ )
760
+ self.assertEqual([value[0][0].value for value in values], list(map(str, paths)))
761
+ self.assertEqual(cache.misses, 8)
762
+
763
+ def test_exact_decode_cache_singleflights_matching_parallel_requests(self) -> None:
764
+ class Frame:
765
+ def copy(self):
766
+ return Frame()
767
+
768
+ calls = []
769
+
770
+ def decoder(path, **kwargs):
771
+ calls.append((path, kwargs))
772
+ time.sleep(0.05)
773
+ return [Frame()], {"frame_indices": [1]}
774
+
775
+ cache = RecentWindowCache(8, decoder=decoder)
776
+
777
+ def request(_):
778
+ return cache.get(
779
+ Path("same-video.mp4"),
780
+ visible_until=10.0,
781
+ recent_frames=4,
782
+ sampling_fps=1.0,
783
+ )
784
+
785
+ with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:
786
+ values = list(executor.map(request, range(16)))
787
+
788
+ self.assertEqual(len(calls), 1)
789
+ self.assertEqual(cache.misses, 1)
790
+ self.assertEqual(cache.hits, 15)
791
+ self.assertGreaterEqual(cache.singleflight_hits, 1)
792
+ self.assertEqual(sum(bool(value[2]) for value in values), 15)
793
+
794
+ def test_visible_prefix_is_clamped_to_last_real_frame(self) -> None:
795
+ effective, clamped = clamp_visible_until_to_source(10.0, 2.0, 9)
796
+ self.assertEqual(effective, 4.0)
797
+ self.assertTrue(clamped)
798
+ effective, clamped = clamp_visible_until_to_source(3.0, 2.0, 9)
799
+ self.assertEqual(effective, 3.0)
800
+ self.assertFalse(clamped)
801
+
802
+ def test_explicit_uid_subset_keeps_export_order(self) -> None:
803
+ import tempfile
804
+ from pathlib import Path
805
+
806
+ with tempfile.TemporaryDirectory() as directory:
807
+ path = Path(directory) / "samples.jsonl"
808
+ path.write_text(
809
+ '{"uid":"a"}\n{"uid":"b"}\n{"uid":"c"}\n',
810
+ encoding="utf-8",
811
+ )
812
+ self.assertEqual(
813
+ selected_uids(path, 0, -1, ["c", "a"]),
814
+ ["a", "c"],
815
+ )
816
+
817
+ def test_batched_runner_keeps_auditable_resume_controls(self) -> None:
818
+ source = Path("experiments/evaluate_qwen3vl_recent4_full.py").read_text(
819
+ encoding="utf-8"
820
+ )
821
+ launcher = Path("scripts/run_recent4_single_benchmark.sh").read_text(
822
+ encoding="utf-8"
823
+ )
824
+ self.assertIn('processor.tokenizer.padding_side = "left"', source)
825
+ self.assertIn("validate_processor_contract(processor_path, protocol)", source)
826
+ self.assertIn('"inference_batch_size": args.batch_size', source)
827
+ self.assertIn('"execution_optimization",', source)
828
+ self.assertIn('--uids-file "$uid_file"', launcher)
829
+ self.assertIn('uids[index::count]', launcher)
830
+ self.assertIn('--batch-size "$attempt_batch"', launcher)
831
+ self.assertIn('attempt_batch=$EVAL_BATCH_SIZE', launcher)
832
+ self.assertIn('|| smoke_rc=$?', launcher)
833
+ self.assertIn('|| runner_rc=$?', launcher)
834
+ self.assertIn(
835
+ 'after writing a complete validated result',
836
+ launcher,
837
+ )
838
+
839
+ def test_recent_window_is_half_open_and_causal(self) -> None:
840
+ self.assertEqual(recent_window_bounds(10.0, 4, 1.0), (6.0, 10.0))
841
+ self.assertEqual(recent_window_bounds(1.0, 4, 1.0), (0.0, 1.0))
842
+
843
+ def test_answer_normalization(self) -> None:
844
+ options = ["A. first", "B. second", "C. third", "D. fourth"]
845
+ self.assertEqual(normalize_answer("(C) third", "question", options), "C")
846
+ self.assertEqual(normalize_answer("second", "question", options), "B")
847
+ self.assertEqual(normalize_answer("A map", "question", ["A. compass", "B. A map", "C. sky", "D. radar"]), "B")
848
+ self.assertEqual(
849
+ normalize_answer(
850
+ "Yes",
851
+ "Answer only with “Yes” or “No”.",
852
+ [],
853
+ ),
854
+ "Yes",
855
+ )
856
+
857
+ def test_streaming_prompt_matches_release_contract(self) -> None:
858
+ prompt = render_question(
859
+ "streamingbench",
860
+ "What is visible?",
861
+ ["A. first", "B. second", "C. third", "D. fourth"],
862
+ )
863
+ self.assertIn("advanced video question-answering AI assistant", prompt)
864
+ self.assertTrue(prompt.endswith(
865
+ "Only give the best option's letter (A, B, C, or D) directly."
866
+ ))
867
+
868
+ def test_ovo_mcq_prompt_matches_release_contract(self) -> None:
869
+ prompt = render_question(
870
+ "ovo",
871
+ "What happened?",
872
+ ["one", "two", "three", "four"],
873
+ )
874
+ self.assertEqual(
875
+ prompt,
876
+ "What happened?\nOptions: A. one; B. two; C. three; D. four;"
877
+ "\nOnly give the best option's letter directly.",
878
+ )
879
+
880
+ def test_ovo_requires_full_runtime_asset_attestation(self) -> None:
881
+ launcher = Path("scripts/run_recent4_single_benchmark.sh").read_text(
882
+ encoding="utf-8"
883
+ )
884
+ self.assertIn("OVO_RUNTIME_REPORT", launcher)
885
+ self.assertIn(".expanded_queries == 3035", launcher)
886
+ self.assertIn(".covered_queries == 3035", launcher)
887
+ self.assertIn(".missing_videos == 0", launcher)
888
+ self.assertIn(".annotation_sha256 == $annotation_sha256", launcher)
889
+ self.assertIn(".frozen_export_sha256 == $frozen_export_sha256", launcher)
890
+
891
+
892
+ if __name__ == "__main__":
893
+ unittest.main()
code/tests/test_semantic_answer_loss_mask.py ADDED
@@ -0,0 +1,108 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import torch
4
+
5
+ from experiments.train_qwen3vl_sft import _assistant_only_labels_and_weights
6
+
7
+
8
+ class _CharacterTokenizer:
9
+ im_end_id = 1
10
+
11
+ def __call__(
12
+ self,
13
+ text: str,
14
+ *,
15
+ add_special_tokens: bool = False,
16
+ return_offsets_mapping: bool = False,
17
+ ) -> dict[str, object]:
18
+ del add_special_tokens
19
+ value: dict[str, object] = {
20
+ "input_ids": [ord(character) + 10 for character in text]
21
+ }
22
+ if return_offsets_mapping:
23
+ value["offset_mapping"] = [
24
+ (index, index + 1) for index in range(len(text))
25
+ ]
26
+ return value
27
+
28
+ def convert_tokens_to_ids(self, token: str) -> int | None:
29
+ return self.im_end_id if token == "<|im_end|>" else None
30
+
31
+
32
+ class _BoundaryMergingTokenizer:
33
+ def apply_chat_template(
34
+ self,
35
+ messages: list[dict],
36
+ *,
37
+ tokenize: bool,
38
+ add_generation_prompt: bool,
39
+ ) -> str:
40
+ assert tokenize is False and add_generation_prompt is False
41
+ return "".join(
42
+ item["text"]
43
+ for message in messages
44
+ for item in message["content"]
45
+ if item["type"] == "text"
46
+ )
47
+
48
+ def __call__(
49
+ self,
50
+ text: str,
51
+ *,
52
+ add_special_tokens: bool = False,
53
+ return_offsets_mapping: bool = False,
54
+ ) -> dict[str, object]:
55
+ assert add_special_tokens is False
56
+ if text == "ABC":
57
+ value: dict[str, object] = {"input_ids": [100, 101]}
58
+ if return_offsets_mapping:
59
+ value["offset_mapping"] = [(0, 2), (2, 3)]
60
+ return value
61
+ return {"input_ids": [200 + ord(character) for character in text]}
62
+
63
+ def convert_tokens_to_ids(self, token: str) -> None:
64
+ del token
65
+ return None
66
+
67
+
68
+ def test_only_assistant_semantic_answer_and_terminator_receive_loss() -> None:
69
+ tokenizer = _CharacterTokenizer()
70
+ question = "A"
71
+ answer = "B"
72
+ input_ids = torch.tensor(
73
+ tokenizer(question)["input_ids"]
74
+ + tokenizer(answer)["input_ids"]
75
+ + [tokenizer.im_end_id],
76
+ dtype=torch.long,
77
+ )
78
+ messages = [
79
+ {
80
+ "role": "user",
81
+ "content": [{"type": "text", "text": question}],
82
+ "loss": False,
83
+ },
84
+ {
85
+ "role": "assistant",
86
+ "content": [{"type": "text", "text": answer}],
87
+ "loss": True,
88
+ },
89
+ ]
90
+ labels, weights = _assistant_only_labels_and_weights(
91
+ tokenizer, input_ids, messages
92
+ )
93
+ assert labels.tolist() == [-100, ord("B") + 10, tokenizer.im_end_id]
94
+ assert weights.tolist() == [0.0, 1.0, 1.0]
95
+
96
+
97
+ def test_rendered_offsets_handle_tokens_crossing_message_boundaries() -> None:
98
+ tokenizer = _BoundaryMergingTokenizer()
99
+ messages = [
100
+ {"role": "user", "content": [{"type": "text", "text": "A"}]},
101
+ {"role": "assistant", "content": [{"type": "text", "text": "B"}]},
102
+ {"role": "user", "content": [{"type": "text", "text": "C"}]},
103
+ ]
104
+ labels, weights = _assistant_only_labels_and_weights(
105
+ tokenizer, torch.tensor([100, 101]), messages
106
+ )
107
+ assert labels.tolist() == [100, -100]
108
+ assert weights.tolist() == [1.0, 0.0]
code/tests/test_size_matched_generic_control.py ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from data_pipeline.temporal_scope.build_size_matched_generic_control import (
2
+ select_parent_ids,
3
+ )
4
+
5
+
6
+ def _row(parent_id: str, kind: str) -> dict:
7
+ return {"parents": [{"id": parent_id, "kind": kind}]}
8
+
9
+
10
+ def test_selection_is_deterministic_and_stratified() -> None:
11
+ rows = [
12
+ *[_row(f"source-{i}", "source_semantic_row") for i in range(8)],
13
+ *[
14
+ _row(f"cited-{i}", "cited_nonbenchmark_refresh_projection")
15
+ for i in range(5)
16
+ ],
17
+ ]
18
+ quotas = {
19
+ "source_semantic_row": 4,
20
+ "cited_nonbenchmark_refresh_projection": 2,
21
+ }
22
+ first, counts = select_parent_ids(rows, seed="control-v1", quotas=quotas)
23
+ second, _ = select_parent_ids(list(reversed(rows)), seed="control-v1", quotas=quotas)
24
+ assert first == second
25
+ assert len(first) == 6
26
+ assert counts == quotas
code/tests/test_source_aligned_sft_release.py ADDED
@@ -0,0 +1,125 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from pathlib import Path
2
+
3
+ from data_pipeline.temporal_scope.build_source_aligned_sft_release import (
4
+ SourceAlignedSFTError,
5
+ _canonical_media_path,
6
+ _semantic_identity,
7
+ _teacher_used_for_labels,
8
+ )
9
+ import pytest
10
+
11
+
12
+ def _row(image: str = "media/one.jpg", question: str = "Which one?") -> dict:
13
+ return {
14
+ "messages": [
15
+ {
16
+ "role": "user",
17
+ "loss": False,
18
+ "content": [
19
+ {"type": "image", "image": image},
20
+ {"type": "text", "text": question},
21
+ ],
22
+ },
23
+ {
24
+ "role": "assistant",
25
+ "loss": True,
26
+ "content": [{"type": "text", "text": "A"}],
27
+ },
28
+ ]
29
+ }
30
+
31
+
32
+ def test_semantic_identity_binds_pixels_question_and_answer() -> None:
33
+ inventory = {"media/one.jpg": {"sha256": "a" * 64}}
34
+ baseline = _semantic_identity(_row(), inventory)
35
+ assert baseline == _semantic_identity(_row(question=" Which one? "), inventory)
36
+ assert baseline != _semantic_identity(_row(question="Which event?"), inventory)
37
+ changed = {"media/one.jpg": {"sha256": "b" * 64}}
38
+ assert baseline != _semantic_identity(_row(), changed)
39
+
40
+
41
+ def test_canonical_media_path_is_content_addressed() -> None:
42
+ digest = "ab" + "1" * 62
43
+ assert _canonical_media_path("legacy/frame.JPG", digest) == (
44
+ f"media/sha256/ab/{digest}.jpg"
45
+ )
46
+
47
+
48
+ def test_teacher_label_provenance_is_strict_and_propagatable() -> None:
49
+ assert _teacher_used_for_labels(
50
+ {"contracts": {"teacher_used_for_labels": True}}
51
+ )
52
+ assert not _teacher_used_for_labels(
53
+ {"contracts": {"teacher_used_for_labels": False}}
54
+ )
55
+ with pytest.raises(SourceAlignedSFTError):
56
+ _teacher_used_for_labels({"contracts": {}})
57
+
58
+
59
+ def test_legacy_v62di_loader_defaults_narrow_but_supports_committed_full_pool() -> None:
60
+ from data_pipeline.temporal_scope.build_source_aligned_sft_release import _load_legacy_v62di_source
61
+ with pytest.raises(SourceAlignedSFTError, match="Private legacy adapters"):
62
+ _load_legacy_v62di_source(Path("synthetic-unused-source"))
63
+
64
+
65
+ def test_legacy_repcount_normalization_is_hash_pinned_and_field_narrow() -> None:
66
+ from data_pipeline.temporal_scope.build_source_aligned_sft_release import _load_legacy_repcount_source
67
+ with pytest.raises(SourceAlignedSFTError, match="Private legacy adapters"):
68
+ _load_legacy_repcount_source(Path("synthetic-unused-source"))
69
+
70
+
71
+ def test_pinned_legacy_inputs_count_as_sealed_sources() -> None:
72
+ text = Path(
73
+ "data_pipeline/temporal_scope/build_source_aligned_sft_release.py"
74
+ ).read_text()
75
+ assert "sealed_source_count" in text
76
+ assert 'getattr(args, "legacy_v62di_source", None)' in text
77
+ assert 'getattr(args, "legacy_repcount_source", None)' in text
78
+
79
+
80
+ def test_hash_bound_parent_selection_allows_one_source_filtering_only() -> None:
81
+ text = Path(
82
+ "data_pipeline/temporal_scope/build_source_aligned_sft_release.py"
83
+ ).read_text()
84
+ assert 'getattr(args, "include_parent_selection", None) is None' in text
85
+ assert "unless a parent selection is supplied" in text
86
+
87
+
88
+ def test_composite_source_stage_filter_is_range_and_manifest_bound() -> None:
89
+ text = Path(
90
+ "data_pipeline/temporal_scope/build_source_aligned_sft_release.py"
91
+ ).read_text()
92
+ assert "--include-source-stage-release-id" in text
93
+ assert "requested source stages are absent" in text
94
+ assert "source curriculum stage row count diverged" in text
95
+ assert '"included_source_stage_release_ids"' in text
96
+
97
+
98
+ def test_capability_parent_selection_is_hash_bound_and_fail_closed() -> None:
99
+ text = Path(
100
+ "data_pipeline/temporal_scope/build_source_aligned_sft_release.py"
101
+ ).read_text()
102
+ assert "--include-parent-selection" in text
103
+ assert "iclr_video.capability_parent_selection.v0" in text
104
+ assert "iclr_video.capability_parent_selection.v1" in text
105
+ assert "source_release_membership_verified" in text
106
+ assert "selected parent IDs were absent" in text
107
+ assert '"parent_selection": parent_selection_record' in text
108
+
109
+
110
+ def test_ordered_curriculum_is_one_run_and_preserves_source_order() -> None:
111
+ text = Path(
112
+ "data_pipeline/temporal_scope/build_source_aligned_sft_release.py"
113
+ ).read_text()
114
+ assert '"training_runs_required": 1' in text
115
+ assert '"source_stage_then_source_row"' in text
116
+ assert "if not ordered_curriculum" in text
117
+
118
+
119
+ def test_additive_composition_can_retain_exposures_without_curriculum_order() -> None:
120
+ text = Path(
121
+ "data_pipeline/temporal_scope/build_source_aligned_sft_release.py"
122
+ ).read_text()
123
+ assert "--retain-semantic-exposures" in text
124
+ assert "ordered_curriculum or retain_semantic_exposures" in text
125
+ assert '"retain_semantic_exposures": retain_semantic_exposures' in text
code/tests/test_third_view_disagreement_agent.py ADDED
@@ -0,0 +1,78 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from experiments.materialize_third_view_disagreement_agent import materialize
2
+
3
+
4
+ IDENTITY = {
5
+ "benchmark": "demo",
6
+ "model_manifest_sha256": "m",
7
+ "annotation_sha256": "a",
8
+ "evaluation_protocol_sha256": "p",
9
+ "annotation": "annotation.json",
10
+ "evaluation_protocol": {},
11
+ "model": "model",
12
+ "processor": "processor",
13
+ }
14
+
15
+
16
+ def _result(rows: list[dict]) -> dict:
17
+ return {**IDENTITY, "rows": rows}
18
+
19
+
20
+ def test_third_view_commits_only_on_acquired_disagreement() -> None:
21
+ recent = _result(
22
+ [
23
+ {"uid": "u0", "prediction": "A", "visual_tokens": 10},
24
+ {"uid": "u1", "prediction": "D", "visual_tokens": 10},
25
+ ]
26
+ )
27
+ event = _result(
28
+ [
29
+ {
30
+ "uid": "u0",
31
+ "prediction": "B",
32
+ "error": None,
33
+ "visual_tokens": 20,
34
+ "sampling": {"frame_indices": list(range(20))},
35
+ }
36
+ ]
37
+ )
38
+ base = _result(
39
+ [
40
+ {
41
+ "uid": "u0",
42
+ "prediction": "A",
43
+ "agent_policy": {"escalated": True},
44
+ "media_calls": 2,
45
+ "num_turns": 2,
46
+ "model_processed_frames": 24,
47
+ "sampling": {"frame_indices": list(range(20))},
48
+ },
49
+ {"uid": "u1", "prediction": "D", "media_calls": 1},
50
+ ]
51
+ )
52
+ third = _result(
53
+ [
54
+ {
55
+ "uid": "u0",
56
+ "prediction": "C",
57
+ "raw_response": "C",
58
+ "error": None,
59
+ "query_time": 31.0,
60
+ "frames_observed": 32,
61
+ "visual_tokens": 30,
62
+ "sampling": {
63
+ "frame_indices": list(range(32)),
64
+ "frame_timestamps": list(map(float, range(32))),
65
+ },
66
+ }
67
+ ]
68
+ )
69
+ result = materialize(recent, event, base, third)
70
+ rows = {row["uid"]: row for row in result["rows"]}
71
+ assert rows["u0"]["prediction"] == "C"
72
+ assert rows["u0"]["media_calls"] == 3
73
+ assert rows["u0"]["model_processed_frames"] == 56
74
+ assert rows["u0"]["unique_frames"] == 32
75
+ assert rows["u0"]["visual_tokens"] == 60
76
+ assert rows["u1"]["prediction"] == "D"
77
+ assert rows["u1"]["third_view_policy"]["triggered"] is False
78
+ assert result["policy"]["question_task_group_or_benchmark_router"] is False