Image-Text-to-Text
Transformers
Safetensors
qwen3_vl
qwen3-vl
viss
streaming-video
visual-question-answering
causal-video
conversational
Instructions to use mis2/ViSS-8B with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use mis2/ViSS-8B with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("image-text-to-text", model="mis2/ViSS-8B") messages = [ { "role": "user", "content": [ {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/p-blog/candy.JPG"}, {"type": "text", "text": "What animal is on the candy?"} ] }, ] pipe(text=messages)# pip install -U transformers accelerate # Load model directly from transformers import AutoProcessor, AutoModelForMultimodalLM processor = AutoProcessor.from_pretrained("mis2/ViSS-8B") model = AutoModelForMultimodalLM.from_pretrained("mis2/ViSS-8B", device_map="auto") messages = [ { "role": "user", "content": [ {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/p-blog/candy.JPG"}, {"type": "text", "text": "What animal is on the candy?"} ] }, ] inputs = processor.apply_chat_template( messages, add_generation_prompt=True, tokenize=True, return_dict=True, return_tensors="pt", ).to(model.device) outputs = model.generate(**inputs, max_new_tokens=256) print(processor.decode(outputs[0][inputs["input_ids"].shape[-1]:])) - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- vLLM
How to use mis2/ViSS-8B with vLLM:
Install from pip and serve model
# Install vLLM from pip: pip install vllm # Start the vLLM server: vllm serve "mis2/ViSS-8B" # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:8000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "mis2/ViSS-8B", "messages": [ { "role": "user", "content": [ { "type": "text", "text": "Describe this image in one sentence." }, { "type": "image_url", "image_url": { "url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg" } } ] } ] }'Use Docker
docker model run hf.co/mis2/ViSS-8B
- SGLang
How to use mis2/ViSS-8B with SGLang:
Install from pip and serve model
# Install SGLang from pip: pip install sglang # Start the SGLang server: python3 -m sglang.launch_server \ --model-path "mis2/ViSS-8B" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "mis2/ViSS-8B", "messages": [ { "role": "user", "content": [ { "type": "text", "text": "Describe this image in one sentence." }, { "type": "image_url", "image_url": { "url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg" } } ] } ] }'Use Docker images
docker run --gpus all \ --shm-size 32g \ -p 30000:30000 \ -v ~/.cache/huggingface:/root/.cache/huggingface \ --env "HF_TOKEN=<secret>" \ --ipc=host \ lmsysorg/sglang:latest \ python3 -m sglang.launch_server \ --model-path "mis2/ViSS-8B" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "mis2/ViSS-8B", "messages": [ { "role": "user", "content": [ { "type": "text", "text": "Describe this image in one sentence." }, { "type": "image_url", "image_url": { "url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg" } } ] } ] }' - Docker Model Runner
How to use mis2/ViSS-8B with Docker Model Runner:
docker model run hf.co/mis2/ViSS-8B
Release ViSS-8B weights, processors and source code
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- .gitattributes +2 -0
- CITATION.cff +29 -0
- LICENSE +202 -0
- README.md +113 -0
- added_tokens.json +28 -0
- chat_template.jinja +120 -0
- code/.gitignore +20 -0
- code/LICENSE +202 -0
- code/NOTICE +16 -0
- code/README.md +135 -0
- code/configs/evaluation/recent4.example.json +74 -0
- code/configs/training/deepspeed_zero3_bf16.json +16 -0
- code/data_pipeline/__init__.py +1 -0
- code/data_pipeline/temporal_scope/__init__.py +1 -0
- code/data_pipeline/temporal_scope/build_size_matched_generic_control.py +107 -0
- code/data_pipeline/temporal_scope/build_source_aligned_sft_release.py +722 -0
- code/data_pipeline/temporal_scope/build_vst_prefix32_semantic.py +525 -0
- code/data_pipeline/temporal_scope/filter_semantic_release_by_blind_audit.py +329 -0
- code/data_pipeline/temporal_scope/filter_semantic_release_by_option_consensus.py +298 -0
- code/data_pipeline/temporal_scope/select_quality_capability_rows.py +420 -0
- code/data_pipeline/temporal_scope/semantic_mcq.py +90 -0
- code/docs/DATA.md +63 -0
- code/docs/EVALUATION.md +114 -0
- code/docs/INSTALL.md +59 -0
- code/docs/TRAINING.md +50 -0
- code/experiments/analyze_uncertainty_temporal_escalation.py +451 -0
- code/experiments/evaluate_qwen3vl_recent4_full.py +0 -0
- code/experiments/materialize_confidence_gated_third_view_agent.py +259 -0
- code/experiments/materialize_third_view_disagreement_agent.py +203 -0
- code/experiments/materialize_tvs_view_oracle.py +118 -0
- code/experiments/materialize_uncertainty_temporal_agent.py +465 -0
- code/experiments/select_uncertainty_escalation_uids.py +192 -0
- code/experiments/train_qwen3vl_sft.py +1091 -0
- code/experiments/tvs_correctness.py +80 -0
- code/pyproject.toml +32 -0
- code/scripts/__init__.py +1 -0
- code/scripts/configure_evaluation_protocol.py +62 -0
- code/scripts/merge_eval_shards.py +483 -0
- code/scripts/run_recent4_single_benchmark.sh +380 -0
- code/scripts/score_official_benchmark_predictions.py +348 -0
- code/scripts/validate_evaluation_protocol.py +28 -0
- code/tests/test_confidence_gated_third_view_agent.py +77 -0
- code/tests/test_filter_semantic_release_by_option_consensus.py +84 -0
- code/tests/test_materialize_uncertainty_temporal_agent.py +551 -0
- code/tests/test_public_protocol.py +47 -0
- code/tests/test_recent4_runner.py +893 -0
- code/tests/test_semantic_answer_loss_mask.py +108 -0
- code/tests/test_size_matched_generic_control.py +26 -0
- code/tests/test_source_aligned_sft_release.py +125 -0
- code/tests/test_third_view_disagreement_agent.py +78 -0
.gitattributes
CHANGED
|
@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
+
locked_processor/tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
| 37 |
+
tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
CITATION.cff
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
cff-version: 1.2.0
|
| 2 |
+
message: "Please cite the ViSS manuscript when using this model and code."
|
| 3 |
+
title: "Supervising the Visible State: Separating Representation from View Selection in Streaming Video"
|
| 4 |
+
type: software
|
| 5 |
+
authors:
|
| 6 |
+
- family-names: Hu
|
| 7 |
+
given-names: Yihao
|
| 8 |
+
- family-names: Wang
|
| 9 |
+
given-names: Pan
|
| 10 |
+
- family-names: Lin
|
| 11 |
+
given-names: Yitong
|
| 12 |
+
- family-names: Wen
|
| 13 |
+
given-names: Zhihao
|
| 14 |
+
- family-names: Liu
|
| 15 |
+
given-names: Deyuan
|
| 16 |
+
- family-names: Wang
|
| 17 |
+
given-names: Hang
|
| 18 |
+
- family-names: Wan
|
| 19 |
+
given-names: Mingda
|
| 20 |
+
- family-names: Zhao
|
| 21 |
+
given-names: Yiwen
|
| 22 |
+
- family-names: Li
|
| 23 |
+
given-names: Yutong
|
| 24 |
+
- family-names: Mao
|
| 25 |
+
given-names: Yuchen
|
| 26 |
+
- family-names: Lin
|
| 27 |
+
given-names: Tao
|
| 28 |
+
license: Apache-2.0
|
| 29 |
+
url: "https://huggingface.co/mis2/ViSS-8B"
|
LICENSE
ADDED
|
@@ -0,0 +1,202 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
Apache License
|
| 3 |
+
Version 2.0, January 2004
|
| 4 |
+
http://www.apache.org/licenses/
|
| 5 |
+
|
| 6 |
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
| 7 |
+
|
| 8 |
+
1. Definitions.
|
| 9 |
+
|
| 10 |
+
"License" shall mean the terms and conditions for use, reproduction,
|
| 11 |
+
and distribution as defined by Sections 1 through 9 of this document.
|
| 12 |
+
|
| 13 |
+
"Licensor" shall mean the copyright owner or entity authorized by
|
| 14 |
+
the copyright owner that is granting the License.
|
| 15 |
+
|
| 16 |
+
"Legal Entity" shall mean the union of the acting entity and all
|
| 17 |
+
other entities that control, are controlled by, or are under common
|
| 18 |
+
control with that entity. For the purposes of this definition,
|
| 19 |
+
"control" means (i) the power, direct or indirect, to cause the
|
| 20 |
+
direction or management of such entity, whether by contract or
|
| 21 |
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
| 22 |
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
| 23 |
+
|
| 24 |
+
"You" (or "Your") shall mean an individual or Legal Entity
|
| 25 |
+
exercising permissions granted by this License.
|
| 26 |
+
|
| 27 |
+
"Source" form shall mean the preferred form for making modifications,
|
| 28 |
+
including but not limited to software source code, documentation
|
| 29 |
+
source, and configuration files.
|
| 30 |
+
|
| 31 |
+
"Object" form shall mean any form resulting from mechanical
|
| 32 |
+
transformation or translation of a Source form, including but
|
| 33 |
+
not limited to compiled object code, generated documentation,
|
| 34 |
+
and conversions to other media types.
|
| 35 |
+
|
| 36 |
+
"Work" shall mean the work of authorship, whether in Source or
|
| 37 |
+
Object form, made available under the License, as indicated by a
|
| 38 |
+
copyright notice that is included in or attached to the work
|
| 39 |
+
(an example is provided in the Appendix below).
|
| 40 |
+
|
| 41 |
+
"Derivative Works" shall mean any work, whether in Source or Object
|
| 42 |
+
form, that is based on (or derived from) the Work and for which the
|
| 43 |
+
editorial revisions, annotations, elaborations, or other modifications
|
| 44 |
+
represent, as a whole, an original work of authorship. For the purposes
|
| 45 |
+
of this License, Derivative Works shall not include works that remain
|
| 46 |
+
separable from, or merely link (or bind by name) to the interfaces of,
|
| 47 |
+
the Work and Derivative Works thereof.
|
| 48 |
+
|
| 49 |
+
"Contribution" shall mean any work of authorship, including
|
| 50 |
+
the original version of the Work and any modifications or additions
|
| 51 |
+
to that Work or Derivative Works thereof, that is intentionally
|
| 52 |
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
| 53 |
+
or by an individual or Legal Entity authorized to submit on behalf of
|
| 54 |
+
the copyright owner. For the purposes of this definition, "submitted"
|
| 55 |
+
means any form of electronic, verbal, or written communication sent
|
| 56 |
+
to the Licensor or its representatives, including but not limited to
|
| 57 |
+
communication on electronic mailing lists, source code control systems,
|
| 58 |
+
and issue tracking systems that are managed by, or on behalf of, the
|
| 59 |
+
Licensor for the purpose of discussing and improving the Work, but
|
| 60 |
+
excluding communication that is conspicuously marked or otherwise
|
| 61 |
+
designated in writing by the copyright owner as "Not a Contribution."
|
| 62 |
+
|
| 63 |
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
| 64 |
+
on behalf of whom a Contribution has been received by Licensor and
|
| 65 |
+
subsequently incorporated within the Work.
|
| 66 |
+
|
| 67 |
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
| 68 |
+
this License, each Contributor hereby grants to You a perpetual,
|
| 69 |
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
| 70 |
+
copyright license to reproduce, prepare Derivative Works of,
|
| 71 |
+
publicly display, publicly perform, sublicense, and distribute the
|
| 72 |
+
Work and such Derivative Works in Source or Object form.
|
| 73 |
+
|
| 74 |
+
3. Grant of Patent License. Subject to the terms and conditions of
|
| 75 |
+
this License, each Contributor hereby grants to You a perpetual,
|
| 76 |
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
| 77 |
+
(except as stated in this section) patent license to make, have made,
|
| 78 |
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
| 79 |
+
where such license applies only to those patent claims licensable
|
| 80 |
+
by such Contributor that are necessarily infringed by their
|
| 81 |
+
Contribution(s) alone or by combination of their Contribution(s)
|
| 82 |
+
with the Work to which such Contribution(s) was submitted. If You
|
| 83 |
+
institute patent litigation against any entity (including a
|
| 84 |
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
| 85 |
+
or a Contribution incorporated within the Work constitutes direct
|
| 86 |
+
or contributory patent infringement, then any patent licenses
|
| 87 |
+
granted to You under this License for that Work shall terminate
|
| 88 |
+
as of the date such litigation is filed.
|
| 89 |
+
|
| 90 |
+
4. Redistribution. You may reproduce and distribute copies of the
|
| 91 |
+
Work or Derivative Works thereof in any medium, with or without
|
| 92 |
+
modifications, and in Source or Object form, provided that You
|
| 93 |
+
meet the following conditions:
|
| 94 |
+
|
| 95 |
+
(a) You must give any other recipients of the Work or
|
| 96 |
+
Derivative Works a copy of this License; and
|
| 97 |
+
|
| 98 |
+
(b) You must cause any modified files to carry prominent notices
|
| 99 |
+
stating that You changed the files; and
|
| 100 |
+
|
| 101 |
+
(c) You must retain, in the Source form of any Derivative Works
|
| 102 |
+
that You distribute, all copyright, patent, trademark, and
|
| 103 |
+
attribution notices from the Source form of the Work,
|
| 104 |
+
excluding those notices that do not pertain to any part of
|
| 105 |
+
the Derivative Works; and
|
| 106 |
+
|
| 107 |
+
(d) If the Work includes a "NOTICE" text file as part of its
|
| 108 |
+
distribution, then any Derivative Works that You distribute must
|
| 109 |
+
include a readable copy of the attribution notices contained
|
| 110 |
+
within such NOTICE file, excluding those notices that do not
|
| 111 |
+
pertain to any part of the Derivative Works, in at least one
|
| 112 |
+
of the following places: within a NOTICE text file distributed
|
| 113 |
+
as part of the Derivative Works; within the Source form or
|
| 114 |
+
documentation, if provided along with the Derivative Works; or,
|
| 115 |
+
within a display generated by the Derivative Works, if and
|
| 116 |
+
wherever such third-party notices normally appear. The contents
|
| 117 |
+
of the NOTICE file are for informational purposes only and
|
| 118 |
+
do not modify the License. You may add Your own attribution
|
| 119 |
+
notices within Derivative Works that You distribute, alongside
|
| 120 |
+
or as an addendum to the NOTICE text from the Work, provided
|
| 121 |
+
that such additional attribution notices cannot be construed
|
| 122 |
+
as modifying the License.
|
| 123 |
+
|
| 124 |
+
You may add Your own copyright statement to Your modifications and
|
| 125 |
+
may provide additional or different license terms and conditions
|
| 126 |
+
for use, reproduction, or distribution of Your modifications, or
|
| 127 |
+
for any such Derivative Works as a whole, provided Your use,
|
| 128 |
+
reproduction, and distribution of the Work otherwise complies with
|
| 129 |
+
the conditions stated in this License.
|
| 130 |
+
|
| 131 |
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
| 132 |
+
any Contribution intentionally submitted for inclusion in the Work
|
| 133 |
+
by You to the Licensor shall be under the terms and conditions of
|
| 134 |
+
this License, without any additional terms or conditions.
|
| 135 |
+
Notwithstanding the above, nothing herein shall supersede or modify
|
| 136 |
+
the terms of any separate license agreement you may have executed
|
| 137 |
+
with Licensor regarding such Contributions.
|
| 138 |
+
|
| 139 |
+
6. Trademarks. This License does not grant permission to use the trade
|
| 140 |
+
names, trademarks, service marks, or product names of the Licensor,
|
| 141 |
+
except as required for reasonable and customary use in describing the
|
| 142 |
+
origin of the Work and reproducing the content of the NOTICE file.
|
| 143 |
+
|
| 144 |
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
| 145 |
+
agreed to in writing, Licensor provides the Work (and each
|
| 146 |
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
| 147 |
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
| 148 |
+
implied, including, without limitation, any warranties or conditions
|
| 149 |
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
| 150 |
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
| 151 |
+
appropriateness of using or redistributing the Work and assume any
|
| 152 |
+
risks associated with Your exercise of permissions under this License.
|
| 153 |
+
|
| 154 |
+
8. Limitation of Liability. In no event and under no legal theory,
|
| 155 |
+
whether in tort (including negligence), contract, or otherwise,
|
| 156 |
+
unless required by applicable law (such as deliberate and grossly
|
| 157 |
+
negligent acts) or agreed to in writing, shall any Contributor be
|
| 158 |
+
liable to You for damages, including any direct, indirect, special,
|
| 159 |
+
incidental, or consequential damages of any character arising as a
|
| 160 |
+
result of this License or out of the use or inability to use the
|
| 161 |
+
Work (including but not limited to damages for loss of goodwill,
|
| 162 |
+
work stoppage, computer failure or malfunction, or any and all
|
| 163 |
+
other commercial damages or losses), even if such Contributor
|
| 164 |
+
has been advised of the possibility of such damages.
|
| 165 |
+
|
| 166 |
+
9. Accepting Warranty or Additional Liability. While redistributing
|
| 167 |
+
the Work or Derivative Works thereof, You may choose to offer,
|
| 168 |
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
| 169 |
+
or other liability obligations and/or rights consistent with this
|
| 170 |
+
License. However, in accepting such obligations, You may act only
|
| 171 |
+
on Your own behalf and on Your sole responsibility, not on behalf
|
| 172 |
+
of any other Contributor, and only if You agree to indemnify,
|
| 173 |
+
defend, and hold each Contributor harmless for any liability
|
| 174 |
+
incurred by, or claims asserted against, such Contributor by reason
|
| 175 |
+
of your accepting any such warranty or additional liability.
|
| 176 |
+
|
| 177 |
+
END OF TERMS AND CONDITIONS
|
| 178 |
+
|
| 179 |
+
APPENDIX: How to apply the Apache License to your work.
|
| 180 |
+
|
| 181 |
+
To apply the Apache License to your work, attach the following
|
| 182 |
+
boilerplate notice, with the fields enclosed by brackets "[]"
|
| 183 |
+
replaced with your own identifying information. (Don't include
|
| 184 |
+
the brackets!) The text should be enclosed in the appropriate
|
| 185 |
+
comment syntax for the file format. We also recommend that a
|
| 186 |
+
file or class name and description of purpose be included on the
|
| 187 |
+
same "printed page" as the copyright notice for easier
|
| 188 |
+
identification within third-party archives.
|
| 189 |
+
|
| 190 |
+
Copyright [yyyy] [name of copyright owner]
|
| 191 |
+
|
| 192 |
+
Licensed under the Apache License, Version 2.0 (the "License");
|
| 193 |
+
you may not use this file except in compliance with the License.
|
| 194 |
+
You may obtain a copy of the License at
|
| 195 |
+
|
| 196 |
+
http://www.apache.org/licenses/LICENSE-2.0
|
| 197 |
+
|
| 198 |
+
Unless required by applicable law or agreed to in writing, software
|
| 199 |
+
distributed under the License is distributed on an "AS IS" BASIS,
|
| 200 |
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 201 |
+
See the License for the specific language governing permissions and
|
| 202 |
+
limitations under the License.
|
README.md
ADDED
|
@@ -0,0 +1,113 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
license: apache-2.0
|
| 3 |
+
base_model: Qwen/Qwen3-VL-8B-Instruct
|
| 4 |
+
library_name: transformers
|
| 5 |
+
pipeline_tag: image-text-to-text
|
| 6 |
+
tags:
|
| 7 |
+
- qwen3-vl
|
| 8 |
+
- viss
|
| 9 |
+
- streaming-video
|
| 10 |
+
- visual-question-answering
|
| 11 |
+
- causal-video
|
| 12 |
+
---
|
| 13 |
+
|
| 14 |
+
# ViSS-8B
|
| 15 |
+
|
| 16 |
+
**Supervising the Visible State: Separating Representation from View Selection in Streaming Video**
|
| 17 |
+
|
| 18 |
+
Yihao Hu*, Pan Wang*, Yitong Lin*, Zhihao Wen†, Deyuan Liu, Hang Wang,
|
| 19 |
+
Mingda Wan, Yiwen Zhao, Yutong Li, Yuchen Mao, Tao Lin†
|
| 20 |
+
|
| 21 |
+
*Equal contribution. †Corresponding author.*
|
| 22 |
+
|
| 23 |
+
[Collection](https://huggingface.co/collections/mis2/viss-6adc8caf041ae0c501c9a11b) ·
|
| 24 |
+
[Source code](https://huggingface.co/mis2/ViSS-8B/tree/main/code) ·
|
| 25 |
+
[Installation](code/docs/INSTALL.md) ·
|
| 26 |
+
[Training](code/docs/TRAINING.md) ·
|
| 27 |
+
[Evaluation](code/docs/EVALUATION.md)
|
| 28 |
+
|
| 29 |
+
## Overview
|
| 30 |
+
|
| 31 |
+
Visible-State Supervision (ViSS) trains a vision–language model on
|
| 32 |
+
source-grounded questions about event identity, order, accumulation,
|
| 33 |
+
relation changes, and whether the available evidence suffices to answer.
|
| 34 |
+
Test-Time Causal View Search (TVS) keeps the model fixed and acquires
|
| 35 |
+
complementary recent, event, and compact-history views.
|
| 36 |
+
|
| 37 |
+
The model is initialized from Qwen3-VL-8B-Instruct and trained for one epoch
|
| 38 |
+
on 7,677 examples, with learning rate 2e-6 and global batch size 12.
|
| 39 |
+
The vision tower is frozen; the language model and multimodal projector
|
| 40 |
+
are trainable. Model-only ViSS and TVS use the same checkpoint.
|
| 41 |
+
|
| 42 |
+
## Released files
|
| 43 |
+
|
| 44 |
+
- Four original BF16 safetensors shards (17.53 GB in total).
|
| 45 |
+
- Model configuration, tokenizer, chat template, and native processor.
|
| 46 |
+
- `locked_processor/`: the processor used for the recent-view evaluation.
|
| 47 |
+
- `code/`: training, inference, TVS composition, evaluation adapters,
|
| 48 |
+
public configuration templates, documentation, and synthetic tests.
|
| 49 |
+
|
| 50 |
+
The release contains no training annotations, raw media, company experiment
|
| 51 |
+
records, prediction files, evaluation summaries, optimizer states, or internal
|
| 52 |
+
training manifests. Training data are not being released in this version.
|
| 53 |
+
|
| 54 |
+
## Download
|
| 55 |
+
|
| 56 |
+
```bash
|
| 57 |
+
hf download mis2/ViSS-8B --local-dir checkpoints/ViSS-8B
|
| 58 |
+
```
|
| 59 |
+
|
| 60 |
+
## Load the model
|
| 61 |
+
|
| 62 |
+
Use a Transformers version supporting Qwen3-VL; the retained model
|
| 63 |
+
configuration specifies Transformers 4.57.3.
|
| 64 |
+
|
| 65 |
+
```python
|
| 66 |
+
import torch
|
| 67 |
+
from transformers import AutoProcessor, Qwen3VLForConditionalGeneration
|
| 68 |
+
|
| 69 |
+
model_id = "mis2/ViSS-8B"
|
| 70 |
+
processor = AutoProcessor.from_pretrained(
|
| 71 |
+
model_id, subfolder="locked_processor"
|
| 72 |
+
)
|
| 73 |
+
model = Qwen3VLForConditionalGeneration.from_pretrained(
|
| 74 |
+
model_id,
|
| 75 |
+
dtype=torch.bfloat16,
|
| 76 |
+
device_map="auto",
|
| 77 |
+
)
|
| 78 |
+
```
|
| 79 |
+
|
| 80 |
+
The root processor is the native training processor. The
|
| 81 |
+
`locked_processor/` subdirectory preserves the recent-view evaluation
|
| 82 |
+
processor. The supplied evaluator implements causal sampling and benchmark
|
| 83 |
+
prompt conventions; loading the processor alone does not implement TVS.
|
| 84 |
+
|
| 85 |
+
## Use the code
|
| 86 |
+
|
| 87 |
+
```bash
|
| 88 |
+
cd checkpoints/ViSS-8B/code
|
| 89 |
+
python -m pip install -e ".[inference,test]"
|
| 90 |
+
python -m pytest -q tests
|
| 91 |
+
```
|
| 92 |
+
|
| 93 |
+
See the evaluation guide to bind the public configuration template to your
|
| 94 |
+
own processor, benchmark query exports, and model. Benchmark media and
|
| 95 |
+
official scoring installations must be obtained separately. TVS additionally
|
| 96 |
+
requires calibration inputs prepared independently of benchmark evaluation.
|
| 97 |
+
|
| 98 |
+
## License
|
| 99 |
+
|
| 100 |
+
The source code follows the project's Apache-2.0 license. The model is
|
| 101 |
+
derived from Qwen3-VL-8B-Instruct, which is also distributed under Apache-2.0.
|
| 102 |
+
External benchmark and media terms remain with their respective providers.
|
| 103 |
+
|
| 104 |
+
## Citation
|
| 105 |
+
|
| 106 |
+
```bibtex
|
| 107 |
+
@misc{hu2026viss,
|
| 108 |
+
title={Supervising the Visible State: Separating Representation from View Selection in Streaming Video},
|
| 109 |
+
author={Yihao Hu and Pan Wang and Yitong Lin and Zhihao Wen and Deyuan Liu and Hang Wang and Mingda Wan and Yiwen Zhao and Yutong Li and Yuchen Mao and Tao Lin},
|
| 110 |
+
year={2026},
|
| 111 |
+
note={Manuscript}
|
| 112 |
+
}
|
| 113 |
+
```
|
added_tokens.json
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"</think>": 151668,
|
| 3 |
+
"</tool_call>": 151658,
|
| 4 |
+
"</tool_response>": 151666,
|
| 5 |
+
"<think>": 151667,
|
| 6 |
+
"<tool_call>": 151657,
|
| 7 |
+
"<tool_response>": 151665,
|
| 8 |
+
"<|box_end|>": 151649,
|
| 9 |
+
"<|box_start|>": 151648,
|
| 10 |
+
"<|endoftext|>": 151643,
|
| 11 |
+
"<|file_sep|>": 151664,
|
| 12 |
+
"<|fim_middle|>": 151660,
|
| 13 |
+
"<|fim_pad|>": 151662,
|
| 14 |
+
"<|fim_prefix|>": 151659,
|
| 15 |
+
"<|fim_suffix|>": 151661,
|
| 16 |
+
"<|im_end|>": 151645,
|
| 17 |
+
"<|im_start|>": 151644,
|
| 18 |
+
"<|image_pad|>": 151655,
|
| 19 |
+
"<|object_ref_end|>": 151647,
|
| 20 |
+
"<|object_ref_start|>": 151646,
|
| 21 |
+
"<|quad_end|>": 151651,
|
| 22 |
+
"<|quad_start|>": 151650,
|
| 23 |
+
"<|repo_name|>": 151663,
|
| 24 |
+
"<|video_pad|>": 151656,
|
| 25 |
+
"<|vision_end|>": 151653,
|
| 26 |
+
"<|vision_pad|>": 151654,
|
| 27 |
+
"<|vision_start|>": 151652
|
| 28 |
+
}
|
chat_template.jinja
ADDED
|
@@ -0,0 +1,120 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{%- if tools %}
|
| 2 |
+
{{- '<|im_start|>system\n' }}
|
| 3 |
+
{%- if messages[0].role == 'system' %}
|
| 4 |
+
{%- if messages[0].content is string %}
|
| 5 |
+
{{- messages[0].content }}
|
| 6 |
+
{%- else %}
|
| 7 |
+
{%- for content in messages[0].content %}
|
| 8 |
+
{%- if 'text' in content %}
|
| 9 |
+
{{- content.text }}
|
| 10 |
+
{%- endif %}
|
| 11 |
+
{%- endfor %}
|
| 12 |
+
{%- endif %}
|
| 13 |
+
{{- '\n\n' }}
|
| 14 |
+
{%- endif %}
|
| 15 |
+
{{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
|
| 16 |
+
{%- for tool in tools %}
|
| 17 |
+
{{- "\n" }}
|
| 18 |
+
{{- tool | tojson }}
|
| 19 |
+
{%- endfor %}
|
| 20 |
+
{{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
|
| 21 |
+
{%- else %}
|
| 22 |
+
{%- if messages[0].role == 'system' %}
|
| 23 |
+
{{- '<|im_start|>system\n' }}
|
| 24 |
+
{%- if messages[0].content is string %}
|
| 25 |
+
{{- messages[0].content }}
|
| 26 |
+
{%- else %}
|
| 27 |
+
{%- for content in messages[0].content %}
|
| 28 |
+
{%- if 'text' in content %}
|
| 29 |
+
{{- content.text }}
|
| 30 |
+
{%- endif %}
|
| 31 |
+
{%- endfor %}
|
| 32 |
+
{%- endif %}
|
| 33 |
+
{{- '<|im_end|>\n' }}
|
| 34 |
+
{%- endif %}
|
| 35 |
+
{%- endif %}
|
| 36 |
+
{%- set image_count = namespace(value=0) %}
|
| 37 |
+
{%- set video_count = namespace(value=0) %}
|
| 38 |
+
{%- for message in messages %}
|
| 39 |
+
{%- if message.role == "user" %}
|
| 40 |
+
{{- '<|im_start|>' + message.role + '\n' }}
|
| 41 |
+
{%- if message.content is string %}
|
| 42 |
+
{{- message.content }}
|
| 43 |
+
{%- else %}
|
| 44 |
+
{%- for content in message.content %}
|
| 45 |
+
{%- if content.type == 'image' or 'image' in content or 'image_url' in content %}
|
| 46 |
+
{%- set image_count.value = image_count.value + 1 %}
|
| 47 |
+
{%- if add_vision_id %}Picture {{ image_count.value }}: {% endif -%}
|
| 48 |
+
<|vision_start|><|image_pad|><|vision_end|>
|
| 49 |
+
{%- elif content.type == 'video' or 'video' in content %}
|
| 50 |
+
{%- set video_count.value = video_count.value + 1 %}
|
| 51 |
+
{%- if add_vision_id %}Video {{ video_count.value }}: {% endif -%}
|
| 52 |
+
<|vision_start|><|video_pad|><|vision_end|>
|
| 53 |
+
{%- elif 'text' in content %}
|
| 54 |
+
{{- content.text }}
|
| 55 |
+
{%- endif %}
|
| 56 |
+
{%- endfor %}
|
| 57 |
+
{%- endif %}
|
| 58 |
+
{{- '<|im_end|>\n' }}
|
| 59 |
+
{%- elif message.role == "assistant" %}
|
| 60 |
+
{{- '<|im_start|>' + message.role + '\n' }}
|
| 61 |
+
{%- if message.content is string %}
|
| 62 |
+
{{- message.content }}
|
| 63 |
+
{%- else %}
|
| 64 |
+
{%- for content_item in message.content %}
|
| 65 |
+
{%- if 'text' in content_item %}
|
| 66 |
+
{{- content_item.text }}
|
| 67 |
+
{%- endif %}
|
| 68 |
+
{%- endfor %}
|
| 69 |
+
{%- endif %}
|
| 70 |
+
{%- if message.tool_calls %}
|
| 71 |
+
{%- for tool_call in message.tool_calls %}
|
| 72 |
+
{%- if (loop.first and message.content) or (not loop.first) %}
|
| 73 |
+
{{- '\n' }}
|
| 74 |
+
{%- endif %}
|
| 75 |
+
{%- if tool_call.function %}
|
| 76 |
+
{%- set tool_call = tool_call.function %}
|
| 77 |
+
{%- endif %}
|
| 78 |
+
{{- '<tool_call>\n{"name": "' }}
|
| 79 |
+
{{- tool_call.name }}
|
| 80 |
+
{{- '", "arguments": ' }}
|
| 81 |
+
{%- if tool_call.arguments is string %}
|
| 82 |
+
{{- tool_call.arguments }}
|
| 83 |
+
{%- else %}
|
| 84 |
+
{{- tool_call.arguments | tojson }}
|
| 85 |
+
{%- endif %}
|
| 86 |
+
{{- '}\n</tool_call>' }}
|
| 87 |
+
{%- endfor %}
|
| 88 |
+
{%- endif %}
|
| 89 |
+
{{- '<|im_end|>\n' }}
|
| 90 |
+
{%- elif message.role == "tool" %}
|
| 91 |
+
{%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
|
| 92 |
+
{{- '<|im_start|>user' }}
|
| 93 |
+
{%- endif %}
|
| 94 |
+
{{- '\n<tool_response>\n' }}
|
| 95 |
+
{%- if message.content is string %}
|
| 96 |
+
{{- message.content }}
|
| 97 |
+
{%- else %}
|
| 98 |
+
{%- for content in message.content %}
|
| 99 |
+
{%- if content.type == 'image' or 'image' in content or 'image_url' in content %}
|
| 100 |
+
{%- set image_count.value = image_count.value + 1 %}
|
| 101 |
+
{%- if add_vision_id %}Picture {{ image_count.value }}: {% endif -%}
|
| 102 |
+
<|vision_start|><|image_pad|><|vision_end|>
|
| 103 |
+
{%- elif content.type == 'video' or 'video' in content %}
|
| 104 |
+
{%- set video_count.value = video_count.value + 1 %}
|
| 105 |
+
{%- if add_vision_id %}Video {{ video_count.value }}: {% endif -%}
|
| 106 |
+
<|vision_start|><|video_pad|><|vision_end|>
|
| 107 |
+
{%- elif 'text' in content %}
|
| 108 |
+
{{- content.text }}
|
| 109 |
+
{%- endif %}
|
| 110 |
+
{%- endfor %}
|
| 111 |
+
{%- endif %}
|
| 112 |
+
{{- '\n</tool_response>' }}
|
| 113 |
+
{%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
|
| 114 |
+
{{- '<|im_end|>\n' }}
|
| 115 |
+
{%- endif %}
|
| 116 |
+
{%- endif %}
|
| 117 |
+
{%- endfor %}
|
| 118 |
+
{%- if add_generation_prompt %}
|
| 119 |
+
{{- '<|im_start|>assistant\n' }}
|
| 120 |
+
{%- endif %}
|
code/.gitignore
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
__pycache__/
|
| 2 |
+
*.py[cod]
|
| 3 |
+
.pytest_cache/
|
| 4 |
+
*.egg-info/
|
| 5 |
+
build/
|
| 6 |
+
dist/
|
| 7 |
+
.venv/
|
| 8 |
+
outputs/
|
| 9 |
+
results/
|
| 10 |
+
logs/
|
| 11 |
+
checkpoints/
|
| 12 |
+
data/
|
| 13 |
+
records/
|
| 14 |
+
runs/
|
| 15 |
+
manifests/
|
| 16 |
+
configs/evaluation/recent4_full_v1.json
|
| 17 |
+
release_manifest.json
|
| 18 |
+
*.log
|
| 19 |
+
.env
|
| 20 |
+
.env.*
|
code/LICENSE
ADDED
|
@@ -0,0 +1,202 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
Apache License
|
| 3 |
+
Version 2.0, January 2004
|
| 4 |
+
http://www.apache.org/licenses/
|
| 5 |
+
|
| 6 |
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
| 7 |
+
|
| 8 |
+
1. Definitions.
|
| 9 |
+
|
| 10 |
+
"License" shall mean the terms and conditions for use, reproduction,
|
| 11 |
+
and distribution as defined by Sections 1 through 9 of this document.
|
| 12 |
+
|
| 13 |
+
"Licensor" shall mean the copyright owner or entity authorized by
|
| 14 |
+
the copyright owner that is granting the License.
|
| 15 |
+
|
| 16 |
+
"Legal Entity" shall mean the union of the acting entity and all
|
| 17 |
+
other entities that control, are controlled by, or are under common
|
| 18 |
+
control with that entity. For the purposes of this definition,
|
| 19 |
+
"control" means (i) the power, direct or indirect, to cause the
|
| 20 |
+
direction or management of such entity, whether by contract or
|
| 21 |
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
| 22 |
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
| 23 |
+
|
| 24 |
+
"You" (or "Your") shall mean an individual or Legal Entity
|
| 25 |
+
exercising permissions granted by this License.
|
| 26 |
+
|
| 27 |
+
"Source" form shall mean the preferred form for making modifications,
|
| 28 |
+
including but not limited to software source code, documentation
|
| 29 |
+
source, and configuration files.
|
| 30 |
+
|
| 31 |
+
"Object" form shall mean any form resulting from mechanical
|
| 32 |
+
transformation or translation of a Source form, including but
|
| 33 |
+
not limited to compiled object code, generated documentation,
|
| 34 |
+
and conversions to other media types.
|
| 35 |
+
|
| 36 |
+
"Work" shall mean the work of authorship, whether in Source or
|
| 37 |
+
Object form, made available under the License, as indicated by a
|
| 38 |
+
copyright notice that is included in or attached to the work
|
| 39 |
+
(an example is provided in the Appendix below).
|
| 40 |
+
|
| 41 |
+
"Derivative Works" shall mean any work, whether in Source or Object
|
| 42 |
+
form, that is based on (or derived from) the Work and for which the
|
| 43 |
+
editorial revisions, annotations, elaborations, or other modifications
|
| 44 |
+
represent, as a whole, an original work of authorship. For the purposes
|
| 45 |
+
of this License, Derivative Works shall not include works that remain
|
| 46 |
+
separable from, or merely link (or bind by name) to the interfaces of,
|
| 47 |
+
the Work and Derivative Works thereof.
|
| 48 |
+
|
| 49 |
+
"Contribution" shall mean any work of authorship, including
|
| 50 |
+
the original version of the Work and any modifications or additions
|
| 51 |
+
to that Work or Derivative Works thereof, that is intentionally
|
| 52 |
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
| 53 |
+
or by an individual or Legal Entity authorized to submit on behalf of
|
| 54 |
+
the copyright owner. For the purposes of this definition, "submitted"
|
| 55 |
+
means any form of electronic, verbal, or written communication sent
|
| 56 |
+
to the Licensor or its representatives, including but not limited to
|
| 57 |
+
communication on electronic mailing lists, source code control systems,
|
| 58 |
+
and issue tracking systems that are managed by, or on behalf of, the
|
| 59 |
+
Licensor for the purpose of discussing and improving the Work, but
|
| 60 |
+
excluding communication that is conspicuously marked or otherwise
|
| 61 |
+
designated in writing by the copyright owner as "Not a Contribution."
|
| 62 |
+
|
| 63 |
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
| 64 |
+
on behalf of whom a Contribution has been received by Licensor and
|
| 65 |
+
subsequently incorporated within the Work.
|
| 66 |
+
|
| 67 |
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
| 68 |
+
this License, each Contributor hereby grants to You a perpetual,
|
| 69 |
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
| 70 |
+
copyright license to reproduce, prepare Derivative Works of,
|
| 71 |
+
publicly display, publicly perform, sublicense, and distribute the
|
| 72 |
+
Work and such Derivative Works in Source or Object form.
|
| 73 |
+
|
| 74 |
+
3. Grant of Patent License. Subject to the terms and conditions of
|
| 75 |
+
this License, each Contributor hereby grants to You a perpetual,
|
| 76 |
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
| 77 |
+
(except as stated in this section) patent license to make, have made,
|
| 78 |
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
| 79 |
+
where such license applies only to those patent claims licensable
|
| 80 |
+
by such Contributor that are necessarily infringed by their
|
| 81 |
+
Contribution(s) alone or by combination of their Contribution(s)
|
| 82 |
+
with the Work to which such Contribution(s) was submitted. If You
|
| 83 |
+
institute patent litigation against any entity (including a
|
| 84 |
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
| 85 |
+
or a Contribution incorporated within the Work constitutes direct
|
| 86 |
+
or contributory patent infringement, then any patent licenses
|
| 87 |
+
granted to You under this License for that Work shall terminate
|
| 88 |
+
as of the date such litigation is filed.
|
| 89 |
+
|
| 90 |
+
4. Redistribution. You may reproduce and distribute copies of the
|
| 91 |
+
Work or Derivative Works thereof in any medium, with or without
|
| 92 |
+
modifications, and in Source or Object form, provided that You
|
| 93 |
+
meet the following conditions:
|
| 94 |
+
|
| 95 |
+
(a) You must give any other recipients of the Work or
|
| 96 |
+
Derivative Works a copy of this License; and
|
| 97 |
+
|
| 98 |
+
(b) You must cause any modified files to carry prominent notices
|
| 99 |
+
stating that You changed the files; and
|
| 100 |
+
|
| 101 |
+
(c) You must retain, in the Source form of any Derivative Works
|
| 102 |
+
that You distribute, all copyright, patent, trademark, and
|
| 103 |
+
attribution notices from the Source form of the Work,
|
| 104 |
+
excluding those notices that do not pertain to any part of
|
| 105 |
+
the Derivative Works; and
|
| 106 |
+
|
| 107 |
+
(d) If the Work includes a "NOTICE" text file as part of its
|
| 108 |
+
distribution, then any Derivative Works that You distribute must
|
| 109 |
+
include a readable copy of the attribution notices contained
|
| 110 |
+
within such NOTICE file, excluding those notices that do not
|
| 111 |
+
pertain to any part of the Derivative Works, in at least one
|
| 112 |
+
of the following places: within a NOTICE text file distributed
|
| 113 |
+
as part of the Derivative Works; within the Source form or
|
| 114 |
+
documentation, if provided along with the Derivative Works; or,
|
| 115 |
+
within a display generated by the Derivative Works, if and
|
| 116 |
+
wherever such third-party notices normally appear. The contents
|
| 117 |
+
of the NOTICE file are for informational purposes only and
|
| 118 |
+
do not modify the License. You may add Your own attribution
|
| 119 |
+
notices within Derivative Works that You distribute, alongside
|
| 120 |
+
or as an addendum to the NOTICE text from the Work, provided
|
| 121 |
+
that such additional attribution notices cannot be construed
|
| 122 |
+
as modifying the License.
|
| 123 |
+
|
| 124 |
+
You may add Your own copyright statement to Your modifications and
|
| 125 |
+
may provide additional or different license terms and conditions
|
| 126 |
+
for use, reproduction, or distribution of Your modifications, or
|
| 127 |
+
for any such Derivative Works as a whole, provided Your use,
|
| 128 |
+
reproduction, and distribution of the Work otherwise complies with
|
| 129 |
+
the conditions stated in this License.
|
| 130 |
+
|
| 131 |
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
| 132 |
+
any Contribution intentionally submitted for inclusion in the Work
|
| 133 |
+
by You to the Licensor shall be under the terms and conditions of
|
| 134 |
+
this License, without any additional terms or conditions.
|
| 135 |
+
Notwithstanding the above, nothing herein shall supersede or modify
|
| 136 |
+
the terms of any separate license agreement you may have executed
|
| 137 |
+
with Licensor regarding such Contributions.
|
| 138 |
+
|
| 139 |
+
6. Trademarks. This License does not grant permission to use the trade
|
| 140 |
+
names, trademarks, service marks, or product names of the Licensor,
|
| 141 |
+
except as required for reasonable and customary use in describing the
|
| 142 |
+
origin of the Work and reproducing the content of the NOTICE file.
|
| 143 |
+
|
| 144 |
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
| 145 |
+
agreed to in writing, Licensor provides the Work (and each
|
| 146 |
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
| 147 |
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
| 148 |
+
implied, including, without limitation, any warranties or conditions
|
| 149 |
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
| 150 |
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
| 151 |
+
appropriateness of using or redistributing the Work and assume any
|
| 152 |
+
risks associated with Your exercise of permissions under this License.
|
| 153 |
+
|
| 154 |
+
8. Limitation of Liability. In no event and under no legal theory,
|
| 155 |
+
whether in tort (including negligence), contract, or otherwise,
|
| 156 |
+
unless required by applicable law (such as deliberate and grossly
|
| 157 |
+
negligent acts) or agreed to in writing, shall any Contributor be
|
| 158 |
+
liable to You for damages, including any direct, indirect, special,
|
| 159 |
+
incidental, or consequential damages of any character arising as a
|
| 160 |
+
result of this License or out of the use or inability to use the
|
| 161 |
+
Work (including but not limited to damages for loss of goodwill,
|
| 162 |
+
work stoppage, computer failure or malfunction, or any and all
|
| 163 |
+
other commercial damages or losses), even if such Contributor
|
| 164 |
+
has been advised of the possibility of such damages.
|
| 165 |
+
|
| 166 |
+
9. Accepting Warranty or Additional Liability. While redistributing
|
| 167 |
+
the Work or Derivative Works thereof, You may choose to offer,
|
| 168 |
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
| 169 |
+
or other liability obligations and/or rights consistent with this
|
| 170 |
+
License. However, in accepting such obligations, You may act only
|
| 171 |
+
on Your own behalf and on Your sole responsibility, not on behalf
|
| 172 |
+
of any other Contributor, and only if You agree to indemnify,
|
| 173 |
+
defend, and hold each Contributor harmless for any liability
|
| 174 |
+
incurred by, or claims asserted against, such Contributor by reason
|
| 175 |
+
of your accepting any such warranty or additional liability.
|
| 176 |
+
|
| 177 |
+
END OF TERMS AND CONDITIONS
|
| 178 |
+
|
| 179 |
+
APPENDIX: How to apply the Apache License to your work.
|
| 180 |
+
|
| 181 |
+
To apply the Apache License to your work, attach the following
|
| 182 |
+
boilerplate notice, with the fields enclosed by brackets "[]"
|
| 183 |
+
replaced with your own identifying information. (Don't include
|
| 184 |
+
the brackets!) The text should be enclosed in the appropriate
|
| 185 |
+
comment syntax for the file format. We also recommend that a
|
| 186 |
+
file or class name and description of purpose be included on the
|
| 187 |
+
same "printed page" as the copyright notice for easier
|
| 188 |
+
identification within third-party archives.
|
| 189 |
+
|
| 190 |
+
Copyright [yyyy] [name of copyright owner]
|
| 191 |
+
|
| 192 |
+
Licensed under the Apache License, Version 2.0 (the "License");
|
| 193 |
+
you may not use this file except in compliance with the License.
|
| 194 |
+
You may obtain a copy of the License at
|
| 195 |
+
|
| 196 |
+
http://www.apache.org/licenses/LICENSE-2.0
|
| 197 |
+
|
| 198 |
+
Unless required by applicable law or agreed to in writing, software
|
| 199 |
+
distributed under the License is distributed on an "AS IS" BASIS,
|
| 200 |
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 201 |
+
See the License for the specific language governing permissions and
|
| 202 |
+
limitations under the License.
|
code/NOTICE
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
ViSS: Visible-State Supervision and Test-Time Causal View Search
|
| 2 |
+
|
| 3 |
+
This source release is derived from the ViSS research repository and carries
|
| 4 |
+
that repository's Apache License 2.0, supplied in LICENSE.
|
| 5 |
+
|
| 6 |
+
The code uses external libraries including PyTorch, Transformers, Accelerate,
|
| 7 |
+
DeepSpeed, qwen-vl-utils, Decord, NumPy, and Pillow. Those projects retain
|
| 8 |
+
their own licenses. Qwen3-VL model weights, training media, benchmark data,
|
| 9 |
+
and official benchmark scorers are not redistributed in this archive.
|
| 10 |
+
|
| 11 |
+
Evaluation adapters preserve the StreamingBench/SimpleStream prompt interface
|
| 12 |
+
and OVO-Bench task interfaces described in the evaluation protocol.
|
| 13 |
+
Users must obtain the corresponding resources under their original terms.
|
| 14 |
+
|
| 15 |
+
The README organization takes inspiration from the OneStreamer repository.
|
| 16 |
+
No OneStreamer implementation, model weights, data, or figures are included.
|
code/README.md
ADDED
|
@@ -0,0 +1,135 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
<div align="center">
|
| 2 |
+
|
| 3 |
+
# ViSS
|
| 4 |
+
### Supervising the Visible State: Separating Representation from View Selection in Streaming Video
|
| 5 |
+
|
| 6 |
+
Yihao Hu*, Pan Wang*, Yitong Lin*, Zhihao Wen†, Deyuan Liu, Hang Wang,
|
| 7 |
+
Mingda WAN, Yiwen Zhao, Yutong Li, Yuchen Mao, Tao Lin†
|
| 8 |
+
|
| 9 |
+
<sup>* Equal contribution. † Corresponding author.</sup>
|
| 10 |
+
|
| 11 |
+
[Overview](#overview) · [Codebase](#codebase) · [Quick start](#quick-start) ·
|
| 12 |
+
[Models and data](#models-and-data) · [Training](docs/TRAINING.md) ·
|
| 13 |
+
[Evaluation](docs/EVALUATION.md) · [Citation](#citation)
|
| 14 |
+
|
| 15 |
+
</div>
|
| 16 |
+
|
| 17 |
+
## Overview
|
| 18 |
+
|
| 19 |
+
**Visible-State Supervision (ViSS)** trains a vision–language model on
|
| 20 |
+
source-grounded questions about event identity, order, accumulation,
|
| 21 |
+
relation changes, and whether the visible evidence is sufficient.
|
| 22 |
+
|
| 23 |
+
**Test-Time Causal View Search (TVS)** keeps the trained model fixed and
|
| 24 |
+
acquires complementary causal views. It starts with recent frames, requests
|
| 25 |
+
an event view when needed, and consults compact history on disagreement.
|
| 26 |
+
Acquisition and answer commitment use separately calibrated thresholds.
|
| 27 |
+
|
| 28 |
+
This release provides data construction, training, inference, policy replay,
|
| 29 |
+
and scoring code. It contains no saved benchmark scores, prediction outputs,
|
| 30 |
+
or training logs. Scoring utilities and synthetic unit tests are included.
|
| 31 |
+
|
| 32 |
+
## Codebase
|
| 33 |
+
|
| 34 |
+
| Directory | Purpose | Guide |
|
| 35 |
+
| --- | --- | --- |
|
| 36 |
+
| `data_pipeline/` | Build source-grounded probes and matched generic controls | [Training](docs/TRAINING.md) |
|
| 37 |
+
| `experiments/` | Train the model, generate causal-view proposals, and apply TVS | [Evaluation](docs/EVALUATION.md) |
|
| 38 |
+
| `viss_core/` | Record validation, media handling, and benchmark adapters | [Data preparation](docs/DATA.md) |
|
| 39 |
+
| `scripts/` | Evaluation launchers, shard merging, and official scoring | [Evaluation](docs/EVALUATION.md) |
|
| 40 |
+
| `configs/` | Evaluation protocol and DeepSpeed configuration | [Installation](docs/INSTALL.md) |
|
| 41 |
+
| `tests/` | CPU checks of data contracts, sampling, loss masks, and view policies | [Quick start](#quick-start) |
|
| 42 |
+
|
| 43 |
+
The repository contains no internal training manifests, run records, data
|
| 44 |
+
inventories, calibration outputs, or file identities from company experiments.
|
| 45 |
+
The evaluation configuration is a public template; each user binds it to
|
| 46 |
+
their own inputs. See [Data preparation](docs/DATA.md).
|
| 47 |
+
|
| 48 |
+
## Quick start
|
| 49 |
+
|
| 50 |
+
Run these commands from the extracted repository root.
|
| 51 |
+
|
| 52 |
+
### Install and check the source
|
| 53 |
+
|
| 54 |
+
```bash
|
| 55 |
+
python -m venv .venv
|
| 56 |
+
source .venv/bin/activate
|
| 57 |
+
python -m pip install --upgrade pip
|
| 58 |
+
python -m pip install -e ".[test]"
|
| 59 |
+
python -m pytest -q tests
|
| 60 |
+
```
|
| 61 |
+
|
| 62 |
+
The unit tests use synthetic inputs and do not download model weights or
|
| 63 |
+
benchmark media. GPU inference and training require the additional setup in
|
| 64 |
+
[Installation](docs/INSTALL.md).
|
| 65 |
+
|
| 66 |
+
### Generate predictions and score separately
|
| 67 |
+
|
| 68 |
+
After preparing your model, annotations, query exports, media, and
|
| 69 |
+
[local evaluation protocol](docs/EVALUATION.md#1-generate-recent-view-proposals):
|
| 70 |
+
|
| 71 |
+
```bash
|
| 72 |
+
python -m experiments.evaluate_qwen3vl_recent4_full \
|
| 73 |
+
--benchmark streamingbench \
|
| 74 |
+
--annotation data/streamingbench/annotations.json \
|
| 75 |
+
--samples data/streamingbench/queries.jsonl \
|
| 76 |
+
--video-root data/streamingbench/videos \
|
| 77 |
+
--model checkpoints/viss \
|
| 78 |
+
--processor checkpoints/viss/locked_processor \
|
| 79 |
+
--protocol configs/evaluation/recent4_full_v1.json \
|
| 80 |
+
--frame-policy recent4 \
|
| 81 |
+
--visual-input-mode image-list \
|
| 82 |
+
--record-choice-confidence \
|
| 83 |
+
--output outputs/streamingbench_recent.json
|
| 84 |
+
```
|
| 85 |
+
|
| 86 |
+
The paths above are placeholders for separately prepared assets. Bind the
|
| 87 |
+
processor and query exports used in your evaluation before running inference.
|
| 88 |
+
The [evaluation guide](docs/EVALUATION.md) explains model
|
| 89 |
+
identity, TVS inputs, official scorers, and full-population checks.
|
| 90 |
+
|
| 91 |
+
## Models and data
|
| 92 |
+
|
| 93 |
+
| Resource | Included in this archive? | Preparation |
|
| 94 |
+
| --- | --- | --- |
|
| 95 |
+
| Source code, tests, and protocol | Yes | This repository |
|
| 96 |
+
| Qwen3-VL-8B-Instruct initialization | No | Obtain the official model |
|
| 97 |
+
| Trained ViSS weights and locked processor | No | Obtain separately or train ViSS |
|
| 98 |
+
| Accepted training records and media | No | Follow [data preparation](docs/DATA.md) |
|
| 99 |
+
| Benchmark annotations, videos, and official scorers | No | Obtain from the benchmark owners |
|
| 100 |
+
| TVS calibration and model-identity artifacts | No | Supply artifacts matching your evaluation |
|
| 101 |
+
|
| 102 |
+
No public download endpoint for ViSS-specific weights or training records is
|
| 103 |
+
asserted by this source package.
|
| 104 |
+
|
| 105 |
+
## Training
|
| 106 |
+
|
| 107 |
+
The paper recipe is a fresh one-epoch run from Qwen3-VL-8B-Instruct.
|
| 108 |
+
It freezes the vision tower and updates the language model and multimodal
|
| 109 |
+
projector. [Training](docs/TRAINING.md) gives the two-GPU command and explains
|
| 110 |
+
the required sealed training release.
|
| 111 |
+
|
| 112 |
+
Some serialized records retain `streameva.*` schema identifiers. These are
|
| 113 |
+
record-format identifiers; the corresponding implementation in this release
|
| 114 |
+
is under `viss_core/`.
|
| 115 |
+
|
| 116 |
+
## License and acknowledgements
|
| 117 |
+
|
| 118 |
+
The project code uses the repository's [Apache-2.0 license](LICENSE).
|
| 119 |
+
Third-party software, media, models, and benchmark assets retain their
|
| 120 |
+
respective terms; see [NOTICE](NOTICE).
|
| 121 |
+
|
| 122 |
+
We acknowledge Qwen3-VL, StreamingBench, OVO-Bench, and SimpleStream.
|
| 123 |
+
The documentation organization follows the separation of setup, inference,
|
| 124 |
+
and evaluation used in [OneStreamer](https://github.com/MCG-NJU/OneStreamer).
|
| 125 |
+
|
| 126 |
+
## Citation
|
| 127 |
+
|
| 128 |
+
```bibtex
|
| 129 |
+
@misc{hu2026viss,
|
| 130 |
+
title={Supervising the Visible State: Separating Representation from View Selection in Streaming Video},
|
| 131 |
+
author={Yihao Hu and Pan Wang and Yitong Lin and Zhihao Wen and Deyuan Liu and Hang Wang and Mingda Wan and Yiwen Zhao and Yutong Li and Yuchen Mao and Tao Lin},
|
| 132 |
+
year={2026},
|
| 133 |
+
note={Manuscript}
|
| 134 |
+
}
|
| 135 |
+
```
|
code/configs/evaluation/recent4.example.json
ADDED
|
@@ -0,0 +1,74 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"format": "iclr_video.locked_evaluation_protocol.v1",
|
| 3 |
+
"protocol_id": "recent4_full_v1",
|
| 4 |
+
"status": "template",
|
| 5 |
+
"applies_to": "all baseline and candidate checkpoints",
|
| 6 |
+
"inference": {
|
| 7 |
+
"causal_prefix_only": true,
|
| 8 |
+
"sampling_fps": 1.0,
|
| 9 |
+
"context_policy": "recent_n",
|
| 10 |
+
"recent_frames": 4,
|
| 11 |
+
"timestamp_rule": "match the SimpleStream release: decode a four-second half-open causal prefix window [max(0,t-4),t) at nominal 1 FPS and retain exactly four independently encoded image frames; only prefixes shorter than two raw frames repeat frame zero",
|
| 12 |
+
"allow_future_frames": false,
|
| 13 |
+
"allow_global_frames": false,
|
| 14 |
+
"allow_extra_frame_inspection": false,
|
| 15 |
+
"allow_persistent_visual_or_text_memory": false,
|
| 16 |
+
"visual_input_mode": "four independent image frames using qwen-vl-utils fetch_video (image_patch_size=14), never Qwen video tokens",
|
| 17 |
+
"processor_contract": {
|
| 18 |
+
"preprocessor_config_sha256": "",
|
| 19 |
+
"image_processor_type": "Qwen2VLImageProcessorFast",
|
| 20 |
+
"patch_size": 16,
|
| 21 |
+
"merge_size": 2,
|
| 22 |
+
"temporal_patch_size": 2,
|
| 23 |
+
"shortest_edge": 65536,
|
| 24 |
+
"longest_edge": 16777216,
|
| 25 |
+
"reference_visual_tokens_per_frame": 576,
|
| 26 |
+
"reason": "Bind the processor used by your evaluation before inference."
|
| 27 |
+
},
|
| 28 |
+
"prompt_source": "StreamingBench SimpleStream release MCQ template; OVO backward/realtime SimpleStream release one-letter template; OVO forward frozen official task wrapper; no system message",
|
| 29 |
+
"decode": {
|
| 30 |
+
"do_sample": false,
|
| 31 |
+
"temperature": 0.0
|
| 32 |
+
}
|
| 33 |
+
},
|
| 34 |
+
"benchmarks": {
|
| 35 |
+
"streamingbench": {
|
| 36 |
+
"scope": "Real-Time full",
|
| 37 |
+
"export": "data/streamingbench/queries.jsonl",
|
| 38 |
+
"export_sha256": "",
|
| 39 |
+
"expected_queries": 2495,
|
| 40 |
+
"expected_groups": {
|
| 41 |
+
"real_time_visual": 2495
|
| 42 |
+
},
|
| 43 |
+
"primary_metric": "official Real-Time accuracy"
|
| 44 |
+
},
|
| 45 |
+
"ovo": {
|
| 46 |
+
"scope": "all 12 tasks: Real-Time, Backward, and Forward",
|
| 47 |
+
"export": "data/ovo/queries.jsonl",
|
| 48 |
+
"export_sha256": "",
|
| 49 |
+
"expected_queries": 3035,
|
| 50 |
+
"expected_tasks": {
|
| 51 |
+
"ACR": 109,
|
| 52 |
+
"ASI": 148,
|
| 53 |
+
"ATR": 116,
|
| 54 |
+
"CRR": 240,
|
| 55 |
+
"EPM": 297,
|
| 56 |
+
"FPD": 101,
|
| 57 |
+
"HLD": 186,
|
| 58 |
+
"OCR": 149,
|
| 59 |
+
"OJR": 184,
|
| 60 |
+
"REC": 698,
|
| 61 |
+
"SSR": 629,
|
| 62 |
+
"STU": 178
|
| 63 |
+
},
|
| 64 |
+
"primary_metric": "official Total Avg",
|
| 65 |
+
"secondary_metrics": [
|
| 66 |
+
"micro Overall",
|
| 67 |
+
"Real-Time Avg",
|
| 68 |
+
"Backward Avg",
|
| 69 |
+
"Forward Avg",
|
| 70 |
+
"RT/BT Avg"
|
| 71 |
+
]
|
| 72 |
+
}
|
| 73 |
+
}
|
| 74 |
+
}
|
code/configs/training/deepspeed_zero3_bf16.json
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"bf16": {"enabled": "auto"},
|
| 3 |
+
"train_micro_batch_size_per_gpu": "auto",
|
| 4 |
+
"train_batch_size": "auto",
|
| 5 |
+
"gradient_accumulation_steps": "auto",
|
| 6 |
+
"gradient_clipping": "auto",
|
| 7 |
+
"zero_optimization": {
|
| 8 |
+
"stage": 3,
|
| 9 |
+
"overlap_comm": true,
|
| 10 |
+
"contiguous_gradients": true,
|
| 11 |
+
"reduce_bucket_size": "auto",
|
| 12 |
+
"stage3_prefetch_bucket_size": "auto",
|
| 13 |
+
"stage3_param_persistence_threshold": "auto",
|
| 14 |
+
"stage3_gather_16bit_weights_on_model_save": true
|
| 15 |
+
}
|
| 16 |
+
}
|
code/data_pipeline/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
"""Stable, user-facing entry points for reproducing StreamEVA data."""
|
code/data_pipeline/temporal_scope/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
"""Deterministic semantic-SFT and public typed-contract compilers."""
|
code/data_pipeline/temporal_scope/build_size_matched_generic_control.py
ADDED
|
@@ -0,0 +1,107 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Select a deterministic, source-stratified generic temporal SFT control."""
|
| 3 |
+
|
| 4 |
+
from __future__ import annotations
|
| 5 |
+
|
| 6 |
+
import argparse
|
| 7 |
+
import hashlib
|
| 8 |
+
import json
|
| 9 |
+
from collections import defaultdict
|
| 10 |
+
from pathlib import Path
|
| 11 |
+
from typing import Any
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
SUPPORTED_PARENT_KINDS = (
|
| 15 |
+
"source_semantic_row",
|
| 16 |
+
"cited_nonbenchmark_refresh_projection",
|
| 17 |
+
)
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
def _rank(seed: str, parent_id: str) -> str:
|
| 21 |
+
return hashlib.sha256(f"{seed}\0{parent_id}".encode()).hexdigest()
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
def select_parent_ids(
|
| 25 |
+
rows: list[dict[str, Any]], *, seed: str, quotas: dict[str, int]
|
| 26 |
+
) -> tuple[list[str], dict[str, int]]:
|
| 27 |
+
grouped: dict[str, list[str]] = defaultdict(list)
|
| 28 |
+
seen: set[str] = set()
|
| 29 |
+
for row in rows:
|
| 30 |
+
parents = row.get("parents") or []
|
| 31 |
+
if len(parents) != 1:
|
| 32 |
+
raise ValueError("generic control requires exactly one parent per row")
|
| 33 |
+
parent_id = str(parents[0].get("id") or "")
|
| 34 |
+
parent_kind = str(parents[0].get("kind") or "")
|
| 35 |
+
if not parent_id or parent_kind not in SUPPORTED_PARENT_KINDS:
|
| 36 |
+
raise ValueError("generic control parent provenance is malformed")
|
| 37 |
+
if parent_id in seen:
|
| 38 |
+
raise ValueError("generic control parent IDs must be unique")
|
| 39 |
+
seen.add(parent_id)
|
| 40 |
+
grouped[parent_kind].append(parent_id)
|
| 41 |
+
|
| 42 |
+
selected: list[str] = []
|
| 43 |
+
counts: dict[str, int] = {}
|
| 44 |
+
for kind in SUPPORTED_PARENT_KINDS:
|
| 45 |
+
quota = int(quotas.get(kind, 0))
|
| 46 |
+
candidates = grouped[kind]
|
| 47 |
+
if quota <= 0 or quota > len(candidates):
|
| 48 |
+
raise ValueError(f"invalid quota for {kind}: {quota}/{len(candidates)}")
|
| 49 |
+
ranked = sorted(candidates, key=lambda value: (_rank(seed, value), value))
|
| 50 |
+
chosen = ranked[:quota]
|
| 51 |
+
selected.extend(chosen)
|
| 52 |
+
counts[kind] = len(chosen)
|
| 53 |
+
return sorted(selected), counts
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
def main() -> None:
|
| 57 |
+
parser = argparse.ArgumentParser()
|
| 58 |
+
parser.add_argument("--source", type=Path, required=True)
|
| 59 |
+
parser.add_argument("--output", type=Path, required=True)
|
| 60 |
+
parser.add_argument("--seed", default="viss-generic-control-7677-v1")
|
| 61 |
+
parser.add_argument("--source-semantic-rows", type=int, default=5903)
|
| 62 |
+
parser.add_argument("--cited-refresh-rows", type=int, default=1774)
|
| 63 |
+
args = parser.parse_args()
|
| 64 |
+
|
| 65 |
+
source = args.source.resolve(strict=True)
|
| 66 |
+
trainer = source / "trainer/sft.jsonl"
|
| 67 |
+
manifest = source / "manifest.json"
|
| 68 |
+
if args.output.exists():
|
| 69 |
+
raise FileExistsError(args.output)
|
| 70 |
+
rows = [json.loads(line) for line in trainer.read_text().splitlines() if line]
|
| 71 |
+
selected, counts = select_parent_ids(
|
| 72 |
+
rows,
|
| 73 |
+
seed=args.seed,
|
| 74 |
+
quotas={
|
| 75 |
+
"source_semantic_row": args.source_semantic_rows,
|
| 76 |
+
"cited_nonbenchmark_refresh_projection": args.cited_refresh_rows,
|
| 77 |
+
},
|
| 78 |
+
)
|
| 79 |
+
if len(selected) != args.source_semantic_rows + args.cited_refresh_rows:
|
| 80 |
+
raise AssertionError("generic control selection size diverged")
|
| 81 |
+
|
| 82 |
+
payload = {
|
| 83 |
+
"format": "iclr_video.capability_parent_selection.v1",
|
| 84 |
+
"status": "sealed_deduplicated_union",
|
| 85 |
+
"purpose": "size_matched_generic_temporal_sft_control",
|
| 86 |
+
"seed": args.seed,
|
| 87 |
+
"source_release": str(source),
|
| 88 |
+
"source_release_id": json.loads(manifest.read_text()).get("release_id"),
|
| 89 |
+
"parent_ids": selected,
|
| 90 |
+
"profiles": list(SUPPORTED_PARENT_KINDS),
|
| 91 |
+
"counts_by_profile": counts,
|
| 92 |
+
"total_rows": len(selected),
|
| 93 |
+
"contracts": {
|
| 94 |
+
"benchmark_content_used_for_selection": False,
|
| 95 |
+
"source_release_membership_verified": True,
|
| 96 |
+
"model_predictions_used_for_selection": False,
|
| 97 |
+
"task_labels_used_for_selection": False,
|
| 98 |
+
"selection_is_sha256_ranked_within_source_kind": True,
|
| 99 |
+
},
|
| 100 |
+
}
|
| 101 |
+
args.output.parent.mkdir(parents=True, exist_ok=True)
|
| 102 |
+
args.output.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n")
|
| 103 |
+
print(json.dumps({"output": str(args.output), "rows": len(selected), "counts": counts}))
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
if __name__ == "__main__":
|
| 107 |
+
main()
|
code/data_pipeline/temporal_scope/build_source_aligned_sft_release.py
ADDED
|
@@ -0,0 +1,722 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Compose one source-aligned, benchmark-firewalled pure-VLM SFT release.
|
| 3 |
+
|
| 4 |
+
The inputs are already sealed semantic releases built from non-benchmark
|
| 5 |
+
training videos. This compiler keeps the original semantic labels, removes
|
| 6 |
+
overlap by cited source identity and exact pixel/question/answer identity,
|
| 7 |
+
canonicalizes media paths by SHA256, and emits the only JSONL consumed by one
|
| 8 |
+
joint SFT run. Benchmark examples and auditor predictions are never labels.
|
| 9 |
+
"""
|
| 10 |
+
|
| 11 |
+
from __future__ import annotations
|
| 12 |
+
|
| 13 |
+
import argparse
|
| 14 |
+
import collections
|
| 15 |
+
import copy
|
| 16 |
+
import hashlib
|
| 17 |
+
import json
|
| 18 |
+
import math
|
| 19 |
+
import os
|
| 20 |
+
import random
|
| 21 |
+
import shutil
|
| 22 |
+
import tempfile
|
| 23 |
+
from pathlib import Path
|
| 24 |
+
from typing import Any
|
| 25 |
+
|
| 26 |
+
from viss_core.data_v0.canonical import (
|
| 27 |
+
attach_content_sha256,
|
| 28 |
+
canonical_json_bytes,
|
| 29 |
+
canonical_json_line,
|
| 30 |
+
file_sha256,
|
| 31 |
+
read_jsonl,
|
| 32 |
+
stable_id,
|
| 33 |
+
write_jsonl,
|
| 34 |
+
)
|
| 35 |
+
from viss_core.data_v0.schema import validate_row
|
| 36 |
+
from viss_core.data_v0.semantic_release import (
|
| 37 |
+
EXPECTED_PUBLIC_UPSTREAM_MANIFEST_SHA256,
|
| 38 |
+
FORMAT,
|
| 39 |
+
validate_semantic_release,
|
| 40 |
+
)
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
class SourceAlignedSFTError(ValueError):
|
| 44 |
+
"""Raised when source releases cannot be composed without leakage."""
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
def _target(row: dict[str, Any]) -> str:
|
| 48 |
+
values = [
|
| 49 |
+
str(item.get("text") or "")
|
| 50 |
+
for message in row.get("messages") or []
|
| 51 |
+
if message.get("role") == "assistant" and message.get("loss") is True
|
| 52 |
+
for item in message.get("content") or []
|
| 53 |
+
if item.get("type") == "text"
|
| 54 |
+
]
|
| 55 |
+
if len(values) != 1:
|
| 56 |
+
raise SourceAlignedSFTError("semantic row has no unique target")
|
| 57 |
+
return values[0]
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
def _question(row: dict[str, Any]) -> str:
|
| 61 |
+
values = [
|
| 62 |
+
str(item.get("text") or "")
|
| 63 |
+
for message in row.get("messages") or []
|
| 64 |
+
if message.get("role") == "user"
|
| 65 |
+
for item in message.get("content") or []
|
| 66 |
+
if item.get("type") == "text"
|
| 67 |
+
]
|
| 68 |
+
if len(values) != 1:
|
| 69 |
+
raise SourceAlignedSFTError("semantic row has no unique public prompt")
|
| 70 |
+
return " ".join(values[0].split())
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
def _images(row: dict[str, Any]) -> list[str]:
|
| 74 |
+
return [
|
| 75 |
+
str(item["image"])
|
| 76 |
+
for message in row.get("messages") or []
|
| 77 |
+
for item in message.get("content") or []
|
| 78 |
+
if item.get("type") == "image"
|
| 79 |
+
]
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
def _cited_parents(row: dict[str, Any]) -> set[str]:
|
| 83 |
+
return {
|
| 84 |
+
str(parent["id"])
|
| 85 |
+
for parent in row.get("parents") or []
|
| 86 |
+
if parent.get("kind") == "cited_nonbenchmark_refresh_projection"
|
| 87 |
+
and parent.get("id")
|
| 88 |
+
}
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
def _semantic_identity(
|
| 92 |
+
row: dict[str, Any], inventory: dict[str, dict[str, Any]]
|
| 93 |
+
) -> str:
|
| 94 |
+
try:
|
| 95 |
+
image_sha256s = [str(inventory[path]["sha256"]) for path in _images(row)]
|
| 96 |
+
except KeyError as exc:
|
| 97 |
+
raise SourceAlignedSFTError("row references undeclared source media") from exc
|
| 98 |
+
payload = {
|
| 99 |
+
"answer": _target(row),
|
| 100 |
+
"image_sha256s": image_sha256s,
|
| 101 |
+
"question": _question(row),
|
| 102 |
+
}
|
| 103 |
+
return hashlib.sha256(canonical_json_bytes(payload)).hexdigest()
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
def _canonical_media_path(relative: str, digest: str) -> str:
|
| 107 |
+
suffix = Path(relative).suffix.casefold()
|
| 108 |
+
if not suffix or len(suffix) > 10:
|
| 109 |
+
raise SourceAlignedSFTError("source media suffix is invalid")
|
| 110 |
+
return (Path("media/sha256") / digest[:2] / f"{digest}{suffix}").as_posix()
|
| 111 |
+
|
| 112 |
+
|
| 113 |
+
def _materialize(source: Path, destination: Path, *, mode: str) -> None:
|
| 114 |
+
if source.is_symlink() or not source.is_file():
|
| 115 |
+
raise SourceAlignedSFTError(f"unsafe source media: {source}")
|
| 116 |
+
destination.parent.mkdir(parents=True, exist_ok=True)
|
| 117 |
+
if mode in {"hardlink", "auto"}:
|
| 118 |
+
try:
|
| 119 |
+
os.link(source, destination)
|
| 120 |
+
return
|
| 121 |
+
except OSError:
|
| 122 |
+
if mode == "hardlink":
|
| 123 |
+
raise
|
| 124 |
+
shutil.copy2(source, destination)
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
def _lineage_key(manifest: dict[str, Any]) -> tuple[str, str, str]:
|
| 128 |
+
lineage = manifest.get("model_lineage") or {}
|
| 129 |
+
return (
|
| 130 |
+
str(lineage.get("required_sha256s_sha256") or ""),
|
| 131 |
+
str(lineage.get("upstream_repo") or ""),
|
| 132 |
+
str(lineage.get("upstream_revision") or ""),
|
| 133 |
+
)
|
| 134 |
+
|
| 135 |
+
|
| 136 |
+
def _teacher_used_for_labels(manifest: dict[str, Any]) -> bool:
|
| 137 |
+
value = (manifest.get("contracts") or {}).get("teacher_used_for_labels")
|
| 138 |
+
if not isinstance(value, bool):
|
| 139 |
+
raise SourceAlignedSFTError(
|
| 140 |
+
"source teacher-label provenance is absent or malformed"
|
| 141 |
+
)
|
| 142 |
+
return value
|
| 143 |
+
|
| 144 |
+
|
| 145 |
+
def _load_legacy_v62di_source(root: Path, *, mode: str = "v62di_only"):
|
| 146 |
+
raise SourceAlignedSFTError("Private legacy adapters are not distributed; supply a standard source release")
|
| 147 |
+
|
| 148 |
+
|
| 149 |
+
def _load_legacy_repcount_source(root: Path):
|
| 150 |
+
raise SourceAlignedSFTError("Private legacy adapters are not distributed; supply a standard source release")
|
| 151 |
+
|
| 152 |
+
|
| 153 |
+
def build(args: argparse.Namespace) -> dict[str, Any]:
|
| 154 |
+
sealed_source_count = (
|
| 155 |
+
len(args.source)
|
| 156 |
+
+ int(getattr(args, "legacy_v62di_source", None) is not None)
|
| 157 |
+
+ int(getattr(args, "legacy_repcount_source", None) is not None)
|
| 158 |
+
)
|
| 159 |
+
# A hash-bound parent selection is also a legitimate one-source filtering
|
| 160 |
+
# operation. Unfiltered releases remain compositions and therefore still
|
| 161 |
+
# require at least two sealed inputs.
|
| 162 |
+
if sealed_source_count < 2 and getattr(args, "include_parent_selection", None) is None:
|
| 163 |
+
raise SourceAlignedSFTError(
|
| 164 |
+
"at least two sealed sources are required unless a parent selection is supplied"
|
| 165 |
+
)
|
| 166 |
+
if args.copy_mode not in {"copy", "hardlink", "auto"}:
|
| 167 |
+
raise SourceAlignedSFTError("unsupported copy mode")
|
| 168 |
+
sample_loss_weight_cap = float(args.sample_loss_weight_cap)
|
| 169 |
+
if not math.isfinite(sample_loss_weight_cap) or sample_loss_weight_cap < 0:
|
| 170 |
+
raise SourceAlignedSFTError(
|
| 171 |
+
"sample loss weight cap must be finite and non-negative"
|
| 172 |
+
)
|
| 173 |
+
excluded_parent_kinds = {
|
| 174 |
+
str(kind).strip()
|
| 175 |
+
for kind in getattr(args, "exclude_parent_kind", [])
|
| 176 |
+
if str(kind).strip()
|
| 177 |
+
}
|
| 178 |
+
included_source_stage_release_ids = {
|
| 179 |
+
str(release_id).strip()
|
| 180 |
+
for release_id in getattr(args, "include_source_stage_release_id", [])
|
| 181 |
+
if str(release_id).strip()
|
| 182 |
+
}
|
| 183 |
+
parent_selection_path = getattr(args, "include_parent_selection", None)
|
| 184 |
+
selected_parent_ids: set[str] = set()
|
| 185 |
+
parent_selection_record: dict[str, Any] | None = None
|
| 186 |
+
if parent_selection_path is not None:
|
| 187 |
+
parent_selection_path = parent_selection_path.resolve(strict=True)
|
| 188 |
+
selection = json.loads(parent_selection_path.read_text(encoding="utf-8"))
|
| 189 |
+
if selection.get("format") not in {
|
| 190 |
+
"iclr_video.capability_parent_selection.v0",
|
| 191 |
+
"iclr_video.capability_parent_selection.v1",
|
| 192 |
+
}:
|
| 193 |
+
raise SourceAlignedSFTError("parent selection format diverged")
|
| 194 |
+
if selection.get("format").endswith(".v1") and (
|
| 195 |
+
selection.get("status") != "sealed_deduplicated_union"
|
| 196 |
+
or (selection.get("contracts") or {}).get(
|
| 197 |
+
"benchmark_content_used_for_selection"
|
| 198 |
+
)
|
| 199 |
+
is not False
|
| 200 |
+
or (selection.get("contracts") or {}).get(
|
| 201 |
+
"source_release_membership_verified"
|
| 202 |
+
)
|
| 203 |
+
is not True
|
| 204 |
+
):
|
| 205 |
+
raise SourceAlignedSFTError("v1 parent selection contract diverged")
|
| 206 |
+
parent_ids = selection.get("parent_ids") or []
|
| 207 |
+
if (
|
| 208 |
+
not isinstance(parent_ids, list)
|
| 209 |
+
or not parent_ids
|
| 210 |
+
or any(not isinstance(value, str) or not value for value in parent_ids)
|
| 211 |
+
or len(set(parent_ids)) != len(parent_ids)
|
| 212 |
+
):
|
| 213 |
+
raise SourceAlignedSFTError("parent selection IDs are malformed")
|
| 214 |
+
selected_parent_ids = set(parent_ids)
|
| 215 |
+
parent_selection_record = {
|
| 216 |
+
"path": str(parent_selection_path),
|
| 217 |
+
"sha256": file_sha256(parent_selection_path),
|
| 218 |
+
"parent_ids": len(selected_parent_ids),
|
| 219 |
+
"profiles": list(
|
| 220 |
+
selection.get("profiles")
|
| 221 |
+
or sorted((selection.get("counts_by_profile") or {}).keys())
|
| 222 |
+
),
|
| 223 |
+
}
|
| 224 |
+
output = args.output.resolve()
|
| 225 |
+
if output.exists():
|
| 226 |
+
raise FileExistsError(output)
|
| 227 |
+
|
| 228 |
+
ordered_curriculum = bool(getattr(args, "ordered_curriculum", False))
|
| 229 |
+
retain_semantic_exposures = bool(
|
| 230 |
+
getattr(args, "retain_semantic_exposures", False)
|
| 231 |
+
)
|
| 232 |
+
legacy_v62di_source = getattr(args, "legacy_v62di_source", None)
|
| 233 |
+
legacy_v62di_mode = getattr(args, "legacy_v62di_mode", "v62di_only")
|
| 234 |
+
legacy_repcount_source = getattr(args, "legacy_repcount_source", None)
|
| 235 |
+
if legacy_v62di_source is not None and not ordered_curriculum:
|
| 236 |
+
raise SourceAlignedSFTError(
|
| 237 |
+
"legacy V62DI input is allowed only for ordered curriculum"
|
| 238 |
+
)
|
| 239 |
+
if legacy_repcount_source is not None and not ordered_curriculum:
|
| 240 |
+
raise SourceAlignedSFTError(
|
| 241 |
+
"legacy RepCount input is allowed only for ordered curriculum"
|
| 242 |
+
)
|
| 243 |
+
expected_lineage: tuple[str, str, str] | None = None
|
| 244 |
+
seen_identity: set[str] = set()
|
| 245 |
+
seen_cited_parent: dict[str, str] = {}
|
| 246 |
+
accepted: list[tuple[dict[str, Any], Path, dict[str, dict[str, Any]], str]] = []
|
| 247 |
+
source_records = []
|
| 248 |
+
rejected: collections.Counter[str] = collections.Counter()
|
| 249 |
+
observed_selected_parent_ids: set[str] = set()
|
| 250 |
+
|
| 251 |
+
if legacy_v62di_source is not None:
|
| 252 |
+
legacy_root = legacy_v62di_source.resolve(strict=True)
|
| 253 |
+
legacy_rows, legacy_inventory, legacy_record = _load_legacy_v62di_source(
|
| 254 |
+
legacy_root, mode=legacy_v62di_mode
|
| 255 |
+
)
|
| 256 |
+
for row in legacy_rows:
|
| 257 |
+
identity = _semantic_identity(row, legacy_inventory)
|
| 258 |
+
accepted.append((row, legacy_root, legacy_inventory, identity))
|
| 259 |
+
seen_identity.add(identity)
|
| 260 |
+
source_records.append(legacy_record)
|
| 261 |
+
|
| 262 |
+
source_specs: list[tuple[str, Path]] = []
|
| 263 |
+
for source_index, raw_root in enumerate(args.source):
|
| 264 |
+
source_specs.append(("semantic", raw_root))
|
| 265 |
+
if source_index == 0 and legacy_repcount_source is not None:
|
| 266 |
+
source_specs.append(("legacy_repcount", legacy_repcount_source))
|
| 267 |
+
|
| 268 |
+
for source_kind, raw_root in source_specs:
|
| 269 |
+
root = raw_root.resolve(strict=True)
|
| 270 |
+
if source_kind == "legacy_repcount":
|
| 271 |
+
rows, inventory, source_record = _load_legacy_repcount_source(root)
|
| 272 |
+
manifest = json.loads((root / "manifest.json").read_text(encoding="utf-8"))
|
| 273 |
+
else:
|
| 274 |
+
manifest = validate_semantic_release(root, verify_media_hashes=False)
|
| 275 |
+
lineage_key = _lineage_key(manifest)
|
| 276 |
+
if expected_lineage is None:
|
| 277 |
+
expected_lineage = lineage_key
|
| 278 |
+
elif lineage_key != expected_lineage:
|
| 279 |
+
raise SourceAlignedSFTError("source model lineages diverged")
|
| 280 |
+
if lineage_key[0] != EXPECTED_PUBLIC_UPSTREAM_MANIFEST_SHA256:
|
| 281 |
+
raise SourceAlignedSFTError("source does not descend from pinned Instruct")
|
| 282 |
+
if (manifest.get("contracts") or {}).get("benchmark_rows_or_labels_admitted", 0) != 0:
|
| 283 |
+
raise SourceAlignedSFTError("source admits benchmark rows or labels")
|
| 284 |
+
teacher_used_for_labels = _teacher_used_for_labels(manifest)
|
| 285 |
+
|
| 286 |
+
if source_kind == "semantic":
|
| 287 |
+
inventory_rows = read_jsonl(
|
| 288 |
+
root / "manifests/media_inventory.jsonl", require_canonical=True
|
| 289 |
+
)
|
| 290 |
+
inventory = {str(item["path"]): item for item in inventory_rows}
|
| 291 |
+
if len(inventory) != len(inventory_rows):
|
| 292 |
+
raise SourceAlignedSFTError("source media inventory paths collide")
|
| 293 |
+
rows = read_jsonl(
|
| 294 |
+
root / "trainer/sft.jsonl",
|
| 295 |
+
verify_content_hashes=True,
|
| 296 |
+
require_canonical=True,
|
| 297 |
+
)
|
| 298 |
+
if included_source_stage_release_ids:
|
| 299 |
+
stages = (manifest.get("inputs") or {}).get("curriculum_stages") or []
|
| 300 |
+
selected_stages = [
|
| 301 |
+
stage
|
| 302 |
+
for stage in stages
|
| 303 |
+
if str(stage.get("release_id"))
|
| 304 |
+
in included_source_stage_release_ids
|
| 305 |
+
]
|
| 306 |
+
selected_ids = {
|
| 307 |
+
str(stage["release_id"]) for stage in selected_stages
|
| 308 |
+
}
|
| 309 |
+
missing_ids = included_source_stage_release_ids - selected_ids
|
| 310 |
+
if missing_ids:
|
| 311 |
+
raise SourceAlignedSFTError(
|
| 312 |
+
"requested source stages are absent: "
|
| 313 |
+
+ ",".join(sorted(missing_ids))
|
| 314 |
+
)
|
| 315 |
+
selected_rows = []
|
| 316 |
+
for stage in sorted(
|
| 317 |
+
selected_stages, key=lambda item: int(item["start_row"])
|
| 318 |
+
):
|
| 319 |
+
start = int(stage["start_row"])
|
| 320 |
+
end = int(stage["end_row_exclusive"])
|
| 321 |
+
if start < 0 or end <= start or end > len(rows):
|
| 322 |
+
raise SourceAlignedSFTError(
|
| 323 |
+
"source curriculum stage range is invalid"
|
| 324 |
+
)
|
| 325 |
+
if end - start != int(stage["rows"]):
|
| 326 |
+
raise SourceAlignedSFTError(
|
| 327 |
+
"source curriculum stage row count diverged"
|
| 328 |
+
)
|
| 329 |
+
selected_rows.extend(rows[start:end])
|
| 330 |
+
rows = selected_rows
|
| 331 |
+
accepted_before = len(accepted)
|
| 332 |
+
for row in rows:
|
| 333 |
+
row_parent_ids = {
|
| 334 |
+
str(parent.get("id") or "") for parent in row.get("parents") or []
|
| 335 |
+
}
|
| 336 |
+
selected_matches = row_parent_ids & selected_parent_ids
|
| 337 |
+
if selected_parent_ids and not selected_matches:
|
| 338 |
+
rejected["outside_parent_selection"] += 1
|
| 339 |
+
continue
|
| 340 |
+
observed_selected_parent_ids.update(selected_matches)
|
| 341 |
+
row_parent_kinds = {
|
| 342 |
+
str(parent.get("kind") or "") for parent in row.get("parents") or []
|
| 343 |
+
}
|
| 344 |
+
excluded = sorted(row_parent_kinds & excluded_parent_kinds)
|
| 345 |
+
if excluded:
|
| 346 |
+
for kind in excluded:
|
| 347 |
+
rejected[f"excluded_parent_kind:{kind}"] += 1
|
| 348 |
+
continue
|
| 349 |
+
identity = _semantic_identity(row, inventory)
|
| 350 |
+
cited = _cited_parents(row)
|
| 351 |
+
conflicts = {
|
| 352 |
+
seen_cited_parent[parent]
|
| 353 |
+
for parent in cited
|
| 354 |
+
if parent in seen_cited_parent
|
| 355 |
+
}
|
| 356 |
+
if conflicts and conflicts != {identity}:
|
| 357 |
+
raise SourceAlignedSFTError(
|
| 358 |
+
"one cited source parent maps to conflicting semantic rows"
|
| 359 |
+
)
|
| 360 |
+
retain_repeated_rows = ordered_curriculum or retain_semantic_exposures
|
| 361 |
+
if identity in seen_identity and not retain_repeated_rows:
|
| 362 |
+
rejected["duplicate_semantic_identity"] += 1
|
| 363 |
+
for parent in cited:
|
| 364 |
+
seen_cited_parent.setdefault(parent, identity)
|
| 365 |
+
continue
|
| 366 |
+
if conflicts and not retain_repeated_rows:
|
| 367 |
+
rejected["duplicate_cited_source_parent"] += 1
|
| 368 |
+
continue
|
| 369 |
+
seen_identity.add(identity)
|
| 370 |
+
for parent in cited:
|
| 371 |
+
seen_cited_parent[parent] = identity
|
| 372 |
+
accepted.append((row, root, inventory, identity))
|
| 373 |
+
record = (
|
| 374 |
+
source_record
|
| 375 |
+
if source_kind == "legacy_repcount"
|
| 376 |
+
else {
|
| 377 |
+
"accepted_rows": len(accepted) - accepted_before,
|
| 378 |
+
"manifest_sha256": file_sha256(root / "manifest.json"),
|
| 379 |
+
"release_id": manifest["release_id"],
|
| 380 |
+
"root": str(root),
|
| 381 |
+
"source_rows": len(rows),
|
| 382 |
+
"teacher_used_for_labels": teacher_used_for_labels,
|
| 383 |
+
"trainer_sha256": file_sha256(root / "trainer/sft.jsonl"),
|
| 384 |
+
}
|
| 385 |
+
)
|
| 386 |
+
record["accepted_rows"] = len(accepted) - accepted_before
|
| 387 |
+
source_records.append(record)
|
| 388 |
+
|
| 389 |
+
if selected_parent_ids:
|
| 390 |
+
missing_selected_parent_ids = selected_parent_ids - observed_selected_parent_ids
|
| 391 |
+
if missing_selected_parent_ids:
|
| 392 |
+
raise SourceAlignedSFTError(
|
| 393 |
+
f"{len(missing_selected_parent_ids)} selected parent IDs were absent"
|
| 394 |
+
)
|
| 395 |
+
|
| 396 |
+
within_stage_shuffle_seed = int(
|
| 397 |
+
getattr(args, "shuffle_within_stage_seed", -1)
|
| 398 |
+
)
|
| 399 |
+
if within_stage_shuffle_seed >= 0:
|
| 400 |
+
shuffled: list[
|
| 401 |
+
tuple[dict[str, Any], Path, dict[str, dict[str, Any]], str]
|
| 402 |
+
] = []
|
| 403 |
+
cursor = 0
|
| 404 |
+
for stage_index, record in enumerate(source_records):
|
| 405 |
+
count = int(record["accepted_rows"])
|
| 406 |
+
stage = accepted[cursor : cursor + count]
|
| 407 |
+
random.Random(within_stage_shuffle_seed + stage_index).shuffle(stage)
|
| 408 |
+
shuffled.extend(stage)
|
| 409 |
+
cursor += count
|
| 410 |
+
if cursor != len(accepted):
|
| 411 |
+
raise SourceAlignedSFTError("curriculum stage row accounting diverged")
|
| 412 |
+
accepted = shuffled
|
| 413 |
+
|
| 414 |
+
assert expected_lineage is not None
|
| 415 |
+
output.parent.mkdir(parents=True, exist_ok=True)
|
| 416 |
+
scratch = Path(tempfile.mkdtemp(prefix=f".{output.name}.partial.", dir=output.parent))
|
| 417 |
+
try:
|
| 418 |
+
output_inventory: dict[str, dict[str, Any]] = {}
|
| 419 |
+
media_sources: dict[str, Path] = {}
|
| 420 |
+
output_rows = []
|
| 421 |
+
by_source: collections.Counter[str] = collections.Counter()
|
| 422 |
+
weights_before: collections.Counter[float] = collections.Counter()
|
| 423 |
+
weights_after: collections.Counter[float] = collections.Counter()
|
| 424 |
+
reweighted_rows = 0
|
| 425 |
+
source_stage = {
|
| 426 |
+
item["root"]: index for index, item in enumerate(source_records)
|
| 427 |
+
}
|
| 428 |
+
source_row_index: collections.Counter[str] = collections.Counter()
|
| 429 |
+
repeated_semantic_exposures = len(accepted) - len(seen_identity)
|
| 430 |
+
stage_cursor = 0
|
| 431 |
+
curriculum_stages = []
|
| 432 |
+
for stage_index, record in enumerate(source_records):
|
| 433 |
+
stage_rows = int(record["accepted_rows"])
|
| 434 |
+
curriculum_stages.append(
|
| 435 |
+
{
|
| 436 |
+
"end_row_exclusive": stage_cursor + stage_rows,
|
| 437 |
+
"index": stage_index,
|
| 438 |
+
"release_id": record["release_id"],
|
| 439 |
+
"rows": stage_rows,
|
| 440 |
+
"start_row": stage_cursor,
|
| 441 |
+
}
|
| 442 |
+
)
|
| 443 |
+
stage_cursor += stage_rows
|
| 444 |
+
if stage_cursor != len(accepted):
|
| 445 |
+
raise SourceAlignedSFTError("curriculum manifest row accounting diverged")
|
| 446 |
+
for row, root, inventory, identity in accepted:
|
| 447 |
+
messages = copy.deepcopy(row["messages"])
|
| 448 |
+
for message in messages:
|
| 449 |
+
for item in message.get("content") or []:
|
| 450 |
+
if item.get("type") != "image":
|
| 451 |
+
continue
|
| 452 |
+
source_relative = str(item["image"])
|
| 453 |
+
declaration = inventory[source_relative]
|
| 454 |
+
digest = str(declaration["sha256"])
|
| 455 |
+
canonical_relative = _canonical_media_path(source_relative, digest)
|
| 456 |
+
previous = output_inventory.get(canonical_relative)
|
| 457 |
+
normalized = {
|
| 458 |
+
"bytes": int(declaration["bytes"]),
|
| 459 |
+
"path": canonical_relative,
|
| 460 |
+
"sha256": digest,
|
| 461 |
+
}
|
| 462 |
+
if previous is not None and previous != normalized:
|
| 463 |
+
raise SourceAlignedSFTError("canonical media declarations conflict")
|
| 464 |
+
output_inventory.setdefault(canonical_relative, normalized)
|
| 465 |
+
media_sources.setdefault(canonical_relative, root / source_relative)
|
| 466 |
+
item["image"] = canonical_relative
|
| 467 |
+
source_weight = float(row.get("sample_loss_weight", 1.0))
|
| 468 |
+
output_weight = (
|
| 469 |
+
min(source_weight, sample_loss_weight_cap)
|
| 470 |
+
if sample_loss_weight_cap > 0
|
| 471 |
+
else source_weight
|
| 472 |
+
)
|
| 473 |
+
weights_before[source_weight] += 1
|
| 474 |
+
weights_after[output_weight] += 1
|
| 475 |
+
reweighted_rows += int(output_weight != source_weight)
|
| 476 |
+
stage_index = source_stage[str(root)]
|
| 477 |
+
row_index = source_row_index[str(root)]
|
| 478 |
+
source_row_index[str(root)] += 1
|
| 479 |
+
uid_parts: tuple[Any, ...] = (identity,)
|
| 480 |
+
if ordered_curriculum or retain_semantic_exposures:
|
| 481 |
+
uid_parts = (stage_index, row_index, identity)
|
| 482 |
+
clean = attach_content_sha256(
|
| 483 |
+
{
|
| 484 |
+
"schema": "streameva.sft_row.v1",
|
| 485 |
+
"uid": stable_id(
|
| 486 |
+
"sourcealignedsft", *uid_parts, digest_length=24
|
| 487 |
+
),
|
| 488 |
+
"component": "preservation",
|
| 489 |
+
"group_id": stable_id(
|
| 490 |
+
"sourcealignedsftgroup", *uid_parts, digest_length=24
|
| 491 |
+
),
|
| 492 |
+
"split": "train",
|
| 493 |
+
"messages": messages,
|
| 494 |
+
"sample_loss_weight": output_weight,
|
| 495 |
+
"parents": copy.deepcopy(row.get("parents") or []),
|
| 496 |
+
}
|
| 497 |
+
)
|
| 498 |
+
validate_row(clean)
|
| 499 |
+
output_rows.append(clean)
|
| 500 |
+
source_release = next(
|
| 501 |
+
item["release_id"]
|
| 502 |
+
for item in source_records
|
| 503 |
+
if item["root"] == str(root)
|
| 504 |
+
)
|
| 505 |
+
by_source[source_release] += 1
|
| 506 |
+
|
| 507 |
+
if not ordered_curriculum:
|
| 508 |
+
output_rows.sort(key=lambda row: str(row["uid"]))
|
| 509 |
+
if len({str(row["uid"]) for row in output_rows}) != len(output_rows):
|
| 510 |
+
raise SourceAlignedSFTError("output UIDs collide")
|
| 511 |
+
write_jsonl(output_rows, scratch / "trainer/sft.jsonl")
|
| 512 |
+
inventory_rows = [output_inventory[path] for path in sorted(output_inventory)]
|
| 513 |
+
write_jsonl(inventory_rows, scratch / "manifests/media_inventory.jsonl")
|
| 514 |
+
for item in inventory_rows:
|
| 515 |
+
relative = str(item["path"])
|
| 516 |
+
destination = scratch / relative
|
| 517 |
+
_materialize(media_sources[relative], destination, mode=args.copy_mode)
|
| 518 |
+
if (
|
| 519 |
+
destination.stat().st_size != int(item["bytes"])
|
| 520 |
+
or file_sha256(destination) != item["sha256"]
|
| 521 |
+
):
|
| 522 |
+
raise SourceAlignedSFTError("materialized media hash diverged")
|
| 523 |
+
|
| 524 |
+
answers = collections.Counter(_target(row) for row in output_rows)
|
| 525 |
+
frames = collections.Counter(str(len(_images(row))) for row in output_rows)
|
| 526 |
+
manifest = {
|
| 527 |
+
"artifacts": {
|
| 528 |
+
"manifests/media_inventory.jsonl": file_sha256(
|
| 529 |
+
scratch / "manifests/media_inventory.jsonl"
|
| 530 |
+
),
|
| 531 |
+
"trainer/sft.jsonl": file_sha256(scratch / "trainer/sft.jsonl"),
|
| 532 |
+
},
|
| 533 |
+
"contracts": {
|
| 534 |
+
"agent_action_targets": 0,
|
| 535 |
+
"assistant_targets_plain_semantic_only": True,
|
| 536 |
+
"benchmark_examples_admitted": 0,
|
| 537 |
+
"benchmark_firewall_inherited": True,
|
| 538 |
+
"counterfactual_rows_admitted": 0,
|
| 539 |
+
"environment_state_actor_targets": 0,
|
| 540 |
+
"legacy_stage_metadata_admitted": 0,
|
| 541 |
+
"loss_on_assistant_only": True,
|
| 542 |
+
"policy_holdout_admitted": 0,
|
| 543 |
+
"private_verifier_actor_targets": 0,
|
| 544 |
+
"all_sample_loss_weights_one": set(weights_after) == {1.0},
|
| 545 |
+
"sample_loss_weight_cap": (
|
| 546 |
+
sample_loss_weight_cap if sample_loss_weight_cap > 0 else None
|
| 547 |
+
),
|
| 548 |
+
"sample_loss_weight_rows_changed": reweighted_rows,
|
| 549 |
+
"self_contained_media": True,
|
| 550 |
+
"teacher_used_for_labels": any(
|
| 551 |
+
bool(item["teacher_used_for_labels"])
|
| 552 |
+
for item in source_records
|
| 553 |
+
),
|
| 554 |
+
"training_runs_required": 1,
|
| 555 |
+
"ordered_curriculum": ordered_curriculum,
|
| 556 |
+
"retain_semantic_exposures": retain_semantic_exposures,
|
| 557 |
+
"repeated_semantic_exposures": repeated_semantic_exposures,
|
| 558 |
+
"shuffle_within_stage_seed": (
|
| 559 |
+
within_stage_shuffle_seed
|
| 560 |
+
if within_stage_shuffle_seed >= 0
|
| 561 |
+
else None
|
| 562 |
+
),
|
| 563 |
+
},
|
| 564 |
+
"counts": {
|
| 565 |
+
"media_bytes": sum(int(item["bytes"]) for item in inventory_rows),
|
| 566 |
+
"media_files": len(inventory_rows),
|
| 567 |
+
"trainer_rows": len(output_rows),
|
| 568 |
+
},
|
| 569 |
+
"distributions": {
|
| 570 |
+
"answer": dict(sorted(answers.items())),
|
| 571 |
+
"frames_per_row": dict(sorted(frames.items())),
|
| 572 |
+
"sample_loss_weight": {
|
| 573 |
+
str(weight): count
|
| 574 |
+
for weight, count in sorted(weights_after.items())
|
| 575 |
+
},
|
| 576 |
+
"source_release": dict(sorted(by_source.items())),
|
| 577 |
+
},
|
| 578 |
+
"format": FORMAT,
|
| 579 |
+
"inputs": {
|
| 580 |
+
"benchmark_rows_or_labels_admitted": 0,
|
| 581 |
+
"builder_sha256": file_sha256(Path(__file__).resolve(strict=True)),
|
| 582 |
+
"deduplication": {
|
| 583 |
+
"cited_source_parent": True,
|
| 584 |
+
"exact_pixel_question_answer_identity": True,
|
| 585 |
+
"retain_semantic_exposures": retain_semantic_exposures,
|
| 586 |
+
"rejected": dict(sorted(rejected.items())),
|
| 587 |
+
},
|
| 588 |
+
"excluded_parent_kinds": sorted(excluded_parent_kinds),
|
| 589 |
+
"included_source_stage_release_ids": sorted(
|
| 590 |
+
included_source_stage_release_ids
|
| 591 |
+
),
|
| 592 |
+
"parent_selection": parent_selection_record,
|
| 593 |
+
"final_release_must_restart_from": "Qwen/Qwen3-VL-8B-Instruct",
|
| 594 |
+
"rows_relabelled": 0,
|
| 595 |
+
"rows_reweighted": reweighted_rows,
|
| 596 |
+
"sample_loss_weight_policy": {
|
| 597 |
+
"cap": (
|
| 598 |
+
sample_loss_weight_cap
|
| 599 |
+
if sample_loss_weight_cap > 0
|
| 600 |
+
else None
|
| 601 |
+
),
|
| 602 |
+
"source_distribution": {
|
| 603 |
+
str(weight): count
|
| 604 |
+
for weight, count in sorted(weights_before.items())
|
| 605 |
+
},
|
| 606 |
+
"output_distribution": {
|
| 607 |
+
str(weight): count
|
| 608 |
+
for weight, count in sorted(weights_after.items())
|
| 609 |
+
},
|
| 610 |
+
},
|
| 611 |
+
"sources": source_records,
|
| 612 |
+
"curriculum_stages": curriculum_stages,
|
| 613 |
+
"trainer_row_order": (
|
| 614 |
+
"source_stage_then_source_row"
|
| 615 |
+
if ordered_curriculum
|
| 616 |
+
else "canonical_uid"
|
| 617 |
+
),
|
| 618 |
+
},
|
| 619 |
+
"license": {
|
| 620 |
+
"public_media_redistribution_claimed": False,
|
| 621 |
+
"release_plan": "code_recipe_selection_and_source_checksums",
|
| 622 |
+
"scope": "internal_noncommercial_research_only",
|
| 623 |
+
},
|
| 624 |
+
"model_lineage": {
|
| 625 |
+
"required_initialization": "Qwen/Qwen3-VL-8B-Instruct",
|
| 626 |
+
"required_sha256s_sha256": expected_lineage[0],
|
| 627 |
+
"upstream_repo": expected_lineage[1],
|
| 628 |
+
"upstream_revision": expected_lineage[2],
|
| 629 |
+
},
|
| 630 |
+
"public_release_allowed": False,
|
| 631 |
+
"release_id": args.release_id,
|
| 632 |
+
"status": "sealed_internal_training_release",
|
| 633 |
+
"training_allowed": True,
|
| 634 |
+
"training_scope": "pure_vlm_semantic_sft_only",
|
| 635 |
+
}
|
| 636 |
+
(scratch / "manifest.json").write_bytes(canonical_json_line(manifest))
|
| 637 |
+
validate_semantic_release(scratch, verify_media_hashes=True)
|
| 638 |
+
os.replace(scratch, output)
|
| 639 |
+
return manifest | {"output": str(output)}
|
| 640 |
+
except BaseException:
|
| 641 |
+
shutil.rmtree(scratch, ignore_errors=True)
|
| 642 |
+
raise
|
| 643 |
+
|
| 644 |
+
|
| 645 |
+
def main() -> None:
|
| 646 |
+
parser = argparse.ArgumentParser(description=__doc__)
|
| 647 |
+
parser.add_argument("--source", type=Path, action="append", required=True)
|
| 648 |
+
parser.add_argument("--output", type=Path, required=True)
|
| 649 |
+
parser.add_argument("--release-id", required=True)
|
| 650 |
+
parser.add_argument(
|
| 651 |
+
"--copy-mode", choices=("copy", "hardlink", "auto"), default="hardlink"
|
| 652 |
+
)
|
| 653 |
+
parser.add_argument(
|
| 654 |
+
"--sample-loss-weight-cap",
|
| 655 |
+
type=float,
|
| 656 |
+
default=0.0,
|
| 657 |
+
help="Cap inherited sample weights; zero preserves source weights.",
|
| 658 |
+
)
|
| 659 |
+
parser.add_argument(
|
| 660 |
+
"--exclude-parent-kind",
|
| 661 |
+
action="append",
|
| 662 |
+
default=[],
|
| 663 |
+
help="Exclude rows carrying this exact parent kind; may be repeated.",
|
| 664 |
+
)
|
| 665 |
+
parser.add_argument(
|
| 666 |
+
"--include-source-stage-release-id",
|
| 667 |
+
action="append",
|
| 668 |
+
default=[],
|
| 669 |
+
help=(
|
| 670 |
+
"For a composite semantic source, admit only rows inside the "
|
| 671 |
+
"named hash-bound curriculum stages; may be repeated."
|
| 672 |
+
),
|
| 673 |
+
)
|
| 674 |
+
parser.add_argument(
|
| 675 |
+
"--include-parent-selection",
|
| 676 |
+
type=Path,
|
| 677 |
+
help=(
|
| 678 |
+
"Hash-bind a capability selection and admit only semantic rows "
|
| 679 |
+
"whose cited parent ID is listed in it."
|
| 680 |
+
),
|
| 681 |
+
)
|
| 682 |
+
parser.add_argument(
|
| 683 |
+
"--ordered-curriculum",
|
| 684 |
+
action="store_true",
|
| 685 |
+
help="Preserve source-stage order and retain repeated stage exposures.",
|
| 686 |
+
)
|
| 687 |
+
parser.add_argument(
|
| 688 |
+
"--retain-semantic-exposures",
|
| 689 |
+
action="store_true",
|
| 690 |
+
help=(
|
| 691 |
+
"Retain every source row as an intentional training exposure while "
|
| 692 |
+
"keeping the normal globally canonicalized row order."
|
| 693 |
+
),
|
| 694 |
+
)
|
| 695 |
+
parser.add_argument(
|
| 696 |
+
"--legacy-v62di-source",
|
| 697 |
+
type=Path,
|
| 698 |
+
help="Pinned legacy release; admits only its 2,320 V62DI rows.",
|
| 699 |
+
)
|
| 700 |
+
parser.add_argument(
|
| 701 |
+
"--legacy-v62di-mode",
|
| 702 |
+
choices=("v62di_only", "full"),
|
| 703 |
+
default="v62di_only",
|
| 704 |
+
help="Use only V62DI rows or the complete committed 3,594-row pool.",
|
| 705 |
+
)
|
| 706 |
+
parser.add_argument(
|
| 707 |
+
"--legacy-repcount-source",
|
| 708 |
+
type=Path,
|
| 709 |
+
help="Pinned 2,172-row legacy RepCount source, inserted after source 1.",
|
| 710 |
+
)
|
| 711 |
+
parser.add_argument(
|
| 712 |
+
"--shuffle-within-stage-seed",
|
| 713 |
+
type=int,
|
| 714 |
+
default=-1,
|
| 715 |
+
help="Deterministically shuffle each stage while preserving stage order.",
|
| 716 |
+
)
|
| 717 |
+
args = parser.parse_args()
|
| 718 |
+
print(json.dumps(build(args), indent=2, sort_keys=True))
|
| 719 |
+
|
| 720 |
+
|
| 721 |
+
if __name__ == "__main__":
|
| 722 |
+
main()
|
code/data_pipeline/temporal_scope/build_vst_prefix32_semantic.py
ADDED
|
@@ -0,0 +1,525 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Build a source-disjoint, pixel-grounded VST prefix-32 semantic pack.
|
| 3 |
+
|
| 4 |
+
The upstream VST metadata provides cumulative count states. A recent-only
|
| 5 |
+
window cannot in general support those labels, so this compiler exposes a
|
| 6 |
+
uniform causal prefix ending strictly before the query time. It emits only
|
| 7 |
+
plain semantic answers (open integer and balanced MCQ); no agent action or
|
| 8 |
+
private state is ever an actor target.
|
| 9 |
+
"""
|
| 10 |
+
|
| 11 |
+
from __future__ import annotations
|
| 12 |
+
|
| 13 |
+
import argparse
|
| 14 |
+
import collections
|
| 15 |
+
import concurrent.futures
|
| 16 |
+
import hashlib
|
| 17 |
+
import io
|
| 18 |
+
import json
|
| 19 |
+
import math
|
| 20 |
+
import os
|
| 21 |
+
import shutil
|
| 22 |
+
import tempfile
|
| 23 |
+
from collections.abc import Mapping, Sequence
|
| 24 |
+
from pathlib import Path
|
| 25 |
+
from typing import Any
|
| 26 |
+
|
| 27 |
+
from PIL import Image
|
| 28 |
+
|
| 29 |
+
from data_pipeline.temporal_scope.semantic_mcq import (
|
| 30 |
+
DIRECT_PROMPT_PREFIX,
|
| 31 |
+
DIRECT_PROMPT_SUFFIX,
|
| 32 |
+
rebalance_answer_slot,
|
| 33 |
+
)
|
| 34 |
+
from viss_core.data_v0.canonical import (
|
| 35 |
+
attach_content_sha256,
|
| 36 |
+
canonical_json_line,
|
| 37 |
+
file_sha256,
|
| 38 |
+
stable_id,
|
| 39 |
+
write_jsonl,
|
| 40 |
+
)
|
| 41 |
+
from viss_core.data_v0.schema import validate_row
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
FORMAT = "streameva.vst_prefix32_semantic.v1"
|
| 45 |
+
PREFLIGHT_FORMAT = "streameva.vst_count_metadata_preflight.v1"
|
| 46 |
+
OPTION_LETTERS = "ABCD"
|
| 47 |
+
TARGET_COUNTS = tuple(range(11))
|
| 48 |
+
PREFIX_FRAME_MAX_PIXELS = 128 * 28 * 28
|
| 49 |
+
PREFIX_FRAME_MAX_SIDE = 504
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
class VSTPrefixSemanticError(ValueError):
|
| 53 |
+
"""Raised when a prefix-32 semantic candidate violates its contract."""
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
def main() -> None:
|
| 57 |
+
parser = argparse.ArgumentParser(description=__doc__)
|
| 58 |
+
parser.add_argument("--selection", type=Path, required=True)
|
| 59 |
+
parser.add_argument("--preflight", type=Path, required=True)
|
| 60 |
+
parser.add_argument("--media-root", type=Path, required=True)
|
| 61 |
+
parser.add_argument("--output", type=Path, required=True)
|
| 62 |
+
parser.add_argument("--frames", type=int, default=32)
|
| 63 |
+
parser.add_argument("--train-per-count", type=int, default=92)
|
| 64 |
+
parser.add_argument("--holdout-per-count", type=int, default=5)
|
| 65 |
+
parser.add_argument("--workers", type=int, default=16)
|
| 66 |
+
parser.add_argument("--seed", type=int, default=173)
|
| 67 |
+
args = parser.parse_args()
|
| 68 |
+
print(json.dumps(materialize(args), indent=2, sort_keys=True))
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
def select_available_states(
|
| 72 |
+
rows: Sequence[Mapping[str, Any]],
|
| 73 |
+
*,
|
| 74 |
+
media_root: Path,
|
| 75 |
+
train_per_count: int,
|
| 76 |
+
holdout_per_count: int,
|
| 77 |
+
seed: int,
|
| 78 |
+
) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
|
| 79 |
+
"""Select balanced, source-disjoint train and holdout states."""
|
| 80 |
+
|
| 81 |
+
if train_per_count <= 0 or holdout_per_count <= 0:
|
| 82 |
+
raise VSTPrefixSemanticError("train/holdout quotas must be positive")
|
| 83 |
+
by_count: dict[int, list[dict[str, Any]]] = {
|
| 84 |
+
value: [] for value in TARGET_COUNTS
|
| 85 |
+
}
|
| 86 |
+
seen_videos: set[str] = set()
|
| 87 |
+
seen_ids: set[str] = set()
|
| 88 |
+
for raw in rows:
|
| 89 |
+
row = dict(raw)
|
| 90 |
+
answer = int(row.get("answer", -1))
|
| 91 |
+
relative = Path(str(row.get("source_video_relative") or ""))
|
| 92 |
+
candidate_id = str(row.get("candidate_id") or "")
|
| 93 |
+
if (
|
| 94 |
+
answer not in by_count
|
| 95 |
+
or not candidate_id
|
| 96 |
+
or relative.is_absolute()
|
| 97 |
+
or ".." in relative.parts
|
| 98 |
+
or relative.suffix.casefold() != ".mp4"
|
| 99 |
+
):
|
| 100 |
+
raise VSTPrefixSemanticError("selection row is malformed")
|
| 101 |
+
if candidate_id in seen_ids or relative.as_posix() in seen_videos:
|
| 102 |
+
raise VSTPrefixSemanticError("selection IDs/videos must be unique")
|
| 103 |
+
seen_ids.add(candidate_id)
|
| 104 |
+
seen_videos.add(relative.as_posix())
|
| 105 |
+
source = (media_root / relative).resolve()
|
| 106 |
+
if not source.is_file() or source.is_symlink():
|
| 107 |
+
continue
|
| 108 |
+
row["source_video"] = str(source)
|
| 109 |
+
by_count[answer].append(row)
|
| 110 |
+
|
| 111 |
+
train: list[dict[str, Any]] = []
|
| 112 |
+
holdout: list[dict[str, Any]] = []
|
| 113 |
+
required = train_per_count + holdout_per_count
|
| 114 |
+
for answer in TARGET_COUNTS:
|
| 115 |
+
values = by_count[answer]
|
| 116 |
+
values.sort(
|
| 117 |
+
key=lambda row: hashlib.sha256(
|
| 118 |
+
f"{seed}|{answer}|{row['candidate_id']}".encode()
|
| 119 |
+
).digest()
|
| 120 |
+
)
|
| 121 |
+
if len(values) < required:
|
| 122 |
+
raise VSTPrefixSemanticError(
|
| 123 |
+
f"count {answer} has {len(values)} available videos; need {required}"
|
| 124 |
+
)
|
| 125 |
+
for row in values[:holdout_per_count]:
|
| 126 |
+
value = dict(row)
|
| 127 |
+
value["split"] = "policy_holdout"
|
| 128 |
+
holdout.append(value)
|
| 129 |
+
for row in values[holdout_per_count:required]:
|
| 130 |
+
value = dict(row)
|
| 131 |
+
value["split"] = "train"
|
| 132 |
+
train.append(value)
|
| 133 |
+
return (
|
| 134 |
+
sorted(train, key=lambda row: str(row["candidate_id"])),
|
| 135 |
+
sorted(holdout, key=lambda row: str(row["candidate_id"])),
|
| 136 |
+
)
|
| 137 |
+
|
| 138 |
+
|
| 139 |
+
def uniform_causal_indices(
|
| 140 |
+
*, source_total: int, source_fps: float, query_time: float, frames: int
|
| 141 |
+
) -> tuple[list[int], list[float]]:
|
| 142 |
+
"""Return exactly ``frames`` uniform indices whose timestamps are causal."""
|
| 143 |
+
|
| 144 |
+
if (
|
| 145 |
+
source_total <= 0
|
| 146 |
+
or not math.isfinite(source_fps)
|
| 147 |
+
or source_fps <= 0
|
| 148 |
+
or not math.isfinite(query_time)
|
| 149 |
+
or query_time <= 0
|
| 150 |
+
or frames <= 1
|
| 151 |
+
):
|
| 152 |
+
raise VSTPrefixSemanticError("invalid video sampling metadata")
|
| 153 |
+
# ceil(t * fps) - 1 is strictly earlier than t, including exact boundaries.
|
| 154 |
+
last = min(source_total - 1, max(0, math.ceil(query_time * source_fps) - 1))
|
| 155 |
+
if last == 0:
|
| 156 |
+
indices = [0] * frames
|
| 157 |
+
else:
|
| 158 |
+
indices = [
|
| 159 |
+
int(round(position * last / (frames - 1))) for position in range(frames)
|
| 160 |
+
]
|
| 161 |
+
timestamps = [index / source_fps for index in indices]
|
| 162 |
+
if any(timestamp >= query_time + 1e-9 for timestamp in timestamps):
|
| 163 |
+
raise VSTPrefixSemanticError("prefix sampler exposed a future frame")
|
| 164 |
+
return indices, timestamps
|
| 165 |
+
|
| 166 |
+
|
| 167 |
+
def project_semantic_rows(
|
| 168 |
+
row: Mapping[str, Any],
|
| 169 |
+
*,
|
| 170 |
+
relative_paths: Sequence[str],
|
| 171 |
+
frame_timestamps: Sequence[float],
|
| 172 |
+
source_sha256: str,
|
| 173 |
+
selection_sha256: str,
|
| 174 |
+
) -> list[dict[str, Any]]:
|
| 175 |
+
"""Project one grounded count state into open and balanced-MCQ rows."""
|
| 176 |
+
|
| 177 |
+
if len(relative_paths) != 32 or len(frame_timestamps) != 32:
|
| 178 |
+
raise VSTPrefixSemanticError("prefix projection requires exactly 32 frames")
|
| 179 |
+
answer = int(row["answer"])
|
| 180 |
+
if answer not in TARGET_COUNTS:
|
| 181 |
+
raise VSTPrefixSemanticError("count answer is outside 0--10")
|
| 182 |
+
candidate_id = str(row["candidate_id"])
|
| 183 |
+
split = str(row["split"])
|
| 184 |
+
if split not in {"train", "policy_holdout"}:
|
| 185 |
+
raise VSTPrefixSemanticError("invalid semantic split")
|
| 186 |
+
images = [
|
| 187 |
+
{
|
| 188 |
+
"type": "image",
|
| 189 |
+
"image": path,
|
| 190 |
+
"frame_id": stable_id(
|
| 191 |
+
"vstprefixframe", candidate_id, index, timestamp, digest_length=20
|
| 192 |
+
),
|
| 193 |
+
}
|
| 194 |
+
for index, (path, timestamp) in enumerate(
|
| 195 |
+
zip(relative_paths, frame_timestamps, strict=True)
|
| 196 |
+
)
|
| 197 |
+
]
|
| 198 |
+
parent = {
|
| 199 |
+
"kind": "vst_count_metadata_state",
|
| 200 |
+
"id": stable_id("vstprefixsource", candidate_id, digest_length=20),
|
| 201 |
+
"sha256": selection_sha256,
|
| 202 |
+
}
|
| 203 |
+
group_id = stable_id("vstprefixgroup", candidate_id, digest_length=20)
|
| 204 |
+
question = " ".join(str(row["question"]).split())
|
| 205 |
+
answer_text = str(answer)
|
| 206 |
+
open_row = attach_content_sha256(
|
| 207 |
+
{
|
| 208 |
+
"schema": "streameva.sft_row.v1",
|
| 209 |
+
"uid": stable_id("sftvstprefixopen", candidate_id, digest_length=20),
|
| 210 |
+
"component": "preservation",
|
| 211 |
+
"group_id": group_id,
|
| 212 |
+
"split": split,
|
| 213 |
+
"decision_loss_profile": "full",
|
| 214 |
+
"messages": [
|
| 215 |
+
{
|
| 216 |
+
"role": "user",
|
| 217 |
+
"content": [
|
| 218 |
+
*images,
|
| 219 |
+
{
|
| 220 |
+
"type": "text",
|
| 221 |
+
"text": (
|
| 222 |
+
f"{question}\nAnswer with one integer from 0 to 10 "
|
| 223 |
+
"and no other text."
|
| 224 |
+
),
|
| 225 |
+
},
|
| 226 |
+
],
|
| 227 |
+
"loss": False,
|
| 228 |
+
},
|
| 229 |
+
{
|
| 230 |
+
"role": "assistant",
|
| 231 |
+
"content": [{"type": "text", "text": answer_text}],
|
| 232 |
+
"loss": True,
|
| 233 |
+
},
|
| 234 |
+
],
|
| 235 |
+
"sample_loss_weight": 1.0,
|
| 236 |
+
"parents": [parent],
|
| 237 |
+
}
|
| 238 |
+
)
|
| 239 |
+
|
| 240 |
+
distractors = [value for value in TARGET_COUNTS if value != answer]
|
| 241 |
+
distractors.sort(
|
| 242 |
+
key=lambda value: (
|
| 243 |
+
abs(value - answer),
|
| 244 |
+
hashlib.sha256(f"{candidate_id}|{value}".encode()).digest(),
|
| 245 |
+
)
|
| 246 |
+
)
|
| 247 |
+
options, mcq_answer = rebalance_answer_slot(
|
| 248 |
+
[answer_text, *[str(value) for value in distractors[:3]]],
|
| 249 |
+
answer="A",
|
| 250 |
+
answer_slot=int(hashlib.sha256(candidate_id.encode()).hexdigest(), 16) % 4,
|
| 251 |
+
seed=f"{candidate_id}|prefix32-mcq-v1",
|
| 252 |
+
)
|
| 253 |
+
option_text = "\n".join(
|
| 254 |
+
f"{letter}. {value}"
|
| 255 |
+
for letter, value in zip(OPTION_LETTERS, options, strict=True)
|
| 256 |
+
)
|
| 257 |
+
mcq_prompt = (
|
| 258 |
+
f"{DIRECT_PROMPT_PREFIX}\n\nQuestion: {question}\n\n"
|
| 259 |
+
f"Options:\n{option_text}\n\n{DIRECT_PROMPT_SUFFIX}"
|
| 260 |
+
)
|
| 261 |
+
mcq_row = attach_content_sha256(
|
| 262 |
+
{
|
| 263 |
+
"schema": "streameva.sft_row.v1",
|
| 264 |
+
"uid": stable_id("sftvstprefixmcq", candidate_id, digest_length=20),
|
| 265 |
+
"component": "preservation",
|
| 266 |
+
"group_id": group_id,
|
| 267 |
+
"split": split,
|
| 268 |
+
"decision_loss_profile": "full",
|
| 269 |
+
"messages": [
|
| 270 |
+
{
|
| 271 |
+
"role": "user",
|
| 272 |
+
"content": [*images, {"type": "text", "text": mcq_prompt}],
|
| 273 |
+
"loss": False,
|
| 274 |
+
},
|
| 275 |
+
{
|
| 276 |
+
"role": "assistant",
|
| 277 |
+
"content": [{"type": "text", "text": mcq_answer}],
|
| 278 |
+
"loss": True,
|
| 279 |
+
},
|
| 280 |
+
],
|
| 281 |
+
"sample_loss_weight": 1.0,
|
| 282 |
+
"parents": [parent],
|
| 283 |
+
}
|
| 284 |
+
)
|
| 285 |
+
for output in (open_row, mcq_row):
|
| 286 |
+
validate_row(output)
|
| 287 |
+
# Bind source pixels in the manifest-facing auxiliary state without leaking
|
| 288 |
+
# an absolute path into the strict trainer row.
|
| 289 |
+
del source_sha256
|
| 290 |
+
return [open_row, mcq_row]
|
| 291 |
+
|
| 292 |
+
|
| 293 |
+
def materialize(args: argparse.Namespace) -> dict[str, Any]:
|
| 294 |
+
selection = args.selection.resolve()
|
| 295 |
+
preflight_path = args.preflight.resolve()
|
| 296 |
+
media_root = args.media_root.resolve()
|
| 297 |
+
output = args.output.resolve()
|
| 298 |
+
if output.exists():
|
| 299 |
+
raise FileExistsError(output)
|
| 300 |
+
if args.frames != 32:
|
| 301 |
+
raise VSTPrefixSemanticError("this release is frozen to exactly 32 frames")
|
| 302 |
+
if not 1 <= args.workers <= 32:
|
| 303 |
+
raise VSTPrefixSemanticError("workers must lie in [1, 32]")
|
| 304 |
+
preflight = json.loads(preflight_path.read_text(encoding="utf-8"))
|
| 305 |
+
declaration = preflight.get("selection_artifact", {})
|
| 306 |
+
if (
|
| 307 |
+
preflight.get("format") != PREFLIGHT_FORMAT
|
| 308 |
+
or preflight.get("status") != "pass_metadata_firewall_and_selection"
|
| 309 |
+
or preflight.get("contracts", {}).get("basename_hits_admitted") != 0
|
| 310 |
+
or preflight.get("contracts", {}).get("question_hits_admitted") != 0
|
| 311 |
+
or declaration.get("sha256") != file_sha256(selection)
|
| 312 |
+
):
|
| 313 |
+
raise VSTPrefixSemanticError("metadata preflight is not valid or hash-bound")
|
| 314 |
+
rows = [
|
| 315 |
+
json.loads(line)
|
| 316 |
+
for line in selection.read_text(encoding="utf-8").splitlines()
|
| 317 |
+
if line.strip()
|
| 318 |
+
]
|
| 319 |
+
train, holdout = select_available_states(
|
| 320 |
+
rows,
|
| 321 |
+
media_root=media_root,
|
| 322 |
+
train_per_count=args.train_per_count,
|
| 323 |
+
holdout_per_count=args.holdout_per_count,
|
| 324 |
+
seed=args.seed,
|
| 325 |
+
)
|
| 326 |
+
selected = [*train, *holdout]
|
| 327 |
+
output.parent.mkdir(parents=True, exist_ok=True)
|
| 328 |
+
scratch = Path(tempfile.mkdtemp(prefix=f".{output.name}.partial.", dir=output.parent))
|
| 329 |
+
try:
|
| 330 |
+
media_out = scratch / "media"
|
| 331 |
+
|
| 332 |
+
def decode(row: Mapping[str, Any]) -> tuple[str, dict[str, Any]]:
|
| 333 |
+
return str(row["candidate_id"]), _decode_prefix(row, media_out, args.frames)
|
| 334 |
+
|
| 335 |
+
decoded: dict[str, dict[str, Any]] = {}
|
| 336 |
+
errors: list[dict[str, str]] = []
|
| 337 |
+
with concurrent.futures.ThreadPoolExecutor(max_workers=args.workers) as pool:
|
| 338 |
+
futures = {pool.submit(decode, row): row for row in selected}
|
| 339 |
+
for future in concurrent.futures.as_completed(futures):
|
| 340 |
+
row = futures[future]
|
| 341 |
+
try:
|
| 342 |
+
candidate_id, state = future.result()
|
| 343 |
+
decoded[candidate_id] = state
|
| 344 |
+
except Exception as exc: # pragma: no cover - remote media failures
|
| 345 |
+
errors.append(
|
| 346 |
+
{
|
| 347 |
+
"candidate_id": str(row["candidate_id"]),
|
| 348 |
+
"reason": f"{type(exc).__name__}: {exc}",
|
| 349 |
+
}
|
| 350 |
+
)
|
| 351 |
+
write_jsonl(errors, scratch / "decode_errors.jsonl")
|
| 352 |
+
if errors or len(decoded) != len(selected):
|
| 353 |
+
raise VSTPrefixSemanticError(f"prefix decode failed for {len(errors)} rows")
|
| 354 |
+
|
| 355 |
+
train_rows: list[dict[str, Any]] = []
|
| 356 |
+
holdout_rows: list[dict[str, Any]] = []
|
| 357 |
+
source_inventory = []
|
| 358 |
+
selection_sha = file_sha256(selection)
|
| 359 |
+
for row in selected:
|
| 360 |
+
state = decoded[str(row["candidate_id"])]
|
| 361 |
+
source_sha = file_sha256(str(row["source_video"]))
|
| 362 |
+
source_inventory.append(
|
| 363 |
+
{
|
| 364 |
+
"candidate_id": row["candidate_id"],
|
| 365 |
+
"source_video_relative": row["source_video_relative"],
|
| 366 |
+
"source_sha256": source_sha,
|
| 367 |
+
"split": row["split"],
|
| 368 |
+
"query_time": row["query_time"],
|
| 369 |
+
"answer": row["answer"],
|
| 370 |
+
"frame_indices": state["frame_indices"],
|
| 371 |
+
"frame_timestamps": state["frame_timestamps"],
|
| 372 |
+
}
|
| 373 |
+
)
|
| 374 |
+
projected = project_semantic_rows(
|
| 375 |
+
row,
|
| 376 |
+
relative_paths=state["relative_paths"],
|
| 377 |
+
frame_timestamps=state["frame_timestamps"],
|
| 378 |
+
source_sha256=source_sha,
|
| 379 |
+
selection_sha256=selection_sha,
|
| 380 |
+
)
|
| 381 |
+
(train_rows if row["split"] == "train" else holdout_rows).extend(projected)
|
| 382 |
+
write_jsonl(train_rows, scratch / "trainer/sft.jsonl", sort_key="uid")
|
| 383 |
+
write_jsonl(
|
| 384 |
+
holdout_rows, scratch / "holdout/policy_holdout.jsonl", sort_key="uid"
|
| 385 |
+
)
|
| 386 |
+
write_jsonl(
|
| 387 |
+
source_inventory,
|
| 388 |
+
scratch / "source_inventory.jsonl",
|
| 389 |
+
sort_key="candidate_id",
|
| 390 |
+
)
|
| 391 |
+
media_files = sorted(path for path in media_out.rglob("*.jpg") if path.is_file())
|
| 392 |
+
manifest = {
|
| 393 |
+
"format": FORMAT,
|
| 394 |
+
"status": "materialized_internal_research_candidate",
|
| 395 |
+
"training_allowed": True,
|
| 396 |
+
"public_release_allowed": False,
|
| 397 |
+
"protocol": {
|
| 398 |
+
"causal_prefix_only": True,
|
| 399 |
+
"frames": 32,
|
| 400 |
+
"per_frame_max_pixels": PREFIX_FRAME_MAX_PIXELS,
|
| 401 |
+
"sampling": "exact_uniform_indices_over_[0,query_time)",
|
| 402 |
+
"visual_input_mode": "independent_image_frames",
|
| 403 |
+
},
|
| 404 |
+
"counts": {
|
| 405 |
+
"train_source_states": len(train),
|
| 406 |
+
"holdout_source_states": len(holdout),
|
| 407 |
+
"train_rows": len(train_rows),
|
| 408 |
+
"holdout_rows": len(holdout_rows),
|
| 409 |
+
"media_files": len(media_files),
|
| 410 |
+
"train_by_answer": dict(
|
| 411 |
+
sorted(collections.Counter(str(row["answer"]) for row in train).items())
|
| 412 |
+
),
|
| 413 |
+
"holdout_by_answer": dict(
|
| 414 |
+
sorted(
|
| 415 |
+
collections.Counter(str(row["answer"]) for row in holdout).items()
|
| 416 |
+
)
|
| 417 |
+
),
|
| 418 |
+
},
|
| 419 |
+
"inputs": {
|
| 420 |
+
"selection_sha256": selection_sha,
|
| 421 |
+
"preflight_sha256": file_sha256(preflight_path),
|
| 422 |
+
"builder_sha256": file_sha256(Path(__file__)),
|
| 423 |
+
},
|
| 424 |
+
"artifacts": {
|
| 425 |
+
"trainer/sft.jsonl": file_sha256(scratch / "trainer/sft.jsonl"),
|
| 426 |
+
"holdout/policy_holdout.jsonl": file_sha256(
|
| 427 |
+
scratch / "holdout/policy_holdout.jsonl"
|
| 428 |
+
),
|
| 429 |
+
"source_inventory.jsonl": file_sha256(
|
| 430 |
+
scratch / "source_inventory.jsonl"
|
| 431 |
+
),
|
| 432 |
+
"decode_errors.jsonl": file_sha256(scratch / "decode_errors.jsonl"),
|
| 433 |
+
},
|
| 434 |
+
"contracts": {
|
| 435 |
+
"all_labels_metadata_derived": True,
|
| 436 |
+
"all_pixels_strictly_before_query": True,
|
| 437 |
+
"benchmark_firewall_inherited": True,
|
| 438 |
+
"holdout_source_disjoint": True,
|
| 439 |
+
"no_agent_actions": True,
|
| 440 |
+
"no_private_state_actor_targets": True,
|
| 441 |
+
"source_videos_globally_unique": True,
|
| 442 |
+
"teacher_used_for_labels": False,
|
| 443 |
+
},
|
| 444 |
+
"license": {
|
| 445 |
+
"public_media_redistribution_claimed": False,
|
| 446 |
+
"release_mode": "recipe_plus_source_checksums",
|
| 447 |
+
},
|
| 448 |
+
}
|
| 449 |
+
(scratch / "manifest.json").write_bytes(canonical_json_line(manifest))
|
| 450 |
+
os.replace(scratch, output)
|
| 451 |
+
return {**manifest, "output": str(output)}
|
| 452 |
+
except Exception:
|
| 453 |
+
shutil.rmtree(scratch, ignore_errors=True)
|
| 454 |
+
raise
|
| 455 |
+
|
| 456 |
+
|
| 457 |
+
def _decode_prefix(
|
| 458 |
+
row: Mapping[str, Any], media_root: Path, frames: int
|
| 459 |
+
) -> dict[str, Any]:
|
| 460 |
+
from decord import VideoReader
|
| 461 |
+
|
| 462 |
+
source = Path(str(row["source_video"]))
|
| 463 |
+
reader = VideoReader(str(source))
|
| 464 |
+
try:
|
| 465 |
+
fps = float(reader.get_avg_fps())
|
| 466 |
+
total = len(reader)
|
| 467 |
+
indices, timestamps = uniform_causal_indices(
|
| 468 |
+
source_total=total,
|
| 469 |
+
source_fps=fps,
|
| 470 |
+
query_time=float(row["query_time"]),
|
| 471 |
+
frames=frames,
|
| 472 |
+
)
|
| 473 |
+
arrays = reader.get_batch(indices).asnumpy()
|
| 474 |
+
finally:
|
| 475 |
+
del reader
|
| 476 |
+
relative_paths = []
|
| 477 |
+
for array in arrays:
|
| 478 |
+
image = Image.fromarray(array)
|
| 479 |
+
height, width = image.height, image.width
|
| 480 |
+
resized_height, resized_width = _rounded_size(height, width)
|
| 481 |
+
if (resized_height, resized_width) != (height, width):
|
| 482 |
+
image = image.resize((resized_width, resized_height), Image.Resampling.LANCZOS)
|
| 483 |
+
buffer = io.BytesIO()
|
| 484 |
+
image.save(buffer, format="JPEG", quality=95, subsampling=0, optimize=False)
|
| 485 |
+
payload = buffer.getvalue()
|
| 486 |
+
digest = hashlib.sha256(payload).hexdigest()
|
| 487 |
+
relative = Path("media") / digest[:2] / f"{digest}.jpg"
|
| 488 |
+
destination = media_root.parent / relative
|
| 489 |
+
destination.parent.mkdir(parents=True, exist_ok=True)
|
| 490 |
+
try:
|
| 491 |
+
with destination.open("xb") as handle:
|
| 492 |
+
handle.write(payload)
|
| 493 |
+
except FileExistsError:
|
| 494 |
+
pass
|
| 495 |
+
if file_sha256(destination) != digest:
|
| 496 |
+
raise VSTPrefixSemanticError("decoded frame hash mismatch")
|
| 497 |
+
relative_paths.append(relative.as_posix())
|
| 498 |
+
return {
|
| 499 |
+
"relative_paths": relative_paths,
|
| 500 |
+
"frame_indices": indices,
|
| 501 |
+
"frame_timestamps": timestamps,
|
| 502 |
+
}
|
| 503 |
+
|
| 504 |
+
|
| 505 |
+
def _rounded_size(height: int, width: int) -> tuple[int, int]:
|
| 506 |
+
factor = 28
|
| 507 |
+
max_pixels = PREFIX_FRAME_MAX_PIXELS
|
| 508 |
+
max_side = PREFIX_FRAME_MAX_SIDE
|
| 509 |
+
scale = min(1.0, max_side / max(height, width))
|
| 510 |
+
if height * width * scale * scale > max_pixels:
|
| 511 |
+
scale = min(scale, math.sqrt(max_pixels / (height * width)))
|
| 512 |
+
new_height = max(factor, int(round(height * scale / factor)) * factor)
|
| 513 |
+
new_width = max(factor, int(round(width * scale / factor)) * factor)
|
| 514 |
+
while new_height * new_width > max_pixels:
|
| 515 |
+
if new_height >= new_width and new_height > factor:
|
| 516 |
+
new_height -= factor
|
| 517 |
+
elif new_width > factor:
|
| 518 |
+
new_width -= factor
|
| 519 |
+
else:
|
| 520 |
+
break
|
| 521 |
+
return new_height, new_width
|
| 522 |
+
|
| 523 |
+
|
| 524 |
+
if __name__ == "__main__":
|
| 525 |
+
main()
|
code/data_pipeline/temporal_scope/filter_semantic_release_by_blind_audit.py
ADDED
|
@@ -0,0 +1,329 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Seal the high-confidence agreement subset of a blind-audited SFT release.
|
| 3 |
+
|
| 4 |
+
The visual auditor never receives the source answer. This builder also never
|
| 5 |
+
uses the auditor prediction as a new label: it only admits an existing source
|
| 6 |
+
row when a complete, hash-bound audit independently recovered that row's
|
| 7 |
+
answer. Partial audits are deliberately ineligible for materialization.
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
from __future__ import annotations
|
| 11 |
+
|
| 12 |
+
import argparse
|
| 13 |
+
import collections
|
| 14 |
+
import hashlib
|
| 15 |
+
import json
|
| 16 |
+
import os
|
| 17 |
+
import shutil
|
| 18 |
+
import tempfile
|
| 19 |
+
from pathlib import Path
|
| 20 |
+
from typing import Any
|
| 21 |
+
|
| 22 |
+
from viss_core.data_v0.canonical import (
|
| 23 |
+
canonical_json_line,
|
| 24 |
+
file_sha256,
|
| 25 |
+
read_jsonl,
|
| 26 |
+
write_jsonl,
|
| 27 |
+
)
|
| 28 |
+
from viss_core.data_v0.semantic_release import (
|
| 29 |
+
FORMAT,
|
| 30 |
+
validate_semantic_release,
|
| 31 |
+
)
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
AUDIT_ROW_FORMAT = "streameva.semantic_release.blind_visual_audit_row.v0"
|
| 35 |
+
AUDIT_SUMMARY_FORMAT = "streameva.semantic_release.blind_visual_audit.v0"
|
| 36 |
+
FILTER_FORMAT = "streameva.semantic_release.blind_visual_filter.v0"
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
class BlindAuditFilterError(ValueError):
|
| 40 |
+
"""Raised when an audit is incomplete or not independent of its labels."""
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def _target(row: dict[str, Any]) -> str:
|
| 44 |
+
values = [
|
| 45 |
+
str(item.get("text") or "")
|
| 46 |
+
for message in row.get("messages") or []
|
| 47 |
+
if message.get("role") == "assistant" and message.get("loss") is True
|
| 48 |
+
for item in message.get("content") or []
|
| 49 |
+
if item.get("type") == "text"
|
| 50 |
+
]
|
| 51 |
+
if len(values) != 1:
|
| 52 |
+
raise BlindAuditFilterError("semantic row has no unique target")
|
| 53 |
+
return values[0]
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
def _images(rows: list[dict[str, Any]]) -> set[str]:
|
| 57 |
+
return {
|
| 58 |
+
str(item["image"])
|
| 59 |
+
for row in rows
|
| 60 |
+
for message in row["messages"]
|
| 61 |
+
for item in message["content"]
|
| 62 |
+
if item.get("type") == "image"
|
| 63 |
+
}
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
def _answer_balanced_subset(
|
| 67 |
+
rows: list[dict[str, Any]], *, count: int, seed: str
|
| 68 |
+
) -> list[dict[str, Any]]:
|
| 69 |
+
if count <= 0 or count > len(rows):
|
| 70 |
+
raise BlindAuditFilterError("subset row count exceeds audited agreements")
|
| 71 |
+
by_answer: dict[str, list[dict[str, Any]]] = collections.defaultdict(list)
|
| 72 |
+
for row in rows:
|
| 73 |
+
by_answer[_target(row)].append(row)
|
| 74 |
+
if set(by_answer) != set("ABCD"):
|
| 75 |
+
raise BlindAuditFilterError("answer-balanced subset requires A/B/C/D rows")
|
| 76 |
+
base, remainder = divmod(count, 4)
|
| 77 |
+
selected: list[dict[str, Any]] = []
|
| 78 |
+
for index, answer in enumerate("ABCD"):
|
| 79 |
+
quota = base + int(index < remainder)
|
| 80 |
+
ranked = sorted(
|
| 81 |
+
by_answer[answer],
|
| 82 |
+
key=lambda row: hashlib.sha256(
|
| 83 |
+
f"blind-filter-subset:{seed}:{row['uid']}".encode()
|
| 84 |
+
).hexdigest(),
|
| 85 |
+
)
|
| 86 |
+
if len(ranked) < quota:
|
| 87 |
+
raise BlindAuditFilterError(
|
| 88 |
+
f"answer {answer} has {len(ranked)} rows but needs {quota}"
|
| 89 |
+
)
|
| 90 |
+
selected.extend(ranked[:quota])
|
| 91 |
+
return sorted(selected, key=lambda row: str(row["uid"]))
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
def _materialize(source: Path, destination: Path, *, mode: str) -> None:
|
| 95 |
+
destination.parent.mkdir(parents=True, exist_ok=True)
|
| 96 |
+
if source.is_symlink() or not source.is_file():
|
| 97 |
+
raise BlindAuditFilterError(f"unsafe source media: {source}")
|
| 98 |
+
if mode in {"hardlink", "auto"}:
|
| 99 |
+
try:
|
| 100 |
+
os.link(source, destination)
|
| 101 |
+
return
|
| 102 |
+
except OSError:
|
| 103 |
+
if mode == "hardlink":
|
| 104 |
+
raise
|
| 105 |
+
shutil.copy2(source, destination)
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
def build(args: argparse.Namespace) -> dict[str, Any]:
|
| 109 |
+
source = args.source.resolve(strict=True)
|
| 110 |
+
audit_path = args.audit.resolve(strict=True)
|
| 111 |
+
summary_path = args.audit_summary.resolve(strict=True)
|
| 112 |
+
output = args.output.resolve()
|
| 113 |
+
if output.exists():
|
| 114 |
+
raise FileExistsError(output)
|
| 115 |
+
if args.copy_mode not in {"copy", "hardlink", "auto"}:
|
| 116 |
+
raise BlindAuditFilterError("unsupported copy mode")
|
| 117 |
+
allowed_confidence = set(args.confidence)
|
| 118 |
+
if not allowed_confidence or not allowed_confidence <= {"high", "medium"}:
|
| 119 |
+
raise BlindAuditFilterError("confidence must be high and/or medium")
|
| 120 |
+
if not 0.0 <= args.min_source_accuracy <= 1.0:
|
| 121 |
+
raise BlindAuditFilterError("minimum source accuracy must lie in [0, 1]")
|
| 122 |
+
if args.min_rows <= 0:
|
| 123 |
+
raise BlindAuditFilterError("minimum retained rows must be positive")
|
| 124 |
+
|
| 125 |
+
source_manifest = validate_semantic_release(source, verify_media_hashes=False)
|
| 126 |
+
source_trainer = source / "trainer/sft.jsonl"
|
| 127 |
+
source_rows = read_jsonl(
|
| 128 |
+
source_trainer, verify_content_hashes=True, require_canonical=True
|
| 129 |
+
)
|
| 130 |
+
rows_by_uid = {str(row["uid"]): row for row in source_rows}
|
| 131 |
+
if len(rows_by_uid) != len(source_rows):
|
| 132 |
+
raise BlindAuditFilterError("source UIDs are not unique")
|
| 133 |
+
|
| 134 |
+
summary = json.loads(summary_path.read_text(encoding="utf-8"))
|
| 135 |
+
required_summary = {
|
| 136 |
+
"format": AUDIT_SUMMARY_FORMAT,
|
| 137 |
+
"status": "admit_semantic_release",
|
| 138 |
+
"admitted": True,
|
| 139 |
+
"release_id": source_manifest["release_id"],
|
| 140 |
+
"release_manifest_sha256": file_sha256(source / "manifest.json"),
|
| 141 |
+
"trainer_sha256": file_sha256(source_trainer),
|
| 142 |
+
"sample_rows": len(source_rows),
|
| 143 |
+
"valid_rows": len(source_rows),
|
| 144 |
+
"errors": 0,
|
| 145 |
+
"expected_answers_sent_to_auditor": False,
|
| 146 |
+
"auditor_outputs_admitted_to_training": False,
|
| 147 |
+
"benchmark_rows_used": 0,
|
| 148 |
+
"result_rows_sha256": file_sha256(audit_path),
|
| 149 |
+
}
|
| 150 |
+
if any(summary.get(key) != value for key, value in required_summary.items()):
|
| 151 |
+
raise BlindAuditFilterError("audit summary is partial, rejected, or unbound")
|
| 152 |
+
accuracy = float(summary.get("accuracy", -1.0))
|
| 153 |
+
if accuracy < args.min_source_accuracy:
|
| 154 |
+
raise BlindAuditFilterError(
|
| 155 |
+
f"blind source agreement {accuracy:.4f} is below "
|
| 156 |
+
f"{args.min_source_accuracy:.4f}"
|
| 157 |
+
)
|
| 158 |
+
|
| 159 |
+
audit_rows = read_jsonl(audit_path, require_canonical=True)
|
| 160 |
+
audit_by_uid: dict[str, dict[str, Any]] = {}
|
| 161 |
+
for item in audit_rows:
|
| 162 |
+
uid = str(item.get("uid") or "")
|
| 163 |
+
if (
|
| 164 |
+
item.get("format") != AUDIT_ROW_FORMAT
|
| 165 |
+
or not uid
|
| 166 |
+
or uid in audit_by_uid
|
| 167 |
+
or uid not in rows_by_uid
|
| 168 |
+
):
|
| 169 |
+
raise BlindAuditFilterError("audit row identity diverged")
|
| 170 |
+
expected = _target(rows_by_uid[uid])
|
| 171 |
+
prediction = item.get("prediction")
|
| 172 |
+
if (
|
| 173 |
+
item.get("expected") != expected
|
| 174 |
+
or item.get("error") is not None
|
| 175 |
+
or item.get("confidence") not in {"high", "medium", "low"}
|
| 176 |
+
or item.get("correct") is not (prediction == expected)
|
| 177 |
+
):
|
| 178 |
+
raise BlindAuditFilterError("audit decision is malformed")
|
| 179 |
+
audit_by_uid[uid] = item
|
| 180 |
+
if set(audit_by_uid) != set(rows_by_uid):
|
| 181 |
+
raise BlindAuditFilterError("blind audit does not cover every source row")
|
| 182 |
+
|
| 183 |
+
selected = [
|
| 184 |
+
row
|
| 185 |
+
for uid, row in rows_by_uid.items()
|
| 186 |
+
if audit_by_uid[uid]["correct"] is True
|
| 187 |
+
and audit_by_uid[uid]["confidence"] in allowed_confidence
|
| 188 |
+
]
|
| 189 |
+
selected.sort(key=lambda row: str(row["uid"]))
|
| 190 |
+
eligible_rows = len(selected)
|
| 191 |
+
max_rows = getattr(args, "max_rows", None)
|
| 192 |
+
selection_seed = str(getattr(args, "selection_seed", "blind-filter-subset-v0"))
|
| 193 |
+
if max_rows is not None:
|
| 194 |
+
selected = _answer_balanced_subset(
|
| 195 |
+
selected, count=int(max_rows), seed=selection_seed
|
| 196 |
+
)
|
| 197 |
+
if len(selected) < args.min_rows:
|
| 198 |
+
raise BlindAuditFilterError(
|
| 199 |
+
f"only {len(selected)} high-confidence rows survived; "
|
| 200 |
+
f"requires {args.min_rows}"
|
| 201 |
+
)
|
| 202 |
+
|
| 203 |
+
source_inventory_path = source / "manifests/media_inventory.jsonl"
|
| 204 |
+
source_inventory = {
|
| 205 |
+
str(item["path"]): item
|
| 206 |
+
for item in read_jsonl(source_inventory_path, require_canonical=True)
|
| 207 |
+
}
|
| 208 |
+
referenced = _images(selected)
|
| 209 |
+
if not referenced <= set(source_inventory):
|
| 210 |
+
raise BlindAuditFilterError("selected media is absent from source inventory")
|
| 211 |
+
|
| 212 |
+
output.parent.mkdir(parents=True, exist_ok=True)
|
| 213 |
+
scratch = Path(tempfile.mkdtemp(prefix=f".{output.name}.partial.", dir=output.parent))
|
| 214 |
+
try:
|
| 215 |
+
write_jsonl(selected, scratch / "trainer/sft.jsonl")
|
| 216 |
+
selected_inventory = [source_inventory[path] for path in sorted(referenced)]
|
| 217 |
+
write_jsonl(selected_inventory, scratch / "manifests/media_inventory.jsonl")
|
| 218 |
+
for item in selected_inventory:
|
| 219 |
+
relative = Path(str(item["path"]))
|
| 220 |
+
source_file = source / relative
|
| 221 |
+
destination = scratch / relative
|
| 222 |
+
_materialize(source_file, destination, mode=args.copy_mode)
|
| 223 |
+
if (
|
| 224 |
+
destination.stat().st_size != int(item["bytes"])
|
| 225 |
+
or file_sha256(destination) != item["sha256"]
|
| 226 |
+
):
|
| 227 |
+
raise BlindAuditFilterError("materialized media hash diverged")
|
| 228 |
+
|
| 229 |
+
answers = collections.Counter(_target(row) for row in selected)
|
| 230 |
+
frames = collections.Counter(
|
| 231 |
+
sum(
|
| 232 |
+
item.get("type") == "image"
|
| 233 |
+
for message in row["messages"]
|
| 234 |
+
for item in message["content"]
|
| 235 |
+
)
|
| 236 |
+
for row in selected
|
| 237 |
+
)
|
| 238 |
+
confidence = collections.Counter(
|
| 239 |
+
str(audit_by_uid[str(row["uid"])]["confidence"]) for row in selected
|
| 240 |
+
)
|
| 241 |
+
manifest = {
|
| 242 |
+
"artifacts": {
|
| 243 |
+
"manifests/media_inventory.jsonl": file_sha256(
|
| 244 |
+
scratch / "manifests/media_inventory.jsonl"
|
| 245 |
+
),
|
| 246 |
+
"trainer/sft.jsonl": file_sha256(scratch / "trainer/sft.jsonl"),
|
| 247 |
+
},
|
| 248 |
+
"contracts": source_manifest["contracts"],
|
| 249 |
+
"counts": {
|
| 250 |
+
"media_bytes": sum(int(item["bytes"]) for item in selected_inventory),
|
| 251 |
+
"media_files": len(selected_inventory),
|
| 252 |
+
"trainer_rows": len(selected),
|
| 253 |
+
},
|
| 254 |
+
"distributions": {
|
| 255 |
+
"answer": dict(sorted(answers.items())),
|
| 256 |
+
"blind_auditor_confidence": dict(sorted(confidence.items())),
|
| 257 |
+
"frames_per_row": {
|
| 258 |
+
str(key): value for key, value in sorted(frames.items())
|
| 259 |
+
},
|
| 260 |
+
},
|
| 261 |
+
"format": FORMAT,
|
| 262 |
+
"inputs": {
|
| 263 |
+
**source_manifest["inputs"],
|
| 264 |
+
"blind_visual_filter": {
|
| 265 |
+
"format": FILTER_FORMAT,
|
| 266 |
+
"source_release_id": source_manifest["release_id"],
|
| 267 |
+
"source_manifest_sha256": file_sha256(source / "manifest.json"),
|
| 268 |
+
"source_trainer_sha256": file_sha256(source_trainer),
|
| 269 |
+
"audit_sha256": file_sha256(audit_path),
|
| 270 |
+
"audit_summary_sha256": file_sha256(summary_path),
|
| 271 |
+
"auditor_model": str(summary.get("auditor_model") or ""),
|
| 272 |
+
"auditor_received_expected_answers": False,
|
| 273 |
+
"auditor_predictions_used_as_labels": False,
|
| 274 |
+
"complete_source_coverage_required": True,
|
| 275 |
+
"eligible_rows_after_audit_filter": eligible_rows,
|
| 276 |
+
"filter_builder_sha256": file_sha256(Path(__file__).resolve(strict=True)),
|
| 277 |
+
"max_rows": max_rows,
|
| 278 |
+
"source_agreement_accuracy": accuracy,
|
| 279 |
+
"subset_selection_seed": selection_seed if max_rows is not None else None,
|
| 280 |
+
"subset_selection_strategy": (
|
| 281 |
+
"answer_balanced_stable_hash" if max_rows is not None else "all"
|
| 282 |
+
),
|
| 283 |
+
"allowed_confidence": sorted(allowed_confidence),
|
| 284 |
+
"source_rows": len(source_rows),
|
| 285 |
+
"selected_rows": len(selected),
|
| 286 |
+
},
|
| 287 |
+
},
|
| 288 |
+
"license": source_manifest["license"],
|
| 289 |
+
"model_lineage": source_manifest["model_lineage"],
|
| 290 |
+
"public_release_allowed": False,
|
| 291 |
+
"release_id": args.release_id,
|
| 292 |
+
"status": "sealed_internal_training_release",
|
| 293 |
+
"training_allowed": True,
|
| 294 |
+
"training_scope": "pure_vlm_semantic_sft_only",
|
| 295 |
+
}
|
| 296 |
+
(scratch / "manifest.json").write_bytes(canonical_json_line(manifest))
|
| 297 |
+
validate_semantic_release(scratch, verify_media_hashes=True)
|
| 298 |
+
os.replace(scratch, output)
|
| 299 |
+
return manifest
|
| 300 |
+
except Exception:
|
| 301 |
+
shutil.rmtree(scratch, ignore_errors=True)
|
| 302 |
+
raise
|
| 303 |
+
|
| 304 |
+
|
| 305 |
+
def main() -> None:
|
| 306 |
+
parser = argparse.ArgumentParser(description=__doc__)
|
| 307 |
+
parser.add_argument("--source", type=Path, required=True)
|
| 308 |
+
parser.add_argument("--audit", type=Path, required=True)
|
| 309 |
+
parser.add_argument("--audit-summary", type=Path, required=True)
|
| 310 |
+
parser.add_argument("--output", type=Path, required=True)
|
| 311 |
+
parser.add_argument("--release-id", required=True)
|
| 312 |
+
parser.add_argument(
|
| 313 |
+
"--confidence", action="append", choices=("high", "medium"), default=[]
|
| 314 |
+
)
|
| 315 |
+
parser.add_argument("--min-source-accuracy", type=float, default=0.90)
|
| 316 |
+
parser.add_argument("--min-rows", type=int, default=1)
|
| 317 |
+
parser.add_argument("--max-rows", type=int)
|
| 318 |
+
parser.add_argument("--selection-seed", default="blind-filter-subset-v0")
|
| 319 |
+
parser.add_argument(
|
| 320 |
+
"--copy-mode", choices=("copy", "hardlink", "auto"), default="hardlink"
|
| 321 |
+
)
|
| 322 |
+
args = parser.parse_args()
|
| 323 |
+
if not args.confidence:
|
| 324 |
+
args.confidence = ["high"]
|
| 325 |
+
print(json.dumps(build(args), indent=2, sort_keys=True))
|
| 326 |
+
|
| 327 |
+
|
| 328 |
+
if __name__ == "__main__":
|
| 329 |
+
main()
|
code/data_pipeline/temporal_scope/filter_semantic_release_by_option_consensus.py
ADDED
|
@@ -0,0 +1,298 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Seal source-labelled rows that survive an option-invariant visual audit.
|
| 3 |
+
|
| 4 |
+
The two-presentation auditor never receives the source answer. This builder
|
| 5 |
+
also never treats an auditor prediction as a replacement label: a source row
|
| 6 |
+
is eligible only when both option orders independently recover the unchanged
|
| 7 |
+
source answer with an allowed confidence level.
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
from __future__ import annotations
|
| 11 |
+
|
| 12 |
+
import argparse
|
| 13 |
+
import collections
|
| 14 |
+
import json
|
| 15 |
+
import os
|
| 16 |
+
import shutil
|
| 17 |
+
import tempfile
|
| 18 |
+
from pathlib import Path
|
| 19 |
+
from typing import Any
|
| 20 |
+
|
| 21 |
+
from data_pipeline.temporal_scope.filter_semantic_release_by_blind_audit import (
|
| 22 |
+
BlindAuditFilterError,
|
| 23 |
+
_answer_balanced_subset,
|
| 24 |
+
_images,
|
| 25 |
+
_materialize,
|
| 26 |
+
_target,
|
| 27 |
+
)
|
| 28 |
+
from viss_core.data_v0.canonical import (
|
| 29 |
+
canonical_json_line,
|
| 30 |
+
file_sha256,
|
| 31 |
+
read_jsonl,
|
| 32 |
+
write_jsonl,
|
| 33 |
+
)
|
| 34 |
+
from viss_core.data_v0.semantic_release import FORMAT, validate_semantic_release
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
AUDIT_ROW_FORMAT = "streameva.semantic_release.option_consensus_audit_row.v0"
|
| 38 |
+
AUDIT_SUMMARY_FORMAT = "streameva.semantic_release.option_consensus_audit.v0"
|
| 39 |
+
FILTER_FORMAT = "streameva.semantic_release.option_consensus_filter.v0"
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def _validate_presentations(
|
| 43 |
+
*,
|
| 44 |
+
source_rows: list[dict[str, Any]],
|
| 45 |
+
audit_rows: list[dict[str, Any]],
|
| 46 |
+
allowed_confidence: set[str],
|
| 47 |
+
) -> tuple[list[dict[str, Any]], int]:
|
| 48 |
+
rows_by_uid = {str(row["uid"]): row for row in source_rows}
|
| 49 |
+
if len(rows_by_uid) != len(source_rows):
|
| 50 |
+
raise BlindAuditFilterError("source UIDs are not unique")
|
| 51 |
+
presentations: dict[str, dict[int, dict[str, Any]]] = collections.defaultdict(dict)
|
| 52 |
+
for item in audit_rows:
|
| 53 |
+
uid = str(item.get("uid") or "")
|
| 54 |
+
presentation = item.get("presentation")
|
| 55 |
+
if (
|
| 56 |
+
item.get("format") != AUDIT_ROW_FORMAT
|
| 57 |
+
or uid not in rows_by_uid
|
| 58 |
+
or presentation not in {0, 1}
|
| 59 |
+
or presentation in presentations[uid]
|
| 60 |
+
):
|
| 61 |
+
raise BlindAuditFilterError("option-consensus audit identity diverged")
|
| 62 |
+
expected = _target(rows_by_uid[uid])
|
| 63 |
+
prediction = item.get("prediction_source")
|
| 64 |
+
mapping = item.get("displayed_to_source")
|
| 65 |
+
if (
|
| 66 |
+
item.get("expected_source") != expected
|
| 67 |
+
or item.get("error") is not None
|
| 68 |
+
or item.get("confidence") not in {"high", "medium", "low"}
|
| 69 |
+
or item.get("correct_source") is not (prediction == expected)
|
| 70 |
+
or not isinstance(mapping, list)
|
| 71 |
+
or sorted(mapping) != list("ABCD")
|
| 72 |
+
or (presentation == 0 and mapping != list("ABCD"))
|
| 73 |
+
or (presentation == 1 and mapping == list("ABCD"))
|
| 74 |
+
):
|
| 75 |
+
raise BlindAuditFilterError("option-consensus audit decision is malformed")
|
| 76 |
+
presentations[uid][int(presentation)] = item
|
| 77 |
+
if set(presentations) != set(rows_by_uid) or any(
|
| 78 |
+
set(values) != {0, 1} for values in presentations.values()
|
| 79 |
+
):
|
| 80 |
+
raise BlindAuditFilterError("option-consensus audit lacks complete two-order coverage")
|
| 81 |
+
|
| 82 |
+
stable_rows = 0
|
| 83 |
+
selected: list[dict[str, Any]] = []
|
| 84 |
+
for uid, source_row in rows_by_uid.items():
|
| 85 |
+
pair = [presentations[uid][0], presentations[uid][1]]
|
| 86 |
+
predictions = {str(item["prediction_source"]) for item in pair}
|
| 87 |
+
if (
|
| 88 |
+
len(predictions) == 1
|
| 89 |
+
and all(str(item["confidence"]) in allowed_confidence for item in pair)
|
| 90 |
+
):
|
| 91 |
+
stable_rows += 1
|
| 92 |
+
if next(iter(predictions)) == _target(source_row):
|
| 93 |
+
selected.append(source_row)
|
| 94 |
+
selected.sort(key=lambda row: str(row["uid"]))
|
| 95 |
+
return selected, stable_rows
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
def build(args: argparse.Namespace) -> dict[str, Any]:
|
| 99 |
+
source = args.source.resolve(strict=True)
|
| 100 |
+
audit_path = args.audit.resolve(strict=True)
|
| 101 |
+
summary_path = args.audit_summary.resolve(strict=True)
|
| 102 |
+
output = args.output.resolve()
|
| 103 |
+
if output.exists():
|
| 104 |
+
raise FileExistsError(output)
|
| 105 |
+
if args.copy_mode not in {"copy", "hardlink", "auto"}:
|
| 106 |
+
raise BlindAuditFilterError("unsupported copy mode")
|
| 107 |
+
allowed_confidence = set(args.confidence)
|
| 108 |
+
if not allowed_confidence or not allowed_confidence <= {"high", "medium"}:
|
| 109 |
+
raise BlindAuditFilterError("confidence must be high and/or medium")
|
| 110 |
+
if not 0.0 <= args.min_consensus_source_accuracy <= 1.0:
|
| 111 |
+
raise BlindAuditFilterError("minimum consensus accuracy must lie in [0, 1]")
|
| 112 |
+
if args.min_rows <= 0:
|
| 113 |
+
raise BlindAuditFilterError("minimum retained rows must be positive")
|
| 114 |
+
|
| 115 |
+
source_manifest = validate_semantic_release(source, verify_media_hashes=False)
|
| 116 |
+
source_trainer = source / "trainer/sft.jsonl"
|
| 117 |
+
source_rows = read_jsonl(
|
| 118 |
+
source_trainer, verify_content_hashes=True, require_canonical=True
|
| 119 |
+
)
|
| 120 |
+
summary = json.loads(summary_path.read_text(encoding="utf-8"))
|
| 121 |
+
required_summary = {
|
| 122 |
+
"format": AUDIT_SUMMARY_FORMAT,
|
| 123 |
+
"status": "complete_option_consensus_audit",
|
| 124 |
+
"release_id": source_manifest["release_id"],
|
| 125 |
+
"release_manifest_sha256": file_sha256(source / "manifest.json"),
|
| 126 |
+
"trainer_sha256": file_sha256(source_trainer),
|
| 127 |
+
"sample_rows": len(source_rows),
|
| 128 |
+
"presentations": 2 * len(source_rows),
|
| 129 |
+
"errors": 0,
|
| 130 |
+
"expected_answers_sent_to_auditor": False,
|
| 131 |
+
"auditor_outputs_admitted_to_training": False,
|
| 132 |
+
"benchmark_rows_used": 0,
|
| 133 |
+
"result_rows_sha256": file_sha256(audit_path),
|
| 134 |
+
}
|
| 135 |
+
if any(summary.get(key) != value for key, value in required_summary.items()):
|
| 136 |
+
raise BlindAuditFilterError("consensus audit is partial, unsafe, or unbound")
|
| 137 |
+
request_contract = summary.get("request_contract") or {}
|
| 138 |
+
if (
|
| 139 |
+
request_contract.get("presentations_per_row") != 2
|
| 140 |
+
or request_contract.get("second_option_order_non_identity") is not True
|
| 141 |
+
or request_contract.get("temperature") != 0
|
| 142 |
+
):
|
| 143 |
+
raise BlindAuditFilterError("consensus request contract diverged")
|
| 144 |
+
|
| 145 |
+
audit_rows = read_jsonl(audit_path, require_canonical=True)
|
| 146 |
+
selected, stable_rows = _validate_presentations(
|
| 147 |
+
source_rows=source_rows,
|
| 148 |
+
audit_rows=audit_rows,
|
| 149 |
+
allowed_confidence=allowed_confidence,
|
| 150 |
+
)
|
| 151 |
+
summary_stable = int(summary.get("high_confidence_option_consensus_rows", -1))
|
| 152 |
+
if allowed_confidence == {"high"} and stable_rows != summary_stable:
|
| 153 |
+
raise BlindAuditFilterError("auditor summary stable-row count diverged")
|
| 154 |
+
source_agreements = len(selected)
|
| 155 |
+
consensus_accuracy = source_agreements / stable_rows if stable_rows else 0.0
|
| 156 |
+
if consensus_accuracy < args.min_consensus_source_accuracy:
|
| 157 |
+
raise BlindAuditFilterError(
|
| 158 |
+
f"stable consensus source agreement {consensus_accuracy:.4f} is below "
|
| 159 |
+
f"{args.min_consensus_source_accuracy:.4f}"
|
| 160 |
+
)
|
| 161 |
+
eligible_rows = len(selected)
|
| 162 |
+
if args.max_rows is not None:
|
| 163 |
+
selected = _answer_balanced_subset(
|
| 164 |
+
selected, count=int(args.max_rows), seed=args.selection_seed
|
| 165 |
+
)
|
| 166 |
+
if len(selected) < args.min_rows:
|
| 167 |
+
raise BlindAuditFilterError(
|
| 168 |
+
f"only {len(selected)} option-invariant source agreements survived; "
|
| 169 |
+
f"requires {args.min_rows}"
|
| 170 |
+
)
|
| 171 |
+
|
| 172 |
+
source_inventory_path = source / "manifests/media_inventory.jsonl"
|
| 173 |
+
source_inventory = {
|
| 174 |
+
str(item["path"]): item
|
| 175 |
+
for item in read_jsonl(source_inventory_path, require_canonical=True)
|
| 176 |
+
}
|
| 177 |
+
referenced = _images(selected)
|
| 178 |
+
if not referenced <= set(source_inventory):
|
| 179 |
+
raise BlindAuditFilterError("selected media is absent from source inventory")
|
| 180 |
+
|
| 181 |
+
output.parent.mkdir(parents=True, exist_ok=True)
|
| 182 |
+
scratch = Path(tempfile.mkdtemp(prefix=f".{output.name}.partial.", dir=output.parent))
|
| 183 |
+
try:
|
| 184 |
+
write_jsonl(selected, scratch / "trainer/sft.jsonl")
|
| 185 |
+
selected_inventory = [source_inventory[path] for path in sorted(referenced)]
|
| 186 |
+
write_jsonl(selected_inventory, scratch / "manifests/media_inventory.jsonl")
|
| 187 |
+
for item in selected_inventory:
|
| 188 |
+
relative = Path(str(item["path"]))
|
| 189 |
+
destination = scratch / relative
|
| 190 |
+
_materialize(source / relative, destination, mode=args.copy_mode)
|
| 191 |
+
if (
|
| 192 |
+
destination.stat().st_size != int(item["bytes"])
|
| 193 |
+
or file_sha256(destination) != item["sha256"]
|
| 194 |
+
):
|
| 195 |
+
raise BlindAuditFilterError("materialized media hash diverged")
|
| 196 |
+
|
| 197 |
+
answers = collections.Counter(_target(row) for row in selected)
|
| 198 |
+
frames = collections.Counter(
|
| 199 |
+
sum(
|
| 200 |
+
item.get("type") == "image"
|
| 201 |
+
for message in row["messages"]
|
| 202 |
+
for item in message["content"]
|
| 203 |
+
)
|
| 204 |
+
for row in selected
|
| 205 |
+
)
|
| 206 |
+
manifest = {
|
| 207 |
+
"artifacts": {
|
| 208 |
+
"manifests/media_inventory.jsonl": file_sha256(
|
| 209 |
+
scratch / "manifests/media_inventory.jsonl"
|
| 210 |
+
),
|
| 211 |
+
"trainer/sft.jsonl": file_sha256(scratch / "trainer/sft.jsonl"),
|
| 212 |
+
},
|
| 213 |
+
"contracts": source_manifest["contracts"],
|
| 214 |
+
"counts": {
|
| 215 |
+
"media_bytes": sum(int(item["bytes"]) for item in selected_inventory),
|
| 216 |
+
"media_files": len(selected_inventory),
|
| 217 |
+
"trainer_rows": len(selected),
|
| 218 |
+
},
|
| 219 |
+
"distributions": {
|
| 220 |
+
"answer": dict(sorted(answers.items())),
|
| 221 |
+
"frames_per_row": {
|
| 222 |
+
str(key): value for key, value in sorted(frames.items())
|
| 223 |
+
},
|
| 224 |
+
},
|
| 225 |
+
"format": FORMAT,
|
| 226 |
+
"inputs": {
|
| 227 |
+
**source_manifest["inputs"],
|
| 228 |
+
"option_consensus_filter": {
|
| 229 |
+
"format": FILTER_FORMAT,
|
| 230 |
+
"source_release_id": source_manifest["release_id"],
|
| 231 |
+
"source_manifest_sha256": file_sha256(source / "manifest.json"),
|
| 232 |
+
"source_trainer_sha256": file_sha256(source_trainer),
|
| 233 |
+
"audit_sha256": file_sha256(audit_path),
|
| 234 |
+
"audit_summary_sha256": file_sha256(summary_path),
|
| 235 |
+
"auditor_model": str(summary.get("auditor_model") or ""),
|
| 236 |
+
"auditor_received_expected_answers": False,
|
| 237 |
+
"auditor_predictions_used_as_labels": False,
|
| 238 |
+
"presentations_per_row": 2,
|
| 239 |
+
"option_invariance_required": True,
|
| 240 |
+
"source_agreement_required": True,
|
| 241 |
+
"stable_consensus_rows": stable_rows,
|
| 242 |
+
"stable_consensus_source_agreements": source_agreements,
|
| 243 |
+
"stable_consensus_source_accuracy": consensus_accuracy,
|
| 244 |
+
"eligible_rows_after_filter": eligible_rows,
|
| 245 |
+
"allowed_confidence": sorted(allowed_confidence),
|
| 246 |
+
"max_rows": args.max_rows,
|
| 247 |
+
"subset_selection_seed": (
|
| 248 |
+
args.selection_seed if args.max_rows is not None else None
|
| 249 |
+
),
|
| 250 |
+
"filter_builder_sha256": file_sha256(Path(__file__).resolve(strict=True)),
|
| 251 |
+
"shared_filter_helpers_sha256": file_sha256(
|
| 252 |
+
Path(__file__)
|
| 253 |
+
.with_name("filter_semantic_release_by_blind_audit.py")
|
| 254 |
+
.resolve(strict=True)
|
| 255 |
+
),
|
| 256 |
+
},
|
| 257 |
+
},
|
| 258 |
+
"license": source_manifest["license"],
|
| 259 |
+
"model_lineage": source_manifest["model_lineage"],
|
| 260 |
+
"public_release_allowed": False,
|
| 261 |
+
"release_id": args.release_id,
|
| 262 |
+
"status": "sealed_internal_training_release",
|
| 263 |
+
"training_allowed": True,
|
| 264 |
+
"training_scope": "pure_vlm_semantic_sft_only",
|
| 265 |
+
}
|
| 266 |
+
(scratch / "manifest.json").write_bytes(canonical_json_line(manifest))
|
| 267 |
+
validate_semantic_release(scratch, verify_media_hashes=True)
|
| 268 |
+
os.replace(scratch, output)
|
| 269 |
+
return manifest
|
| 270 |
+
except Exception:
|
| 271 |
+
shutil.rmtree(scratch, ignore_errors=True)
|
| 272 |
+
raise
|
| 273 |
+
|
| 274 |
+
|
| 275 |
+
def main() -> None:
|
| 276 |
+
parser = argparse.ArgumentParser(description=__doc__)
|
| 277 |
+
parser.add_argument("--source", type=Path, required=True)
|
| 278 |
+
parser.add_argument("--audit", type=Path, required=True)
|
| 279 |
+
parser.add_argument("--audit-summary", type=Path, required=True)
|
| 280 |
+
parser.add_argument("--output", type=Path, required=True)
|
| 281 |
+
parser.add_argument("--release-id", required=True)
|
| 282 |
+
parser.add_argument(
|
| 283 |
+
"--confidence", action="append", choices=("high", "medium"), default=[]
|
| 284 |
+
)
|
| 285 |
+
parser.add_argument("--min-consensus-source-accuracy", type=float, default=0.80)
|
| 286 |
+
parser.add_argument("--min-rows", type=int, default=1)
|
| 287 |
+
parser.add_argument("--max-rows", type=int)
|
| 288 |
+
parser.add_argument("--selection-seed", default="option-consensus-filter-v0")
|
| 289 |
+
parser.add_argument(
|
| 290 |
+
"--copy-mode", choices=("copy", "hardlink", "auto"), default="hardlink"
|
| 291 |
+
)
|
| 292 |
+
args = parser.parse_args()
|
| 293 |
+
manifest = build(args)
|
| 294 |
+
print(json.dumps(manifest, ensure_ascii=False, indent=2, sort_keys=True))
|
| 295 |
+
|
| 296 |
+
|
| 297 |
+
if __name__ == "__main__":
|
| 298 |
+
main()
|
code/data_pipeline/temporal_scope/select_quality_capability_rows.py
ADDED
|
@@ -0,0 +1,420 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Freeze a source-disjoint holdout and a quality-ranked capability selection.
|
| 3 |
+
|
| 4 |
+
This selector never reads benchmark rows, labels, predictions, task IDs, or
|
| 5 |
+
question templates. It joins an already sealed non-benchmark teacher release
|
| 6 |
+
back to its generation evidence, removes nominal capability rows whose answer
|
| 7 |
+
was available too early, and reserves a policy holdout before selecting SFT
|
| 8 |
+
rows. A final training selection additionally requires a complete student
|
| 9 |
+
screen over the structural candidate set.
|
| 10 |
+
"""
|
| 11 |
+
|
| 12 |
+
from __future__ import annotations
|
| 13 |
+
|
| 14 |
+
import argparse
|
| 15 |
+
import collections
|
| 16 |
+
import hashlib
|
| 17 |
+
import json
|
| 18 |
+
from pathlib import Path
|
| 19 |
+
from typing import Any, Iterable
|
| 20 |
+
|
| 21 |
+
from viss_core.data_v0.canonical import (
|
| 22 |
+
canonical_json_line,
|
| 23 |
+
file_sha256,
|
| 24 |
+
read_jsonl,
|
| 25 |
+
stable_id,
|
| 26 |
+
)
|
| 27 |
+
from viss_core.data_v0.semantic_release import validate_semantic_release
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
SELECTION_FORMAT = "iclr_video.quality_capability_selection.v1"
|
| 31 |
+
HOLDOUT_FORMAT = "iclr_video.nonbenchmark_policy_holdout.v1"
|
| 32 |
+
STUDENT_SCREEN_FORMAT = "iclr_video.nonbenchmark_student_screen.v1"
|
| 33 |
+
PROFILE_KIND = "nonbenchmark_capability_profile"
|
| 34 |
+
PARENT_KIND = "teacher_generated_nonbenchmark_visual_mcq"
|
| 35 |
+
PROFILES = (
|
| 36 |
+
"backward_epm",
|
| 37 |
+
"backward_asi",
|
| 38 |
+
"rec_counting",
|
| 39 |
+
"sequence_crr",
|
| 40 |
+
"sequence_ssr",
|
| 41 |
+
)
|
| 42 |
+
DEFAULT_HOLDOUT = {profile: 60 for profile in PROFILES}
|
| 43 |
+
DEFAULT_TRAIN = {"backward_epm": 120, "backward_asi": 180}
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
class QualityCapabilitySelectionError(ValueError):
|
| 47 |
+
pass
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def _target(row: dict[str, Any]) -> str:
|
| 51 |
+
values = [
|
| 52 |
+
str(item.get("text") or "").strip()
|
| 53 |
+
for message in row.get("messages") or []
|
| 54 |
+
if message.get("role") == "assistant" and message.get("loss") is True
|
| 55 |
+
for item in message.get("content") or []
|
| 56 |
+
if item.get("type") == "text"
|
| 57 |
+
]
|
| 58 |
+
if len(values) != 1 or values[0] not in set("ABCD"):
|
| 59 |
+
raise QualityCapabilitySelectionError("row lacks one A-D target")
|
| 60 |
+
return values[0]
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
def _single_parent(row: dict[str, Any], kind: str) -> str:
|
| 64 |
+
values = [
|
| 65 |
+
str(parent.get("id") or "")
|
| 66 |
+
for parent in row.get("parents") or []
|
| 67 |
+
if parent.get("kind") == kind
|
| 68 |
+
]
|
| 69 |
+
if len(values) != 1 or not values[0]:
|
| 70 |
+
raise QualityCapabilitySelectionError(f"row lacks one {kind} parent")
|
| 71 |
+
return values[0]
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
def _rank(seed: str, profile: str, answer: str, uid: str) -> str:
|
| 75 |
+
return hashlib.sha256(
|
| 76 |
+
f"quality-capability-v1:{seed}:{profile}:{answer}:{uid}".encode()
|
| 77 |
+
).hexdigest()
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
def _balanced_select(
|
| 81 |
+
rows: Iterable[dict[str, Any]],
|
| 82 |
+
*,
|
| 83 |
+
quota: int,
|
| 84 |
+
seed: str,
|
| 85 |
+
profile: str,
|
| 86 |
+
) -> list[dict[str, Any]]:
|
| 87 |
+
if quota <= 0 or quota % 4:
|
| 88 |
+
raise QualityCapabilitySelectionError("balanced quota must be positive / 4")
|
| 89 |
+
by_answer: dict[str, list[dict[str, Any]]] = collections.defaultdict(list)
|
| 90 |
+
for row in rows:
|
| 91 |
+
by_answer[str(row["answer"])].append(row)
|
| 92 |
+
per_answer = quota // 4
|
| 93 |
+
selected = []
|
| 94 |
+
for answer in "ABCD":
|
| 95 |
+
values = sorted(
|
| 96 |
+
by_answer.get(answer, []),
|
| 97 |
+
key=lambda row: (_rank(seed, profile, answer, str(row["uid"])), str(row["uid"])),
|
| 98 |
+
)
|
| 99 |
+
if len(values) < per_answer:
|
| 100 |
+
raise QualityCapabilitySelectionError(
|
| 101 |
+
f"{profile}/{answer} has {len(values)} rows; requires {per_answer}"
|
| 102 |
+
)
|
| 103 |
+
selected.extend(values[:per_answer])
|
| 104 |
+
return sorted(selected, key=lambda row: str(row["uid"]))
|
| 105 |
+
|
| 106 |
+
|
| 107 |
+
def _image_paths(request: dict[str, Any]) -> list[str]:
|
| 108 |
+
return [
|
| 109 |
+
str(image)
|
| 110 |
+
for group in request.get("groups") or []
|
| 111 |
+
for image in group.get("images") or []
|
| 112 |
+
]
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
def _structural_candidate(
|
| 116 |
+
request: dict[str, Any], generation: dict[str, Any]
|
| 117 |
+
) -> tuple[bool, dict[str, Any]]:
|
| 118 |
+
profile = str(request.get("target_capability") or "")
|
| 119 |
+
groups = [
|
| 120 |
+
tuple(str(image) for image in group.get("images") or [])
|
| 121 |
+
for group in request.get("groups") or []
|
| 122 |
+
]
|
| 123 |
+
flattened = [image for group in groups for image in group]
|
| 124 |
+
evidence_groups = generation.get("evidence_groups") or []
|
| 125 |
+
first_sufficient = generation.get("first_sufficient_group")
|
| 126 |
+
minimum_sufficient = {
|
| 127 |
+
"backward_epm": 1,
|
| 128 |
+
"backward_asi": 2,
|
| 129 |
+
"rec_counting": 1,
|
| 130 |
+
"sequence_crr": 1,
|
| 131 |
+
"sequence_ssr": 1,
|
| 132 |
+
}.get(profile)
|
| 133 |
+
diagnostics = {
|
| 134 |
+
"first_sufficient_group": first_sufficient,
|
| 135 |
+
"evidence_groups": len(evidence_groups),
|
| 136 |
+
"observation_groups": len(groups),
|
| 137 |
+
"distinct_observation_groups": len(set(groups)),
|
| 138 |
+
"image_positions": len(flattened),
|
| 139 |
+
"unique_images": len(set(flattened)),
|
| 140 |
+
"duplicate_image_positions": len(flattened) - len(set(flattened)),
|
| 141 |
+
}
|
| 142 |
+
eligible = (
|
| 143 |
+
minimum_sufficient is not None
|
| 144 |
+
and generation.get("decision") == "accept"
|
| 145 |
+
and generation.get("confidence") == "high"
|
| 146 |
+
and isinstance(first_sufficient, int)
|
| 147 |
+
and first_sufficient >= minimum_sufficient
|
| 148 |
+
and len(evidence_groups) >= 2
|
| 149 |
+
and len(set(groups)) >= 2
|
| 150 |
+
and len(set(flattened)) >= 6
|
| 151 |
+
)
|
| 152 |
+
return bool(eligible), diagnostics
|
| 153 |
+
|
| 154 |
+
|
| 155 |
+
def _student_keep(row: dict[str, Any], audit: dict[str, Any]) -> bool:
|
| 156 |
+
if audit.get("error") is not None:
|
| 157 |
+
return False
|
| 158 |
+
target = str(row["answer"])
|
| 159 |
+
if str(audit.get("target") or "") != target:
|
| 160 |
+
raise QualityCapabilitySelectionError("student audit target diverges")
|
| 161 |
+
prediction = str(audit.get("prediction") or "")
|
| 162 |
+
margin = float(audit.get("visible_option_margin"))
|
| 163 |
+
if not 0.0 <= margin <= 1.0:
|
| 164 |
+
raise QualityCapabilitySelectionError("student audit margin is outside [0,1]")
|
| 165 |
+
return prediction != target or margin <= 0.25
|
| 166 |
+
|
| 167 |
+
|
| 168 |
+
def _parse_quota(value: str) -> tuple[str, int]:
|
| 169 |
+
try:
|
| 170 |
+
profile, raw = value.rsplit("=", 1)
|
| 171 |
+
count = int(raw)
|
| 172 |
+
except (TypeError, ValueError) as exc:
|
| 173 |
+
raise argparse.ArgumentTypeError("quota must be PROFILE=COUNT") from exc
|
| 174 |
+
if profile not in PROFILES or count <= 0 or count % 4:
|
| 175 |
+
raise argparse.ArgumentTypeError("quota must use a known profile and be / 4")
|
| 176 |
+
return profile, count
|
| 177 |
+
|
| 178 |
+
|
| 179 |
+
def _excluded_parent_ids(releases: list[Path]) -> set[str]:
|
| 180 |
+
excluded: set[str] = set()
|
| 181 |
+
for raw in releases:
|
| 182 |
+
root = raw.resolve(strict=True)
|
| 183 |
+
validate_semantic_release(root, verify_media_hashes=False)
|
| 184 |
+
for row in read_jsonl(
|
| 185 |
+
root / "trainer/sft.jsonl",
|
| 186 |
+
verify_content_hashes=True,
|
| 187 |
+
require_canonical=True,
|
| 188 |
+
):
|
| 189 |
+
excluded.update(
|
| 190 |
+
str(parent["id"])
|
| 191 |
+
for parent in row.get("parents") or []
|
| 192 |
+
if parent.get("kind") == PARENT_KIND and parent.get("id")
|
| 193 |
+
)
|
| 194 |
+
return excluded
|
| 195 |
+
|
| 196 |
+
|
| 197 |
+
def build(args: argparse.Namespace) -> tuple[dict[str, Any], dict[str, Any]]:
|
| 198 |
+
release = args.release.resolve(strict=True)
|
| 199 |
+
pack = args.pack.resolve(strict=True)
|
| 200 |
+
generation_path = args.generation.resolve(strict=True)
|
| 201 |
+
release_manifest = validate_semantic_release(release, verify_media_hashes=False)
|
| 202 |
+
pack_manifest_path = pack / "manifest.json"
|
| 203 |
+
pack_manifest = json.loads(pack_manifest_path.read_text(encoding="utf-8"))
|
| 204 |
+
if (
|
| 205 |
+
(release_manifest.get("inputs") or {}).get("pack_manifest_sha256")
|
| 206 |
+
!= file_sha256(pack_manifest_path)
|
| 207 |
+
or (release_manifest.get("inputs") or {}).get("generation_sha256")
|
| 208 |
+
!= file_sha256(generation_path)
|
| 209 |
+
):
|
| 210 |
+
raise QualityCapabilitySelectionError("release is not bound to pack/generation")
|
| 211 |
+
requests = read_jsonl(pack / "requests.jsonl", require_canonical=True)
|
| 212 |
+
generation_rows = read_jsonl(generation_path, require_canonical=True)
|
| 213 |
+
trainer_rows = read_jsonl(
|
| 214 |
+
release / "trainer/sft.jsonl",
|
| 215 |
+
verify_content_hashes=True,
|
| 216 |
+
require_canonical=True,
|
| 217 |
+
)
|
| 218 |
+
request_by_uid = {str(row["uid"]): row for row in requests}
|
| 219 |
+
generation_by_uid = {str(row["uid"]): row for row in generation_rows}
|
| 220 |
+
trainer_by_uid = {str(row["uid"]): row for row in trainer_rows}
|
| 221 |
+
if len(request_by_uid) != len(requests) or len(generation_by_uid) != len(generation_rows):
|
| 222 |
+
raise QualityCapabilitySelectionError("request/generation UIDs collide")
|
| 223 |
+
|
| 224 |
+
excluded = _excluded_parent_ids(list(args.exclude_release or []))
|
| 225 |
+
joined = []
|
| 226 |
+
for request_uid, request in request_by_uid.items():
|
| 227 |
+
trainer_uid = stable_id("benchmarkdistillsft", request_uid, digest_length=24)
|
| 228 |
+
trainer = trainer_by_uid.get(trainer_uid)
|
| 229 |
+
generation = generation_by_uid.get(request_uid)
|
| 230 |
+
if trainer is None:
|
| 231 |
+
continue
|
| 232 |
+
if generation is None:
|
| 233 |
+
raise QualityCapabilitySelectionError("filtered row lacks generation record")
|
| 234 |
+
parent_id = _single_parent(trainer, PARENT_KIND)
|
| 235 |
+
profile = _single_parent(trainer, PROFILE_KIND)
|
| 236 |
+
if profile != str(request.get("target_capability") or ""):
|
| 237 |
+
raise QualityCapabilitySelectionError("request/trainer profile diverges")
|
| 238 |
+
if parent_id in excluded:
|
| 239 |
+
continue
|
| 240 |
+
joined.append(
|
| 241 |
+
{
|
| 242 |
+
"uid": trainer_uid,
|
| 243 |
+
"request_uid": request_uid,
|
| 244 |
+
"parent_id": parent_id,
|
| 245 |
+
"profile": profile,
|
| 246 |
+
"answer": _target(trainer),
|
| 247 |
+
"source_video_basename": str(request["source_video_basename"]),
|
| 248 |
+
"request": request,
|
| 249 |
+
"generation": generation,
|
| 250 |
+
}
|
| 251 |
+
)
|
| 252 |
+
|
| 253 |
+
holdout_quotas = dict(args.holdout_quota or DEFAULT_HOLDOUT)
|
| 254 |
+
train_quotas = dict(args.train_quota or DEFAULT_TRAIN)
|
| 255 |
+
holdout_rows = []
|
| 256 |
+
for profile, quota in sorted(holdout_quotas.items()):
|
| 257 |
+
holdout_rows.extend(
|
| 258 |
+
_balanced_select(
|
| 259 |
+
(row for row in joined if row["profile"] == profile),
|
| 260 |
+
quota=quota,
|
| 261 |
+
seed=args.seed + ":holdout",
|
| 262 |
+
profile=profile,
|
| 263 |
+
)
|
| 264 |
+
)
|
| 265 |
+
holdout_uids = {str(row["uid"]) for row in holdout_rows}
|
| 266 |
+
holdout_videos = {str(row["source_video_basename"]) for row in holdout_rows}
|
| 267 |
+
if len(holdout_uids) != len(holdout_rows) or len(holdout_videos) != len(holdout_rows):
|
| 268 |
+
raise QualityCapabilitySelectionError("policy holdout is not source-video unique")
|
| 269 |
+
|
| 270 |
+
structural = []
|
| 271 |
+
for row in joined:
|
| 272 |
+
if row["uid"] in holdout_uids:
|
| 273 |
+
continue
|
| 274 |
+
eligible, diagnostics = _structural_candidate(
|
| 275 |
+
row["request"], row["generation"]
|
| 276 |
+
)
|
| 277 |
+
if eligible:
|
| 278 |
+
structural.append({**row, "diagnostics": diagnostics})
|
| 279 |
+
|
| 280 |
+
audit_by_uid: dict[str, dict[str, Any]] | None = None
|
| 281 |
+
audit_record = None
|
| 282 |
+
if args.student_audit is not None:
|
| 283 |
+
audit_path = args.student_audit.resolve(strict=True)
|
| 284 |
+
audit_value = json.loads(audit_path.read_text(encoding="utf-8"))
|
| 285 |
+
if (
|
| 286 |
+
audit_value.get("format") != STUDENT_SCREEN_FORMAT
|
| 287 |
+
or audit_value.get("benchmark_rows_or_labels_used") is not False
|
| 288 |
+
or audit_value.get("source_release_manifest_sha256")
|
| 289 |
+
!= file_sha256(release / "manifest.json")
|
| 290 |
+
):
|
| 291 |
+
raise QualityCapabilitySelectionError("student screen contract diverges")
|
| 292 |
+
audit_rows = list(audit_value.get("rows") or [])
|
| 293 |
+
audit_by_uid = {str(row.get("uid") or ""): row for row in audit_rows}
|
| 294 |
+
if set(audit_by_uid) != {str(row["uid"]) for row in structural}:
|
| 295 |
+
raise QualityCapabilitySelectionError("student screen coverage diverges")
|
| 296 |
+
audit_record = {"path": str(audit_path), "sha256": file_sha256(audit_path)}
|
| 297 |
+
|
| 298 |
+
selected_rows = []
|
| 299 |
+
if audit_by_uid is not None:
|
| 300 |
+
hard = [row for row in structural if _student_keep(row, audit_by_uid[row["uid"]])]
|
| 301 |
+
for profile, quota in sorted(train_quotas.items()):
|
| 302 |
+
selected_rows.extend(
|
| 303 |
+
_balanced_select(
|
| 304 |
+
(row for row in hard if row["profile"] == profile),
|
| 305 |
+
quota=quota,
|
| 306 |
+
seed=args.seed + ":train",
|
| 307 |
+
profile=profile,
|
| 308 |
+
)
|
| 309 |
+
)
|
| 310 |
+
|
| 311 |
+
def public_row(row: dict[str, Any]) -> dict[str, Any]:
|
| 312 |
+
return {
|
| 313 |
+
key: row[key]
|
| 314 |
+
for key in (
|
| 315 |
+
"uid",
|
| 316 |
+
"request_uid",
|
| 317 |
+
"parent_id",
|
| 318 |
+
"profile",
|
| 319 |
+
"answer",
|
| 320 |
+
"source_video_basename",
|
| 321 |
+
)
|
| 322 |
+
} | ({"diagnostics": row["diagnostics"]} if "diagnostics" in row else {})
|
| 323 |
+
|
| 324 |
+
holdout = {
|
| 325 |
+
"format": HOLDOUT_FORMAT,
|
| 326 |
+
"status": "sealed_before_training_selection",
|
| 327 |
+
"source_release": str(release),
|
| 328 |
+
"source_release_manifest_sha256": file_sha256(release / "manifest.json"),
|
| 329 |
+
"selection_seed": args.seed + ":holdout",
|
| 330 |
+
"quotas": holdout_quotas,
|
| 331 |
+
"rows": [public_row(row) for row in sorted(holdout_rows, key=lambda row: row["uid"])],
|
| 332 |
+
"parent_ids": sorted(row["parent_id"] for row in holdout_rows),
|
| 333 |
+
"contracts": {
|
| 334 |
+
"source_video_disjoint_from_sft_selection": True,
|
| 335 |
+
"may_enter_sft_or_rl": False,
|
| 336 |
+
"benchmark_rows_or_labels_used": False,
|
| 337 |
+
},
|
| 338 |
+
}
|
| 339 |
+
selection = {
|
| 340 |
+
"format": SELECTION_FORMAT,
|
| 341 |
+
"status": (
|
| 342 |
+
"sealed_nonbenchmark_quality_selection"
|
| 343 |
+
if audit_by_uid is not None
|
| 344 |
+
else "awaiting_complete_student_screen"
|
| 345 |
+
),
|
| 346 |
+
"training_allowed": audit_by_uid is not None,
|
| 347 |
+
"source_release": str(release),
|
| 348 |
+
"source_release_manifest_sha256": file_sha256(release / "manifest.json"),
|
| 349 |
+
"pack_manifest_sha256": file_sha256(pack_manifest_path),
|
| 350 |
+
"generation_sha256": file_sha256(generation_path),
|
| 351 |
+
"excluded_parent_ids": len(excluded),
|
| 352 |
+
"holdout_parent_ids_sha256": hashlib.sha256(
|
| 353 |
+
canonical_json_line(sorted(holdout["parent_ids"]))
|
| 354 |
+
).hexdigest(),
|
| 355 |
+
"structural_candidates": [
|
| 356 |
+
public_row(row) for row in sorted(structural, key=lambda row: row["uid"])
|
| 357 |
+
],
|
| 358 |
+
"structural_counts_by_profile": dict(
|
| 359 |
+
sorted(collections.Counter(row["profile"] for row in structural).items())
|
| 360 |
+
),
|
| 361 |
+
"student_screen": audit_record,
|
| 362 |
+
"train_quotas": train_quotas,
|
| 363 |
+
"selected_rows": [
|
| 364 |
+
public_row(row) for row in sorted(selected_rows, key=lambda row: row["uid"])
|
| 365 |
+
],
|
| 366 |
+
"parent_ids": sorted(row["parent_id"] for row in selected_rows),
|
| 367 |
+
"transformation": "deduplicate_identical_images_keep_first_chronological_occurrence",
|
| 368 |
+
"contracts": {
|
| 369 |
+
"benchmark_rows_labels_predictions_or_tasks_used": False,
|
| 370 |
+
"counterfactual_rows": 0,
|
| 371 |
+
"holdout_overlap": 0,
|
| 372 |
+
"already_trained_parent_overlap": 0,
|
| 373 |
+
"teacher_confidence_high": True,
|
| 374 |
+
"two_order_option_consensus_inherited": True,
|
| 375 |
+
},
|
| 376 |
+
}
|
| 377 |
+
return holdout, selection
|
| 378 |
+
|
| 379 |
+
|
| 380 |
+
def main() -> None:
|
| 381 |
+
parser = argparse.ArgumentParser(description=__doc__)
|
| 382 |
+
parser.add_argument("--release", type=Path, required=True)
|
| 383 |
+
parser.add_argument("--pack", type=Path, required=True)
|
| 384 |
+
parser.add_argument("--generation", type=Path, required=True)
|
| 385 |
+
parser.add_argument("--exclude-release", type=Path, action="append", default=[])
|
| 386 |
+
parser.add_argument("--student-audit", type=Path)
|
| 387 |
+
parser.add_argument("--holdout-quota", type=_parse_quota, action="append")
|
| 388 |
+
parser.add_argument("--train-quota", type=_parse_quota, action="append")
|
| 389 |
+
parser.add_argument("--seed", default="quality-capability-v1")
|
| 390 |
+
parser.add_argument("--holdout-output", type=Path, required=True)
|
| 391 |
+
parser.add_argument("--selection-output", type=Path, required=True)
|
| 392 |
+
args = parser.parse_args()
|
| 393 |
+
if args.holdout_quota:
|
| 394 |
+
args.holdout_quota = dict(args.holdout_quota)
|
| 395 |
+
if args.train_quota:
|
| 396 |
+
args.train_quota = dict(args.train_quota)
|
| 397 |
+
for path in (args.holdout_output, args.selection_output):
|
| 398 |
+
if path.exists():
|
| 399 |
+
raise FileExistsError(path)
|
| 400 |
+
holdout, selection = build(args)
|
| 401 |
+
args.holdout_output.parent.mkdir(parents=True, exist_ok=True)
|
| 402 |
+
args.selection_output.parent.mkdir(parents=True, exist_ok=True)
|
| 403 |
+
args.holdout_output.write_bytes(canonical_json_line(holdout))
|
| 404 |
+
args.selection_output.write_bytes(canonical_json_line(selection))
|
| 405 |
+
print(
|
| 406 |
+
json.dumps(
|
| 407 |
+
{
|
| 408 |
+
"holdout_rows": len(holdout["rows"]),
|
| 409 |
+
"structural_counts_by_profile": selection["structural_counts_by_profile"],
|
| 410 |
+
"selected_rows": len(selection["selected_rows"]),
|
| 411 |
+
"training_allowed": selection["training_allowed"],
|
| 412 |
+
},
|
| 413 |
+
indent=2,
|
| 414 |
+
sort_keys=True,
|
| 415 |
+
)
|
| 416 |
+
)
|
| 417 |
+
|
| 418 |
+
|
| 419 |
+
if __name__ == "__main__":
|
| 420 |
+
main()
|
code/data_pipeline/temporal_scope/semantic_mcq.py
ADDED
|
@@ -0,0 +1,90 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Shared pure-VLM prompt and deterministic MCQ utilities."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import hashlib
|
| 6 |
+
import re
|
| 7 |
+
from collections.abc import Sequence
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
DIRECT_PROMPT_PREFIX = (
|
| 11 |
+
"You are an advanced video question-answering AI assistant. You have been "
|
| 12 |
+
"provided with causal frames from the video and a multiple-choice question. "
|
| 13 |
+
"Analyze only the visible frames and provide the best answer."
|
| 14 |
+
)
|
| 15 |
+
DIRECT_PROMPT_SUFFIX = "Only give the best option's letter (A, B, C, or D) directly."
|
| 16 |
+
OPTION_LETTERS = "ABCD"
|
| 17 |
+
DIRECT_PROMPT_RE = re.compile(
|
| 18 |
+
r"(?:^|\n)Question:\s*(?P<question>.*?)\n+\s*Options:\s*\n"
|
| 19 |
+
r"(?P<options>.*?)\n+\s*Only give the best option's letter",
|
| 20 |
+
re.DOTALL,
|
| 21 |
+
)
|
| 22 |
+
OPTION_LINE_RE = re.compile(r"^\s*([A-D])[.):]\s*(.+?)\s*$")
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
class SemanticMCQError(ValueError):
|
| 26 |
+
"""Raised when a deterministic semantic MCQ cannot be constructed."""
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def parse_direct_prompt(value: str) -> tuple[str, list[str]]:
|
| 30 |
+
"""Recover one canonical public question and its four visible options."""
|
| 31 |
+
|
| 32 |
+
match = DIRECT_PROMPT_RE.search(str(value))
|
| 33 |
+
if match is None:
|
| 34 |
+
raise SemanticMCQError("canonical direct prompt is malformed")
|
| 35 |
+
question = " ".join(match.group("question").split())
|
| 36 |
+
parsed: dict[str, str] = {}
|
| 37 |
+
for line in match.group("options").splitlines():
|
| 38 |
+
option = OPTION_LINE_RE.fullmatch(line)
|
| 39 |
+
if option is None:
|
| 40 |
+
if line.strip():
|
| 41 |
+
raise SemanticMCQError("canonical option line is malformed")
|
| 42 |
+
continue
|
| 43 |
+
letter, text = option.groups()
|
| 44 |
+
if letter in parsed:
|
| 45 |
+
raise SemanticMCQError("canonical option letter is duplicated")
|
| 46 |
+
parsed[letter] = " ".join(text.split())
|
| 47 |
+
if not question or list(parsed) != list(OPTION_LETTERS):
|
| 48 |
+
raise SemanticMCQError("canonical prompt lacks one question and A--D options")
|
| 49 |
+
options = [parsed[letter] for letter in OPTION_LETTERS]
|
| 50 |
+
if any(not option for option in options) or len(set(options)) != len(options):
|
| 51 |
+
raise SemanticMCQError("canonical prompt options are empty or duplicated")
|
| 52 |
+
return question, options
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def rebalance_answer_slot(
|
| 56 |
+
options: Sequence[str],
|
| 57 |
+
*,
|
| 58 |
+
answer: str,
|
| 59 |
+
answer_slot: int,
|
| 60 |
+
seed: str,
|
| 61 |
+
) -> tuple[list[str], str]:
|
| 62 |
+
"""Move the target to a fixed slot and hash-order the three distractors."""
|
| 63 |
+
|
| 64 |
+
if (
|
| 65 |
+
len(options) != len(OPTION_LETTERS)
|
| 66 |
+
or answer not in OPTION_LETTERS
|
| 67 |
+
or answer_slot not in range(len(OPTION_LETTERS))
|
| 68 |
+
or len(set(options)) != len(OPTION_LETTERS)
|
| 69 |
+
):
|
| 70 |
+
raise SemanticMCQError("invalid four-option answer-space contract")
|
| 71 |
+
target_index = OPTION_LETTERS.index(answer)
|
| 72 |
+
target = str(options[target_index])
|
| 73 |
+
distractors = [
|
| 74 |
+
(index, str(option))
|
| 75 |
+
for index, option in enumerate(options)
|
| 76 |
+
if index != target_index
|
| 77 |
+
]
|
| 78 |
+
distractors.sort(
|
| 79 |
+
key=lambda item: hashlib.sha256(
|
| 80 |
+
f"{seed}|{item[0]}|{item[1]}".encode("utf-8")
|
| 81 |
+
).digest()
|
| 82 |
+
)
|
| 83 |
+
balanced: list[str | None] = [None] * len(OPTION_LETTERS)
|
| 84 |
+
balanced[answer_slot] = target
|
| 85 |
+
remaining = [index for index, value in enumerate(balanced) if value is None]
|
| 86 |
+
for slot, (_, option) in zip(remaining, distractors, strict=True):
|
| 87 |
+
balanced[slot] = option
|
| 88 |
+
if any(value is None for value in balanced):
|
| 89 |
+
raise SemanticMCQError("balanced options are incomplete")
|
| 90 |
+
return [str(value) for value in balanced], OPTION_LETTERS[answer_slot]
|
code/docs/DATA.md
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Data preparation
|
| 2 |
+
|
| 3 |
+
The code package distributes construction and validation code. Training
|
| 4 |
+
records, source media, benchmark exports, and calibration artifacts are
|
| 5 |
+
external inputs.
|
| 6 |
+
|
| 7 |
+
## Training records
|
| 8 |
+
|
| 9 |
+
The trainer consumes a sealed semantic release. Its validation checks the
|
| 10 |
+
manifest, source inventory, user/assistant message schema, media identity,
|
| 11 |
+
and permitted targets. See:
|
| 12 |
+
|
| 13 |
+
- `viss_core/data_v0/schema.py`
|
| 14 |
+
- `viss_core/data_v0/semantic_release.py`
|
| 15 |
+
- `viss_core/data_v0/trainer_adapter.py`
|
| 16 |
+
|
| 17 |
+
The source-aligned builder combines already prepared source releases:
|
| 18 |
+
|
| 19 |
+
```bash
|
| 20 |
+
python -m data_pipeline.temporal_scope.build_source_aligned_sft_release \
|
| 21 |
+
--source data/prepared_source_a \
|
| 22 |
+
--source data/prepared_source_b \
|
| 23 |
+
--output data/viss_release \
|
| 24 |
+
--release-id my_viss_release \
|
| 25 |
+
--copy-mode copy
|
| 26 |
+
```
|
| 27 |
+
|
| 28 |
+
`prepared_source_a` and `prepared_source_b` are placeholders for valid
|
| 29 |
+
source releases, not raw-video directories. This generic example does not
|
| 30 |
+
reconstruct the paper's accepted training mixture by itself.
|
| 31 |
+
|
| 32 |
+
The paper describes the training mixture. Company training records and their
|
| 33 |
+
manifests are not distributed. The training guide assumes users have prepared
|
| 34 |
+
their own valid source releases, media, example weights, and ordering.
|
| 35 |
+
|
| 36 |
+
Set `VISS_UPSTREAM_MANIFEST_SHA256` to the trusted upstream identity for your
|
| 37 |
+
training inputs before starting the builder or trainer. The validator retains
|
| 38 |
+
the identity check, but the code package embeds no company-specific digest.
|
| 39 |
+
Private legacy source adapters are excluded; supply the standard semantic
|
| 40 |
+
release format instead.
|
| 41 |
+
|
| 42 |
+
## Evaluation assets
|
| 43 |
+
|
| 44 |
+
For each benchmark, prepare:
|
| 45 |
+
|
| 46 |
+
1. Official annotations and authorized video files.
|
| 47 |
+
2. A query export using the fields read by `viss_core/benchmarks.py`.
|
| 48 |
+
3. The benchmark's official scoring installation.
|
| 49 |
+
4. Model and processor files compatible with the locked evaluation protocol.
|
| 50 |
+
|
| 51 |
+
`configs/evaluation/recent4.example.json` describes the public sampling,
|
| 52 |
+
query-count, prompt, and processor conventions. It contains no run dates,
|
| 53 |
+
company file identities, or query records. Follow the evaluation guide to
|
| 54 |
+
create a local configuration bound to your own query exports and processor.
|
| 55 |
+
The generated configuration is excluded from publication.
|
| 56 |
+
|
| 57 |
+
## TVS calibration
|
| 58 |
+
|
| 59 |
+
The TVS policy consumes acquisition/commit calibration artifacts and aligned
|
| 60 |
+
view predictions from one model. Calibrate on source-disjoint data, then
|
| 61 |
+
freeze the thresholds for benchmark evaluation. The archive includes the
|
| 62 |
+
selection and analysis implementation, but not the paper's calibration
|
| 63 |
+
records or precomputed predictions.
|
code/docs/EVALUATION.md
ADDED
|
@@ -0,0 +1,114 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Inference, TVS, and scoring
|
| 2 |
+
|
| 3 |
+
Run from the repository root after [installation](INSTALL.md) and
|
| 4 |
+
[data preparation](DATA.md). The code separates model inference, view-policy
|
| 5 |
+
composition, and official scoring.
|
| 6 |
+
|
| 7 |
+
## 1. Generate recent-view proposals
|
| 8 |
+
|
| 9 |
+
Create a local configuration from the public template and your own assets:
|
| 10 |
+
|
| 11 |
+
```bash
|
| 12 |
+
python -m scripts.configure_evaluation_protocol \
|
| 13 |
+
--streaming-samples data/streamingbench/queries.jsonl \
|
| 14 |
+
--ovo-samples data/ovo/queries.jsonl \
|
| 15 |
+
--processor-config checkpoints/viss/locked_processor/preprocessor_config.json \
|
| 16 |
+
--output configs/evaluation/recent4_full_v1.json
|
| 17 |
+
```
|
| 18 |
+
|
| 19 |
+
The generated file contains local paths and file identities. It remains
|
| 20 |
+
private and is excluded from the source archive. Then validate coverage:
|
| 21 |
+
|
| 22 |
+
```bash
|
| 23 |
+
python -m scripts.validate_evaluation_protocol \
|
| 24 |
+
--protocol configs/evaluation/recent4_full_v1.json
|
| 25 |
+
```
|
| 26 |
+
|
| 27 |
+
This check requires the external benchmark exports. It is not a data-free
|
| 28 |
+
installation test.
|
| 29 |
+
|
| 30 |
+
Use the command in the main README with the prepared checkpoint and locked
|
| 31 |
+
processor. The runner accepts `--benchmark streamingbench` or
|
| 32 |
+
`--benchmark ovo`; annotations, query exports, and media must match.
|
| 33 |
+
|
| 34 |
+
Use `--record-choice-confidence` when the proposals will be passed to TVS.
|
| 35 |
+
For a reproducibility run, supply `--model-manifest` with the matching model
|
| 36 |
+
identity artifact. The same identity must be retained across all views.
|
| 37 |
+
The runner also supports `--start-index` and `--end-index` for sharding.
|
| 38 |
+
|
| 39 |
+
The optional multi-GPU launcher is:
|
| 40 |
+
|
| 41 |
+
```text
|
| 42 |
+
bash scripts/run_recent4_single_benchmark.sh \
|
| 43 |
+
REPO MODEL MODEL_MANIFEST OUTPUT PYTHON BENCHMARK GPU_CSV MODEL_ID
|
| 44 |
+
```
|
| 45 |
+
|
| 46 |
+
It additionally requires the benchmark paths documented by its environment
|
| 47 |
+
variable checks. Start with the Python entry point when preparing new assets.
|
| 48 |
+
|
| 49 |
+
## 2. Apply the TVS acquisition and commitment policy
|
| 50 |
+
|
| 51 |
+
The event policy is `causalbudget20`; the compact-history policy is
|
| 52 |
+
`causalhybrid32`. Use the same checkpoint and the view encodings prescribed
|
| 53 |
+
by your protocol. Selecting a policy name alone does not fix image resolution
|
| 54 |
+
or the processor configuration.
|
| 55 |
+
|
| 56 |
+
Given recent-view predictions and a frozen calibration artifact:
|
| 57 |
+
|
| 58 |
+
```bash
|
| 59 |
+
python -m experiments.select_uncertainty_escalation_uids \
|
| 60 |
+
--recent outputs/recent.json \
|
| 61 |
+
--threshold data/calibration/thresholds.json \
|
| 62 |
+
--samples data/benchmark/queries.jsonl \
|
| 63 |
+
--output-uids outputs/event_uids.json \
|
| 64 |
+
--output-samples outputs/event_queries.jsonl \
|
| 65 |
+
--output-audit outputs/selection.json
|
| 66 |
+
```
|
| 67 |
+
|
| 68 |
+
Run event-view inference on the selected queries. Then compose the first
|
| 69 |
+
two stages:
|
| 70 |
+
|
| 71 |
+
```bash
|
| 72 |
+
python -m experiments.materialize_uncertainty_temporal_agent \
|
| 73 |
+
--recent outputs/recent.json \
|
| 74 |
+
--temporal outputs/event.json \
|
| 75 |
+
--selection-audit outputs/selection.json \
|
| 76 |
+
--threshold-audit data/calibration/thresholds.json \
|
| 77 |
+
--output outputs/two_view.json
|
| 78 |
+
```
|
| 79 |
+
|
| 80 |
+
Once the required compact-history proposals are available:
|
| 81 |
+
|
| 82 |
+
```bash
|
| 83 |
+
python -m experiments.materialize_confidence_gated_third_view_agent \
|
| 84 |
+
--recent outputs/recent.json \
|
| 85 |
+
--event outputs/event.json \
|
| 86 |
+
--base-agent outputs/two_view.json \
|
| 87 |
+
--third-view outputs/history.json \
|
| 88 |
+
--threshold-audit data/calibration/thresholds.json \
|
| 89 |
+
--output outputs/tvs.json
|
| 90 |
+
```
|
| 91 |
+
|
| 92 |
+
These commands name user-generated files; no prediction or score artifacts
|
| 93 |
+
are shipped. Materialization combines existing proposals and does not call
|
| 94 |
+
the VLM. `materialize_tvs_view_oracle.py` uses reference correctness for
|
| 95 |
+
analysis and is not the deployed selection policy.
|
| 96 |
+
|
| 97 |
+
## 3. Score saved predictions
|
| 98 |
+
|
| 99 |
+
```bash
|
| 100 |
+
python -m scripts.score_official_benchmark_predictions \
|
| 101 |
+
--benchmark streamingbench \
|
| 102 |
+
--annotation data/streamingbench/annotations.json \
|
| 103 |
+
--predictions outputs/tvs.json \
|
| 104 |
+
--official-root external/StreamingBench \
|
| 105 |
+
--output-json outputs/streamingbench_score.json
|
| 106 |
+
```
|
| 107 |
+
|
| 108 |
+
Use `--benchmark ovo` and the corresponding official assets for OVO-Bench.
|
| 109 |
+
The adapter checks query coverage and invokes the original scorer. For OVO,
|
| 110 |
+
the primary metric is the task-then-group macro average. The official scorer
|
| 111 |
+
may have additional dependencies; install those from its own repository.
|
| 112 |
+
|
| 113 |
+
Generated predictions and scores are your run outputs. They are excluded
|
| 114 |
+
from version control and from the source-release archive.
|
code/docs/INSTALL.md
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Installation
|
| 2 |
+
|
| 3 |
+
Use Python 3.10 or later. Run commands from the source root; editable
|
| 4 |
+
installation keeps the configuration and script paths available.
|
| 5 |
+
|
| 6 |
+
## CPU source checks
|
| 7 |
+
|
| 8 |
+
```bash
|
| 9 |
+
python -m venv .venv
|
| 10 |
+
source .venv/bin/activate
|
| 11 |
+
python -m pip install --upgrade pip
|
| 12 |
+
python -m pip install -e ".[test]"
|
| 13 |
+
python -m pytest -q tests
|
| 14 |
+
```
|
| 15 |
+
|
| 16 |
+
These tests exercise code contracts with synthetic inputs. They do not
|
| 17 |
+
validate a trained checkpoint or execute a full benchmark.
|
| 18 |
+
|
| 19 |
+
## GPU inference
|
| 20 |
+
|
| 21 |
+
Install a CUDA-enabled PyTorch build appropriate for the machine first,
|
| 22 |
+
then install the inference extras:
|
| 23 |
+
|
| 24 |
+
```bash
|
| 25 |
+
python -m pip install -e ".[inference]"
|
| 26 |
+
```
|
| 27 |
+
|
| 28 |
+
The inference extras pin PyTorch 2.7.0 and Transformers 4.57.3.
|
| 29 |
+
The PyTorch version follows the repository runtime requirements; the
|
| 30 |
+
Transformers version follows the retained ViSS model configuration.
|
| 31 |
+
The extras declare Accelerate, qwen-vl-utils, and Decord as well.
|
| 32 |
+
They specify an installation recipe, not a claim that every GPU/driver
|
| 33 |
+
combination has been validated.
|
| 34 |
+
|
| 35 |
+
Install FFmpeg and ensure `ffprobe` is on `PATH`. Multi-GPU shell launchers
|
| 36 |
+
also use Bash 4 or newer, `jq`, `sha256sum`, and `nvidia-smi`.
|
| 37 |
+
|
| 38 |
+
## Training and attention
|
| 39 |
+
|
| 40 |
+
```bash
|
| 41 |
+
python -m pip install -e ".[inference,training]"
|
| 42 |
+
python -m pip install flash-attn==2.7.4.post1 --no-build-isolation
|
| 43 |
+
```
|
| 44 |
+
|
| 45 |
+
FlashAttention installation needs a compatible CUDA toolkit and compiler.
|
| 46 |
+
The trainer defaults to `flash_attention_2`. Use `--attn-implementation sdpa`
|
| 47 |
+
for a development run when FlashAttention is unavailable; this changes the
|
| 48 |
+
runtime configuration.
|
| 49 |
+
|
| 50 |
+
The package includes no model weights. Before a GPU run, confirm that the
|
| 51 |
+
selected checkpoint loads with `AutoModelForImageTextToText` and that the
|
| 52 |
+
processor matches the intended evaluation contract.
|
| 53 |
+
|
| 54 |
+
## Environment records
|
| 55 |
+
|
| 56 |
+
Save the installed package versions and launch command alongside your own
|
| 57 |
+
runs. Generated outputs belong under `outputs/` or `checkpoints/` and are
|
| 58 |
+
excluded from the source archive. Do not commit credentials, benchmark media,
|
| 59 |
+
or local machine paths.
|
code/docs/TRAINING.md
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Training ViSS
|
| 2 |
+
|
| 3 |
+
Install the training environment described in [Installation](INSTALL.md) and
|
| 4 |
+
prepare a sealed training release as described in [Data preparation](DATA.md).
|
| 5 |
+
|
| 6 |
+
## Paper recipe
|
| 7 |
+
|
| 8 |
+
The manuscript specifies one epoch, learning rate `2e-6`, cosine scheduling,
|
| 9 |
+
warmup ratio `0.03`, seed `59`, maximum length `8192`, maximum image pixels
|
| 10 |
+
`200704`, and a global batch of `12`. The vision tower is frozen; the language
|
| 11 |
+
model and multimodal projector are trainable.
|
| 12 |
+
|
| 13 |
+
For two GPUs, a microbatch of one and six accumulation steps give the stated
|
| 14 |
+
global batch:
|
| 15 |
+
|
| 16 |
+
```bash
|
| 17 |
+
torchrun --nproc_per_node=2 -m experiments.train_qwen3vl_sft \
|
| 18 |
+
--model checkpoints/Qwen3-VL-8B-Instruct \
|
| 19 |
+
--train data/viss_release/train.jsonl \
|
| 20 |
+
--output checkpoints/viss \
|
| 21 |
+
--epochs 1 \
|
| 22 |
+
--lr 2e-6 \
|
| 23 |
+
--lr-scheduler-type cosine \
|
| 24 |
+
--warmup-ratio 0.03 \
|
| 25 |
+
--per-device-batch-size 1 \
|
| 26 |
+
--gradient-accumulation-steps 6 \
|
| 27 |
+
--max-length 8192 \
|
| 28 |
+
--max-pixels 200704 \
|
| 29 |
+
--seed 59 \
|
| 30 |
+
--train-order sequential \
|
| 31 |
+
--freeze-vision-tower \
|
| 32 |
+
--train-vision-projector \
|
| 33 |
+
--attn-implementation flash_attention_2 \
|
| 34 |
+
--deepspeed configs/training/deepspeed_zero3_bf16.json
|
| 35 |
+
```
|
| 36 |
+
|
| 37 |
+
Replace `train.jsonl` with the actual row file inside the sealed release.
|
| 38 |
+
The sequential setting preserves prepared row order. The paper preparation
|
| 39 |
+
uses a parent stage followed by capability rows, with stage shuffle seed 59;
|
| 40 |
+
that ordering must already be reflected in the input release.
|
| 41 |
+
|
| 42 |
+
Changing the GPU count requires adjusting accumulation if the global batch
|
| 43 |
+
is to remain fixed. Check the trainer's complete options with:
|
| 44 |
+
|
| 45 |
+
```bash
|
| 46 |
+
python -m experiments.train_qwen3vl_sft --help
|
| 47 |
+
```
|
| 48 |
+
|
| 49 |
+
The training entry point requires Transformers even for `--help`. Training
|
| 50 |
+
outputs and logs are generated at runtime and are not part of this archive.
|
code/experiments/analyze_uncertainty_temporal_escalation.py
ADDED
|
@@ -0,0 +1,451 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Audit a task-agnostic recent-to-temporal escalation policy.
|
| 3 |
+
|
| 4 |
+
The online policy sees only the recent-view first-token generation margin. Gold labels are
|
| 5 |
+
used offline to fit one global threshold on a deterministic calibration split
|
| 6 |
+
and to score the disjoint exploratory split. Task, group, benchmark identity,
|
| 7 |
+
and answer-space semantics are never policy inputs.
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
from __future__ import annotations
|
| 11 |
+
|
| 12 |
+
import argparse
|
| 13 |
+
import hashlib
|
| 14 |
+
import json
|
| 15 |
+
from collections import defaultdict
|
| 16 |
+
from pathlib import Path
|
| 17 |
+
from typing import Any
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
def parse_args() -> argparse.Namespace:
|
| 21 |
+
parser = argparse.ArgumentParser(description=__doc__)
|
| 22 |
+
parser.add_argument("--recent", type=Path, required=True)
|
| 23 |
+
parser.add_argument("--temporal", type=Path, required=True)
|
| 24 |
+
parser.add_argument("--output", type=Path, required=True)
|
| 25 |
+
parser.add_argument(
|
| 26 |
+
"--calibration-modulus",
|
| 27 |
+
type=int,
|
| 28 |
+
default=2,
|
| 29 |
+
help="SHA256(uid) modulo this value; bucket zero fits the threshold.",
|
| 30 |
+
)
|
| 31 |
+
return parser.parse_args()
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def load_result(path: Path) -> dict[str, Any]:
|
| 35 |
+
value = json.loads(path.read_text(encoding="utf-8"))
|
| 36 |
+
if not isinstance(value, dict) or not isinstance(value.get("rows"), list):
|
| 37 |
+
raise ValueError(f"{path}: expected benchmark result with rows")
|
| 38 |
+
if int(value.get("num_errors", 0)) != 0:
|
| 39 |
+
raise ValueError(f"{path}: result contains inference errors")
|
| 40 |
+
return value
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def index_rows(result: dict[str, Any]) -> dict[str, dict[str, Any]]:
|
| 44 |
+
indexed = {str(row.get("uid")): row for row in result["rows"]}
|
| 45 |
+
if "" in indexed or len(indexed) != len(result["rows"]):
|
| 46 |
+
raise ValueError("result contains missing or duplicate UIDs")
|
| 47 |
+
return indexed
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def index_scores(result: dict[str, Any]) -> dict[str, dict[str, Any]]:
|
| 51 |
+
rows = (result.get("score") or {}).get("rows")
|
| 52 |
+
if not isinstance(rows, list):
|
| 53 |
+
raise ValueError("result is missing score.rows")
|
| 54 |
+
indexed = {str(row.get("query_id")): row for row in rows}
|
| 55 |
+
if "" in indexed or len(indexed) != len(rows):
|
| 56 |
+
raise ValueError("score rows contain missing or duplicate query IDs")
|
| 57 |
+
return indexed
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
def index_temporal_scores(result: dict[str, Any]) -> dict[str, dict[str, Any]]:
|
| 61 |
+
score_rows = (result.get("score") or {}).get("rows")
|
| 62 |
+
if isinstance(score_rows, list):
|
| 63 |
+
return index_scores(result)
|
| 64 |
+
rows = result.get("rows")
|
| 65 |
+
if not isinstance(rows, list) or any("correct" not in row for row in rows):
|
| 66 |
+
raise ValueError("temporal result lacks official or inherited correctness")
|
| 67 |
+
indexed = {str(row.get("uid")): row for row in rows}
|
| 68 |
+
if "" in indexed or len(indexed) != len(rows):
|
| 69 |
+
raise ValueError("temporal score rows contain missing or duplicate UIDs")
|
| 70 |
+
return indexed
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
def calibration_bucket(uid: str, modulus: int) -> int:
|
| 74 |
+
if modulus < 2:
|
| 75 |
+
raise ValueError("calibration modulus must be at least two")
|
| 76 |
+
return int(hashlib.sha256(uid.encode("utf-8")).hexdigest(), 16) % modulus
|
| 77 |
+
|
| 78 |
+
|
| 79 |
+
def choice_margin(row: dict[str, Any]) -> float:
|
| 80 |
+
# New confirmatory runs use the same full-vocabulary first-token margin for
|
| 81 |
+
# MCQ and open-ended questions. Older exploratory artifacts contain only
|
| 82 |
+
# the option-letter diagnostic and remain readable for historical audits.
|
| 83 |
+
confidence = row.get("generation_confidence") or row.get("choice_confidence")
|
| 84 |
+
if not isinstance(confidence, dict):
|
| 85 |
+
raise ValueError(f"{row.get('uid')}: missing generation confidence")
|
| 86 |
+
if confidence.get("task_or_group_used") is not False:
|
| 87 |
+
raise ValueError(f"{row.get('uid')}: confidence provenance is not task blind")
|
| 88 |
+
value = float(confidence.get("top2_margin"))
|
| 89 |
+
if not 0.0 <= value <= 1.0:
|
| 90 |
+
raise ValueError(f"{row.get('uid')}: invalid generation margin")
|
| 91 |
+
return value
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
def select_prediction(
|
| 95 |
+
recent_row: dict[str, Any], temporal_row: dict[str, Any], threshold: float
|
| 96 |
+
) -> tuple[str, bool]:
|
| 97 |
+
escalate = choice_margin(recent_row) <= threshold
|
| 98 |
+
source = temporal_row if escalate else recent_row
|
| 99 |
+
return str(
|
| 100 |
+
source.get("prediction") or source.get("selected_prediction") or ""
|
| 101 |
+
), escalate
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
def confidence_gain(
|
| 105 |
+
recent_row: dict[str, Any], temporal_row: dict[str, Any]
|
| 106 |
+
) -> float:
|
| 107 |
+
"""Return the task-blind confidence change after acquiring the temporal view."""
|
| 108 |
+
|
| 109 |
+
return choice_margin(temporal_row) - choice_margin(recent_row)
|
| 110 |
+
|
| 111 |
+
|
| 112 |
+
def select_acquire_commit_prediction(
|
| 113 |
+
recent_row: dict[str, Any],
|
| 114 |
+
temporal_row: dict[str, Any],
|
| 115 |
+
acquire_threshold: float,
|
| 116 |
+
commit_threshold: float,
|
| 117 |
+
) -> tuple[str, bool, bool, float]:
|
| 118 |
+
"""Apply one shared acquire threshold and one shared safe-commit threshold."""
|
| 119 |
+
|
| 120 |
+
acquired = choice_margin(recent_row) <= acquire_threshold
|
| 121 |
+
recent_prediction = str(
|
| 122 |
+
recent_row.get("prediction") or recent_row.get("selected_prediction") or ""
|
| 123 |
+
)
|
| 124 |
+
temporal_prediction = str(
|
| 125 |
+
temporal_row.get("prediction")
|
| 126 |
+
or temporal_row.get("selected_prediction")
|
| 127 |
+
or ""
|
| 128 |
+
)
|
| 129 |
+
gain = confidence_gain(recent_row, temporal_row)
|
| 130 |
+
committed = bool(
|
| 131 |
+
acquired
|
| 132 |
+
and temporal_prediction != recent_prediction
|
| 133 |
+
and gain > commit_threshold
|
| 134 |
+
)
|
| 135 |
+
return (
|
| 136 |
+
temporal_prediction if committed else recent_prediction,
|
| 137 |
+
acquired,
|
| 138 |
+
committed,
|
| 139 |
+
gain,
|
| 140 |
+
)
|
| 141 |
+
|
| 142 |
+
|
| 143 |
+
def evaluate_threshold(
|
| 144 |
+
uids: list[str],
|
| 145 |
+
*,
|
| 146 |
+
recent: dict[str, dict[str, Any]],
|
| 147 |
+
temporal: dict[str, dict[str, Any]],
|
| 148 |
+
recent_scores: dict[str, dict[str, Any]],
|
| 149 |
+
temporal_scores: dict[str, dict[str, Any]],
|
| 150 |
+
threshold: float,
|
| 151 |
+
) -> tuple[int, int]:
|
| 152 |
+
correct = 0
|
| 153 |
+
escalated = 0
|
| 154 |
+
for uid in uids:
|
| 155 |
+
_, did_escalate = select_prediction(
|
| 156 |
+
recent[uid], temporal[uid], threshold
|
| 157 |
+
)
|
| 158 |
+
official = temporal_scores[uid] if did_escalate else recent_scores[uid]
|
| 159 |
+
correct += int(bool(official["correct"]))
|
| 160 |
+
escalated += int(did_escalate)
|
| 161 |
+
return correct, escalated
|
| 162 |
+
|
| 163 |
+
|
| 164 |
+
def fit_threshold(
|
| 165 |
+
calibration_uids: list[str],
|
| 166 |
+
*,
|
| 167 |
+
recent: dict[str, dict[str, Any]],
|
| 168 |
+
temporal: dict[str, dict[str, Any]],
|
| 169 |
+
recent_scores: dict[str, dict[str, Any]],
|
| 170 |
+
temporal_scores: dict[str, dict[str, Any]],
|
| 171 |
+
) -> tuple[float, list[dict[str, Any]]]:
|
| 172 |
+
margins = sorted({choice_margin(recent[uid]) for uid in calibration_uids})
|
| 173 |
+
candidates = [-1.0, *margins, 1.0]
|
| 174 |
+
sweep = []
|
| 175 |
+
for threshold in candidates:
|
| 176 |
+
correct, escalated = evaluate_threshold(
|
| 177 |
+
calibration_uids,
|
| 178 |
+
recent=recent,
|
| 179 |
+
temporal=temporal,
|
| 180 |
+
recent_scores=recent_scores,
|
| 181 |
+
temporal_scores=temporal_scores,
|
| 182 |
+
threshold=threshold,
|
| 183 |
+
)
|
| 184 |
+
sweep.append(
|
| 185 |
+
{
|
| 186 |
+
"threshold": threshold,
|
| 187 |
+
"correct": correct,
|
| 188 |
+
"total": len(calibration_uids),
|
| 189 |
+
"accuracy": correct / len(calibration_uids),
|
| 190 |
+
"escalated": escalated,
|
| 191 |
+
}
|
| 192 |
+
)
|
| 193 |
+
best = max(
|
| 194 |
+
sweep,
|
| 195 |
+
key=lambda row: (
|
| 196 |
+
int(row["correct"]),
|
| 197 |
+
-int(row["escalated"]),
|
| 198 |
+
-float(row["threshold"]),
|
| 199 |
+
),
|
| 200 |
+
)
|
| 201 |
+
return float(best["threshold"]), sweep
|
| 202 |
+
|
| 203 |
+
|
| 204 |
+
def evaluate_acquire_commit_thresholds(
|
| 205 |
+
uids: list[str],
|
| 206 |
+
*,
|
| 207 |
+
recent: dict[str, dict[str, Any]],
|
| 208 |
+
temporal: dict[str, dict[str, Any]],
|
| 209 |
+
recent_scores: dict[str, dict[str, Any]],
|
| 210 |
+
temporal_scores: dict[str, dict[str, Any]],
|
| 211 |
+
acquire_threshold: float,
|
| 212 |
+
commit_threshold: float,
|
| 213 |
+
) -> tuple[int, int, int]:
|
| 214 |
+
correct = 0
|
| 215 |
+
acquired = 0
|
| 216 |
+
committed = 0
|
| 217 |
+
for uid in uids:
|
| 218 |
+
_, did_acquire, did_commit, _ = select_acquire_commit_prediction(
|
| 219 |
+
recent[uid],
|
| 220 |
+
temporal[uid],
|
| 221 |
+
acquire_threshold,
|
| 222 |
+
commit_threshold,
|
| 223 |
+
)
|
| 224 |
+
official = temporal_scores[uid] if did_commit else recent_scores[uid]
|
| 225 |
+
correct += int(bool(official["correct"]))
|
| 226 |
+
acquired += int(did_acquire)
|
| 227 |
+
committed += int(did_commit)
|
| 228 |
+
return correct, acquired, committed
|
| 229 |
+
|
| 230 |
+
|
| 231 |
+
def fit_acquire_commit_thresholds(
|
| 232 |
+
calibration_uids: list[str],
|
| 233 |
+
*,
|
| 234 |
+
recent: dict[str, dict[str, Any]],
|
| 235 |
+
temporal: dict[str, dict[str, Any]],
|
| 236 |
+
recent_scores: dict[str, dict[str, Any]],
|
| 237 |
+
temporal_scores: dict[str, dict[str, Any]],
|
| 238 |
+
) -> tuple[float, float, list[dict[str, Any]]]:
|
| 239 |
+
acquire_candidates = [
|
| 240 |
+
-1.000001,
|
| 241 |
+
*sorted({choice_margin(recent[uid]) for uid in calibration_uids}),
|
| 242 |
+
1.000001,
|
| 243 |
+
]
|
| 244 |
+
commit_candidates = [
|
| 245 |
+
-1.000001,
|
| 246 |
+
*sorted(
|
| 247 |
+
{confidence_gain(recent[uid], temporal[uid]) for uid in calibration_uids}
|
| 248 |
+
),
|
| 249 |
+
1.000001,
|
| 250 |
+
]
|
| 251 |
+
sweep = []
|
| 252 |
+
for acquire_threshold in acquire_candidates:
|
| 253 |
+
for commit_threshold in commit_candidates:
|
| 254 |
+
correct, acquired, committed = evaluate_acquire_commit_thresholds(
|
| 255 |
+
calibration_uids,
|
| 256 |
+
recent=recent,
|
| 257 |
+
temporal=temporal,
|
| 258 |
+
recent_scores=recent_scores,
|
| 259 |
+
temporal_scores=temporal_scores,
|
| 260 |
+
acquire_threshold=acquire_threshold,
|
| 261 |
+
commit_threshold=commit_threshold,
|
| 262 |
+
)
|
| 263 |
+
sweep.append(
|
| 264 |
+
{
|
| 265 |
+
"acquire_threshold": acquire_threshold,
|
| 266 |
+
"commit_threshold": commit_threshold,
|
| 267 |
+
"correct": correct,
|
| 268 |
+
"total": len(calibration_uids),
|
| 269 |
+
"accuracy": correct / len(calibration_uids),
|
| 270 |
+
"acquired": acquired,
|
| 271 |
+
"committed": committed,
|
| 272 |
+
}
|
| 273 |
+
)
|
| 274 |
+
best = max(
|
| 275 |
+
sweep,
|
| 276 |
+
key=lambda row: (
|
| 277 |
+
int(row["correct"]),
|
| 278 |
+
-int(row["acquired"]),
|
| 279 |
+
-int(row["committed"]),
|
| 280 |
+
-float(row["acquire_threshold"]),
|
| 281 |
+
float(row["commit_threshold"]),
|
| 282 |
+
),
|
| 283 |
+
)
|
| 284 |
+
return (
|
| 285 |
+
float(best["acquire_threshold"]),
|
| 286 |
+
float(best["commit_threshold"]),
|
| 287 |
+
sweep,
|
| 288 |
+
)
|
| 289 |
+
|
| 290 |
+
|
| 291 |
+
def main() -> None:
|
| 292 |
+
args = parse_args()
|
| 293 |
+
recent_result = load_result(args.recent)
|
| 294 |
+
temporal_result = load_result(args.temporal)
|
| 295 |
+
for field in ("benchmark", "model_manifest_sha256", "samples_sha256"):
|
| 296 |
+
if recent_result.get(field) != temporal_result.get(field):
|
| 297 |
+
raise ValueError(f"recent/temporal identity mismatch: {field}")
|
| 298 |
+
|
| 299 |
+
recent = index_rows(recent_result)
|
| 300 |
+
temporal = index_rows(temporal_result)
|
| 301 |
+
recent_scores = index_scores(recent_result)
|
| 302 |
+
temporal_scores = index_temporal_scores(temporal_result)
|
| 303 |
+
if (
|
| 304 |
+
set(recent) != set(temporal)
|
| 305 |
+
or set(recent) != set(recent_scores)
|
| 306 |
+
or set(recent) != set(temporal_scores)
|
| 307 |
+
):
|
| 308 |
+
raise ValueError("recent, temporal, and score UID sets must match exactly")
|
| 309 |
+
if any(row.get("task") is None for row in recent_scores.values()):
|
| 310 |
+
raise ValueError("score rows require task labels for reporting only")
|
| 311 |
+
|
| 312 |
+
ordered_uids = [str(row["uid"]) for row in recent_result["rows"]]
|
| 313 |
+
calibration_uids = [
|
| 314 |
+
uid
|
| 315 |
+
for uid in ordered_uids
|
| 316 |
+
if calibration_bucket(uid, args.calibration_modulus) == 0
|
| 317 |
+
]
|
| 318 |
+
evaluation_uids = [uid for uid in ordered_uids if uid not in calibration_uids]
|
| 319 |
+
if not calibration_uids or not evaluation_uids:
|
| 320 |
+
raise ValueError("deterministic split produced an empty partition")
|
| 321 |
+
|
| 322 |
+
acquire_threshold, commit_threshold, sweep = fit_acquire_commit_thresholds(
|
| 323 |
+
calibration_uids,
|
| 324 |
+
recent=recent,
|
| 325 |
+
temporal=temporal,
|
| 326 |
+
recent_scores=recent_scores,
|
| 327 |
+
temporal_scores=temporal_scores,
|
| 328 |
+
)
|
| 329 |
+
rows = []
|
| 330 |
+
calibration_set = set(calibration_uids)
|
| 331 |
+
by_task: dict[str, list[int]] = defaultdict(lambda: [0, 0])
|
| 332 |
+
for uid in ordered_uids:
|
| 333 |
+
prediction, escalated, committed, gain = select_acquire_commit_prediction(
|
| 334 |
+
recent[uid], temporal[uid], acquire_threshold, commit_threshold
|
| 335 |
+
)
|
| 336 |
+
official = temporal_scores[uid] if committed else recent_scores[uid]
|
| 337 |
+
correct = bool(official["correct"])
|
| 338 |
+
partition = "calibration" if uid in calibration_set else "evaluation"
|
| 339 |
+
task = str(recent_scores[uid]["task"])
|
| 340 |
+
if partition == "evaluation":
|
| 341 |
+
by_task[task][0] += int(correct)
|
| 342 |
+
by_task[task][1] += 1
|
| 343 |
+
rows.append(
|
| 344 |
+
{
|
| 345 |
+
"uid": uid,
|
| 346 |
+
"partition": partition,
|
| 347 |
+
"margin": choice_margin(recent[uid]),
|
| 348 |
+
"temporal_margin": choice_margin(temporal[uid]),
|
| 349 |
+
"confidence_gain": gain,
|
| 350 |
+
"acquire_threshold": acquire_threshold,
|
| 351 |
+
"commit_threshold": commit_threshold,
|
| 352 |
+
"escalated": escalated,
|
| 353 |
+
"committed": committed,
|
| 354 |
+
"joint_required_after_escalation": bool(
|
| 355 |
+
escalated and temporal[uid].get("joint_required", False)
|
| 356 |
+
),
|
| 357 |
+
"prediction": prediction,
|
| 358 |
+
"correct": correct,
|
| 359 |
+
"task": task,
|
| 360 |
+
"task_used_by_policy": False,
|
| 361 |
+
}
|
| 362 |
+
)
|
| 363 |
+
|
| 364 |
+
def partition_summary(partition: str) -> dict[str, Any]:
|
| 365 |
+
chosen = [row for row in rows if row["partition"] == partition]
|
| 366 |
+
correct = sum(int(row["correct"]) for row in chosen)
|
| 367 |
+
escalated = sum(int(row["escalated"]) for row in chosen)
|
| 368 |
+
committed = sum(int(row["committed"]) for row in chosen)
|
| 369 |
+
joint_calls = sum(
|
| 370 |
+
int(row["joint_required_after_escalation"]) for row in chosen
|
| 371 |
+
)
|
| 372 |
+
return {
|
| 373 |
+
"correct": correct,
|
| 374 |
+
"total": len(chosen),
|
| 375 |
+
"accuracy": correct / len(chosen),
|
| 376 |
+
"escalated": escalated,
|
| 377 |
+
"escalation_rate": escalated / len(chosen),
|
| 378 |
+
"committed": committed,
|
| 379 |
+
"commit_rate": committed / len(chosen),
|
| 380 |
+
"joint_calls": joint_calls,
|
| 381 |
+
"mean_model_calls": (len(chosen) + escalated + joint_calls)
|
| 382 |
+
/ len(chosen),
|
| 383 |
+
}
|
| 384 |
+
|
| 385 |
+
internal_calibration = (
|
| 386 |
+
recent_result.get("benchmark") == "internal_policy_holdout"
|
| 387 |
+
and recent_result.get("status")
|
| 388 |
+
== "source_disjoint_internal_controller_calibration"
|
| 389 |
+
and temporal_result.get("status")
|
| 390 |
+
== "source_disjoint_internal_controller_calibration"
|
| 391 |
+
)
|
| 392 |
+
output = {
|
| 393 |
+
"format": "streameva.uncertainty_temporal_escalation_audit.v0",
|
| 394 |
+
"status": (
|
| 395 |
+
"source_disjoint_internal_threshold_calibration"
|
| 396 |
+
if internal_calibration
|
| 397 |
+
else "benchmark_selected_exploratory_not_confirmatory"
|
| 398 |
+
),
|
| 399 |
+
"eligible_as_frozen_benchmark_controller_config": internal_calibration,
|
| 400 |
+
"benchmark": recent_result.get("benchmark"),
|
| 401 |
+
"model_manifest_sha256": recent_result.get("model_manifest_sha256"),
|
| 402 |
+
"policy": {
|
| 403 |
+
"first_view": (
|
| 404 |
+
"recorded_initial_causal_view" if internal_calibration else "recent"
|
| 405 |
+
),
|
| 406 |
+
"second_view": (
|
| 407 |
+
"all_recorded_causal_views"
|
| 408 |
+
if internal_calibration
|
| 409 |
+
else temporal_result.get("inference_policy")
|
| 410 |
+
),
|
| 411 |
+
"trigger": "recent_first_step_generation_top2_margin_leq_global_threshold",
|
| 412 |
+
"threshold": acquire_threshold,
|
| 413 |
+
"acquire_threshold": acquire_threshold,
|
| 414 |
+
"commit_threshold": commit_threshold,
|
| 415 |
+
"commit_signal": "temporal_minus_recent_full_vocabulary_top2_margin",
|
| 416 |
+
"commit_rule": "changed_answer_and_confidence_gain_gt_global_threshold",
|
| 417 |
+
"task_group_or_question_router": False,
|
| 418 |
+
"gold_available_at_runtime": False,
|
| 419 |
+
},
|
| 420 |
+
"split": {
|
| 421 |
+
"method": "sha256(uid) modulo calibration_modulus",
|
| 422 |
+
"calibration_modulus": args.calibration_modulus,
|
| 423 |
+
"calibration_bucket": 0,
|
| 424 |
+
},
|
| 425 |
+
"calibration": partition_summary("calibration"),
|
| 426 |
+
"evaluation": partition_summary("evaluation"),
|
| 427 |
+
"evaluation_by_task_reporting_only": {
|
| 428 |
+
task: {"correct": values[0], "total": values[1], "accuracy": values[0] / values[1]}
|
| 429 |
+
for task, values in sorted(by_task.items())
|
| 430 |
+
},
|
| 431 |
+
"calibration_sweep": sweep,
|
| 432 |
+
"rows": rows,
|
| 433 |
+
}
|
| 434 |
+
args.output.parent.mkdir(parents=True, exist_ok=True)
|
| 435 |
+
args.output.write_text(
|
| 436 |
+
json.dumps(output, indent=2, sort_keys=True) + "\n", encoding="utf-8"
|
| 437 |
+
)
|
| 438 |
+
print(
|
| 439 |
+
json.dumps(
|
| 440 |
+
{
|
| 441 |
+
"acquire_threshold": acquire_threshold,
|
| 442 |
+
"commit_threshold": commit_threshold,
|
| 443 |
+
**output["evaluation"],
|
| 444 |
+
},
|
| 445 |
+
sort_keys=True,
|
| 446 |
+
)
|
| 447 |
+
)
|
| 448 |
+
|
| 449 |
+
|
| 450 |
+
if __name__ == "__main__":
|
| 451 |
+
main()
|
code/experiments/evaluate_qwen3vl_recent4_full.py
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
code/experiments/materialize_confidence_gated_third_view_agent.py
ADDED
|
@@ -0,0 +1,259 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Materialize a task-blind confidence-gated third-view temporal agent.
|
| 3 |
+
|
| 4 |
+
The third view is acquired on the same answer-disagreement cohort as the direct
|
| 5 |
+
third-view agent. Its proposal is committed only when its first-token margin
|
| 6 |
+
improves over the margin of the view that supplied the current answer by more
|
| 7 |
+
than one source-disjoint, globally frozen threshold. Runtime policy inputs do
|
| 8 |
+
not include task, benchmark identity, or gold labels.
|
| 9 |
+
"""
|
| 10 |
+
|
| 11 |
+
from __future__ import annotations
|
| 12 |
+
|
| 13 |
+
import argparse
|
| 14 |
+
import copy
|
| 15 |
+
import json
|
| 16 |
+
from pathlib import Path
|
| 17 |
+
from statistics import fmean
|
| 18 |
+
from typing import Any
|
| 19 |
+
|
| 20 |
+
from experiments.analyze_uncertainty_temporal_escalation import choice_margin
|
| 21 |
+
from experiments.materialize_third_view_disagreement_agent import (
|
| 22 |
+
IDENTITY_FIELDS,
|
| 23 |
+
_frame_indices,
|
| 24 |
+
_index,
|
| 25 |
+
)
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
def materialize(
|
| 29 |
+
recent: dict[str, Any],
|
| 30 |
+
event: dict[str, Any],
|
| 31 |
+
base_agent: dict[str, Any],
|
| 32 |
+
third: dict[str, Any],
|
| 33 |
+
threshold: dict[str, Any],
|
| 34 |
+
) -> dict[str, Any]:
|
| 35 |
+
for field in IDENTITY_FIELDS:
|
| 36 |
+
values = {value.get(field) for value in (recent, event, base_agent, third)}
|
| 37 |
+
if len(values) != 1:
|
| 38 |
+
raise ValueError(f"view identity mismatch: {field}")
|
| 39 |
+
if (
|
| 40 |
+
threshold.get("status") != "source_disjoint_internal_threshold_calibration"
|
| 41 |
+
or threshold.get("eligible_as_frozen_benchmark_controller_config") is not True
|
| 42 |
+
or threshold.get("model_manifest_sha256")
|
| 43 |
+
!= recent.get("model_manifest_sha256")
|
| 44 |
+
):
|
| 45 |
+
raise ValueError("threshold is not an identity-aligned internal calibration")
|
| 46 |
+
commit_threshold = float(threshold["policy"]["commit_threshold"])
|
| 47 |
+
|
| 48 |
+
recent_rows, event_rows, base_rows, third_rows = map(
|
| 49 |
+
_index, (recent, event, base_agent, third)
|
| 50 |
+
)
|
| 51 |
+
if set(base_rows) != set(recent_rows) or not set(event_rows) <= set(recent_rows):
|
| 52 |
+
raise ValueError("base/recent/event UID coverage diverged")
|
| 53 |
+
disagreements = {
|
| 54 |
+
uid
|
| 55 |
+
for uid, row in event_rows.items()
|
| 56 |
+
if str(row.get("prediction") or "")
|
| 57 |
+
!= str(recent_rows[uid].get("prediction") or "")
|
| 58 |
+
}
|
| 59 |
+
if not disagreements <= set(third_rows):
|
| 60 |
+
raise ValueError("third-view coverage must include the event disagreement cohort")
|
| 61 |
+
|
| 62 |
+
output_rows: list[dict[str, Any]] = []
|
| 63 |
+
committed_count = 0
|
| 64 |
+
for recent_row in recent.get("rows") or []:
|
| 65 |
+
uid = str(recent_row["uid"])
|
| 66 |
+
base = copy.deepcopy(base_rows[uid])
|
| 67 |
+
if uid not in disagreements:
|
| 68 |
+
base["third_view_policy"] = {
|
| 69 |
+
"triggered": False,
|
| 70 |
+
"reason": "no_acquired_event_disagreement",
|
| 71 |
+
"question_task_group_or_benchmark_router": False,
|
| 72 |
+
}
|
| 73 |
+
output_rows.append(base)
|
| 74 |
+
continue
|
| 75 |
+
|
| 76 |
+
event_row = event_rows[uid]
|
| 77 |
+
third_row = third_rows[uid]
|
| 78 |
+
if event_row.get("error") or third_row.get("error"):
|
| 79 |
+
raise ValueError(f"{uid}: acquired view contains an error")
|
| 80 |
+
if not bool((base.get("agent_policy") or {}).get("escalated")):
|
| 81 |
+
raise ValueError(f"{uid}: third view was triggered outside acquired cohort")
|
| 82 |
+
timestamps = (third_row.get("sampling") or {}).get("frame_timestamps") or []
|
| 83 |
+
if timestamps and max(map(float, timestamps)) > float(third_row["query_time"]) + 1e-3:
|
| 84 |
+
raise ValueError(f"{uid}: third view crosses the causal frontier")
|
| 85 |
+
|
| 86 |
+
recent_prediction = str(recent_row.get("prediction") or "")
|
| 87 |
+
event_prediction = str(event_row.get("prediction") or "")
|
| 88 |
+
current_prediction = str(base.get("prediction") or "")
|
| 89 |
+
third_prediction = str(third_row.get("prediction") or "")
|
| 90 |
+
if not all((recent_prediction, event_prediction, current_prediction, third_prediction)):
|
| 91 |
+
raise ValueError(f"{uid}: an agent proposal is empty")
|
| 92 |
+
if recent_prediction == event_prediction:
|
| 93 |
+
raise ValueError(f"{uid}: third view lacks a disagreement trigger")
|
| 94 |
+
if current_prediction == recent_prediction:
|
| 95 |
+
current_source = recent_row
|
| 96 |
+
current_view = "recent4"
|
| 97 |
+
elif current_prediction == event_prediction:
|
| 98 |
+
current_source = event_row
|
| 99 |
+
current_view = "causalbudget20"
|
| 100 |
+
else:
|
| 101 |
+
raise ValueError(f"{uid}: current answer is not supplied by recent or event view")
|
| 102 |
+
|
| 103 |
+
confidence_gain = choice_margin(third_row) - choice_margin(current_source)
|
| 104 |
+
committed = bool(
|
| 105 |
+
third_prediction != current_prediction
|
| 106 |
+
and confidence_gain > commit_threshold
|
| 107 |
+
)
|
| 108 |
+
if committed:
|
| 109 |
+
committed_count += 1
|
| 110 |
+
base["prediction"] = third_prediction
|
| 111 |
+
base["raw_response"] = third_row.get("raw_response")
|
| 112 |
+
unique_indices = _frame_indices(event_row) | _frame_indices(third_row)
|
| 113 |
+
base.update(
|
| 114 |
+
{
|
| 115 |
+
"current_prediction": current_prediction,
|
| 116 |
+
"third_view_prediction": third_prediction,
|
| 117 |
+
"third_view_policy": {
|
| 118 |
+
"triggered": True,
|
| 119 |
+
"trigger": "acquired_event_prediction_differs_from_recent_prediction",
|
| 120 |
+
"commit": "global_source_disjoint_confidence_gain",
|
| 121 |
+
"current_view": current_view,
|
| 122 |
+
"current_margin": choice_margin(current_source),
|
| 123 |
+
"third_margin": choice_margin(third_row),
|
| 124 |
+
"confidence_gain": confidence_gain,
|
| 125 |
+
"commit_threshold": commit_threshold,
|
| 126 |
+
"committed": committed,
|
| 127 |
+
"question_task_group_or_benchmark_router": False,
|
| 128 |
+
"gold_available_to_policy": False,
|
| 129 |
+
},
|
| 130 |
+
"media_calls": int(base.get("media_calls") or 2) + 1,
|
| 131 |
+
"num_turns": int(base.get("num_turns") or 2) + 1,
|
| 132 |
+
"model_processed_frames": int(base.get("model_processed_frames") or 24)
|
| 133 |
+
+ int(third_row.get("frames_observed") or 32),
|
| 134 |
+
"unique_frames": len(unique_indices) if unique_indices else 32,
|
| 135 |
+
"latency_seconds": float(base.get("latency_seconds") or 0.0)
|
| 136 |
+
+ float(third_row.get("latency_seconds") or 0.0),
|
| 137 |
+
"visual_tokens": int(recent_row.get("visual_tokens") or 0)
|
| 138 |
+
+ int(event_row.get("visual_tokens") or 0)
|
| 139 |
+
+ int(third_row.get("visual_tokens") or 0),
|
| 140 |
+
}
|
| 141 |
+
)
|
| 142 |
+
output_rows.append(base)
|
| 143 |
+
|
| 144 |
+
if any(not str(row.get("prediction") or "") for row in output_rows):
|
| 145 |
+
raise ValueError("materialized agent contains an empty prediction")
|
| 146 |
+
triggered = len(disagreements)
|
| 147 |
+
return {
|
| 148 |
+
**{
|
| 149 |
+
key: base_agent.get(key)
|
| 150 |
+
for key in (
|
| 151 |
+
"annotation",
|
| 152 |
+
"annotation_sha256",
|
| 153 |
+
"benchmark",
|
| 154 |
+
"evaluation_protocol",
|
| 155 |
+
"evaluation_protocol_sha256",
|
| 156 |
+
"model",
|
| 157 |
+
"model_manifest_sha256",
|
| 158 |
+
"processor",
|
| 159 |
+
)
|
| 160 |
+
},
|
| 161 |
+
"format": "streameva.confidence_gated_third_view_agent.v0",
|
| 162 |
+
"status": "benchmark_selected_router_free_development_agent",
|
| 163 |
+
"classification": "benchmark_selected_development_not_confirmatory",
|
| 164 |
+
"comparable_under_locked_recent4_protocol": False,
|
| 165 |
+
"policy": {
|
| 166 |
+
"first_view": "recent4",
|
| 167 |
+
"second_view": "frozen_uncertainty_triggered_causalbudget20",
|
| 168 |
+
"third_view": "causalbudget32r8_on_acquired_answer_disagreement",
|
| 169 |
+
"third_view_commit": "global_source_disjoint_confidence_gain",
|
| 170 |
+
"commit_threshold": commit_threshold,
|
| 171 |
+
"question_task_group_or_benchmark_router": False,
|
| 172 |
+
"benchmark_identity_available_to_policy": False,
|
| 173 |
+
"gold_available_to_policy": False,
|
| 174 |
+
},
|
| 175 |
+
"num_samples": len(output_rows),
|
| 176 |
+
"num_errors": 0,
|
| 177 |
+
"third_view_triggered": triggered,
|
| 178 |
+
"third_view_input_rows": len(third_rows),
|
| 179 |
+
"third_view_unused_precomputed_rows": len(set(third_rows) - disagreements),
|
| 180 |
+
"third_view_committed": committed_count,
|
| 181 |
+
"third_view_trigger_rate": triggered / len(output_rows),
|
| 182 |
+
"third_view_commit_rate": committed_count / triggered if triggered else 0.0,
|
| 183 |
+
"mean_model_calls": fmean(int(row.get("media_calls") or 1) for row in output_rows),
|
| 184 |
+
"mean_unique_frames": fmean(int(row.get("unique_frames") or 4) for row in output_rows),
|
| 185 |
+
"mean_model_processed_frames": fmean(
|
| 186 |
+
int(row.get("model_processed_frames") or 4) for row in output_rows
|
| 187 |
+
),
|
| 188 |
+
"rows": output_rows,
|
| 189 |
+
}
|
| 190 |
+
|
| 191 |
+
|
| 192 |
+
def main() -> None:
|
| 193 |
+
parser = argparse.ArgumentParser(description=__doc__)
|
| 194 |
+
parser.add_argument("--recent", type=Path, required=True)
|
| 195 |
+
parser.add_argument("--event", type=Path, required=True)
|
| 196 |
+
parser.add_argument("--base-agent", type=Path, required=True)
|
| 197 |
+
parser.add_argument("--third-view", type=Path, required=True)
|
| 198 |
+
parser.add_argument("--threshold-audit", type=Path, required=True)
|
| 199 |
+
parser.add_argument("--third-view-model-identity-sha256")
|
| 200 |
+
parser.add_argument("--output", type=Path, required=True)
|
| 201 |
+
args = parser.parse_args()
|
| 202 |
+
if args.output.exists():
|
| 203 |
+
raise FileExistsError(args.output)
|
| 204 |
+
values = [
|
| 205 |
+
json.loads(path.read_text(encoding="utf-8"))
|
| 206 |
+
for path in (
|
| 207 |
+
args.recent,
|
| 208 |
+
args.event,
|
| 209 |
+
args.base_agent,
|
| 210 |
+
args.third_view,
|
| 211 |
+
args.threshold_audit,
|
| 212 |
+
)
|
| 213 |
+
]
|
| 214 |
+
identity_override = None
|
| 215 |
+
if args.third_view_model_identity_sha256:
|
| 216 |
+
if len(args.third_view_model_identity_sha256) != 64 or any(
|
| 217 |
+
value not in "0123456789abcdef"
|
| 218 |
+
for value in args.third_view_model_identity_sha256
|
| 219 |
+
):
|
| 220 |
+
raise ValueError("third-view model identity is malformed")
|
| 221 |
+
third = values[3]
|
| 222 |
+
if third.get("model_manifest_sha256") not in {
|
| 223 |
+
None,
|
| 224 |
+
args.third_view_model_identity_sha256,
|
| 225 |
+
}:
|
| 226 |
+
raise ValueError("third-view model identity override diverges")
|
| 227 |
+
if third.get("model_manifest_sha256") is None:
|
| 228 |
+
third["model_manifest_sha256"] = args.third_view_model_identity_sha256
|
| 229 |
+
identity_override = {
|
| 230 |
+
"field": "model_manifest_sha256",
|
| 231 |
+
"source": "previously_sealed_checkpoint_identity",
|
| 232 |
+
"value": args.third_view_model_identity_sha256,
|
| 233 |
+
}
|
| 234 |
+
result = materialize(*values)
|
| 235 |
+
result["third_view_identity_override"] = identity_override
|
| 236 |
+
result["artifacts"] = {
|
| 237 |
+
"recent": str(args.recent.resolve()),
|
| 238 |
+
"event": str(args.event.resolve()),
|
| 239 |
+
"base_agent": str(args.base_agent.resolve()),
|
| 240 |
+
"third_view": str(args.third_view.resolve()),
|
| 241 |
+
"threshold_audit": str(args.threshold_audit.resolve()),
|
| 242 |
+
}
|
| 243 |
+
args.output.parent.mkdir(parents=True, exist_ok=True)
|
| 244 |
+
args.output.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n")
|
| 245 |
+
print(
|
| 246 |
+
json.dumps(
|
| 247 |
+
{
|
| 248 |
+
"output": str(args.output),
|
| 249 |
+
"num_samples": result["num_samples"],
|
| 250 |
+
"triggered": result["third_view_triggered"],
|
| 251 |
+
"committed": result["third_view_committed"],
|
| 252 |
+
},
|
| 253 |
+
sort_keys=True,
|
| 254 |
+
)
|
| 255 |
+
)
|
| 256 |
+
|
| 257 |
+
|
| 258 |
+
if __name__ == "__main__":
|
| 259 |
+
main()
|
code/experiments/materialize_third_view_disagreement_agent.py
ADDED
|
@@ -0,0 +1,203 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Materialize a router-free third-view agent on event-view disagreements.
|
| 3 |
+
|
| 4 |
+
The frozen first stage decides whether to acquire the event view. A third
|
| 5 |
+
32-frame view is acquired only when that event proposal disagrees with the
|
| 6 |
+
recent proposal, and its answer is committed directly. Runtime decisions use
|
| 7 |
+
predictions and acquisition state only; task names, benchmark identity, and
|
| 8 |
+
gold labels are never inputs to the policy.
|
| 9 |
+
"""
|
| 10 |
+
|
| 11 |
+
from __future__ import annotations
|
| 12 |
+
|
| 13 |
+
import argparse
|
| 14 |
+
import copy
|
| 15 |
+
import json
|
| 16 |
+
from pathlib import Path
|
| 17 |
+
from statistics import fmean
|
| 18 |
+
from typing import Any
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
IDENTITY_FIELDS = (
|
| 22 |
+
"benchmark",
|
| 23 |
+
"model_manifest_sha256",
|
| 24 |
+
"annotation_sha256",
|
| 25 |
+
"evaluation_protocol_sha256",
|
| 26 |
+
)
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def _index(value: dict[str, Any]) -> dict[str, dict[str, Any]]:
|
| 30 |
+
rows = value.get("rows") or []
|
| 31 |
+
indexed = {str(row.get("uid") or ""): row for row in rows}
|
| 32 |
+
if "" in indexed or len(indexed) != len(rows):
|
| 33 |
+
raise ValueError("result rows contain empty or duplicate UIDs")
|
| 34 |
+
return indexed
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def _frame_indices(row: dict[str, Any]) -> set[int]:
|
| 38 |
+
return {int(value) for value in (row.get("sampling") or {}).get("frame_indices") or []}
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def materialize(
|
| 42 |
+
recent: dict[str, Any],
|
| 43 |
+
event: dict[str, Any],
|
| 44 |
+
base_agent: dict[str, Any],
|
| 45 |
+
third: dict[str, Any],
|
| 46 |
+
) -> dict[str, Any]:
|
| 47 |
+
for field in IDENTITY_FIELDS:
|
| 48 |
+
values = {value.get(field) for value in (recent, event, base_agent, third)}
|
| 49 |
+
if len(values) != 1:
|
| 50 |
+
raise ValueError(f"view identity mismatch: {field}")
|
| 51 |
+
recent_rows, event_rows, base_rows, third_rows = map(
|
| 52 |
+
_index, (recent, event, base_agent, third)
|
| 53 |
+
)
|
| 54 |
+
if set(base_rows) != set(recent_rows) or not set(event_rows) <= set(recent_rows):
|
| 55 |
+
raise ValueError("base/recent/event UID coverage diverged")
|
| 56 |
+
disagreements = {
|
| 57 |
+
uid
|
| 58 |
+
for uid, row in event_rows.items()
|
| 59 |
+
if str(row.get("prediction") or "")
|
| 60 |
+
!= str(recent_rows[uid].get("prediction") or "")
|
| 61 |
+
}
|
| 62 |
+
if set(third_rows) != disagreements:
|
| 63 |
+
raise ValueError("third-view coverage must equal the event disagreement cohort")
|
| 64 |
+
|
| 65 |
+
output_rows = []
|
| 66 |
+
for recent_row in recent.get("rows") or []:
|
| 67 |
+
uid = str(recent_row["uid"])
|
| 68 |
+
base = copy.deepcopy(base_rows[uid])
|
| 69 |
+
if uid not in third_rows:
|
| 70 |
+
base["third_view_policy"] = {
|
| 71 |
+
"triggered": False,
|
| 72 |
+
"reason": "no_acquired_event_disagreement",
|
| 73 |
+
"question_task_group_or_benchmark_router": False,
|
| 74 |
+
}
|
| 75 |
+
output_rows.append(base)
|
| 76 |
+
continue
|
| 77 |
+
|
| 78 |
+
proposal = event_rows[uid]
|
| 79 |
+
adjudicator = third_rows[uid]
|
| 80 |
+
if proposal.get("error") or adjudicator.get("error"):
|
| 81 |
+
raise ValueError(f"{uid}: acquired view contains an error")
|
| 82 |
+
if not bool((base.get("agent_policy") or {}).get("escalated")):
|
| 83 |
+
raise ValueError(f"{uid}: third view was triggered outside the acquired cohort")
|
| 84 |
+
timestamps = (adjudicator.get("sampling") or {}).get("frame_timestamps") or []
|
| 85 |
+
if timestamps and max(map(float, timestamps)) > float(adjudicator["query_time"]) + 1e-3:
|
| 86 |
+
raise ValueError(f"{uid}: third view crosses the causal frontier")
|
| 87 |
+
recent_prediction = str(recent_row.get("prediction") or "")
|
| 88 |
+
event_prediction = str(proposal.get("prediction") or "")
|
| 89 |
+
third_prediction = str(adjudicator.get("prediction") or "")
|
| 90 |
+
if not recent_prediction or not event_prediction or not third_prediction:
|
| 91 |
+
raise ValueError(f"{uid}: an agent proposal is empty")
|
| 92 |
+
if recent_prediction == event_prediction:
|
| 93 |
+
raise ValueError(f"{uid}: third view lacks a disagreement trigger")
|
| 94 |
+
|
| 95 |
+
unique_indices = _frame_indices(proposal) | _frame_indices(adjudicator)
|
| 96 |
+
base.update(
|
| 97 |
+
{
|
| 98 |
+
"prediction": third_prediction,
|
| 99 |
+
"raw_response": adjudicator.get("raw_response"),
|
| 100 |
+
"third_view_prediction": third_prediction,
|
| 101 |
+
"third_view_policy": {
|
| 102 |
+
"triggered": True,
|
| 103 |
+
"trigger": "acquired_event_prediction_differs_from_recent_prediction",
|
| 104 |
+
"commit": "direct_third_view_prediction",
|
| 105 |
+
"recent_prediction": recent_prediction,
|
| 106 |
+
"event_prediction": event_prediction,
|
| 107 |
+
"question_task_group_or_benchmark_router": False,
|
| 108 |
+
"gold_available_to_policy": False,
|
| 109 |
+
},
|
| 110 |
+
"media_calls": int(base.get("media_calls") or 2) + 1,
|
| 111 |
+
"num_turns": int(base.get("num_turns") or 2) + 1,
|
| 112 |
+
"model_processed_frames": int(base.get("model_processed_frames") or 24)
|
| 113 |
+
+ int(adjudicator.get("frames_observed") or 32),
|
| 114 |
+
"unique_frames": len(unique_indices) if unique_indices else 32,
|
| 115 |
+
"latency_seconds": float(base.get("latency_seconds") or 0.0)
|
| 116 |
+
+ float(adjudicator.get("latency_seconds") or 0.0),
|
| 117 |
+
"visual_tokens": int(recent_row.get("visual_tokens") or 0)
|
| 118 |
+
+ int(proposal.get("visual_tokens") or 0)
|
| 119 |
+
+ int(adjudicator.get("visual_tokens") or 0),
|
| 120 |
+
}
|
| 121 |
+
)
|
| 122 |
+
output_rows.append(base)
|
| 123 |
+
if any(not str(row.get("prediction") or "") for row in output_rows):
|
| 124 |
+
raise ValueError("materialized agent contains an empty prediction")
|
| 125 |
+
|
| 126 |
+
triggered = len(third_rows)
|
| 127 |
+
return {
|
| 128 |
+
**{
|
| 129 |
+
key: base_agent.get(key)
|
| 130 |
+
for key in (
|
| 131 |
+
"annotation",
|
| 132 |
+
"annotation_sha256",
|
| 133 |
+
"benchmark",
|
| 134 |
+
"evaluation_protocol",
|
| 135 |
+
"evaluation_protocol_sha256",
|
| 136 |
+
"model",
|
| 137 |
+
"model_manifest_sha256",
|
| 138 |
+
"processor",
|
| 139 |
+
)
|
| 140 |
+
},
|
| 141 |
+
"format": "streameva.third_view_disagreement_agent.v0",
|
| 142 |
+
"status": "benchmark_selected_router_free_development_agent",
|
| 143 |
+
"classification": "benchmark_selected_development_not_confirmatory",
|
| 144 |
+
"comparable_under_locked_recent4_protocol": False,
|
| 145 |
+
"policy": {
|
| 146 |
+
"first_view": "recent4",
|
| 147 |
+
"second_view": "frozen_uncertainty_triggered_causalbudget20",
|
| 148 |
+
"third_view": "causalbudget32r8_on_acquired_answer_disagreement",
|
| 149 |
+
"third_view_commit": "direct_prediction",
|
| 150 |
+
"question_task_group_or_benchmark_router": False,
|
| 151 |
+
"benchmark_identity_available_to_policy": False,
|
| 152 |
+
"gold_available_to_policy": False,
|
| 153 |
+
},
|
| 154 |
+
"num_samples": len(output_rows),
|
| 155 |
+
"num_errors": 0,
|
| 156 |
+
"third_view_triggered": triggered,
|
| 157 |
+
"third_view_trigger_rate": triggered / len(output_rows),
|
| 158 |
+
"mean_model_calls": fmean(int(row.get("media_calls") or 1) for row in output_rows),
|
| 159 |
+
"mean_unique_frames": fmean(int(row.get("unique_frames") or 4) for row in output_rows),
|
| 160 |
+
"mean_model_processed_frames": fmean(
|
| 161 |
+
int(row.get("model_processed_frames") or 4) for row in output_rows
|
| 162 |
+
),
|
| 163 |
+
"rows": output_rows,
|
| 164 |
+
}
|
| 165 |
+
|
| 166 |
+
|
| 167 |
+
def main() -> None:
|
| 168 |
+
parser = argparse.ArgumentParser(description=__doc__)
|
| 169 |
+
parser.add_argument("--recent", type=Path, required=True)
|
| 170 |
+
parser.add_argument("--event", type=Path, required=True)
|
| 171 |
+
parser.add_argument("--base-agent", type=Path, required=True)
|
| 172 |
+
parser.add_argument("--third-view", type=Path, required=True)
|
| 173 |
+
parser.add_argument("--output", type=Path, required=True)
|
| 174 |
+
args = parser.parse_args()
|
| 175 |
+
if args.output.exists():
|
| 176 |
+
raise FileExistsError(args.output)
|
| 177 |
+
values = [
|
| 178 |
+
json.loads(path.read_text(encoding="utf-8"))
|
| 179 |
+
for path in (args.recent, args.event, args.base_agent, args.third_view)
|
| 180 |
+
]
|
| 181 |
+
result = materialize(*values)
|
| 182 |
+
result["artifacts"] = {
|
| 183 |
+
"recent": str(args.recent.resolve()),
|
| 184 |
+
"event": str(args.event.resolve()),
|
| 185 |
+
"base_agent": str(args.base_agent.resolve()),
|
| 186 |
+
"third_view": str(args.third_view.resolve()),
|
| 187 |
+
}
|
| 188 |
+
args.output.parent.mkdir(parents=True, exist_ok=True)
|
| 189 |
+
args.output.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n")
|
| 190 |
+
print(
|
| 191 |
+
json.dumps(
|
| 192 |
+
{
|
| 193 |
+
"output": str(args.output),
|
| 194 |
+
"num_samples": result["num_samples"],
|
| 195 |
+
"third_view_triggered": result["third_view_triggered"],
|
| 196 |
+
},
|
| 197 |
+
sort_keys=True,
|
| 198 |
+
)
|
| 199 |
+
)
|
| 200 |
+
|
| 201 |
+
|
| 202 |
+
if __name__ == "__main__":
|
| 203 |
+
main()
|
code/experiments/materialize_tvs_view_oracle.py
ADDED
|
@@ -0,0 +1,118 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Materialize a strict post-hoc oracle over three complete causal views."""
|
| 3 |
+
|
| 4 |
+
from __future__ import annotations
|
| 5 |
+
|
| 6 |
+
import argparse
|
| 7 |
+
import copy
|
| 8 |
+
import json
|
| 9 |
+
from pathlib import Path
|
| 10 |
+
from statistics import fmean
|
| 11 |
+
from typing import Any
|
| 12 |
+
|
| 13 |
+
try:
|
| 14 |
+
from experiments.tvs_correctness import is_correct, load_gold
|
| 15 |
+
except ModuleNotFoundError: # Direct script execution without repository PYTHONPATH.
|
| 16 |
+
from tvs_correctness import is_correct, load_gold
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
IDENTITY_FIELDS = (
|
| 20 |
+
"benchmark",
|
| 21 |
+
"model_manifest_sha256",
|
| 22 |
+
"annotation_sha256",
|
| 23 |
+
"evaluation_protocol_sha256",
|
| 24 |
+
)
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def _index(value: dict[str, Any]) -> dict[str, dict[str, Any]]:
|
| 28 |
+
rows = value.get("rows") or []
|
| 29 |
+
result = {str(row["uid"]): row for row in rows}
|
| 30 |
+
if len(result) != len(rows):
|
| 31 |
+
raise ValueError("duplicate view UID")
|
| 32 |
+
return result
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def materialize(
|
| 36 |
+
recent: dict[str, Any],
|
| 37 |
+
event: dict[str, Any],
|
| 38 |
+
history: dict[str, Any],
|
| 39 |
+
gold: dict[str, dict[str, Any]],
|
| 40 |
+
) -> dict[str, Any]:
|
| 41 |
+
views = [_index(value) for value in (recent, event, history)]
|
| 42 |
+
uid_set = set(views[0])
|
| 43 |
+
if any(set(view) != uid_set for view in views[1:]) or set(gold) != uid_set:
|
| 44 |
+
raise ValueError("strict oracle requires exact three-view and gold coverage")
|
| 45 |
+
for field in IDENTITY_FIELDS:
|
| 46 |
+
expected = recent.get(field)
|
| 47 |
+
if expected in (None, ""):
|
| 48 |
+
raise ValueError(f"recent view lacks identity: {field}")
|
| 49 |
+
for value in (event, history):
|
| 50 |
+
observed = value.get(field)
|
| 51 |
+
if observed not in (None, "", expected):
|
| 52 |
+
raise ValueError(f"view identity mismatch: {field}")
|
| 53 |
+
|
| 54 |
+
rows = []
|
| 55 |
+
chosen_counts = [0, 0, 0]
|
| 56 |
+
for source in recent.get("rows") or []:
|
| 57 |
+
uid = str(source["uid"])
|
| 58 |
+
candidates = [view[uid] for view in views]
|
| 59 |
+
correct = [is_correct(row.get("prediction"), gold[uid]) for row in candidates]
|
| 60 |
+
chosen_index = next((i for i, value in enumerate(correct) if value), 0)
|
| 61 |
+
chosen_counts[chosen_index] += 1
|
| 62 |
+
output = copy.deepcopy(candidates[chosen_index])
|
| 63 |
+
output.update(
|
| 64 |
+
{
|
| 65 |
+
"uid": uid,
|
| 66 |
+
"prediction": str(candidates[chosen_index].get("prediction") or ""),
|
| 67 |
+
"media_calls": 3,
|
| 68 |
+
"model_processed_frames": sum(
|
| 69 |
+
int(row.get("frames_observed") or fallback)
|
| 70 |
+
for row, fallback in zip(candidates, (4, 20, 32), strict=True)
|
| 71 |
+
),
|
| 72 |
+
"tvs_oracle": {
|
| 73 |
+
"analysis_only": True,
|
| 74 |
+
"gold_available_to_selector": True,
|
| 75 |
+
"correct_proposals": correct,
|
| 76 |
+
"chosen_view": ("recent", "event", "history")[chosen_index],
|
| 77 |
+
},
|
| 78 |
+
}
|
| 79 |
+
)
|
| 80 |
+
rows.append(output)
|
| 81 |
+
return {
|
| 82 |
+
**{field: recent.get(field) for field in IDENTITY_FIELDS},
|
| 83 |
+
"format": "iclr_video.strict_three_view_oracle.v1",
|
| 84 |
+
"status": "complete",
|
| 85 |
+
"analysis_only": True,
|
| 86 |
+
"num_samples": len(rows),
|
| 87 |
+
"num_errors": 0,
|
| 88 |
+
"mean_model_calls": 3.0,
|
| 89 |
+
"mean_processed_frames": fmean(row["model_processed_frames"] for row in rows),
|
| 90 |
+
"chosen_view_counts": dict(zip(("recent", "event", "history"), chosen_counts, strict=True)),
|
| 91 |
+
"rows": rows,
|
| 92 |
+
}
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
def main() -> None:
|
| 96 |
+
parser = argparse.ArgumentParser()
|
| 97 |
+
parser.add_argument("--benchmark", choices=("streamingbench", "ovo"), required=True)
|
| 98 |
+
parser.add_argument("--annotation", type=Path, required=True)
|
| 99 |
+
parser.add_argument("--recent", type=Path, required=True)
|
| 100 |
+
parser.add_argument("--event", type=Path, required=True)
|
| 101 |
+
parser.add_argument("--history", type=Path, required=True)
|
| 102 |
+
parser.add_argument("--output", type=Path, required=True)
|
| 103 |
+
args = parser.parse_args()
|
| 104 |
+
if args.output.exists():
|
| 105 |
+
raise FileExistsError(args.output)
|
| 106 |
+
values = [json.loads(path.read_text()) for path in (args.recent, args.event, args.history)]
|
| 107 |
+
result = materialize(*values, load_gold(args.benchmark, args.annotation))
|
| 108 |
+
args.output.parent.mkdir(parents=True, exist_ok=True)
|
| 109 |
+
args.output.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n")
|
| 110 |
+
print(json.dumps({
|
| 111 |
+
"rows": result["num_samples"],
|
| 112 |
+
"mean_model_calls": result["mean_model_calls"],
|
| 113 |
+
"mean_processed_frames": result["mean_processed_frames"],
|
| 114 |
+
}, sort_keys=True))
|
| 115 |
+
|
| 116 |
+
|
| 117 |
+
if __name__ == "__main__":
|
| 118 |
+
main()
|
code/experiments/materialize_uncertainty_temporal_agent.py
ADDED
|
@@ -0,0 +1,465 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Materialize one full prediction set from recent and triggered temporal calls."""
|
| 3 |
+
|
| 4 |
+
from __future__ import annotations
|
| 5 |
+
|
| 6 |
+
import argparse
|
| 7 |
+
import json
|
| 8 |
+
from pathlib import Path
|
| 9 |
+
from statistics import fmean
|
| 10 |
+
from typing import Any
|
| 11 |
+
|
| 12 |
+
from experiments.evaluate_qwen3vl_recent4_full import atomic_write
|
| 13 |
+
from experiments.analyze_uncertainty_temporal_escalation import choice_margin
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
def parse_args() -> argparse.Namespace:
|
| 17 |
+
parser = argparse.ArgumentParser(description=__doc__)
|
| 18 |
+
parser.add_argument("--recent", type=Path, required=True)
|
| 19 |
+
parser.add_argument("--temporal", type=Path, required=True)
|
| 20 |
+
parser.add_argument("--selection-audit", type=Path, required=True)
|
| 21 |
+
parser.add_argument("--threshold-audit", type=Path)
|
| 22 |
+
parser.add_argument(
|
| 23 |
+
"--temporal-model-identity-sha256",
|
| 24 |
+
help=(
|
| 25 |
+
"Previously sealed model identity to attach when the temporal result "
|
| 26 |
+
"was launched without embedding it; avoids rehashing the model."
|
| 27 |
+
),
|
| 28 |
+
)
|
| 29 |
+
parser.add_argument(
|
| 30 |
+
"--commit-threshold",
|
| 31 |
+
type=float,
|
| 32 |
+
help=(
|
| 33 |
+
"Optional fixed proposal-minus-current margin threshold. This is "
|
| 34 |
+
"required for an unlabeled fixed-budget selection and overrides "
|
| 35 |
+
"the commit threshold stored in an internal calibration."
|
| 36 |
+
),
|
| 37 |
+
)
|
| 38 |
+
parser.add_argument(
|
| 39 |
+
"--fallback-recent-on-temporal-error",
|
| 40 |
+
action="store_true",
|
| 41 |
+
help=(
|
| 42 |
+
"Fail closed to the aligned recent4 prediction when a preselected "
|
| 43 |
+
"temporal call has a recorded runtime error. The fallback is fixed "
|
| 44 |
+
"before scoring and never reads benchmark labels."
|
| 45 |
+
),
|
| 46 |
+
)
|
| 47 |
+
parser.add_argument("--output", type=Path, required=True)
|
| 48 |
+
return parser.parse_args()
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
def selected_view_name(policy: str, mode: str, *, escalated: bool) -> str:
|
| 52 |
+
if not escalated:
|
| 53 |
+
return "recent4"
|
| 54 |
+
if policy in {"causalbudget20", "causalbudget32r8"}:
|
| 55 |
+
suffix = "32_recent8" if policy == "causalbudget32r8" else "20"
|
| 56 |
+
mode_name = {
|
| 57 |
+
"native-video": "native_video",
|
| 58 |
+
"image-list": "image_list",
|
| 59 |
+
"hybrid-history-video-recent-images": (
|
| 60 |
+
"hybrid_history_video_recent_images"
|
| 61 |
+
),
|
| 62 |
+
"lowres-history-recent-images": "lowres_history_recent_images",
|
| 63 |
+
}[mode]
|
| 64 |
+
return f"causal_temporal{suffix}_{mode_name}"
|
| 65 |
+
if policy in {"causalhybrid32", "causalhybrid32r8"}:
|
| 66 |
+
recent_frames = 8 if policy == "causalhybrid32r8" else 4
|
| 67 |
+
history_frames = 32 - recent_frames
|
| 68 |
+
if mode == "image-list":
|
| 69 |
+
return (
|
| 70 |
+
f"causal_uniform_history{history_frames}_plus_recent"
|
| 71 |
+
f"{recent_frames}_image_list"
|
| 72 |
+
)
|
| 73 |
+
return f"causal_hybrid_history{history_frames}_plus_recent{recent_frames}"
|
| 74 |
+
if policy == "prefix32":
|
| 75 |
+
return "causal_prefix32"
|
| 76 |
+
raise ValueError(f"unsupported second-view policy: {policy}")
|
| 77 |
+
|
| 78 |
+
|
| 79 |
+
def second_view_policy_name(policy: str, mode: str) -> str:
|
| 80 |
+
if policy in {"causalbudget20", "causalbudget32r8"}:
|
| 81 |
+
suffix = "32_recent8" if policy == "causalbudget32r8" else "20"
|
| 82 |
+
prefix = {
|
| 83 |
+
"native-video": "native_video",
|
| 84 |
+
"image-list": "image_list",
|
| 85 |
+
"hybrid-history-video-recent-images": (
|
| 86 |
+
"hybrid_history_video_recent_images"
|
| 87 |
+
),
|
| 88 |
+
"lowres-history-recent-images": "lowres_history_recent_images",
|
| 89 |
+
}[mode]
|
| 90 |
+
return f"{prefix}_causal_temporal{suffix}"
|
| 91 |
+
if policy in {"causalhybrid32", "causalhybrid32r8"}:
|
| 92 |
+
if mode == "hybrid-history-video-recent-images":
|
| 93 |
+
return (
|
| 94 |
+
"hybrid_history_video_recent_images_causal_hybrid32r8"
|
| 95 |
+
if policy == "causalhybrid32r8"
|
| 96 |
+
else "hybrid_history_video_recent_images_causal_hybrid32"
|
| 97 |
+
)
|
| 98 |
+
return (
|
| 99 |
+
"image_list_uniform_history24_plus_recent8"
|
| 100 |
+
if policy == "causalhybrid32r8"
|
| 101 |
+
else "image_list_uniform_history28_plus_recent4"
|
| 102 |
+
)
|
| 103 |
+
if policy == "prefix32":
|
| 104 |
+
return "image_list_causal_prefix32"
|
| 105 |
+
raise ValueError(f"unsupported second-view policy: {policy}")
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
def materialize(
|
| 109 |
+
recent: dict[str, Any],
|
| 110 |
+
temporal: dict[str, Any],
|
| 111 |
+
selection: dict[str, Any],
|
| 112 |
+
threshold: dict[str, Any] | None,
|
| 113 |
+
*,
|
| 114 |
+
commit_threshold_override: float | None = None,
|
| 115 |
+
fallback_recent_on_temporal_error: bool = False,
|
| 116 |
+
) -> dict[str, Any]:
|
| 117 |
+
for field in (
|
| 118 |
+
"benchmark",
|
| 119 |
+
"model_manifest_sha256",
|
| 120 |
+
"annotation_sha256",
|
| 121 |
+
"evaluation_protocol_sha256",
|
| 122 |
+
):
|
| 123 |
+
if recent.get(field) != temporal.get(field):
|
| 124 |
+
raise ValueError(f"recent/temporal identity mismatch: {field}")
|
| 125 |
+
selection_format = selection.get("format")
|
| 126 |
+
fixed_quantile = None
|
| 127 |
+
frozen_threshold = None
|
| 128 |
+
if threshold is not None:
|
| 129 |
+
if (
|
| 130 |
+
threshold.get("status")
|
| 131 |
+
!= "source_disjoint_internal_threshold_calibration"
|
| 132 |
+
or threshold.get("eligible_as_frozen_benchmark_controller_config")
|
| 133 |
+
is not True
|
| 134 |
+
or threshold.get("model_manifest_sha256")
|
| 135 |
+
!= recent.get("model_manifest_sha256")
|
| 136 |
+
):
|
| 137 |
+
raise ValueError("threshold is not an eligible internal calibration")
|
| 138 |
+
frozen_threshold = float(threshold["policy"]["threshold"])
|
| 139 |
+
commit_threshold = threshold["policy"].get("commit_threshold")
|
| 140 |
+
if selection_format not in (None, "streameva.internal_threshold_uid_selection.v0"):
|
| 141 |
+
raise ValueError("threshold calibration and selection mode diverge")
|
| 142 |
+
else:
|
| 143 |
+
if selection_format != "streameva.unlabeled_budget_uid_selection.v0":
|
| 144 |
+
raise ValueError("selection is not an unlabeled fixed-budget audit")
|
| 145 |
+
fixed_quantile = float(selection.get("quantile"))
|
| 146 |
+
if not 0.0 < fixed_quantile <= 1.0:
|
| 147 |
+
raise ValueError("fixed uncertainty quantile is invalid")
|
| 148 |
+
if selection.get("benchmark_identity_used_for_policy_parameters") is not False:
|
| 149 |
+
raise ValueError("fixed-budget acquisition used benchmark identity")
|
| 150 |
+
if selection.get("cohort_labels_read_for_selection") is not False:
|
| 151 |
+
raise ValueError("fixed-budget acquisition used cohort labels")
|
| 152 |
+
if commit_threshold_override is None:
|
| 153 |
+
raise ValueError("fixed-budget selection requires a commit threshold")
|
| 154 |
+
commit_threshold = commit_threshold_override
|
| 155 |
+
if commit_threshold_override is not None:
|
| 156 |
+
commit_threshold = commit_threshold_override
|
| 157 |
+
if commit_threshold is not None:
|
| 158 |
+
commit_threshold = float(commit_threshold)
|
| 159 |
+
if (
|
| 160 |
+
selection.get("status")
|
| 161 |
+
!= "frozen_before_benchmark_temporal_inference"
|
| 162 |
+
or selection.get("benchmark") != recent.get("benchmark")
|
| 163 |
+
or selection.get("model_manifest_sha256")
|
| 164 |
+
!= recent.get("model_manifest_sha256")
|
| 165 |
+
or selection.get("score_or_gold_fields_read_for_selection") is not False
|
| 166 |
+
):
|
| 167 |
+
raise ValueError("selection audit is not frozen or identity-aligned")
|
| 168 |
+
if threshold is not None and float(selection.get("threshold")) != frozen_threshold:
|
| 169 |
+
raise ValueError("selection threshold diverges from internal calibration")
|
| 170 |
+
|
| 171 |
+
recent_rows = {str(row["uid"]): row for row in recent.get("rows") or []}
|
| 172 |
+
temporal_rows = {str(row["uid"]): row for row in temporal.get("rows") or []}
|
| 173 |
+
decisions = {str(row["uid"]): row for row in selection.get("rows") or []}
|
| 174 |
+
if (
|
| 175 |
+
len(recent_rows) != len(recent.get("rows") or [])
|
| 176 |
+
or len(temporal_rows) != len(temporal.get("rows") or [])
|
| 177 |
+
or len(decisions) != len(selection.get("rows") or [])
|
| 178 |
+
):
|
| 179 |
+
raise ValueError("duplicate UID in input artifacts")
|
| 180 |
+
if set(decisions) != set(recent_rows):
|
| 181 |
+
raise ValueError("selection audit does not cover the recent result")
|
| 182 |
+
selected = {uid for uid, row in decisions.items() if bool(row["escalated"])}
|
| 183 |
+
temporal_input_count = len(temporal_rows)
|
| 184 |
+
if not selected.issubset(temporal_rows):
|
| 185 |
+
missing = sorted(selected - set(temporal_rows))
|
| 186 |
+
raise ValueError(
|
| 187 |
+
f"temporal inference is missing {len(missing)} selected UIDs"
|
| 188 |
+
)
|
| 189 |
+
temporal_rows = {uid: temporal_rows[uid] for uid in selected}
|
| 190 |
+
temporal_error_uids = {
|
| 191 |
+
uid for uid, row in temporal_rows.items() if bool(row.get("error"))
|
| 192 |
+
}
|
| 193 |
+
if temporal_error_uids and not fallback_recent_on_temporal_error:
|
| 194 |
+
raise ValueError(
|
| 195 |
+
f"{len(temporal_error_uids)} selected temporal rows contain errors"
|
| 196 |
+
)
|
| 197 |
+
valid_temporal_rows = {
|
| 198 |
+
uid: row for uid, row in temporal_rows.items() if uid not in temporal_error_uids
|
| 199 |
+
}
|
| 200 |
+
if not valid_temporal_rows:
|
| 201 |
+
raise ValueError("no valid temporal row remains to establish the second-view policy")
|
| 202 |
+
second_view_policies = {
|
| 203 |
+
str((row.get("sampling") or {}).get("frame_policy") or "")
|
| 204 |
+
for row in valid_temporal_rows.values()
|
| 205 |
+
}
|
| 206 |
+
if second_view_policies not in (
|
| 207 |
+
{"causalbudget20"},
|
| 208 |
+
{"causalbudget32r8"},
|
| 209 |
+
{"prefix32"},
|
| 210 |
+
{"causalhybrid32"},
|
| 211 |
+
{"causalhybrid32r8"},
|
| 212 |
+
):
|
| 213 |
+
raise ValueError(
|
| 214 |
+
"second-view rows must share causalbudget20, causalbudget32r8, prefix32, or "
|
| 215 |
+
"causalhybrid32*"
|
| 216 |
+
)
|
| 217 |
+
second_view_policy = next(iter(second_view_policies))
|
| 218 |
+
second_view_modes = {
|
| 219 |
+
str(row.get("visual_input_mode") or "")
|
| 220 |
+
for row in valid_temporal_rows.values()
|
| 221 |
+
}
|
| 222 |
+
if len(second_view_modes) != 1:
|
| 223 |
+
raise ValueError("second-view rows do not share one visual input mode")
|
| 224 |
+
second_view_mode = next(iter(second_view_modes))
|
| 225 |
+
if second_view_mode not in {
|
| 226 |
+
"native-video",
|
| 227 |
+
"hybrid-history-video-recent-images",
|
| 228 |
+
"lowres-history-recent-images",
|
| 229 |
+
"image-list",
|
| 230 |
+
}:
|
| 231 |
+
raise ValueError("second-view visual input mode is missing or unsupported")
|
| 232 |
+
|
| 233 |
+
rows = []
|
| 234 |
+
for recent_row in recent.get("rows") or []:
|
| 235 |
+
uid = str(recent_row["uid"])
|
| 236 |
+
decision = decisions[uid]
|
| 237 |
+
escalated = bool(decision["escalated"])
|
| 238 |
+
temporal_error_fallback = escalated and uid in temporal_error_uids
|
| 239 |
+
proposal = (
|
| 240 |
+
recent_row
|
| 241 |
+
if temporal_error_fallback or not escalated
|
| 242 |
+
else temporal_rows[uid]
|
| 243 |
+
)
|
| 244 |
+
sampling = proposal.get("sampling") or {}
|
| 245 |
+
if sampling.get("frame_timestamps") and max(sampling["frame_timestamps"]) > float(
|
| 246 |
+
proposal["query_time"]
|
| 247 |
+
) + 1e-3:
|
| 248 |
+
raise ValueError(f"{uid}: selected inference crosses the causal frontier")
|
| 249 |
+
recent_frames = int(recent_row.get("frames_observed") or 4)
|
| 250 |
+
chosen_frames = int(proposal.get("frames_observed") or recent_frames)
|
| 251 |
+
probe_frames = 0
|
| 252 |
+
extra_probe_frames = 0
|
| 253 |
+
if escalated and not temporal_error_fallback:
|
| 254 |
+
if sampling.get("frame_policy") != second_view_policy:
|
| 255 |
+
raise ValueError(f"{uid}: second-view policy diverged")
|
| 256 |
+
if second_view_policy in {"causalbudget20", "causalbudget32r8"}:
|
| 257 |
+
candidate_indices = sampling.get("candidate_frame_indices")
|
| 258 |
+
if not isinstance(candidate_indices, list) or not candidate_indices:
|
| 259 |
+
raise ValueError(f"{uid}: temporal row lacks probe-frame audit")
|
| 260 |
+
probe_frames = len(candidate_indices)
|
| 261 |
+
extra_probe_frames = int(sampling.get("extra_frames_inspected", -1))
|
| 262 |
+
if extra_probe_frames != max(0, probe_frames - chosen_frames):
|
| 263 |
+
raise ValueError(f"{uid}: temporal probe accounting diverged")
|
| 264 |
+
elif second_view_policy == "prefix32" and chosen_frames != 32:
|
| 265 |
+
raise ValueError(f"{uid}: prefix32 row does not expose 32 frames")
|
| 266 |
+
elif second_view_policy in {
|
| 267 |
+
"causalhybrid32",
|
| 268 |
+
"causalhybrid32r8",
|
| 269 |
+
} and not (
|
| 270 |
+
1 <= chosen_frames <= 32
|
| 271 |
+
):
|
| 272 |
+
raise ValueError(f"{uid}: causalhybrid32* frame budget is invalid")
|
| 273 |
+
recent_prediction = str(recent_row.get("prediction") or "")
|
| 274 |
+
proposal_prediction = str(proposal.get("prediction") or "")
|
| 275 |
+
confidence_gain = None
|
| 276 |
+
committed = escalated and not temporal_error_fallback
|
| 277 |
+
if escalated and not temporal_error_fallback and commit_threshold is not None:
|
| 278 |
+
confidence_gain = choice_margin(proposal) - choice_margin(recent_row)
|
| 279 |
+
committed = bool(
|
| 280 |
+
proposal_prediction != recent_prediction
|
| 281 |
+
and confidence_gain > commit_threshold
|
| 282 |
+
)
|
| 283 |
+
final_prediction = proposal_prediction if committed else recent_prediction
|
| 284 |
+
rows.append(
|
| 285 |
+
{
|
| 286 |
+
**proposal,
|
| 287 |
+
"uid": uid,
|
| 288 |
+
"prediction": final_prediction,
|
| 289 |
+
"current_prediction": recent_prediction,
|
| 290 |
+
"proposal_prediction": proposal_prediction if escalated else None,
|
| 291 |
+
"agent_policy": {
|
| 292 |
+
"margin": float(decision["margin"]),
|
| 293 |
+
"threshold": frozen_threshold,
|
| 294 |
+
"uncertainty_quantile": fixed_quantile,
|
| 295 |
+
"uncertainty_rank": decision.get("uncertainty_rank"),
|
| 296 |
+
"escalated": escalated,
|
| 297 |
+
"temporal_error_fallback": temporal_error_fallback,
|
| 298 |
+
"temporal_error": (
|
| 299 |
+
str(temporal_rows[uid].get("error"))
|
| 300 |
+
if temporal_error_fallback
|
| 301 |
+
else None
|
| 302 |
+
),
|
| 303 |
+
"commit_threshold": commit_threshold,
|
| 304 |
+
"confidence_gain": confidence_gain,
|
| 305 |
+
"committed": committed,
|
| 306 |
+
"selected_view": (
|
| 307 |
+
"recent4_temporal_error_fallback"
|
| 308 |
+
if temporal_error_fallback
|
| 309 |
+
else selected_view_name(
|
| 310 |
+
second_view_policy, second_view_mode, escalated=escalated
|
| 311 |
+
)
|
| 312 |
+
),
|
| 313 |
+
"question_task_group_or_benchmark_router": False,
|
| 314 |
+
},
|
| 315 |
+
"unique_frames": chosen_frames,
|
| 316 |
+
"model_processed_frames": (
|
| 317 |
+
recent_frames + chosen_frames
|
| 318 |
+
if escalated and not temporal_error_fallback
|
| 319 |
+
else recent_frames
|
| 320 |
+
),
|
| 321 |
+
"low_resolution_probe_frames": probe_frames,
|
| 322 |
+
"extra_probe_frames_not_visible_to_vlm": extra_probe_frames,
|
| 323 |
+
"total_decoded_frame_instances": (
|
| 324 |
+
recent_frames + probe_frames + chosen_frames
|
| 325 |
+
if escalated and not temporal_error_fallback
|
| 326 |
+
else recent_frames
|
| 327 |
+
),
|
| 328 |
+
"media_calls": 2 if escalated else 1,
|
| 329 |
+
"num_turns": 2 if escalated else 1,
|
| 330 |
+
"latency_seconds": float(recent_row.get("latency_seconds") or 0.0)
|
| 331 |
+
+ (
|
| 332 |
+
float(temporal_rows[uid].get("latency_seconds") or 0.0)
|
| 333 |
+
if escalated
|
| 334 |
+
else 0.0
|
| 335 |
+
),
|
| 336 |
+
}
|
| 337 |
+
)
|
| 338 |
+
if any(not row["prediction"] for row in rows):
|
| 339 |
+
raise ValueError("materialized agent contains an empty prediction")
|
| 340 |
+
escalated_count = len(selected)
|
| 341 |
+
return {
|
| 342 |
+
"format": "streameva.uncertainty_temporal_agent_eval.v0",
|
| 343 |
+
"status": (
|
| 344 |
+
"internally_calibrated_router_free_agent"
|
| 345 |
+
if threshold is not None
|
| 346 |
+
else "fixed_budget_router_free_agent"
|
| 347 |
+
),
|
| 348 |
+
"benchmark": recent.get("benchmark"),
|
| 349 |
+
"model": recent.get("model"),
|
| 350 |
+
"model_manifest_sha256": recent.get("model_manifest_sha256"),
|
| 351 |
+
"processor": recent.get("processor"),
|
| 352 |
+
"annotation": recent.get("annotation"),
|
| 353 |
+
"annotation_sha256": recent.get("annotation_sha256"),
|
| 354 |
+
"evaluation_protocol": recent.get("evaluation_protocol"),
|
| 355 |
+
"evaluation_protocol_sha256": recent.get("evaluation_protocol_sha256"),
|
| 356 |
+
"comparable_under_locked_recent4_protocol": False,
|
| 357 |
+
"policy": {
|
| 358 |
+
"first_view": "recent4",
|
| 359 |
+
"uncertainty": "first_step_full_vocabulary_top2_margin",
|
| 360 |
+
"threshold": frozen_threshold,
|
| 361 |
+
"uncertainty_quantile": fixed_quantile,
|
| 362 |
+
"commit_threshold": commit_threshold,
|
| 363 |
+
"second_view": second_view_policy_name(
|
| 364 |
+
second_view_policy, second_view_mode
|
| 365 |
+
),
|
| 366 |
+
"commit_rule": (
|
| 367 |
+
"changed_answer_and_confidence_gain_gt_global_threshold"
|
| 368 |
+
if commit_threshold is not None
|
| 369 |
+
else "legacy_always_accept_temporal_proposal"
|
| 370 |
+
),
|
| 371 |
+
"question_task_group_or_benchmark_router": False,
|
| 372 |
+
"benchmark_identity_available_to_policy": False,
|
| 373 |
+
"gold_available_to_policy": False,
|
| 374 |
+
"temporal_error_policy": (
|
| 375 |
+
"fail_closed_to_recent4"
|
| 376 |
+
if fallback_recent_on_temporal_error
|
| 377 |
+
else "reject_materialization"
|
| 378 |
+
),
|
| 379 |
+
},
|
| 380 |
+
"num_samples": len(rows),
|
| 381 |
+
"num_errors": 0,
|
| 382 |
+
"escalated": escalated_count,
|
| 383 |
+
"temporal_input_num_rows": temporal_input_count,
|
| 384 |
+
"temporal_unused_num_rows": temporal_input_count - escalated_count,
|
| 385 |
+
"temporal_error_fallback_count": len(temporal_error_uids),
|
| 386 |
+
"escalation_rate": escalated_count / len(rows),
|
| 387 |
+
"mean_model_calls": 1.0 + escalated_count / len(rows),
|
| 388 |
+
"mean_unique_frames": fmean(row["unique_frames"] for row in rows),
|
| 389 |
+
"mean_model_processed_frames": fmean(
|
| 390 |
+
row["model_processed_frames"] for row in rows
|
| 391 |
+
),
|
| 392 |
+
"mean_low_resolution_probe_frames": fmean(
|
| 393 |
+
row["low_resolution_probe_frames"] for row in rows
|
| 394 |
+
),
|
| 395 |
+
"mean_extra_probe_frames_not_visible_to_vlm": fmean(
|
| 396 |
+
row["extra_probe_frames_not_visible_to_vlm"] for row in rows
|
| 397 |
+
),
|
| 398 |
+
"mean_total_decoded_frame_instances": fmean(
|
| 399 |
+
row["total_decoded_frame_instances"] for row in rows
|
| 400 |
+
),
|
| 401 |
+
"rows": rows,
|
| 402 |
+
}
|
| 403 |
+
|
| 404 |
+
|
| 405 |
+
def main() -> None:
|
| 406 |
+
args = parse_args()
|
| 407 |
+
recent = json.loads(args.recent.read_text(encoding="utf-8"))
|
| 408 |
+
temporal = json.loads(args.temporal.read_text(encoding="utf-8"))
|
| 409 |
+
identity_override_applied = False
|
| 410 |
+
if args.temporal_model_identity_sha256 is not None:
|
| 411 |
+
identity = args.temporal_model_identity_sha256.strip().lower()
|
| 412 |
+
if len(identity) != 64 or any(
|
| 413 |
+
character not in "0123456789abcdef" for character in identity
|
| 414 |
+
):
|
| 415 |
+
raise ValueError("temporal model identity must be a 64-character SHA-256")
|
| 416 |
+
embedded_identity = temporal.get("model_manifest_sha256")
|
| 417 |
+
if embedded_identity not in (None, "", identity):
|
| 418 |
+
raise ValueError("temporal model identity override conflicts with artifact")
|
| 419 |
+
if embedded_identity in (None, ""):
|
| 420 |
+
temporal["model_manifest_sha256"] = identity
|
| 421 |
+
identity_override_applied = True
|
| 422 |
+
selection = json.loads(args.selection_audit.read_text(encoding="utf-8"))
|
| 423 |
+
threshold = (
|
| 424 |
+
json.loads(args.threshold_audit.read_text(encoding="utf-8"))
|
| 425 |
+
if args.threshold_audit is not None
|
| 426 |
+
else None
|
| 427 |
+
)
|
| 428 |
+
output = materialize(
|
| 429 |
+
recent,
|
| 430 |
+
temporal,
|
| 431 |
+
selection,
|
| 432 |
+
threshold,
|
| 433 |
+
commit_threshold_override=args.commit_threshold,
|
| 434 |
+
fallback_recent_on_temporal_error=args.fallback_recent_on_temporal_error,
|
| 435 |
+
)
|
| 436 |
+
output["artifacts"] = {
|
| 437 |
+
"recent": {"path": str(args.recent.resolve())},
|
| 438 |
+
"temporal": {"path": str(args.temporal.resolve())},
|
| 439 |
+
"selection_audit": {"path": str(args.selection_audit.resolve())},
|
| 440 |
+
}
|
| 441 |
+
if args.threshold_audit is not None:
|
| 442 |
+
output["artifacts"]["threshold_audit"] = {
|
| 443 |
+
"path": str(args.threshold_audit.resolve()),
|
| 444 |
+
}
|
| 445 |
+
output["artifact_checksums_skipped"] = True
|
| 446 |
+
output["artifact_checksum_skip_reason"] = (
|
| 447 |
+
"native artifacts succeeded and identities are embedded in their manifests"
|
| 448 |
+
)
|
| 449 |
+
output["temporal_model_identity_override_applied"] = identity_override_applied
|
| 450 |
+
atomic_write(args.output, output)
|
| 451 |
+
print(
|
| 452 |
+
json.dumps(
|
| 453 |
+
{
|
| 454 |
+
"benchmark": output["benchmark"],
|
| 455 |
+
"num_samples": output["num_samples"],
|
| 456 |
+
"escalated": output["escalated"],
|
| 457 |
+
"mean_model_calls": output["mean_model_calls"],
|
| 458 |
+
},
|
| 459 |
+
sort_keys=True,
|
| 460 |
+
)
|
| 461 |
+
)
|
| 462 |
+
|
| 463 |
+
|
| 464 |
+
if __name__ == "__main__":
|
| 465 |
+
main()
|
code/experiments/select_uncertainty_escalation_uids.py
ADDED
|
@@ -0,0 +1,192 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Select temporal-view calls using a threshold or fixed uncertainty budget."""
|
| 3 |
+
|
| 4 |
+
from __future__ import annotations
|
| 5 |
+
|
| 6 |
+
import argparse
|
| 7 |
+
import json
|
| 8 |
+
import math
|
| 9 |
+
from pathlib import Path
|
| 10 |
+
from typing import Any
|
| 11 |
+
|
| 12 |
+
from experiments.analyze_uncertainty_temporal_escalation import choice_margin
|
| 13 |
+
from experiments.evaluate_qwen3vl_recent4_full import atomic_write, sha256_file
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
def parse_args() -> argparse.Namespace:
|
| 17 |
+
parser = argparse.ArgumentParser(description=__doc__)
|
| 18 |
+
parser.add_argument("--recent", type=Path, required=True)
|
| 19 |
+
policy = parser.add_mutually_exclusive_group(required=True)
|
| 20 |
+
policy.add_argument("--threshold", type=Path)
|
| 21 |
+
policy.add_argument(
|
| 22 |
+
"--quantile",
|
| 23 |
+
type=float,
|
| 24 |
+
help="Acquire the lowest-margin fraction without reading cohort labels.",
|
| 25 |
+
)
|
| 26 |
+
parser.add_argument("--samples", type=Path, required=True)
|
| 27 |
+
parser.add_argument("--output-uids", type=Path, required=True)
|
| 28 |
+
parser.add_argument("--output-samples", type=Path, required=True)
|
| 29 |
+
parser.add_argument("--output-audit", type=Path, required=True)
|
| 30 |
+
return parser.parse_args()
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
def load_frozen_threshold(path: Path, model_manifest_sha256: str | None) -> float:
|
| 34 |
+
value = json.loads(path.read_text(encoding="utf-8"))
|
| 35 |
+
if (
|
| 36 |
+
value.get("status") != "source_disjoint_internal_threshold_calibration"
|
| 37 |
+
or value.get("eligible_as_frozen_benchmark_controller_config") is not True
|
| 38 |
+
or value.get("benchmark") != "internal_policy_holdout"
|
| 39 |
+
or value.get("model_manifest_sha256") != model_manifest_sha256
|
| 40 |
+
):
|
| 41 |
+
raise ValueError("threshold is not an eligible source-disjoint calibration")
|
| 42 |
+
threshold = float(value["policy"]["threshold"])
|
| 43 |
+
if not 0.0 <= threshold <= 1.0:
|
| 44 |
+
raise ValueError("invalid calibrated threshold")
|
| 45 |
+
return threshold
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
def select_rows(
|
| 49 |
+
recent: dict[str, Any], *, threshold: float
|
| 50 |
+
) -> tuple[list[str], list[dict[str, Any]]]:
|
| 51 |
+
selected = []
|
| 52 |
+
audit_rows = []
|
| 53 |
+
for row in recent.get("rows") or []:
|
| 54 |
+
uid = str(row.get("uid") or "")
|
| 55 |
+
if not uid:
|
| 56 |
+
raise ValueError("recent result has a row without uid")
|
| 57 |
+
margin = choice_margin(row)
|
| 58 |
+
escalated = margin <= threshold
|
| 59 |
+
if escalated:
|
| 60 |
+
selected.append(uid)
|
| 61 |
+
audit_rows.append(
|
| 62 |
+
{
|
| 63 |
+
"uid": uid,
|
| 64 |
+
"margin": margin,
|
| 65 |
+
"threshold": threshold,
|
| 66 |
+
"escalated": escalated,
|
| 67 |
+
}
|
| 68 |
+
)
|
| 69 |
+
if len(audit_rows) != int(recent.get("num_samples", -1)):
|
| 70 |
+
raise ValueError("recent result sample count diverged")
|
| 71 |
+
if int(recent.get("num_errors", -1)) != 0:
|
| 72 |
+
raise ValueError("recent result contains inference errors")
|
| 73 |
+
if len({row["uid"] for row in audit_rows}) != len(audit_rows):
|
| 74 |
+
raise ValueError("recent result contains duplicate uids")
|
| 75 |
+
return selected, audit_rows
|
| 76 |
+
|
| 77 |
+
|
| 78 |
+
def select_rows_by_quantile(
|
| 79 |
+
recent: dict[str, Any], *, quantile: float
|
| 80 |
+
) -> tuple[list[str], list[dict[str, Any]], float]:
|
| 81 |
+
"""Select an exact bottom-margin budget with deterministic UID tie-breaking."""
|
| 82 |
+
|
| 83 |
+
if not 0.0 < quantile <= 1.0:
|
| 84 |
+
raise ValueError("quantile must be in (0, 1]")
|
| 85 |
+
rows = list(recent.get("rows") or [])
|
| 86 |
+
if len(rows) != int(recent.get("num_samples", -1)):
|
| 87 |
+
raise ValueError("recent result sample count diverged")
|
| 88 |
+
if int(recent.get("num_errors", -1)) != 0:
|
| 89 |
+
raise ValueError("recent result contains inference errors")
|
| 90 |
+
values = []
|
| 91 |
+
for row in rows:
|
| 92 |
+
uid = str(row.get("uid") or "")
|
| 93 |
+
if not uid:
|
| 94 |
+
raise ValueError("recent result has a row without uid")
|
| 95 |
+
values.append((choice_margin(row), uid))
|
| 96 |
+
if len({uid for _, uid in values}) != len(values):
|
| 97 |
+
raise ValueError("recent result contains duplicate uids")
|
| 98 |
+
budget = max(1, math.ceil(len(values) * quantile))
|
| 99 |
+
ranked = sorted(values, key=lambda item: (item[0], item[1]))
|
| 100 |
+
selected_set = {uid for _, uid in ranked[:budget]}
|
| 101 |
+
cutoff = float(ranked[budget - 1][0])
|
| 102 |
+
selected = [str(row["uid"]) for row in rows if str(row["uid"]) in selected_set]
|
| 103 |
+
rank_by_uid = {uid: rank for rank, (_, uid) in enumerate(ranked, 1)}
|
| 104 |
+
audit_rows = [
|
| 105 |
+
{
|
| 106 |
+
"uid": str(row["uid"]),
|
| 107 |
+
"margin": choice_margin(row),
|
| 108 |
+
"uncertainty_rank": rank_by_uid[str(row["uid"])],
|
| 109 |
+
"quantile": quantile,
|
| 110 |
+
"escalated": str(row["uid"]) in selected_set,
|
| 111 |
+
}
|
| 112 |
+
for row in rows
|
| 113 |
+
]
|
| 114 |
+
return selected, audit_rows, cutoff
|
| 115 |
+
|
| 116 |
+
|
| 117 |
+
def main() -> None:
|
| 118 |
+
args = parse_args()
|
| 119 |
+
recent = json.loads(args.recent.read_text(encoding="utf-8"))
|
| 120 |
+
threshold = None
|
| 121 |
+
quantile = None
|
| 122 |
+
cutoff = None
|
| 123 |
+
if args.threshold is not None:
|
| 124 |
+
threshold = load_frozen_threshold(
|
| 125 |
+
args.threshold, recent.get("model_manifest_sha256")
|
| 126 |
+
)
|
| 127 |
+
selected, audit_rows = select_rows(recent, threshold=threshold)
|
| 128 |
+
else:
|
| 129 |
+
quantile = float(args.quantile)
|
| 130 |
+
selected, audit_rows, cutoff = select_rows_by_quantile(
|
| 131 |
+
recent, quantile=quantile
|
| 132 |
+
)
|
| 133 |
+
selected_set = set(selected)
|
| 134 |
+
sample_rows = [
|
| 135 |
+
json.loads(line)
|
| 136 |
+
for line in args.samples.read_text(encoding="utf-8").splitlines()
|
| 137 |
+
if line.strip()
|
| 138 |
+
]
|
| 139 |
+
ordered_uids = [str(row.get("uid") or "") for row in sample_rows]
|
| 140 |
+
recent_uids = [str(row["uid"]) for row in recent["rows"]]
|
| 141 |
+
if ordered_uids != recent_uids:
|
| 142 |
+
raise ValueError("recent result does not follow the frozen sample order")
|
| 143 |
+
selected_samples = [row for row in sample_rows if str(row["uid"]) in selected_set]
|
| 144 |
+
if [str(row["uid"]) for row in selected_samples] != selected:
|
| 145 |
+
raise ValueError("selected UID ordering diverged")
|
| 146 |
+
args.output_uids.parent.mkdir(parents=True, exist_ok=True)
|
| 147 |
+
args.output_uids.write_text("\n".join(selected) + "\n", encoding="utf-8")
|
| 148 |
+
args.output_samples.write_text(
|
| 149 |
+
"".join(
|
| 150 |
+
json.dumps(row, sort_keys=True, ensure_ascii=False) + "\n"
|
| 151 |
+
for row in selected_samples
|
| 152 |
+
),
|
| 153 |
+
encoding="utf-8",
|
| 154 |
+
)
|
| 155 |
+
atomic_write(
|
| 156 |
+
args.output_audit,
|
| 157 |
+
{
|
| 158 |
+
"format": (
|
| 159 |
+
"streameva.internal_threshold_uid_selection.v0"
|
| 160 |
+
if threshold is not None
|
| 161 |
+
else "streameva.unlabeled_budget_uid_selection.v0"
|
| 162 |
+
),
|
| 163 |
+
"status": "frozen_before_benchmark_temporal_inference",
|
| 164 |
+
"benchmark": recent.get("benchmark"),
|
| 165 |
+
"model_manifest_sha256": recent.get("model_manifest_sha256"),
|
| 166 |
+
"threshold": threshold,
|
| 167 |
+
"quantile": quantile,
|
| 168 |
+
"empirical_margin_cutoff": cutoff,
|
| 169 |
+
"threshold_audit": (
|
| 170 |
+
str(args.threshold.resolve()) if args.threshold is not None else None
|
| 171 |
+
),
|
| 172 |
+
"threshold_audit_sha256": (
|
| 173 |
+
sha256_file(args.threshold) if args.threshold is not None else None
|
| 174 |
+
),
|
| 175 |
+
"recent_result": str(args.recent.resolve()),
|
| 176 |
+
"recent_result_sha256": sha256_file(args.recent),
|
| 177 |
+
"samples_sha256": sha256_file(args.samples),
|
| 178 |
+
"question_task_group_or_benchmark_router": False,
|
| 179 |
+
"score_or_gold_fields_read_for_selection": False,
|
| 180 |
+
"benchmark_identity_used_for_policy_parameters": False,
|
| 181 |
+
"cohort_labels_read_for_selection": False,
|
| 182 |
+
"total": len(audit_rows),
|
| 183 |
+
"selected": len(selected),
|
| 184 |
+
"selection_rate": len(selected) / len(audit_rows),
|
| 185 |
+
"rows": audit_rows,
|
| 186 |
+
},
|
| 187 |
+
)
|
| 188 |
+
print(json.dumps({"total": len(audit_rows), "selected": len(selected)}))
|
| 189 |
+
|
| 190 |
+
|
| 191 |
+
if __name__ == "__main__":
|
| 192 |
+
main()
|
code/experiments/train_qwen3vl_sft.py
ADDED
|
@@ -0,0 +1,1091 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Audited full-parameter Qwen3-VL trainer for the pure-VLM backbone."""
|
| 3 |
+
|
| 4 |
+
from __future__ import annotations
|
| 5 |
+
|
| 6 |
+
import argparse
|
| 7 |
+
import inspect
|
| 8 |
+
import json
|
| 9 |
+
import math
|
| 10 |
+
import os
|
| 11 |
+
import shutil
|
| 12 |
+
from dataclasses import dataclass
|
| 13 |
+
from pathlib import Path
|
| 14 |
+
from typing import Any, Dict, List
|
| 15 |
+
|
| 16 |
+
import torch
|
| 17 |
+
from PIL import Image
|
| 18 |
+
from torch.utils.data import Dataset, SequentialSampler
|
| 19 |
+
|
| 20 |
+
try:
|
| 21 |
+
from transformers import (
|
| 22 |
+
AutoModelForImageTextToText,
|
| 23 |
+
AutoProcessor,
|
| 24 |
+
Trainer,
|
| 25 |
+
TrainingArguments,
|
| 26 |
+
set_seed,
|
| 27 |
+
)
|
| 28 |
+
except ModuleNotFoundError:
|
| 29 |
+
AutoModelForImageTextToText = None
|
| 30 |
+
AutoProcessor = None
|
| 31 |
+
TrainingArguments = None
|
| 32 |
+
set_seed = None
|
| 33 |
+
|
| 34 |
+
class Trainer: # type: ignore[no-redef]
|
| 35 |
+
"""Import-only fallback for dependency-light contract tests."""
|
| 36 |
+
|
| 37 |
+
pass
|
| 38 |
+
|
| 39 |
+
from viss_core.data_v0.trainer_adapter import (
|
| 40 |
+
infer_release_root,
|
| 41 |
+
load_training_rows,
|
| 42 |
+
normalize_messages,
|
| 43 |
+
)
|
| 44 |
+
class MediaSemanticDataset(Dataset):
|
| 45 |
+
"""Load only a sealed pure-semantic JSONL release."""
|
| 46 |
+
|
| 47 |
+
def __init__(
|
| 48 |
+
self,
|
| 49 |
+
path: Path,
|
| 50 |
+
processor: Any,
|
| 51 |
+
*,
|
| 52 |
+
max_samples: int = 0,
|
| 53 |
+
media_root: Path | None = None,
|
| 54 |
+
development_unsealed_overlay: bool = False,
|
| 55 |
+
) -> None:
|
| 56 |
+
self.development_unsealed_overlay = development_unsealed_overlay
|
| 57 |
+
if development_unsealed_overlay:
|
| 58 |
+
self.rows = [
|
| 59 |
+
json.loads(line)
|
| 60 |
+
for line in path.read_text(encoding="utf-8").splitlines()
|
| 61 |
+
if line.strip()
|
| 62 |
+
]
|
| 63 |
+
else:
|
| 64 |
+
self.rows = _load_training_rows(path)
|
| 65 |
+
if max_samples > 0:
|
| 66 |
+
self.rows = self.rows[:max_samples]
|
| 67 |
+
self.processor = processor
|
| 68 |
+
inferred_root = _infer_release_root(path)
|
| 69 |
+
if development_unsealed_overlay:
|
| 70 |
+
if media_root is None:
|
| 71 |
+
raise RuntimeError("development overlay requires --media-root")
|
| 72 |
+
self.media_root = media_root.resolve(strict=True)
|
| 73 |
+
else:
|
| 74 |
+
if inferred_root is None:
|
| 75 |
+
raise RuntimeError("semantic release root could not be inferred")
|
| 76 |
+
if media_root is not None and media_root.resolve() != inferred_root:
|
| 77 |
+
raise RuntimeError("--media-root may not override sealed release media")
|
| 78 |
+
self.media_root = inferred_root
|
| 79 |
+
|
| 80 |
+
def __len__(self) -> int:
|
| 81 |
+
return len(self.rows)
|
| 82 |
+
|
| 83 |
+
def __getitem__(self, index: int) -> Dict[str, Any]:
|
| 84 |
+
row = self.rows[index]
|
| 85 |
+
raw_weight = row.get("sample_loss_weight")
|
| 86 |
+
sample_loss_weight = float(1.0 if raw_weight is None else raw_weight)
|
| 87 |
+
if not math.isfinite(sample_loss_weight) or sample_loss_weight <= 0:
|
| 88 |
+
raise RuntimeError(
|
| 89 |
+
"train-ready contract violated: "
|
| 90 |
+
f"uid={row.get('uid') or f'row_{index}'} "
|
| 91 |
+
f"sample_loss_weight={sample_loss_weight}"
|
| 92 |
+
)
|
| 93 |
+
return {
|
| 94 |
+
"uid": str(row.get("uid") or f"row_{index}"),
|
| 95 |
+
"messages": (
|
| 96 |
+
_normalize_development_messages(row["messages"])
|
| 97 |
+
if self.development_unsealed_overlay
|
| 98 |
+
else _normalize_messages(row["messages"], media_root=self.media_root)
|
| 99 |
+
),
|
| 100 |
+
"sample_loss_weight": sample_loss_weight,
|
| 101 |
+
}
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
@dataclass
|
| 105 |
+
class MediaSemanticCollator:
|
| 106 |
+
processor: Any
|
| 107 |
+
max_length: int
|
| 108 |
+
|
| 109 |
+
def __call__(self, features: List[Dict[str, Any]]) -> Dict[str, torch.Tensor]:
|
| 110 |
+
encoded = []
|
| 111 |
+
labels_list = []
|
| 112 |
+
weights_list = []
|
| 113 |
+
for feature in features:
|
| 114 |
+
messages = feature["messages"]
|
| 115 |
+
item = _encode_processor_item(self.processor, messages)
|
| 116 |
+
if item["input_ids"].numel() > self.max_length:
|
| 117 |
+
raise RuntimeError(
|
| 118 |
+
f"train-ready contract violated: uid={feature['uid']} "
|
| 119 |
+
f"tokens={item['input_ids'].numel()} max_length={self.max_length}"
|
| 120 |
+
)
|
| 121 |
+
labels, loss_weights = _assistant_only_labels_and_weights(
|
| 122 |
+
getattr(self.processor, "tokenizer", self.processor),
|
| 123 |
+
item["input_ids"],
|
| 124 |
+
messages,
|
| 125 |
+
)
|
| 126 |
+
if not torch.any(labels != -100):
|
| 127 |
+
raise RuntimeError(
|
| 128 |
+
f"train-ready contract violated: uid={feature['uid']} "
|
| 129 |
+
"has no assistant labels"
|
| 130 |
+
)
|
| 131 |
+
encoded.append(item)
|
| 132 |
+
labels_list.append(labels)
|
| 133 |
+
weights_list.append(loss_weights)
|
| 134 |
+
|
| 135 |
+
tokenizer = getattr(self.processor, "tokenizer", self.processor)
|
| 136 |
+
if tokenizer.pad_token_id is None:
|
| 137 |
+
raise RuntimeError("tokenizer has no pad token")
|
| 138 |
+
maximum = max(item["input_ids"].numel() for item in encoded)
|
| 139 |
+
batch: Dict[str, list[torch.Tensor]] = {
|
| 140 |
+
"input_ids": [],
|
| 141 |
+
"attention_mask": [],
|
| 142 |
+
"labels": [],
|
| 143 |
+
"loss_weights": [],
|
| 144 |
+
"sample_loss_weight": [],
|
| 145 |
+
}
|
| 146 |
+
pixel_values = []
|
| 147 |
+
image_grid_thw = []
|
| 148 |
+
for item, labels, loss_weights, feature in zip(
|
| 149 |
+
encoded, labels_list, weights_list, features, strict=True
|
| 150 |
+
):
|
| 151 |
+
pad = maximum - item["input_ids"].numel()
|
| 152 |
+
batch["input_ids"].append(
|
| 153 |
+
torch.nn.functional.pad(
|
| 154 |
+
item["input_ids"], (0, pad), value=tokenizer.pad_token_id
|
| 155 |
+
)
|
| 156 |
+
)
|
| 157 |
+
batch["attention_mask"].append(
|
| 158 |
+
torch.nn.functional.pad(item["attention_mask"], (0, pad), value=0)
|
| 159 |
+
)
|
| 160 |
+
batch["labels"].append(
|
| 161 |
+
torch.nn.functional.pad(labels, (0, pad), value=-100)
|
| 162 |
+
)
|
| 163 |
+
batch["loss_weights"].append(
|
| 164 |
+
torch.nn.functional.pad(loss_weights, (0, pad), value=0.0)
|
| 165 |
+
)
|
| 166 |
+
batch["sample_loss_weight"].append(
|
| 167 |
+
torch.tensor(
|
| 168 |
+
float(feature["sample_loss_weight"]), dtype=torch.float32
|
| 169 |
+
)
|
| 170 |
+
)
|
| 171 |
+
if "pixel_values" in item:
|
| 172 |
+
pixel_values.append(item["pixel_values"])
|
| 173 |
+
if "image_grid_thw" in item:
|
| 174 |
+
image_grid_thw.append(item["image_grid_thw"])
|
| 175 |
+
output = {key: torch.stack(value) for key, value in batch.items()}
|
| 176 |
+
if pixel_values:
|
| 177 |
+
output["pixel_values"] = torch.cat(pixel_values, dim=0)
|
| 178 |
+
if image_grid_thw:
|
| 179 |
+
output["image_grid_thw"] = torch.cat(image_grid_thw, dim=0)
|
| 180 |
+
return output
|
| 181 |
+
|
| 182 |
+
|
| 183 |
+
def main() -> None:
|
| 184 |
+
if any(
|
| 185 |
+
value is None
|
| 186 |
+
for value in (
|
| 187 |
+
AutoModelForImageTextToText,
|
| 188 |
+
AutoProcessor,
|
| 189 |
+
TrainingArguments,
|
| 190 |
+
set_seed,
|
| 191 |
+
)
|
| 192 |
+
):
|
| 193 |
+
raise RuntimeError(
|
| 194 |
+
"training requires transformers; the dependency-light import path "
|
| 195 |
+
"supports only data and loss-mask contract tests"
|
| 196 |
+
)
|
| 197 |
+
parser = argparse.ArgumentParser(
|
| 198 |
+
description="Full-parameter Qwen3-VL pure-semantic SFT."
|
| 199 |
+
)
|
| 200 |
+
parser.add_argument("--model", type=Path, required=True)
|
| 201 |
+
parser.add_argument("--train", type=Path, required=True)
|
| 202 |
+
parser.add_argument("--val", type=Path)
|
| 203 |
+
parser.add_argument("--media-root", type=Path)
|
| 204 |
+
parser.add_argument("--val-media-root", type=Path)
|
| 205 |
+
parser.add_argument(
|
| 206 |
+
"--development-unsealed-overlay",
|
| 207 |
+
action="store_true",
|
| 208 |
+
help="Development only: read an unsealed JSONL overlay with absolute media paths.",
|
| 209 |
+
)
|
| 210 |
+
parser.add_argument("--output", type=Path, required=True)
|
| 211 |
+
parser.add_argument("--max-length", type=int, default=8192)
|
| 212 |
+
parser.add_argument("--max-pixels", type=int, default=0)
|
| 213 |
+
parser.add_argument("--per-device-batch-size", type=int, default=1)
|
| 214 |
+
parser.add_argument("--gradient-accumulation-steps", type=int, default=8)
|
| 215 |
+
parser.add_argument("--epochs", type=float, default=1.0)
|
| 216 |
+
parser.add_argument("--lr", type=float, default=1e-4)
|
| 217 |
+
parser.add_argument(
|
| 218 |
+
"--vision-projector-lr-scale",
|
| 219 |
+
type=float,
|
| 220 |
+
default=1.0,
|
| 221 |
+
help=(
|
| 222 |
+
"Multiply the optimizer learning rate for trainable visual "
|
| 223 |
+
"merger/deepstack projector parameters."
|
| 224 |
+
),
|
| 225 |
+
)
|
| 226 |
+
parser.add_argument(
|
| 227 |
+
"--lr-scheduler-type",
|
| 228 |
+
choices=("cosine", "constant_with_warmup"),
|
| 229 |
+
default="cosine",
|
| 230 |
+
)
|
| 231 |
+
parser.add_argument(
|
| 232 |
+
"--train-order", choices=("shuffled", "sequential"), default="shuffled"
|
| 233 |
+
)
|
| 234 |
+
parser.add_argument(
|
| 235 |
+
"--stage-lr-plan",
|
| 236 |
+
type=Path,
|
| 237 |
+
help="Audited row-boundary learning-rate plan for one ordered run.",
|
| 238 |
+
)
|
| 239 |
+
parser.add_argument("--warmup-ratio", type=float, default=0.03)
|
| 240 |
+
parser.add_argument("--deepspeed", type=Path, required=True)
|
| 241 |
+
parser.add_argument("--freeze-vision-tower", action="store_true")
|
| 242 |
+
parser.add_argument("--train-vision-projector", action="store_true")
|
| 243 |
+
parser.add_argument("--logging-steps", type=int, default=10)
|
| 244 |
+
parser.add_argument("--save-steps", type=int, default=250)
|
| 245 |
+
parser.add_argument("--save-total-limit", type=int, default=2)
|
| 246 |
+
parser.add_argument("--skip-final-save", action="store_true")
|
| 247 |
+
parser.add_argument("--max-train-samples", type=int, default=0)
|
| 248 |
+
parser.add_argument("--max-val-samples", type=int, default=0)
|
| 249 |
+
parser.add_argument("--max-steps", type=int, default=-1)
|
| 250 |
+
parser.add_argument("--seed", type=int, default=42)
|
| 251 |
+
parser.add_argument("--attn-implementation", default="flash_attention_2")
|
| 252 |
+
parser.add_argument("--dataloader-num-workers", type=int, default=0)
|
| 253 |
+
args = parser.parse_args()
|
| 254 |
+
|
| 255 |
+
if args.max_length <= 0 or args.max_pixels < 0:
|
| 256 |
+
raise ValueError("length must be positive and max-pixels non-negative")
|
| 257 |
+
if (
|
| 258 |
+
not math.isfinite(args.vision_projector_lr_scale)
|
| 259 |
+
or args.vision_projector_lr_scale <= 0
|
| 260 |
+
):
|
| 261 |
+
raise ValueError("vision-projector-lr-scale must be finite and positive")
|
| 262 |
+
if args.train_vision_projector and not args.freeze_vision_tower:
|
| 263 |
+
raise ValueError("--train-vision-projector requires --freeze-vision-tower")
|
| 264 |
+
if args.vision_projector_lr_scale != 1.0 and not args.train_vision_projector:
|
| 265 |
+
raise ValueError(
|
| 266 |
+
"a non-default vision-projector LR scale requires a trainable projector"
|
| 267 |
+
)
|
| 268 |
+
if not args.deepspeed.is_file():
|
| 269 |
+
raise FileNotFoundError(args.deepspeed)
|
| 270 |
+
|
| 271 |
+
set_seed(args.seed)
|
| 272 |
+
has_eval = args.val is not None and args.val.exists()
|
| 273 |
+
training_args = _training_args(args, has_eval)
|
| 274 |
+
processor = AutoProcessor.from_pretrained(args.model, trust_remote_code=True)
|
| 275 |
+
_configure_processor_pixel_budget(processor, args.max_pixels)
|
| 276 |
+
tokenizer = getattr(processor, "tokenizer", processor)
|
| 277 |
+
if tokenizer.pad_token_id is None:
|
| 278 |
+
tokenizer.pad_token = tokenizer.eos_token
|
| 279 |
+
model = AutoModelForImageTextToText.from_pretrained(
|
| 280 |
+
args.model,
|
| 281 |
+
dtype=torch.bfloat16,
|
| 282 |
+
trust_remote_code=True,
|
| 283 |
+
attn_implementation=args.attn_implementation,
|
| 284 |
+
)
|
| 285 |
+
model.config.use_cache = False
|
| 286 |
+
model.gradient_checkpointing_enable()
|
| 287 |
+
if args.freeze_vision_tower:
|
| 288 |
+
model.visual.requires_grad_(False)
|
| 289 |
+
if args.train_vision_projector:
|
| 290 |
+
for name in ("merger", "deepstack_merger_list"):
|
| 291 |
+
module = getattr(model.visual, name, None)
|
| 292 |
+
if module is None:
|
| 293 |
+
raise RuntimeError(
|
| 294 |
+
f"Qwen3-VL visual module lacks projector component {name!r}"
|
| 295 |
+
)
|
| 296 |
+
module.requires_grad_(True)
|
| 297 |
+
_audit_trainable_parameters(model, args)
|
| 298 |
+
|
| 299 |
+
if _is_main_process():
|
| 300 |
+
trainable = sum(
|
| 301 |
+
_logical_parameter_numel(parameter)
|
| 302 |
+
for parameter in model.parameters()
|
| 303 |
+
if parameter.requires_grad
|
| 304 |
+
)
|
| 305 |
+
total = sum(
|
| 306 |
+
_logical_parameter_numel(parameter) for parameter in model.parameters()
|
| 307 |
+
)
|
| 308 |
+
if total <= 0:
|
| 309 |
+
raise RuntimeError("could not determine logical parameter count")
|
| 310 |
+
print(
|
| 311 |
+
{
|
| 312 |
+
"training_mode": "full_parameter_no_adapter",
|
| 313 |
+
"vision_tower_frozen": args.freeze_vision_tower,
|
| 314 |
+
"trainable_parameters": trainable,
|
| 315 |
+
"total_parameters": total,
|
| 316 |
+
"trainable_fraction": trainable / total,
|
| 317 |
+
},
|
| 318 |
+
flush=True,
|
| 319 |
+
)
|
| 320 |
+
|
| 321 |
+
train_dataset = MediaSemanticDataset(
|
| 322 |
+
args.train,
|
| 323 |
+
processor,
|
| 324 |
+
max_samples=args.max_train_samples,
|
| 325 |
+
media_root=args.media_root,
|
| 326 |
+
development_unsealed_overlay=args.development_unsealed_overlay,
|
| 327 |
+
)
|
| 328 |
+
global_batch = (
|
| 329 |
+
args.per_device_batch_size
|
| 330 |
+
* args.gradient_accumulation_steps
|
| 331 |
+
* int(os.environ.get("WORLD_SIZE", "1"))
|
| 332 |
+
)
|
| 333 |
+
stage_lr_plan = (
|
| 334 |
+
_load_stage_lr_plan(
|
| 335 |
+
args.stage_lr_plan,
|
| 336 |
+
release_manifest=inferred_root_manifest(train_dataset.media_root),
|
| 337 |
+
dataset_rows=len(train_dataset),
|
| 338 |
+
global_batch=global_batch,
|
| 339 |
+
optimizer_learning_rate=args.lr,
|
| 340 |
+
warmup_ratio=args.warmup_ratio,
|
| 341 |
+
)
|
| 342 |
+
if args.stage_lr_plan is not None
|
| 343 |
+
else None
|
| 344 |
+
)
|
| 345 |
+
eval_dataset = (
|
| 346 |
+
MediaSemanticDataset(
|
| 347 |
+
args.val,
|
| 348 |
+
processor,
|
| 349 |
+
max_samples=args.max_val_samples,
|
| 350 |
+
media_root=args.val_media_root,
|
| 351 |
+
)
|
| 352 |
+
if has_eval
|
| 353 |
+
else None
|
| 354 |
+
)
|
| 355 |
+
trainer = SemanticTokenTrainer(
|
| 356 |
+
model=model,
|
| 357 |
+
args=training_args,
|
| 358 |
+
train_dataset=train_dataset,
|
| 359 |
+
eval_dataset=eval_dataset,
|
| 360 |
+
data_collator=MediaSemanticCollator(processor, args.max_length),
|
| 361 |
+
train_order=args.train_order,
|
| 362 |
+
stage_lr_plan=stage_lr_plan,
|
| 363 |
+
vision_projector_lr_scale=args.vision_projector_lr_scale,
|
| 364 |
+
vision_projector_parameter_ids={
|
| 365 |
+
id(parameter)
|
| 366 |
+
for name in ("merger", "deepstack_merger_list")
|
| 367 |
+
for parameter in getattr(model.visual, name).parameters()
|
| 368 |
+
}
|
| 369 |
+
if args.train_vision_projector
|
| 370 |
+
else set(),
|
| 371 |
+
)
|
| 372 |
+
trainer.train()
|
| 373 |
+
if args.skip_final_save:
|
| 374 |
+
if _is_main_process():
|
| 375 |
+
print(
|
| 376 |
+
{
|
| 377 |
+
"output": str(args.output),
|
| 378 |
+
"training_mode": "full_parameter_no_adapter",
|
| 379 |
+
"final_model": None,
|
| 380 |
+
"systems_smoke_skipped_final_save": True,
|
| 381 |
+
},
|
| 382 |
+
flush=True,
|
| 383 |
+
)
|
| 384 |
+
return
|
| 385 |
+
_release_save_reserve()
|
| 386 |
+
final_dir = args.output / "final_model"
|
| 387 |
+
trainer.save_model(final_dir)
|
| 388 |
+
processor.save_pretrained(final_dir)
|
| 389 |
+
if _is_main_process():
|
| 390 |
+
print(
|
| 391 |
+
{
|
| 392 |
+
"output": str(args.output),
|
| 393 |
+
"training_mode": "full_parameter_no_adapter",
|
| 394 |
+
"final_model": str(final_dir),
|
| 395 |
+
},
|
| 396 |
+
flush=True,
|
| 397 |
+
)
|
| 398 |
+
|
| 399 |
+
|
| 400 |
+
def _release_save_reserve() -> None:
|
| 401 |
+
raw = os.environ.get("STREAMEVA_SAVE_RESERVE_PATH")
|
| 402 |
+
if raw and _is_main_process():
|
| 403 |
+
path = Path(raw).resolve()
|
| 404 |
+
if (
|
| 405 |
+
path.name != "save_reserve.bin"
|
| 406 |
+
or path.parent.parent != Path("/tmp")
|
| 407 |
+
or not path.parent.name.startswith("streameva_")
|
| 408 |
+
):
|
| 409 |
+
raise RuntimeError("save reserve path is outside the short runtime root")
|
| 410 |
+
path.unlink(missing_ok=True)
|
| 411 |
+
os.sync()
|
| 412 |
+
print(
|
| 413 |
+
{
|
| 414 |
+
"save_reserve_released": str(path),
|
| 415 |
+
"free_bytes": shutil.disk_usage(path.parent).free,
|
| 416 |
+
},
|
| 417 |
+
flush=True,
|
| 418 |
+
)
|
| 419 |
+
if torch.distributed.is_available() and torch.distributed.is_initialized():
|
| 420 |
+
torch.distributed.barrier()
|
| 421 |
+
|
| 422 |
+
|
| 423 |
+
def _audit_trainable_parameters(model: Any, args: argparse.Namespace) -> None:
|
| 424 |
+
if not all(
|
| 425 |
+
parameter.requires_grad for parameter in model.language_model.parameters()
|
| 426 |
+
):
|
| 427 |
+
raise RuntimeError("full-parameter contract froze a language parameter")
|
| 428 |
+
if not all(parameter.requires_grad for parameter in model.lm_head.parameters()):
|
| 429 |
+
raise RuntimeError("full-parameter contract froze an LM-head parameter")
|
| 430 |
+
if not args.freeze_vision_tower:
|
| 431 |
+
return
|
| 432 |
+
projector_ids: set[int] = set()
|
| 433 |
+
if args.train_vision_projector:
|
| 434 |
+
for name in ("merger", "deepstack_merger_list"):
|
| 435 |
+
projector_ids.update(
|
| 436 |
+
id(parameter)
|
| 437 |
+
for parameter in getattr(model.visual, name).parameters()
|
| 438 |
+
)
|
| 439 |
+
for parameter in model.visual.parameters():
|
| 440 |
+
if parameter.requires_grad != (id(parameter) in projector_ids):
|
| 441 |
+
raise RuntimeError("frozen-vision/trainable-projector contract violated")
|
| 442 |
+
|
| 443 |
+
|
| 444 |
+
def _load_training_rows(
|
| 445 |
+
path: Path, *, training_schema: str = "semantic"
|
| 446 |
+
) -> List[Dict[str, Any]]:
|
| 447 |
+
if training_schema != "semantic":
|
| 448 |
+
raise ValueError(f"unsupported training schema: {training_schema}")
|
| 449 |
+
return load_training_rows(path)
|
| 450 |
+
|
| 451 |
+
|
| 452 |
+
def _normalize_development_messages(messages: Any) -> List[Dict[str, Any]]:
|
| 453 |
+
"""Normalize an explicit development overlay without weakening release loading."""
|
| 454 |
+
if not isinstance(messages, list):
|
| 455 |
+
raise TypeError("development messages must be a list")
|
| 456 |
+
roles = [message.get("role") for message in messages]
|
| 457 |
+
if roles != ["user", "assistant"]:
|
| 458 |
+
raise ValueError("development overlay accepts pure-VLM rows only")
|
| 459 |
+
output: List[Dict[str, Any]] = []
|
| 460 |
+
for message in messages:
|
| 461 |
+
content: List[Dict[str, Any]] = []
|
| 462 |
+
for item in message.get("content") or []:
|
| 463 |
+
if item.get("type") == "text":
|
| 464 |
+
content.append({"type": "text", "text": str(item.get("text") or "")})
|
| 465 |
+
continue
|
| 466 |
+
if item.get("type") != "image" or message["role"] != "user":
|
| 467 |
+
raise ValueError("development overlay contains unsupported content")
|
| 468 |
+
image_path = Path(str(item.get("image") or ""))
|
| 469 |
+
if not image_path.is_absolute() or not image_path.is_file():
|
| 470 |
+
raise RuntimeError("development overlay image must be an existing absolute path")
|
| 471 |
+
content.append(
|
| 472 |
+
{"type": "image", "image": Image.open(image_path).convert("RGB")}
|
| 473 |
+
)
|
| 474 |
+
if not content:
|
| 475 |
+
raise ValueError("development overlay message content is empty")
|
| 476 |
+
output.append(
|
| 477 |
+
{
|
| 478 |
+
"role": str(message["role"]),
|
| 479 |
+
"content": content,
|
| 480 |
+
"loss": bool(message.get("loss")),
|
| 481 |
+
}
|
| 482 |
+
)
|
| 483 |
+
return output
|
| 484 |
+
|
| 485 |
+
|
| 486 |
+
def _infer_release_root(
|
| 487 |
+
path: Path, *, training_schema: str = "semantic"
|
| 488 |
+
) -> Path | None:
|
| 489 |
+
if training_schema != "semantic":
|
| 490 |
+
raise ValueError(f"unsupported training schema: {training_schema}")
|
| 491 |
+
return infer_release_root(path)
|
| 492 |
+
|
| 493 |
+
|
| 494 |
+
def _normalize_messages(
|
| 495 |
+
messages: Any,
|
| 496 |
+
*,
|
| 497 |
+
media_root: Path | None = None,
|
| 498 |
+
training_schema: str = "semantic",
|
| 499 |
+
) -> List[Dict[str, Any]]:
|
| 500 |
+
if training_schema != "semantic":
|
| 501 |
+
raise ValueError(f"unsupported training schema: {training_schema}")
|
| 502 |
+
return normalize_messages(messages, media_root=media_root)
|
| 503 |
+
|
| 504 |
+
|
| 505 |
+
def _load_images(messages: List[Dict[str, Any]]) -> List[Image.Image]:
|
| 506 |
+
images = []
|
| 507 |
+
for message in messages:
|
| 508 |
+
for item in message.get("content", []):
|
| 509 |
+
if not isinstance(item, dict) or item.get("type") != "image":
|
| 510 |
+
continue
|
| 511 |
+
image = item.get("image")
|
| 512 |
+
if isinstance(image, Image.Image):
|
| 513 |
+
images.append(image)
|
| 514 |
+
elif image:
|
| 515 |
+
images.append(Image.open(image).convert("RGB"))
|
| 516 |
+
return images
|
| 517 |
+
|
| 518 |
+
|
| 519 |
+
def _configure_processor_pixel_budget(
|
| 520 |
+
processor: Any, max_pixels: int
|
| 521 |
+
) -> Dict[str, int]:
|
| 522 |
+
image_processor = getattr(processor, "image_processor", None)
|
| 523 |
+
size = getattr(image_processor, "size", None)
|
| 524 |
+
if image_processor is None or not isinstance(size, dict):
|
| 525 |
+
raise RuntimeError("processor has no configurable image size contract")
|
| 526 |
+
shortest = int(size.get("shortest_edge", 0))
|
| 527 |
+
longest = int(size.get("longest_edge", 0))
|
| 528 |
+
if shortest <= 0 or longest <= 0 or shortest > longest:
|
| 529 |
+
raise RuntimeError("processor image size contract is invalid")
|
| 530 |
+
requested = longest if max_pixels == 0 else int(max_pixels)
|
| 531 |
+
if requested < shortest:
|
| 532 |
+
raise ValueError(f"max_pixels={requested} is below shortest_edge={shortest}")
|
| 533 |
+
image_processor.size = {
|
| 534 |
+
"shortest_edge": shortest,
|
| 535 |
+
"longest_edge": requested,
|
| 536 |
+
}
|
| 537 |
+
return dict(image_processor.size)
|
| 538 |
+
|
| 539 |
+
|
| 540 |
+
def _encode_processor_item(
|
| 541 |
+
processor: Any, messages: List[Dict[str, Any]]
|
| 542 |
+
) -> Dict[str, torch.Tensor]:
|
| 543 |
+
rendered = processor.apply_chat_template(
|
| 544 |
+
messages,
|
| 545 |
+
tokenize=False,
|
| 546 |
+
add_generation_prompt=False,
|
| 547 |
+
)
|
| 548 |
+
images = _load_images(messages)
|
| 549 |
+
if rendered.count("<|image_pad|>") != len(images):
|
| 550 |
+
raise RuntimeError("chat-template image slots and message images differ")
|
| 551 |
+
kwargs: Dict[str, Any] = {
|
| 552 |
+
"text": rendered,
|
| 553 |
+
"return_dict": True,
|
| 554 |
+
"return_tensors": "pt",
|
| 555 |
+
}
|
| 556 |
+
if images:
|
| 557 |
+
kwargs["images"] = images
|
| 558 |
+
item = processor(**kwargs)
|
| 559 |
+
return {
|
| 560 |
+
key: (
|
| 561 |
+
value.squeeze(0)
|
| 562 |
+
if hasattr(value, "dim")
|
| 563 |
+
and value.dim() > 1
|
| 564 |
+
and key in {"input_ids", "attention_mask"}
|
| 565 |
+
else value
|
| 566 |
+
)
|
| 567 |
+
for key, value in item.items()
|
| 568 |
+
}
|
| 569 |
+
|
| 570 |
+
|
| 571 |
+
def _locate_rendered_text_token_positions(
|
| 572 |
+
tokenizer: Any,
|
| 573 |
+
input_ids: torch.Tensor,
|
| 574 |
+
messages: List[Dict[str, Any]],
|
| 575 |
+
) -> Dict[tuple[int, int], List[int]] | None:
|
| 576 |
+
"""Align message text spans through chat rendering and visual expansion."""
|
| 577 |
+
|
| 578 |
+
if not hasattr(tokenizer, "apply_chat_template"):
|
| 579 |
+
return None
|
| 580 |
+
try:
|
| 581 |
+
rendered = tokenizer.apply_chat_template(
|
| 582 |
+
messages,
|
| 583 |
+
tokenize=False,
|
| 584 |
+
add_generation_prompt=False,
|
| 585 |
+
)
|
| 586 |
+
encoded = tokenizer(
|
| 587 |
+
rendered,
|
| 588 |
+
add_special_tokens=False,
|
| 589 |
+
return_offsets_mapping=True,
|
| 590 |
+
)
|
| 591 |
+
except (KeyError, NotImplementedError, TypeError, ValueError):
|
| 592 |
+
return None
|
| 593 |
+
template_ids = encoded.get("input_ids")
|
| 594 |
+
offsets = encoded.get("offset_mapping")
|
| 595 |
+
if template_ids is None or offsets is None:
|
| 596 |
+
return None
|
| 597 |
+
if hasattr(template_ids, "tolist"):
|
| 598 |
+
template_ids = template_ids.tolist()
|
| 599 |
+
if hasattr(offsets, "tolist"):
|
| 600 |
+
offsets = offsets.tolist()
|
| 601 |
+
if template_ids and isinstance(template_ids[0], list):
|
| 602 |
+
template_ids = template_ids[0]
|
| 603 |
+
if offsets and isinstance(offsets[0][0], list):
|
| 604 |
+
offsets = offsets[0]
|
| 605 |
+
|
| 606 |
+
actual_ids = input_ids.tolist()
|
| 607 |
+
visual_pad_ids = {
|
| 608 |
+
token_id
|
| 609 |
+
for token in ("<|image_pad|>", "<|video_pad|>")
|
| 610 |
+
for token_id in [tokenizer.convert_tokens_to_ids(token)]
|
| 611 |
+
if isinstance(token_id, int) and token_id >= 0
|
| 612 |
+
}
|
| 613 |
+
template_to_actual: Dict[int, int] = {}
|
| 614 |
+
actual_index = 0
|
| 615 |
+
for template_index, token_id in enumerate(template_ids):
|
| 616 |
+
if actual_index >= len(actual_ids) or actual_ids[actual_index] != token_id:
|
| 617 |
+
raise RuntimeError("rendered template and processor input IDs diverged")
|
| 618 |
+
template_to_actual[template_index] = actual_index
|
| 619 |
+
actual_index += 1
|
| 620 |
+
if token_id in visual_pad_ids:
|
| 621 |
+
while actual_index < len(actual_ids) and actual_ids[actual_index] == token_id:
|
| 622 |
+
actual_index += 1
|
| 623 |
+
if actual_index != len(actual_ids):
|
| 624 |
+
raise RuntimeError("processor input IDs contain non-template tokens")
|
| 625 |
+
|
| 626 |
+
output: Dict[tuple[int, int], List[int]] = {}
|
| 627 |
+
character_cursor = 0
|
| 628 |
+
for message_index, message in enumerate(messages):
|
| 629 |
+
text_items = [
|
| 630 |
+
item
|
| 631 |
+
for item in message.get("content", [])
|
| 632 |
+
if isinstance(item, dict) and item.get("type") == "text"
|
| 633 |
+
]
|
| 634 |
+
for text_index, item in enumerate(text_items):
|
| 635 |
+
text = str(item.get("text", ""))
|
| 636 |
+
start = rendered.find(text, character_cursor)
|
| 637 |
+
if start < 0:
|
| 638 |
+
raise RuntimeError("message text is absent from the rendered template")
|
| 639 |
+
end = start + len(text)
|
| 640 |
+
positions = [
|
| 641 |
+
template_to_actual[index]
|
| 642 |
+
for index, pair in enumerate(offsets)
|
| 643 |
+
if int(pair[0]) < end and int(pair[1]) > start
|
| 644 |
+
]
|
| 645 |
+
if text and not positions:
|
| 646 |
+
raise RuntimeError("rendered message text has no token positions")
|
| 647 |
+
output[(message_index, text_index)] = positions
|
| 648 |
+
character_cursor = end
|
| 649 |
+
return output
|
| 650 |
+
|
| 651 |
+
|
| 652 |
+
def _assistant_only_labels(
|
| 653 |
+
tokenizer: Any,
|
| 654 |
+
input_ids: torch.Tensor,
|
| 655 |
+
messages: List[Dict[str, Any]],
|
| 656 |
+
) -> torch.Tensor:
|
| 657 |
+
labels, _ = _assistant_only_labels_and_weights(tokenizer, input_ids, messages)
|
| 658 |
+
return labels
|
| 659 |
+
|
| 660 |
+
|
| 661 |
+
def _assistant_only_labels_and_weights(
|
| 662 |
+
tokenizer: Any,
|
| 663 |
+
input_ids: torch.Tensor,
|
| 664 |
+
messages: List[Dict[str, Any]],
|
| 665 |
+
) -> tuple[torch.Tensor, torch.Tensor]:
|
| 666 |
+
"""Supervise assistant semantic text and its turn terminator only."""
|
| 667 |
+
|
| 668 |
+
labels = torch.full_like(input_ids, -100)
|
| 669 |
+
weights = torch.zeros_like(input_ids, dtype=torch.float32)
|
| 670 |
+
ids = input_ids.tolist()
|
| 671 |
+
cursor = 0
|
| 672 |
+
rendered_positions = _locate_rendered_text_token_positions(
|
| 673 |
+
tokenizer, input_ids, messages
|
| 674 |
+
)
|
| 675 |
+
for message_index, message in enumerate(messages):
|
| 676 |
+
is_assistant = message.get("role") == "assistant"
|
| 677 |
+
supervise = is_assistant and message.get("loss", True) is not False
|
| 678 |
+
text_items = [
|
| 679 |
+
item
|
| 680 |
+
for item in message.get("content", [])
|
| 681 |
+
if isinstance(item, dict) and item.get("type") == "text"
|
| 682 |
+
]
|
| 683 |
+
for text_index, item in enumerate(text_items):
|
| 684 |
+
text = str(item.get("text", ""))
|
| 685 |
+
if rendered_positions is not None:
|
| 686 |
+
positions = rendered_positions[(message_index, text_index)]
|
| 687 |
+
start = positions[0] if positions else cursor
|
| 688 |
+
semantic_end = positions[-1] + 1 if positions else cursor
|
| 689 |
+
else:
|
| 690 |
+
target = tokenizer(text, add_special_tokens=False)["input_ids"]
|
| 691 |
+
start = _find_subsequence(ids, target, cursor)
|
| 692 |
+
if start < 0:
|
| 693 |
+
raise RuntimeError("could not locate message text in role order")
|
| 694 |
+
semantic_end = min(start + len(target), labels.numel())
|
| 695 |
+
end = semantic_end
|
| 696 |
+
im_end_id = tokenizer.convert_tokens_to_ids("<|im_end|>")
|
| 697 |
+
if (
|
| 698 |
+
is_assistant
|
| 699 |
+
and text_index == len(text_items) - 1
|
| 700 |
+
and im_end_id is not None
|
| 701 |
+
and end < labels.numel()
|
| 702 |
+
and ids[end] == im_end_id
|
| 703 |
+
):
|
| 704 |
+
end += 1
|
| 705 |
+
cursor = end
|
| 706 |
+
if supervise:
|
| 707 |
+
labels[start:end] = input_ids[start:end]
|
| 708 |
+
weights[start:end] = 1.0
|
| 709 |
+
return labels, weights
|
| 710 |
+
|
| 711 |
+
|
| 712 |
+
def _find_subsequence(haystack: List[int], needle: List[int], start: int) -> int:
|
| 713 |
+
if not needle:
|
| 714 |
+
return -1
|
| 715 |
+
for index in range(max(start, 0), len(haystack) - len(needle) + 1):
|
| 716 |
+
if haystack[index : index + len(needle)] == needle:
|
| 717 |
+
return index
|
| 718 |
+
return -1
|
| 719 |
+
|
| 720 |
+
|
| 721 |
+
def _split_optimizer_groups_for_projector(
|
| 722 |
+
param_groups: list[dict[str, Any]],
|
| 723 |
+
*,
|
| 724 |
+
projector_parameter_ids: set[int],
|
| 725 |
+
learning_rate_scale: float,
|
| 726 |
+
) -> tuple[list[dict[str, Any]], int]:
|
| 727 |
+
"""Preserve group options while isolating/scaling projector parameters."""
|
| 728 |
+
|
| 729 |
+
replacement_groups: list[dict[str, Any]] = []
|
| 730 |
+
projector_parameters = 0
|
| 731 |
+
for original in param_groups:
|
| 732 |
+
projector = [
|
| 733 |
+
parameter
|
| 734 |
+
for parameter in original["params"]
|
| 735 |
+
if id(parameter) in projector_parameter_ids
|
| 736 |
+
]
|
| 737 |
+
backbone = [
|
| 738 |
+
parameter
|
| 739 |
+
for parameter in original["params"]
|
| 740 |
+
if id(parameter) not in projector_parameter_ids
|
| 741 |
+
]
|
| 742 |
+
if backbone:
|
| 743 |
+
group = dict(original)
|
| 744 |
+
group["params"] = backbone
|
| 745 |
+
replacement_groups.append(group)
|
| 746 |
+
if projector:
|
| 747 |
+
group = dict(original)
|
| 748 |
+
group["params"] = projector
|
| 749 |
+
group["lr"] = float(group["lr"]) * learning_rate_scale
|
| 750 |
+
if "initial_lr" in group:
|
| 751 |
+
group["initial_lr"] = (
|
| 752 |
+
float(group["initial_lr"]) * learning_rate_scale
|
| 753 |
+
)
|
| 754 |
+
replacement_groups.append(group)
|
| 755 |
+
projector_parameters += len(projector)
|
| 756 |
+
return replacement_groups, projector_parameters
|
| 757 |
+
|
| 758 |
+
|
| 759 |
+
class SemanticTokenTrainer(Trainer):
|
| 760 |
+
def __init__(
|
| 761 |
+
self,
|
| 762 |
+
*args: Any,
|
| 763 |
+
train_order: str = "shuffled",
|
| 764 |
+
stage_lr_plan: dict[str, Any] | None = None,
|
| 765 |
+
vision_projector_lr_scale: float = 1.0,
|
| 766 |
+
vision_projector_parameter_ids: set[int] | None = None,
|
| 767 |
+
**kwargs: Any,
|
| 768 |
+
) -> None:
|
| 769 |
+
if train_order not in {"shuffled", "sequential"}:
|
| 770 |
+
raise ValueError("unsupported train order")
|
| 771 |
+
if stage_lr_plan is not None and train_order != "sequential":
|
| 772 |
+
raise ValueError("stage learning-rate plan requires sequential stage order")
|
| 773 |
+
self.train_order = train_order
|
| 774 |
+
self.stage_lr_plan = stage_lr_plan
|
| 775 |
+
self.vision_projector_lr_scale = float(vision_projector_lr_scale)
|
| 776 |
+
self.vision_projector_parameter_ids = set(
|
| 777 |
+
vision_projector_parameter_ids or set()
|
| 778 |
+
)
|
| 779 |
+
self._projector_lr_groups_applied = False
|
| 780 |
+
if (
|
| 781 |
+
self.vision_projector_lr_scale != 1.0
|
| 782 |
+
and not self.vision_projector_parameter_ids
|
| 783 |
+
):
|
| 784 |
+
raise ValueError("projector LR scaling requires projector parameters")
|
| 785 |
+
super().__init__(*args, **kwargs)
|
| 786 |
+
|
| 787 |
+
def create_optimizer(self) -> Any:
|
| 788 |
+
"""Split Trainer's established decay groups and scale projector LR only."""
|
| 789 |
+
|
| 790 |
+
optimizer = super().create_optimizer()
|
| 791 |
+
if self.vision_projector_lr_scale == 1.0:
|
| 792 |
+
return optimizer
|
| 793 |
+
if self._projector_lr_groups_applied:
|
| 794 |
+
return optimizer
|
| 795 |
+
if optimizer is None:
|
| 796 |
+
raise RuntimeError("optimizer is absent while applying projector LR")
|
| 797 |
+
|
| 798 |
+
replacement_groups, projector_parameters = (
|
| 799 |
+
_split_optimizer_groups_for_projector(
|
| 800 |
+
list(optimizer.param_groups),
|
| 801 |
+
projector_parameter_ids=self.vision_projector_parameter_ids,
|
| 802 |
+
learning_rate_scale=self.vision_projector_lr_scale,
|
| 803 |
+
)
|
| 804 |
+
)
|
| 805 |
+
if projector_parameters != len(self.vision_projector_parameter_ids):
|
| 806 |
+
raise RuntimeError(
|
| 807 |
+
"optimizer projector parameter coverage diverged: "
|
| 808 |
+
f"{projector_parameters} != "
|
| 809 |
+
f"{len(self.vision_projector_parameter_ids)}"
|
| 810 |
+
)
|
| 811 |
+
optimizer.param_groups[:] = replacement_groups
|
| 812 |
+
self._projector_lr_groups_applied = True
|
| 813 |
+
if _is_main_process():
|
| 814 |
+
print(
|
| 815 |
+
{
|
| 816 |
+
"optimizer_lr_policy": "split_visual_projector",
|
| 817 |
+
"base_learning_rate": self.args.learning_rate,
|
| 818 |
+
"vision_projector_lr_scale": self.vision_projector_lr_scale,
|
| 819 |
+
"vision_projector_learning_rate": (
|
| 820 |
+
self.args.learning_rate
|
| 821 |
+
* self.vision_projector_lr_scale
|
| 822 |
+
),
|
| 823 |
+
"projector_parameter_tensors": projector_parameters,
|
| 824 |
+
"optimizer_parameter_groups": len(replacement_groups),
|
| 825 |
+
},
|
| 826 |
+
flush=True,
|
| 827 |
+
)
|
| 828 |
+
return optimizer
|
| 829 |
+
|
| 830 |
+
def create_scheduler(
|
| 831 |
+
self, num_training_steps: int, optimizer: Any = None
|
| 832 |
+
) -> Any:
|
| 833 |
+
if self.stage_lr_plan is None:
|
| 834 |
+
try:
|
| 835 |
+
return super().create_scheduler(num_training_steps, optimizer)
|
| 836 |
+
except TypeError:
|
| 837 |
+
return super().create_scheduler(num_training_steps)
|
| 838 |
+
if num_training_steps != int(self.stage_lr_plan["total_steps"]):
|
| 839 |
+
raise RuntimeError(
|
| 840 |
+
"stage LR plan and Trainer optimization steps diverged: "
|
| 841 |
+
f"{self.stage_lr_plan['total_steps']} != {num_training_steps}"
|
| 842 |
+
)
|
| 843 |
+
if self.lr_scheduler is None:
|
| 844 |
+
active_optimizer = self.optimizer if optimizer is None else optimizer
|
| 845 |
+
if active_optimizer is None:
|
| 846 |
+
raise RuntimeError("optimizer is absent while creating stage scheduler")
|
| 847 |
+
self.lr_scheduler = torch.optim.lr_scheduler.LambdaLR(
|
| 848 |
+
active_optimizer,
|
| 849 |
+
lr_lambda=lambda step: _stage_lr_multiplier(
|
| 850 |
+
self.stage_lr_plan, int(step)
|
| 851 |
+
),
|
| 852 |
+
)
|
| 853 |
+
return self.lr_scheduler
|
| 854 |
+
|
| 855 |
+
def _get_train_sampler(self, train_dataset: Any = None) -> Any:
|
| 856 |
+
dataset = self.train_dataset if train_dataset is None else train_dataset
|
| 857 |
+
if self.train_order == "sequential":
|
| 858 |
+
return SequentialSampler(dataset)
|
| 859 |
+
try:
|
| 860 |
+
return super()._get_train_sampler(train_dataset)
|
| 861 |
+
except TypeError:
|
| 862 |
+
return super()._get_train_sampler()
|
| 863 |
+
|
| 864 |
+
def compute_loss(
|
| 865 |
+
self,
|
| 866 |
+
model: Any,
|
| 867 |
+
inputs: Dict[str, Any],
|
| 868 |
+
return_outputs: bool = False,
|
| 869 |
+
num_items_in_batch: Any = None,
|
| 870 |
+
) -> Any:
|
| 871 |
+
del num_items_in_batch
|
| 872 |
+
labels = inputs.pop("labels")
|
| 873 |
+
loss_weights = inputs.pop("loss_weights")
|
| 874 |
+
sample_loss_weight = inputs.pop("sample_loss_weight").float()
|
| 875 |
+
outputs = model(**inputs)
|
| 876 |
+
shift_logits = outputs.logits[..., :-1, :]
|
| 877 |
+
shift_labels = labels[..., 1:].contiguous()
|
| 878 |
+
shift_weights = loss_weights[..., 1:].contiguous().float()
|
| 879 |
+
loss = _sparse_weighted_token_loss(
|
| 880 |
+
shift_logits, shift_labels, shift_weights, sample_loss_weight
|
| 881 |
+
)
|
| 882 |
+
return (loss, outputs) if return_outputs else loss
|
| 883 |
+
|
| 884 |
+
|
| 885 |
+
def _sparse_weighted_token_loss(
|
| 886 |
+
logits: torch.Tensor,
|
| 887 |
+
labels: torch.Tensor,
|
| 888 |
+
loss_weights: torch.Tensor,
|
| 889 |
+
sample_loss_weight: torch.Tensor,
|
| 890 |
+
) -> torch.Tensor:
|
| 891 |
+
"""Compute the same weighted CE without densifying every prompt token.
|
| 892 |
+
|
| 893 |
+
Qwen emits logits for the visual tokens, prompt and assistant answer, while
|
| 894 |
+
the semantic SFT contract supervises only assistant answer positions. The
|
| 895 |
+
previous implementation cast the complete ``batch x sequence x vocab``
|
| 896 |
+
tensor to fp32 and then discarded all ignored positions. Selecting active
|
| 897 |
+
positions first is mathematically equivalent and avoids the transient
|
| 898 |
+
13--20 GiB allocation seen on long examples.
|
| 899 |
+
"""
|
| 900 |
+
|
| 901 |
+
if logits.ndim != 3 or labels.shape != logits.shape[:2]:
|
| 902 |
+
raise RuntimeError("logits and labels do not align")
|
| 903 |
+
if loss_weights.shape != labels.shape:
|
| 904 |
+
raise RuntimeError("token loss weights do not align with labels")
|
| 905 |
+
if sample_loss_weight.shape != (logits.shape[0],):
|
| 906 |
+
raise RuntimeError("sample loss weights do not align with the batch")
|
| 907 |
+
|
| 908 |
+
active = labels.ne(-100) & loss_weights.gt(0)
|
| 909 |
+
active_batch, _ = torch.nonzero(active, as_tuple=True)
|
| 910 |
+
if active_batch.numel() == 0:
|
| 911 |
+
raise RuntimeError("batch has no positively weighted assistant labels")
|
| 912 |
+
active_logits = logits[active].float()
|
| 913 |
+
active_labels = labels[active]
|
| 914 |
+
active_weights = loss_weights[active].float()
|
| 915 |
+
token_loss = torch.nn.functional.cross_entropy(
|
| 916 |
+
active_logits, active_labels, reduction="none"
|
| 917 |
+
)
|
| 918 |
+
numerator = torch.zeros(
|
| 919 |
+
logits.shape[0], device=logits.device, dtype=torch.float32
|
| 920 |
+
).scatter_add_(0, active_batch, token_loss * active_weights)
|
| 921 |
+
denominator = torch.zeros_like(numerator).scatter_add_(
|
| 922 |
+
0, active_batch, active_weights
|
| 923 |
+
)
|
| 924 |
+
per_sample_loss = numerator / denominator.clamp_min(1e-8)
|
| 925 |
+
return (per_sample_loss * sample_loss_weight.float()).mean()
|
| 926 |
+
|
| 927 |
+
|
| 928 |
+
def inferred_root_manifest(release_root: Path) -> dict[str, Any]:
|
| 929 |
+
path = release_root / "manifest.json"
|
| 930 |
+
if not path.is_file():
|
| 931 |
+
raise RuntimeError("sealed release manifest is absent")
|
| 932 |
+
return json.loads(path.read_text(encoding="utf-8"))
|
| 933 |
+
|
| 934 |
+
|
| 935 |
+
def _load_stage_lr_plan(
|
| 936 |
+
path: Path,
|
| 937 |
+
*,
|
| 938 |
+
release_manifest: dict[str, Any],
|
| 939 |
+
dataset_rows: int,
|
| 940 |
+
global_batch: int,
|
| 941 |
+
optimizer_learning_rate: float,
|
| 942 |
+
warmup_ratio: float,
|
| 943 |
+
) -> dict[str, Any]:
|
| 944 |
+
"""Validate and compile row boundaries into optimizer-step boundaries."""
|
| 945 |
+
|
| 946 |
+
raw = json.loads(path.read_text(encoding="utf-8"))
|
| 947 |
+
if raw.get("format") != "iclr_video.stage_lr_plan.v1":
|
| 948 |
+
raise RuntimeError("unsupported stage LR plan format")
|
| 949 |
+
if int(raw.get("dataset_rows", -1)) != dataset_rows or global_batch <= 0:
|
| 950 |
+
raise RuntimeError("stage LR plan dataset/global-batch contract diverged")
|
| 951 |
+
if int(raw.get("global_batch", -1)) != global_batch:
|
| 952 |
+
raise RuntimeError("stage LR plan global batch diverged")
|
| 953 |
+
manifest_stages = (release_manifest.get("inputs") or {}).get(
|
| 954 |
+
"curriculum_stages"
|
| 955 |
+
)
|
| 956 |
+
stages = raw.get("stages")
|
| 957 |
+
if not isinstance(stages, list) or not stages or not isinstance(manifest_stages, list):
|
| 958 |
+
raise RuntimeError("stage LR plan or release stages are absent")
|
| 959 |
+
if len(stages) != len(manifest_stages):
|
| 960 |
+
raise RuntimeError("stage LR plan count diverged from release")
|
| 961 |
+
compiled = []
|
| 962 |
+
previous_row = 0
|
| 963 |
+
previous_step = 0
|
| 964 |
+
maximum_lr = 0.0
|
| 965 |
+
for index, (stage, manifest_stage) in enumerate(
|
| 966 |
+
zip(stages, manifest_stages, strict=True)
|
| 967 |
+
):
|
| 968 |
+
end_row = int(stage.get("end_row_exclusive", -1))
|
| 969 |
+
learning_rate = float(stage.get("learning_rate", 0.0))
|
| 970 |
+
if (
|
| 971 |
+
int(stage.get("index", -1)) != index
|
| 972 |
+
or int(manifest_stage.get("index", -1)) != index
|
| 973 |
+
or str(stage.get("release_id")) != str(manifest_stage.get("release_id"))
|
| 974 |
+
or end_row != int(manifest_stage.get("end_row_exclusive", -1))
|
| 975 |
+
or end_row <= previous_row
|
| 976 |
+
or not math.isfinite(learning_rate)
|
| 977 |
+
or learning_rate <= 0
|
| 978 |
+
):
|
| 979 |
+
raise RuntimeError("stage LR entry diverged from release manifest")
|
| 980 |
+
end_step = math.ceil(end_row / global_batch)
|
| 981 |
+
if end_step <= previous_step:
|
| 982 |
+
raise RuntimeError("stage has no optimizer steps")
|
| 983 |
+
stage_steps = end_step - previous_step
|
| 984 |
+
warmup_steps = math.ceil(stage_steps * warmup_ratio)
|
| 985 |
+
compiled.append(
|
| 986 |
+
{
|
| 987 |
+
"end_step_exclusive": end_step,
|
| 988 |
+
"index": index,
|
| 989 |
+
"learning_rate": learning_rate,
|
| 990 |
+
"release_id": str(stage["release_id"]),
|
| 991 |
+
"start_step": previous_step,
|
| 992 |
+
"warmup_steps": warmup_steps,
|
| 993 |
+
}
|
| 994 |
+
)
|
| 995 |
+
maximum_lr = max(maximum_lr, learning_rate)
|
| 996 |
+
previous_row = end_row
|
| 997 |
+
previous_step = end_step
|
| 998 |
+
if previous_row != dataset_rows:
|
| 999 |
+
raise RuntimeError("stage LR plan does not cover the complete dataset")
|
| 1000 |
+
if not math.isclose(
|
| 1001 |
+
maximum_lr, optimizer_learning_rate, rel_tol=1e-9, abs_tol=0.0
|
| 1002 |
+
):
|
| 1003 |
+
raise RuntimeError("optimizer LR must equal the maximum stage LR")
|
| 1004 |
+
return {
|
| 1005 |
+
"base_learning_rate": optimizer_learning_rate,
|
| 1006 |
+
"format": raw["format"],
|
| 1007 |
+
"stages": compiled,
|
| 1008 |
+
"total_steps": previous_step,
|
| 1009 |
+
}
|
| 1010 |
+
|
| 1011 |
+
|
| 1012 |
+
def _stage_lr_multiplier(plan: dict[str, Any], step: int) -> float:
|
| 1013 |
+
"""Return a per-stage warmup/cosine multiplier for one optimizer step."""
|
| 1014 |
+
|
| 1015 |
+
base_lr = float(plan["base_learning_rate"])
|
| 1016 |
+
for stage in plan["stages"]:
|
| 1017 |
+
start = int(stage["start_step"])
|
| 1018 |
+
end = int(stage["end_step_exclusive"])
|
| 1019 |
+
if step >= end:
|
| 1020 |
+
continue
|
| 1021 |
+
offset = max(step - start, 0)
|
| 1022 |
+
duration = end - start
|
| 1023 |
+
warmup = int(stage["warmup_steps"])
|
| 1024 |
+
scale = float(stage["learning_rate"]) / base_lr
|
| 1025 |
+
if warmup > 0 and offset < warmup:
|
| 1026 |
+
return scale * (offset / warmup)
|
| 1027 |
+
decay_steps = max(duration - warmup, 1)
|
| 1028 |
+
progress = min(max((offset - warmup) / decay_steps, 0.0), 1.0)
|
| 1029 |
+
return scale * 0.5 * (1.0 + math.cos(math.pi * progress))
|
| 1030 |
+
return 0.0
|
| 1031 |
+
|
| 1032 |
+
|
| 1033 |
+
def _training_args(args: argparse.Namespace, has_eval: bool) -> Any:
|
| 1034 |
+
kwargs: Dict[str, Any] = {
|
| 1035 |
+
"output_dir": str(args.output),
|
| 1036 |
+
"num_train_epochs": args.epochs,
|
| 1037 |
+
"per_device_train_batch_size": args.per_device_batch_size,
|
| 1038 |
+
"gradient_accumulation_steps": args.gradient_accumulation_steps,
|
| 1039 |
+
"learning_rate": args.lr,
|
| 1040 |
+
"warmup_ratio": args.warmup_ratio,
|
| 1041 |
+
"bf16": True,
|
| 1042 |
+
"logging_steps": args.logging_steps,
|
| 1043 |
+
"save_steps": args.save_steps,
|
| 1044 |
+
"save_total_limit": args.save_total_limit,
|
| 1045 |
+
"max_steps": args.max_steps,
|
| 1046 |
+
"gradient_checkpointing": True,
|
| 1047 |
+
"lr_scheduler_type": args.lr_scheduler_type,
|
| 1048 |
+
"optim": "adamw_torch",
|
| 1049 |
+
"weight_decay": 0.0,
|
| 1050 |
+
"max_grad_norm": 1.0,
|
| 1051 |
+
"tf32": True,
|
| 1052 |
+
"save_only_model": True,
|
| 1053 |
+
"report_to": "none",
|
| 1054 |
+
"remove_unused_columns": False,
|
| 1055 |
+
"ddp_find_unused_parameters": False,
|
| 1056 |
+
"dataloader_num_workers": args.dataloader_num_workers,
|
| 1057 |
+
"dataloader_pin_memory": True,
|
| 1058 |
+
"seed": args.seed,
|
| 1059 |
+
"deepspeed": str(args.deepspeed),
|
| 1060 |
+
}
|
| 1061 |
+
signature = inspect.signature(TrainingArguments.__init__)
|
| 1062 |
+
if args.save_steps >= 999999 and "save_strategy" in signature.parameters:
|
| 1063 |
+
kwargs["save_strategy"] = "no"
|
| 1064 |
+
if args.dataloader_num_workers > 0:
|
| 1065 |
+
if "dataloader_persistent_workers" in signature.parameters:
|
| 1066 |
+
kwargs["dataloader_persistent_workers"] = True
|
| 1067 |
+
if "dataloader_prefetch_factor" in signature.parameters:
|
| 1068 |
+
kwargs["dataloader_prefetch_factor"] = 2
|
| 1069 |
+
if "data_seed" in signature.parameters:
|
| 1070 |
+
kwargs["data_seed"] = args.seed
|
| 1071 |
+
evaluation_key = (
|
| 1072 |
+
"eval_strategy"
|
| 1073 |
+
if "eval_strategy" in signature.parameters
|
| 1074 |
+
else "evaluation_strategy"
|
| 1075 |
+
)
|
| 1076 |
+
kwargs[evaluation_key] = "steps" if has_eval else "no"
|
| 1077 |
+
if has_eval:
|
| 1078 |
+
kwargs["eval_steps"] = max(args.save_steps, args.logging_steps)
|
| 1079 |
+
return TrainingArguments(**kwargs)
|
| 1080 |
+
|
| 1081 |
+
|
| 1082 |
+
def _is_main_process() -> bool:
|
| 1083 |
+
return int(os.environ.get("RANK", "0")) == 0
|
| 1084 |
+
|
| 1085 |
+
|
| 1086 |
+
def _logical_parameter_numel(parameter: torch.nn.Parameter) -> int:
|
| 1087 |
+
return int(getattr(parameter, "ds_numel", parameter.numel()))
|
| 1088 |
+
|
| 1089 |
+
|
| 1090 |
+
if __name__ == "__main__":
|
| 1091 |
+
main()
|
code/experiments/tvs_correctness.py
ADDED
|
@@ -0,0 +1,80 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Official-protocol per-query correctness helpers for TVS analyses."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import json
|
| 6 |
+
import re
|
| 7 |
+
from pathlib import Path
|
| 8 |
+
from typing import Any
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
OVO_FORWARD_TASKS = {"REC", "SSR", "CRR"}
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
def _mcq_letter(value: Any) -> str | None:
|
| 15 |
+
text = str(value or "").strip()
|
| 16 |
+
match = re.match(r"^[\(\[]?([A-D])(?:[\)\].:\s-]|$)", text, re.IGNORECASE)
|
| 17 |
+
return match.group(1).upper() if match else None
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
def load_gold(benchmark: str, annotation: Path) -> dict[str, dict[str, Any]]:
|
| 21 |
+
data = json.loads(annotation.read_text())
|
| 22 |
+
gold: dict[str, dict[str, Any]] = {}
|
| 23 |
+
if benchmark == "streamingbench":
|
| 24 |
+
for video_index, entry in enumerate(data):
|
| 25 |
+
for question_index, question in enumerate(entry.get("questions") or []):
|
| 26 |
+
uid = f"streaming_{video_index}_{question_index}"
|
| 27 |
+
gold[uid] = {
|
| 28 |
+
"kind": "streaming_mcq",
|
| 29 |
+
"answer": str(question["answer"]),
|
| 30 |
+
"task": str(question.get("task_type") or ""),
|
| 31 |
+
}
|
| 32 |
+
return gold
|
| 33 |
+
if benchmark != "ovo":
|
| 34 |
+
raise ValueError(f"unsupported benchmark: {benchmark}")
|
| 35 |
+
for item in data:
|
| 36 |
+
task = str(item.get("task") or "").upper()
|
| 37 |
+
if task not in OVO_FORWARD_TASKS:
|
| 38 |
+
uid = f"ovo_{item['id']}"
|
| 39 |
+
gold[uid] = {
|
| 40 |
+
"kind": "ovo_mcq",
|
| 41 |
+
"answer": chr(ord("A") + int(item["gt"])),
|
| 42 |
+
"task": task,
|
| 43 |
+
}
|
| 44 |
+
continue
|
| 45 |
+
for index, test_info in enumerate(item.get("test_info") or []):
|
| 46 |
+
uid = f"ovo_{item['id']}_{index}"
|
| 47 |
+
if task == "REC":
|
| 48 |
+
gold[uid] = {
|
| 49 |
+
"kind": "ovo_rec",
|
| 50 |
+
"answer": str(test_info["count"]),
|
| 51 |
+
"task": task,
|
| 52 |
+
}
|
| 53 |
+
else:
|
| 54 |
+
gold[uid] = {
|
| 55 |
+
"kind": "ovo_binary",
|
| 56 |
+
"answer": int(test_info["type"]),
|
| 57 |
+
"task": task,
|
| 58 |
+
}
|
| 59 |
+
return gold
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
def is_correct(prediction: Any, gold: dict[str, Any]) -> bool:
|
| 63 |
+
text = str(prediction or "")
|
| 64 |
+
kind = gold["kind"]
|
| 65 |
+
if kind == "streaming_mcq":
|
| 66 |
+
letter = _mcq_letter(text)
|
| 67 |
+
response = letter if letter is not None else text
|
| 68 |
+
return response == str(gold["answer"])
|
| 69 |
+
if kind == "ovo_mcq":
|
| 70 |
+
letter = _mcq_letter(text)
|
| 71 |
+
response = letter if letter is not None else text
|
| 72 |
+
return str(gold["answer"]) in response
|
| 73 |
+
if kind == "ovo_rec":
|
| 74 |
+
return "".join(re.findall(r"\d+", text)) == str(gold["answer"])
|
| 75 |
+
if kind == "ovo_binary":
|
| 76 |
+
positive = int(gold["answer"]) == 1
|
| 77 |
+
if (text == "Y" and positive) or (text == "N" and not positive):
|
| 78 |
+
return True
|
| 79 |
+
return ("Yes" if positive else "No") in text
|
| 80 |
+
raise ValueError(f"unknown gold kind: {kind}")
|
code/pyproject.toml
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[build-system]
|
| 2 |
+
requires = ["setuptools>=61", "wheel"]
|
| 3 |
+
build-backend = "setuptools.build_meta"
|
| 4 |
+
|
| 5 |
+
[project]
|
| 6 |
+
name = "viss-code"
|
| 7 |
+
version = "0.3.1"
|
| 8 |
+
description = "Visible-State Supervision and Test-Time Causal View Search"
|
| 9 |
+
readme = "README.md"
|
| 10 |
+
requires-python = ">=3.10"
|
| 11 |
+
license = {file = "LICENSE"}
|
| 12 |
+
dependencies = ["numpy>=1.24,<3", "Pillow>=10,<12"]
|
| 13 |
+
|
| 14 |
+
[project.optional-dependencies]
|
| 15 |
+
test = ["pytest>=8,<10", "torch>=2.7,<3"]
|
| 16 |
+
inference = [
|
| 17 |
+
"torch==2.7.0",
|
| 18 |
+
"transformers==4.57.3",
|
| 19 |
+
"accelerate==1.8.1",
|
| 20 |
+
"qwen-vl-utils>=0.0.11,<0.1",
|
| 21 |
+
"decord==0.6.0",
|
| 22 |
+
]
|
| 23 |
+
training = ["deepspeed>=0.16,<0.18"]
|
| 24 |
+
|
| 25 |
+
[tool.setuptools.packages.find]
|
| 26 |
+
where = ["."]
|
| 27 |
+
include = ["viss_core*", "experiments*", "data_pipeline*", "scripts*"]
|
| 28 |
+
namespaces = true
|
| 29 |
+
|
| 30 |
+
[tool.pytest.ini_options]
|
| 31 |
+
testpaths = ["tests"]
|
| 32 |
+
pythonpath = ["."]
|
code/scripts/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
"""Version-neutral command-line helpers for the Data V0 release."""
|
code/scripts/configure_evaluation_protocol.py
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Bind the public protocol template to the user's own evaluation assets."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import argparse
|
| 6 |
+
import json
|
| 7 |
+
from pathlib import Path
|
| 8 |
+
|
| 9 |
+
from experiments.evaluate_qwen3vl_recent4_full import (
|
| 10 |
+
sha256_file,
|
| 11 |
+
validate_processor_contract,
|
| 12 |
+
)
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
def configure(
|
| 16 |
+
*,
|
| 17 |
+
template: Path,
|
| 18 |
+
streaming_samples: Path,
|
| 19 |
+
ovo_samples: Path,
|
| 20 |
+
processor_config: Path,
|
| 21 |
+
output: Path,
|
| 22 |
+
) -> None:
|
| 23 |
+
protocol = json.loads(template.read_text(encoding="utf-8"))
|
| 24 |
+
for benchmark, path in (
|
| 25 |
+
("streamingbench", streaming_samples),
|
| 26 |
+
("ovo", ovo_samples),
|
| 27 |
+
):
|
| 28 |
+
path = path.resolve(strict=True)
|
| 29 |
+
protocol["benchmarks"][benchmark]["export"] = str(path)
|
| 30 |
+
protocol["benchmarks"][benchmark]["export_sha256"] = sha256_file(path)
|
| 31 |
+
processor_config = processor_config.resolve(strict=True)
|
| 32 |
+
if processor_config.name != "preprocessor_config.json":
|
| 33 |
+
raise ValueError("Pass the processor's preprocessor_config.json")
|
| 34 |
+
protocol["inference"]["processor_contract"]["preprocessor_config_sha256"] = (
|
| 35 |
+
sha256_file(processor_config)
|
| 36 |
+
)
|
| 37 |
+
validate_processor_contract(processor_config.parent, protocol)
|
| 38 |
+
protocol["status"] = "locked"
|
| 39 |
+
output.parent.mkdir(parents=True, exist_ok=True)
|
| 40 |
+
with output.open("x", encoding="utf-8") as handle:
|
| 41 |
+
json.dump(protocol, handle, indent=2)
|
| 42 |
+
handle.write("\n")
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def main() -> None:
|
| 46 |
+
parser = argparse.ArgumentParser(description=__doc__)
|
| 47 |
+
parser.add_argument(
|
| 48 |
+
"--template",
|
| 49 |
+
type=Path,
|
| 50 |
+
default=Path("configs/evaluation/recent4.example.json"),
|
| 51 |
+
)
|
| 52 |
+
parser.add_argument("--streaming-samples", type=Path, required=True)
|
| 53 |
+
parser.add_argument("--ovo-samples", type=Path, required=True)
|
| 54 |
+
parser.add_argument("--processor-config", type=Path, required=True)
|
| 55 |
+
parser.add_argument("--output", type=Path, required=True)
|
| 56 |
+
args = parser.parse_args()
|
| 57 |
+
configure(**vars(args))
|
| 58 |
+
print(f"Created local evaluation protocol: {args.output}")
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
if __name__ == "__main__":
|
| 62 |
+
main()
|
code/scripts/merge_eval_shards.py
ADDED
|
@@ -0,0 +1,483 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Merge disjoint agent-evaluation shards into canonical official UID order."""
|
| 3 |
+
|
| 4 |
+
from __future__ import annotations
|
| 5 |
+
|
| 6 |
+
import argparse
|
| 7 |
+
import hashlib
|
| 8 |
+
import json
|
| 9 |
+
import os
|
| 10 |
+
from collections import defaultdict
|
| 11 |
+
from pathlib import Path
|
| 12 |
+
from statistics import fmean
|
| 13 |
+
from typing import Any
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
IDENTITY_FIELDS = (
|
| 17 |
+
"format",
|
| 18 |
+
"classification",
|
| 19 |
+
"benchmark_selected",
|
| 20 |
+
"source_policy_gate_admitted",
|
| 21 |
+
"schema_version",
|
| 22 |
+
"benchmark",
|
| 23 |
+
"model",
|
| 24 |
+
"model_config_sha256",
|
| 25 |
+
"model_manifest_sha256",
|
| 26 |
+
"processor",
|
| 27 |
+
"processor_sha256",
|
| 28 |
+
"processor_contract",
|
| 29 |
+
"controller_processor",
|
| 30 |
+
"controller_processor_sha256",
|
| 31 |
+
"answer_processor",
|
| 32 |
+
"answer_processor_contract",
|
| 33 |
+
"processor_geometry",
|
| 34 |
+
"adapter",
|
| 35 |
+
"evaluator",
|
| 36 |
+
"evaluation_protocol",
|
| 37 |
+
"evaluation_protocol_sha256",
|
| 38 |
+
"protocol_validation",
|
| 39 |
+
"policy_hash",
|
| 40 |
+
"policy_head",
|
| 41 |
+
"policy_head_sha256",
|
| 42 |
+
"policy_report_sha256",
|
| 43 |
+
"policy_calibration_sha256",
|
| 44 |
+
"global_action_threshold",
|
| 45 |
+
"policy_inputs",
|
| 46 |
+
"budget",
|
| 47 |
+
"policy_training_report",
|
| 48 |
+
"one_shared_commit_policy",
|
| 49 |
+
"benchmark_task_or_group_router",
|
| 50 |
+
"question_keyword_router",
|
| 51 |
+
"answer_space_router",
|
| 52 |
+
"gold_answer_available_to_commit_policy",
|
| 53 |
+
"commit_policy",
|
| 54 |
+
"unparseable_or_error_policy",
|
| 55 |
+
"maximum_unique_frames",
|
| 56 |
+
"direct_prediction_cache",
|
| 57 |
+
"direct_prediction_cache_sha256",
|
| 58 |
+
"source_gate",
|
| 59 |
+
"source_gate_sha256",
|
| 60 |
+
"global_action_thresholds",
|
| 61 |
+
"acquisition_identity",
|
| 62 |
+
"action_abi_schema_sha256",
|
| 63 |
+
"inference_policy_hash",
|
| 64 |
+
"inference_policy",
|
| 65 |
+
"policy_config",
|
| 66 |
+
"comparable_under_locked_recent4_protocol",
|
| 67 |
+
"generation_config",
|
| 68 |
+
"execution_optimization",
|
| 69 |
+
"system_prompt_sha256",
|
| 70 |
+
"prompt_contract_sha256",
|
| 71 |
+
"launch_marker",
|
| 72 |
+
"launch_marker_sha256",
|
| 73 |
+
"annotation",
|
| 74 |
+
"annotation_sha256",
|
| 75 |
+
"samples",
|
| 76 |
+
"samples_sha256",
|
| 77 |
+
"runner_sha256",
|
| 78 |
+
"software",
|
| 79 |
+
"policy",
|
| 80 |
+
"identity_hash",
|
| 81 |
+
"protocol_sha256",
|
| 82 |
+
"evaluator_sha256",
|
| 83 |
+
"generation",
|
| 84 |
+
)
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
def sha256(path: Path) -> str:
|
| 88 |
+
digest = hashlib.sha256()
|
| 89 |
+
with path.open("rb") as handle:
|
| 90 |
+
for block in iter(lambda: handle.read(8 << 20), b""):
|
| 91 |
+
digest.update(block)
|
| 92 |
+
return digest.hexdigest()
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
def load_expected_uids(path: Path) -> list[str]:
|
| 96 |
+
uids: list[str] = []
|
| 97 |
+
for line_number, line in enumerate(
|
| 98 |
+
path.read_text(encoding="utf-8").splitlines(), start=1
|
| 99 |
+
):
|
| 100 |
+
if not line.strip():
|
| 101 |
+
continue
|
| 102 |
+
uid = str(json.loads(line).get("uid") or "")
|
| 103 |
+
if not uid:
|
| 104 |
+
raise ValueError(f"{path}:{line_number} has no uid")
|
| 105 |
+
uids.append(uid)
|
| 106 |
+
if len(uids) != len(set(uids)):
|
| 107 |
+
raise ValueError(f"{path} contains duplicate uids")
|
| 108 |
+
return uids
|
| 109 |
+
|
| 110 |
+
|
| 111 |
+
def grouped_accuracy(rows: list[dict[str, Any]], field: str) -> dict[str, Any]:
|
| 112 |
+
grouped: dict[str, list[dict[str, Any]]] = defaultdict(list)
|
| 113 |
+
for row in rows:
|
| 114 |
+
grouped[str(row.get(field) or "unknown")].append(row)
|
| 115 |
+
result: dict[str, Any] = {}
|
| 116 |
+
for name in sorted(grouped):
|
| 117 |
+
group_rows = grouped[name]
|
| 118 |
+
correct = sum(int(row.get("correct") or 0) for row in group_rows)
|
| 119 |
+
total = len(group_rows)
|
| 120 |
+
result[name] = {
|
| 121 |
+
"accuracy": correct / total if total else 0.0,
|
| 122 |
+
"correct": correct,
|
| 123 |
+
"total": total,
|
| 124 |
+
}
|
| 125 |
+
return result
|
| 126 |
+
|
| 127 |
+
|
| 128 |
+
def aggregate_score(
|
| 129 |
+
score_rows: list[dict[str, Any]], benchmark: str
|
| 130 |
+
) -> dict[str, Any]:
|
| 131 |
+
correct = sum(int(row.get("correct") or 0) for row in score_rows)
|
| 132 |
+
total = len(score_rows)
|
| 133 |
+
by_task = grouped_accuracy(score_rows, "task")
|
| 134 |
+
by_group = grouped_accuracy(score_rows, "group")
|
| 135 |
+
score: dict[str, Any] = {
|
| 136 |
+
"accuracy": correct / total if total else 0.0,
|
| 137 |
+
"by_group": by_group,
|
| 138 |
+
"by_task": by_task,
|
| 139 |
+
"num_queries": total,
|
| 140 |
+
"rows": score_rows,
|
| 141 |
+
}
|
| 142 |
+
if benchmark == "ovo":
|
| 143 |
+
tasks_by_group: dict[str, set[str]] = defaultdict(set)
|
| 144 |
+
for row in score_rows:
|
| 145 |
+
tasks_by_group[str(row.get("group") or "unknown")].add(
|
| 146 |
+
str(row.get("task") or "unknown")
|
| 147 |
+
)
|
| 148 |
+
official_groups: dict[str, Any] = {}
|
| 149 |
+
for group in sorted(tasks_by_group):
|
| 150 |
+
task_names = sorted(tasks_by_group[group])
|
| 151 |
+
official_groups[group] = {
|
| 152 |
+
"accuracy": fmean(by_task[name]["accuracy"] for name in task_names),
|
| 153 |
+
"num_tasks": len(task_names),
|
| 154 |
+
}
|
| 155 |
+
score["ovo_official"] = {
|
| 156 |
+
"accuracy": fmean(
|
| 157 |
+
value["accuracy"] for value in official_groups.values()
|
| 158 |
+
),
|
| 159 |
+
"by_group": official_groups,
|
| 160 |
+
}
|
| 161 |
+
return score
|
| 162 |
+
|
| 163 |
+
|
| 164 |
+
def aggregate_efficiency(rows: list[dict[str, Any]]) -> dict[str, float]:
|
| 165 |
+
def mean(field: str, *fallbacks: str) -> float:
|
| 166 |
+
def value(row: dict[str, Any]) -> float:
|
| 167 |
+
for name in (field, *fallbacks):
|
| 168 |
+
if row.get(name) is not None:
|
| 169 |
+
return float(row[name])
|
| 170 |
+
return 0.0
|
| 171 |
+
|
| 172 |
+
return fmean(value(row) for row in rows) if rows else 0.0
|
| 173 |
+
|
| 174 |
+
result = {
|
| 175 |
+
"avg_frames_observed": mean(
|
| 176 |
+
"frames_observed", "unique_frames_acquired", "unique_frames"
|
| 177 |
+
),
|
| 178 |
+
"avg_model_processed_frames": mean(
|
| 179 |
+
"model_processed_frames", "processed_frames"
|
| 180 |
+
),
|
| 181 |
+
"avg_media_calls": mean("media_calls"),
|
| 182 |
+
"avg_total_tokens": (
|
| 183 |
+
mean("total_tokens")
|
| 184 |
+
if any(row.get("total_tokens") is not None for row in rows)
|
| 185 |
+
else mean("input_tokens") + mean("output_tokens")
|
| 186 |
+
),
|
| 187 |
+
"avg_turns": mean("num_turns"),
|
| 188 |
+
}
|
| 189 |
+
if any(row.get("commit_latency_seconds") is not None for row in rows):
|
| 190 |
+
result["avg_commit_latency_seconds"] = mean("commit_latency_seconds")
|
| 191 |
+
if any(row.get("commit_visual_tokens") is not None for row in rows):
|
| 192 |
+
result["avg_commit_visual_tokens"] = mean("commit_visual_tokens")
|
| 193 |
+
if any(row.get("latency_seconds") is not None for row in rows):
|
| 194 |
+
result["avg_latency_seconds"] = mean("latency_seconds")
|
| 195 |
+
return result
|
| 196 |
+
|
| 197 |
+
|
| 198 |
+
def aggregate_agent_behavior(rows: list[dict[str, Any]]) -> dict[str, Any]:
|
| 199 |
+
terminal_counts: dict[str, int] = defaultdict(int)
|
| 200 |
+
for row in rows:
|
| 201 |
+
terminal_counts[str(row.get("policy_terminal_event") or "unknown")] += 1
|
| 202 |
+
total = len(rows)
|
| 203 |
+
failures = sum(
|
| 204 |
+
terminal_counts[event]
|
| 205 |
+
for event in ("malformed_action", "illegal_observe", "missing_answer")
|
| 206 |
+
)
|
| 207 |
+
return {
|
| 208 |
+
"terminal_counts": dict(sorted(terminal_counts.items())),
|
| 209 |
+
"num_policy_failures": failures,
|
| 210 |
+
"strict_terminal_answer_rate": (
|
| 211 |
+
terminal_counts["answered"] / total if total else 0.0
|
| 212 |
+
),
|
| 213 |
+
"malformed_action_rate": (
|
| 214 |
+
terminal_counts["malformed_action"] / total if total else 0.0
|
| 215 |
+
),
|
| 216 |
+
"illegal_observe_rate": (
|
| 217 |
+
terminal_counts["illegal_observe"] / total if total else 0.0
|
| 218 |
+
),
|
| 219 |
+
"missing_answer_rate": (
|
| 220 |
+
terminal_counts["missing_answer"] / total if total else 0.0
|
| 221 |
+
),
|
| 222 |
+
}
|
| 223 |
+
|
| 224 |
+
|
| 225 |
+
def aggregate_execution_statistics(
|
| 226 |
+
shard_statistics: list[dict[str, Any]],
|
| 227 |
+
) -> dict[str, Any]:
|
| 228 |
+
"""Sum disjoint-shard cache counters and per-row stage timings."""
|
| 229 |
+
|
| 230 |
+
counter_fields = (
|
| 231 |
+
"decode_window_cache_hits",
|
| 232 |
+
"decode_window_cache_misses",
|
| 233 |
+
"decode_window_singleflight_hits",
|
| 234 |
+
"temporal_budget_cache_hits",
|
| 235 |
+
"temporal_budget_cache_misses",
|
| 236 |
+
"video_reader_cache_hits",
|
| 237 |
+
"video_reader_cache_misses",
|
| 238 |
+
"stage_timing_samples",
|
| 239 |
+
)
|
| 240 |
+
stage_totals: dict[str, float] = defaultdict(float)
|
| 241 |
+
result = {
|
| 242 |
+
field: sum(int(stats.get(field) or 0) for stats in shard_statistics)
|
| 243 |
+
for field in counter_fields
|
| 244 |
+
}
|
| 245 |
+
for stats in shard_statistics:
|
| 246 |
+
for name, seconds in (stats.get("stage_seconds_total") or {}).items():
|
| 247 |
+
stage_totals[str(name)] += float(seconds or 0.0)
|
| 248 |
+
result["stage_seconds_total"] = {
|
| 249 |
+
name: round(seconds, 6) for name, seconds in sorted(stage_totals.items())
|
| 250 |
+
}
|
| 251 |
+
return result
|
| 252 |
+
|
| 253 |
+
|
| 254 |
+
def main() -> None:
|
| 255 |
+
parser = argparse.ArgumentParser(description=__doc__)
|
| 256 |
+
parser.add_argument("--inputs", nargs="+", type=Path, required=True)
|
| 257 |
+
parser.add_argument("--output", type=Path, required=True)
|
| 258 |
+
parser.add_argument("--expected-count", type=int, default=-1)
|
| 259 |
+
parser.add_argument(
|
| 260 |
+
"--expected-uids",
|
| 261 |
+
type=Path,
|
| 262 |
+
help="Official JSONL export whose UID order becomes the canonical order.",
|
| 263 |
+
)
|
| 264 |
+
args = parser.parse_args()
|
| 265 |
+
|
| 266 |
+
rows_by_uid: dict[str, dict[str, Any]] = {}
|
| 267 |
+
score_by_uid: dict[str, dict[str, Any]] = {}
|
| 268 |
+
seen_in: dict[str, str] = {}
|
| 269 |
+
metadata: list[dict[str, Any]] = []
|
| 270 |
+
execution_statistics: list[dict[str, Any]] = []
|
| 271 |
+
row_repairs: list[dict[str, Any]] = []
|
| 272 |
+
identity: dict[str, Any] | None = None
|
| 273 |
+
benchmark: str | None = None
|
| 274 |
+
|
| 275 |
+
for path in args.inputs:
|
| 276 |
+
data = json.loads(path.read_text(encoding="utf-8"))
|
| 277 |
+
# A sharded run intentionally records a different ``samples`` JSONL
|
| 278 |
+
# path (and therefore a different samples hash) in every shard. When
|
| 279 |
+
# a frozen full-export is supplied via --expected-uids, that export is
|
| 280 |
+
# the canonical merged sample manifest; all other identity fields
|
| 281 |
+
# must still match exactly.
|
| 282 |
+
comparable_identity_fields = tuple(
|
| 283 |
+
field
|
| 284 |
+
for field in IDENTITY_FIELDS
|
| 285 |
+
if args.expected_uids is None
|
| 286 |
+
or field not in {"samples", "samples_sha256"}
|
| 287 |
+
)
|
| 288 |
+
current_identity = {
|
| 289 |
+
field: data.get(field) for field in comparable_identity_fields
|
| 290 |
+
}
|
| 291 |
+
if identity is None:
|
| 292 |
+
identity = current_identity
|
| 293 |
+
benchmark = str(data.get("benchmark") or "") or None
|
| 294 |
+
elif current_identity != identity:
|
| 295 |
+
changed = [
|
| 296 |
+
field
|
| 297 |
+
for field in comparable_identity_fields
|
| 298 |
+
if current_identity.get(field) != identity.get(field)
|
| 299 |
+
]
|
| 300 |
+
# Annotation files are frequently remounted from a volatile
|
| 301 |
+
# location (for example /dev/shm) to persistent storage when an
|
| 302 |
+
# evaluation is resumed. The absolute path is not part of the
|
| 303 |
+
# benchmark identity when both shards carry the same non-empty
|
| 304 |
+
# content hash. Keep the hash itself in the frozen identity so
|
| 305 |
+
# different annotation contents remain a hard failure.
|
| 306 |
+
if (
|
| 307 |
+
"annotation" in changed
|
| 308 |
+
and current_identity.get("annotation_sha256")
|
| 309 |
+
and current_identity.get("annotation_sha256")
|
| 310 |
+
== identity.get("annotation_sha256")
|
| 311 |
+
):
|
| 312 |
+
changed.remove("annotation")
|
| 313 |
+
if not changed:
|
| 314 |
+
current_identity = identity
|
| 315 |
+
else:
|
| 316 |
+
raise ValueError(
|
| 317 |
+
f"{path} does not match the frozen shard identity: {changed}"
|
| 318 |
+
)
|
| 319 |
+
shard_rows = data.get("rows")
|
| 320 |
+
if not isinstance(shard_rows, list):
|
| 321 |
+
raise ValueError(f"{path} has no rows list")
|
| 322 |
+
for row in shard_rows:
|
| 323 |
+
uid = str(row.get("uid") or "")
|
| 324 |
+
if not uid:
|
| 325 |
+
raise ValueError(f"{path} contains a row without uid")
|
| 326 |
+
if uid in rows_by_uid:
|
| 327 |
+
raise ValueError(f"duplicate uid {uid!r} in {seen_in[uid]} and {path}")
|
| 328 |
+
rows_by_uid[uid] = row
|
| 329 |
+
seen_in[uid] = str(path)
|
| 330 |
+
row_benchmark = str(row.get("benchmark") or "")
|
| 331 |
+
if row_benchmark:
|
| 332 |
+
if benchmark is None:
|
| 333 |
+
benchmark = row_benchmark
|
| 334 |
+
elif row_benchmark != benchmark:
|
| 335 |
+
raise ValueError(
|
| 336 |
+
f"mixed benchmarks in shards: {benchmark!r} and {row_benchmark!r}"
|
| 337 |
+
)
|
| 338 |
+
shard_score_rows = (data.get("score") or {}).get("rows") or []
|
| 339 |
+
if not isinstance(shard_score_rows, list):
|
| 340 |
+
raise ValueError(f"{path} score.rows is not a list")
|
| 341 |
+
for score_row in shard_score_rows:
|
| 342 |
+
uid = str(score_row.get("query_id") or "")
|
| 343 |
+
if not uid:
|
| 344 |
+
raise ValueError(f"{path} contains a score row without query_id")
|
| 345 |
+
if uid in score_by_uid:
|
| 346 |
+
raise ValueError(f"duplicate score query_id {uid!r}")
|
| 347 |
+
score_by_uid[uid] = score_row
|
| 348 |
+
metadata.append(
|
| 349 |
+
{
|
| 350 |
+
"path": str(path),
|
| 351 |
+
"sha256": sha256(path),
|
| 352 |
+
"num_samples": len(shard_rows),
|
| 353 |
+
"sample_slice": data.get("sample_slice"),
|
| 354 |
+
"samples": data.get("samples"),
|
| 355 |
+
"samples_sha256": data.get("samples_sha256"),
|
| 356 |
+
"annotation": data.get("annotation"),
|
| 357 |
+
"annotation_sha256": data.get("annotation_sha256"),
|
| 358 |
+
}
|
| 359 |
+
)
|
| 360 |
+
if isinstance(data.get("execution_statistics"), dict):
|
| 361 |
+
execution_statistics.append(data["execution_statistics"])
|
| 362 |
+
row_repairs.extend(data.get("row_repairs") or [])
|
| 363 |
+
|
| 364 |
+
if args.expected_count >= 0 and len(rows_by_uid) != args.expected_count:
|
| 365 |
+
raise ValueError(
|
| 366 |
+
f"merged {len(rows_by_uid)} rows, expected {args.expected_count}"
|
| 367 |
+
)
|
| 368 |
+
|
| 369 |
+
expected_uids = (
|
| 370 |
+
load_expected_uids(args.expected_uids)
|
| 371 |
+
if args.expected_uids is not None
|
| 372 |
+
else list(rows_by_uid)
|
| 373 |
+
)
|
| 374 |
+
expected_set = set(expected_uids)
|
| 375 |
+
actual_set = set(rows_by_uid)
|
| 376 |
+
if actual_set != expected_set:
|
| 377 |
+
missing = [uid for uid in expected_uids if uid not in actual_set]
|
| 378 |
+
extra = sorted(actual_set - expected_set)
|
| 379 |
+
raise ValueError(
|
| 380 |
+
f"official UID mismatch: missing={missing[:20]} extra={extra[:20]}"
|
| 381 |
+
)
|
| 382 |
+
if set(score_by_uid) != expected_set:
|
| 383 |
+
missing = [uid for uid in expected_uids if uid not in score_by_uid]
|
| 384 |
+
extra = sorted(set(score_by_uid) - expected_set)
|
| 385 |
+
raise ValueError(
|
| 386 |
+
f"score UID mismatch: missing={missing[:20]} extra={extra[:20]}"
|
| 387 |
+
)
|
| 388 |
+
|
| 389 |
+
rows = [rows_by_uid[uid] for uid in expected_uids]
|
| 390 |
+
score_rows = [score_by_uid[uid] for uid in expected_uids]
|
| 391 |
+
empty_predictions = [
|
| 392 |
+
str(row.get("uid"))
|
| 393 |
+
for row in rows
|
| 394 |
+
if not str(row.get("prediction") or "").strip()
|
| 395 |
+
]
|
| 396 |
+
errors = [
|
| 397 |
+
row for row in rows if row.get("error") or row.get("commit_error")
|
| 398 |
+
]
|
| 399 |
+
merged_identity = dict(identity or {})
|
| 400 |
+
if args.expected_uids is not None:
|
| 401 |
+
merged_identity["samples"] = str(args.expected_uids)
|
| 402 |
+
merged_identity["samples_sha256"] = sha256(args.expected_uids)
|
| 403 |
+
result = {
|
| 404 |
+
"benchmark": benchmark,
|
| 405 |
+
**merged_identity,
|
| 406 |
+
"canonical_official_uid_order": args.expected_uids is not None,
|
| 407 |
+
"efficiency": aggregate_efficiency(rows),
|
| 408 |
+
"empty_prediction_uids": empty_predictions,
|
| 409 |
+
"num_empty_predictions": len(empty_predictions),
|
| 410 |
+
"num_errors": len(errors),
|
| 411 |
+
"num_future_frame_violations": sum(
|
| 412 |
+
bool(row.get("future_frame_violation")) for row in rows
|
| 413 |
+
),
|
| 414 |
+
"num_samples": len(rows),
|
| 415 |
+
"strict_terminal_answer_rate": (
|
| 416 |
+
sum(bool(row.get("strict_terminal_answer")) for row in rows) / len(rows)
|
| 417 |
+
if rows else 0.0
|
| 418 |
+
),
|
| 419 |
+
"score": aggregate_score(score_rows, benchmark or ""),
|
| 420 |
+
"row_repairs": row_repairs,
|
| 421 |
+
"source_shards": metadata,
|
| 422 |
+
"expected_uid_source": (
|
| 423 |
+
{
|
| 424 |
+
"path": str(args.expected_uids),
|
| 425 |
+
"sha256": sha256(args.expected_uids),
|
| 426 |
+
}
|
| 427 |
+
if args.expected_uids is not None
|
| 428 |
+
else None
|
| 429 |
+
),
|
| 430 |
+
"rows": rows,
|
| 431 |
+
}
|
| 432 |
+
if execution_statistics:
|
| 433 |
+
result["execution_statistics"] = aggregate_execution_statistics(
|
| 434 |
+
execution_statistics
|
| 435 |
+
)
|
| 436 |
+
if any("policy_terminal_event" in row for row in rows):
|
| 437 |
+
result["agent_behavior"] = aggregate_agent_behavior(rows)
|
| 438 |
+
result["strict_terminal_answer_rate"] = result["agent_behavior"][
|
| 439 |
+
"strict_terminal_answer_rate"
|
| 440 |
+
]
|
| 441 |
+
if any("commit_action" in row for row in rows):
|
| 442 |
+
result.update(
|
| 443 |
+
{
|
| 444 |
+
"num_commit_errors": sum(
|
| 445 |
+
bool(row.get("commit_error")) for row in rows
|
| 446 |
+
),
|
| 447 |
+
"num_unparsed": sum(
|
| 448 |
+
row.get("commit_parsed") is False for row in rows
|
| 449 |
+
),
|
| 450 |
+
"decision_counts": {
|
| 451 |
+
action: sum(row.get("commit_action") == action for row in rows)
|
| 452 |
+
for action in (
|
| 453 |
+
"not_applicable",
|
| 454 |
+
"unchanged",
|
| 455 |
+
"retain",
|
| 456 |
+
"commit",
|
| 457 |
+
)
|
| 458 |
+
},
|
| 459 |
+
}
|
| 460 |
+
)
|
| 461 |
+
args.output.parent.mkdir(parents=True, exist_ok=True)
|
| 462 |
+
temporary = args.output.with_suffix(args.output.suffix + f".tmp.{os.getpid()}")
|
| 463 |
+
temporary.write_text(
|
| 464 |
+
json.dumps(result, indent=2, sort_keys=True, ensure_ascii=False) + "\n",
|
| 465 |
+
encoding="utf-8",
|
| 466 |
+
)
|
| 467 |
+
temporary.replace(args.output)
|
| 468 |
+
print(
|
| 469 |
+
json.dumps(
|
| 470 |
+
{
|
| 471 |
+
"output": str(args.output),
|
| 472 |
+
"num_samples": len(rows),
|
| 473 |
+
"num_errors": len(errors),
|
| 474 |
+
"num_empty_predictions": len(empty_predictions),
|
| 475 |
+
"canonical_official_uid_order": args.expected_uids is not None,
|
| 476 |
+
},
|
| 477 |
+
sort_keys=True,
|
| 478 |
+
)
|
| 479 |
+
)
|
| 480 |
+
|
| 481 |
+
|
| 482 |
+
if __name__ == "__main__":
|
| 483 |
+
main()
|
code/scripts/run_recent4_single_benchmark.sh
ADDED
|
@@ -0,0 +1,380 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env bash
|
| 2 |
+
set -euo pipefail
|
| 3 |
+
|
| 4 |
+
# Run one half of the locked recent4_full_v1 evaluation. Two independent
|
| 5 |
+
# invocations may therefore evaluate StreamingBench and OVO concurrently on
|
| 6 |
+
# disjoint GPU sets while retaining the exact common runner and official
|
| 7 |
+
# scorers used by the serial baseline launcher.
|
| 8 |
+
|
| 9 |
+
if [[ $# -ne 8 ]]; then
|
| 10 |
+
echo "usage: $0 REPO MODEL MODEL_MANIFEST OUTPUT PYTHON BENCHMARK GPU_CSV MODEL_ID" >&2
|
| 11 |
+
exit 2
|
| 12 |
+
fi
|
| 13 |
+
|
| 14 |
+
REPO=$1
|
| 15 |
+
MODEL=$2
|
| 16 |
+
MODEL_MANIFEST=$3
|
| 17 |
+
OUTPUT=$4
|
| 18 |
+
PYTHON=$5
|
| 19 |
+
BENCHMARK=$6
|
| 20 |
+
GPU_CSV=$7
|
| 21 |
+
MODEL_ID=$8
|
| 22 |
+
PROCESSOR=${STREAMEVA_PROCESSOR:-$MODEL}
|
| 23 |
+
EVAL_BATCH_SIZE=${STREAMEVA_EVAL_BATCH_SIZE:-16}
|
| 24 |
+
DECODE_PREFETCH_BATCHES=${STREAMEVA_DECODE_PREFETCH_BATCHES:-2}
|
| 25 |
+
FLUSH_EVERY_BATCHES=${STREAMEVA_EVAL_FLUSH_EVERY_BATCHES:-16}
|
| 26 |
+
RECORD_CHOICE_CONFIDENCE=${STREAMEVA_RECORD_CHOICE_CONFIDENCE:-1}
|
| 27 |
+
FRAME_POLICY=${STREAMEVA_FRAME_POLICY:-recent4}
|
| 28 |
+
VISUAL_INPUT_MODE=${STREAMEVA_VISUAL_INPUT_MODE:-image-list}
|
| 29 |
+
ALLOW_SHARED_GPU=${STREAMEVA_ALLOW_SHARED_GPU:-0}
|
| 30 |
+
MIN_FREE_MIB=${STREAMEVA_MIN_FREE_MIB:-35840}
|
| 31 |
+
PROTOCOL=$REPO/configs/evaluation/recent4_full_v1.json
|
| 32 |
+
RUNNER=${STREAMEVA_EVALUATOR_ENTRYPOINT:-$REPO/experiments/evaluate_qwen3vl_recent4_full.py}
|
| 33 |
+
MODEL_IDENTITY_SHA256=${STREAMEVA_MODEL_IDENTITY_SHA256:-}
|
| 34 |
+
|
| 35 |
+
# Keep non-login tmux jobs aligned with the CUDA wheel runtime used by
|
| 36 |
+
# training. In particular, libcusparse from the wheel requires the matching
|
| 37 |
+
# bundled nvJitLink library to precede system CUDA libraries.
|
| 38 |
+
PYTHON_SITE=$($PYTHON -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')
|
| 39 |
+
NVIDIA_LIB_DIRS=$(find "$PYTHON_SITE/nvidia" -mindepth 2 -maxdepth 2 \
|
| 40 |
+
-type d -name lib -print 2>/dev/null | LC_ALL=C sort | paste -sd: -)
|
| 41 |
+
if [[ -n $NVIDIA_LIB_DIRS ]]; then
|
| 42 |
+
export LD_LIBRARY_PATH="$NVIDIA_LIB_DIRS${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
|
| 43 |
+
fi
|
| 44 |
+
[[ $EVAL_BATCH_SIZE =~ ^[1-9][0-9]*$ ]]
|
| 45 |
+
[[ $DECODE_PREFETCH_BATCHES =~ ^[0-9]+$ ]]
|
| 46 |
+
[[ $FLUSH_EVERY_BATCHES =~ ^[1-9][0-9]*$ ]]
|
| 47 |
+
[[ $RECORD_CHOICE_CONFIDENCE =~ ^[01]$ ]]
|
| 48 |
+
[[ $FRAME_POLICY =~ ^(recent4|wide8|wide12|wide16|prefix8|prefix16|prefix32|prefix64|causalbudget20)$ ]]
|
| 49 |
+
[[ $VISUAL_INPUT_MODE =~ ^(image-list|native-video|hybrid-history-video-recent-images)$ ]]
|
| 50 |
+
[[ $ALLOW_SHARED_GPU =~ ^[01]$ ]]
|
| 51 |
+
[[ $MIN_FREE_MIB =~ ^[1-9][0-9]*$ ]]
|
| 52 |
+
|
| 53 |
+
case "$BENCHMARK" in
|
| 54 |
+
streamingbench)
|
| 55 |
+
ANNOTATION=${STREAM_ANNOTATION:?Set STREAM_ANNOTATION to the StreamingBench annotation file}
|
| 56 |
+
SAMPLES=${STREAM_SAMPLES:?Set STREAM_SAMPLES to the frozen StreamingBench query export}
|
| 57 |
+
VIDEO_ROOT=${STREAM_VIDEO_ROOT:?Set STREAM_VIDEO_ROOT to authorized benchmark media}
|
| 58 |
+
OFFICIAL_ROOT=${STREAM_OFFICIAL_ROOT:?Set STREAM_OFFICIAL_ROOT to the official scorer}
|
| 59 |
+
EXPECTED=2495
|
| 60 |
+
SMOKE_UIDS=(streaming_0_0 streaming_8_0 streaming_61_4)
|
| 61 |
+
;;
|
| 62 |
+
ovo)
|
| 63 |
+
OVO_ROOT=${OVO_ROOT:?Set OVO_ROOT to authorized OVO runtime assets}
|
| 64 |
+
ANNOTATION=${OVO_ANNOTATION:-$OVO_ROOT/data/ovo_bench_new.json}
|
| 65 |
+
OVO_RUNTIME_REPORT=${OVO_RUNTIME_REPORT:-$OVO_ROOT/data/runtime_assets.json}
|
| 66 |
+
SAMPLES=${OVO_SAMPLES:?Set OVO_SAMPLES to the frozen OVO query export}
|
| 67 |
+
VIDEO_ROOT=${OVO_VIDEO_ROOT:-$OVO_ROOT}
|
| 68 |
+
OFFICIAL_ROOT=${OVO_OFFICIAL_ROOT:-$OVO_ROOT}
|
| 69 |
+
EXPECTED=3035
|
| 70 |
+
SMOKE_UIDS=(ovo_0 ovo_1117 ovo_1468_0 ovo_1516_0 ovo_1558_0)
|
| 71 |
+
;;
|
| 72 |
+
*) echo "BENCHMARK must be streamingbench or ovo" >&2; exit 2 ;;
|
| 73 |
+
esac
|
| 74 |
+
|
| 75 |
+
IFS=, read -r -a GPUS <<<"$GPU_CSV"
|
| 76 |
+
[[ ${#GPUS[@]} -gt 0 ]] || exit 2
|
| 77 |
+
declare -A SEEN=()
|
| 78 |
+
for gpu in "${GPUS[@]}"; do
|
| 79 |
+
[[ $gpu =~ ^[0-7]$ || $gpu =~ ^GPU-[0-9A-Fa-f-]{36}$ ]] || {
|
| 80 |
+
echo "GPU_CSV must contain local indices or GPU UUIDs" >&2
|
| 81 |
+
exit 2
|
| 82 |
+
}
|
| 83 |
+
if [[ -n ${SEEN[$gpu]:-} && $ALLOW_SHARED_GPU -eq 0 ]]; then
|
| 84 |
+
echo "GPU_CSV may repeat an index only with STREAMEVA_ALLOW_SHARED_GPU=1" >&2
|
| 85 |
+
exit 2
|
| 86 |
+
fi
|
| 87 |
+
SEEN[$gpu]=1
|
| 88 |
+
done
|
| 89 |
+
[[ $MODEL_ID =~ ^[A-Za-z0-9._-]+$ ]] || exit 2
|
| 90 |
+
for path in "$MODEL/config.json" \
|
| 91 |
+
"$PROCESSOR/preprocessor_config.json" "$PROTOCOL" "$RUNNER" \
|
| 92 |
+
"$ANNOTATION" "$SAMPLES" "$VIDEO_ROOT" "$OFFICIAL_ROOT"; do
|
| 93 |
+
test -e "$path"
|
| 94 |
+
done
|
| 95 |
+
if [[ $MODEL_MANIFEST == none ]]; then
|
| 96 |
+
[[ $MODEL_IDENTITY_SHA256 =~ ^[0-9a-f]{64}$ ]] || {
|
| 97 |
+
echo "manifest-free evaluation requires STREAMEVA_MODEL_IDENTITY_SHA256" >&2
|
| 98 |
+
exit 2
|
| 99 |
+
}
|
| 100 |
+
MODEL_IDENTITY_ARGS=(--model-identity-sha256 "$MODEL_IDENTITY_SHA256")
|
| 101 |
+
else
|
| 102 |
+
test -s "$MODEL_MANIFEST"
|
| 103 |
+
MODEL_IDENTITY_ARGS=(--model-manifest "$MODEL_MANIFEST")
|
| 104 |
+
fi
|
| 105 |
+
if [[ $BENCHMARK == ovo ]]; then
|
| 106 |
+
test -s "$OVO_RUNTIME_REPORT"
|
| 107 |
+
ANNOTATION_SHA256=$(sha256sum "$ANNOTATION" | cut -d' ' -f1)
|
| 108 |
+
FROZEN_EXPORT_SHA256=$(sha256sum "$SAMPLES" | cut -d' ' -f1)
|
| 109 |
+
jq -e --arg annotation_sha256 "$ANNOTATION_SHA256" \
|
| 110 |
+
--arg frozen_export_sha256 "$FROZEN_EXPORT_SHA256" '
|
| 111 |
+
.status == "pass"
|
| 112 |
+
and .expanded_queries == 3035
|
| 113 |
+
and .covered_queries == 3035
|
| 114 |
+
and .coverage == 1
|
| 115 |
+
and .missing_videos == 0
|
| 116 |
+
and .annotation_sha256 == $annotation_sha256
|
| 117 |
+
and .frozen_export_sha256 == $frozen_export_sha256
|
| 118 |
+
' "$OVO_RUNTIME_REPORT" >/dev/null
|
| 119 |
+
fi
|
| 120 |
+
test -x "$PYTHON"
|
| 121 |
+
export PYTHONPATH=$REPO${PYTHONPATH:+:$PYTHONPATH}
|
| 122 |
+
|
| 123 |
+
if [[ ${PREFLIGHT_ONLY:-0} == 1 ]]; then
|
| 124 |
+
"$PYTHON" -c 'import PIL, qwen_vl_utils, torch, transformers'
|
| 125 |
+
printf '{"status":"cpu_preflight_pass","benchmark":"%s","protocol":"recent4_full_v1","batch_size":%s,"decode_prefetch_batches":%s}\n' \
|
| 126 |
+
"$BENCHMARK" "$EVAL_BATCH_SIZE" "$DECODE_PREFETCH_BATCHES"
|
| 127 |
+
exit 0
|
| 128 |
+
fi
|
| 129 |
+
|
| 130 |
+
mkdir -p "$OUTPUT/logs" "$OUTPUT/$BENCHMARK"
|
| 131 |
+
if [[ -e $OUTPUT/$BENCHMARK/complete ]]; then
|
| 132 |
+
echo "$BENCHMARK is already complete"
|
| 133 |
+
exit 0
|
| 134 |
+
fi
|
| 135 |
+
if [[ $MODEL_MANIFEST == none ]]; then
|
| 136 |
+
:
|
| 137 |
+
elif [[ -n ${PREVERIFIED_MODEL_MANIFEST_SHA256:-} ]]; then
|
| 138 |
+
[[ $PREVERIFIED_MODEL_MANIFEST_SHA256 =~ ^[0-9a-f]{64}$ ]] || exit 2
|
| 139 |
+
[[ $(sha256sum "$MODEL_MANIFEST" | cut -d' ' -f1) == \
|
| 140 |
+
"$PREVERIFIED_MODEL_MANIFEST_SHA256" ]] || {
|
| 141 |
+
echo "preverified model manifest hash mismatch" >&2
|
| 142 |
+
exit 3
|
| 143 |
+
}
|
| 144 |
+
else
|
| 145 |
+
(cd "$MODEL" && sha256sum --quiet -c SHA256SUMS)
|
| 146 |
+
fi
|
| 147 |
+
if [[ $MODEL_MANIFEST != none ]]; then
|
| 148 |
+
MODEL_IDENTITY_SHA256=$(sha256sum "$MODEL_MANIFEST" | cut -d' ' -f1)
|
| 149 |
+
fi
|
| 150 |
+
"$PYTHON" "$REPO/scripts/validate_evaluation_protocol.py" --protocol "$PROTOCOL" \
|
| 151 |
+
>"$OUTPUT/$BENCHMARK/protocol_validation.json"
|
| 152 |
+
|
| 153 |
+
wait_gpu_free() {
|
| 154 |
+
local gpu=$1 pids used free
|
| 155 |
+
while true; do
|
| 156 |
+
pids=$(nvidia-smi -i "$gpu" --query-compute-apps=pid \
|
| 157 |
+
--format=csv,noheader,nounits 2>/dev/null | tr -d '[:space:]')
|
| 158 |
+
used=$(nvidia-smi -i "$gpu" --query-gpu=memory.used \
|
| 159 |
+
--format=csv,noheader,nounits 2>/dev/null | tr -d '[:space:]')
|
| 160 |
+
free=$(nvidia-smi -i "$gpu" --query-gpu=memory.free \
|
| 161 |
+
--format=csv,noheader,nounits 2>/dev/null | tr -d '[:space:]')
|
| 162 |
+
if [[ $ALLOW_SHARED_GPU -eq 0 ]]; then
|
| 163 |
+
if [[ -z $pids && ${used:-999999} -le 16 ]]; then return 0; fi
|
| 164 |
+
elif [[ ${free:-0} -ge $MIN_FREE_MIB ]]; then
|
| 165 |
+
return 0
|
| 166 |
+
fi
|
| 167 |
+
echo "$(date -u +%FT%TZ) waiting for GPU $gpu (used=${used:-unknown} MiB free=${free:-unknown} MiB shared=$ALLOW_SHARED_GPU minimum_free=$MIN_FREE_MIB MiB)"
|
| 168 |
+
sleep 20
|
| 169 |
+
done
|
| 170 |
+
}
|
| 171 |
+
|
| 172 |
+
validate_result() {
|
| 173 |
+
"$PYTHON" - "$1" "$2" "$FRAME_POLICY" "$VISUAL_INPUT_MODE" "$RECORD_CHOICE_CONFIDENCE" <<'PY'
|
| 174 |
+
import json, sys
|
| 175 |
+
x = json.load(open(sys.argv[1], encoding="utf-8"))
|
| 176 |
+
assert x["evaluation_protocol"] == "recent4_full_v1"
|
| 177 |
+
assert x["num_samples"] == int(sys.argv[2])
|
| 178 |
+
assert x["num_errors"] == 0
|
| 179 |
+
assert all(str(row.get("prediction") or "").strip() for row in x["rows"])
|
| 180 |
+
requested_policy = sys.argv[3]
|
| 181 |
+
visual_input_mode = sys.argv[4]
|
| 182 |
+
record_choice_confidence = bool(int(sys.argv[5]))
|
| 183 |
+
assert x["processor_geometry"]["visual_input_mode"] == visual_input_mode
|
| 184 |
+
assert x["comparable_under_locked_recent4_protocol"] == (
|
| 185 |
+
requested_policy == "recent4" and visual_input_mode == "image-list"
|
| 186 |
+
)
|
| 187 |
+
for row in x["rows"]:
|
| 188 |
+
sampling = row["sampling"]
|
| 189 |
+
if requested_policy == "prefix8":
|
| 190 |
+
assert sampling["num_sampled_frames"] == 8
|
| 191 |
+
elif requested_policy == "prefix16":
|
| 192 |
+
assert sampling["num_sampled_frames"] == 16
|
| 193 |
+
elif requested_policy == "prefix32":
|
| 194 |
+
assert sampling["num_sampled_frames"] == 32
|
| 195 |
+
elif requested_policy == "prefix64":
|
| 196 |
+
assert sampling["num_sampled_frames"] == 64
|
| 197 |
+
elif requested_policy == "causalbudget20":
|
| 198 |
+
assert 1 <= sampling["num_sampled_frames"] <= 20
|
| 199 |
+
assert sampling["controller"]["benchmark_task_or_group_router"] is False
|
| 200 |
+
assert sampling["controller"]["future_frame_violations"] == 0
|
| 201 |
+
else:
|
| 202 |
+
assert sampling["num_sampled_frames"] == 4
|
| 203 |
+
assert max(sampling["frame_timestamps"]) <= row["query_time"] + 1e-3
|
| 204 |
+
assert row["visual_input_mode"] == visual_input_mode
|
| 205 |
+
assert bool(row.get("generation_confidence")) == record_choice_confidence
|
| 206 |
+
assert bool(row["video_grid_thw"]) == (
|
| 207 |
+
visual_input_mode in {"native-video", "hybrid-history-video-recent-images"}
|
| 208 |
+
and bool(row.get("native_video_metadata"))
|
| 209 |
+
)
|
| 210 |
+
PY
|
| 211 |
+
}
|
| 212 |
+
|
| 213 |
+
CONFIDENCE_ARGS=()
|
| 214 |
+
if [[ $RECORD_CHOICE_CONFIDENCE -eq 1 ]]; then
|
| 215 |
+
CONFIDENCE_ARGS+=(--record-choice-confidence)
|
| 216 |
+
fi
|
| 217 |
+
|
| 218 |
+
SMOKE_OUTPUT=$OUTPUT/$BENCHMARK/smoke.json
|
| 219 |
+
if ! test -s "$SMOKE_OUTPUT" \
|
| 220 |
+
|| ! validate_result "$SMOKE_OUTPUT" "${#SMOKE_UIDS[@]}"; then
|
| 221 |
+
wait_gpu_free "${GPUS[0]}"
|
| 222 |
+
smoke_batch=$EVAL_BATCH_SIZE
|
| 223 |
+
smoke_rc=1
|
| 224 |
+
for smoke_attempt in 1 2 3; do
|
| 225 |
+
smoke_rc=0
|
| 226 |
+
CUDA_VISIBLE_DEVICES=${GPUS[0]} PYTHONPATH=$REPO PYTHONUNBUFFERED=1 \
|
| 227 |
+
TOKENIZERS_PARALLELISM=false OMP_NUM_THREADS=1 \
|
| 228 |
+
PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \
|
| 229 |
+
"$PYTHON" "$RUNNER" --benchmark "$BENCHMARK" \
|
| 230 |
+
--annotation "$ANNOTATION" --samples "$SAMPLES" \
|
| 231 |
+
--video-root "$VIDEO_ROOT" --model "$MODEL" --processor "$PROCESSOR" \
|
| 232 |
+
"${MODEL_IDENTITY_ARGS[@]}" --protocol "$PROTOCOL" \
|
| 233 |
+
--output "$SMOKE_OUTPUT" --uids "${SMOKE_UIDS[@]}" \
|
| 234 |
+
--frame-policy "$FRAME_POLICY" \
|
| 235 |
+
--visual-input-mode "$VISUAL_INPUT_MODE" \
|
| 236 |
+
--batch-size "$smoke_batch" \
|
| 237 |
+
--decode-prefetch-batches "$DECODE_PREFETCH_BATCHES" \
|
| 238 |
+
--flush-every-batches "$FLUSH_EVERY_BATCHES" \
|
| 239 |
+
"${CONFIDENCE_ARGS[@]}" \
|
| 240 |
+
--resume --retry-errors >>"$OUTPUT/logs/${BENCHMARK}_smoke.log" 2>&1 || smoke_rc=$?
|
| 241 |
+
if validate_result "$SMOKE_OUTPUT" "${#SMOKE_UIDS[@]}"; then break; fi
|
| 242 |
+
echo "warning: $BENCHMARK smoke attempt=$smoke_attempt batch=$smoke_batch failed rc=$smoke_rc" >&2
|
| 243 |
+
if (( smoke_batch > 1 )); then smoke_batch=$(( (smoke_batch + 1) / 2 )); fi
|
| 244 |
+
done
|
| 245 |
+
if validate_result "$SMOKE_OUTPUT" "${#SMOKE_UIDS[@]}"; then
|
| 246 |
+
if [[ $smoke_rc -ne 0 ]]; then
|
| 247 |
+
echo "warning: $BENCHMARK smoke exited rc=$smoke_rc after writing a complete validated result" >&2
|
| 248 |
+
fi
|
| 249 |
+
else
|
| 250 |
+
echo "$BENCHMARK smoke failed validation after runner rc=$smoke_rc" >&2
|
| 251 |
+
if [[ $smoke_rc -eq 0 ]]; then smoke_rc=1; fi
|
| 252 |
+
exit "$smoke_rc"
|
| 253 |
+
fi
|
| 254 |
+
fi
|
| 255 |
+
|
| 256 |
+
SHARD_UID_ROOT=$OUTPUT/$BENCHMARK/shard_uids
|
| 257 |
+
mkdir -p "$SHARD_UID_ROOT"
|
| 258 |
+
"$PYTHON" - "$SAMPLES" "$SHARD_UID_ROOT" "${#GPUS[@]}" <<'PY'
|
| 259 |
+
import json, sys
|
| 260 |
+
from pathlib import Path
|
| 261 |
+
source, output, count = Path(sys.argv[1]), Path(sys.argv[2]), int(sys.argv[3])
|
| 262 |
+
uids = [str(json.loads(line)["uid"]) for line in source.read_text().splitlines() if line]
|
| 263 |
+
assert len(uids) == len(set(uids))
|
| 264 |
+
for index in range(count):
|
| 265 |
+
shard = uids[index::count]
|
| 266 |
+
(output / f"shard{index}.uids").write_text("\n".join(shard) + "\n")
|
| 267 |
+
assert sum(1 for path in output.glob("shard*.uids") for line in path.read_text().splitlines() if line) == len(uids)
|
| 268 |
+
PY
|
| 269 |
+
|
| 270 |
+
run_shard() {
|
| 271 |
+
local index=$1 gpu=$2 uid_file=$3 expected=$4 output log attempt runner_rc attempt_batch
|
| 272 |
+
output=$OUTPUT/$BENCHMARK/shard${index}.json
|
| 273 |
+
log=$OUTPUT/logs/${BENCHMARK}_shard${index}.log
|
| 274 |
+
wait_gpu_free "$gpu"
|
| 275 |
+
attempt_batch=$EVAL_BATCH_SIZE
|
| 276 |
+
for attempt in 1 2 3; do
|
| 277 |
+
runner_rc=0
|
| 278 |
+
CUDA_VISIBLE_DEVICES=$gpu PYTHONPATH=$REPO PYTHONUNBUFFERED=1 \
|
| 279 |
+
TOKENIZERS_PARALLELISM=false OMP_NUM_THREADS=1 \
|
| 280 |
+
PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \
|
| 281 |
+
"$PYTHON" "$RUNNER" --benchmark "$BENCHMARK" \
|
| 282 |
+
--annotation "$ANNOTATION" --samples "$SAMPLES" \
|
| 283 |
+
--video-root "$VIDEO_ROOT" --model "$MODEL" --processor "$PROCESSOR" \
|
| 284 |
+
"${MODEL_IDENTITY_ARGS[@]}" --protocol "$PROTOCOL" \
|
| 285 |
+
--output "$output" --uids-file "$uid_file" \
|
| 286 |
+
--frame-policy "$FRAME_POLICY" \
|
| 287 |
+
--visual-input-mode "$VISUAL_INPUT_MODE" \
|
| 288 |
+
--batch-size "$attempt_batch" \
|
| 289 |
+
--decode-prefetch-batches "$DECODE_PREFETCH_BATCHES" \
|
| 290 |
+
--flush-every-batches "$FLUSH_EVERY_BATCHES" \
|
| 291 |
+
"${CONFIDENCE_ARGS[@]}" \
|
| 292 |
+
--resume --retry-errors >>"$log" 2>&1 || runner_rc=$?
|
| 293 |
+
if validate_result "$output" "$expected"; then
|
| 294 |
+
if [[ $runner_rc -ne 0 ]]; then
|
| 295 |
+
echo "warning: $BENCHMARK shard$index exited rc=$runner_rc after writing a complete validated result" >&2
|
| 296 |
+
fi
|
| 297 |
+
return 0
|
| 298 |
+
fi
|
| 299 |
+
echo "warning: $BENCHMARK shard$index attempt=$attempt batch=$attempt_batch failed validation after runner rc=$runner_rc" >&2
|
| 300 |
+
if (( attempt_batch > 1 )); then attempt_batch=$(( (attempt_batch + 1) / 2 )); fi
|
| 301 |
+
done
|
| 302 |
+
return 1
|
| 303 |
+
}
|
| 304 |
+
|
| 305 |
+
pids=()
|
| 306 |
+
shards=()
|
| 307 |
+
for index in "${!GPUS[@]}"; do
|
| 308 |
+
uid_file=$SHARD_UID_ROOT/shard${index}.uids
|
| 309 |
+
expected=$(awk 'NF {count++} END {print count+0}' "$uid_file")
|
| 310 |
+
shards+=("$OUTPUT/$BENCHMARK/shard${index}.json")
|
| 311 |
+
run_shard "$index" "${GPUS[$index]}" "$uid_file" "$expected" &
|
| 312 |
+
pids+=("$!")
|
| 313 |
+
done
|
| 314 |
+
failed=0
|
| 315 |
+
for index in "${!pids[@]}"; do
|
| 316 |
+
pid=${pids[$index]}
|
| 317 |
+
if wait "$pid"; then
|
| 318 |
+
echo "$(date -u +%FT%TZ) $BENCHMARK shard$index exited rc=0"
|
| 319 |
+
else
|
| 320 |
+
rc=$?
|
| 321 |
+
echo "$(date -u +%FT%TZ) $BENCHMARK shard$index exited rc=$rc" >&2
|
| 322 |
+
failed=1
|
| 323 |
+
fi
|
| 324 |
+
done
|
| 325 |
+
[[ $failed -eq 0 ]]
|
| 326 |
+
|
| 327 |
+
"$PYTHON" "$REPO/scripts/merge_eval_shards.py" --inputs "${shards[@]}" \
|
| 328 |
+
--output "$OUTPUT/$BENCHMARK/full.json" --expected-count "$EXPECTED" \
|
| 329 |
+
--expected-uids "$SAMPLES"
|
| 330 |
+
validate_result "$OUTPUT/$BENCHMARK/full.json" "$EXPECTED"
|
| 331 |
+
"$PYTHON" "$REPO/scripts/score_official_benchmark_predictions.py" \
|
| 332 |
+
--benchmark "$BENCHMARK" --official-root "$OFFICIAL_ROOT" \
|
| 333 |
+
--annotation "$ANNOTATION" --predictions "$OUTPUT/$BENCHMARK/full.json" \
|
| 334 |
+
--output-json "$OUTPUT/$BENCHMARK/official_score.json" \
|
| 335 |
+
--official-input-json "$OUTPUT/$BENCHMARK/official_input.json" \
|
| 336 |
+
>"$OUTPUT/logs/${BENCHMARK}_official_score.log" 2>&1
|
| 337 |
+
|
| 338 |
+
RUNTIME_REPORT_ARG=${OVO_RUNTIME_REPORT:-}
|
| 339 |
+
"$PYTHON" - "$OUTPUT/$BENCHMARK" "$MODEL_IDENTITY_SHA256" "$PROTOCOL" \
|
| 340 |
+
"$MODEL_ID" "$BENCHMARK" "$PROCESSOR" "$RUNTIME_REPORT_ARG" <<'PY'
|
| 341 |
+
import hashlib, json, sys
|
| 342 |
+
from datetime import datetime, timezone
|
| 343 |
+
from pathlib import Path
|
| 344 |
+
out, model_identity, protocol = Path(sys.argv[1]), sys.argv[2], Path(sys.argv[3])
|
| 345 |
+
model_id, benchmark, processor = sys.argv[4:7]
|
| 346 |
+
processor = Path(processor)
|
| 347 |
+
runtime_report = Path(sys.argv[7]) if sys.argv[7] else None
|
| 348 |
+
score = json.loads((out / "official_score.json").read_text())
|
| 349 |
+
full = json.loads((out / "full.json").read_text())
|
| 350 |
+
sha = lambda p: hashlib.sha256(Path(p).read_bytes()).hexdigest()
|
| 351 |
+
summary = {
|
| 352 |
+
"format": "iclr_video.single_benchmark.v2",
|
| 353 |
+
"benchmark": benchmark,
|
| 354 |
+
"model_id": model_id,
|
| 355 |
+
"protocol_id": "recent4_full_v1",
|
| 356 |
+
"inference_policy": full["inference_policy"],
|
| 357 |
+
"inference_policy_hash": full["inference_policy_hash"],
|
| 358 |
+
"policy_config": full["policy_config"],
|
| 359 |
+
"comparable_under_locked_recent4_protocol": full.get(
|
| 360 |
+
"comparable_under_locked_recent4_protocol"
|
| 361 |
+
),
|
| 362 |
+
"model_identity_sha256": model_identity,
|
| 363 |
+
"processor_path": str(processor),
|
| 364 |
+
"processor_config_sha256": sha(processor / "preprocessor_config.json"),
|
| 365 |
+
"processor_manifest_sha256": (
|
| 366 |
+
sha(processor / "SHA256SUMS")
|
| 367 |
+
if (processor / "SHA256SUMS").is_file()
|
| 368 |
+
else None
|
| 369 |
+
),
|
| 370 |
+
"protocol_sha256": sha(protocol),
|
| 371 |
+
"benchmark_runtime_assets_sha256": (
|
| 372 |
+
sha(runtime_report) if runtime_report is not None else None
|
| 373 |
+
),
|
| 374 |
+
"completed_at": datetime.now(timezone.utc).isoformat(),
|
| 375 |
+
"official_score": score,
|
| 376 |
+
}
|
| 377 |
+
(out / "summary.json").write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n")
|
| 378 |
+
PY
|
| 379 |
+
touch "$OUTPUT/$BENCHMARK/complete"
|
| 380 |
+
echo "$(date -u +%FT%TZ) completed $BENCHMARK for $MODEL_ID"
|
code/scripts/score_official_benchmark_predictions.py
ADDED
|
@@ -0,0 +1,348 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
|
| 4 |
+
import argparse
|
| 5 |
+
import copy
|
| 6 |
+
import hashlib
|
| 7 |
+
import importlib.util
|
| 8 |
+
import io
|
| 9 |
+
import json
|
| 10 |
+
import re
|
| 11 |
+
import subprocess
|
| 12 |
+
import sys
|
| 13 |
+
import tempfile
|
| 14 |
+
from contextlib import redirect_stdout
|
| 15 |
+
from pathlib import Path
|
| 16 |
+
from statistics import mean
|
| 17 |
+
from types import SimpleNamespace
|
| 18 |
+
from typing import Any
|
| 19 |
+
|
| 20 |
+
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
| 21 |
+
|
| 22 |
+
from viss_core.benchmarks import load_prediction_file
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
OVO_GROUP_TASKS = {
|
| 26 |
+
"backward": ("EPM", "ASI", "HLD"),
|
| 27 |
+
"realtime": ("OCR", "ACR", "ATR", "STU", "FPD", "OJR"),
|
| 28 |
+
"forward": ("REC", "SSR", "CRR"),
|
| 29 |
+
}
|
| 30 |
+
|
| 31 |
+
INVALID_EMPTY_MODEL_RESPONSE = "__INVALID_EMPTY_MODEL_RESPONSE__"
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def main() -> None:
|
| 35 |
+
parser = argparse.ArgumentParser(
|
| 36 |
+
description="Adapt agent predictions to the original benchmark formats and invoke their scorers."
|
| 37 |
+
)
|
| 38 |
+
parser.add_argument("--benchmark", required=True, choices=("streamingbench", "ovo"))
|
| 39 |
+
parser.add_argument("--official-root", type=Path, required=True)
|
| 40 |
+
parser.add_argument("--annotation", type=Path, required=True)
|
| 41 |
+
parser.add_argument("--predictions", type=Path, required=True)
|
| 42 |
+
parser.add_argument("--output-json", type=Path, required=True)
|
| 43 |
+
parser.add_argument("--official-input-json", type=Path, default=None)
|
| 44 |
+
args = parser.parse_args()
|
| 45 |
+
|
| 46 |
+
if args.benchmark == "streamingbench":
|
| 47 |
+
report, official_input = score_streamingbench(
|
| 48 |
+
args.official_root, args.annotation, args.predictions
|
| 49 |
+
)
|
| 50 |
+
else:
|
| 51 |
+
report, official_input = score_ovo(args.official_root, args.annotation, args.predictions)
|
| 52 |
+
|
| 53 |
+
args.output_json.parent.mkdir(parents=True, exist_ok=True)
|
| 54 |
+
args.output_json.write_text(
|
| 55 |
+
json.dumps(report, indent=2, ensure_ascii=False, sort_keys=True) + "\n",
|
| 56 |
+
encoding="utf-8",
|
| 57 |
+
)
|
| 58 |
+
if args.official_input_json:
|
| 59 |
+
args.official_input_json.parent.mkdir(parents=True, exist_ok=True)
|
| 60 |
+
args.official_input_json.write_text(
|
| 61 |
+
json.dumps(official_input, indent=2, ensure_ascii=False, sort_keys=True) + "\n",
|
| 62 |
+
encoding="utf-8",
|
| 63 |
+
)
|
| 64 |
+
print(json.dumps(report, indent=2, ensure_ascii=False, sort_keys=True))
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
def score_streamingbench(
|
| 68 |
+
official_root: Path, annotation: Path, predictions_path: Path
|
| 69 |
+
) -> tuple[dict[str, Any], Any]:
|
| 70 |
+
predictions = load_prediction_file(predictions_path)
|
| 71 |
+
data = json.loads(annotation.read_text(encoding="utf-8"))
|
| 72 |
+
model_key = "AgenticSFT"
|
| 73 |
+
expected_ids: list[str] = []
|
| 74 |
+
empty_model_response_ids: list[str] = []
|
| 75 |
+
for video_index, entry in enumerate(data):
|
| 76 |
+
for question_index, question in enumerate(entry.get("questions") or []):
|
| 77 |
+
query_id = f"streaming_{video_index}_{question_index}"
|
| 78 |
+
expected_ids.append(query_id)
|
| 79 |
+
raw = predictions.get(query_id)
|
| 80 |
+
letter = _mcq_letter(raw, question.get("options") or [])
|
| 81 |
+
response = letter if letter is not None else str(raw or "")
|
| 82 |
+
if not response:
|
| 83 |
+
# The original scorer skips false-y responses instead of counting
|
| 84 |
+
# them as incorrect. Preserve the model failure without imputing an
|
| 85 |
+
# answer by passing an explicit invalid value through the scorer.
|
| 86 |
+
empty_model_response_ids.append(query_id)
|
| 87 |
+
response = INVALID_EMPTY_MODEL_RESPONSE
|
| 88 |
+
question[model_key] = [response]
|
| 89 |
+
_require_exact_coverage(predictions, expected_ids, "StreamingBench")
|
| 90 |
+
|
| 91 |
+
count_script = official_root / "src" / "data" / "count.py"
|
| 92 |
+
if not count_script.is_file():
|
| 93 |
+
raise FileNotFoundError(count_script)
|
| 94 |
+
with tempfile.TemporaryDirectory(prefix="streamingbench_official_score_") as tmp:
|
| 95 |
+
tmp_path = Path(tmp)
|
| 96 |
+
official_input = tmp_path / "predictions.json"
|
| 97 |
+
official_input.write_text(json.dumps(data, ensure_ascii=False), encoding="utf-8")
|
| 98 |
+
completed = subprocess.run(
|
| 99 |
+
[
|
| 100 |
+
sys.executable,
|
| 101 |
+
str(count_script),
|
| 102 |
+
"--model",
|
| 103 |
+
model_key,
|
| 104 |
+
"--task",
|
| 105 |
+
"real",
|
| 106 |
+
"--src",
|
| 107 |
+
str(official_input),
|
| 108 |
+
],
|
| 109 |
+
cwd=tmp_path,
|
| 110 |
+
check=True,
|
| 111 |
+
text=True,
|
| 112 |
+
capture_output=True,
|
| 113 |
+
)
|
| 114 |
+
stats = json.loads((tmp_path / f"{model_key}_stats.json").read_text(encoding="utf-8"))
|
| 115 |
+
|
| 116 |
+
total = stats.get("total") or {}
|
| 117 |
+
by_task = {
|
| 118 |
+
task: {
|
| 119 |
+
"accuracy": float(values["accuracy"]),
|
| 120 |
+
"correct": float(values["correct"]),
|
| 121 |
+
"total": int(values["total"]),
|
| 122 |
+
}
|
| 123 |
+
for task, values in stats.items()
|
| 124 |
+
if task != "total"
|
| 125 |
+
}
|
| 126 |
+
if int(total.get("total", 0)) != len(expected_ids):
|
| 127 |
+
raise RuntimeError(
|
| 128 |
+
f"official StreamingBench scorer counted {total.get('total')} of {len(expected_ids)} queries"
|
| 129 |
+
)
|
| 130 |
+
report = {
|
| 131 |
+
"benchmark": "streamingbench",
|
| 132 |
+
"official_scorer": str(count_script),
|
| 133 |
+
"official_scorer_sha256": _sha256(count_script),
|
| 134 |
+
"official_repository_commit": _git_commit(official_root),
|
| 135 |
+
"official_stdout": completed.stdout.strip(),
|
| 136 |
+
"num_queries": len(expected_ids),
|
| 137 |
+
"num_predictions": len(expected_ids),
|
| 138 |
+
"coverage": 1.0,
|
| 139 |
+
"num_empty_model_responses": len(empty_model_response_ids),
|
| 140 |
+
"empty_model_response_ids": empty_model_response_ids,
|
| 141 |
+
"empty_response_scoring": "counted_as_incorrect_without_answer_imputation",
|
| 142 |
+
"accuracy": float(total["accuracy"]),
|
| 143 |
+
"correct": float(total["correct"]),
|
| 144 |
+
"total": int(total["total"]),
|
| 145 |
+
"by_task": by_task,
|
| 146 |
+
}
|
| 147 |
+
return report, data
|
| 148 |
+
|
| 149 |
+
|
| 150 |
+
def score_ovo(
|
| 151 |
+
official_root: Path, annotation: Path, predictions_path: Path
|
| 152 |
+
) -> tuple[dict[str, Any], Any]:
|
| 153 |
+
predictions = load_prediction_file(predictions_path)
|
| 154 |
+
annotations = json.loads(annotation.read_text(encoding="utf-8"))
|
| 155 |
+
results: dict[str, list[dict[str, Any]]] = {key: [] for key in OVO_GROUP_TASKS}
|
| 156 |
+
expected_ids: list[str] = []
|
| 157 |
+
empty_model_response_ids: list[str] = []
|
| 158 |
+
for source in annotations:
|
| 159 |
+
item = copy.deepcopy(source)
|
| 160 |
+
task = str(item.get("task") or "").upper()
|
| 161 |
+
group = next((name for name, tasks in OVO_GROUP_TASKS.items() if task in tasks), None)
|
| 162 |
+
if group is None:
|
| 163 |
+
raise ValueError(f"unknown OVO task: {task}")
|
| 164 |
+
if group == "forward":
|
| 165 |
+
for index, test_info in enumerate(item.get("test_info") or []):
|
| 166 |
+
query_id = f"ovo_{item['id']}_{index}"
|
| 167 |
+
expected_ids.append(query_id)
|
| 168 |
+
response = str(predictions.get(query_id) or "")
|
| 169 |
+
if not response:
|
| 170 |
+
empty_model_response_ids.append(query_id)
|
| 171 |
+
test_info["response"] = response
|
| 172 |
+
else:
|
| 173 |
+
query_id = f"ovo_{item['id']}"
|
| 174 |
+
expected_ids.append(query_id)
|
| 175 |
+
letter = _mcq_letter(predictions.get(query_id), item.get("options") or [])
|
| 176 |
+
response = letter if letter is not None else str(predictions.get(query_id) or "")
|
| 177 |
+
if not response:
|
| 178 |
+
empty_model_response_ids.append(query_id)
|
| 179 |
+
item["response"] = response
|
| 180 |
+
item["ground_truth"] = chr(ord("A") + int(item["gt"]))
|
| 181 |
+
results[group].append(item)
|
| 182 |
+
_require_exact_coverage(predictions, expected_ids, "OVO-Bench")
|
| 183 |
+
|
| 184 |
+
score_module_path = official_root / "utils" / "OVOBenchScore.py"
|
| 185 |
+
if not score_module_path.is_file():
|
| 186 |
+
raise FileNotFoundError(score_module_path)
|
| 187 |
+
spec = importlib.util.spec_from_file_location("official_ovo_score", score_module_path)
|
| 188 |
+
if spec is None or spec.loader is None:
|
| 189 |
+
raise ImportError(score_module_path)
|
| 190 |
+
module = importlib.util.module_from_spec(spec)
|
| 191 |
+
spec.loader.exec_module(module)
|
| 192 |
+
scorer = module.OVOBenchOfflineScore(SimpleNamespace(model="AgenticSFT"), copy.deepcopy(results))
|
| 193 |
+
|
| 194 |
+
_, backward_scores = scorer.calculate_score_backward_realtime(copy.deepcopy(results["backward"]))
|
| 195 |
+
_, realtime_scores = scorer.calculate_score_backward_realtime(copy.deepcopy(results["realtime"]))
|
| 196 |
+
_, forward_scores = scorer.calculate_score_forward(copy.deepcopy(results["forward"]))
|
| 197 |
+
score_maps = {
|
| 198 |
+
"backward": backward_scores,
|
| 199 |
+
"realtime": realtime_scores,
|
| 200 |
+
"forward": forward_scores,
|
| 201 |
+
}
|
| 202 |
+
by_task: dict[str, dict[str, Any]] = {}
|
| 203 |
+
official_by_group: dict[str, dict[str, Any]] = {}
|
| 204 |
+
for group, expected_tasks in OVO_GROUP_TASKS.items():
|
| 205 |
+
scores = score_maps[group]
|
| 206 |
+
if set(scores) != set(expected_tasks):
|
| 207 |
+
raise RuntimeError(f"official OVO scorer tasks for {group}: {sorted(scores)}")
|
| 208 |
+
task_accuracies = []
|
| 209 |
+
for task in expected_tasks:
|
| 210 |
+
values = scores[task]
|
| 211 |
+
accuracy = sum(values) / len(values)
|
| 212 |
+
by_task[task] = {
|
| 213 |
+
"accuracy": accuracy,
|
| 214 |
+
"correct": float(sum(values)),
|
| 215 |
+
"total": len(values),
|
| 216 |
+
}
|
| 217 |
+
task_accuracies.append(accuracy)
|
| 218 |
+
official_by_group[group] = {
|
| 219 |
+
"accuracy": mean(task_accuracies),
|
| 220 |
+
"num_tasks": len(task_accuracies),
|
| 221 |
+
}
|
| 222 |
+
official_overall = mean(value["accuracy"] for value in official_by_group.values())
|
| 223 |
+
derived_metrics = _ovo_derived_metrics(by_task, official_by_group)
|
| 224 |
+
|
| 225 |
+
stdout_buffer = io.StringIO()
|
| 226 |
+
with redirect_stdout(stdout_buffer):
|
| 227 |
+
module.OVOBenchOfflineScore(
|
| 228 |
+
SimpleNamespace(model="AgenticSFT"), copy.deepcopy(results)
|
| 229 |
+
).score()
|
| 230 |
+
official_stdout = stdout_buffer.getvalue().strip()
|
| 231 |
+
_verify_ovo_stdout(official_stdout, by_task, official_by_group, official_overall)
|
| 232 |
+
|
| 233 |
+
report = {
|
| 234 |
+
"benchmark": "ovo",
|
| 235 |
+
"official_scorer": str(score_module_path),
|
| 236 |
+
"official_scorer_sha256": _sha256(score_module_path),
|
| 237 |
+
"official_repository_commit": _git_commit(official_root),
|
| 238 |
+
"official_stdout": official_stdout,
|
| 239 |
+
"num_queries": len(expected_ids),
|
| 240 |
+
"num_predictions": len(expected_ids),
|
| 241 |
+
"coverage": 1.0,
|
| 242 |
+
"num_empty_model_responses": len(empty_model_response_ids),
|
| 243 |
+
"empty_model_response_ids": empty_model_response_ids,
|
| 244 |
+
"empty_response_scoring": "passed_unchanged_to_original_scorer_without_answer_imputation",
|
| 245 |
+
"by_task": by_task,
|
| 246 |
+
"ovo_official": {
|
| 247 |
+
"name": "Total Avg",
|
| 248 |
+
"aggregation": "mean of Real-Time, Backward, and Forward task-macro averages",
|
| 249 |
+
"accuracy": official_overall,
|
| 250 |
+
"by_group": official_by_group,
|
| 251 |
+
},
|
| 252 |
+
**derived_metrics,
|
| 253 |
+
}
|
| 254 |
+
return report, results
|
| 255 |
+
|
| 256 |
+
|
| 257 |
+
def _ovo_derived_metrics(
|
| 258 |
+
by_task: dict[str, dict[str, Any]],
|
| 259 |
+
by_group: dict[str, dict[str, Any]],
|
| 260 |
+
) -> dict[str, dict[str, Any]]:
|
| 261 |
+
micro_correct = sum(float(value["correct"]) for value in by_task.values())
|
| 262 |
+
micro_total = sum(int(value["total"]) for value in by_task.values())
|
| 263 |
+
if micro_total <= 0:
|
| 264 |
+
raise RuntimeError("OVO-Bench has no scored predictions")
|
| 265 |
+
rt_bt_accuracy = mean(
|
| 266 |
+
float(by_group[group]["accuracy"]) for group in ("realtime", "backward")
|
| 267 |
+
)
|
| 268 |
+
return {
|
| 269 |
+
"ovo_micro_overall": {
|
| 270 |
+
"name": "Overall",
|
| 271 |
+
"aggregation": "micro average over all expanded OVO queries",
|
| 272 |
+
"accuracy": micro_correct / micro_total,
|
| 273 |
+
"correct": micro_correct,
|
| 274 |
+
"total": micro_total,
|
| 275 |
+
},
|
| 276 |
+
"ovo_rt_bt_avg": {
|
| 277 |
+
"name": "RT/BT Avg",
|
| 278 |
+
"aggregation": "mean of Real-Time and Backward task-macro averages",
|
| 279 |
+
"accuracy": rt_bt_accuracy,
|
| 280 |
+
},
|
| 281 |
+
}
|
| 282 |
+
|
| 283 |
+
|
| 284 |
+
def _require_exact_coverage(
|
| 285 |
+
predictions: dict[str, Any], expected_ids: list[str], benchmark: str
|
| 286 |
+
) -> None:
|
| 287 |
+
expected = set(expected_ids)
|
| 288 |
+
missing = sorted(expected - set(predictions))
|
| 289 |
+
if missing:
|
| 290 |
+
raise RuntimeError(f"{benchmark} missing {len(missing)} predictions: {missing[:10]}")
|
| 291 |
+
if len(expected_ids) != len(expected):
|
| 292 |
+
raise RuntimeError(f"{benchmark} contains duplicate query IDs")
|
| 293 |
+
|
| 294 |
+
|
| 295 |
+
def _mcq_letter(prediction: Any, _options: list[Any]) -> str | None:
|
| 296 |
+
text = str(prediction or "").strip()
|
| 297 |
+
match = re.match(r"^[\(\[]?([A-D])(?:[\)\].:\s-]|$)", text, flags=re.IGNORECASE)
|
| 298 |
+
if match:
|
| 299 |
+
return match.group(1).upper()
|
| 300 |
+
return None
|
| 301 |
+
|
| 302 |
+
|
| 303 |
+
def _verify_ovo_stdout(
|
| 304 |
+
stdout: str,
|
| 305 |
+
by_task: dict[str, dict[str, Any]],
|
| 306 |
+
by_group: dict[str, dict[str, Any]],
|
| 307 |
+
overall: float,
|
| 308 |
+
) -> None:
|
| 309 |
+
printed_tasks = {
|
| 310 |
+
task: float(value)
|
| 311 |
+
for task, value in re.findall(r"Task:\s*(\w+),\s*Acc:\s*([0-9.]+)", stdout)
|
| 312 |
+
}
|
| 313 |
+
for task, values in by_task.items():
|
| 314 |
+
if printed_tasks.get(task) != round(values["accuracy"] * 100, 2):
|
| 315 |
+
raise RuntimeError(f"official OVO stdout mismatch for {task}")
|
| 316 |
+
labels = {"backward": "Backward", "realtime": "Realtime", "forward": "Forward"}
|
| 317 |
+
for group, label in labels.items():
|
| 318 |
+
match = re.search(rf"{label} Avg\.:\s*([0-9.]+)", stdout)
|
| 319 |
+
if match is None or float(match.group(1)) != round(by_group[group]["accuracy"] * 100, 2):
|
| 320 |
+
raise RuntimeError(f"official OVO stdout mismatch for {group} Avg")
|
| 321 |
+
match = re.search(r"Total Avg\.:\s*([0-9.]+)", stdout)
|
| 322 |
+
if match is None or float(match.group(1)) != round(overall * 100, 2):
|
| 323 |
+
raise RuntimeError("official OVO stdout mismatch for Total Avg")
|
| 324 |
+
|
| 325 |
+
|
| 326 |
+
def _sha256(path: Path) -> str:
|
| 327 |
+
digest = hashlib.sha256()
|
| 328 |
+
with path.open("rb") as handle:
|
| 329 |
+
for block in iter(lambda: handle.read(8 << 20), b""):
|
| 330 |
+
digest.update(block)
|
| 331 |
+
return digest.hexdigest()
|
| 332 |
+
|
| 333 |
+
|
| 334 |
+
def _git_commit(root: Path) -> str | None:
|
| 335 |
+
completed = subprocess.run(
|
| 336 |
+
["git", "-C", str(root), "rev-parse", "HEAD"],
|
| 337 |
+
text=True,
|
| 338 |
+
capture_output=True,
|
| 339 |
+
check=False,
|
| 340 |
+
)
|
| 341 |
+
if completed.returncode != 0:
|
| 342 |
+
return None
|
| 343 |
+
commit = completed.stdout.strip()
|
| 344 |
+
return commit or None
|
| 345 |
+
|
| 346 |
+
|
| 347 |
+
if __name__ == "__main__":
|
| 348 |
+
main()
|
code/scripts/validate_evaluation_protocol.py
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
|
| 4 |
+
import argparse
|
| 5 |
+
import json
|
| 6 |
+
from pathlib import Path
|
| 7 |
+
|
| 8 |
+
from viss_core.evaluation_protocol import LOCKED_PROTOCOL, validate_locked_protocol
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
def main() -> None:
|
| 12 |
+
parser = argparse.ArgumentParser(
|
| 13 |
+
description="Validate the immutable exports and rules of the locked evaluation protocol."
|
| 14 |
+
)
|
| 15 |
+
parser.add_argument("--protocol", type=Path, default=LOCKED_PROTOCOL)
|
| 16 |
+
args = parser.parse_args()
|
| 17 |
+
print(
|
| 18 |
+
json.dumps(
|
| 19 |
+
validate_locked_protocol(args.protocol),
|
| 20 |
+
indent=2,
|
| 21 |
+
ensure_ascii=False,
|
| 22 |
+
sort_keys=True,
|
| 23 |
+
)
|
| 24 |
+
)
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
if __name__ == "__main__":
|
| 28 |
+
main()
|
code/tests/test_confidence_gated_third_view_agent.py
ADDED
|
@@ -0,0 +1,77 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from experiments.materialize_confidence_gated_third_view_agent import materialize
|
| 2 |
+
|
| 3 |
+
|
| 4 |
+
def _confidence(margin):
|
| 5 |
+
return {"top2_margin": margin, "task_or_group_used": False}
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
def _row(uid, prediction, margin, *, escalated=True):
|
| 9 |
+
return {
|
| 10 |
+
"uid": uid,
|
| 11 |
+
"prediction": prediction,
|
| 12 |
+
"generation_confidence": _confidence(margin),
|
| 13 |
+
"sampling": {"frame_indices": [1], "frame_timestamps": [0.0]},
|
| 14 |
+
"query_time": 1.0,
|
| 15 |
+
"frames_observed": 1,
|
| 16 |
+
"agent_policy": {"escalated": escalated},
|
| 17 |
+
}
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
def _value(rows):
|
| 21 |
+
return {
|
| 22 |
+
"benchmark": "streamingbench",
|
| 23 |
+
"model_manifest_sha256": "model",
|
| 24 |
+
"annotation_sha256": "annotation",
|
| 25 |
+
"evaluation_protocol_sha256": "protocol",
|
| 26 |
+
"rows": rows,
|
| 27 |
+
}
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def _threshold(commit=-0.1):
|
| 31 |
+
return {
|
| 32 |
+
"status": "source_disjoint_internal_threshold_calibration",
|
| 33 |
+
"eligible_as_frozen_benchmark_controller_config": True,
|
| 34 |
+
"model_manifest_sha256": "model",
|
| 35 |
+
"policy": {"commit_threshold": commit},
|
| 36 |
+
}
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
def test_third_view_uses_current_answer_source_margin() -> None:
|
| 40 |
+
recent = _value([_row("x", "A", 0.8), _row("y", "A", 0.2)])
|
| 41 |
+
event = _value([_row("x", "B", 0.5), _row("y", "B", 0.8)])
|
| 42 |
+
base = _value([_row("x", "A", 0.5), _row("y", "B", 0.8)])
|
| 43 |
+
third = _value([_row("x", "C", 0.4), _row("y", "C", 0.6)])
|
| 44 |
+
result = materialize(recent, event, base, third, _threshold(-0.3))
|
| 45 |
+
rows = {row["uid"]: row for row in result["rows"]}
|
| 46 |
+
assert rows["x"]["prediction"] == "A"
|
| 47 |
+
assert rows["x"]["third_view_policy"]["current_view"] == "recent4"
|
| 48 |
+
assert rows["x"]["third_view_policy"]["committed"] is False
|
| 49 |
+
assert rows["y"]["prediction"] == "C"
|
| 50 |
+
assert rows["y"]["third_view_policy"]["current_view"] == "causalbudget20"
|
| 51 |
+
assert rows["y"]["third_view_policy"]["committed"] is True
|
| 52 |
+
assert result["third_view_committed"] == 1
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def test_unchanged_third_answer_never_counts_as_commit() -> None:
|
| 56 |
+
recent = _value([_row("x", "A", 0.2)])
|
| 57 |
+
event = _value([_row("x", "B", 0.3)])
|
| 58 |
+
base = _value([_row("x", "B", 0.3)])
|
| 59 |
+
third = _value([_row("x", "B", 0.9)])
|
| 60 |
+
result = materialize(recent, event, base, third, _threshold())
|
| 61 |
+
assert result["rows"][0]["prediction"] == "B"
|
| 62 |
+
assert result["rows"][0]["third_view_policy"]["committed"] is False
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def test_full_precomputed_third_view_only_triggers_on_event_disagreement() -> None:
|
| 66 |
+
recent = _value([_row("x", "A", 0.2), _row("z", "A", 0.4)])
|
| 67 |
+
event = _value([_row("x", "B", 0.3), _row("z", "A", 0.5)])
|
| 68 |
+
base = _value([_row("x", "B", 0.3), _row("z", "A", 0.5)])
|
| 69 |
+
third = _value([_row("x", "C", 0.9), _row("z", "C", 0.9)])
|
| 70 |
+
result = materialize(recent, event, base, third, _threshold())
|
| 71 |
+
rows = {row["uid"]: row for row in result["rows"]}
|
| 72 |
+
assert rows["x"]["prediction"] == "C"
|
| 73 |
+
assert rows["z"]["prediction"] == "A"
|
| 74 |
+
assert rows["z"]["third_view_policy"]["triggered"] is False
|
| 75 |
+
assert result["third_view_triggered"] == 1
|
| 76 |
+
assert result["third_view_input_rows"] == 2
|
| 77 |
+
assert result["third_view_unused_precomputed_rows"] == 1
|
code/tests/test_filter_semantic_release_by_option_consensus.py
ADDED
|
@@ -0,0 +1,84 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from copy import deepcopy
|
| 2 |
+
|
| 3 |
+
import pytest
|
| 4 |
+
|
| 5 |
+
from data_pipeline.temporal_scope.filter_semantic_release_by_blind_audit import (
|
| 6 |
+
BlindAuditFilterError,
|
| 7 |
+
)
|
| 8 |
+
from data_pipeline.temporal_scope.filter_semantic_release_by_option_consensus import (
|
| 9 |
+
_validate_presentations,
|
| 10 |
+
)
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
def _source(uid: str, answer: str) -> dict:
|
| 14 |
+
return {
|
| 15 |
+
"uid": uid,
|
| 16 |
+
"messages": [
|
| 17 |
+
{"role": "user", "content": [{"type": "text", "text": "q"}]},
|
| 18 |
+
{
|
| 19 |
+
"role": "assistant",
|
| 20 |
+
"loss": True,
|
| 21 |
+
"content": [{"type": "text", "text": answer}],
|
| 22 |
+
},
|
| 23 |
+
],
|
| 24 |
+
}
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def _audit(uid: str, presentation: int, answer: str, *, confidence: str = "high") -> dict:
|
| 28 |
+
mapping = list("ABCD") if presentation == 0 else list("BCDA")
|
| 29 |
+
return {
|
| 30 |
+
"format": "streameva.semantic_release.option_consensus_audit_row.v0",
|
| 31 |
+
"uid": uid,
|
| 32 |
+
"presentation": presentation,
|
| 33 |
+
"displayed_to_source": mapping,
|
| 34 |
+
"expected_source": "B",
|
| 35 |
+
"prediction_source": answer,
|
| 36 |
+
"confidence": confidence,
|
| 37 |
+
"error": None,
|
| 38 |
+
"correct_source": answer == "B",
|
| 39 |
+
}
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def test_option_consensus_selects_only_stable_source_agreements() -> None:
|
| 43 |
+
sources = [_source("kept", "B"), _source("wrong", "B"), _source("unstable", "B")]
|
| 44 |
+
audits = [
|
| 45 |
+
_audit("kept", 0, "B"),
|
| 46 |
+
_audit("kept", 1, "B"),
|
| 47 |
+
_audit("wrong", 0, "C"),
|
| 48 |
+
_audit("wrong", 1, "C"),
|
| 49 |
+
_audit("unstable", 0, "B"),
|
| 50 |
+
_audit("unstable", 1, "C"),
|
| 51 |
+
]
|
| 52 |
+
selected, stable = _validate_presentations(
|
| 53 |
+
source_rows=sources, audit_rows=audits, allowed_confidence={"high"}
|
| 54 |
+
)
|
| 55 |
+
assert [row["uid"] for row in selected] == ["kept"]
|
| 56 |
+
assert stable == 2
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
def test_option_consensus_rejects_missing_or_identity_second_order() -> None:
|
| 60 |
+
source = _source("u", "B")
|
| 61 |
+
first = _audit("u", 0, "B")
|
| 62 |
+
with pytest.raises(BlindAuditFilterError, match="complete two-order"):
|
| 63 |
+
_validate_presentations(
|
| 64 |
+
source_rows=[source], audit_rows=[first], allowed_confidence={"high"}
|
| 65 |
+
)
|
| 66 |
+
second = _audit("u", 1, "B")
|
| 67 |
+
second["displayed_to_source"] = list("ABCD")
|
| 68 |
+
with pytest.raises(BlindAuditFilterError, match="malformed"):
|
| 69 |
+
_validate_presentations(
|
| 70 |
+
source_rows=[source],
|
| 71 |
+
audit_rows=[first, second],
|
| 72 |
+
allowed_confidence={"high"},
|
| 73 |
+
)
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
def test_option_consensus_rejects_tampered_correctness_flag() -> None:
|
| 77 |
+
source = _source("u", "B")
|
| 78 |
+
rows = [_audit("u", 0, "B"), _audit("u", 1, "B")]
|
| 79 |
+
tampered = deepcopy(rows)
|
| 80 |
+
tampered[1]["correct_source"] = False
|
| 81 |
+
with pytest.raises(BlindAuditFilterError, match="malformed"):
|
| 82 |
+
_validate_presentations(
|
| 83 |
+
source_rows=[source], audit_rows=tampered, allowed_confidence={"high"}
|
| 84 |
+
)
|
code/tests/test_materialize_uncertainty_temporal_agent.py
ADDED
|
@@ -0,0 +1,551 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from experiments.materialize_uncertainty_temporal_agent import materialize
|
| 2 |
+
|
| 3 |
+
|
| 4 |
+
def _result(rows: list[dict]) -> dict:
|
| 5 |
+
return {
|
| 6 |
+
"benchmark": "x",
|
| 7 |
+
"model_manifest_sha256": "m",
|
| 8 |
+
"annotation_sha256": "a",
|
| 9 |
+
"evaluation_protocol_sha256": "p",
|
| 10 |
+
"rows": rows,
|
| 11 |
+
}
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
def test_materializer_uses_temporal_only_for_preselected_uids() -> None:
|
| 15 |
+
recent = _result(
|
| 16 |
+
[
|
| 17 |
+
{"uid": "a", "prediction": "A", "query_time": 10, "frames_observed": 4},
|
| 18 |
+
{"uid": "b", "prediction": "B", "query_time": 10, "frames_observed": 4},
|
| 19 |
+
]
|
| 20 |
+
)
|
| 21 |
+
temporal = _result(
|
| 22 |
+
[
|
| 23 |
+
{
|
| 24 |
+
"uid": "a",
|
| 25 |
+
"prediction": "C",
|
| 26 |
+
"query_time": 10,
|
| 27 |
+
"frames_observed": 20,
|
| 28 |
+
"sampling": {
|
| 29 |
+
"frame_policy": "causalbudget20",
|
| 30 |
+
"frame_timestamps": [0, 10],
|
| 31 |
+
"candidate_frame_indices": list(range(64)),
|
| 32 |
+
"extra_frames_inspected": 44,
|
| 33 |
+
},
|
| 34 |
+
"visual_input_mode": "native-video",
|
| 35 |
+
}
|
| 36 |
+
]
|
| 37 |
+
)
|
| 38 |
+
selection = {
|
| 39 |
+
"status": "frozen_before_benchmark_temporal_inference",
|
| 40 |
+
"threshold": 0.3,
|
| 41 |
+
"benchmark": "x",
|
| 42 |
+
"model_manifest_sha256": "m",
|
| 43 |
+
"score_or_gold_fields_read_for_selection": False,
|
| 44 |
+
"rows": [
|
| 45 |
+
{"uid": "a", "margin": 0.2, "escalated": True},
|
| 46 |
+
{"uid": "b", "margin": 0.8, "escalated": False},
|
| 47 |
+
],
|
| 48 |
+
}
|
| 49 |
+
threshold = {
|
| 50 |
+
"status": "source_disjoint_internal_threshold_calibration",
|
| 51 |
+
"eligible_as_frozen_benchmark_controller_config": True,
|
| 52 |
+
"model_manifest_sha256": "m",
|
| 53 |
+
"policy": {"threshold": 0.3},
|
| 54 |
+
}
|
| 55 |
+
output = materialize(recent, temporal, selection, threshold)
|
| 56 |
+
assert [row["prediction"] for row in output["rows"]] == ["C", "B"]
|
| 57 |
+
assert output["escalated"] == 1
|
| 58 |
+
assert output["mean_model_calls"] == 1.5
|
| 59 |
+
assert output["mean_low_resolution_probe_frames"] == 32
|
| 60 |
+
assert output["mean_extra_probe_frames_not_visible_to_vlm"] == 22
|
| 61 |
+
assert output["mean_model_processed_frames"] == 14
|
| 62 |
+
assert output["mean_total_decoded_frame_instances"] == 46
|
| 63 |
+
assert output["rows"][0]["total_decoded_frame_instances"] == 88
|
| 64 |
+
assert output["rows"][1]["total_decoded_frame_instances"] == 4
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
def test_materializer_rejects_lower_confidence_changed_proposal() -> None:
|
| 68 |
+
recent = _result(
|
| 69 |
+
[
|
| 70 |
+
{
|
| 71 |
+
"uid": "a",
|
| 72 |
+
"prediction": "A",
|
| 73 |
+
"query_time": 10,
|
| 74 |
+
"frames_observed": 4,
|
| 75 |
+
"generation_confidence": {
|
| 76 |
+
"top2_margin": 0.3,
|
| 77 |
+
"task_or_group_used": False,
|
| 78 |
+
},
|
| 79 |
+
}
|
| 80 |
+
]
|
| 81 |
+
)
|
| 82 |
+
temporal = _result(
|
| 83 |
+
[
|
| 84 |
+
{
|
| 85 |
+
"uid": "a",
|
| 86 |
+
"prediction": "C",
|
| 87 |
+
"query_time": 10,
|
| 88 |
+
"frames_observed": 20,
|
| 89 |
+
"generation_confidence": {
|
| 90 |
+
"top2_margin": 0.2,
|
| 91 |
+
"task_or_group_used": False,
|
| 92 |
+
},
|
| 93 |
+
"sampling": {
|
| 94 |
+
"frame_policy": "causalbudget20",
|
| 95 |
+
"frame_timestamps": [0, 10],
|
| 96 |
+
"candidate_frame_indices": list(range(20)),
|
| 97 |
+
"extra_frames_inspected": 0,
|
| 98 |
+
},
|
| 99 |
+
"visual_input_mode": "native-video",
|
| 100 |
+
}
|
| 101 |
+
]
|
| 102 |
+
)
|
| 103 |
+
selection = {
|
| 104 |
+
"status": "frozen_before_benchmark_temporal_inference",
|
| 105 |
+
"threshold": 0.4,
|
| 106 |
+
"benchmark": "x",
|
| 107 |
+
"model_manifest_sha256": "m",
|
| 108 |
+
"score_or_gold_fields_read_for_selection": False,
|
| 109 |
+
"rows": [{"uid": "a", "margin": 0.3, "escalated": True}],
|
| 110 |
+
}
|
| 111 |
+
threshold = {
|
| 112 |
+
"status": "source_disjoint_internal_threshold_calibration",
|
| 113 |
+
"eligible_as_frozen_benchmark_controller_config": True,
|
| 114 |
+
"model_manifest_sha256": "m",
|
| 115 |
+
"policy": {"threshold": 0.4, "commit_threshold": 0.0},
|
| 116 |
+
}
|
| 117 |
+
output = materialize(recent, temporal, selection, threshold)
|
| 118 |
+
assert output["rows"][0]["prediction"] == "A"
|
| 119 |
+
assert output["rows"][0]["proposal_prediction"] == "C"
|
| 120 |
+
assert output["rows"][0]["agent_policy"]["committed"] is False
|
| 121 |
+
|
| 122 |
+
|
| 123 |
+
def test_materializer_names_image_list_event20_interface() -> None:
|
| 124 |
+
recent = _result(
|
| 125 |
+
[{
|
| 126 |
+
"uid": "a", "prediction": "A", "query_time": 10,
|
| 127 |
+
"frames_observed": 4,
|
| 128 |
+
"generation_confidence": {"top2_margin": 0.1, "task_or_group_used": False},
|
| 129 |
+
}]
|
| 130 |
+
)
|
| 131 |
+
temporal = _result(
|
| 132 |
+
[{
|
| 133 |
+
"uid": "a", "prediction": "C", "query_time": 10,
|
| 134 |
+
"frames_observed": 20,
|
| 135 |
+
"generation_confidence": {"top2_margin": 0.4, "task_or_group_used": False},
|
| 136 |
+
"sampling": {
|
| 137 |
+
"frame_policy": "causalbudget20", "frame_timestamps": [0, 10],
|
| 138 |
+
"candidate_frame_indices": list(range(20)), "extra_frames_inspected": 0,
|
| 139 |
+
},
|
| 140 |
+
"visual_input_mode": "image-list",
|
| 141 |
+
}]
|
| 142 |
+
)
|
| 143 |
+
selection = {
|
| 144 |
+
"status": "frozen_before_benchmark_temporal_inference",
|
| 145 |
+
"threshold": 0.5, "benchmark": "x", "model_manifest_sha256": "m",
|
| 146 |
+
"score_or_gold_fields_read_for_selection": False,
|
| 147 |
+
"rows": [{"uid": "a", "margin": 0.1, "escalated": True}],
|
| 148 |
+
}
|
| 149 |
+
threshold = {
|
| 150 |
+
"status": "source_disjoint_internal_threshold_calibration",
|
| 151 |
+
"eligible_as_frozen_benchmark_controller_config": True,
|
| 152 |
+
"model_manifest_sha256": "m",
|
| 153 |
+
"policy": {"threshold": 0.5, "commit_threshold": 0.0},
|
| 154 |
+
}
|
| 155 |
+
output = materialize(recent, temporal, selection, threshold)
|
| 156 |
+
assert output["policy"]["second_view"] == "image_list_causal_temporal20"
|
| 157 |
+
assert output["rows"][0]["agent_policy"]["selected_view"] == (
|
| 158 |
+
"causal_temporal20_image_list"
|
| 159 |
+
)
|
| 160 |
+
|
| 161 |
+
|
| 162 |
+
def test_materializer_names_image_list_event32_recent8_interface() -> None:
|
| 163 |
+
recent = _result(
|
| 164 |
+
[{
|
| 165 |
+
"uid": "a", "prediction": "A", "query_time": 10,
|
| 166 |
+
"frames_observed": 4,
|
| 167 |
+
"generation_confidence": {"top2_margin": 0.1, "task_or_group_used": False},
|
| 168 |
+
}]
|
| 169 |
+
)
|
| 170 |
+
temporal = _result(
|
| 171 |
+
[{
|
| 172 |
+
"uid": "a", "prediction": "C", "query_time": 10,
|
| 173 |
+
"frames_observed": 32,
|
| 174 |
+
"generation_confidence": {"top2_margin": 0.4, "task_or_group_used": False},
|
| 175 |
+
"sampling": {
|
| 176 |
+
"frame_policy": "causalbudget32r8", "frame_timestamps": [0, 10],
|
| 177 |
+
"candidate_frame_indices": list(range(64)),
|
| 178 |
+
"extra_frames_inspected": 32,
|
| 179 |
+
},
|
| 180 |
+
"visual_input_mode": "image-list",
|
| 181 |
+
}]
|
| 182 |
+
)
|
| 183 |
+
selection = {
|
| 184 |
+
"status": "frozen_before_benchmark_temporal_inference",
|
| 185 |
+
"threshold": 0.5, "benchmark": "x", "model_manifest_sha256": "m",
|
| 186 |
+
"score_or_gold_fields_read_for_selection": False,
|
| 187 |
+
"rows": [{"uid": "a", "margin": 0.1, "escalated": True}],
|
| 188 |
+
}
|
| 189 |
+
threshold = {
|
| 190 |
+
"status": "source_disjoint_internal_threshold_calibration",
|
| 191 |
+
"eligible_as_frozen_benchmark_controller_config": True,
|
| 192 |
+
"model_manifest_sha256": "m",
|
| 193 |
+
"policy": {"threshold": 0.5, "commit_threshold": 0.0},
|
| 194 |
+
}
|
| 195 |
+
output = materialize(recent, temporal, selection, threshold)
|
| 196 |
+
assert output["policy"]["second_view"] == (
|
| 197 |
+
"image_list_causal_temporal32_recent8"
|
| 198 |
+
)
|
| 199 |
+
assert output["rows"][0]["agent_policy"]["selected_view"] == (
|
| 200 |
+
"causal_temporal32_recent8_image_list"
|
| 201 |
+
)
|
| 202 |
+
|
| 203 |
+
|
| 204 |
+
def test_materializer_accepts_full_lowres_temporal_superset() -> None:
|
| 205 |
+
recent = _result(
|
| 206 |
+
[
|
| 207 |
+
{"uid": "a", "prediction": "A", "query_time": 10, "frames_observed": 4},
|
| 208 |
+
{"uid": "b", "prediction": "B", "query_time": 10, "frames_observed": 4},
|
| 209 |
+
]
|
| 210 |
+
)
|
| 211 |
+
temporal_rows = []
|
| 212 |
+
for uid, prediction in (("a", "C"), ("b", "D")):
|
| 213 |
+
temporal_rows.append(
|
| 214 |
+
{
|
| 215 |
+
"uid": uid,
|
| 216 |
+
"prediction": prediction,
|
| 217 |
+
"query_time": 10,
|
| 218 |
+
"frames_observed": 32,
|
| 219 |
+
"sampling": {
|
| 220 |
+
"frame_policy": "causalbudget32r8",
|
| 221 |
+
"frame_timestamps": [0, 10],
|
| 222 |
+
"candidate_frame_indices": list(range(64)),
|
| 223 |
+
"extra_frames_inspected": 32,
|
| 224 |
+
},
|
| 225 |
+
"visual_input_mode": "lowres-history-recent-images",
|
| 226 |
+
}
|
| 227 |
+
)
|
| 228 |
+
temporal = _result(temporal_rows)
|
| 229 |
+
selection = {
|
| 230 |
+
"status": "frozen_before_benchmark_temporal_inference",
|
| 231 |
+
"threshold": 0.3,
|
| 232 |
+
"benchmark": "x",
|
| 233 |
+
"model_manifest_sha256": "m",
|
| 234 |
+
"score_or_gold_fields_read_for_selection": False,
|
| 235 |
+
"rows": [
|
| 236 |
+
{"uid": "a", "margin": 0.2, "escalated": True},
|
| 237 |
+
{"uid": "b", "margin": 0.8, "escalated": False},
|
| 238 |
+
],
|
| 239 |
+
}
|
| 240 |
+
threshold = {
|
| 241 |
+
"status": "source_disjoint_internal_threshold_calibration",
|
| 242 |
+
"eligible_as_frozen_benchmark_controller_config": True,
|
| 243 |
+
"model_manifest_sha256": "m",
|
| 244 |
+
"policy": {"threshold": 0.3},
|
| 245 |
+
}
|
| 246 |
+
output = materialize(recent, temporal, selection, threshold)
|
| 247 |
+
assert [row["prediction"] for row in output["rows"]] == ["C", "B"]
|
| 248 |
+
assert output["temporal_input_num_rows"] == 2
|
| 249 |
+
assert output["temporal_unused_num_rows"] == 1
|
| 250 |
+
assert output["policy"]["second_view"] == (
|
| 251 |
+
"lowres_history_recent_images_causal_temporal32_recent8"
|
| 252 |
+
)
|
| 253 |
+
assert output["rows"][0]["agent_policy"]["selected_view"] == (
|
| 254 |
+
"causal_temporal32_recent8_lowres_history_recent_images"
|
| 255 |
+
)
|
| 256 |
+
|
| 257 |
+
|
| 258 |
+
def test_materializer_names_uniform32_image_list_interface() -> None:
|
| 259 |
+
recent = _result(
|
| 260 |
+
[{
|
| 261 |
+
"uid": "a", "prediction": "A", "query_time": 10,
|
| 262 |
+
"frames_observed": 4,
|
| 263 |
+
"generation_confidence": {"top2_margin": 0.1, "task_or_group_used": False},
|
| 264 |
+
}]
|
| 265 |
+
)
|
| 266 |
+
temporal = _result(
|
| 267 |
+
[{
|
| 268 |
+
"uid": "a", "prediction": "C", "query_time": 10,
|
| 269 |
+
"frames_observed": 32,
|
| 270 |
+
"generation_confidence": {"top2_margin": 0.4, "task_or_group_used": False},
|
| 271 |
+
"sampling": {
|
| 272 |
+
"frame_policy": "causalhybrid32", "frame_timestamps": [0, 10],
|
| 273 |
+
},
|
| 274 |
+
"visual_input_mode": "image-list",
|
| 275 |
+
}]
|
| 276 |
+
)
|
| 277 |
+
selection = {
|
| 278 |
+
"status": "frozen_before_benchmark_temporal_inference",
|
| 279 |
+
"threshold": 0.5, "benchmark": "x", "model_manifest_sha256": "m",
|
| 280 |
+
"score_or_gold_fields_read_for_selection": False,
|
| 281 |
+
"rows": [{"uid": "a", "margin": 0.1, "escalated": True}],
|
| 282 |
+
}
|
| 283 |
+
threshold = {
|
| 284 |
+
"status": "source_disjoint_internal_threshold_calibration",
|
| 285 |
+
"eligible_as_frozen_benchmark_controller_config": True,
|
| 286 |
+
"model_manifest_sha256": "m",
|
| 287 |
+
"policy": {"threshold": 0.5, "commit_threshold": 0.0},
|
| 288 |
+
}
|
| 289 |
+
output = materialize(recent, temporal, selection, threshold)
|
| 290 |
+
assert output["policy"]["second_view"] == (
|
| 291 |
+
"image_list_uniform_history28_plus_recent4"
|
| 292 |
+
)
|
| 293 |
+
assert output["rows"][0]["agent_policy"]["selected_view"] == (
|
| 294 |
+
"causal_uniform_history28_plus_recent4_image_list"
|
| 295 |
+
)
|
| 296 |
+
|
| 297 |
+
|
| 298 |
+
def test_materializer_names_hybrid32_recent8_interface() -> None:
|
| 299 |
+
recent = _result(
|
| 300 |
+
[{
|
| 301 |
+
"uid": "a", "prediction": "A", "query_time": 10,
|
| 302 |
+
"frames_observed": 4,
|
| 303 |
+
"generation_confidence": {"top2_margin": 0.1, "task_or_group_used": False},
|
| 304 |
+
}]
|
| 305 |
+
)
|
| 306 |
+
temporal = _result(
|
| 307 |
+
[{
|
| 308 |
+
"uid": "a", "prediction": "C", "query_time": 10,
|
| 309 |
+
"frames_observed": 32,
|
| 310 |
+
"generation_confidence": {"top2_margin": 0.4, "task_or_group_used": False},
|
| 311 |
+
"sampling": {
|
| 312 |
+
"frame_policy": "causalhybrid32r8", "frame_timestamps": [0, 10],
|
| 313 |
+
},
|
| 314 |
+
"visual_input_mode": "hybrid-history-video-recent-images",
|
| 315 |
+
}]
|
| 316 |
+
)
|
| 317 |
+
selection = {
|
| 318 |
+
"status": "frozen_before_benchmark_temporal_inference",
|
| 319 |
+
"threshold": 0.5, "benchmark": "x", "model_manifest_sha256": "m",
|
| 320 |
+
"score_or_gold_fields_read_for_selection": False,
|
| 321 |
+
"rows": [{"uid": "a", "margin": 0.1, "escalated": True}],
|
| 322 |
+
}
|
| 323 |
+
threshold = {
|
| 324 |
+
"status": "source_disjoint_internal_threshold_calibration",
|
| 325 |
+
"eligible_as_frozen_benchmark_controller_config": True,
|
| 326 |
+
"model_manifest_sha256": "m",
|
| 327 |
+
"policy": {"threshold": 0.5, "commit_threshold": 0.0},
|
| 328 |
+
}
|
| 329 |
+
output = materialize(recent, temporal, selection, threshold)
|
| 330 |
+
assert output["policy"]["second_view"] == (
|
| 331 |
+
"hybrid_history_video_recent_images_causal_hybrid32r8"
|
| 332 |
+
)
|
| 333 |
+
assert output["rows"][0]["agent_policy"]["selected_view"] == (
|
| 334 |
+
"causal_hybrid_history24_plus_recent8"
|
| 335 |
+
)
|
| 336 |
+
|
| 337 |
+
|
| 338 |
+
def test_materializer_accepts_task_blind_fixed_budget_selection() -> None:
|
| 339 |
+
recent = _result(
|
| 340 |
+
[
|
| 341 |
+
{
|
| 342 |
+
"uid": "a",
|
| 343 |
+
"prediction": "A",
|
| 344 |
+
"query_time": 10,
|
| 345 |
+
"frames_observed": 4,
|
| 346 |
+
"generation_confidence": {
|
| 347 |
+
"top2_margin": 0.1,
|
| 348 |
+
"task_or_group_used": False,
|
| 349 |
+
},
|
| 350 |
+
},
|
| 351 |
+
{
|
| 352 |
+
"uid": "b",
|
| 353 |
+
"prediction": "B",
|
| 354 |
+
"query_time": 10,
|
| 355 |
+
"frames_observed": 4,
|
| 356 |
+
},
|
| 357 |
+
]
|
| 358 |
+
)
|
| 359 |
+
temporal = _result(
|
| 360 |
+
[
|
| 361 |
+
{
|
| 362 |
+
"uid": "a",
|
| 363 |
+
"prediction": "C",
|
| 364 |
+
"query_time": 10,
|
| 365 |
+
"frames_observed": 20,
|
| 366 |
+
"generation_confidence": {
|
| 367 |
+
"top2_margin": 0.3,
|
| 368 |
+
"task_or_group_used": False,
|
| 369 |
+
},
|
| 370 |
+
"sampling": {
|
| 371 |
+
"frame_policy": "causalbudget20",
|
| 372 |
+
"frame_timestamps": [0, 10],
|
| 373 |
+
"candidate_frame_indices": list(range(64)),
|
| 374 |
+
"extra_frames_inspected": 44,
|
| 375 |
+
},
|
| 376 |
+
"visual_input_mode": "native-video",
|
| 377 |
+
}
|
| 378 |
+
]
|
| 379 |
+
)
|
| 380 |
+
selection = {
|
| 381 |
+
"format": "streameva.unlabeled_budget_uid_selection.v0",
|
| 382 |
+
"status": "frozen_before_benchmark_temporal_inference",
|
| 383 |
+
"benchmark": "x",
|
| 384 |
+
"model_manifest_sha256": "m",
|
| 385 |
+
"quantile": 0.5,
|
| 386 |
+
"score_or_gold_fields_read_for_selection": False,
|
| 387 |
+
"benchmark_identity_used_for_policy_parameters": False,
|
| 388 |
+
"cohort_labels_read_for_selection": False,
|
| 389 |
+
"rows": [
|
| 390 |
+
{
|
| 391 |
+
"uid": "a",
|
| 392 |
+
"margin": 0.1,
|
| 393 |
+
"uncertainty_rank": 1,
|
| 394 |
+
"escalated": True,
|
| 395 |
+
},
|
| 396 |
+
{
|
| 397 |
+
"uid": "b",
|
| 398 |
+
"margin": 0.7,
|
| 399 |
+
"uncertainty_rank": 2,
|
| 400 |
+
"escalated": False,
|
| 401 |
+
},
|
| 402 |
+
],
|
| 403 |
+
}
|
| 404 |
+
output = materialize(
|
| 405 |
+
recent,
|
| 406 |
+
temporal,
|
| 407 |
+
selection,
|
| 408 |
+
None,
|
| 409 |
+
commit_threshold_override=0.0,
|
| 410 |
+
)
|
| 411 |
+
assert [row["prediction"] for row in output["rows"]] == ["C", "B"]
|
| 412 |
+
assert output["status"] == "fixed_budget_router_free_agent"
|
| 413 |
+
assert output["policy"]["uncertainty_quantile"] == 0.5
|
| 414 |
+
assert output["rows"][0]["agent_policy"]["uncertainty_rank"] == 1
|
| 415 |
+
|
| 416 |
+
|
| 417 |
+
def test_materializer_accounts_for_prefix32_without_hidden_probe() -> None:
|
| 418 |
+
recent = _result(
|
| 419 |
+
[
|
| 420 |
+
{
|
| 421 |
+
"uid": "a",
|
| 422 |
+
"prediction": "A",
|
| 423 |
+
"query_time": 10,
|
| 424 |
+
"frames_observed": 4,
|
| 425 |
+
"generation_confidence": {
|
| 426 |
+
"top2_margin": 0.1,
|
| 427 |
+
"task_or_group_used": False,
|
| 428 |
+
},
|
| 429 |
+
}
|
| 430 |
+
]
|
| 431 |
+
)
|
| 432 |
+
temporal = _result(
|
| 433 |
+
[
|
| 434 |
+
{
|
| 435 |
+
"uid": "a",
|
| 436 |
+
"prediction": "B",
|
| 437 |
+
"query_time": 10,
|
| 438 |
+
"frames_observed": 32,
|
| 439 |
+
"generation_confidence": {
|
| 440 |
+
"top2_margin": 0.5,
|
| 441 |
+
"task_or_group_used": False,
|
| 442 |
+
},
|
| 443 |
+
"sampling": {
|
| 444 |
+
"frame_policy": "prefix32",
|
| 445 |
+
"frame_timestamps": [0, 10],
|
| 446 |
+
},
|
| 447 |
+
"visual_input_mode": "image-list",
|
| 448 |
+
}
|
| 449 |
+
]
|
| 450 |
+
)
|
| 451 |
+
selection = {
|
| 452 |
+
"format": "streameva.unlabeled_budget_uid_selection.v0",
|
| 453 |
+
"status": "frozen_before_benchmark_temporal_inference",
|
| 454 |
+
"benchmark": "x",
|
| 455 |
+
"model_manifest_sha256": "m",
|
| 456 |
+
"quantile": 1.0,
|
| 457 |
+
"score_or_gold_fields_read_for_selection": False,
|
| 458 |
+
"benchmark_identity_used_for_policy_parameters": False,
|
| 459 |
+
"cohort_labels_read_for_selection": False,
|
| 460 |
+
"rows": [
|
| 461 |
+
{"uid": "a", "margin": 0.1, "uncertainty_rank": 1, "escalated": True}
|
| 462 |
+
],
|
| 463 |
+
}
|
| 464 |
+
output = materialize(
|
| 465 |
+
recent,
|
| 466 |
+
temporal,
|
| 467 |
+
selection,
|
| 468 |
+
None,
|
| 469 |
+
commit_threshold_override=-1.0,
|
| 470 |
+
)
|
| 471 |
+
row = output["rows"][0]
|
| 472 |
+
assert row["prediction"] == "B"
|
| 473 |
+
assert row["agent_policy"]["selected_view"] == "causal_prefix32"
|
| 474 |
+
assert row["low_resolution_probe_frames"] == 0
|
| 475 |
+
assert row["model_processed_frames"] == 36
|
| 476 |
+
assert row["total_decoded_frame_instances"] == 36
|
| 477 |
+
|
| 478 |
+
|
| 479 |
+
def test_materializer_can_fail_closed_to_recent_on_temporal_error() -> None:
|
| 480 |
+
recent = _result(
|
| 481 |
+
[
|
| 482 |
+
{
|
| 483 |
+
"uid": "a",
|
| 484 |
+
"prediction": "A",
|
| 485 |
+
"query_time": 10,
|
| 486 |
+
"frames_observed": 4,
|
| 487 |
+
},
|
| 488 |
+
{
|
| 489 |
+
"uid": "b",
|
| 490 |
+
"prediction": "B",
|
| 491 |
+
"query_time": 10,
|
| 492 |
+
"frames_observed": 4,
|
| 493 |
+
},
|
| 494 |
+
]
|
| 495 |
+
)
|
| 496 |
+
temporal = _result(
|
| 497 |
+
[
|
| 498 |
+
{
|
| 499 |
+
"uid": "a",
|
| 500 |
+
"prediction": "",
|
| 501 |
+
"error": "OutOfMemoryError",
|
| 502 |
+
"query_time": 10,
|
| 503 |
+
"frames_observed": 0,
|
| 504 |
+
},
|
| 505 |
+
{
|
| 506 |
+
"uid": "b",
|
| 507 |
+
"prediction": "C",
|
| 508 |
+
"query_time": 10,
|
| 509 |
+
"frames_observed": 20,
|
| 510 |
+
"sampling": {
|
| 511 |
+
"frame_policy": "causalbudget20",
|
| 512 |
+
"frame_timestamps": [0, 10],
|
| 513 |
+
"candidate_frame_indices": list(range(20)),
|
| 514 |
+
"extra_frames_inspected": 0,
|
| 515 |
+
},
|
| 516 |
+
"visual_input_mode": "image-list",
|
| 517 |
+
},
|
| 518 |
+
]
|
| 519 |
+
)
|
| 520 |
+
selection = {
|
| 521 |
+
"status": "frozen_before_benchmark_temporal_inference",
|
| 522 |
+
"threshold": 0.3,
|
| 523 |
+
"benchmark": "x",
|
| 524 |
+
"model_manifest_sha256": "m",
|
| 525 |
+
"score_or_gold_fields_read_for_selection": False,
|
| 526 |
+
"rows": [
|
| 527 |
+
{"uid": "a", "margin": 0.1, "escalated": True},
|
| 528 |
+
{"uid": "b", "margin": 0.2, "escalated": True},
|
| 529 |
+
],
|
| 530 |
+
}
|
| 531 |
+
threshold = {
|
| 532 |
+
"status": "source_disjoint_internal_threshold_calibration",
|
| 533 |
+
"eligible_as_frozen_benchmark_controller_config": True,
|
| 534 |
+
"model_manifest_sha256": "m",
|
| 535 |
+
"policy": {"threshold": 0.3},
|
| 536 |
+
}
|
| 537 |
+
output = materialize(
|
| 538 |
+
recent,
|
| 539 |
+
temporal,
|
| 540 |
+
selection,
|
| 541 |
+
threshold,
|
| 542 |
+
fallback_recent_on_temporal_error=True,
|
| 543 |
+
)
|
| 544 |
+
assert [row["prediction"] for row in output["rows"]] == ["A", "C"]
|
| 545 |
+
assert output["temporal_error_fallback_count"] == 1
|
| 546 |
+
assert output["policy"]["temporal_error_policy"] == "fail_closed_to_recent4"
|
| 547 |
+
assert output["rows"][0]["agent_policy"]["temporal_error_fallback"] is True
|
| 548 |
+
assert output["rows"][0]["agent_policy"]["committed"] is False
|
| 549 |
+
assert output["rows"][0]["agent_policy"]["selected_view"] == (
|
| 550 |
+
"recent4_temporal_error_fallback"
|
| 551 |
+
)
|
code/tests/test_public_protocol.py
ADDED
|
@@ -0,0 +1,47 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import json
|
| 2 |
+
from pathlib import Path
|
| 3 |
+
|
| 4 |
+
import pytest
|
| 5 |
+
|
| 6 |
+
from experiments.evaluate_qwen3vl_recent4_full import validate_processor_contract
|
| 7 |
+
from scripts.configure_evaluation_protocol import configure
|
| 8 |
+
from viss_core.evaluation_protocol import ProtocolValidationError, load_locked_protocol
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
TEMPLATE = Path(__file__).resolve().parents[1] / "configs/evaluation/recent4.example.json"
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
def test_unconfigured_template_cannot_be_used_as_a_locked_protocol():
|
| 15 |
+
with pytest.raises(ProtocolValidationError, match="not locked"):
|
| 16 |
+
load_locked_protocol(TEMPLATE)
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def test_user_configured_protocol_preserves_processor_identity_check(tmp_path):
|
| 20 |
+
template = json.loads(TEMPLATE.read_text())
|
| 21 |
+
contract = template["inference"]["processor_contract"]
|
| 22 |
+
processor = tmp_path / "processor"
|
| 23 |
+
processor.mkdir()
|
| 24 |
+
config = processor / "preprocessor_config.json"
|
| 25 |
+
config.write_text(json.dumps({
|
| 26 |
+
key: contract[key]
|
| 27 |
+
for key in ["image_processor_type", "patch_size", "merge_size", "temporal_patch_size"]
|
| 28 |
+
} | {"size": {
|
| 29 |
+
key: contract[key] for key in ["shortest_edge", "longest_edge"]
|
| 30 |
+
}}))
|
| 31 |
+
streaming = tmp_path / "streaming.jsonl"
|
| 32 |
+
ovo = tmp_path / "ovo.jsonl"
|
| 33 |
+
for path in (streaming, ovo):
|
| 34 |
+
path.write_text('{"uid": "synthetic-query"}\n')
|
| 35 |
+
output = tmp_path / "configured.json"
|
| 36 |
+
configure(
|
| 37 |
+
template=TEMPLATE,
|
| 38 |
+
streaming_samples=streaming,
|
| 39 |
+
ovo_samples=ovo,
|
| 40 |
+
processor_config=config,
|
| 41 |
+
output=output,
|
| 42 |
+
)
|
| 43 |
+
protocol = load_locked_protocol(output)
|
| 44 |
+
validate_processor_contract(processor, protocol)
|
| 45 |
+
config.write_text(config.read_text() + " ")
|
| 46 |
+
with pytest.raises(ValueError, match="SHA-256 diverged"):
|
| 47 |
+
validate_processor_contract(processor, protocol)
|
code/tests/test_recent4_runner.py
ADDED
|
@@ -0,0 +1,893 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import concurrent.futures
|
| 2 |
+
import time
|
| 3 |
+
import unittest
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
from types import SimpleNamespace
|
| 6 |
+
from unittest.mock import patch
|
| 7 |
+
|
| 8 |
+
from experiments.evaluate_qwen3vl_recent4_full import (
|
| 9 |
+
CausalTemporalBudgetCache,
|
| 10 |
+
DecordReaderPool,
|
| 11 |
+
RecentWindowCache,
|
| 12 |
+
_qwen_video_resize_limits,
|
| 13 |
+
frame_policy_sampling_fps,
|
| 14 |
+
resolve_frame_policy,
|
| 15 |
+
clamp_visible_until_to_source,
|
| 16 |
+
normalize_answer,
|
| 17 |
+
probe_source_video,
|
| 18 |
+
recent_window_bounds,
|
| 19 |
+
render_question,
|
| 20 |
+
render_semantic_revisit_prompt,
|
| 21 |
+
selected_uids,
|
| 22 |
+
build_visual_messages,
|
| 23 |
+
compose_temporal_storyboard,
|
| 24 |
+
downscale_temporal_history,
|
| 25 |
+
parse_semantic_revisit_positions,
|
| 26 |
+
native_video_metadata_payload,
|
| 27 |
+
partition_hybrid_frames,
|
| 28 |
+
render_anonymous_candidate_prompt,
|
| 29 |
+
render_candidate_binary_verification_prompt,
|
| 30 |
+
render_candidate_rejection_verification_prompt,
|
| 31 |
+
render_candidate_pairwise_verification_prompt,
|
| 32 |
+
render_candidate_revisit_prompt,
|
| 33 |
+
choose_binary_verified_candidate,
|
| 34 |
+
render_temporal_review_prompt,
|
| 35 |
+
render_temporal_deliberation_prompt,
|
| 36 |
+
render_candidate_evidence_deliberation_prompt,
|
| 37 |
+
decode_causal_temporal_budget,
|
| 38 |
+
decode_causal_hybrid_prefix,
|
| 39 |
+
load_causalbudget_index_plan,
|
| 40 |
+
load_anonymous_candidate_proposals,
|
| 41 |
+
uniform_prefix_indices,
|
| 42 |
+
)
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
class Recent4RunnerTest(unittest.TestCase):
|
| 46 |
+
def test_anonymous_candidate_sources_may_be_sparse(self) -> None:
|
| 47 |
+
import json
|
| 48 |
+
import tempfile
|
| 49 |
+
|
| 50 |
+
with tempfile.TemporaryDirectory() as directory:
|
| 51 |
+
path = Path(directory) / "partial.json"
|
| 52 |
+
path.write_text(
|
| 53 |
+
json.dumps(
|
| 54 |
+
{
|
| 55 |
+
"benchmark": "streamingbench",
|
| 56 |
+
"annotation_sha256": "annotation",
|
| 57 |
+
"model_manifest_sha256": "model",
|
| 58 |
+
"num_errors": 0,
|
| 59 |
+
"rows": [{"uid": "q0", "prediction": "A"}],
|
| 60 |
+
}
|
| 61 |
+
)
|
| 62 |
+
)
|
| 63 |
+
proposals, inputs = load_anonymous_candidate_proposals(
|
| 64 |
+
paths=[path],
|
| 65 |
+
wanted=["q0", "q1"],
|
| 66 |
+
benchmark="streamingbench",
|
| 67 |
+
annotation_sha256="annotation",
|
| 68 |
+
model_manifest_sha256="model",
|
| 69 |
+
)
|
| 70 |
+
self.assertEqual(proposals, [{"q0": "A"}])
|
| 71 |
+
self.assertEqual(inputs[0]["covered_rows"], 1)
|
| 72 |
+
self.assertEqual(inputs[0]["requested_rows"], 2)
|
| 73 |
+
self.assertTrue(inputs[0]["sparse_candidate_source"])
|
| 74 |
+
|
| 75 |
+
def test_semantic_revisit_parser_is_bounded_and_fills_deterministically(self):
|
| 76 |
+
self.assertEqual(
|
| 77 |
+
parse_semantic_revisit_positions(
|
| 78 |
+
"Frames: 7, 2, 7, 99",
|
| 79 |
+
history_count=8,
|
| 80 |
+
revisit_count=4,
|
| 81 |
+
fallback_positions=[8, 6, 4, 2],
|
| 82 |
+
),
|
| 83 |
+
[2, 6, 7, 8],
|
| 84 |
+
)
|
| 85 |
+
|
| 86 |
+
def test_semantic_revisit_instruction_overrides_answer_format(self):
|
| 87 |
+
prompt = render_semantic_revisit_prompt(
|
| 88 |
+
"Question\nOnly give the best option's letter directly.",
|
| 89 |
+
history_count=24,
|
| 90 |
+
revisit_count=8,
|
| 91 |
+
)
|
| 92 |
+
self.assertTrue(prompt.endswith("`Frames: i,j,...` and nothing else."))
|
| 93 |
+
|
| 94 |
+
def test_semantic_revisit_storyboard_instruction_has_global_labels(self):
|
| 95 |
+
prompt = render_semantic_revisit_prompt(
|
| 96 |
+
"Question",
|
| 97 |
+
history_count=24,
|
| 98 |
+
revisit_count=8,
|
| 99 |
+
planner_mode="storyboard16",
|
| 100 |
+
)
|
| 101 |
+
self.assertIn("global frame number", prompt)
|
| 102 |
+
self.assertIn("numbered 1 through 24", prompt)
|
| 103 |
+
|
| 104 |
+
def test_storyboard_preserves_chronology_and_fixed_geometry(self) -> None:
|
| 105 |
+
from PIL import Image
|
| 106 |
+
|
| 107 |
+
frames = [Image.new("RGB", (32, 16), color) for color in ("red", "green", "blue")]
|
| 108 |
+
sheet = compose_temporal_storyboard(
|
| 109 |
+
frames, [0.0, 1.0, 2.0], columns=2, tile_width=40, tile_height=20
|
| 110 |
+
)
|
| 111 |
+
self.assertEqual(sheet.size, (80, 80))
|
| 112 |
+
self.assertEqual(sheet.getpixel((20, 10)), (255, 0, 0))
|
| 113 |
+
self.assertEqual(sheet.getpixel((60, 10)), (0, 128, 0))
|
| 114 |
+
self.assertEqual(sheet.getpixel((20, 50)), (0, 0, 255))
|
| 115 |
+
|
| 116 |
+
offset_sheet = compose_temporal_storyboard(
|
| 117 |
+
frames[:2], [3.0, 4.0], columns=2, label_start=17
|
| 118 |
+
)
|
| 119 |
+
self.assertEqual(offset_sheet.size[0], 448)
|
| 120 |
+
|
| 121 |
+
def test_lowres_history_keeps_recent_frames_exact(self) -> None:
|
| 122 |
+
from PIL import Image
|
| 123 |
+
|
| 124 |
+
frames = [Image.new("RGB", (640, 360), color) for color in ("red", "blue")]
|
| 125 |
+
output = downscale_temporal_history(frames, [3, 4], {4}, max_edge=224)
|
| 126 |
+
self.assertEqual(output[0].size, (224, 126))
|
| 127 |
+
self.assertIs(output[1], frames[1])
|
| 128 |
+
self.assertEqual(frames[0].size, (640, 360))
|
| 129 |
+
|
| 130 |
+
def test_deliberation_replaces_conflicting_direct_only_instruction(self) -> None:
|
| 131 |
+
prompt = render_temporal_deliberation_prompt(
|
| 132 |
+
"Question\nOnly give the best option's letter (A, B, C, or D) directly."
|
| 133 |
+
)
|
| 134 |
+
self.assertNotIn("Only give", prompt)
|
| 135 |
+
self.assertIn("Analyze the visible causal sequence", prompt)
|
| 136 |
+
self.assertIn("Answer: X", prompt)
|
| 137 |
+
|
| 138 |
+
def test_anonymous_candidates_can_be_verified_with_one_deliberation_contract(
|
| 139 |
+
self,
|
| 140 |
+
) -> None:
|
| 141 |
+
prompt, ordered = render_anonymous_candidate_prompt(
|
| 142 |
+
"Question\nOnly give the best option's letter directly.",
|
| 143 |
+
predictions=["A", "C", "A"],
|
| 144 |
+
uid="q0",
|
| 145 |
+
)
|
| 146 |
+
prompt = render_temporal_deliberation_prompt(prompt)
|
| 147 |
+
self.assertEqual(sorted(ordered), ["A", "C"])
|
| 148 |
+
self.assertIn("Candidate 1", prompt)
|
| 149 |
+
self.assertIn("Answer: X", prompt)
|
| 150 |
+
|
| 151 |
+
def test_candidate_evidence_deliberation_requires_pixel_cited_elimination(
|
| 152 |
+
self,
|
| 153 |
+
) -> None:
|
| 154 |
+
prompt, ordered = render_anonymous_candidate_prompt(
|
| 155 |
+
"Question\nOnly give the best option's letter directly.",
|
| 156 |
+
predictions=["B", "D", "B"],
|
| 157 |
+
uid="q1",
|
| 158 |
+
)
|
| 159 |
+
prompt = render_candidate_evidence_deliberation_prompt(prompt)
|
| 160 |
+
self.assertEqual(sorted(ordered), ["B", "D"])
|
| 161 |
+
self.assertNotIn("Only give", prompt)
|
| 162 |
+
self.assertIn("untrusted hypothesis", prompt)
|
| 163 |
+
self.assertIn("at most ten words per candidate", prompt)
|
| 164 |
+
self.assertIn("do not narrate", prompt)
|
| 165 |
+
self.assertIn("Answer: X", prompt)
|
| 166 |
+
|
| 167 |
+
def test_binary_candidate_verifier_is_isolated_and_removes_direct_contract(self):
|
| 168 |
+
prompt = render_candidate_binary_verification_prompt(
|
| 169 |
+
"Question: Where is it?\nOptions:\nA. left\nB. right\n"
|
| 170 |
+
"Only give the best option's letter directly.",
|
| 171 |
+
candidate="B",
|
| 172 |
+
options=["A. left", "B. right"],
|
| 173 |
+
)
|
| 174 |
+
self.assertNotIn("Only give", prompt)
|
| 175 |
+
self.assertIn("Candidate answer under isolated verification: B. right", prompt)
|
| 176 |
+
self.assertTrue(prompt.endswith("Answer exactly Yes or No."))
|
| 177 |
+
|
| 178 |
+
def test_binary_candidate_commit_is_source_order_invariant(self):
|
| 179 |
+
first = choose_binary_verified_candidate(
|
| 180 |
+
uid="q0", candidate_probabilities={"A": 0.3, "C": 0.8}
|
| 181 |
+
)
|
| 182 |
+
second = choose_binary_verified_candidate(
|
| 183 |
+
uid="q0", candidate_probabilities={"C": 0.8, "A": 0.3}
|
| 184 |
+
)
|
| 185 |
+
self.assertEqual(first, "C")
|
| 186 |
+
self.assertEqual(first, second)
|
| 187 |
+
|
| 188 |
+
def test_antipodal_candidate_prompt_reverses_only_the_decision_polarity(self):
|
| 189 |
+
prompt = render_candidate_rejection_verification_prompt(
|
| 190 |
+
"Question: Where is it?\nOptions:\nA. left\nB. right\n"
|
| 191 |
+
"Only give the best option's letter directly.",
|
| 192 |
+
candidate="B",
|
| 193 |
+
options=["A. left", "B. right"],
|
| 194 |
+
)
|
| 195 |
+
self.assertNotIn("Only give", prompt)
|
| 196 |
+
self.assertIn("isolated rejection check: B. right", prompt)
|
| 197 |
+
self.assertIn("should be rejected as the best answer", prompt)
|
| 198 |
+
self.assertTrue(prompt.endswith("Answer exactly Yes or No."))
|
| 199 |
+
|
| 200 |
+
def test_pairwise_candidate_prompt_is_source_anonymous_and_order_explicit(self):
|
| 201 |
+
prompt = render_candidate_pairwise_verification_prompt(
|
| 202 |
+
"Question: Where is it?\nOptions:\nA. left\nB. right\n"
|
| 203 |
+
"Only give the best option's letter directly.",
|
| 204 |
+
first_candidate="B",
|
| 205 |
+
second_candidate="A",
|
| 206 |
+
options=["A. left", "B. right"],
|
| 207 |
+
)
|
| 208 |
+
self.assertNotIn("Only give", prompt)
|
| 209 |
+
self.assertIn("Choice A: B. right", prompt)
|
| 210 |
+
self.assertIn("Choice B: A. left", prompt)
|
| 211 |
+
self.assertTrue(prompt.endswith("Answer exactly A or B."))
|
| 212 |
+
|
| 213 |
+
def test_candidate_conditioned_revisit_is_bounded_and_answer_free(self):
|
| 214 |
+
prompt = render_candidate_revisit_prompt(
|
| 215 |
+
"Question: What happened?\nOptions:\nA. left\nB. right\n"
|
| 216 |
+
"Only give the best option's letter directly.",
|
| 217 |
+
candidate="A",
|
| 218 |
+
options=["A. left", "B. right"],
|
| 219 |
+
history_count=24,
|
| 220 |
+
revisit_count=4,
|
| 221 |
+
)
|
| 222 |
+
self.assertNotIn("Only give", prompt)
|
| 223 |
+
self.assertIn("Candidate answer under isolated verification: A. left", prompt)
|
| 224 |
+
self.assertIn("Return exactly 4 distinct frame numbers", prompt)
|
| 225 |
+
self.assertTrue(prompt.endswith("and nothing else."))
|
| 226 |
+
|
| 227 |
+
def test_deliberative_explicit_answer_has_priority_for_open_outputs(self) -> None:
|
| 228 |
+
self.assertEqual(
|
| 229 |
+
normalize_answer(
|
| 230 |
+
"There are 2 early events, then one more. Answer: 3",
|
| 231 |
+
"Answer with a single number indicating the total count.",
|
| 232 |
+
[],
|
| 233 |
+
),
|
| 234 |
+
"3",
|
| 235 |
+
)
|
| 236 |
+
self.assertEqual(
|
| 237 |
+
normalize_answer(
|
| 238 |
+
"The first condition suggests no, but the final state does. Answer: Yes",
|
| 239 |
+
'Please answer only with "Yes" or "No".',
|
| 240 |
+
[],
|
| 241 |
+
),
|
| 242 |
+
"Yes",
|
| 243 |
+
)
|
| 244 |
+
|
| 245 |
+
def test_causalbudget_index_plan_requires_question_blind_complete_coverage(
|
| 246 |
+
self,
|
| 247 |
+
) -> None:
|
| 248 |
+
import json
|
| 249 |
+
import tempfile
|
| 250 |
+
|
| 251 |
+
with tempfile.TemporaryDirectory() as directory:
|
| 252 |
+
path = Path(directory) / "plan.json"
|
| 253 |
+
path.write_text(
|
| 254 |
+
json.dumps(
|
| 255 |
+
{
|
| 256 |
+
"format": "iclr_video.causalbudget20_index_plan.v0",
|
| 257 |
+
"status": "complete_query_independent_execution_cache",
|
| 258 |
+
"benchmark": "ovo",
|
| 259 |
+
"contracts": {
|
| 260 |
+
"question_text_read_for_plan": False,
|
| 261 |
+
"task_or_group_read_for_plan": False,
|
| 262 |
+
"model_prediction_read_for_plan": False,
|
| 263 |
+
"gold_answer_read": False,
|
| 264 |
+
"future_frames_read": False,
|
| 265 |
+
},
|
| 266 |
+
"rows": [{"uid": "u1", "sampling": {}}],
|
| 267 |
+
}
|
| 268 |
+
)
|
| 269 |
+
)
|
| 270 |
+
rows, artifact = load_causalbudget_index_plan(
|
| 271 |
+
path, benchmark="ovo", wanted=["u1"]
|
| 272 |
+
)
|
| 273 |
+
self.assertEqual(set(rows), {"u1"})
|
| 274 |
+
self.assertEqual(artifact["path"], str(path.resolve()))
|
| 275 |
+
|
| 276 |
+
value = json.loads(path.read_text())
|
| 277 |
+
value.update(
|
| 278 |
+
{
|
| 279 |
+
"format": "iclr_video.causalbudget_index_plan.v1",
|
| 280 |
+
"model_frames": 32,
|
| 281 |
+
"recent_frames": 8,
|
| 282 |
+
}
|
| 283 |
+
)
|
| 284 |
+
path.write_text(json.dumps(value))
|
| 285 |
+
rows, _ = load_causalbudget_index_plan(
|
| 286 |
+
path,
|
| 287 |
+
benchmark="ovo",
|
| 288 |
+
wanted=["u1"],
|
| 289 |
+
model_frames=32,
|
| 290 |
+
recent_frames=8,
|
| 291 |
+
)
|
| 292 |
+
self.assertEqual(set(rows), {"u1"})
|
| 293 |
+
|
| 294 |
+
def test_temporal_budget_cache_reuses_exact_query_independent_prefix(self) -> None:
|
| 295 |
+
decoder_path = (
|
| 296 |
+
"experiments.evaluate_qwen3vl_recent4_full."
|
| 297 |
+
"decode_causal_temporal_budget"
|
| 298 |
+
)
|
| 299 |
+
with patch(
|
| 300 |
+
decoder_path,
|
| 301 |
+
return_value=([{"pixels": [1]}], {"frame_indices": [3]}),
|
| 302 |
+
) as decoder:
|
| 303 |
+
cache = CausalTemporalBudgetCache(2)
|
| 304 |
+
frames, sampling, hit = cache.get(Path("v.mp4"), visible_until=4.0)
|
| 305 |
+
self.assertFalse(hit)
|
| 306 |
+
frames[0]["pixels"] = [2]
|
| 307 |
+
sampling["frame_indices"].append(4)
|
| 308 |
+
frames2, sampling2, hit2 = cache.get(
|
| 309 |
+
Path("v.mp4"), visible_until=4.0
|
| 310 |
+
)
|
| 311 |
+
self.assertTrue(hit2)
|
| 312 |
+
self.assertEqual(frames2, [{"pixels": [1]}])
|
| 313 |
+
self.assertEqual(sampling2, {"frame_indices": [3]})
|
| 314 |
+
cache.get(Path("v.mp4"), visible_until=5.0)
|
| 315 |
+
self.assertEqual(decoder.call_count, 2)
|
| 316 |
+
|
| 317 |
+
def test_native_video_turn_has_one_video_placeholder_and_no_route(self) -> None:
|
| 318 |
+
frames = [object(), object(), object(), object()]
|
| 319 |
+
messages = build_visual_messages(
|
| 320 |
+
frames=frames,
|
| 321 |
+
prompt="question",
|
| 322 |
+
visual_input_mode="native-video",
|
| 323 |
+
)
|
| 324 |
+
self.assertEqual(messages[0]["content"][0], {"type": "video"})
|
| 325 |
+
self.assertEqual(messages[0]["content"][1]["text"], "question")
|
| 326 |
+
|
| 327 |
+
def test_hybrid_turn_keeps_history_as_video_and_recent_as_images(self) -> None:
|
| 328 |
+
history = [object(), object()]
|
| 329 |
+
recent = [object(), object(), object(), object()]
|
| 330 |
+
messages = build_visual_messages(
|
| 331 |
+
frames=[*history, *recent],
|
| 332 |
+
history_frames=history,
|
| 333 |
+
recent_frames=recent,
|
| 334 |
+
prompt="question",
|
| 335 |
+
visual_input_mode="hybrid-history-video-recent-images",
|
| 336 |
+
)
|
| 337 |
+
content = messages[0]["content"]
|
| 338 |
+
self.assertEqual(content[0], {"type": "video"})
|
| 339 |
+
self.assertEqual([item["type"] for item in content[1:5]], ["image"] * 4)
|
| 340 |
+
self.assertEqual(content[-1]["text"], "question")
|
| 341 |
+
|
| 342 |
+
def test_hybrid_singleton_history_falls_back_to_ordered_image(self) -> None:
|
| 343 |
+
frames = ["history", "r1", "r2", "r3", "r4"]
|
| 344 |
+
history, indices, images, fallback = partition_hybrid_frames(
|
| 345 |
+
frames,
|
| 346 |
+
{
|
| 347 |
+
"frame_indices": [1, 2, 3, 4, 5],
|
| 348 |
+
"recent_frame_indices": [2, 3, 4, 5],
|
| 349 |
+
},
|
| 350 |
+
temporal_patch_size=2,
|
| 351 |
+
)
|
| 352 |
+
self.assertEqual(history, [])
|
| 353 |
+
self.assertEqual(indices, [])
|
| 354 |
+
self.assertEqual(images, frames)
|
| 355 |
+
self.assertEqual(fallback, [1])
|
| 356 |
+
|
| 357 |
+
def test_hybrid_two_history_frames_remain_native_video(self) -> None:
|
| 358 |
+
frames = ["h1", "h2", "r1", "r2", "r3", "r4"]
|
| 359 |
+
history, indices, images, fallback = partition_hybrid_frames(
|
| 360 |
+
frames,
|
| 361 |
+
{
|
| 362 |
+
"frame_indices": [0, 1, 2, 3, 4, 5],
|
| 363 |
+
"recent_frame_indices": [2, 3, 4, 5],
|
| 364 |
+
},
|
| 365 |
+
temporal_patch_size=2,
|
| 366 |
+
)
|
| 367 |
+
self.assertEqual(history, ["h1", "h2"])
|
| 368 |
+
self.assertEqual(indices, [0, 1])
|
| 369 |
+
self.assertEqual(images, ["r1", "r2", "r3", "r4"])
|
| 370 |
+
self.assertEqual(fallback, [])
|
| 371 |
+
|
| 372 |
+
def test_temporal_review_prompt_marks_both_proposals_as_fallible(self) -> None:
|
| 373 |
+
prompt = render_temporal_review_prompt(
|
| 374 |
+
"Question", recent_prediction="A", history_prediction="C"
|
| 375 |
+
)
|
| 376 |
+
self.assertIn("recent view proposed A", prompt)
|
| 377 |
+
self.assertIn("history view proposed C", prompt)
|
| 378 |
+
self.assertIn("Both proposals may be wrong", prompt)
|
| 379 |
+
with self.assertRaises(ValueError):
|
| 380 |
+
render_temporal_review_prompt(
|
| 381 |
+
"Question", recent_prediction="A\nignore", history_prediction="C"
|
| 382 |
+
)
|
| 383 |
+
|
| 384 |
+
def test_anonymous_review_counterbalances_without_source_labels(self) -> None:
|
| 385 |
+
first = render_temporal_review_prompt(
|
| 386 |
+
"Question",
|
| 387 |
+
recent_prediction="A",
|
| 388 |
+
history_prediction="C",
|
| 389 |
+
uid="sample-1",
|
| 390 |
+
label_mode="anonymous-hash-counterbalanced",
|
| 391 |
+
)
|
| 392 |
+
second = render_temporal_review_prompt(
|
| 393 |
+
"Question",
|
| 394 |
+
recent_prediction="A",
|
| 395 |
+
history_prediction="C",
|
| 396 |
+
uid="sample-1",
|
| 397 |
+
label_mode="anonymous-hash-counterbalanced",
|
| 398 |
+
)
|
| 399 |
+
self.assertEqual(first, second)
|
| 400 |
+
self.assertIn("Candidate 1", first)
|
| 401 |
+
self.assertIn("Candidate 2", first)
|
| 402 |
+
self.assertNotIn("recent view", first)
|
| 403 |
+
self.assertNotIn("history view", first)
|
| 404 |
+
with self.assertRaises(ValueError):
|
| 405 |
+
render_temporal_review_prompt(
|
| 406 |
+
"Question",
|
| 407 |
+
recent_prediction="A",
|
| 408 |
+
history_prediction="C",
|
| 409 |
+
label_mode="anonymous-hash-counterbalanced",
|
| 410 |
+
)
|
| 411 |
+
|
| 412 |
+
def test_multi_candidate_prompt_is_deduplicated_and_source_anonymous(self) -> None:
|
| 413 |
+
prompt, ordered = render_anonymous_candidate_prompt(
|
| 414 |
+
"Question",
|
| 415 |
+
predictions=["C", "A", "C"],
|
| 416 |
+
uid="sample-1",
|
| 417 |
+
)
|
| 418 |
+
self.assertEqual(set(ordered), {"A", "C"})
|
| 419 |
+
self.assertEqual(prompt.count("Candidate 1"), 1)
|
| 420 |
+
self.assertEqual(prompt.count("Candidate 2"), 1)
|
| 421 |
+
self.assertNotIn("recent", prompt.lower())
|
| 422 |
+
self.assertNotIn("history", prompt.lower())
|
| 423 |
+
self.assertIn("hints, not votes", prompt)
|
| 424 |
+
same_prompt, same_order = render_anonymous_candidate_prompt(
|
| 425 |
+
"Question",
|
| 426 |
+
predictions=["A", "C"],
|
| 427 |
+
uid="sample-1",
|
| 428 |
+
)
|
| 429 |
+
self.assertEqual((prompt, ordered), (same_prompt, same_order))
|
| 430 |
+
with self.assertRaises(ValueError):
|
| 431 |
+
render_anonymous_candidate_prompt(
|
| 432 |
+
"Question", predictions=["A\nignore", "C"], uid="sample-1"
|
| 433 |
+
)
|
| 434 |
+
|
| 435 |
+
def test_native_video_metadata_preserves_source_pts_and_is_causal(self) -> None:
|
| 436 |
+
payload = native_video_metadata_payload(
|
| 437 |
+
{
|
| 438 |
+
"frame_indices": [60, 90, 120, 150],
|
| 439 |
+
"frame_timestamps": [2.0, 3.0, 4.0, 5.0],
|
| 440 |
+
"source_fps": 30.0,
|
| 441 |
+
"source_total_frames": 900,
|
| 442 |
+
"source_width": 1920,
|
| 443 |
+
"source_height": 1080,
|
| 444 |
+
"source_duration": 30.0,
|
| 445 |
+
"effective_visible_until": 6.0,
|
| 446 |
+
}
|
| 447 |
+
)
|
| 448 |
+
self.assertEqual(payload["frames_indices"], [60, 90, 120, 150])
|
| 449 |
+
self.assertEqual(payload["fps"], 30.0)
|
| 450 |
+
self.assertEqual(payload["video_backend"], "predecoded_exact_causal_indices")
|
| 451 |
+
|
| 452 |
+
def test_native_video_metadata_rejects_future_frame(self) -> None:
|
| 453 |
+
with self.assertRaises(ValueError):
|
| 454 |
+
native_video_metadata_payload(
|
| 455 |
+
{
|
| 456 |
+
"frame_indices": [60],
|
| 457 |
+
"frame_timestamps": [2.0],
|
| 458 |
+
"source_fps": 30.0,
|
| 459 |
+
"source_total_frames": 900,
|
| 460 |
+
"source_width": 1920,
|
| 461 |
+
"source_height": 1080,
|
| 462 |
+
"source_duration": 30.0,
|
| 463 |
+
"effective_visible_until": 1.0,
|
| 464 |
+
}
|
| 465 |
+
)
|
| 466 |
+
|
| 467 |
+
def test_causal_budget_reserves_exact_recent_indices_without_task_router(self) -> None:
|
| 468 |
+
source = {
|
| 469 |
+
"source_fps": 10.0,
|
| 470 |
+
"source_total_frames": 100,
|
| 471 |
+
"source_duration": 10.0,
|
| 472 |
+
"source_last_frame_timestamp": 9.9,
|
| 473 |
+
"requested_visible_until": 6.0,
|
| 474 |
+
"effective_visible_until": 6.0,
|
| 475 |
+
"visible_until_clamped_to_source": False,
|
| 476 |
+
"window_start": 0.0,
|
| 477 |
+
"window_end_exclusive": 6.0,
|
| 478 |
+
"source_height": 100,
|
| 479 |
+
"source_width": 100,
|
| 480 |
+
}
|
| 481 |
+
prefix = [0, 10, 20, 30, 40, 50]
|
| 482 |
+
recent = [30, 40, 50, 60]
|
| 483 |
+
probe_features = [[float(i), 1.0] for i in range(7)]
|
| 484 |
+
selected_frames = [f"frame-{i}" for i in range(7)]
|
| 485 |
+
with patch(
|
| 486 |
+
"experiments.evaluate_qwen3vl_recent4_full.exact_causal_window_indices",
|
| 487 |
+
side_effect=[(prefix, source), (recent, source)],
|
| 488 |
+
), patch(
|
| 489 |
+
"experiments.evaluate_qwen3vl_recent4_full.decode_exact_indices_ffmpeg",
|
| 490 |
+
side_effect=[probe_features, selected_frames],
|
| 491 |
+
):
|
| 492 |
+
frames, sampling = decode_causal_temporal_budget(
|
| 493 |
+
Path("video.mp4"), visible_until=6.0
|
| 494 |
+
)
|
| 495 |
+
self.assertEqual(frames, selected_frames)
|
| 496 |
+
self.assertTrue({30, 40, 50, 60}.issubset(sampling["frame_indices"]))
|
| 497 |
+
self.assertFalse(
|
| 498 |
+
sampling["controller"]["benchmark_task_or_group_router"]
|
| 499 |
+
)
|
| 500 |
+
self.assertEqual(
|
| 501 |
+
sampling["controller"]["policy_config"]["query_weight"], 0.0
|
| 502 |
+
)
|
| 503 |
+
self.assertEqual(sampling["controller"]["future_frame_violations"], 0)
|
| 504 |
+
|
| 505 |
+
def test_uniform_history_has_unique_indices_and_no_padding(self) -> None:
|
| 506 |
+
self.assertEqual(uniform_prefix_indices(-1, 28), [])
|
| 507 |
+
self.assertEqual(uniform_prefix_indices(2, 28), [0, 1, 2])
|
| 508 |
+
values = uniform_prefix_indices(99, 28)
|
| 509 |
+
self.assertEqual(len(values), 28)
|
| 510 |
+
self.assertEqual(values, sorted(set(values)))
|
| 511 |
+
self.assertEqual((values[0], values[-1]), (0, 99))
|
| 512 |
+
|
| 513 |
+
def test_hybrid32_preserves_recent4_as_separate_exact_indices(self) -> None:
|
| 514 |
+
source = {
|
| 515 |
+
"source_fps": 10.0,
|
| 516 |
+
"source_total_frames": 100,
|
| 517 |
+
"source_duration": 10.0,
|
| 518 |
+
"source_last_frame_timestamp": 9.9,
|
| 519 |
+
"requested_visible_until": 6.0,
|
| 520 |
+
"effective_visible_until": 6.0,
|
| 521 |
+
"visible_until_clamped_to_source": False,
|
| 522 |
+
"window_start": 2.0,
|
| 523 |
+
"window_end_exclusive": 6.0,
|
| 524 |
+
"source_height": 100,
|
| 525 |
+
"source_width": 100,
|
| 526 |
+
}
|
| 527 |
+
recent = [30, 40, 50, 60]
|
| 528 |
+
with patch(
|
| 529 |
+
"experiments.evaluate_qwen3vl_recent4_full.exact_causal_window_indices",
|
| 530 |
+
return_value=(recent, source),
|
| 531 |
+
), patch(
|
| 532 |
+
"experiments.evaluate_qwen3vl_recent4_full.decode_exact_indices_ffmpeg",
|
| 533 |
+
side_effect=lambda _path, indices: [f"frame-{value}" for value in indices],
|
| 534 |
+
):
|
| 535 |
+
frames, sampling = decode_causal_hybrid_prefix(
|
| 536 |
+
Path("video.mp4"), visible_until=6.0
|
| 537 |
+
)
|
| 538 |
+
self.assertEqual(sampling["recent_frame_indices"], recent)
|
| 539 |
+
self.assertEqual(len(sampling["history_frame_indices"]), 28)
|
| 540 |
+
self.assertEqual(len(sampling["frame_indices"]), 32)
|
| 541 |
+
self.assertEqual(sampling["frame_indices"], sorted(set(sampling["frame_indices"])))
|
| 542 |
+
self.assertEqual(len(frames), 32)
|
| 543 |
+
self.assertFalse(sampling["controller"]["benchmark_task_or_group_router"])
|
| 544 |
+
self.assertFalse(sampling["controller"]["question_keyword_router"])
|
| 545 |
+
|
| 546 |
+
def test_qwen_video_resize_limits_supports_pixel_api(self) -> None:
|
| 547 |
+
module = SimpleNamespace(
|
| 548 |
+
IMAGE_FACTOR=28,
|
| 549 |
+
VIDEO_MIN_PIXELS=100_352,
|
| 550 |
+
VIDEO_MAX_PIXELS=602_112,
|
| 551 |
+
VIDEO_TOTAL_PIXELS=90_316_800,
|
| 552 |
+
FRAME_FACTOR=2,
|
| 553 |
+
)
|
| 554 |
+
self.assertEqual(
|
| 555 |
+
_qwen_video_resize_limits(module, 32),
|
| 556 |
+
(28, 100_352, 602_112),
|
| 557 |
+
)
|
| 558 |
+
|
| 559 |
+
def test_qwen_video_resize_limits_supports_token_api(self) -> None:
|
| 560 |
+
module = SimpleNamespace(
|
| 561 |
+
SPATIAL_MERGE_SIZE=2,
|
| 562 |
+
VIDEO_MIN_TOKEN_NUM=128,
|
| 563 |
+
VIDEO_MAX_TOKEN_NUM=768,
|
| 564 |
+
MODEL_SEQ_LEN=128_000,
|
| 565 |
+
FRAME_FACTOR=2,
|
| 566 |
+
)
|
| 567 |
+
factor, minimum, maximum = _qwen_video_resize_limits(module, 32)
|
| 568 |
+
self.assertEqual((factor, minimum), (28, 100_352))
|
| 569 |
+
self.assertEqual(maximum, 602_112)
|
| 570 |
+
|
| 571 |
+
def test_training_free_active4_policies_only_change_official_sampling_fps(self) -> None:
|
| 572 |
+
self.assertEqual(frame_policy_sampling_fps("recent4"), 1.0)
|
| 573 |
+
self.assertEqual(frame_policy_sampling_fps("wide8"), 0.5)
|
| 574 |
+
self.assertAlmostEqual(frame_policy_sampling_fps("wide12"), 1.0 / 3.0)
|
| 575 |
+
self.assertEqual(frame_policy_sampling_fps("wide16"), 0.25)
|
| 576 |
+
|
| 577 |
+
def test_modelonly_frame_policy_is_question_independent(self) -> None:
|
| 578 |
+
self.assertEqual(resolve_frame_policy("recent4"), "recent4")
|
| 579 |
+
self.assertEqual(resolve_frame_policy("wide12"), "wide12")
|
| 580 |
+
self.assertEqual(resolve_frame_policy("prefix8"), "prefix8")
|
| 581 |
+
self.assertEqual(resolve_frame_policy("prefix32"), "prefix32")
|
| 582 |
+
self.assertEqual(resolve_frame_policy("causalbudget20"), "causalbudget20")
|
| 583 |
+
self.assertEqual(
|
| 584 |
+
resolve_frame_policy("causalbudget32r8"), "causalbudget32r8"
|
| 585 |
+
)
|
| 586 |
+
self.assertEqual(resolve_frame_policy("causalhybrid32"), "causalhybrid32")
|
| 587 |
+
self.assertEqual(
|
| 588 |
+
resolve_frame_policy("causalhybrid32r8"), "causalhybrid32r8"
|
| 589 |
+
)
|
| 590 |
+
with self.assertRaises(ValueError):
|
| 591 |
+
resolve_frame_policy("adaptive_query_budget32")
|
| 592 |
+
|
| 593 |
+
def test_source_probe_has_small_bounded_cache(self) -> None:
|
| 594 |
+
self.assertEqual(probe_source_video.cache_parameters()["maxsize"], 8)
|
| 595 |
+
|
| 596 |
+
def test_decord_reader_pool_bounds_native_decoder_threads(self) -> None:
|
| 597 |
+
calls = []
|
| 598 |
+
|
| 599 |
+
class Reader:
|
| 600 |
+
pass
|
| 601 |
+
|
| 602 |
+
def video_reader(path, *, num_threads):
|
| 603 |
+
calls.append((path, num_threads))
|
| 604 |
+
return Reader()
|
| 605 |
+
|
| 606 |
+
pool = DecordReaderPool(2, num_threads=3)
|
| 607 |
+
with patch.dict(
|
| 608 |
+
"sys.modules", {"decord": SimpleNamespace(VideoReader=video_reader)}
|
| 609 |
+
):
|
| 610 |
+
first, first_hit = pool.reader("a.mp4")
|
| 611 |
+
second, second_hit = pool.reader("a.mp4")
|
| 612 |
+
|
| 613 |
+
self.assertIs(first, second)
|
| 614 |
+
self.assertFalse(first_hit)
|
| 615 |
+
self.assertTrue(second_hit)
|
| 616 |
+
self.assertEqual(calls, [("a.mp4", 3)])
|
| 617 |
+
|
| 618 |
+
def test_decord_reader_pool_serializes_same_video_prefetch(self) -> None:
|
| 619 |
+
state_lock = __import__("threading").Lock()
|
| 620 |
+
active = 0
|
| 621 |
+
peak = 0
|
| 622 |
+
instances = 0
|
| 623 |
+
|
| 624 |
+
class Frame:
|
| 625 |
+
shape = (16, 32, 3)
|
| 626 |
+
|
| 627 |
+
class Reader:
|
| 628 |
+
def get_avg_fps(self):
|
| 629 |
+
nonlocal active, peak
|
| 630 |
+
with state_lock:
|
| 631 |
+
active += 1
|
| 632 |
+
peak = max(peak, active)
|
| 633 |
+
time.sleep(0.01)
|
| 634 |
+
with state_lock:
|
| 635 |
+
active -= 1
|
| 636 |
+
return 30.0
|
| 637 |
+
|
| 638 |
+
def __len__(self):
|
| 639 |
+
return 10
|
| 640 |
+
|
| 641 |
+
def __getitem__(self, index):
|
| 642 |
+
return Frame()
|
| 643 |
+
|
| 644 |
+
def video_reader(path, *, num_threads):
|
| 645 |
+
nonlocal instances
|
| 646 |
+
instances += 1
|
| 647 |
+
return Reader()
|
| 648 |
+
|
| 649 |
+
pool = DecordReaderPool(2, num_threads=3)
|
| 650 |
+
with patch.dict(
|
| 651 |
+
"sys.modules", {"decord": SimpleNamespace(VideoReader=video_reader)}
|
| 652 |
+
):
|
| 653 |
+
with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor:
|
| 654 |
+
values = list(executor.map(pool.metadata, ["a.mp4"] * 8))
|
| 655 |
+
|
| 656 |
+
self.assertEqual(values, [(30.0, 10, 16, 32)] * 8)
|
| 657 |
+
self.assertEqual(instances, 1)
|
| 658 |
+
self.assertEqual(peak, 1)
|
| 659 |
+
|
| 660 |
+
def test_exact_decode_cache_reuses_only_matching_sampling_request(self) -> None:
|
| 661 |
+
from pathlib import Path
|
| 662 |
+
|
| 663 |
+
class Frame:
|
| 664 |
+
def __init__(self, value: int) -> None:
|
| 665 |
+
self.value = value
|
| 666 |
+
|
| 667 |
+
def copy(self):
|
| 668 |
+
return Frame(self.value)
|
| 669 |
+
|
| 670 |
+
calls = []
|
| 671 |
+
|
| 672 |
+
def decoder(path, **kwargs):
|
| 673 |
+
calls.append((path, kwargs))
|
| 674 |
+
return [Frame(len(calls))], {"frame_indices": [len(calls)]}
|
| 675 |
+
|
| 676 |
+
cache = RecentWindowCache(2, decoder=decoder)
|
| 677 |
+
first_frames, first_sampling, first_hit = cache.get(
|
| 678 |
+
Path("video.mp4"),
|
| 679 |
+
visible_until=10.0,
|
| 680 |
+
recent_frames=4,
|
| 681 |
+
sampling_fps=1.0,
|
| 682 |
+
)
|
| 683 |
+
first_frames[0].value = 99
|
| 684 |
+
first_sampling["frame_indices"][0] = 99
|
| 685 |
+
second_frames, second_sampling, second_hit = cache.get(
|
| 686 |
+
Path("video.mp4"),
|
| 687 |
+
visible_until=10.0,
|
| 688 |
+
recent_frames=4,
|
| 689 |
+
sampling_fps=1.0,
|
| 690 |
+
)
|
| 691 |
+
_, _, third_hit = cache.get(
|
| 692 |
+
Path("video.mp4"),
|
| 693 |
+
visible_until=11.0,
|
| 694 |
+
recent_frames=4,
|
| 695 |
+
sampling_fps=1.0,
|
| 696 |
+
)
|
| 697 |
+
|
| 698 |
+
self.assertFalse(first_hit)
|
| 699 |
+
self.assertTrue(second_hit)
|
| 700 |
+
self.assertFalse(third_hit)
|
| 701 |
+
self.assertEqual(len(calls), 2)
|
| 702 |
+
self.assertEqual(second_frames[0].value, 1)
|
| 703 |
+
self.assertEqual(second_sampling["frame_indices"], [1])
|
| 704 |
+
self.assertEqual(second_sampling["frame_policy"], "recent4")
|
| 705 |
+
self.assertEqual(second_sampling["policy_sampling_fps"], 1.0)
|
| 706 |
+
self.assertEqual((cache.hits, cache.misses), (1, 2))
|
| 707 |
+
|
| 708 |
+
def test_prefix_policy_uses_dedicated_sequential_decoder(self) -> None:
|
| 709 |
+
cache = RecentWindowCache(0)
|
| 710 |
+
decoded = ([object()], {"frame_indices": [0]})
|
| 711 |
+
with patch(
|
| 712 |
+
"experiments.evaluate_qwen3vl_recent4_full.decode_prefix_window_ffmpeg",
|
| 713 |
+
return_value=decoded,
|
| 714 |
+
) as decoder:
|
| 715 |
+
frames, sampling, hit = cache.get(
|
| 716 |
+
Path("video.mp4"),
|
| 717 |
+
visible_until=10.0,
|
| 718 |
+
recent_frames=4,
|
| 719 |
+
sampling_fps=1.0,
|
| 720 |
+
frame_policy="prefix32",
|
| 721 |
+
)
|
| 722 |
+
|
| 723 |
+
self.assertIs(frames, decoded[0])
|
| 724 |
+
self.assertFalse(hit)
|
| 725 |
+
self.assertEqual(sampling["frame_policy"], "prefix32")
|
| 726 |
+
decoder.assert_called_once_with(
|
| 727 |
+
Path("video.mp4"),
|
| 728 |
+
visible_until=10.0,
|
| 729 |
+
recent_frames=32,
|
| 730 |
+
sampling_fps=3.2,
|
| 731 |
+
)
|
| 732 |
+
|
| 733 |
+
def test_exact_decode_cache_is_safe_for_parallel_prefix_misses(self) -> None:
|
| 734 |
+
class Frame:
|
| 735 |
+
def __init__(self, value: str) -> None:
|
| 736 |
+
self.value = value
|
| 737 |
+
|
| 738 |
+
def copy(self):
|
| 739 |
+
return Frame(self.value)
|
| 740 |
+
|
| 741 |
+
def decoder(path, **kwargs):
|
| 742 |
+
time.sleep(0.01)
|
| 743 |
+
return [Frame(str(path))], {"frame_indices": [str(path)]}
|
| 744 |
+
|
| 745 |
+
cache = RecentWindowCache(8, decoder=decoder)
|
| 746 |
+
paths = [Path(f"video-{index}.mp4") for index in range(8)]
|
| 747 |
+
with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor:
|
| 748 |
+
values = list(
|
| 749 |
+
executor.map(
|
| 750 |
+
lambda path: cache.get(
|
| 751 |
+
path,
|
| 752 |
+
visible_until=10.0,
|
| 753 |
+
recent_frames=32,
|
| 754 |
+
sampling_fps=3.2,
|
| 755 |
+
frame_policy="prefix32",
|
| 756 |
+
),
|
| 757 |
+
paths,
|
| 758 |
+
)
|
| 759 |
+
)
|
| 760 |
+
self.assertEqual([value[0][0].value for value in values], list(map(str, paths)))
|
| 761 |
+
self.assertEqual(cache.misses, 8)
|
| 762 |
+
|
| 763 |
+
def test_exact_decode_cache_singleflights_matching_parallel_requests(self) -> None:
|
| 764 |
+
class Frame:
|
| 765 |
+
def copy(self):
|
| 766 |
+
return Frame()
|
| 767 |
+
|
| 768 |
+
calls = []
|
| 769 |
+
|
| 770 |
+
def decoder(path, **kwargs):
|
| 771 |
+
calls.append((path, kwargs))
|
| 772 |
+
time.sleep(0.05)
|
| 773 |
+
return [Frame()], {"frame_indices": [1]}
|
| 774 |
+
|
| 775 |
+
cache = RecentWindowCache(8, decoder=decoder)
|
| 776 |
+
|
| 777 |
+
def request(_):
|
| 778 |
+
return cache.get(
|
| 779 |
+
Path("same-video.mp4"),
|
| 780 |
+
visible_until=10.0,
|
| 781 |
+
recent_frames=4,
|
| 782 |
+
sampling_fps=1.0,
|
| 783 |
+
)
|
| 784 |
+
|
| 785 |
+
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:
|
| 786 |
+
values = list(executor.map(request, range(16)))
|
| 787 |
+
|
| 788 |
+
self.assertEqual(len(calls), 1)
|
| 789 |
+
self.assertEqual(cache.misses, 1)
|
| 790 |
+
self.assertEqual(cache.hits, 15)
|
| 791 |
+
self.assertGreaterEqual(cache.singleflight_hits, 1)
|
| 792 |
+
self.assertEqual(sum(bool(value[2]) for value in values), 15)
|
| 793 |
+
|
| 794 |
+
def test_visible_prefix_is_clamped_to_last_real_frame(self) -> None:
|
| 795 |
+
effective, clamped = clamp_visible_until_to_source(10.0, 2.0, 9)
|
| 796 |
+
self.assertEqual(effective, 4.0)
|
| 797 |
+
self.assertTrue(clamped)
|
| 798 |
+
effective, clamped = clamp_visible_until_to_source(3.0, 2.0, 9)
|
| 799 |
+
self.assertEqual(effective, 3.0)
|
| 800 |
+
self.assertFalse(clamped)
|
| 801 |
+
|
| 802 |
+
def test_explicit_uid_subset_keeps_export_order(self) -> None:
|
| 803 |
+
import tempfile
|
| 804 |
+
from pathlib import Path
|
| 805 |
+
|
| 806 |
+
with tempfile.TemporaryDirectory() as directory:
|
| 807 |
+
path = Path(directory) / "samples.jsonl"
|
| 808 |
+
path.write_text(
|
| 809 |
+
'{"uid":"a"}\n{"uid":"b"}\n{"uid":"c"}\n',
|
| 810 |
+
encoding="utf-8",
|
| 811 |
+
)
|
| 812 |
+
self.assertEqual(
|
| 813 |
+
selected_uids(path, 0, -1, ["c", "a"]),
|
| 814 |
+
["a", "c"],
|
| 815 |
+
)
|
| 816 |
+
|
| 817 |
+
def test_batched_runner_keeps_auditable_resume_controls(self) -> None:
|
| 818 |
+
source = Path("experiments/evaluate_qwen3vl_recent4_full.py").read_text(
|
| 819 |
+
encoding="utf-8"
|
| 820 |
+
)
|
| 821 |
+
launcher = Path("scripts/run_recent4_single_benchmark.sh").read_text(
|
| 822 |
+
encoding="utf-8"
|
| 823 |
+
)
|
| 824 |
+
self.assertIn('processor.tokenizer.padding_side = "left"', source)
|
| 825 |
+
self.assertIn("validate_processor_contract(processor_path, protocol)", source)
|
| 826 |
+
self.assertIn('"inference_batch_size": args.batch_size', source)
|
| 827 |
+
self.assertIn('"execution_optimization",', source)
|
| 828 |
+
self.assertIn('--uids-file "$uid_file"', launcher)
|
| 829 |
+
self.assertIn('uids[index::count]', launcher)
|
| 830 |
+
self.assertIn('--batch-size "$attempt_batch"', launcher)
|
| 831 |
+
self.assertIn('attempt_batch=$EVAL_BATCH_SIZE', launcher)
|
| 832 |
+
self.assertIn('|| smoke_rc=$?', launcher)
|
| 833 |
+
self.assertIn('|| runner_rc=$?', launcher)
|
| 834 |
+
self.assertIn(
|
| 835 |
+
'after writing a complete validated result',
|
| 836 |
+
launcher,
|
| 837 |
+
)
|
| 838 |
+
|
| 839 |
+
def test_recent_window_is_half_open_and_causal(self) -> None:
|
| 840 |
+
self.assertEqual(recent_window_bounds(10.0, 4, 1.0), (6.0, 10.0))
|
| 841 |
+
self.assertEqual(recent_window_bounds(1.0, 4, 1.0), (0.0, 1.0))
|
| 842 |
+
|
| 843 |
+
def test_answer_normalization(self) -> None:
|
| 844 |
+
options = ["A. first", "B. second", "C. third", "D. fourth"]
|
| 845 |
+
self.assertEqual(normalize_answer("(C) third", "question", options), "C")
|
| 846 |
+
self.assertEqual(normalize_answer("second", "question", options), "B")
|
| 847 |
+
self.assertEqual(normalize_answer("A map", "question", ["A. compass", "B. A map", "C. sky", "D. radar"]), "B")
|
| 848 |
+
self.assertEqual(
|
| 849 |
+
normalize_answer(
|
| 850 |
+
"Yes",
|
| 851 |
+
"Answer only with “Yes” or “No”.",
|
| 852 |
+
[],
|
| 853 |
+
),
|
| 854 |
+
"Yes",
|
| 855 |
+
)
|
| 856 |
+
|
| 857 |
+
def test_streaming_prompt_matches_release_contract(self) -> None:
|
| 858 |
+
prompt = render_question(
|
| 859 |
+
"streamingbench",
|
| 860 |
+
"What is visible?",
|
| 861 |
+
["A. first", "B. second", "C. third", "D. fourth"],
|
| 862 |
+
)
|
| 863 |
+
self.assertIn("advanced video question-answering AI assistant", prompt)
|
| 864 |
+
self.assertTrue(prompt.endswith(
|
| 865 |
+
"Only give the best option's letter (A, B, C, or D) directly."
|
| 866 |
+
))
|
| 867 |
+
|
| 868 |
+
def test_ovo_mcq_prompt_matches_release_contract(self) -> None:
|
| 869 |
+
prompt = render_question(
|
| 870 |
+
"ovo",
|
| 871 |
+
"What happened?",
|
| 872 |
+
["one", "two", "three", "four"],
|
| 873 |
+
)
|
| 874 |
+
self.assertEqual(
|
| 875 |
+
prompt,
|
| 876 |
+
"What happened?\nOptions: A. one; B. two; C. three; D. four;"
|
| 877 |
+
"\nOnly give the best option's letter directly.",
|
| 878 |
+
)
|
| 879 |
+
|
| 880 |
+
def test_ovo_requires_full_runtime_asset_attestation(self) -> None:
|
| 881 |
+
launcher = Path("scripts/run_recent4_single_benchmark.sh").read_text(
|
| 882 |
+
encoding="utf-8"
|
| 883 |
+
)
|
| 884 |
+
self.assertIn("OVO_RUNTIME_REPORT", launcher)
|
| 885 |
+
self.assertIn(".expanded_queries == 3035", launcher)
|
| 886 |
+
self.assertIn(".covered_queries == 3035", launcher)
|
| 887 |
+
self.assertIn(".missing_videos == 0", launcher)
|
| 888 |
+
self.assertIn(".annotation_sha256 == $annotation_sha256", launcher)
|
| 889 |
+
self.assertIn(".frozen_export_sha256 == $frozen_export_sha256", launcher)
|
| 890 |
+
|
| 891 |
+
|
| 892 |
+
if __name__ == "__main__":
|
| 893 |
+
unittest.main()
|
code/tests/test_semantic_answer_loss_mask.py
ADDED
|
@@ -0,0 +1,108 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import torch
|
| 4 |
+
|
| 5 |
+
from experiments.train_qwen3vl_sft import _assistant_only_labels_and_weights
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
class _CharacterTokenizer:
|
| 9 |
+
im_end_id = 1
|
| 10 |
+
|
| 11 |
+
def __call__(
|
| 12 |
+
self,
|
| 13 |
+
text: str,
|
| 14 |
+
*,
|
| 15 |
+
add_special_tokens: bool = False,
|
| 16 |
+
return_offsets_mapping: bool = False,
|
| 17 |
+
) -> dict[str, object]:
|
| 18 |
+
del add_special_tokens
|
| 19 |
+
value: dict[str, object] = {
|
| 20 |
+
"input_ids": [ord(character) + 10 for character in text]
|
| 21 |
+
}
|
| 22 |
+
if return_offsets_mapping:
|
| 23 |
+
value["offset_mapping"] = [
|
| 24 |
+
(index, index + 1) for index in range(len(text))
|
| 25 |
+
]
|
| 26 |
+
return value
|
| 27 |
+
|
| 28 |
+
def convert_tokens_to_ids(self, token: str) -> int | None:
|
| 29 |
+
return self.im_end_id if token == "<|im_end|>" else None
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
class _BoundaryMergingTokenizer:
|
| 33 |
+
def apply_chat_template(
|
| 34 |
+
self,
|
| 35 |
+
messages: list[dict],
|
| 36 |
+
*,
|
| 37 |
+
tokenize: bool,
|
| 38 |
+
add_generation_prompt: bool,
|
| 39 |
+
) -> str:
|
| 40 |
+
assert tokenize is False and add_generation_prompt is False
|
| 41 |
+
return "".join(
|
| 42 |
+
item["text"]
|
| 43 |
+
for message in messages
|
| 44 |
+
for item in message["content"]
|
| 45 |
+
if item["type"] == "text"
|
| 46 |
+
)
|
| 47 |
+
|
| 48 |
+
def __call__(
|
| 49 |
+
self,
|
| 50 |
+
text: str,
|
| 51 |
+
*,
|
| 52 |
+
add_special_tokens: bool = False,
|
| 53 |
+
return_offsets_mapping: bool = False,
|
| 54 |
+
) -> dict[str, object]:
|
| 55 |
+
assert add_special_tokens is False
|
| 56 |
+
if text == "ABC":
|
| 57 |
+
value: dict[str, object] = {"input_ids": [100, 101]}
|
| 58 |
+
if return_offsets_mapping:
|
| 59 |
+
value["offset_mapping"] = [(0, 2), (2, 3)]
|
| 60 |
+
return value
|
| 61 |
+
return {"input_ids": [200 + ord(character) for character in text]}
|
| 62 |
+
|
| 63 |
+
def convert_tokens_to_ids(self, token: str) -> None:
|
| 64 |
+
del token
|
| 65 |
+
return None
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
def test_only_assistant_semantic_answer_and_terminator_receive_loss() -> None:
|
| 69 |
+
tokenizer = _CharacterTokenizer()
|
| 70 |
+
question = "A"
|
| 71 |
+
answer = "B"
|
| 72 |
+
input_ids = torch.tensor(
|
| 73 |
+
tokenizer(question)["input_ids"]
|
| 74 |
+
+ tokenizer(answer)["input_ids"]
|
| 75 |
+
+ [tokenizer.im_end_id],
|
| 76 |
+
dtype=torch.long,
|
| 77 |
+
)
|
| 78 |
+
messages = [
|
| 79 |
+
{
|
| 80 |
+
"role": "user",
|
| 81 |
+
"content": [{"type": "text", "text": question}],
|
| 82 |
+
"loss": False,
|
| 83 |
+
},
|
| 84 |
+
{
|
| 85 |
+
"role": "assistant",
|
| 86 |
+
"content": [{"type": "text", "text": answer}],
|
| 87 |
+
"loss": True,
|
| 88 |
+
},
|
| 89 |
+
]
|
| 90 |
+
labels, weights = _assistant_only_labels_and_weights(
|
| 91 |
+
tokenizer, input_ids, messages
|
| 92 |
+
)
|
| 93 |
+
assert labels.tolist() == [-100, ord("B") + 10, tokenizer.im_end_id]
|
| 94 |
+
assert weights.tolist() == [0.0, 1.0, 1.0]
|
| 95 |
+
|
| 96 |
+
|
| 97 |
+
def test_rendered_offsets_handle_tokens_crossing_message_boundaries() -> None:
|
| 98 |
+
tokenizer = _BoundaryMergingTokenizer()
|
| 99 |
+
messages = [
|
| 100 |
+
{"role": "user", "content": [{"type": "text", "text": "A"}]},
|
| 101 |
+
{"role": "assistant", "content": [{"type": "text", "text": "B"}]},
|
| 102 |
+
{"role": "user", "content": [{"type": "text", "text": "C"}]},
|
| 103 |
+
]
|
| 104 |
+
labels, weights = _assistant_only_labels_and_weights(
|
| 105 |
+
tokenizer, torch.tensor([100, 101]), messages
|
| 106 |
+
)
|
| 107 |
+
assert labels.tolist() == [100, -100]
|
| 108 |
+
assert weights.tolist() == [1.0, 0.0]
|
code/tests/test_size_matched_generic_control.py
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from data_pipeline.temporal_scope.build_size_matched_generic_control import (
|
| 2 |
+
select_parent_ids,
|
| 3 |
+
)
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
def _row(parent_id: str, kind: str) -> dict:
|
| 7 |
+
return {"parents": [{"id": parent_id, "kind": kind}]}
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
def test_selection_is_deterministic_and_stratified() -> None:
|
| 11 |
+
rows = [
|
| 12 |
+
*[_row(f"source-{i}", "source_semantic_row") for i in range(8)],
|
| 13 |
+
*[
|
| 14 |
+
_row(f"cited-{i}", "cited_nonbenchmark_refresh_projection")
|
| 15 |
+
for i in range(5)
|
| 16 |
+
],
|
| 17 |
+
]
|
| 18 |
+
quotas = {
|
| 19 |
+
"source_semantic_row": 4,
|
| 20 |
+
"cited_nonbenchmark_refresh_projection": 2,
|
| 21 |
+
}
|
| 22 |
+
first, counts = select_parent_ids(rows, seed="control-v1", quotas=quotas)
|
| 23 |
+
second, _ = select_parent_ids(list(reversed(rows)), seed="control-v1", quotas=quotas)
|
| 24 |
+
assert first == second
|
| 25 |
+
assert len(first) == 6
|
| 26 |
+
assert counts == quotas
|
code/tests/test_source_aligned_sft_release.py
ADDED
|
@@ -0,0 +1,125 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from pathlib import Path
|
| 2 |
+
|
| 3 |
+
from data_pipeline.temporal_scope.build_source_aligned_sft_release import (
|
| 4 |
+
SourceAlignedSFTError,
|
| 5 |
+
_canonical_media_path,
|
| 6 |
+
_semantic_identity,
|
| 7 |
+
_teacher_used_for_labels,
|
| 8 |
+
)
|
| 9 |
+
import pytest
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
def _row(image: str = "media/one.jpg", question: str = "Which one?") -> dict:
|
| 13 |
+
return {
|
| 14 |
+
"messages": [
|
| 15 |
+
{
|
| 16 |
+
"role": "user",
|
| 17 |
+
"loss": False,
|
| 18 |
+
"content": [
|
| 19 |
+
{"type": "image", "image": image},
|
| 20 |
+
{"type": "text", "text": question},
|
| 21 |
+
],
|
| 22 |
+
},
|
| 23 |
+
{
|
| 24 |
+
"role": "assistant",
|
| 25 |
+
"loss": True,
|
| 26 |
+
"content": [{"type": "text", "text": "A"}],
|
| 27 |
+
},
|
| 28 |
+
]
|
| 29 |
+
}
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def test_semantic_identity_binds_pixels_question_and_answer() -> None:
|
| 33 |
+
inventory = {"media/one.jpg": {"sha256": "a" * 64}}
|
| 34 |
+
baseline = _semantic_identity(_row(), inventory)
|
| 35 |
+
assert baseline == _semantic_identity(_row(question=" Which one? "), inventory)
|
| 36 |
+
assert baseline != _semantic_identity(_row(question="Which event?"), inventory)
|
| 37 |
+
changed = {"media/one.jpg": {"sha256": "b" * 64}}
|
| 38 |
+
assert baseline != _semantic_identity(_row(), changed)
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def test_canonical_media_path_is_content_addressed() -> None:
|
| 42 |
+
digest = "ab" + "1" * 62
|
| 43 |
+
assert _canonical_media_path("legacy/frame.JPG", digest) == (
|
| 44 |
+
f"media/sha256/ab/{digest}.jpg"
|
| 45 |
+
)
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
def test_teacher_label_provenance_is_strict_and_propagatable() -> None:
|
| 49 |
+
assert _teacher_used_for_labels(
|
| 50 |
+
{"contracts": {"teacher_used_for_labels": True}}
|
| 51 |
+
)
|
| 52 |
+
assert not _teacher_used_for_labels(
|
| 53 |
+
{"contracts": {"teacher_used_for_labels": False}}
|
| 54 |
+
)
|
| 55 |
+
with pytest.raises(SourceAlignedSFTError):
|
| 56 |
+
_teacher_used_for_labels({"contracts": {}})
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
def test_legacy_v62di_loader_defaults_narrow_but_supports_committed_full_pool() -> None:
|
| 60 |
+
from data_pipeline.temporal_scope.build_source_aligned_sft_release import _load_legacy_v62di_source
|
| 61 |
+
with pytest.raises(SourceAlignedSFTError, match="Private legacy adapters"):
|
| 62 |
+
_load_legacy_v62di_source(Path("synthetic-unused-source"))
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def test_legacy_repcount_normalization_is_hash_pinned_and_field_narrow() -> None:
|
| 66 |
+
from data_pipeline.temporal_scope.build_source_aligned_sft_release import _load_legacy_repcount_source
|
| 67 |
+
with pytest.raises(SourceAlignedSFTError, match="Private legacy adapters"):
|
| 68 |
+
_load_legacy_repcount_source(Path("synthetic-unused-source"))
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
def test_pinned_legacy_inputs_count_as_sealed_sources() -> None:
|
| 72 |
+
text = Path(
|
| 73 |
+
"data_pipeline/temporal_scope/build_source_aligned_sft_release.py"
|
| 74 |
+
).read_text()
|
| 75 |
+
assert "sealed_source_count" in text
|
| 76 |
+
assert 'getattr(args, "legacy_v62di_source", None)' in text
|
| 77 |
+
assert 'getattr(args, "legacy_repcount_source", None)' in text
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
def test_hash_bound_parent_selection_allows_one_source_filtering_only() -> None:
|
| 81 |
+
text = Path(
|
| 82 |
+
"data_pipeline/temporal_scope/build_source_aligned_sft_release.py"
|
| 83 |
+
).read_text()
|
| 84 |
+
assert 'getattr(args, "include_parent_selection", None) is None' in text
|
| 85 |
+
assert "unless a parent selection is supplied" in text
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
def test_composite_source_stage_filter_is_range_and_manifest_bound() -> None:
|
| 89 |
+
text = Path(
|
| 90 |
+
"data_pipeline/temporal_scope/build_source_aligned_sft_release.py"
|
| 91 |
+
).read_text()
|
| 92 |
+
assert "--include-source-stage-release-id" in text
|
| 93 |
+
assert "requested source stages are absent" in text
|
| 94 |
+
assert "source curriculum stage row count diverged" in text
|
| 95 |
+
assert '"included_source_stage_release_ids"' in text
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
def test_capability_parent_selection_is_hash_bound_and_fail_closed() -> None:
|
| 99 |
+
text = Path(
|
| 100 |
+
"data_pipeline/temporal_scope/build_source_aligned_sft_release.py"
|
| 101 |
+
).read_text()
|
| 102 |
+
assert "--include-parent-selection" in text
|
| 103 |
+
assert "iclr_video.capability_parent_selection.v0" in text
|
| 104 |
+
assert "iclr_video.capability_parent_selection.v1" in text
|
| 105 |
+
assert "source_release_membership_verified" in text
|
| 106 |
+
assert "selected parent IDs were absent" in text
|
| 107 |
+
assert '"parent_selection": parent_selection_record' in text
|
| 108 |
+
|
| 109 |
+
|
| 110 |
+
def test_ordered_curriculum_is_one_run_and_preserves_source_order() -> None:
|
| 111 |
+
text = Path(
|
| 112 |
+
"data_pipeline/temporal_scope/build_source_aligned_sft_release.py"
|
| 113 |
+
).read_text()
|
| 114 |
+
assert '"training_runs_required": 1' in text
|
| 115 |
+
assert '"source_stage_then_source_row"' in text
|
| 116 |
+
assert "if not ordered_curriculum" in text
|
| 117 |
+
|
| 118 |
+
|
| 119 |
+
def test_additive_composition_can_retain_exposures_without_curriculum_order() -> None:
|
| 120 |
+
text = Path(
|
| 121 |
+
"data_pipeline/temporal_scope/build_source_aligned_sft_release.py"
|
| 122 |
+
).read_text()
|
| 123 |
+
assert "--retain-semantic-exposures" in text
|
| 124 |
+
assert "ordered_curriculum or retain_semantic_exposures" in text
|
| 125 |
+
assert '"retain_semantic_exposures": retain_semantic_exposures' in text
|
code/tests/test_third_view_disagreement_agent.py
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from experiments.materialize_third_view_disagreement_agent import materialize
|
| 2 |
+
|
| 3 |
+
|
| 4 |
+
IDENTITY = {
|
| 5 |
+
"benchmark": "demo",
|
| 6 |
+
"model_manifest_sha256": "m",
|
| 7 |
+
"annotation_sha256": "a",
|
| 8 |
+
"evaluation_protocol_sha256": "p",
|
| 9 |
+
"annotation": "annotation.json",
|
| 10 |
+
"evaluation_protocol": {},
|
| 11 |
+
"model": "model",
|
| 12 |
+
"processor": "processor",
|
| 13 |
+
}
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
def _result(rows: list[dict]) -> dict:
|
| 17 |
+
return {**IDENTITY, "rows": rows}
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
def test_third_view_commits_only_on_acquired_disagreement() -> None:
|
| 21 |
+
recent = _result(
|
| 22 |
+
[
|
| 23 |
+
{"uid": "u0", "prediction": "A", "visual_tokens": 10},
|
| 24 |
+
{"uid": "u1", "prediction": "D", "visual_tokens": 10},
|
| 25 |
+
]
|
| 26 |
+
)
|
| 27 |
+
event = _result(
|
| 28 |
+
[
|
| 29 |
+
{
|
| 30 |
+
"uid": "u0",
|
| 31 |
+
"prediction": "B",
|
| 32 |
+
"error": None,
|
| 33 |
+
"visual_tokens": 20,
|
| 34 |
+
"sampling": {"frame_indices": list(range(20))},
|
| 35 |
+
}
|
| 36 |
+
]
|
| 37 |
+
)
|
| 38 |
+
base = _result(
|
| 39 |
+
[
|
| 40 |
+
{
|
| 41 |
+
"uid": "u0",
|
| 42 |
+
"prediction": "A",
|
| 43 |
+
"agent_policy": {"escalated": True},
|
| 44 |
+
"media_calls": 2,
|
| 45 |
+
"num_turns": 2,
|
| 46 |
+
"model_processed_frames": 24,
|
| 47 |
+
"sampling": {"frame_indices": list(range(20))},
|
| 48 |
+
},
|
| 49 |
+
{"uid": "u1", "prediction": "D", "media_calls": 1},
|
| 50 |
+
]
|
| 51 |
+
)
|
| 52 |
+
third = _result(
|
| 53 |
+
[
|
| 54 |
+
{
|
| 55 |
+
"uid": "u0",
|
| 56 |
+
"prediction": "C",
|
| 57 |
+
"raw_response": "C",
|
| 58 |
+
"error": None,
|
| 59 |
+
"query_time": 31.0,
|
| 60 |
+
"frames_observed": 32,
|
| 61 |
+
"visual_tokens": 30,
|
| 62 |
+
"sampling": {
|
| 63 |
+
"frame_indices": list(range(32)),
|
| 64 |
+
"frame_timestamps": list(map(float, range(32))),
|
| 65 |
+
},
|
| 66 |
+
}
|
| 67 |
+
]
|
| 68 |
+
)
|
| 69 |
+
result = materialize(recent, event, base, third)
|
| 70 |
+
rows = {row["uid"]: row for row in result["rows"]}
|
| 71 |
+
assert rows["u0"]["prediction"] == "C"
|
| 72 |
+
assert rows["u0"]["media_calls"] == 3
|
| 73 |
+
assert rows["u0"]["model_processed_frames"] == 56
|
| 74 |
+
assert rows["u0"]["unique_frames"] == 32
|
| 75 |
+
assert rows["u0"]["visual_tokens"] == 60
|
| 76 |
+
assert rows["u1"]["prediction"] == "D"
|
| 77 |
+
assert rows["u1"]["third_view_policy"]["triggered"] is False
|
| 78 |
+
assert result["policy"]["question_task_group_or_benchmark_router"] is False
|