diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..a6edf747f20dcecfc5015924ddeaf9b8b843fe68 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text +unmerged/tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/COMPLETE.json b/COMPLETE.json new file mode 100644 index 0000000000000000000000000000000000000000..2d4deed450491b0badfb73f212511d81baed3de3 --- /dev/null +++ b/COMPLETE.json @@ -0,0 +1,6 @@ +{ + "model": "Valen-4B", + "validated": true, + "files": 21, + "source_checkpoint_sha256": "b4193b4e907f44b913fd54a8d6b998e7050a1528306269a000f1e5e8bd8dffc7" +} diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..f938136e3adacfd92be087f6e113b5d6d97f678f --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2026 Alibaba Cloud + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/MODEL_USAGE.md b/MODEL_USAGE.md new file mode 100644 index 0000000000000000000000000000000000000000..f629d9c1519c5eeea68c7c50fbf3569101db6186 --- /dev/null +++ b/MODEL_USAGE.md @@ -0,0 +1,49 @@ +# Valen-4B + +完整合并的 Qwen3.5 + 两层 Mixer，提供 Choice、Noul、Score 决策和 share_state 多题推理。 +Complete Qwen3.5 backbone with merged LoRA, trained visual merger, and the FP32 Mixer head. + +训练版本：1195k + JevBench 30k，两阶段 SFT，最终 step 325。 +全体基座参数、训练参数、分词器及输入编译代码均在本目录。 + +## 加载 / Load + +验证环境：Python 3.10、PyTorch 2.6.0、Transformers 5.4.0、torchvision 0.21.0、Pillow、av 16.1.0。 +使用 Flash Attention 2 时，另需与 PyTorch/CUDA 匹配的 flash-attn。 + +```python +import torch +from transformers import AutoModel + +model = AutoModel.from_pretrained( + "./Valen-4B", + trust_remote_code=True, + dtype=torch.bfloat16, + attn_implementation="sdpa", # 或 flash_attention_2 / or flash_attention_2 +).to("cuda").eval() +# 和导出一致性验证保持相同的 FP32 精度。 / Match export verification precision. +torch.set_float32_matmul_precision("highest") +torch.backends.cudnn.allow_tf32 = False + +response = model.predict({ + "state": "There is one cat in the room.", + "questions": { + "animal": {"type": "choice", "instructions": "Which animal is present?", + "criteria": {"cat": "A cat", "dog": "A dog"}}, + "is_cat": {"type": "noul", "instructions": "There is a cat in the room."}, + "count": {"type": "score", "instructions": "How many cats are present?", + "criteria": ["No cats", "Exactly one cat", "Two or more cats"]}, + }, +}) +print(response["answers"]) +``` + +图像/视频使用 Valen 的 messages、image_url、video_url 输入格式，媒体路径由 `media_root` 解析。 +默认 `execution="shared_state"`；`model.predict(..., execution="question")` 可使用独立问题路径。 +视频沿用训练配置，采样 16 帧。所有 Mixer 参数保持 FP32，基座权重为 BF16。 + +`validation.json` 记录合并前后概率差异、重新加载一致性、文本/图像/视频及三类题型检查。 +`export_manifest.json` 记录来源 checkpoint、基座版本及文件 SHA-256。 +LoRA 以 FP32 累加后保存为 BF16；融合矩阵的舍入顺序与独立 LoRA 分支不同。 +验证样本中最大概率绝对差：0.01969546；预测标签变化：0 / 20。 +该检查只验证导出样本，不能代替完整 benchmark；之前 checkpoint 的成绩属于未融合版本。 diff --git a/README.md b/README.md index 154df8298fab5ecf322016157858e08cd1bccbe1..359f89d8a67a52bf3aabb61f9844bf887852cf90 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,106 @@ --- license: apache-2.0 +base_model: Qwen/Qwen3.5-4B +library_name: transformers +tags: +- valen +- decision-making +- shared-state +- qwen3.5 +- custom-code --- + +# Valen-4B + +Valen-4B combines Qwen3.5-4B with a two-layer MLP-Mixer decision head. It predicts probability distributions for Choice, Noul (yes/no), and Score questions. A shared state can contain text, images, or video and serve multiple questions. + +Valen-4B 基于 Qwen3.5-4B 和两层 MLP-Mixer 决策头，支持图文、文本、视频，以及共享 state 的多题推理。 + +## Training / 训练 + +- Two-stage SFT: decision-head warmup followed by joint LoRA, Mixer, and visual-merger fine-tuning. +- Training data: the previous 1,195,000 records plus JevBench train_v2 30,000 records, totaling 1,225,000 records. This count refers to records; shared-state records may contain multiple questions. +- Warmup: 122,500 records, 67 optimizer steps. Joint SFT: one epoch over all 1,225,000 records, 325 optimizer steps, on 32 H200 GPUs. +- Final checkpoint: joint SFT step 325. This release uses SFT training. +- Backbone: BF16. Mixer: FP32. Videos use 16 sampled frames. + +## Files / 权重格式 + +- Root directory: complete backbone with merged LoRA, trained visual merger, FP32 Mixer, tokenizer, processor, and standalone loading code. +- `unmerged/`: separate PEFT LoRA, Mixer, and trained visual-merger weights, with its own configuration and loading code. Requires the pinned original Qwen3.5-4B base. +- `MODEL_USAGE.md`: inference examples and environment details. +- `validation.json`, `fresh_process_validation.json`: export checks covering text, images, video, Choice/Noul/Score, question/shared_state, and isolated Python loading. +- `export_manifest.json`: source revision and file checksums. + +## Load merged weights / 加载完整合并权重 + +The verified environment uses Python 3.10, PyTorch 2.6.0, Transformers 5.4.0, torchvision 0.21.0, Pillow, and av 16.1.0. Flash Attention 2 requires a compatible flash-attn installation. Private repositories require an account with access. + +```python +import torch +from transformers import AutoModel + +model = AutoModel.from_pretrained( + "Valen-Team/Valen-4B", + trust_remote_code=True, + dtype=torch.bfloat16, + attn_implementation="sdpa", # or "flash_attention_2" +).to("cuda").eval() +torch.set_float32_matmul_precision("highest") +torch.backends.cudnn.allow_tf32 = False + +response = model.predict({ + "state": "There is one cat in the room.", + "questions": { + "animal": {"type": "choice", "instructions": "Which animal is present?", + "criteria": {"cat": "A cat", "dog": "A dog"}}, + "is_cat": {"type": "noul", "instructions": "There is a cat in the room."}, + }, +}, execution="shared_state") +print(response["answers"]) +``` + +For a local copy, replace the repository ID with `"./Valen-4B"`. +`execution="question"` scores questions separately. Images and video use the Valen `messages`, `image_url`, and `video_url` schema; relative media paths are resolved against `media_root`. + +## Load unmerged LoRA / 加载未合并权重 + +Download the repository first; the adapter loader expects a local directory. PEFT 0.18.1 and safetensors are also required. + +```python +import torch +from huggingface_hub import snapshot_download +from transformers import AutoModel + +folder = snapshot_download("Valen-Team/Valen-4B") +model = AutoModel.from_pretrained( + folder + "/unmerged", + trust_remote_code=True, + local_files_only=False, # download the pinned Qwen base if needed + dtype=torch.bfloat16, + attn_implementation="sdpa", +).to("cuda").eval() +torch.set_float32_matmul_precision("highest") +torch.backends.cudnn.allow_tf32 = False +``` + +For an existing local base, pass `base_model_path="./Qwen3.5-4B"` and `local_files_only=True`. The loader verifies its original weight hashes and restores LoRA, Mixer, and visual-merger weights together. `is_trainable=True` enables training those parameters. + +## Reference benchmark results / 参考评测 + +Measured on the original final **unmerged** SFT checkpoint, using shared-state inference for multimodal sets and single-question inference for JevBench: + +| Benchmark | QA | Accuracy | Choice | Noul | Score | +| --- | --- | --- | --- | --- | --- | +| eval_v1 | 2,000 | 83.06% | 85.42% | 83.25% | 54.00% | +| General | 10,000 | 88.68% | 90.68% | 85.76% | 75.60% | +| Video | 7,143 | 87.36% | 90.80% | 93.14% | 72.43% | +| JevBench | 231 | 83.12% | 84.89% | 81.08% | 77.78% | + +JevBench difficulty scores: Easy 100.00% (48/48), Standard 93.06% (67/72), Hard 69.37% (77/111). The overall score is weighted by task count: 192/231. + +The export smoke cases showed a maximum absolute probability difference of 0.01969546 between merged and unmerged inference, with no label changes in those cases. This does not establish full-benchmark equivalence. + +For eval_v1, accuracy uses 1,700 hard-label questions. Its remaining 300 soft-label Score questions contribute probability metrics and are excluded from hard-label accuracy. + +Merged BF16 matrices and separate LoRA branches can produce numerical differences. Export validation records those differences; the merged release has not been independently rerun on the complete benchmarks. The unmerged bundle preserves the original trained tensors and passed reloading checks. diff --git a/batching.py b/batching.py new file mode 100644 index 0000000000000000000000000000000000000000..6bd343e1d7a8479c6bbc45d73b987b1d1d2d5c28 --- /dev/null +++ b/batching.py @@ -0,0 +1,62 @@ +"""Pad language rows and concatenate visual patches. / 文本补齐，视觉 patch 拼接。""" +import torch +from torch.nn.utils.rnn import pad_sequence + + +SEQUENCE_KEYS = {"input_ids", "attention_mask", "mm_token_type_ids"} +VISUAL_KEYS = {"pixel_values", "pixel_values_videos", "image_grid_thw", "video_grid_thw"} + + +def collate_branches(branches, pad_token_id=0): + if not branches: + raise ValueError("Cannot collate an empty branch batch") + inputs = [branch.inputs for branch in branches] + unknown = set().union(*(row.keys() for row in inputs)) - SEQUENCE_KEYS - VISUAL_KEYS + if unknown: + raise ValueError(f"Unsupported batched Qwen inputs: {sorted(unknown)}") + if any(row["input_ids"].ndim != 2 or row["input_ids"].shape[0] != 1 for row in inputs): + raise ValueError("Each compiled branch must contain one language row") + result = {} + for key in SEQUENCE_KEYS: + if key != "input_ids" and not any(key in row for row in inputs): + continue + rows = [] + for row in inputs: + value = row.get(key) + if value is None: + value = (torch.ones_like(row["input_ids"]) if key == "attention_mask" + else torch.zeros_like(row["input_ids"])) + if value.shape != row["input_ids"].shape: + raise ValueError(f"{key} must match the branch input_ids shape") + rows.append(value[0]) + # Right padding keeps all compiler readout positions and spans unchanged. + # 右侧补齐，候选位置和角色区间无需偏移。 + result[key] = pad_sequence(rows, batch_first=True, + padding_value=pad_token_id if key == "input_ids" else 0) + if "attention_mask" not in result: + result["attention_mask"] = pad_sequence([torch.ones_like(r["input_ids"][0]) for r in inputs], + batch_first=True, padding_value=0) + for key in VISUAL_KEYS: + values = [row[key] for row in inputs if key in row] + if values: + result[key] = torch.cat(values, dim=0) + return result + + +def branch_batches(questions, max_tokens): + """Limit padded tokens per backbone forward, keeping Score branch order.""" + if type(max_tokens) is not int or max_tokens <= 0: + raise ValueError("microbatch_max_tokens must be a positive integer") + batch, longest = [], 0 + for question_index, question in enumerate(questions): + if not question.branches: + raise ValueError("A Qwen question requires at least one branch") + for branch in question.branches: + length = branch.inputs["input_ids"].shape[1] + if batch and max(longest, length) * (len(batch) + 1) > max_tokens: + yield batch + batch, longest = [], 0 + batch.append((question_index, branch)) + longest = max(longest, length) + if batch: + yield batch diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..a585dec894e63da457d9440ec6aa7caa16d20860 --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/compiler.py b/compiler.py new file mode 100644 index 0000000000000000000000000000000000000000..a5b9fe08e2cc9cd8564d6325211bf10ebf789089 --- /dev/null +++ b/compiler.py @@ -0,0 +1,239 @@ +"""Compile Qwen question branches or a shared sequence with per-question readouts.""" +from copy import deepcopy +from dataclasses import dataclass +from pathlib import Path +import hashlib +import torch +from .schema import candidates, target_distribution +from .data_types import QuestionSpec + + +@dataclass +class Branch: + inputs: dict + candidate_positions: list + decision_position: int + context_span: tuple | None = None + instruction_span: tuple | None = None + candidate_spans: list | None = None + + +@dataclass(init=False) +class Question(QuestionSpec): + branches: list + + def __init__(self, qid, kind, keys, descriptions, branches, target, is_text): + # Keep the positional constructor used by existing compiler clients. + super().__init__(qid, kind, keys, descriptions, target, is_text) + self.branches = branches + + +@dataclass +class CompiledState: + questions: list + logical_tokens: int + compute_tokens: int + media: list + inputs: dict | None = None + + +class Compiler: + def __init__(self, processor, media_root=".", max_length=8192, media_kwargs=None, verified_media_hashes=None, + execution="question"): + if execution not in {"question", "shared_state"}: + raise ValueError(f"Unknown qwen_execution: {execution}") + self.execution = execution + self.processor = processor + self.tokenizer = processor.tokenizer + self.media_root = Path(media_root) + self.max_length = max_length + self.media_kwargs = media_kwargs or {} + # Optional, pre-audited immutable assets for latency measurements. + # Normal training/inference continues to hash each input at use time. + self.verified_media_hashes = verified_media_hashes or {} + + def _messages(self, state): + messages = [{"role": "user", "content": state}] if isinstance(state, str) else deepcopy(state["messages"]) + media = [] + for message in messages: + if message["role"] not in {"user", "assistant", "system"}: + raise ValueError("Unsupported state role") + if isinstance(message["content"], str): + message["content"] = [{"type": "text", "text": message["content"]}] + for item in message["content"]: + kind = item["type"] + if kind == "text": + # Prevent user text from injecting Qwen control tokens. + if any(t in item["text"] for t in self.tokenizer.all_special_tokens): + raise ValueError("Reserved tokenizer control token in state") + continue + if kind not in {"image_url", "video_url"}: + raise ValueError(f"Unsupported content type: {kind}") + url = item[kind]["url"] + if "://" in url: + raise ValueError("Materialize remote media locally with a hash before training") + path = Path(url) + path = path if path.is_absolute() else self.media_root / path + path = path.resolve(strict=True) + digest = self.verified_media_hashes.get(str(path)) + if digest is None: + digest = hashlib.sha256(path.read_bytes()).hexdigest() + media.append({"path": str(path), "sha256": digest, "type": kind}) + item.clear() + item.update(type=kind.removesuffix("_url"), path=str(path)) + return messages, media + + def compile(self, record, rng=None, labeled_only=False): + selected = [] + for qid, question in record["request"]["questions"].items(): + pairs = candidates(question) + target = record.get("targets", {}).get(qid) + if labeled_only and self.execution == "question" and target_distribution(target, [k for k, _ in pairs]) is None: + continue + if rng is not None and question["type"] == "choice": + rng.shuffle(pairs) + selected.append((qid, question, pairs, target)) + # 标签只决定监督范围，共享序列仍保留所有题。 / Labels select losses, never prompt content. + if not selected or (labeled_only and not any( + target_distribution(target, [k for k, _ in pairs]) is not None + for _, _, pairs, target in selected)): + return CompiledState([], 0, 0, []) + messages, media = self._messages(record["request"]["state"]) + expected_assets = {str((self.media_root / a["path"]).resolve()): a["sha256"] for a in record.get("assets", [])} + for entry in media: + if entry["path"] in expected_assets and expected_assets[entry["path"]] != entry["sha256"]: + raise ValueError(f"Media hash mismatch: {entry['path']}") + base = dict(self.processor.apply_chat_template( + messages, tokenize=True, add_generation_prompt=False, return_dict=True, + return_tensors="pt", processor_kwargs=dict(self.media_kwargs, return_metadata=True))) + # Metadata is audit information, never a backbone keyword argument. + metadata = base.pop("video_metadata", None) + if metadata is not None: + for entry, meta in zip([m for m in media if m["type"] == "video_url"], metadata): + entry["sampling"] = dict(meta) if isinstance(meta, dict) else vars(meta) + sampling = entry["sampling"] + indices = sampling.get("frames_indices") + if indices is not None and sampling.get("fps"): + sampling["frames_indices"] = [int(i) for i in indices] + sampling["timestamps_seconds"] = [int(i) / sampling["fps"] for i in indices] + sampling["visible_range_seconds"] = [min(sampling["timestamps_seconds"]), max(sampling["timestamps_seconds"])] + for name in ("image_grid_thw", "video_grid_thw"): + if name in base: + grid = base[name] + media.append({name: grid.tolist(), "visual_tokens": int(grid.prod(-1).sum()) // 4}) + base_length = base["input_ids"].shape[1] + if self.execution == "shared_state": + return self._compile_shared(selected, base, media, labeled_only) + logical_tokens, compute_tokens = base_length, 0 + compiled = [] + for qid, question, pairs, target in selected: + keys = [k for k, _ in pairs] + branches = [] + groups = [[pair] for pair in pairs] if question["type"] == "score" else [pairs] + for group in groups: + suffix = [] + def append(text, char_span=None): + start = base_length + len(suffix) + if char_span is None: + suffix.extend(self.tokenizer.encode(text, add_special_tokens=False)) + return None + # 保持原有分词，只额外记录区间。 / Preserve token IDs; record role spans. + encoded = self.tokenizer(text, add_special_tokens=False, return_offsets_mapping=True) + suffix.extend(encoded["input_ids"]) + indices = [i for i, (a, b) in enumerate(encoded["offset_mapping"]) + if a < char_span[1] and b > char_span[0]] + if not indices: + raise ValueError("Empty token span for decision role") + return (start + indices[0], start + indices[-1] + 1) + # Canonical segment tokenization makes endpoints exact; no string searching. + suffix.extend(self.tokenizer.encode("<|im_start|>user\n", add_special_tokens=False)) + for value in [question["instructions"]] + [v for pair in group for v in pair]: + if any(t in value for t in self.tokenizer.all_special_tokens): + raise ValueError("Reserved tokenizer control token in question/criteria") + prefix = "Task: " + question["type"] + "\nQuestion: " + instruction_span = append(prefix + question["instructions"] + "\nCandidates:\n", + (len(prefix), len(prefix) + len(question["instructions"]))) + positions, candidate_spans = [], [] + for key, description in group: + # Score never sees its level ID, nor any other level description. + prefix = key + ": " if question["type"] != "score" else "" + candidate_spans.append(append(prefix + description, (len(prefix), len(prefix) + len(description)))) + positions.append(base_length + len(suffix) - 1) + append("\n") + append("Decision:") + length = base_length + len(suffix) + if length > self.max_length: + raise ValueError(f"{qid}: {length} tokens exceed max_length={self.max_length}; no truncation") + inputs = dict(base) + inputs["input_ids"] = torch.cat([base["input_ids"], torch.tensor([suffix], dtype=torch.long)], dim=1) + inputs["attention_mask"] = torch.ones_like(inputs["input_ids"]) + if "mm_token_type_ids" in base: + inputs["mm_token_type_ids"] = torch.cat([base["mm_token_type_ids"], torch.zeros((1, len(suffix)), dtype=torch.long)], dim=1) + branches.append(Branch(inputs, positions, length - 1, (0, base_length), + instruction_span, candidate_spans)) + compute_tokens += length + logical_tokens += len(suffix) + compiled.append(Question(qid, question["type"], keys, [v for _, v in pairs], branches, + target_distribution(target, keys), not media)) + return CompiledState(compiled, logical_tokens, compute_tokens, media) + + def _compile_shared(self, selected, base, media, labeled_only=False): + """All schemas precede all readouts; labels and external IDs never enter inputs. + + This is ordinary causal attention, not isolated question/Score branches. + No answer tokens or LM output layer are required by the decision heads. + """ + base_length = base["input_ids"].shape[1] + suffix, compiled = [], [] + + def append(text, span=None): + start = base_length + len(suffix) + if span is None: + suffix.extend(self.tokenizer.encode(text, add_special_tokens=False)) + return None + encoded = self.tokenizer(text, add_special_tokens=False, return_offsets_mapping=True) + suffix.extend(encoded["input_ids"]) + indices = [i for i, (a, b) in enumerate(encoded["offset_mapping"]) if a < span[1] and b > span[0]] + if not indices: + raise ValueError("Empty token span for decision role") + return start + indices[0], start + indices[-1] + 1 + + append("<|im_start|>user\n") + for index, (qid, question, pairs, target) in enumerate(selected): + for value in [question["instructions"]] + [v for pair in pairs for v in pair]: + if any(t in value for t in self.tokenizer.all_special_tokens): + raise ValueError("Reserved tokenizer control token in question/criteria") + prefix = f"Question {index + 1}\nTask: {question['type']}\nQuestion: " + instruction = append(prefix + question["instructions"] + "\nCandidates:\n", + (len(prefix), len(prefix) + len(question["instructions"]))) + positions, spans = [], [] + for key, description in pairs: + prefix = key + ": " if question["type"] != "score" else "" + spans.append(append(prefix + description, (len(prefix), len(prefix) + len(description)))) + positions.append(base_length + len(suffix) - 1) + append("\n") + # Inputs and the decision position are filled after all schemas are appended. + branch = Branch({}, positions, -1, (0, base_length), instruction, spans) + keys = [key for key, _ in pairs] + compiled.append(Question(qid, question["type"], keys, [v for _, v in pairs], [branch], + target_distribution(target, keys), not media)) + append("<|im_end|>\n<|im_start|>assistant\n") + for index, question in enumerate(compiled): + append(f"Question {index + 1} Decision:") + question.branches[0].decision_position = base_length + len(suffix) - 1 + append("\n") + length = base_length + len(suffix) + if length > self.max_length: + raise ValueError(f"Shared state: {length} tokens exceed max_length={self.max_length}; no truncation") + inputs = dict(base) + inputs["input_ids"] = torch.cat([base["input_ids"], base["input_ids"].new_tensor([suffix])], dim=1) + inputs["attention_mask"] = torch.cat([base["attention_mask"], + base["attention_mask"].new_ones((1, len(suffix)))], dim=1) + if "mm_token_type_ids" in base: + inputs["mm_token_type_ids"] = torch.cat([base["mm_token_type_ids"], + base["mm_token_type_ids"].new_zeros((1, len(suffix)))], dim=1) + for question in compiled: + question.branches[0].inputs = inputs + if labeled_only: + compiled = [question for question in compiled if question.target is not None] + return CompiledState(compiled, length, length, media, inputs) diff --git a/config.json b/config.json new file mode 100644 index 0000000000000000000000000000000000000000..1be34fc1c9d3569e7dc57ce6606eaa9ebe60de42 --- /dev/null +++ b/config.json @@ -0,0 +1,138 @@ +{ + "architectures": [ + "ValenQwenForDecisionMaking" + ], + "auto_map": { + "AutoConfig": "configuration_valen.ValenQwenConfig", + "AutoModel": "modeling_valen.ValenQwenForDecisionMaking" + }, + "dtype": "bfloat16", + "image_token_id": 248056, + "model_type": "valen_qwen", + "qwen_execution": "shared_state", + "text_config": { + "attention_bias": false, + "attention_dropout": 0.0, + "attn_output_gate": true, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 248044, + "full_attention_interval": 4, + "head_dim": 256, + "hidden_act": "silu", + "hidden_size": 2560, + "initializer_range": 0.02, + "intermediate_size": 9216, + "layer_types": [ + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention" + ], + "linear_conv_kernel_dim": 4, + "linear_key_head_dim": 128, + "linear_num_key_heads": 16, + "linear_num_value_heads": 32, + "linear_value_head_dim": 128, + "mamba_ssm_dtype": "float32", + "max_position_embeddings": 262144, + "mlp_only_layers": [], + "model_type": "qwen3_5_text", + "mtp_num_hidden_layers": 1, + "mtp_use_dedicated_embeddings": false, + "num_attention_heads": 16, + "num_hidden_layers": 32, + "num_key_value_heads": 4, + "pad_token_id": null, + "partial_rotary_factor": 0.25, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "mrope_interleaved": true, + "mrope_section": [ + 11, + 11, + 10 + ], + "partial_rotary_factor": 0.25, + "rope_theta": 10000000, + "rope_type": "default" + }, + "tie_word_embeddings": true, + "use_cache": true, + "vocab_size": 248320 + }, + "tie_word_embeddings": true, + "transformers_version": "5.4.0", + "valen_head": { + "head_channel_hidden_dim": 1024, + "head_layers": 2, + "head_token_hidden_dim": 16, + "head_type": "mixer", + "head_width": 256 + }, + "valen_max_length": 32768, + "valen_media_kwargs": { + "images_kwargs": { + "size": { + "longest_edge": 262144, + "shortest_edge": 65536 + } + }, + "videos_kwargs": { + "fps": null, + "num_frames": 16, + "size": { + "longest_edge": 2097152, + "shortest_edge": 131072 + } + } + }, + "video_token_id": 248057, + "vision_config": { + "deepstack_visual_indexes": [], + "depth": 24, + "dtype": "bfloat16", + "hidden_act": "gelu_pytorch_tanh", + "hidden_size": 1024, + "in_channels": 3, + "initializer_range": 0.02, + "intermediate_size": 4096, + "model_type": "qwen3_5", + "num_heads": 16, + "num_position_embeddings": 2304, + "out_hidden_size": 2560, + "patch_size": 16, + "spatial_merge_size": 2, + "temporal_patch_size": 2 + }, + "vision_end_token_id": 248054, + "vision_start_token_id": 248053 +} diff --git a/configuration_valen.py b/configuration_valen.py new file mode 100644 index 0000000000000000000000000000000000000000..a9252326abac70b4afc90518b61999f6ff07284c --- /dev/null +++ b/configuration_valen.py @@ -0,0 +1,16 @@ +"""Standalone Qwen decision configuration. / 独立的 Qwen 决策模型配置。""" +from transformers import Qwen3_5Config + + +class ValenQwenConfig(Qwen3_5Config): + model_type = "valen_qwen" + + def __init__(self, valen_head=None, qwen_execution="shared_state", valen_max_length=32768, + valen_media_kwargs=None, **kwargs): + if qwen_execution not in {"question", "shared_state"}: + raise ValueError("qwen_execution must be question or shared_state") + self.valen_head = valen_head or {"head_type": "mixer"} + self.qwen_execution = qwen_execution + self.valen_max_length = valen_max_length + self.valen_media_kwargs = valen_media_kwargs or {} + super().__init__(**kwargs) diff --git a/data_types.py b/data_types.py new file mode 100644 index 0000000000000000000000000000000000000000..ce1d6a8d6ee0f46600b13865ea941dcd7d803de0 --- /dev/null +++ b/data_types.py @@ -0,0 +1,12 @@ +"""Model-independent decision metadata; encoded tensors stay with each compiler.""" +from dataclasses import dataclass + + +@dataclass +class QuestionSpec: + qid: str + kind: str + keys: list + descriptions: list + target: list | None + is_text: bool diff --git a/export_manifest.json b/export_manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..34502ed249514e3e6a8b2f6f4fc98b77c3e880f6 --- /dev/null +++ b/export_manifest.json @@ -0,0 +1,82 @@ +{ + "format": "valen_qwen_merged_v1", + "created_utc": "2026-10-06T06:52:07.182367+00:00", + "model": "Valen-4B", + "architecture": "Qwen3.5 + two-layer Mixer", + "execution": "shared_state", + "source_checkpoint_sha256": "b4193b4e907f44b913fd54a8d6b998e7050a1528306269a000f1e5e8bd8dffc7", + "training_step": 325, + "training_records": 1225000, + "training_data_sha256": "648cb127fe4830f9faaa6d4595445cc2573ef0c152f8f74c9c776dc8a9223d7f", + "base": { + "repo": "Qwen/Qwen3.5-4B", + "revision": "851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a", + "weight_sha256": { + "model.safetensors-00001-of-00002.safetensors": "26a93f066e1916adb13453dae5a0c707c0fbc71299ed98779571a907b8e74c61", + "model.safetensors-00002-of-00002.safetensors": "cb544bd9bfae93dc59b0f22b292f5933573854a7f9b97835c67060d7d910e188" + }, + "files_sha256": { + ".gitattributes": "34448b82c17d60fec9b65b1f093c115ddbaadc04beb1b0140b6bfed2e012a930", + "LICENSE": "bbedc3fda3305820b977265f01b8619d87570a6739de3a5582c3464840f1e57a", + "README.md": "1406be1b6b8fd8a6545870da516912804756593628a1d0fb0a7965211e82a7bb", + "chat_template.jinja": "a4aee8afcf2e0711942cf848899be66016f8d14a889ff9ede07bca099c28f715", + "config.json": "ddc63e1c717afa86c865bb5e01313d89d72bb53b97ad4a8a03ba8510c0621670", + "merges.txt": "a9d356d7bdf1ef4949e3e748e95b8e10ad9d4e2e838eddc38a0a7b6b94d1db8d", + "model.safetensors-00001-of-00002.safetensors": "26a93f066e1916adb13453dae5a0c707c0fbc71299ed98779571a907b8e74c61", + "model.safetensors-00002-of-00002.safetensors": "cb544bd9bfae93dc59b0f22b292f5933573854a7f9b97835c67060d7d910e188", + "model.safetensors.index.json": "cf3f798ee02ba45f9622aa8892a47369ab667d0afbf154ee7c2212de42e6302d", + "preprocessor_config.json": "27225450ac9c6529872ee1924fcb0962ff5634834f817040f444118116f4e516", + "tokenizer.json": "5f9e4d4901a92b997e463c1f46055088b6cca5ca61a6522d1b9f64c4bb81cb42", + "tokenizer_config.json": "316230d6a809701f4db5ea8f8fc862bc3a6f3229c937c174e674ff3ca0a64ac8", + "video_preprocessor_config.json": "7768af27c1fafa9cc9011c1dc20067e03f8915e03b63504550e11d5066986d13", + "vocab.json": "ce99b4cb2983d118806ce0a8b777a35b093e2000a503ebde25853284c9dfa003" + }, + "verified_from": "download_manifest.json; every LFS SHA256 matched" + }, + "merge": "FP32 base+LoRA accumulation, stored BF16; trained visual merger and FP32 Mixer retained", + "merged_linear_layers": 248, + "runtime_source_sha256": { + "valen/modeling/qwen/heads.py": "63b5970a8bfa121eff94d50ffd35e1fa8387f3af5abccc5f5b75381fb084fc34", + "valen/modeling/qwen/batching.py": "ea45d4724bcafca559fc5a2a18092918b633746771c19da6554c29bc5f2cef64", + "valen/modeling/qwen/model.py": "37b97e52bd16ada6a841fd553454ec0b0e458f3966ca0bef8a1bfe15293c2271", + "valen/data/compilers/qwen.py": "cae0389c407a9e67e1b14776b56d87767046b4ad119e93d138e1ae2f20fcadb2", + "valen/data/schema.py": "253825862b60016e37fd1040492008ad7f303679c092313d8a6527cf509f3b6e", + "valen/data/types.py": "b06ce08081d8f7b2bf36abf302410a96086395e75a04f9e9b1c591db519b1641", + "valen/evaluation/inference.py": "ccb47efd9f029f19a70cc462e262143cbab021deb4b8f4d11d2ecec054cbe553" + }, + "versions": { + "torch": "2.6.0", + "transformers": "5.4.0", + "peft": "0.18.1", + "safetensors": "0.6.2", + "flash-attn": "2.7.4.post1", + "torchvision": "0.21.0", + "Pillow": "11.3.0", + "av": "16.1.0" + }, + "files_sha256": { + "LICENSE": "bbedc3fda3305820b977265f01b8619d87570a6739de3a5582c3464840f1e57a", + "MODEL_USAGE.md": "cc6beb959e63059e68d8e5badb4df45090a22f1b049b1762262deb1988a61590", + "batching.py": "ea45d4724bcafca559fc5a2a18092918b633746771c19da6554c29bc5f2cef64", + "chat_template.jinja": "a4aee8afcf2e0711942cf848899be66016f8d14a889ff9ede07bca099c28f715", + "compiler.py": "f35bab2e71b6beb1228daaa5b7baecdacf04eaf713aee14e8082b97090f910b4", + "config.json": "15dd1b0e94aa6ad330d3d8c50e4f9ea579edbb21fb2ba13c02b32c3e0ede6527", + "configuration_valen.py": "735b02402fb5b6c7a1652f6691ba66d066ef99e84500042faec396ff7a668fe5", + "data_types.py": "b06ce08081d8f7b2bf36abf302410a96086395e75a04f9e9b1c591db519b1641", + "heads.py": "63b5970a8bfa121eff94d50ffd35e1fa8387f3af5abccc5f5b75381fb084fc34", + "model-00001-of-00003.safetensors": "21fef5c5b93c2d9f1e54100f31709aaeb1651058784b9d38c47d9345fdba50d0", + "model-00002-of-00003.safetensors": "9ec6792a776db86b348b97ae3e9ff5feaa055dba2ea28035aac9f1d543623e9b", + "model-00003-of-00003.safetensors": "c7b96a0d85eb01475c0f775a51f88e4455aeae31143cc86c8457723d9019c13e", + "model.safetensors.index.json": "69f50ed2c7dcbbda30cc1b9ec3413ef84f2ac9aa569f357eec9f8ab63d1fdb95", + "modeling_valen.py": "07f87481334cb1e28e9daee036c27bab3654645abf3df7983346b316e7788848", + "processor_config.json": "d89ef49ce9cd37fbf510158e13c1ef063d9286411c1ec9049932dbe0487143b1", + "responses.py": "298016b0070f1000a38d033e1048e038ff5df6fc1dcbe830601105b181f1d228", + "runtime_model.py": "98bf0443ca196900b9f2b056129c84932041b8b53395d111f2fe6d3f2bc1e0a6", + "schema.py": "253825862b60016e37fd1040492008ad7f303679c092313d8a6527cf509f3b6e", + "tokenizer.json": "87a7830d63fcf43bf241c3c5242e96e62dd3fdc29224ca26fed8ea333db72de4", + "tokenizer_config.json": "e98f1901ac6f0adff67b1d540bfa0c36ac1a0cf59eb72ed78146ef89aafa1182", + "validation.json": "0ed2de1bc25b85c42d2329e42eb53bb6bad85e1ee3e2ec598a1dff643a3aa509", + "README.md": "fc0d6e3a980891edd2264d5a705da824cb1c0a31ebf52f44da54903c13d2b4c3", + "fresh_process_validation.json": "5945d5fc669cb573bb47db5fd41c3bb3ceb51b81b6951eea30681c7451505d43" + } +} diff --git a/fresh_process_validation.json b/fresh_process_validation.json new file mode 100644 index 0000000000000000000000000000000000000000..653a095b72586e569f260f197ed3be6999d9d1ce --- /dev/null +++ b/fresh_process_validation.json @@ -0,0 +1,112 @@ +{ + "passed": true, + "isolated_python": true, + "valen_imports_blocked": true, + "all_real_modalities_and_question_types_passed": true, + "sdpa_example_passed": true, + "cases": 20, + "max_probability_abs_error": 0.0, + "max_logit_abs_error": 0.0, + "details": [ + { + "case": "shared_state/text_multi/choice", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0 + }, + { + "case": "shared_state/text_multi/noul", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0 + }, + { + "case": "shared_state/text_multi/score", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0 + }, + { + "case": "shared_state/text_multi/score_many", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0 + }, + { + "case": "shared_state/image_score/decision", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0 + }, + { + "case": "shared_state/image_noul/decision", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0 + }, + { + "case": "shared_state/image_choice/decision", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0 + }, + { + "case": "shared_state/video_multi/q000", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0 + }, + { + "case": "shared_state/video_multi/score_000", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0 + }, + { + "case": "shared_state/video_multi/noul_000", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0 + }, + { + "case": "question/text_multi/choice", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0 + }, + { + "case": "question/text_multi/noul", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0 + }, + { + "case": "question/text_multi/score", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0 + }, + { + "case": "question/text_multi/score_many", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0 + }, + { + "case": "question/image_score/decision", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0 + }, + { + "case": "question/image_noul/decision", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0 + }, + { + "case": "question/image_choice/decision", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0 + }, + { + "case": "question/video_multi/q000", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0 + }, + { + "case": "question/video_multi/score_000", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0 + }, + { + "case": "question/video_multi/noul_000", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0 + } + ] +} diff --git a/heads.py b/heads.py new file mode 100644 index 0000000000000000000000000000000000000000..47627b2dc79bfafeea627683628f949f80ab7692 --- /dev/null +++ b/heads.py @@ -0,0 +1,143 @@ +"""可替换的候选评分头。 / Interchangeable, candidate-wise decision heads.""" +import math + +import torch +from torch import nn + + +def head_signature(config): + """用于构建与恢复校验。 / Canonical architecture settings for loading.""" + kind = config.get("head_type", "bilinear") + defaults = { + "bilinear": {"projection_dim": 256}, + "mlp": {"head_hidden_dim": 512, "head_bottleneck_dim": 128}, + "role_mlp": {"head_width": 256, "head_hidden_dim": 1024}, + "mixer": {"head_width": 256, "head_layers": 2, + "head_token_hidden_dim": 16, "head_channel_hidden_dim": 1024}, + } + if kind not in defaults: + raise ValueError(f"Unknown Qwen head_type: {kind!r}") + values = {key: config.get(key, value) for key, value in defaults[kind].items()} + if any(type(value) is not int or value <= 0 for value in values.values()): + raise ValueError("Decision head dimensions must be positive integers") + return {"head_type": kind, **values} + + +class DecisionHead(nn.Module): + """原有参数名保持兼容。 / Preserve legacy bilinear parameter names.""" + def __init__(self, hidden_size, projection_dim=256): + super().__init__() + self.decision = nn.Linear(hidden_size, projection_dim, bias=False) + self.candidate = nn.Linear(hidden_size, projection_dim, bias=False) + self.scale = math.sqrt(projection_dim) + + def forward(self, hidden, positions, decision_position): + d = self.decision(hidden[0, decision_position].float()) + c = self.candidate(hidden[0, positions].float()) + return (c * d).sum(-1) / self.scale + + def extract_features(self, hidden, branch): + return hidden[:, [*branch.candidate_positions, branch.decision_position], :] + + def score_features(self, features): + return self(features, list(range(features.shape[1] - 1)), features.shape[1] - 1) + + +class MLPHead(nn.Module): + """每个候选共享一个 MLP。 / One shared MLP for arbitrary candidate counts.""" + def __init__(self, hidden_size, hidden_dim=512, bottleneck_dim=128): + super().__init__() + self.decision_norm = nn.LayerNorm(hidden_size) + self.candidate_norm = nn.LayerNorm(hidden_size) + self.network = nn.Sequential(nn.Linear(2 * hidden_size, hidden_dim), nn.GELU(), + nn.Linear(hidden_dim, bottleneck_dim), nn.GELU(), + nn.Linear(bottleneck_dim, 1)) + + def extract_features(self, hidden, branch): + c = hidden[0, branch.candidate_positions] + d = hidden[0, branch.decision_position].expand_as(c) + return torch.stack((d, c), dim=1) + + def score_features(self, features): + d, c = features.float().unbind(1) + return self.network(torch.cat((self.decision_norm(d), self.candidate_norm(c)), -1)).squeeze(-1) + + +def role_features(hidden, branch): + """先池化再投影，可缓存冻结特征。 / Pool before trainable projections.""" + spans = (getattr(branch, "context_span", None), getattr(branch, "instruction_span", None)) + candidates = getattr(branch, "candidate_spans", None) + if any(span is None for span in spans) or not candidates or len(candidates) != len(branch.candidate_positions): + raise ValueError("Role heads require compiler context, instruction and candidate spans") + + def pool(span): + start, end = span + if not 0 <= start < end <= hidden.shape[1]: + raise ValueError(f"Invalid role span: {span}") + return hidden[0, start:end].float().mean(0) + + context, instruction = (pool(span) for span in spans) + decision = hidden[0, branch.decision_position].float() + return torch.stack([torch.stack((context, instruction, pool(span), decision)) for span in candidates]) + + +class RoleHead(nn.Module): + def __init__(self, hidden_size, width): + super().__init__() + self.input_norm = nn.LayerNorm(hidden_size) + self.projection = nn.Linear(hidden_size, width) + self.roles = nn.Parameter(torch.zeros(1, 4, width)) + + def extract_features(self, hidden, branch): + return role_features(hidden, branch) + + def project(self, features): + return self.projection(self.input_norm(features.float())) + self.roles + + +class RoleMLPHead(RoleHead): + """与 Mixer 使用相同输入。 / Control for the Mixer's richer input features.""" + def __init__(self, hidden_size, width=256, hidden_dim=1024): + super().__init__(hidden_size, width) + self.network = nn.Sequential(nn.Linear(4 * width, hidden_dim), nn.GELU(), nn.Linear(hidden_dim, 1)) + + def score_features(self, features): + return self.network(self.project(features).flatten(1)).squeeze(-1) + + +class MixerBlock(nn.Module): + def __init__(self, width, token_hidden, channel_hidden): + super().__init__() + self.token_norm = nn.LayerNorm(width) + self.token_mlp = nn.Sequential(nn.Linear(4, token_hidden), nn.GELU(), nn.Linear(token_hidden, 4)) + self.channel_norm = nn.LayerNorm(width) + self.channel_mlp = nn.Sequential(nn.Linear(width, channel_hidden), nn.GELU(), nn.Linear(channel_hidden, width)) + + def forward(self, x): + # 混合角色，不混合不同候选。 / Mix fixed roles, never candidate rows. + x = x + self.token_mlp(self.token_norm(x).transpose(1, 2)).transpose(1, 2) + return x + self.channel_mlp(self.channel_norm(x)) + + +class MixerHead(RoleHead): + def __init__(self, hidden_size, width=256, layers=2, token_hidden=16, channel_hidden=1024): + super().__init__(hidden_size, width) + self.blocks = nn.Sequential(*(MixerBlock(width, token_hidden, channel_hidden) for _ in range(layers))) + self.output = nn.Sequential(nn.LayerNorm(width), nn.Linear(width, 1)) + + def score_features(self, features): + # 第三行是候选角色。 / Read the updated candidate role. + return self.output(self.blocks(self.project(features))[:, 2]).squeeze(-1) + + +def build_head(hidden_size, config): + settings = head_signature(config) + kind = settings.pop("head_type") + if kind == "bilinear": + return DecisionHead(hidden_size, settings["projection_dim"]) + if kind == "mlp": + return MLPHead(hidden_size, settings["head_hidden_dim"], settings["head_bottleneck_dim"]) + if kind == "role_mlp": + return RoleMLPHead(hidden_size, settings["head_width"], settings["head_hidden_dim"]) + return MixerHead(hidden_size, settings["head_width"], settings["head_layers"], + settings["head_token_hidden_dim"], settings["head_channel_hidden_dim"]) diff --git a/model-00001-of-00003.safetensors b/model-00001-of-00003.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..2b75b810f61570d0dd0d0632dc8f621663f8f3d4 --- /dev/null +++ b/model-00001-of-00003.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:21fef5c5b93c2d9f1e54100f31709aaeb1651058784b9d38c47d9345fdba50d0 +size 3991299376 diff --git a/model-00002-of-00003.safetensors b/model-00002-of-00003.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..34d1812a12904fdf0f2ee7abf769ff673ecd977b --- /dev/null +++ b/model-00002-of-00003.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ec6792a776db86b348b97ae3e9ff5feaa055dba2ea28035aac9f1d543623e9b +size 3979833864 diff --git a/model-00003-of-00003.safetensors b/model-00003-of-00003.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..3bcd19335665e4a03a59b28ba6085623981748d8 --- /dev/null +++ b/model-00003-of-00003.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c7b96a0d85eb01475c0f775a51f88e4455aeae31143cc86c8457723d9019c13e +size 1114356012 diff --git a/model.safetensors.index.json b/model.safetensors.index.json new file mode 100644 index 0000000000000000000000000000000000000000..26e247fc33f850d2b05df4f98f4da0640d52da1d --- /dev/null +++ b/model.safetensors.index.json @@ -0,0 +1,764 @@ +{ + "metadata": { + "total_parameters": 4540981545, + "total_size": 9085395108 + }, + "weight_map": { + "backbone.language_model.embed_tokens.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.0.linear_attn.A_log": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.0.linear_attn.conv1d.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.0.linear_attn.dt_bias": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.0.linear_attn.in_proj_a.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.0.linear_attn.in_proj_b.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.0.linear_attn.in_proj_qkv.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.0.linear_attn.in_proj_z.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.0.linear_attn.norm.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.0.linear_attn.out_proj.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.1.linear_attn.A_log": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.1.linear_attn.conv1d.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.1.linear_attn.dt_bias": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.1.linear_attn.in_proj_a.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.1.linear_attn.in_proj_b.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.1.linear_attn.in_proj_qkv.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.1.linear_attn.in_proj_z.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.1.linear_attn.norm.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.1.linear_attn.out_proj.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.10.input_layernorm.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.10.linear_attn.A_log": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.10.linear_attn.conv1d.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.10.linear_attn.dt_bias": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.10.linear_attn.in_proj_a.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.10.linear_attn.in_proj_b.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.10.linear_attn.in_proj_qkv.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.10.linear_attn.in_proj_z.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.10.linear_attn.norm.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.10.linear_attn.out_proj.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.10.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.10.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.11.input_layernorm.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.11.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.11.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.11.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.11.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.11.self_attn.k_norm.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.11.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.11.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.11.self_attn.q_norm.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.11.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.11.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.12.input_layernorm.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.12.linear_attn.A_log": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.12.linear_attn.conv1d.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.12.linear_attn.dt_bias": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.12.linear_attn.in_proj_a.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.12.linear_attn.in_proj_b.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.12.linear_attn.in_proj_qkv.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.12.linear_attn.in_proj_z.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.12.linear_attn.norm.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.12.linear_attn.out_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.13.linear_attn.A_log": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.13.linear_attn.conv1d.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.13.linear_attn.dt_bias": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.13.linear_attn.in_proj_a.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.13.linear_attn.in_proj_b.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.13.linear_attn.in_proj_qkv.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.13.linear_attn.in_proj_z.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.13.linear_attn.norm.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.13.linear_attn.out_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.14.linear_attn.A_log": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.14.linear_attn.conv1d.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.14.linear_attn.dt_bias": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.14.linear_attn.in_proj_a.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.14.linear_attn.in_proj_b.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.14.linear_attn.in_proj_qkv.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.14.linear_attn.in_proj_z.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.14.linear_attn.norm.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.14.linear_attn.out_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.15.self_attn.k_norm.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.15.self_attn.q_norm.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.16.linear_attn.A_log": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.16.linear_attn.conv1d.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.16.linear_attn.dt_bias": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.16.linear_attn.in_proj_a.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.16.linear_attn.in_proj_b.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.16.linear_attn.in_proj_qkv.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.16.linear_attn.in_proj_z.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.16.linear_attn.norm.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.16.linear_attn.out_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.17.linear_attn.A_log": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.17.linear_attn.conv1d.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.17.linear_attn.dt_bias": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.17.linear_attn.in_proj_a.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.17.linear_attn.in_proj_b.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.17.linear_attn.in_proj_qkv.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.17.linear_attn.in_proj_z.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.17.linear_attn.norm.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.17.linear_attn.out_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.18.linear_attn.A_log": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.18.linear_attn.conv1d.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.18.linear_attn.dt_bias": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.18.linear_attn.in_proj_a.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.18.linear_attn.in_proj_b.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.18.linear_attn.in_proj_qkv.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.18.linear_attn.in_proj_z.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.18.linear_attn.norm.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.18.linear_attn.out_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.19.self_attn.k_norm.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.19.self_attn.q_norm.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.2.linear_attn.A_log": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.2.linear_attn.conv1d.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.2.linear_attn.dt_bias": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.2.linear_attn.in_proj_a.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.2.linear_attn.in_proj_b.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.2.linear_attn.in_proj_qkv.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.2.linear_attn.in_proj_z.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.2.linear_attn.norm.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.2.linear_attn.out_proj.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.20.linear_attn.A_log": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.20.linear_attn.conv1d.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.20.linear_attn.dt_bias": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.20.linear_attn.in_proj_a.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.20.linear_attn.in_proj_b.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.20.linear_attn.in_proj_qkv.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.20.linear_attn.in_proj_z.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.20.linear_attn.norm.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.20.linear_attn.out_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.21.linear_attn.A_log": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.21.linear_attn.conv1d.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.21.linear_attn.dt_bias": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.21.linear_attn.in_proj_a.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.21.linear_attn.in_proj_b.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.21.linear_attn.in_proj_qkv.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.21.linear_attn.in_proj_z.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.21.linear_attn.norm.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.21.linear_attn.out_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.22.input_layernorm.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.22.linear_attn.A_log": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.22.linear_attn.conv1d.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.22.linear_attn.dt_bias": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.22.linear_attn.in_proj_a.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.22.linear_attn.in_proj_b.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.22.linear_attn.in_proj_qkv.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.22.linear_attn.in_proj_z.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.22.linear_attn.norm.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.22.linear_attn.out_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.22.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.22.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.22.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.22.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.23.input_layernorm.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.23.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.23.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.23.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.23.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.23.self_attn.k_norm.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.23.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.23.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.23.self_attn.q_norm.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.23.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.23.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.24.input_layernorm.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.24.linear_attn.A_log": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.24.linear_attn.conv1d.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.24.linear_attn.dt_bias": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.24.linear_attn.in_proj_a.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.24.linear_attn.in_proj_b.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.24.linear_attn.in_proj_qkv.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.24.linear_attn.in_proj_z.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.24.linear_attn.norm.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.24.linear_attn.out_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.24.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.24.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.24.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.24.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.25.input_layernorm.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.25.linear_attn.A_log": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.25.linear_attn.conv1d.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.25.linear_attn.dt_bias": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.25.linear_attn.in_proj_a.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.25.linear_attn.in_proj_b.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.25.linear_attn.in_proj_qkv.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.25.linear_attn.in_proj_z.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.25.linear_attn.norm.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.25.linear_attn.out_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.25.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.25.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.25.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.25.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.26.input_layernorm.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.26.linear_attn.A_log": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.26.linear_attn.conv1d.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.26.linear_attn.dt_bias": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.26.linear_attn.in_proj_a.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.26.linear_attn.in_proj_b.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.26.linear_attn.in_proj_qkv.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.26.linear_attn.in_proj_z.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.26.linear_attn.norm.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.26.linear_attn.out_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.26.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.26.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.26.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.26.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.27.input_layernorm.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.27.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.27.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.27.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.27.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.27.self_attn.k_norm.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.27.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.27.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.27.self_attn.q_norm.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.27.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.27.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.28.input_layernorm.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.28.linear_attn.A_log": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.28.linear_attn.conv1d.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.28.linear_attn.dt_bias": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.28.linear_attn.in_proj_a.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.28.linear_attn.in_proj_b.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.28.linear_attn.in_proj_qkv.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.28.linear_attn.in_proj_z.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.28.linear_attn.norm.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.28.linear_attn.out_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.28.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.28.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.28.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.28.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.29.input_layernorm.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.29.linear_attn.A_log": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.29.linear_attn.conv1d.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.29.linear_attn.dt_bias": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.29.linear_attn.in_proj_a.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.29.linear_attn.in_proj_b.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.29.linear_attn.in_proj_qkv.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.29.linear_attn.in_proj_z.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.29.linear_attn.norm.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.29.linear_attn.out_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.29.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.29.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.29.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.29.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.3.self_attn.k_norm.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.3.self_attn.q_norm.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.30.input_layernorm.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.30.linear_attn.A_log": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.30.linear_attn.conv1d.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.30.linear_attn.dt_bias": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.30.linear_attn.in_proj_a.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.30.linear_attn.in_proj_b.weight": "model-00002-of-00003.safetensors", + "backbone.language_model.layers.30.linear_attn.in_proj_qkv.weight": "model-00003-of-00003.safetensors", + "backbone.language_model.layers.30.linear_attn.in_proj_z.weight": "model-00003-of-00003.safetensors", + "backbone.language_model.layers.30.linear_attn.norm.weight": "model-00003-of-00003.safetensors", + "backbone.language_model.layers.30.linear_attn.out_proj.weight": "model-00003-of-00003.safetensors", + "backbone.language_model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "backbone.language_model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", + "backbone.language_model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors", + "backbone.language_model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "backbone.language_model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors", + "backbone.language_model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "backbone.language_model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", + "backbone.language_model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors", + "backbone.language_model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "backbone.language_model.layers.31.self_attn.k_norm.weight": "model-00003-of-00003.safetensors", + "backbone.language_model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", + "backbone.language_model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", + "backbone.language_model.layers.31.self_attn.q_norm.weight": "model-00003-of-00003.safetensors", + "backbone.language_model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", + "backbone.language_model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", + "backbone.language_model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.4.linear_attn.A_log": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.4.linear_attn.conv1d.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.4.linear_attn.dt_bias": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.4.linear_attn.in_proj_a.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.4.linear_attn.in_proj_b.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.4.linear_attn.in_proj_qkv.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.4.linear_attn.in_proj_z.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.4.linear_attn.norm.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.4.linear_attn.out_proj.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.5.linear_attn.A_log": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.5.linear_attn.conv1d.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.5.linear_attn.dt_bias": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.5.linear_attn.in_proj_a.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.5.linear_attn.in_proj_b.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.5.linear_attn.in_proj_qkv.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.5.linear_attn.in_proj_z.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.5.linear_attn.norm.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.5.linear_attn.out_proj.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.6.linear_attn.A_log": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.6.linear_attn.conv1d.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.6.linear_attn.dt_bias": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.6.linear_attn.in_proj_a.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.6.linear_attn.in_proj_b.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.6.linear_attn.in_proj_qkv.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.6.linear_attn.in_proj_z.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.6.linear_attn.norm.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.6.linear_attn.out_proj.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.7.self_attn.k_norm.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.7.self_attn.q_norm.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.8.linear_attn.A_log": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.8.linear_attn.conv1d.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.8.linear_attn.dt_bias": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.8.linear_attn.in_proj_a.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.8.linear_attn.in_proj_b.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.8.linear_attn.in_proj_qkv.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.8.linear_attn.in_proj_z.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.8.linear_attn.norm.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.8.linear_attn.out_proj.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.9.linear_attn.A_log": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.9.linear_attn.conv1d.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.9.linear_attn.dt_bias": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.9.linear_attn.in_proj_a.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.9.linear_attn.in_proj_b.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.9.linear_attn.in_proj_qkv.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.9.linear_attn.in_proj_z.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.9.linear_attn.norm.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.9.linear_attn.out_proj.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "backbone.language_model.norm.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.0.attn.proj.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.0.attn.proj.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.0.attn.qkv.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.0.attn.qkv.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.0.mlp.linear_fc1.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.0.mlp.linear_fc1.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.0.mlp.linear_fc2.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.0.mlp.linear_fc2.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.0.norm1.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.0.norm1.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.0.norm2.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.0.norm2.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.1.attn.proj.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.1.attn.proj.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.1.attn.qkv.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.1.attn.qkv.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.1.mlp.linear_fc1.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.1.mlp.linear_fc1.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.1.mlp.linear_fc2.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.1.mlp.linear_fc2.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.1.norm1.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.1.norm1.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.1.norm2.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.1.norm2.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.10.attn.proj.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.10.attn.proj.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.10.attn.qkv.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.10.attn.qkv.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.10.mlp.linear_fc1.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.10.mlp.linear_fc1.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.10.mlp.linear_fc2.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.10.mlp.linear_fc2.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.10.norm1.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.10.norm1.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.10.norm2.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.10.norm2.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.11.attn.proj.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.11.attn.proj.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.11.attn.qkv.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.11.attn.qkv.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.11.mlp.linear_fc1.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.11.mlp.linear_fc1.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.11.mlp.linear_fc2.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.11.mlp.linear_fc2.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.11.norm1.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.11.norm1.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.11.norm2.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.11.norm2.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.12.attn.proj.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.12.attn.proj.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.12.attn.qkv.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.12.attn.qkv.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.12.mlp.linear_fc1.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.12.mlp.linear_fc1.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.12.mlp.linear_fc2.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.12.mlp.linear_fc2.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.12.norm1.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.12.norm1.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.12.norm2.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.12.norm2.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.13.attn.proj.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.13.attn.proj.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.13.attn.qkv.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.13.attn.qkv.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.13.mlp.linear_fc1.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.13.mlp.linear_fc1.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.13.mlp.linear_fc2.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.13.mlp.linear_fc2.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.13.norm1.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.13.norm1.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.13.norm2.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.13.norm2.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.14.attn.proj.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.14.attn.proj.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.14.attn.qkv.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.14.attn.qkv.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.14.mlp.linear_fc1.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.14.mlp.linear_fc1.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.14.mlp.linear_fc2.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.14.mlp.linear_fc2.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.14.norm1.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.14.norm1.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.14.norm2.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.14.norm2.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.15.attn.proj.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.15.attn.proj.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.15.attn.qkv.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.15.attn.qkv.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.15.mlp.linear_fc1.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.15.mlp.linear_fc1.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.15.mlp.linear_fc2.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.15.mlp.linear_fc2.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.15.norm1.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.15.norm1.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.15.norm2.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.15.norm2.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.16.attn.proj.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.16.attn.proj.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.16.attn.qkv.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.16.attn.qkv.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.16.mlp.linear_fc1.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.16.mlp.linear_fc1.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.16.mlp.linear_fc2.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.16.mlp.linear_fc2.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.16.norm1.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.16.norm1.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.16.norm2.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.16.norm2.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.17.attn.proj.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.17.attn.proj.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.17.attn.qkv.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.17.attn.qkv.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.17.mlp.linear_fc1.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.17.mlp.linear_fc1.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.17.mlp.linear_fc2.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.17.mlp.linear_fc2.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.17.norm1.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.17.norm1.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.17.norm2.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.17.norm2.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.18.attn.proj.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.18.attn.proj.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.18.attn.qkv.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.18.attn.qkv.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.18.mlp.linear_fc1.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.18.mlp.linear_fc1.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.18.mlp.linear_fc2.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.18.mlp.linear_fc2.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.18.norm1.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.18.norm1.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.18.norm2.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.18.norm2.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.19.attn.proj.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.19.attn.proj.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.19.attn.qkv.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.19.attn.qkv.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.19.mlp.linear_fc1.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.19.mlp.linear_fc1.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.19.mlp.linear_fc2.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.19.mlp.linear_fc2.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.19.norm1.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.19.norm1.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.19.norm2.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.19.norm2.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.2.attn.proj.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.2.attn.proj.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.2.attn.qkv.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.2.attn.qkv.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.2.mlp.linear_fc1.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.2.mlp.linear_fc1.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.2.mlp.linear_fc2.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.2.mlp.linear_fc2.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.2.norm1.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.2.norm1.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.2.norm2.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.2.norm2.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.20.attn.proj.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.20.attn.proj.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.20.attn.qkv.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.20.attn.qkv.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.20.mlp.linear_fc1.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.20.mlp.linear_fc1.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.20.mlp.linear_fc2.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.20.mlp.linear_fc2.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.20.norm1.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.20.norm1.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.20.norm2.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.20.norm2.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.21.attn.proj.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.21.attn.proj.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.21.attn.qkv.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.21.attn.qkv.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.21.mlp.linear_fc1.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.21.mlp.linear_fc1.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.21.mlp.linear_fc2.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.21.mlp.linear_fc2.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.21.norm1.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.21.norm1.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.21.norm2.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.21.norm2.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.22.attn.proj.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.22.attn.proj.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.22.attn.qkv.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.22.attn.qkv.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.22.mlp.linear_fc1.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.22.mlp.linear_fc1.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.22.mlp.linear_fc2.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.22.mlp.linear_fc2.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.22.norm1.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.22.norm1.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.22.norm2.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.22.norm2.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.23.attn.proj.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.23.attn.proj.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.23.attn.qkv.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.23.attn.qkv.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.23.mlp.linear_fc1.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.23.mlp.linear_fc1.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.23.mlp.linear_fc2.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.23.mlp.linear_fc2.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.23.norm1.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.23.norm1.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.23.norm2.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.23.norm2.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.3.attn.proj.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.3.attn.proj.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.3.attn.qkv.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.3.attn.qkv.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.3.mlp.linear_fc1.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.3.mlp.linear_fc1.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.3.mlp.linear_fc2.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.3.mlp.linear_fc2.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.3.norm1.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.3.norm1.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.3.norm2.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.3.norm2.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.4.attn.proj.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.4.attn.proj.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.4.attn.qkv.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.4.attn.qkv.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.4.mlp.linear_fc1.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.4.mlp.linear_fc1.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.4.mlp.linear_fc2.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.4.mlp.linear_fc2.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.4.norm1.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.4.norm1.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.4.norm2.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.4.norm2.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.5.attn.proj.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.5.attn.proj.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.5.attn.qkv.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.5.attn.qkv.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.5.mlp.linear_fc1.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.5.mlp.linear_fc1.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.5.mlp.linear_fc2.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.5.mlp.linear_fc2.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.5.norm1.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.5.norm1.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.5.norm2.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.5.norm2.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.6.attn.proj.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.6.attn.proj.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.6.attn.qkv.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.6.attn.qkv.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.6.mlp.linear_fc1.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.6.mlp.linear_fc1.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.6.mlp.linear_fc2.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.6.mlp.linear_fc2.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.6.norm1.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.6.norm1.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.6.norm2.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.6.norm2.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.7.attn.proj.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.7.attn.proj.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.7.attn.qkv.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.7.attn.qkv.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.7.mlp.linear_fc1.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.7.mlp.linear_fc1.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.7.mlp.linear_fc2.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.7.mlp.linear_fc2.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.7.norm1.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.7.norm1.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.7.norm2.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.7.norm2.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.8.attn.proj.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.8.attn.proj.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.8.attn.qkv.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.8.attn.qkv.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.8.mlp.linear_fc1.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.8.mlp.linear_fc1.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.8.mlp.linear_fc2.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.8.mlp.linear_fc2.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.8.norm1.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.8.norm1.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.8.norm2.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.8.norm2.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.9.attn.proj.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.9.attn.proj.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.9.attn.qkv.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.9.attn.qkv.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.9.mlp.linear_fc1.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.9.mlp.linear_fc1.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.9.mlp.linear_fc2.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.9.mlp.linear_fc2.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.9.norm1.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.9.norm1.weight": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.9.norm2.bias": "model-00003-of-00003.safetensors", + "backbone.visual.blocks.9.norm2.weight": "model-00003-of-00003.safetensors", + "backbone.visual.merger.linear_fc1.bias": "model-00003-of-00003.safetensors", + "backbone.visual.merger.linear_fc1.weight": "model-00003-of-00003.safetensors", + "backbone.visual.merger.linear_fc2.bias": "model-00003-of-00003.safetensors", + "backbone.visual.merger.linear_fc2.weight": "model-00003-of-00003.safetensors", + "backbone.visual.merger.norm.bias": "model-00003-of-00003.safetensors", + "backbone.visual.merger.norm.weight": "model-00003-of-00003.safetensors", + "backbone.visual.patch_embed.proj.bias": "model-00003-of-00003.safetensors", + "backbone.visual.patch_embed.proj.weight": "model-00003-of-00003.safetensors", + "backbone.visual.pos_embed.weight": "model-00003-of-00003.safetensors", + "head.blocks.0.channel_mlp.0.bias": "model-00003-of-00003.safetensors", + "head.blocks.0.channel_mlp.0.weight": "model-00003-of-00003.safetensors", + "head.blocks.0.channel_mlp.2.bias": "model-00003-of-00003.safetensors", + "head.blocks.0.channel_mlp.2.weight": "model-00003-of-00003.safetensors", + "head.blocks.0.channel_norm.bias": "model-00003-of-00003.safetensors", + "head.blocks.0.channel_norm.weight": "model-00003-of-00003.safetensors", + "head.blocks.0.token_mlp.0.bias": "model-00003-of-00003.safetensors", + "head.blocks.0.token_mlp.0.weight": "model-00003-of-00003.safetensors", + "head.blocks.0.token_mlp.2.bias": "model-00003-of-00003.safetensors", + "head.blocks.0.token_mlp.2.weight": "model-00003-of-00003.safetensors", + "head.blocks.0.token_norm.bias": "model-00003-of-00003.safetensors", + "head.blocks.0.token_norm.weight": "model-00003-of-00003.safetensors", + "head.blocks.1.channel_mlp.0.bias": "model-00003-of-00003.safetensors", + "head.blocks.1.channel_mlp.0.weight": "model-00003-of-00003.safetensors", + "head.blocks.1.channel_mlp.2.bias": "model-00003-of-00003.safetensors", + "head.blocks.1.channel_mlp.2.weight": "model-00003-of-00003.safetensors", + "head.blocks.1.channel_norm.bias": "model-00003-of-00003.safetensors", + "head.blocks.1.channel_norm.weight": "model-00003-of-00003.safetensors", + "head.blocks.1.token_mlp.0.bias": "model-00003-of-00003.safetensors", + "head.blocks.1.token_mlp.0.weight": "model-00003-of-00003.safetensors", + "head.blocks.1.token_mlp.2.bias": "model-00003-of-00003.safetensors", + "head.blocks.1.token_mlp.2.weight": "model-00003-of-00003.safetensors", + "head.blocks.1.token_norm.bias": "model-00003-of-00003.safetensors", + "head.blocks.1.token_norm.weight": "model-00003-of-00003.safetensors", + "head.input_norm.bias": "model-00003-of-00003.safetensors", + "head.input_norm.weight": "model-00003-of-00003.safetensors", + "head.output.0.bias": "model-00003-of-00003.safetensors", + "head.output.0.weight": "model-00003-of-00003.safetensors", + "head.output.1.bias": "model-00003-of-00003.safetensors", + "head.output.1.weight": "model-00003-of-00003.safetensors", + "head.projection.bias": "model-00003-of-00003.safetensors", + "head.projection.weight": "model-00003-of-00003.safetensors", + "head.roles": "model-00003-of-00003.safetensors" + } +} diff --git a/modeling_valen.py b/modeling_valen.py new file mode 100644 index 0000000000000000000000000000000000000000..3c8d3a4ae3d4dfde9e3ece865bc57bccd4d1d6cd --- /dev/null +++ b/modeling_valen.py @@ -0,0 +1,84 @@ +"""Complete merged Qwen + decision head. / 已合并的完整 Qwen 和决策头。""" +import torch +from transformers import AutoProcessor +from transformers.models.qwen3_5.modeling_qwen3_5 import Qwen3_5Model, Qwen3_5PreTrainedModel + +from .configuration_valen import ValenQwenConfig +from .heads import build_head +from .runtime_model import ValenQwen +from .compiler import Compiler +from .responses import answer +# List transitive files explicitly for Transformers' local dynamic loader. +# 本地动态加载需要显式列出间接依赖，否则搬移后可能缺少这些模块。 +from .batching import collate_branches +from .schema import candidates +from .data_types import QuestionSpec + + +class ValenQwenForDecisionMaking(Qwen3_5PreTrainedModel): + config_class = ValenQwenConfig + base_model_prefix = "backbone" + model_name = "Valen" + architecture = "qwen" + # The trained Mixer stays FP32 with a BF16 backbone. / BF16 基座、FP32 决策头。 + _keep_in_fp32_modules_strict = ["head"] + + def __init__(self, config, backbone=None, head=None): + super().__init__(config) + if (backbone is None) != (head is None): + raise ValueError("Supply both a merged backbone and a trained head") + self.backbone = backbone if backbone is not None else Qwen3_5Model(config) + self.head = head if head is not None else build_head(config.text_config.hidden_size, config.valen_head) + self.head.float() + self._processor = None + # Supplied weights are already trained: do not reinitialize them. + # 传入的参数已训练完成，不能再次调用初始化覆盖它们。 + if backbone is not None: + for module in self.modules(): + module._is_hf_initialized = True + self.post_init() + + def get_input_embeddings(self): + return self.backbone.get_input_embeddings() + + def set_input_embeddings(self, value): + self.backbone.set_input_embeddings(value) + + def forward(self, question=None, **inputs): + if question is None: + return self.backbone(**inputs) + if hasattr(question, "questions"): + return self.forward_state(question) + return ValenQwen.forward(self, question) + + # Use the exact trained readouts and batching implementation. + # 保留训练时的特征读取和 batch 逻辑。 + extract_features = ValenQwen.extract_features + extract_state_features = ValenQwen.extract_state_features + _state_features = ValenQwen._state_features + forward_state = ValenQwen.forward_state + forward_state_batch = ValenQwen.forward_state_batch + forward_batch = ValenQwen.forward_batch + score_features = ValenQwen.score_features + + def make_compiler(self, media_root=".", execution=None, processor=None): + if processor is None: + if self._processor is None: + self._processor = AutoProcessor.from_pretrained(self.config._name_or_path, trust_remote_code=True) + processor = self._processor + return Compiler(processor, media_root, self.config.valen_max_length, self.config.valen_media_kwargs, + execution=execution or self.config.qwen_execution) + + @torch.inference_mode() + def predict(self, request, *, media_root=".", execution=None, temperature=1.0, processor=None): + self.eval() + # Only request content reaches the compiler. / 只编译输入，不读取标签。 + request = request["request"] if "request" in request else request + compiler = self.make_compiler(media_root, execution, processor) + state = compiler.compile({"request": request}) + logits = self.forward_state(state) if state.inputs is not None else [self(q) for q in state.questions] + answers = {q.qid: answer(q, z, temperature[q.kind] if isinstance(temperature, dict) else temperature) + for q, z in zip(state.questions, logits)} + return {"model": self.model_name, "answers": answers, + "usage": {"input_tokens": state.logical_tokens, "output_tokens": 0}, + "internal_usage": {"compute_tokens": state.compute_tokens}} diff --git a/processor_config.json b/processor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..33818c7f9e991ad735fd240209f4fa73e6c28c50 --- /dev/null +++ b/processor_config.json @@ -0,0 +1,60 @@ +{ + "image_processor": { + "do_convert_rgb": true, + "do_normalize": true, + "do_rescale": true, + "do_resize": true, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_processor_type": "Qwen2VLImageProcessor", + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "merge_size": 2, + "patch_size": 16, + "resample": 3, + "rescale_factor": 0.00392156862745098, + "size": { + "longest_edge": 16777216, + "shortest_edge": 65536 + }, + "temporal_patch_size": 2 + }, + "processor_class": "Qwen3VLProcessor", + "video_processor": { + "do_convert_rgb": true, + "do_normalize": true, + "do_rescale": true, + "do_resize": true, + "do_sample_frames": true, + "fps": 2, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "max_frames": 768, + "merge_size": 2, + "min_frames": 4, + "patch_size": 16, + "resample": 3, + "rescale_factor": 0.00392156862745098, + "return_metadata": false, + "size": { + "longest_edge": 25165824, + "shortest_edge": 4096 + }, + "temporal_patch_size": 2, + "video_processor_type": "Qwen3VLVideoProcessor" + } +} diff --git a/responses.py b/responses.py new file mode 100644 index 0000000000000000000000000000000000000000..fb4398970a9fd3827c185269a2cfa97cc78d755f --- /dev/null +++ b/responses.py @@ -0,0 +1,29 @@ +import math +import torch + +def answer(question, logits, temperature=1.0): + if not math.isfinite(temperature) or temperature <= 0: + raise ValueError("Calibration temperature must be finite and positive") + if logits.ndim != 1 or len(logits) != len(question.keys) or not torch.isfinite(logits).all(): + raise ValueError("Expected one finite logit per candidate") + probs = (logits.float() / temperature).softmax(-1).detach().cpu().tolist() + total = sum(probs) + probs = [p / total for p in probs] + result = {"type": question.kind} + if question.kind == "noul": + result["noul"] = probs[question.keys.index("true")] + return result + result["probabilities"] = dict(zip(question.keys, probs)) + mode = max(range(len(probs)), key=probs.__getitem__) + if question.kind == "choice": + result["choice"] = question.keys[mode] + result["confidence"] = 1.0 if len(probs) == 1 else max(0.0, (max(probs) - 1/len(probs)) / (1 - 1/len(probs))) + else: + result["score"] = sum(i * p for i, p in enumerate(probs)) + result["legend"] = dict(zip(question.keys, question.descriptions)) + # Independently implemented formula from pinned system-one adapter: + # fb52b1030b7fc1f4f1cf39910afa5da54f9835e3, confidence_metrics.py. + distance = sum(p * abs(i - mode) for i, p in enumerate(probs)) + uniform_mad = sum(abs(i - (len(probs)-1)/2) for i in range(len(probs))) / len(probs) + result["confidence"] = max(0.0, 1.0 - distance / uniform_mad) + return result diff --git a/runtime_model.py b/runtime_model.py new file mode 100644 index 0000000000000000000000000000000000000000..38f601b00ea984e6e0f221dfcbd106aa8ed7d775 --- /dev/null +++ b/runtime_model.py @@ -0,0 +1,101 @@ +import torch +from torch import nn +MODEL_NAME = "Valen" +from .heads import DecisionHead, build_head +from .batching import branch_batches, collate_branches + + +class ValenQwen(nn.Module): + """Qwen3.5 backbone with a shared candidate decision head.""" + + model_name = MODEL_NAME + architecture = "qwen" + + def __init__(self, backbone, projection_dim=256, head_config=None): + super().__init__() + self.backbone = backbone + self.head = build_head(backbone.config.text_config.hidden_size, + dict({"projection_dim": projection_dim}, **(head_config or {}))) + + def forward(self, question): + device = next(self.backbone.parameters()).device + outputs = [] + for branch in question.branches: + inputs = {k: v.to(device) if isinstance(v, torch.Tensor) else v for k, v in branch.inputs.items()} + hidden = self.backbone(**inputs, use_cache=False, return_dict=True).last_hidden_state + outputs.append(self.head.score_features(self.head.extract_features(hidden, branch))) + return torch.cat(outputs) + + def extract_features(self, question): + """缓存投影前的特征。 / Cache raw readouts before trainable head layers.""" + device = next(self.backbone.parameters()).device + features = [] + for branch in question.branches: + inputs = {k: v.to(device) if isinstance(v, torch.Tensor) else v for k, v in branch.inputs.items()} + hidden = self.backbone(**inputs, use_cache=False, return_dict=True).last_hidden_state + features.append(self.head.extract_features(hidden, branch)) + return features + + def extract_state_features(self, state): + """一次编码 state，读取各题特征。 / One backbone graph for every question.""" + if not state.questions: + return [] + if state.inputs is None: + raise ValueError("Shared-state features require compiled shared inputs") + device = next(self.backbone.parameters()).device + inputs = {k: v.to(device) if isinstance(v, torch.Tensor) else v for k, v in state.inputs.items()} + hidden = self.backbone(**inputs, use_cache=False, return_dict=True).last_hidden_state + return self._state_features(hidden, state) + + def _state_features(self, hidden, state): + return [[self.head.extract_features(hidden, branch) for branch in q.branches] + for q in state.questions] + + def forward_state(self, state): + return [self.score_features(features) for features in self.extract_state_features(state)] + + def forward_state_batch(self, states, max_tokens=32768): + """不同 state 组成 batch，同一 state 的题目共用一行。 / One row per shared state.""" + device = next(self.backbone.parameters()).device + pad_token_id = getattr(self.backbone.config.text_config, "pad_token_id", None) or 0 + outputs = [[] for _ in states] + representatives = [] + indices = [] + for index, state in enumerate(states): + if not state.questions: + continue + if state.inputs is None or any(branch.inputs is not state.inputs + for q in state.questions for branch in q.branches): + raise ValueError("Shared-state batch requires one shared input per state") + representatives.append(state.questions[0]) + indices.append(index) + for batch in branch_batches(representatives, max_tokens): + inputs = collate_branches([branch for _, branch in batch], pad_token_id) + inputs = {key: value.to(device) for key, value in inputs.items()} + hidden = self.backbone(**inputs, use_cache=False, return_dict=True).last_hidden_state + for row, (index, _) in enumerate(batch): + original = indices[index] + features = self._state_features(hidden[row:row + 1], states[original]) + outputs[original] = [self.score_features(f) for f in features] + return outputs + + def forward_batch(self, questions, max_tokens=32768): + """Forward several QAs together; each keeps its own decision head inputs. + + 多条 QA 并行；每条保留独立的候选、角色区间和 Score 分支。 + """ + device = next(self.backbone.parameters()).device + pad_token_id = getattr(self.backbone.config.text_config, "pad_token_id", None) or 0 + outputs = [[] for _ in questions] + for batch in branch_batches(questions, max_tokens): + inputs = collate_branches([branch for _, branch in batch], pad_token_id) + inputs = {key: value.to(device) for key, value in inputs.items()} + hidden = self.backbone(**inputs, use_cache=False, return_dict=True).last_hidden_state + for row, (question_index, branch) in enumerate(batch): + features = self.head.extract_features(hidden[row:row + 1], branch) + outputs[question_index].append(self.head.score_features(features)) + return [torch.cat(branches) for branches in outputs] + + def score_features(self, features, head=None): + head = self.head if head is None else head + return torch.cat([head.score_features(hidden) for hidden in features]) diff --git a/schema.py b/schema.py new file mode 100644 index 0000000000000000000000000000000000000000..f3c3483045739bd5e38210e4a1ac32d13db44b07 --- /dev/null +++ b/schema.py @@ -0,0 +1,71 @@ +"""Strict JSONL contract. IDs and labels never enter the model prompt.""" +import json +import math +from pathlib import Path + + +def candidates(question): + kind = question.get("type") + if not isinstance(question.get("instructions"), str) or not question["instructions"].strip(): + raise ValueError("instructions must be nonempty text") + criteria = question.get("criteria") + if kind == "noul": + return [("true", "True / 是：满足问题中的条件。"), ("false", "False / 否：不满足问题中的条件。")] + if kind == "choice": + if not isinstance(criteria, dict) or not 1 <= len(criteria) <= 255: + raise ValueError("Choice needs 1–255 named criteria") + result = list(criteria.items()) + elif kind == "score": + if not isinstance(criteria, list) or not 2 <= len(criteria) <= 255: + raise ValueError("Score criteria must be an ordered list of 2–255 descriptions") + result = [(str(i), text) for i, text in enumerate(criteria)] + else: + raise ValueError(f"Unsupported question type: {kind}") + if any(not isinstance(k, str) or not k or not isinstance(v, str) or not v.strip() for k, v in result): + raise ValueError("Candidate names and descriptions must be nonempty strings") + return result + + +def target_distribution(target, keys): + if target is None or target.get("probabilities") is None: + return None + probs = target["probabilities"] + if not isinstance(probs, dict) or set(probs) != set(keys): + raise ValueError(f"Probability keys must exactly match {keys}") + values = [probs[k] for k in keys] + if any(isinstance(v, bool) or not isinstance(v, (int, float)) or not math.isfinite(v) or v < 0 for v in values): + raise ValueError("Probabilities must be finite nonnegative numbers") + if not math.isclose(sum(values), 1.0, abs_tol=1e-6): + raise ValueError("Target probabilities must sum to 1") + return values + + +def validate_record(record, candidate_fn=candidates): + request = record["request"] + state = request["state"] + if not isinstance(state, (str, dict)): + raise ValueError("state must be text or a messages object") + if not isinstance(request["questions"], dict) or not request["questions"]: + raise ValueError("questions must be a nonempty mapping") + targets = record.get("targets", {}) + if set(targets) - set(request["questions"]): + raise ValueError("Unknown target question IDs") + for qid, question in request["questions"].items(): + target_distribution(targets.get(qid), [k for k, _ in candidate_fn(question)]) + if not record.get("group_id"): + raise ValueError("group_id is required for leakage-free splits") + return record + + +def read_jsonl(path, candidate_fn=candidates): + records = [] + with Path(path).open(encoding="utf-8") as stream: + for line_number, line in enumerate(stream, 1): + if line.strip(): + try: + records.append(validate_record(json.loads(line), candidate_fn)) + except (ValueError, KeyError, TypeError, AttributeError) as exc: + raise ValueError(f"{path}:{line_number}: {exc}") from exc + if not records: + raise ValueError("Empty dataset") + return records diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..67741b04f23bfdb46501f748ce27865ec82eccfb --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:87a7830d63fcf43bf241c3c5242e96e62dd3fdc29224ca26fed8ea333db72de4 +size 19989343 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..a068e2468cff426a9b105006e74e044030a6faf4 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,32 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": true, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "processor_class": "Qwen3VLProcessor", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/unmerged/COMPLETE.json b/unmerged/COMPLETE.json new file mode 100644 index 0000000000000000000000000000000000000000..fa8551c9df5becfed54eaa655087f9a246bbd15b --- /dev/null +++ b/unmerged/COMPLETE.json @@ -0,0 +1,4 @@ +{ + "validated": true, + "source_checkpoint_sha256": "b4193b4e907f44b913fd54a8d6b998e7050a1528306269a000f1e5e8bd8dffc7" +} diff --git a/unmerged/LICENSE b/unmerged/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..f938136e3adacfd92be087f6e113b5d6d97f678f --- /dev/null +++ b/unmerged/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2026 Alibaba Cloud + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/unmerged/README.md b/unmerged/README.md new file mode 100644 index 0000000000000000000000000000000000000000..f24c93ea49804cfcded2f7e6874f7f8cba5217a0 --- /dev/null +++ b/unmerged/README.md @@ -0,0 +1,50 @@ +# Valen-4B — unmerged LoRA + +原始 Qwen3.5 基座 + 未合并的 LoRA + FP32 Mixer + 训练后的视觉 merger。 +Original base with separate LoRA, Mixer, and trained visual merger weights. +来源：1195k + JevBench 30k，两阶段 SFT，最终 step 325。 + +## 文件 / Files + +- `lora/adapter_model.safetensors`、`lora/adapter_config.json`：标准 PEFT LoRA。 +- `mixer.safetensors`：完整 FP32 Mixer 决策头。 +- `visual_merger.safetensors`：联合微调时训练的视觉 merger，加载时一并恢复。 +- `config.json`、processor、tokenizer 和 Python 文件：模型配置及独立加载代码。 + +## 完整模型加载 / Load the complete decision model + +环境沿用父目录：PyTorch 2.6.0、Transformers 5.4.0，另需 PEFT 0.18.1 和 safetensors。 +Flash Attention 2 可通过 `attn_implementation="flash_attention_2"` 启用。 + +```python +import torch +from transformers import AutoModel + +model = AutoModel.from_pretrained( + "./Valen-4B/unmerged", + base_model_path="./Qwen3.5-4B", # 原始官方基座 / original published base + trust_remote_code=True, + dtype=torch.bfloat16, + attn_implementation="sdpa", +).to("cuda").eval() +torch.set_float32_matmul_precision("highest") +torch.backends.cudnn.allow_tf32 = False + +result = model.predict({ + "state": "There is one cat in the room.", + "questions": { + "animal": {"type": "choice", "instructions": "Which animal is present?", + "criteria": {"cat": "A cat", "dog": "A dog"}}, + "is_cat": {"type": "noul", "instructions": "There is a cat in the room."}, + }, +}) +print(result["answers"]) +``` + +`base_model_path` 指向原始官方 `Qwen/Qwen3.5-4B` 权重，本目录中的加载器会核对基座 SHA-256。 +省略该参数时使用配置记录的仓库及固定 revision；离线环境需提前缓存该版本。 +默认 share_state，也支持 `model.predict(..., execution="question")`；视频采样 16 帧。 +内部使用 `PeftModel.from_pretrained(backbone.language_model, .../lora)` 加载语言 LoRA，随后恢复 Mixer 和视觉 merger。 +适配器不会被融合。`is_trainable=True` 可启用 LoRA、Mixer 和视觉 merger 的梯度。 + +`validation.json` 记录恢复后与原 checkpoint 的一致性；文件来源和校验和见 `export_manifest.json`。 diff --git a/unmerged/batching.py b/unmerged/batching.py new file mode 100644 index 0000000000000000000000000000000000000000..6bd343e1d7a8479c6bbc45d73b987b1d1d2d5c28 --- /dev/null +++ b/unmerged/batching.py @@ -0,0 +1,62 @@ +"""Pad language rows and concatenate visual patches. / 文本补齐，视觉 patch 拼接。""" +import torch +from torch.nn.utils.rnn import pad_sequence + + +SEQUENCE_KEYS = {"input_ids", "attention_mask", "mm_token_type_ids"} +VISUAL_KEYS = {"pixel_values", "pixel_values_videos", "image_grid_thw", "video_grid_thw"} + + +def collate_branches(branches, pad_token_id=0): + if not branches: + raise ValueError("Cannot collate an empty branch batch") + inputs = [branch.inputs for branch in branches] + unknown = set().union(*(row.keys() for row in inputs)) - SEQUENCE_KEYS - VISUAL_KEYS + if unknown: + raise ValueError(f"Unsupported batched Qwen inputs: {sorted(unknown)}") + if any(row["input_ids"].ndim != 2 or row["input_ids"].shape[0] != 1 for row in inputs): + raise ValueError("Each compiled branch must contain one language row") + result = {} + for key in SEQUENCE_KEYS: + if key != "input_ids" and not any(key in row for row in inputs): + continue + rows = [] + for row in inputs: + value = row.get(key) + if value is None: + value = (torch.ones_like(row["input_ids"]) if key == "attention_mask" + else torch.zeros_like(row["input_ids"])) + if value.shape != row["input_ids"].shape: + raise ValueError(f"{key} must match the branch input_ids shape") + rows.append(value[0]) + # Right padding keeps all compiler readout positions and spans unchanged. + # 右侧补齐，候选位置和角色区间无需偏移。 + result[key] = pad_sequence(rows, batch_first=True, + padding_value=pad_token_id if key == "input_ids" else 0) + if "attention_mask" not in result: + result["attention_mask"] = pad_sequence([torch.ones_like(r["input_ids"][0]) for r in inputs], + batch_first=True, padding_value=0) + for key in VISUAL_KEYS: + values = [row[key] for row in inputs if key in row] + if values: + result[key] = torch.cat(values, dim=0) + return result + + +def branch_batches(questions, max_tokens): + """Limit padded tokens per backbone forward, keeping Score branch order.""" + if type(max_tokens) is not int or max_tokens <= 0: + raise ValueError("microbatch_max_tokens must be a positive integer") + batch, longest = [], 0 + for question_index, question in enumerate(questions): + if not question.branches: + raise ValueError("A Qwen question requires at least one branch") + for branch in question.branches: + length = branch.inputs["input_ids"].shape[1] + if batch and max(longest, length) * (len(batch) + 1) > max_tokens: + yield batch + batch, longest = [], 0 + batch.append((question_index, branch)) + longest = max(longest, length) + if batch: + yield batch diff --git a/unmerged/chat_template.jinja b/unmerged/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..a585dec894e63da457d9440ec6aa7caa16d20860 --- /dev/null +++ b/unmerged/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/unmerged/compiler.py b/unmerged/compiler.py new file mode 100644 index 0000000000000000000000000000000000000000..a5b9fe08e2cc9cd8564d6325211bf10ebf789089 --- /dev/null +++ b/unmerged/compiler.py @@ -0,0 +1,239 @@ +"""Compile Qwen question branches or a shared sequence with per-question readouts.""" +from copy import deepcopy +from dataclasses import dataclass +from pathlib import Path +import hashlib +import torch +from .schema import candidates, target_distribution +from .data_types import QuestionSpec + + +@dataclass +class Branch: + inputs: dict + candidate_positions: list + decision_position: int + context_span: tuple | None = None + instruction_span: tuple | None = None + candidate_spans: list | None = None + + +@dataclass(init=False) +class Question(QuestionSpec): + branches: list + + def __init__(self, qid, kind, keys, descriptions, branches, target, is_text): + # Keep the positional constructor used by existing compiler clients. + super().__init__(qid, kind, keys, descriptions, target, is_text) + self.branches = branches + + +@dataclass +class CompiledState: + questions: list + logical_tokens: int + compute_tokens: int + media: list + inputs: dict | None = None + + +class Compiler: + def __init__(self, processor, media_root=".", max_length=8192, media_kwargs=None, verified_media_hashes=None, + execution="question"): + if execution not in {"question", "shared_state"}: + raise ValueError(f"Unknown qwen_execution: {execution}") + self.execution = execution + self.processor = processor + self.tokenizer = processor.tokenizer + self.media_root = Path(media_root) + self.max_length = max_length + self.media_kwargs = media_kwargs or {} + # Optional, pre-audited immutable assets for latency measurements. + # Normal training/inference continues to hash each input at use time. + self.verified_media_hashes = verified_media_hashes or {} + + def _messages(self, state): + messages = [{"role": "user", "content": state}] if isinstance(state, str) else deepcopy(state["messages"]) + media = [] + for message in messages: + if message["role"] not in {"user", "assistant", "system"}: + raise ValueError("Unsupported state role") + if isinstance(message["content"], str): + message["content"] = [{"type": "text", "text": message["content"]}] + for item in message["content"]: + kind = item["type"] + if kind == "text": + # Prevent user text from injecting Qwen control tokens. + if any(t in item["text"] for t in self.tokenizer.all_special_tokens): + raise ValueError("Reserved tokenizer control token in state") + continue + if kind not in {"image_url", "video_url"}: + raise ValueError(f"Unsupported content type: {kind}") + url = item[kind]["url"] + if "://" in url: + raise ValueError("Materialize remote media locally with a hash before training") + path = Path(url) + path = path if path.is_absolute() else self.media_root / path + path = path.resolve(strict=True) + digest = self.verified_media_hashes.get(str(path)) + if digest is None: + digest = hashlib.sha256(path.read_bytes()).hexdigest() + media.append({"path": str(path), "sha256": digest, "type": kind}) + item.clear() + item.update(type=kind.removesuffix("_url"), path=str(path)) + return messages, media + + def compile(self, record, rng=None, labeled_only=False): + selected = [] + for qid, question in record["request"]["questions"].items(): + pairs = candidates(question) + target = record.get("targets", {}).get(qid) + if labeled_only and self.execution == "question" and target_distribution(target, [k for k, _ in pairs]) is None: + continue + if rng is not None and question["type"] == "choice": + rng.shuffle(pairs) + selected.append((qid, question, pairs, target)) + # 标签只决定监督范围，共享序列仍保留所有题。 / Labels select losses, never prompt content. + if not selected or (labeled_only and not any( + target_distribution(target, [k for k, _ in pairs]) is not None + for _, _, pairs, target in selected)): + return CompiledState([], 0, 0, []) + messages, media = self._messages(record["request"]["state"]) + expected_assets = {str((self.media_root / a["path"]).resolve()): a["sha256"] for a in record.get("assets", [])} + for entry in media: + if entry["path"] in expected_assets and expected_assets[entry["path"]] != entry["sha256"]: + raise ValueError(f"Media hash mismatch: {entry['path']}") + base = dict(self.processor.apply_chat_template( + messages, tokenize=True, add_generation_prompt=False, return_dict=True, + return_tensors="pt", processor_kwargs=dict(self.media_kwargs, return_metadata=True))) + # Metadata is audit information, never a backbone keyword argument. + metadata = base.pop("video_metadata", None) + if metadata is not None: + for entry, meta in zip([m for m in media if m["type"] == "video_url"], metadata): + entry["sampling"] = dict(meta) if isinstance(meta, dict) else vars(meta) + sampling = entry["sampling"] + indices = sampling.get("frames_indices") + if indices is not None and sampling.get("fps"): + sampling["frames_indices"] = [int(i) for i in indices] + sampling["timestamps_seconds"] = [int(i) / sampling["fps"] for i in indices] + sampling["visible_range_seconds"] = [min(sampling["timestamps_seconds"]), max(sampling["timestamps_seconds"])] + for name in ("image_grid_thw", "video_grid_thw"): + if name in base: + grid = base[name] + media.append({name: grid.tolist(), "visual_tokens": int(grid.prod(-1).sum()) // 4}) + base_length = base["input_ids"].shape[1] + if self.execution == "shared_state": + return self._compile_shared(selected, base, media, labeled_only) + logical_tokens, compute_tokens = base_length, 0 + compiled = [] + for qid, question, pairs, target in selected: + keys = [k for k, _ in pairs] + branches = [] + groups = [[pair] for pair in pairs] if question["type"] == "score" else [pairs] + for group in groups: + suffix = [] + def append(text, char_span=None): + start = base_length + len(suffix) + if char_span is None: + suffix.extend(self.tokenizer.encode(text, add_special_tokens=False)) + return None + # 保持原有分词，只额外记录区间。 / Preserve token IDs; record role spans. + encoded = self.tokenizer(text, add_special_tokens=False, return_offsets_mapping=True) + suffix.extend(encoded["input_ids"]) + indices = [i for i, (a, b) in enumerate(encoded["offset_mapping"]) + if a < char_span[1] and b > char_span[0]] + if not indices: + raise ValueError("Empty token span for decision role") + return (start + indices[0], start + indices[-1] + 1) + # Canonical segment tokenization makes endpoints exact; no string searching. + suffix.extend(self.tokenizer.encode("<|im_start|>user\n", add_special_tokens=False)) + for value in [question["instructions"]] + [v for pair in group for v in pair]: + if any(t in value for t in self.tokenizer.all_special_tokens): + raise ValueError("Reserved tokenizer control token in question/criteria") + prefix = "Task: " + question["type"] + "\nQuestion: " + instruction_span = append(prefix + question["instructions"] + "\nCandidates:\n", + (len(prefix), len(prefix) + len(question["instructions"]))) + positions, candidate_spans = [], [] + for key, description in group: + # Score never sees its level ID, nor any other level description. + prefix = key + ": " if question["type"] != "score" else "" + candidate_spans.append(append(prefix + description, (len(prefix), len(prefix) + len(description)))) + positions.append(base_length + len(suffix) - 1) + append("\n") + append("Decision:") + length = base_length + len(suffix) + if length > self.max_length: + raise ValueError(f"{qid}: {length} tokens exceed max_length={self.max_length}; no truncation") + inputs = dict(base) + inputs["input_ids"] = torch.cat([base["input_ids"], torch.tensor([suffix], dtype=torch.long)], dim=1) + inputs["attention_mask"] = torch.ones_like(inputs["input_ids"]) + if "mm_token_type_ids" in base: + inputs["mm_token_type_ids"] = torch.cat([base["mm_token_type_ids"], torch.zeros((1, len(suffix)), dtype=torch.long)], dim=1) + branches.append(Branch(inputs, positions, length - 1, (0, base_length), + instruction_span, candidate_spans)) + compute_tokens += length + logical_tokens += len(suffix) + compiled.append(Question(qid, question["type"], keys, [v for _, v in pairs], branches, + target_distribution(target, keys), not media)) + return CompiledState(compiled, logical_tokens, compute_tokens, media) + + def _compile_shared(self, selected, base, media, labeled_only=False): + """All schemas precede all readouts; labels and external IDs never enter inputs. + + This is ordinary causal attention, not isolated question/Score branches. + No answer tokens or LM output layer are required by the decision heads. + """ + base_length = base["input_ids"].shape[1] + suffix, compiled = [], [] + + def append(text, span=None): + start = base_length + len(suffix) + if span is None: + suffix.extend(self.tokenizer.encode(text, add_special_tokens=False)) + return None + encoded = self.tokenizer(text, add_special_tokens=False, return_offsets_mapping=True) + suffix.extend(encoded["input_ids"]) + indices = [i for i, (a, b) in enumerate(encoded["offset_mapping"]) if a < span[1] and b > span[0]] + if not indices: + raise ValueError("Empty token span for decision role") + return start + indices[0], start + indices[-1] + 1 + + append("<|im_start|>user\n") + for index, (qid, question, pairs, target) in enumerate(selected): + for value in [question["instructions"]] + [v for pair in pairs for v in pair]: + if any(t in value for t in self.tokenizer.all_special_tokens): + raise ValueError("Reserved tokenizer control token in question/criteria") + prefix = f"Question {index + 1}\nTask: {question['type']}\nQuestion: " + instruction = append(prefix + question["instructions"] + "\nCandidates:\n", + (len(prefix), len(prefix) + len(question["instructions"]))) + positions, spans = [], [] + for key, description in pairs: + prefix = key + ": " if question["type"] != "score" else "" + spans.append(append(prefix + description, (len(prefix), len(prefix) + len(description)))) + positions.append(base_length + len(suffix) - 1) + append("\n") + # Inputs and the decision position are filled after all schemas are appended. + branch = Branch({}, positions, -1, (0, base_length), instruction, spans) + keys = [key for key, _ in pairs] + compiled.append(Question(qid, question["type"], keys, [v for _, v in pairs], [branch], + target_distribution(target, keys), not media)) + append("<|im_end|>\n<|im_start|>assistant\n") + for index, question in enumerate(compiled): + append(f"Question {index + 1} Decision:") + question.branches[0].decision_position = base_length + len(suffix) - 1 + append("\n") + length = base_length + len(suffix) + if length > self.max_length: + raise ValueError(f"Shared state: {length} tokens exceed max_length={self.max_length}; no truncation") + inputs = dict(base) + inputs["input_ids"] = torch.cat([base["input_ids"], base["input_ids"].new_tensor([suffix])], dim=1) + inputs["attention_mask"] = torch.cat([base["attention_mask"], + base["attention_mask"].new_ones((1, len(suffix)))], dim=1) + if "mm_token_type_ids" in base: + inputs["mm_token_type_ids"] = torch.cat([base["mm_token_type_ids"], + base["mm_token_type_ids"].new_zeros((1, len(suffix)))], dim=1) + for question in compiled: + question.branches[0].inputs = inputs + if labeled_only: + compiled = [question for question in compiled if question.target is not None] + return CompiledState(compiled, length, length, media, inputs) diff --git a/unmerged/config.json b/unmerged/config.json new file mode 100644 index 0000000000000000000000000000000000000000..7c80462e381738b1fdef603716657e6bbb6943da --- /dev/null +++ b/unmerged/config.json @@ -0,0 +1,163 @@ +{ + "architectures": [ + "ValenQwenWithLoRA" + ], + "auto_map": { + "AutoConfig": "configuration_valen.ValenQwenConfig", + "AutoModel": "modeling_valen_lora.ValenQwenWithLoRA" + }, + "dtype": "bfloat16", + "image_token_id": 248056, + "model_type": "valen_qwen", + "qwen_execution": "shared_state", + "text_config": { + "attention_bias": false, + "attention_dropout": 0.0, + "attn_output_gate": true, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 248044, + "full_attention_interval": 4, + "head_dim": 256, + "hidden_act": "silu", + "hidden_size": 2560, + "initializer_range": 0.02, + "intermediate_size": 9216, + "layer_types": [ + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention" + ], + "linear_conv_kernel_dim": 4, + "linear_key_head_dim": 128, + "linear_num_key_heads": 16, + "linear_num_value_heads": 32, + "linear_value_head_dim": 128, + "mamba_ssm_dtype": "float32", + "max_position_embeddings": 262144, + "mlp_only_layers": [], + "model_type": "qwen3_5_text", + "mtp_num_hidden_layers": 1, + "mtp_use_dedicated_embeddings": false, + "num_attention_heads": 16, + "num_hidden_layers": 32, + "num_key_value_heads": 4, + "pad_token_id": null, + "partial_rotary_factor": 0.25, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "mrope_interleaved": true, + "mrope_section": [ + 11, + 11, + 10 + ], + "partial_rotary_factor": 0.25, + "rope_theta": 10000000, + "rope_type": "default" + }, + "tie_word_embeddings": true, + "use_cache": true, + "vocab_size": 248320 + }, + "tie_word_embeddings": true, + "transformers_version": "5.4.0", + "valen_head": { + "head_channel_hidden_dim": 1024, + "head_layers": 2, + "head_token_hidden_dim": 16, + "head_type": "mixer", + "head_width": 256 + }, + "valen_max_length": 32768, + "valen_media_kwargs": { + "images_kwargs": { + "size": { + "longest_edge": 262144, + "shortest_edge": 65536 + } + }, + "videos_kwargs": { + "fps": null, + "num_frames": 16, + "size": { + "longest_edge": 2097152, + "shortest_edge": 131072 + } + } + }, + "video_token_id": 248057, + "vision_config": { + "deepstack_visual_indexes": [], + "depth": 24, + "dtype": "bfloat16", + "hidden_act": "gelu_pytorch_tanh", + "hidden_size": 1024, + "in_channels": 3, + "initializer_range": 0.02, + "intermediate_size": 4096, + "model_type": "qwen3_5", + "num_heads": 16, + "num_position_embeddings": 2304, + "out_hidden_size": 2560, + "patch_size": 16, + "spatial_merge_size": 2, + "temporal_patch_size": 2 + }, + "vision_end_token_id": 248054, + "vision_start_token_id": 248053, + "valen_base": { + "repo": "Qwen/Qwen3.5-4B", + "revision": "851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a", + "weight_sha256": { + "model.safetensors-00001-of-00002.safetensors": "26a93f066e1916adb13453dae5a0c707c0fbc71299ed98779571a907b8e74c61", + "model.safetensors-00002-of-00002.safetensors": "cb544bd9bfae93dc59b0f22b292f5933573854a7f9b97835c67060d7d910e188" + }, + "files_sha256": { + ".gitattributes": "34448b82c17d60fec9b65b1f093c115ddbaadc04beb1b0140b6bfed2e012a930", + "LICENSE": "bbedc3fda3305820b977265f01b8619d87570a6739de3a5582c3464840f1e57a", + "README.md": "1406be1b6b8fd8a6545870da516912804756593628a1d0fb0a7965211e82a7bb", + "chat_template.jinja": "a4aee8afcf2e0711942cf848899be66016f8d14a889ff9ede07bca099c28f715", + "config.json": "ddc63e1c717afa86c865bb5e01313d89d72bb53b97ad4a8a03ba8510c0621670", + "merges.txt": "a9d356d7bdf1ef4949e3e748e95b8e10ad9d4e2e838eddc38a0a7b6b94d1db8d", + "model.safetensors-00001-of-00002.safetensors": "26a93f066e1916adb13453dae5a0c707c0fbc71299ed98779571a907b8e74c61", + "model.safetensors-00002-of-00002.safetensors": "cb544bd9bfae93dc59b0f22b292f5933573854a7f9b97835c67060d7d910e188", + "model.safetensors.index.json": "cf3f798ee02ba45f9622aa8892a47369ab667d0afbf154ee7c2212de42e6302d", + "preprocessor_config.json": "27225450ac9c6529872ee1924fcb0962ff5634834f817040f444118116f4e516", + "tokenizer.json": "5f9e4d4901a92b997e463c1f46055088b6cca5ca61a6522d1b9f64c4bb81cb42", + "tokenizer_config.json": "316230d6a809701f4db5ea8f8fc862bc3a6f3229c937c174e674ff3ca0a64ac8", + "video_preprocessor_config.json": "7768af27c1fafa9cc9011c1dc20067e03f8915e03b63504550e11d5066986d13", + "vocab.json": "ce99b4cb2983d118806ce0a8b777a35b093e2000a503ebde25853284c9dfa003" + }, + "verified_from": "download_manifest.json; every LFS SHA256 matched" + } +} diff --git a/unmerged/configuration_valen.py b/unmerged/configuration_valen.py new file mode 100644 index 0000000000000000000000000000000000000000..a9252326abac70b4afc90518b61999f6ff07284c --- /dev/null +++ b/unmerged/configuration_valen.py @@ -0,0 +1,16 @@ +"""Standalone Qwen decision configuration. / 独立的 Qwen 决策模型配置。""" +from transformers import Qwen3_5Config + + +class ValenQwenConfig(Qwen3_5Config): + model_type = "valen_qwen" + + def __init__(self, valen_head=None, qwen_execution="shared_state", valen_max_length=32768, + valen_media_kwargs=None, **kwargs): + if qwen_execution not in {"question", "shared_state"}: + raise ValueError("qwen_execution must be question or shared_state") + self.valen_head = valen_head or {"head_type": "mixer"} + self.qwen_execution = qwen_execution + self.valen_max_length = valen_max_length + self.valen_media_kwargs = valen_media_kwargs or {} + super().__init__(**kwargs) diff --git a/unmerged/data_types.py b/unmerged/data_types.py new file mode 100644 index 0000000000000000000000000000000000000000..ce1d6a8d6ee0f46600b13865ea941dcd7d803de0 --- /dev/null +++ b/unmerged/data_types.py @@ -0,0 +1,12 @@ +"""Model-independent decision metadata; encoded tensors stay with each compiler.""" +from dataclasses import dataclass + + +@dataclass +class QuestionSpec: + qid: str + kind: str + keys: list + descriptions: list + target: list | None + is_text: bool diff --git a/unmerged/export_manifest.json b/unmerged/export_manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..bbcca2d926f902cf47572cafe07f47e55def59c3 --- /dev/null +++ b/unmerged/export_manifest.json @@ -0,0 +1,68 @@ +{ + "format": "valen_qwen_unmerged_v1", + "created_utc": "2026-10-06T06:53:45.813755+00:00", + "model": "Valen-4B", + "source_checkpoint_sha256": "b4193b4e907f44b913fd54a8d6b998e7050a1528306269a000f1e5e8bd8dffc7", + "training_step": 325, + "training_records": 1225000, + "training_data_sha256": "648cb127fe4830f9faaa6d4595445cc2573ef0c152f8f74c9c776dc8a9223d7f", + "base": { + "repo": "Qwen/Qwen3.5-4B", + "revision": "851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a", + "weight_sha256": { + "model.safetensors-00001-of-00002.safetensors": "26a93f066e1916adb13453dae5a0c707c0fbc71299ed98779571a907b8e74c61", + "model.safetensors-00002-of-00002.safetensors": "cb544bd9bfae93dc59b0f22b292f5933573854a7f9b97835c67060d7d910e188" + }, + "files_sha256": { + ".gitattributes": "34448b82c17d60fec9b65b1f093c115ddbaadc04beb1b0140b6bfed2e012a930", + "LICENSE": "bbedc3fda3305820b977265f01b8619d87570a6739de3a5582c3464840f1e57a", + "README.md": "1406be1b6b8fd8a6545870da516912804756593628a1d0fb0a7965211e82a7bb", + "chat_template.jinja": "a4aee8afcf2e0711942cf848899be66016f8d14a889ff9ede07bca099c28f715", + "config.json": "ddc63e1c717afa86c865bb5e01313d89d72bb53b97ad4a8a03ba8510c0621670", + "merges.txt": "a9d356d7bdf1ef4949e3e748e95b8e10ad9d4e2e838eddc38a0a7b6b94d1db8d", + "model.safetensors-00001-of-00002.safetensors": "26a93f066e1916adb13453dae5a0c707c0fbc71299ed98779571a907b8e74c61", + "model.safetensors-00002-of-00002.safetensors": "cb544bd9bfae93dc59b0f22b292f5933573854a7f9b97835c67060d7d910e188", + "model.safetensors.index.json": "cf3f798ee02ba45f9622aa8892a47369ab667d0afbf154ee7c2212de42e6302d", + "preprocessor_config.json": "27225450ac9c6529872ee1924fcb0962ff5634834f817040f444118116f4e516", + "tokenizer.json": "5f9e4d4901a92b997e463c1f46055088b6cca5ca61a6522d1b9f64c4bb81cb42", + "tokenizer_config.json": "316230d6a809701f4db5ea8f8fc862bc3a6f3229c937c174e674ff3ca0a64ac8", + "video_preprocessor_config.json": "7768af27c1fafa9cc9011c1dc20067e03f8915e03b63504550e11d5066986d13", + "vocab.json": "ce99b4cb2983d118806ce0a8b777a35b093e2000a503ebde25853284c9dfa003" + }, + "verified_from": "download_manifest.json; every LFS SHA256 matched" + }, + "execution": "shared_state", + "trainable_tensor_count": 535, + "versions": { + "torch": "2.6.0", + "transformers": "5.4.0", + "peft": "0.18.1", + "safetensors": "0.6.2" + }, + "files_sha256": { + "LICENSE": "bbedc3fda3305820b977265f01b8619d87570a6739de3a5582c3464840f1e57a", + "README.md": "19dade0ed4951da63d3f556939f52d16e81d6a77fea4372816754824c52a3983", + "batching.py": "ea45d4724bcafca559fc5a2a18092918b633746771c19da6554c29bc5f2cef64", + "chat_template.jinja": "a4aee8afcf2e0711942cf848899be66016f8d14a889ff9ede07bca099c28f715", + "compiler.py": "f35bab2e71b6beb1228daaa5b7baecdacf04eaf713aee14e8082b97090f910b4", + "config.json": "63d39d4399bf06747f5f06d6b68db7b17fa3e96e8bc219fb9823742a700a06c1", + "configuration_valen.py": "735b02402fb5b6c7a1652f6691ba66d066ef99e84500042faec396ff7a668fe5", + "data_types.py": "b06ce08081d8f7b2bf36abf302410a96086395e75a04f9e9b1c591db519b1641", + "heads.py": "63b5970a8bfa121eff94d50ffd35e1fa8387f3af5abccc5f5b75381fb084fc34", + "lora/README.md": "761f9a02d12a6434a0444d774b01bd6e13cd98c392c154f2442665b8e632819b", + "lora/adapter_config.json": "83e2bc08b862c2cec7326c9f1ef2f44580e422c5f4ba2eea1148c8cd12bc979a", + "lora/adapter_model.safetensors": "b162b622b6d92720a87c678931a88ff05fd5e4b93b356997153c124bfac87708", + "mixer.safetensors": "64d368877bac6f4d9e8662d07ed6f21d3e4e58bec4e7a5dc6b4ac555fb4028d9", + "modeling_valen.py": "07f87481334cb1e28e9daee036c27bab3654645abf3df7983346b316e7788848", + "modeling_valen_lora.py": "0102d4e71596dc7e1f6f3c95e6cf5fb514c50bff9f27c4870f16ad618874d4be", + "processor_config.json": "d89ef49ce9cd37fbf510158e13c1ef063d9286411c1ec9049932dbe0487143b1", + "responses.py": "298016b0070f1000a38d033e1048e038ff5df6fc1dcbe830601105b181f1d228", + "runtime_model.py": "98bf0443ca196900b9f2b056129c84932041b8b53395d111f2fe6d3f2bc1e0a6", + "schema.py": "253825862b60016e37fd1040492008ad7f303679c092313d8a6527cf509f3b6e", + "tokenizer.json": "87a7830d63fcf43bf241c3c5242e96e62dd3fdc29224ca26fed8ea333db72de4", + "tokenizer_config.json": "e98f1901ac6f0adff67b1d540bfa0c36ac1a0cf59eb72ed78146ef89aafa1182", + "validation.json": "6e66db946145cba9c89033b82036e048845ac0f4e81920dcc9a8992a7f88edd2", + "visual_merger.safetensors": "8379c8434e846ff5007de471cd6174092ac0d30699007fbea7780882106774c1", + "fresh_process_validation.json": "d5f7b9a17d0f1011d02177a0dc0af7530912c2a5f191f06ee98a3a94cbd39b0c" + } +} diff --git a/unmerged/fresh_process_validation.json b/unmerged/fresh_process_validation.json new file mode 100644 index 0000000000000000000000000000000000000000..47567ef157db8e34ce3e94b81280eae2acc2f09f --- /dev/null +++ b/unmerged/fresh_process_validation.json @@ -0,0 +1,113 @@ +{ + "passed": true, + "isolated_python": true, + "valen_imports_blocked": true, + "lora_remains_unmerged": true, + "all_real_modalities_and_question_types_passed": true, + "sdpa_example_passed": true, + "cases": 20, + "max_probability_abs_error": 0.0, + "max_logit_abs_error": 0.0, + "details": [ + { + "case": "shared_state/text_multi/choice", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0 + }, + { + "case": "shared_state/text_multi/noul", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0 + }, + { + "case": "shared_state/text_multi/score", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0 + }, + { + "case": "shared_state/text_multi/score_many", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0 + }, + { + "case": "shared_state/image_score/decision", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0 + }, + { + "case": "shared_state/image_noul/decision", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0 + }, + { + "case": "shared_state/image_choice/decision", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0 + }, + { + "case": "shared_state/video_multi/q000", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0 + }, + { + "case": "shared_state/video_multi/score_000", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0 + }, + { + "case": "shared_state/video_multi/noul_000", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0 + }, + { + "case": "question/text_multi/choice", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0 + }, + { + "case": "question/text_multi/noul", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0 + }, + { + "case": "question/text_multi/score", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0 + }, + { + "case": "question/text_multi/score_many", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0 + }, + { + "case": "question/image_score/decision", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0 + }, + { + "case": "question/image_noul/decision", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0 + }, + { + "case": "question/image_choice/decision", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0 + }, + { + "case": "question/video_multi/q000", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0 + }, + { + "case": "question/video_multi/score_000", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0 + }, + { + "case": "question/video_multi/noul_000", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0 + } + ] +} diff --git a/unmerged/heads.py b/unmerged/heads.py new file mode 100644 index 0000000000000000000000000000000000000000..47627b2dc79bfafeea627683628f949f80ab7692 --- /dev/null +++ b/unmerged/heads.py @@ -0,0 +1,143 @@ +"""可替换的候选评分头。 / Interchangeable, candidate-wise decision heads.""" +import math + +import torch +from torch import nn + + +def head_signature(config): + """用于构建与恢复校验。 / Canonical architecture settings for loading.""" + kind = config.get("head_type", "bilinear") + defaults = { + "bilinear": {"projection_dim": 256}, + "mlp": {"head_hidden_dim": 512, "head_bottleneck_dim": 128}, + "role_mlp": {"head_width": 256, "head_hidden_dim": 1024}, + "mixer": {"head_width": 256, "head_layers": 2, + "head_token_hidden_dim": 16, "head_channel_hidden_dim": 1024}, + } + if kind not in defaults: + raise ValueError(f"Unknown Qwen head_type: {kind!r}") + values = {key: config.get(key, value) for key, value in defaults[kind].items()} + if any(type(value) is not int or value <= 0 for value in values.values()): + raise ValueError("Decision head dimensions must be positive integers") + return {"head_type": kind, **values} + + +class DecisionHead(nn.Module): + """原有参数名保持兼容。 / Preserve legacy bilinear parameter names.""" + def __init__(self, hidden_size, projection_dim=256): + super().__init__() + self.decision = nn.Linear(hidden_size, projection_dim, bias=False) + self.candidate = nn.Linear(hidden_size, projection_dim, bias=False) + self.scale = math.sqrt(projection_dim) + + def forward(self, hidden, positions, decision_position): + d = self.decision(hidden[0, decision_position].float()) + c = self.candidate(hidden[0, positions].float()) + return (c * d).sum(-1) / self.scale + + def extract_features(self, hidden, branch): + return hidden[:, [*branch.candidate_positions, branch.decision_position], :] + + def score_features(self, features): + return self(features, list(range(features.shape[1] - 1)), features.shape[1] - 1) + + +class MLPHead(nn.Module): + """每个候选共享一个 MLP。 / One shared MLP for arbitrary candidate counts.""" + def __init__(self, hidden_size, hidden_dim=512, bottleneck_dim=128): + super().__init__() + self.decision_norm = nn.LayerNorm(hidden_size) + self.candidate_norm = nn.LayerNorm(hidden_size) + self.network = nn.Sequential(nn.Linear(2 * hidden_size, hidden_dim), nn.GELU(), + nn.Linear(hidden_dim, bottleneck_dim), nn.GELU(), + nn.Linear(bottleneck_dim, 1)) + + def extract_features(self, hidden, branch): + c = hidden[0, branch.candidate_positions] + d = hidden[0, branch.decision_position].expand_as(c) + return torch.stack((d, c), dim=1) + + def score_features(self, features): + d, c = features.float().unbind(1) + return self.network(torch.cat((self.decision_norm(d), self.candidate_norm(c)), -1)).squeeze(-1) + + +def role_features(hidden, branch): + """先池化再投影，可缓存冻结特征。 / Pool before trainable projections.""" + spans = (getattr(branch, "context_span", None), getattr(branch, "instruction_span", None)) + candidates = getattr(branch, "candidate_spans", None) + if any(span is None for span in spans) or not candidates or len(candidates) != len(branch.candidate_positions): + raise ValueError("Role heads require compiler context, instruction and candidate spans") + + def pool(span): + start, end = span + if not 0 <= start < end <= hidden.shape[1]: + raise ValueError(f"Invalid role span: {span}") + return hidden[0, start:end].float().mean(0) + + context, instruction = (pool(span) for span in spans) + decision = hidden[0, branch.decision_position].float() + return torch.stack([torch.stack((context, instruction, pool(span), decision)) for span in candidates]) + + +class RoleHead(nn.Module): + def __init__(self, hidden_size, width): + super().__init__() + self.input_norm = nn.LayerNorm(hidden_size) + self.projection = nn.Linear(hidden_size, width) + self.roles = nn.Parameter(torch.zeros(1, 4, width)) + + def extract_features(self, hidden, branch): + return role_features(hidden, branch) + + def project(self, features): + return self.projection(self.input_norm(features.float())) + self.roles + + +class RoleMLPHead(RoleHead): + """与 Mixer 使用相同输入。 / Control for the Mixer's richer input features.""" + def __init__(self, hidden_size, width=256, hidden_dim=1024): + super().__init__(hidden_size, width) + self.network = nn.Sequential(nn.Linear(4 * width, hidden_dim), nn.GELU(), nn.Linear(hidden_dim, 1)) + + def score_features(self, features): + return self.network(self.project(features).flatten(1)).squeeze(-1) + + +class MixerBlock(nn.Module): + def __init__(self, width, token_hidden, channel_hidden): + super().__init__() + self.token_norm = nn.LayerNorm(width) + self.token_mlp = nn.Sequential(nn.Linear(4, token_hidden), nn.GELU(), nn.Linear(token_hidden, 4)) + self.channel_norm = nn.LayerNorm(width) + self.channel_mlp = nn.Sequential(nn.Linear(width, channel_hidden), nn.GELU(), nn.Linear(channel_hidden, width)) + + def forward(self, x): + # 混合角色，不混合不同候选。 / Mix fixed roles, never candidate rows. + x = x + self.token_mlp(self.token_norm(x).transpose(1, 2)).transpose(1, 2) + return x + self.channel_mlp(self.channel_norm(x)) + + +class MixerHead(RoleHead): + def __init__(self, hidden_size, width=256, layers=2, token_hidden=16, channel_hidden=1024): + super().__init__(hidden_size, width) + self.blocks = nn.Sequential(*(MixerBlock(width, token_hidden, channel_hidden) for _ in range(layers))) + self.output = nn.Sequential(nn.LayerNorm(width), nn.Linear(width, 1)) + + def score_features(self, features): + # 第三行是候选角色。 / Read the updated candidate role. + return self.output(self.blocks(self.project(features))[:, 2]).squeeze(-1) + + +def build_head(hidden_size, config): + settings = head_signature(config) + kind = settings.pop("head_type") + if kind == "bilinear": + return DecisionHead(hidden_size, settings["projection_dim"]) + if kind == "mlp": + return MLPHead(hidden_size, settings["head_hidden_dim"], settings["head_bottleneck_dim"]) + if kind == "role_mlp": + return RoleMLPHead(hidden_size, settings["head_width"], settings["head_hidden_dim"]) + return MixerHead(hidden_size, settings["head_width"], settings["head_layers"], + settings["head_token_hidden_dim"], settings["head_channel_hidden_dim"]) diff --git a/unmerged/lora/README.md b/unmerged/lora/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7daea40193544e018122f7a2bb1a91aae8fd7e96 --- /dev/null +++ b/unmerged/lora/README.md @@ -0,0 +1,206 @@ +--- +base_model: '' +library_name: peft +tags: +- 'base_model:adapter:' +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/unmerged/lora/adapter_config.json b/unmerged/lora/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..198b360aff5fd62c12ff3209d2b88a60fd6d5348 --- /dev/null +++ b/unmerged/lora/adapter_config.json @@ -0,0 +1,54 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": { + "base_model_class": "Qwen3_5TextModel", + "parent_library": "transformers.models.qwen3_5.modeling_qwen3_5" + }, + "base_model_name_or_path": "Qwen/Qwen3.5-4B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.0, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": "851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a", + "target_modules": [ + "in_proj_qkv", + "out_proj", + "in_proj_z", + "gate_proj", + "o_proj", + "up_proj", + "k_proj", + "in_proj_a", + "in_proj_b", + "v_proj", + "q_proj", + "down_proj" + ], + "target_parameters": null, + "task_type": null, + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/unmerged/lora/adapter_model.safetensors b/unmerged/lora/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c5a50b982b26fa7b8209374d1c27a2f327ec232e --- /dev/null +++ b/unmerged/lora/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b162b622b6d92720a87c678931a88ff05fd5e4b93b356997153c124bfac87708 +size 259784032 diff --git a/unmerged/mixer.safetensors b/unmerged/mixer.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..429c851027a4276ee55abdbc73fb9d3870a35c9a --- /dev/null +++ b/unmerged/mixer.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:64d368877bac6f4d9e8662d07ed6f21d3e4e58bec4e7a5dc6b4ac555fb4028d9 +size 6867012 diff --git a/unmerged/modeling_valen.py b/unmerged/modeling_valen.py new file mode 100644 index 0000000000000000000000000000000000000000..3c8d3a4ae3d4dfde9e3ece865bc57bccd4d1d6cd --- /dev/null +++ b/unmerged/modeling_valen.py @@ -0,0 +1,84 @@ +"""Complete merged Qwen + decision head. / 已合并的完整 Qwen 和决策头。""" +import torch +from transformers import AutoProcessor +from transformers.models.qwen3_5.modeling_qwen3_5 import Qwen3_5Model, Qwen3_5PreTrainedModel + +from .configuration_valen import ValenQwenConfig +from .heads import build_head +from .runtime_model import ValenQwen +from .compiler import Compiler +from .responses import answer +# List transitive files explicitly for Transformers' local dynamic loader. +# 本地动态加载需要显式列出间接依赖，否则搬移后可能缺少这些模块。 +from .batching import collate_branches +from .schema import candidates +from .data_types import QuestionSpec + + +class ValenQwenForDecisionMaking(Qwen3_5PreTrainedModel): + config_class = ValenQwenConfig + base_model_prefix = "backbone" + model_name = "Valen" + architecture = "qwen" + # The trained Mixer stays FP32 with a BF16 backbone. / BF16 基座、FP32 决策头。 + _keep_in_fp32_modules_strict = ["head"] + + def __init__(self, config, backbone=None, head=None): + super().__init__(config) + if (backbone is None) != (head is None): + raise ValueError("Supply both a merged backbone and a trained head") + self.backbone = backbone if backbone is not None else Qwen3_5Model(config) + self.head = head if head is not None else build_head(config.text_config.hidden_size, config.valen_head) + self.head.float() + self._processor = None + # Supplied weights are already trained: do not reinitialize them. + # 传入的参数已训练完成，不能再次调用初始化覆盖它们。 + if backbone is not None: + for module in self.modules(): + module._is_hf_initialized = True + self.post_init() + + def get_input_embeddings(self): + return self.backbone.get_input_embeddings() + + def set_input_embeddings(self, value): + self.backbone.set_input_embeddings(value) + + def forward(self, question=None, **inputs): + if question is None: + return self.backbone(**inputs) + if hasattr(question, "questions"): + return self.forward_state(question) + return ValenQwen.forward(self, question) + + # Use the exact trained readouts and batching implementation. + # 保留训练时的特征读取和 batch 逻辑。 + extract_features = ValenQwen.extract_features + extract_state_features = ValenQwen.extract_state_features + _state_features = ValenQwen._state_features + forward_state = ValenQwen.forward_state + forward_state_batch = ValenQwen.forward_state_batch + forward_batch = ValenQwen.forward_batch + score_features = ValenQwen.score_features + + def make_compiler(self, media_root=".", execution=None, processor=None): + if processor is None: + if self._processor is None: + self._processor = AutoProcessor.from_pretrained(self.config._name_or_path, trust_remote_code=True) + processor = self._processor + return Compiler(processor, media_root, self.config.valen_max_length, self.config.valen_media_kwargs, + execution=execution or self.config.qwen_execution) + + @torch.inference_mode() + def predict(self, request, *, media_root=".", execution=None, temperature=1.0, processor=None): + self.eval() + # Only request content reaches the compiler. / 只编译输入，不读取标签。 + request = request["request"] if "request" in request else request + compiler = self.make_compiler(media_root, execution, processor) + state = compiler.compile({"request": request}) + logits = self.forward_state(state) if state.inputs is not None else [self(q) for q in state.questions] + answers = {q.qid: answer(q, z, temperature[q.kind] if isinstance(temperature, dict) else temperature) + for q, z in zip(state.questions, logits)} + return {"model": self.model_name, "answers": answers, + "usage": {"input_tokens": state.logical_tokens, "output_tokens": 0}, + "internal_usage": {"compute_tokens": state.compute_tokens}} diff --git a/unmerged/modeling_valen_lora.py b/unmerged/modeling_valen_lora.py new file mode 100644 index 0000000000000000000000000000000000000000..c4d6031baef21b588dbb3c293151bb77cebc5a5b --- /dev/null +++ b/unmerged/modeling_valen_lora.py @@ -0,0 +1,92 @@ +"""Original base + separate trained deltas. / 原始基座与未融合的训练权重。""" +import hashlib +from pathlib import Path + +import torch +from peft import PeftModel +from safetensors.torch import load_file +from transformers import Qwen3_5ForConditionalGeneration + +from .configuration_valen import ValenQwenConfig +from .modeling_valen import ValenQwenForDecisionMaking +# Explicit dependencies keep local HF dynamic loading portable. +# 显式列出依赖，保证 HF 本地动态加载能复制完整的运行代码。 +from .heads import build_head +from .compiler import Compiler +from .runtime_model import ValenQwen +from .responses import answer +from .batching import collate_branches +from .schema import candidates +from .data_types import QuestionSpec + + +def _verify_local_base(base_path, expected): + if not Path(base_path).is_dir(): + return + for filename, digest in expected.items(): + path = Path(base_path) / filename + if not path.is_file(): + raise FileNotFoundError(f"Original base shard is required: {path}") + actual = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(8 << 20), b""): + actual.update(chunk) + if actual.hexdigest() != digest: + raise ValueError(f"Base weights differ from the trained revision: {filename}") + + +class ValenQwenWithLoRA(ValenQwenForDecisionMaking): + @classmethod + def from_pretrained(cls, pretrained_model_name_or_path, *model_args, config=None, + base_model_path=None, device=None, dtype=None, attn_implementation=None, + is_trainable=False, local_files_only=True, output_loading_info=False, + verify_base=True, **kwargs): + if model_args: + raise TypeError("Pass loader options as named arguments") + folder = Path(pretrained_model_name_or_path).resolve(strict=True) + if config is None: + config = ValenQwenConfig.from_pretrained(folder, local_files_only=True) + base = str(base_model_path or config.valen_base["repo"]) + dtype = dtype or kwargs.pop("torch_dtype", None) or config.dtype or torch.bfloat16 + if isinstance(dtype, str): + dtype = getattr(torch, dtype) + if dtype != torch.bfloat16: + raise ValueError("Use dtype=torch.bfloat16 to reproduce this adapter bundle") + attention = attn_implementation or config._attn_implementation or "sdpa" + if kwargs.pop("device_map", None) is not None: + raise ValueError("Use device=... or model.to(...) for this adapter bundle") + # AutoModel also supplies flags for loading the bundle's custom code. + # AutoModel 会传入自定义代码标记，它们不应改变原始基座的加载配置。 + for name in ("trust_remote_code", "_from_auto", "_commit_hash", "adapter_kwargs", "revision", "subfolder"): + kwargs.pop(name, None) + base_options = {name:kwargs.pop(name) for name in + ("cache_dir", "token", "force_download", "proxies", "low_cpu_mem_usage") if name in kwargs} + if kwargs: + raise TypeError(f"Unsupported loader options: {sorted(kwargs)}") + if verify_base: + _verify_local_base(base, config.valen_base["weight_sha256"]) + # Load the published conditional model first to preserve language prefixes. + # 先完整加载官方模型，再取出主干，避免语言参数前缀被错误映射。 + container, loading = Qwen3_5ForConditionalGeneration.from_pretrained( + base, revision=config.valen_base["revision"], dtype=dtype, + attn_implementation=attention, local_files_only=local_files_only, + output_loading_info=True, **base_options) + if any(loading.get(key) for key in ("missing_keys", "unexpected_keys", "mismatched_keys", "error_msgs")): + raise ValueError(f"Incomplete original base load: {loading}") + backbone = container.model + del container + backbone.requires_grad_(False) + backbone.language_model = PeftModel.from_pretrained( + backbone.language_model, folder / "lora", is_trainable=is_trainable, + autocast_adapter_dtype=True, torch_device="cpu", local_files_only=True) + backbone.visual.merger.load_state_dict(load_file(folder / "visual_merger.safetensors"), strict=True) + head = build_head(config.text_config.hidden_size, config.valen_head).float() + head.load_state_dict(load_file(folder / "mixer.safetensors"), strict=True) + config._name_or_path = str(folder) + model = cls(config, backbone=backbone, head=head) + model.head.requires_grad_(is_trainable) + model.backbone.visual.merger.requires_grad_(is_trainable) + if device is not None: + model.to(device) + model.eval() + return (model, loading) if output_loading_info else model diff --git a/unmerged/processor_config.json b/unmerged/processor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..33818c7f9e991ad735fd240209f4fa73e6c28c50 --- /dev/null +++ b/unmerged/processor_config.json @@ -0,0 +1,60 @@ +{ + "image_processor": { + "do_convert_rgb": true, + "do_normalize": true, + "do_rescale": true, + "do_resize": true, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_processor_type": "Qwen2VLImageProcessor", + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "merge_size": 2, + "patch_size": 16, + "resample": 3, + "rescale_factor": 0.00392156862745098, + "size": { + "longest_edge": 16777216, + "shortest_edge": 65536 + }, + "temporal_patch_size": 2 + }, + "processor_class": "Qwen3VLProcessor", + "video_processor": { + "do_convert_rgb": true, + "do_normalize": true, + "do_rescale": true, + "do_resize": true, + "do_sample_frames": true, + "fps": 2, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "max_frames": 768, + "merge_size": 2, + "min_frames": 4, + "patch_size": 16, + "resample": 3, + "rescale_factor": 0.00392156862745098, + "return_metadata": false, + "size": { + "longest_edge": 25165824, + "shortest_edge": 4096 + }, + "temporal_patch_size": 2, + "video_processor_type": "Qwen3VLVideoProcessor" + } +} diff --git a/unmerged/responses.py b/unmerged/responses.py new file mode 100644 index 0000000000000000000000000000000000000000..fb4398970a9fd3827c185269a2cfa97cc78d755f --- /dev/null +++ b/unmerged/responses.py @@ -0,0 +1,29 @@ +import math +import torch + +def answer(question, logits, temperature=1.0): + if not math.isfinite(temperature) or temperature <= 0: + raise ValueError("Calibration temperature must be finite and positive") + if logits.ndim != 1 or len(logits) != len(question.keys) or not torch.isfinite(logits).all(): + raise ValueError("Expected one finite logit per candidate") + probs = (logits.float() / temperature).softmax(-1).detach().cpu().tolist() + total = sum(probs) + probs = [p / total for p in probs] + result = {"type": question.kind} + if question.kind == "noul": + result["noul"] = probs[question.keys.index("true")] + return result + result["probabilities"] = dict(zip(question.keys, probs)) + mode = max(range(len(probs)), key=probs.__getitem__) + if question.kind == "choice": + result["choice"] = question.keys[mode] + result["confidence"] = 1.0 if len(probs) == 1 else max(0.0, (max(probs) - 1/len(probs)) / (1 - 1/len(probs))) + else: + result["score"] = sum(i * p for i, p in enumerate(probs)) + result["legend"] = dict(zip(question.keys, question.descriptions)) + # Independently implemented formula from pinned system-one adapter: + # fb52b1030b7fc1f4f1cf39910afa5da54f9835e3, confidence_metrics.py. + distance = sum(p * abs(i - mode) for i, p in enumerate(probs)) + uniform_mad = sum(abs(i - (len(probs)-1)/2) for i in range(len(probs))) / len(probs) + result["confidence"] = max(0.0, 1.0 - distance / uniform_mad) + return result diff --git a/unmerged/runtime_model.py b/unmerged/runtime_model.py new file mode 100644 index 0000000000000000000000000000000000000000..38f601b00ea984e6e0f221dfcbd106aa8ed7d775 --- /dev/null +++ b/unmerged/runtime_model.py @@ -0,0 +1,101 @@ +import torch +from torch import nn +MODEL_NAME = "Valen" +from .heads import DecisionHead, build_head +from .batching import branch_batches, collate_branches + + +class ValenQwen(nn.Module): + """Qwen3.5 backbone with a shared candidate decision head.""" + + model_name = MODEL_NAME + architecture = "qwen" + + def __init__(self, backbone, projection_dim=256, head_config=None): + super().__init__() + self.backbone = backbone + self.head = build_head(backbone.config.text_config.hidden_size, + dict({"projection_dim": projection_dim}, **(head_config or {}))) + + def forward(self, question): + device = next(self.backbone.parameters()).device + outputs = [] + for branch in question.branches: + inputs = {k: v.to(device) if isinstance(v, torch.Tensor) else v for k, v in branch.inputs.items()} + hidden = self.backbone(**inputs, use_cache=False, return_dict=True).last_hidden_state + outputs.append(self.head.score_features(self.head.extract_features(hidden, branch))) + return torch.cat(outputs) + + def extract_features(self, question): + """缓存投影前的特征。 / Cache raw readouts before trainable head layers.""" + device = next(self.backbone.parameters()).device + features = [] + for branch in question.branches: + inputs = {k: v.to(device) if isinstance(v, torch.Tensor) else v for k, v in branch.inputs.items()} + hidden = self.backbone(**inputs, use_cache=False, return_dict=True).last_hidden_state + features.append(self.head.extract_features(hidden, branch)) + return features + + def extract_state_features(self, state): + """一次编码 state，读取各题特征。 / One backbone graph for every question.""" + if not state.questions: + return [] + if state.inputs is None: + raise ValueError("Shared-state features require compiled shared inputs") + device = next(self.backbone.parameters()).device + inputs = {k: v.to(device) if isinstance(v, torch.Tensor) else v for k, v in state.inputs.items()} + hidden = self.backbone(**inputs, use_cache=False, return_dict=True).last_hidden_state + return self._state_features(hidden, state) + + def _state_features(self, hidden, state): + return [[self.head.extract_features(hidden, branch) for branch in q.branches] + for q in state.questions] + + def forward_state(self, state): + return [self.score_features(features) for features in self.extract_state_features(state)] + + def forward_state_batch(self, states, max_tokens=32768): + """不同 state 组成 batch，同一 state 的题目共用一行。 / One row per shared state.""" + device = next(self.backbone.parameters()).device + pad_token_id = getattr(self.backbone.config.text_config, "pad_token_id", None) or 0 + outputs = [[] for _ in states] + representatives = [] + indices = [] + for index, state in enumerate(states): + if not state.questions: + continue + if state.inputs is None or any(branch.inputs is not state.inputs + for q in state.questions for branch in q.branches): + raise ValueError("Shared-state batch requires one shared input per state") + representatives.append(state.questions[0]) + indices.append(index) + for batch in branch_batches(representatives, max_tokens): + inputs = collate_branches([branch for _, branch in batch], pad_token_id) + inputs = {key: value.to(device) for key, value in inputs.items()} + hidden = self.backbone(**inputs, use_cache=False, return_dict=True).last_hidden_state + for row, (index, _) in enumerate(batch): + original = indices[index] + features = self._state_features(hidden[row:row + 1], states[original]) + outputs[original] = [self.score_features(f) for f in features] + return outputs + + def forward_batch(self, questions, max_tokens=32768): + """Forward several QAs together; each keeps its own decision head inputs. + + 多条 QA 并行；每条保留独立的候选、角色区间和 Score 分支。 + """ + device = next(self.backbone.parameters()).device + pad_token_id = getattr(self.backbone.config.text_config, "pad_token_id", None) or 0 + outputs = [[] for _ in questions] + for batch in branch_batches(questions, max_tokens): + inputs = collate_branches([branch for _, branch in batch], pad_token_id) + inputs = {key: value.to(device) for key, value in inputs.items()} + hidden = self.backbone(**inputs, use_cache=False, return_dict=True).last_hidden_state + for row, (question_index, branch) in enumerate(batch): + features = self.head.extract_features(hidden[row:row + 1], branch) + outputs[question_index].append(self.head.score_features(features)) + return [torch.cat(branches) for branches in outputs] + + def score_features(self, features, head=None): + head = self.head if head is None else head + return torch.cat([head.score_features(hidden) for hidden in features]) diff --git a/unmerged/schema.py b/unmerged/schema.py new file mode 100644 index 0000000000000000000000000000000000000000..f3c3483045739bd5e38210e4a1ac32d13db44b07 --- /dev/null +++ b/unmerged/schema.py @@ -0,0 +1,71 @@ +"""Strict JSONL contract. IDs and labels never enter the model prompt.""" +import json +import math +from pathlib import Path + + +def candidates(question): + kind = question.get("type") + if not isinstance(question.get("instructions"), str) or not question["instructions"].strip(): + raise ValueError("instructions must be nonempty text") + criteria = question.get("criteria") + if kind == "noul": + return [("true", "True / 是：满足问题中的条件。"), ("false", "False / 否：不满足问题中的条件。")] + if kind == "choice": + if not isinstance(criteria, dict) or not 1 <= len(criteria) <= 255: + raise ValueError("Choice needs 1–255 named criteria") + result = list(criteria.items()) + elif kind == "score": + if not isinstance(criteria, list) or not 2 <= len(criteria) <= 255: + raise ValueError("Score criteria must be an ordered list of 2–255 descriptions") + result = [(str(i), text) for i, text in enumerate(criteria)] + else: + raise ValueError(f"Unsupported question type: {kind}") + if any(not isinstance(k, str) or not k or not isinstance(v, str) or not v.strip() for k, v in result): + raise ValueError("Candidate names and descriptions must be nonempty strings") + return result + + +def target_distribution(target, keys): + if target is None or target.get("probabilities") is None: + return None + probs = target["probabilities"] + if not isinstance(probs, dict) or set(probs) != set(keys): + raise ValueError(f"Probability keys must exactly match {keys}") + values = [probs[k] for k in keys] + if any(isinstance(v, bool) or not isinstance(v, (int, float)) or not math.isfinite(v) or v < 0 for v in values): + raise ValueError("Probabilities must be finite nonnegative numbers") + if not math.isclose(sum(values), 1.0, abs_tol=1e-6): + raise ValueError("Target probabilities must sum to 1") + return values + + +def validate_record(record, candidate_fn=candidates): + request = record["request"] + state = request["state"] + if not isinstance(state, (str, dict)): + raise ValueError("state must be text or a messages object") + if not isinstance(request["questions"], dict) or not request["questions"]: + raise ValueError("questions must be a nonempty mapping") + targets = record.get("targets", {}) + if set(targets) - set(request["questions"]): + raise ValueError("Unknown target question IDs") + for qid, question in request["questions"].items(): + target_distribution(targets.get(qid), [k for k, _ in candidate_fn(question)]) + if not record.get("group_id"): + raise ValueError("group_id is required for leakage-free splits") + return record + + +def read_jsonl(path, candidate_fn=candidates): + records = [] + with Path(path).open(encoding="utf-8") as stream: + for line_number, line in enumerate(stream, 1): + if line.strip(): + try: + records.append(validate_record(json.loads(line), candidate_fn)) + except (ValueError, KeyError, TypeError, AttributeError) as exc: + raise ValueError(f"{path}:{line_number}: {exc}") from exc + if not records: + raise ValueError("Empty dataset") + return records diff --git a/unmerged/tokenizer.json b/unmerged/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..67741b04f23bfdb46501f748ce27865ec82eccfb --- /dev/null +++ b/unmerged/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:87a7830d63fcf43bf241c3c5242e96e62dd3fdc29224ca26fed8ea333db72de4 +size 19989343 diff --git a/unmerged/tokenizer_config.json b/unmerged/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..a068e2468cff426a9b105006e74e044030a6faf4 --- /dev/null +++ b/unmerged/tokenizer_config.json @@ -0,0 +1,32 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": true, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "processor_class": "Qwen3VLProcessor", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/unmerged/validation.json b/unmerged/validation.json new file mode 100644 index 0000000000000000000000000000000000000000..45c05162d4d9ca42258185d82bedf5a9b8341acf --- /dev/null +++ b/unmerged/validation.json @@ -0,0 +1,136 @@ +{ + "passed": true, + "all_training_deltas_bitwise_reloaded": true, + "all_backbone_buffers_bitwise_preserved": true, + "trainable_tensor_count": 535, + "reload_vs_checkpoint": { + "cases": 20, + "changed_labels": [], + "max_probability_abs_error": 0.0, + "max_logit_abs_error": 0.0, + "details": [ + { + "case": "shared_state/text_multi/choice", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0, + "label_changed": false + }, + { + "case": "shared_state/text_multi/noul", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0, + "label_changed": false + }, + { + "case": "shared_state/text_multi/score", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0, + "label_changed": false + }, + { + "case": "shared_state/text_multi/score_many", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0, + "label_changed": false + }, + { + "case": "shared_state/image_score/decision", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0, + "label_changed": false + }, + { + "case": "shared_state/image_noul/decision", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0, + "label_changed": false + }, + { + "case": "shared_state/image_choice/decision", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0, + "label_changed": false + }, + { + "case": "shared_state/video_multi/q000", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0, + "label_changed": false + }, + { + "case": "shared_state/video_multi/score_000", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0, + "label_changed": false + }, + { + "case": "shared_state/video_multi/noul_000", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0, + "label_changed": false + }, + { + "case": "question/text_multi/choice", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0, + "label_changed": false + }, + { + "case": "question/text_multi/noul", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0, + "label_changed": false + }, + { + "case": "question/text_multi/score", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0, + "label_changed": false + }, + { + "case": "question/text_multi/score_many", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0, + "label_changed": false + }, + { + "case": "question/image_score/decision", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0, + "label_changed": false + }, + { + "case": "question/image_noul/decision", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0, + "label_changed": false + }, + { + "case": "question/image_choice/decision", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0, + "label_changed": false + }, + { + "case": "question/video_multi/q000", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0, + "label_changed": false + }, + { + "case": "question/video_multi/score_000", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0, + "label_changed": false + }, + { + "case": "question/video_multi/noul_000", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0, + "label_changed": false + } + ] + }, + "public_predict_passed": true, + "scope": "Text/image/video; question/shared_state; Choice/Noul/Score, including 12-level Score." +} diff --git a/unmerged/visual_merger.safetensors b/unmerged/visual_merger.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ae93468ea3a068c697e8dbec43a75b8f4b395b96 --- /dev/null +++ b/unmerged/visual_merger.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8379c8434e846ff5007de471cd6174092ac0d30699007fbea7780882106774c1 +size 54543872 diff --git a/validation.json b/validation.json new file mode 100644 index 0000000000000000000000000000000000000000..d0507f520b16ebca237706f9ed5276b619544043 --- /dev/null +++ b/validation.json @@ -0,0 +1,399 @@ +{ + "passed": true, + "merge_vs_adapter": { + "cases": 20, + "changed_labels": [], + "max_probability_abs_error": 0.0196954607963562, + "max_logit_abs_error": 0.2783782482147217, + "details": [ + { + "case": "shared_state/text_multi/choice", + "probability_max_abs": 7.516145706176758e-05, + "logit_max_abs": 0.1412639617919922, + "label_changed": false + }, + { + "case": "shared_state/text_multi/noul", + "probability_max_abs": 0.0020025968551635742, + "logit_max_abs": 0.08715403079986572, + "label_changed": false + }, + { + "case": "shared_state/text_multi/score", + "probability_max_abs": 0.00015294551849365234, + "logit_max_abs": 0.22236299514770508, + "label_changed": false + }, + { + "case": "shared_state/text_multi/score_many", + "probability_max_abs": 0.0007519125938415527, + "logit_max_abs": 0.21488428115844727, + "label_changed": false + }, + { + "case": "shared_state/image_score/decision", + "probability_max_abs": 0.0049536824226379395, + "logit_max_abs": 0.08501756191253662, + "label_changed": false + }, + { + "case": "shared_state/image_noul/decision", + "probability_max_abs": 0.007346063852310181, + "logit_max_abs": 0.053545475006103516, + "label_changed": false + }, + { + "case": "shared_state/image_choice/decision", + "probability_max_abs": 5.960464477539062e-07, + "logit_max_abs": 0.07563447952270508, + "label_changed": false + }, + { + "case": "shared_state/video_multi/q000", + "probability_max_abs": 0.0007462501525878906, + "logit_max_abs": 0.052941322326660156, + "label_changed": false + }, + { + "case": "shared_state/video_multi/score_000", + "probability_max_abs": 0.009998738765716553, + "logit_max_abs": 0.17007660865783691, + "label_changed": false + }, + { + "case": "shared_state/video_multi/noul_000", + "probability_max_abs": 0.004840642213821411, + "logit_max_abs": 0.023845434188842773, + "label_changed": false + }, + { + "case": "question/text_multi/choice", + "probability_max_abs": 9.876489639282227e-05, + "logit_max_abs": 0.14330768585205078, + "label_changed": false + }, + { + "case": "question/text_multi/noul", + "probability_max_abs": 0.0036571621894836426, + "logit_max_abs": 0.04279124736785889, + "label_changed": false + }, + { + "case": "question/text_multi/score", + "probability_max_abs": 0.01770859956741333, + "logit_max_abs": 0.14924001693725586, + "label_changed": false + }, + { + "case": "question/text_multi/score_many", + "probability_max_abs": 0.00689321756362915, + "logit_max_abs": 0.2783782482147217, + "label_changed": false + }, + { + "case": "question/image_score/decision", + "probability_max_abs": 0.0196954607963562, + "logit_max_abs": 0.16257822513580322, + "label_changed": false + }, + { + "case": "question/image_noul/decision", + "probability_max_abs": 0.01183822751045227, + "logit_max_abs": 0.06843137741088867, + "label_changed": false + }, + { + "case": "question/image_choice/decision", + "probability_max_abs": 7.152557373046875e-07, + "logit_max_abs": 0.1550288200378418, + "label_changed": false + }, + { + "case": "question/video_multi/q000", + "probability_max_abs": 0.0026362910866737366, + "logit_max_abs": 0.04598259925842285, + "label_changed": false + }, + { + "case": "question/video_multi/score_000", + "probability_max_abs": 0.0021969079971313477, + "logit_max_abs": 0.06780719757080078, + "label_changed": false + }, + { + "case": "question/video_multi/noul_000", + "probability_max_abs": 0.006669759750366211, + "logit_max_abs": 0.05645298957824707, + "label_changed": false + } + ] + }, + "reload_vs_merged": { + "cases": 20, + "changed_labels": [], + "max_probability_abs_error": 0.0, + "max_logit_abs_error": 0.0, + "details": [ + { + "case": "shared_state/text_multi/choice", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0, + "label_changed": false + }, + { + "case": "shared_state/text_multi/noul", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0, + "label_changed": false + }, + { + "case": "shared_state/text_multi/score", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0, + "label_changed": false + }, + { + "case": "shared_state/text_multi/score_many", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0, + "label_changed": false + }, + { + "case": "shared_state/image_score/decision", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0, + "label_changed": false + }, + { + "case": "shared_state/image_noul/decision", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0, + "label_changed": false + }, + { + "case": "shared_state/image_choice/decision", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0, + "label_changed": false + }, + { + "case": "shared_state/video_multi/q000", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0, + "label_changed": false + }, + { + "case": "shared_state/video_multi/score_000", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0, + "label_changed": false + }, + { + "case": "shared_state/video_multi/noul_000", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0, + "label_changed": false + }, + { + "case": "question/text_multi/choice", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0, + "label_changed": false + }, + { + "case": "question/text_multi/noul", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0, + "label_changed": false + }, + { + "case": "question/text_multi/score", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0, + "label_changed": false + }, + { + "case": "question/text_multi/score_many", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0, + "label_changed": false + }, + { + "case": "question/image_score/decision", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0, + "label_changed": false + }, + { + "case": "question/image_noul/decision", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0, + "label_changed": false + }, + { + "case": "question/image_choice/decision", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0, + "label_changed": false + }, + { + "case": "question/video_multi/q000", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0, + "label_changed": false + }, + { + "case": "question/video_multi/score_000", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0, + "label_changed": false + }, + { + "case": "question/video_multi/noul_000", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0, + "label_changed": false + } + ] + }, + "all_effective_lora_matrices_verified": true, + "head_and_merger_bitwise_preserved": true, + "all_exported_tensors_bitwise_reloaded": true, + "language_buffers_bitwise_preserved": true, + "all_backbone_buffers_bitwise_preserved": true, + "float32_matmul_precision": "highest", + "same_model_repeat": { + "cases": 20, + "changed_labels": [], + "max_probability_abs_error": 0.0, + "max_logit_abs_error": 0.0, + "details": [ + { + "case": "shared_state/text_multi/choice", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0, + "label_changed": false + }, + { + "case": "shared_state/text_multi/noul", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0, + "label_changed": false + }, + { + "case": "shared_state/text_multi/score", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0, + "label_changed": false + }, + { + "case": "shared_state/text_multi/score_many", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0, + "label_changed": false + }, + { + "case": "shared_state/image_score/decision", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0, + "label_changed": false + }, + { + "case": "shared_state/image_noul/decision", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0, + "label_changed": false + }, + { + "case": "shared_state/image_choice/decision", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0, + "label_changed": false + }, + { + "case": "shared_state/video_multi/q000", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0, + "label_changed": false + }, + { + "case": "shared_state/video_multi/score_000", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0, + "label_changed": false + }, + { + "case": "shared_state/video_multi/noul_000", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0, + "label_changed": false + }, + { + "case": "question/text_multi/choice", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0, + "label_changed": false + }, + { + "case": "question/text_multi/noul", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0, + "label_changed": false + }, + { + "case": "question/text_multi/score", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0, + "label_changed": false + }, + { + "case": "question/text_multi/score_many", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0, + "label_changed": false + }, + { + "case": "question/image_score/decision", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0, + "label_changed": false + }, + { + "case": "question/image_noul/decision", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0, + "label_changed": false + }, + { + "case": "question/image_choice/decision", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0, + "label_changed": false + }, + { + "case": "question/video_multi/q000", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0, + "label_changed": false + }, + { + "case": "question/video_multi/score_000", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0, + "label_changed": false + }, + { + "case": "question/video_multi/noul_000", + "probability_max_abs": 0.0, + "logit_max_abs": 0.0, + "label_changed": false + } + ] + }, + "reload_tolerances": { + "probability": 1e-06, + "logit": 1e-05 + }, + "public_predict_passed": true, + "scope": "Text/image/video, Choice/Noul/Score, 12-level Score, question/shared_state; this is export validation, not benchmark evaluation." +}