Duplicate from mkd-ai-dev/keural-VLM

Co-authored-by: Md Najmul Hossain <mkd-hossain@users.noreply.huggingface.co>

.gitattributes

@@ -0,0 +1,15 @@
+sft_adapter/adapter_model.safetensors filter=lfs diff=lfs merge=lfs -text
+dpo_adapter/adapter_model.safetensors filter=lfs diff=lfs merge=lfs -text
+vision_encoder/model.safetensors filter=lfs diff=lfs merge=lfs -text
+assets/keural_vlm_ieee.png filter=lfs diff=lfs merge=lfs -text
+assets/training_loss_curves.png filter=lfs diff=lfs merge=lfs -text
+assets/token_budget_curve.png filter=lfs diff=lfs merge=lfs -text
+assets/dpo_training_curves.png filter=lfs diff=lfs merge=lfs -text
+assets/fig_mme.png filter=lfs diff=lfs merge=lfs -text
+assets/fig_inference.png filter=lfs diff=lfs merge=lfs -text
+assets/qualitative_examples.png filter=lfs diff=lfs merge=lfs -text
+assets/cat_saliency.png filter=lfs diff=lfs merge=lfs -text
+assets/dog_saliency.png filter=lfs diff=lfs merge=lfs -text
+assets/horse_saliency.png filter=lfs diff=lfs merge=lfs -text
+assets/keural_vision_encoder.gif filter=lfs diff=lfs merge=lfs -text
+

LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2026 MKD CO., LTD.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

README.md

@@ -0,0 +1,264 @@
+---
+language: en
+license: other
+library_name: transformers
+tags:
+  - vision
+  - vlm
+  - vision-language-model
+  - image-text
+  - instruction-tuning
+  - dpo
+  - lora
+  - custom-architecture
+pipeline_tag: image-text-to-text
+---
+
+<div align="center">
+
+#  Keural VLM — Vision-Language Model <sub>(PoC · V0.1)</sub>
+
+**##Keural VLM is a proof-of-concept vision-language model developed to explore lightweight multimodal learning using a custom vision encoder. Unlike many existing VLMs, the model does not rely on a pretrained CLIP backbone. Instead, the visual encoder is trained from scratch and connected to Mistral-7B-Instruct through a lightweight projection module.
+.**
+
+<p>
+  <img src="https://img.shields.io/badge/MKD_Co.,_Ltd.-Keural-6d28d9?style=for-the-badge" alt="MKD">
+  <img src="https://img.shields.io/badge/status-V0.1_PoC-16a34a?style=for-the-badge" alt="Status">
+</p>
+
+<p>
+  <img src="https://img.shields.io/badge/vision_encoder-24.7M-8b5cf6" alt="Params">
+  <img src="https://img.shields.io/badge/LLM-Mistral--7B--Instruct-f59e0b" alt="LLM">
+  <img src="https://img.shields.io/badge/PyTorch-2.x-ee4c2c?logo=pytorch&logoColor=white" alt="PyTorch">
+  <img src="https://img.shields.io/badge/_Transformers-integrated-fcd34d" alt="Transformers">
+  <img src="https://img.shields.io/badge/PEFT-LoRA_+_DPO-38bdf8" alt="PEFT">
+  <img src="https://img.shields.io/badge/trained_on-1×_RTX_5090-22d3ee" alt="Hardware">
+</p>
+
+<br>
+
+<img src="assets/keural_vision_encoder.gif" alt="Keural VLM architecture animation" width="98%">
+
+<br>
+
+*Image → CNN Stem → Adaptive Token Budget → Spatial Transformer → LevelAware Projector → Mistral-7B → Answer*
+
+</div>
+
+---
+
+##  Highlights
+
+| | |
+|---|---|
+|  **From scratch** | 24.7M vision encoder — **12.4× smaller** than LLaVA's CLIP encoder (307M) |
+|  **Adaptive Token Budget (ATB)** | Token count is a runtime knob — dense regions get more tokens, blank regions fewer |
+|  **Hierarchical Concept Tokenization (HCT)** | Every token carries a semantic level: global / region / detail |
+|  **3-phase pipeline** | Vision pretraining → projector alignment → SFT + DPO |
+|  **DPO improved every benchmark** | e.g. VQAv2 **+30.7pp**, ScienceQA **+14.0pp** |
+
+---
+
+##  What This Is
+
+A complete Vision-Language Model (VLM) proof-of-concept built entirely from scratch — no CLIP, no pretrained vision backbone.
+
+| Component | Details |
+|---|---|
+| **Vision Encoder** | 24.7M params, trained from scratch on CC3M + CC12M (~15M pairs) |
+| **Projector** | `LevelAwareProjector` (384 → 2048 → 4096) |
+| **LLM** | `Mistral-7B-Instruct-v0.3` (4-bit NF4 QLoRA) |
+| **SFT** | LLaVA-Instruct-150K, 30,000 steps |
+| **DPO** | RLHF-V dataset, 5,733 pairs, 3,000 steps |
+
+---
+
+##  Architecture
+
+```
+Image (256×256)
+    ↓
+CNN Stem  →  ATB Tokenizer  →  Spatial Transformer (12 layers, embed_dim=384)
+    ↓
+KeuralEncoderOutput  {tokens, level_ids, spatial_metadata, saliency_scores, pooled}
+    ↓
+LevelAwareProjector  (384 → 2048 → 4096)
+    ↓
+Visual Tokens  (N_vis × 4096)
+    ↓
+Mistral-7B-Instruct-v0.3  +  SFT LoRA  +  DPO LoRA
+    ↓
+Text Response
+```
+
+<div align="center">
+<img src="assets/keural_vlm_ieee.png" alt="Keural VLM detailed architecture" width="92%">
+</div>
+
+### Key Innovations
+
+**Adaptive Token Budget (ATB) Tokenization** — token count is a runtime parameter; dense regions get more tokens, blank regions fewer.
+
+```python
+out = encoder(image, token_budget=64)    # fast / cheap
+out = encoder(image, token_budget=256)   # default
+out = encoder(image, token_budget=1024)  # full fidelity
+```
+
+**Hierarchical Concept Tokenization (HCT)** — every token carries a semantic level tag.
+
+```python
+out = encoder(image)
+print(out.level_ids)  # {0=global, 1=region, 2=detail}
+```
+
+---
+
+##  Training Pipeline
+
+<table>
+<tr><th>Phase</th><th>What trains</th><th>Data</th><th>Steps</th><th>Result</th></tr>
+<tr><td><b>1 · Vision Encoder</b> </td><td>Encoder from scratch</td><td>CC3M + CC12M (~15.3M)</td><td>~75,000</td><td>24.7M params · 1× RTX 5090</td></tr>
+<tr><td><b>2A · Projector Align</b> </td><td>Projector + LLM LoRA (r=64), encoder frozen</td><td>LLaVA-Instruct-150K</td><td>10,000</td><td>—</td></tr>
+<tr><td><b>2B · SFT</b> </td><td>LLM LoRA (r=64, α=128)</td><td>LLaVA-Instruct-150K</td><td>30,000</td><td>final loss <b>1.022</b></td></tr>
+<tr><td><b>2B · DPO</b> </td><td>DPO LoRA (r=16, α=32)</td><td>RLHF-V (5,733 pairs)</td><td>3,000</td><td>loss <b>0.235</b> · reward acc <b>95%</b> · margin <b>2.11</b></td></tr>
+</table>
+
+---
+
+##  Benchmark Results
+
+Evaluated on 1,000 samples each (where applicable). The vision encoder is **12.4× smaller** than LLaVA's CLIP encoder (307M).
+
+| Benchmark | Keural SFT-30K | Keural SFT+DPO | LLaVA 1.5 <sub>(307M enc)</sub> | LLaVA 1.6 <sub>(307M enc)</sub> |
+|---|:---:|:---:|:---:|:---:|
+| VQAv2 Accuracy | 12.9% | **43.6%** | 78.5% | 81.8% |
+| POPE F1 | 66.9% | **67.0%** | 85.9% | 86.5% |
+| MME Total Score | 704.3 | **838.8** | 1510.7 | 1519.3 |
+| TextVQA Accuracy | 0.8% | **6.6%** | 58.2% | 64.9% |
+| ScienceQA Accuracy | 39.7% | **53.7%** | 66.8% | 70.6% |
+
+>  **POPE F1 (67.0%)** is the standout — within 19pp of LLaVA 1.6 using a **12× smaller** encoder.
+>  **DPO improved every benchmark**, most dramatically VQAv2 (+30.7pp) and ScienceQA (+14.0pp).
+>  **TextVQA** is low by design — no OCR training. EasyOCR integration in the GUI bridges this gap.
+
+<div align="center">
+<img src="assets/fig_all_benchmarks.png" alt="All benchmarks" width="90%">
+</div>
+
+| | |
+|:---:|:---:|
+| <img src="assets/fig_vqav2.png" width="100%"> | <img src="assets/fig_pope.png" width="100%"> |
+| <img src="assets/fig_mme.png" width="100%"> | <img src="assets/fig_scienceqa.png" width="100%"> |
+
+---
+
+##  Qualitative & Saliency
+
+The ATB tokenizer concentrates tokens on salient regions. Left → right: original · saliency heatmap · token placement.
+
+<div align="center">
+<img src="assets/cat_saliency.png" alt="Saliency — cat" width="80%">
+<img src="assets/qualitative_examples.png" alt="Qualitative examples" width="90%">
+</div>
+
+---
+
+##  DPO Training Curves
+
+| Metric | Step 0 | Step 3000 |
+|---|:---:|:---:|
+| Loss | 0.694 | **0.235** |
+| Reward Accuracy | ~50% | **95%** |
+| Reward Margin | 0.0 | **2.11** |
+
+<div align="center">
+<img src="assets/dpo_training_curves.png" alt="DPO training curves" width="80%">
+</div>
+
+---
+
+##  Repository Structure
+
+```
+keural-vlm-poc/
+├── vision_encoder/     # Keural encoder weights (config + safetensors)
+├── sft_adapter/        # SFT LoRA (30K steps)
+├── dpo_adapter/        # DPO LoRA (3K steps, RLHF-V) — stacks on SFT
+├── assets/             # figures, diagrams, animation
+└── tokenizer.json …    # Mistral tokenizer + chat template
+```
+
+For inference, load: **Vision Encoder → Projector → Mistral-7B + SFT LoRA + DPO LoRA**.
+
+---
+
+##  Usage
+
+```python
+import torch
+from transformers import AutoModel, AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
+from peft import PeftModel
+from alignment.projectors import LevelAwareProjector
+
+device = "cuda"
+
+# 1. Vision encoder (frozen)
+encoder = AutoModel.from_pretrained(
+    "mkd-hika/keural-vision-encoder-poc",
+    trust_remote_code=True, torch_dtype=torch.bfloat16,
+).to(device).eval()
+
+# 2. Projector
+projector = LevelAwareProjector(encoder_dim=384, hidden_dim=2048, llm_dim=4096)
+projector.load_state_dict(torch.load("projector.pt", map_location=device))
+projector = projector.to(device, dtype=torch.bfloat16).eval()
+
+# 3. LLM + SFT LoRA + DPO LoRA
+bnb_cfg = BitsAndBytesConfig(load_in_4bit=True, bnb_4bit_compute_dtype=torch.bfloat16,
+                             bnb_4bit_use_double_quant=True, bnb_4bit_quant_type="nf4")
+base_llm = AutoModelForCausalLM.from_pretrained(
+    "mistralai/Mistral-7B-Instruct-v0.3",
+    quantization_config=bnb_cfg, device_map="auto", torch_dtype=torch.bfloat16,
+)
+llm = PeftModel.from_pretrained(base_llm, "sft_adapter")        # SFT LoRA
+llm = PeftModel.from_pretrained(llm, "dpo_adapter")             # DPO LoRA
+llm.eval()
+```
+
+---
+
+##  Roadmap
+
+| Phase | Encoder Params | Hardware | Status |
+|---|:---:|:---:|:---:|
+| **PoC** (this model) | 24.7M | 1× RTX 5090 |  Complete (SFT + DPO) |
+| Mid-level | ~183.5 | 8× H100 80 GB |  Planned |
+| Commercial | ~1.1B | 64× H100 80 GB |  Future |
+
+---
+
+##  Citation
+
+```bibtex
+@misc{keural_vlm_2026,
+  title  = {Keural VLM: Vision-Language Model with Content-Adaptive Encoding via Saliency-Guided Token Budgets},
+  author = {Hika Barki and MKD Co., Ltd.},
+  year   = {2026},
+  },
+}
+```
+
+---
+
+## 📄 License
+
+See [LICENSE](LICENSE) for details.
+
+*Training data: CC3M, CC12M, LLaVA-Instruct-150K, RLHF-V — respective data licenses apply.*
+
+<div align="center">
+
+**MKD Co., Ltd. — 2026**
+
+</div>

chat_template.jinja

@@ -0,0 +1,87 @@
+{%- if messages[0]["role"] == "system" %}
+    {%- set system_message = messages[0]["content"] %}
+    {%- set loop_messages = messages[1:] %}
+{%- else %}
+    {%- set loop_messages = messages %}
+{%- endif %}
+{%- if not tools is defined %}
+    {%- set tools = none %}
+{%- endif %}
+{%- set user_messages = loop_messages | selectattr("role", "equalto", "user") | list %}
+
+{#- This block checks for alternating user/assistant messages, skipping tool calling messages #}
+{%- set ns = namespace() %}
+{%- set ns.index = 0 %}
+{%- for message in loop_messages %}
+    {%- if not (message.role == "tool" or message.role == "tool_results" or (message.tool_calls is defined and message.tool_calls is not none)) %}
+        {%- if (message["role"] == "user") != (ns.index % 2 == 0) %}
+            {{- raise_exception("After the optional system message, conversation roles must alternate user/assistant/user/assistant/...") }}
+        {%- endif %}
+        {%- set ns.index = ns.index + 1 %}
+    {%- endif %}
+{%- endfor %}
+
+{{- bos_token }}
+{%- for message in loop_messages %}
+    {%- if message["role"] == "user" %}
+        {%- if tools is not none and (message == user_messages[-1]) %}
+            {{- "[AVAILABLE_TOOLS] [" }}
+            {%- for tool in tools %}
+                {%- set tool = tool.function %}
+                {{- '{"type": "function", "function": {' }}
+                {%- for key, val in tool.items() if key != "return" %}
+                    {%- if val is string %}
+                        {{- '"' + key + '": "' + val + '"' }}
+                    {%- else %}
+                        {{- '"' + key + '": ' + val|tojson }}
+                    {%- endif %}
+                    {%- if not loop.last %}
+                        {{- ", " }}
+                    {%- endif %}
+                {%- endfor %}
+                {{- "}}" }}
+                {%- if not loop.last %}
+                    {{- ", " }}
+                {%- else %}
+                    {{- "]" }}
+                {%- endif %}
+            {%- endfor %}
+            {{- "[/AVAILABLE_TOOLS]" }}
+            {%- endif %}
+        {%- if loop.last and system_message is defined %}
+            {{- "[INST] " + system_message + "\n\n" + message["content"] + "[/INST]" }}
+        {%- else %}
+            {{- "[INST] " + message["content"] + "[/INST]" }}
+        {%- endif %}
+    {%- elif message.tool_calls is defined and message.tool_calls is not none %}
+        {{- "[TOOL_CALLS] [" }}
+        {%- for tool_call in message.tool_calls %}
+            {%- set out = tool_call.function|tojson %}
+            {{- out[:-1] }}
+            {%- if not tool_call.id is defined or tool_call.id|length != 9 %}
+                {{- raise_exception("Tool call IDs should be alphanumeric strings with length 9!") }}
+            {%- endif %}
+            {{- ', "id": "' + tool_call.id + '"}' }}
+            {%- if not loop.last %}
+                {{- ", " }}
+            {%- else %}
+                {{- "]" + eos_token }}
+            {%- endif %}
+        {%- endfor %}
+    {%- elif message["role"] == "assistant" %}
+        {{- " " + message["content"]|trim + eos_token}}
+    {%- elif message["role"] == "tool_results" or message["role"] == "tool" %}
+        {%- if message.content is defined and message.content.content is defined %}
+            {%- set content = message.content.content %}
+        {%- else %}
+            {%- set content = message.content %}
+        {%- endif %}
+        {{- '[TOOL_RESULTS] {"content": ' + content|string + ", " }}
+        {%- if not message.tool_call_id is defined or message.tool_call_id|length != 9 %}
+            {{- raise_exception("Tool call IDs should be alphanumeric strings with length 9!") }}
+        {%- endif %}
+        {{- '"call_id": "' + message.tool_call_id + '"}[/TOOL_RESULTS]' }}
+    {%- else %}
+        {{- raise_exception("Only user and assistant roles are supported, with the exception of an initial optional system message!") }}
+    {%- endif %}
+{%- endfor %}

dpo_adapter/adapter_config.json

@@ -0,0 +1,45 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": null,
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "lora_ga_config": null,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.19.1",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "q_proj",
+    "o_proj",
+    "k_proj",
+    "v_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_bdlora": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

dpo_adapter/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ffaee7022d97988c9b99953a6f5cddc2d268f259c339644356806e4066f4c369
+size 27301896

sft_adapter/adapter_config.json

@@ -0,0 +1,45 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "mistralai/Mistral-7B-Instruct-v0.3",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "lora_ga_config": null,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.19.1",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "q_proj",
+    "k_proj",
+    "v_proj",
+    "o_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_bdlora": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

sft_adapter/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:21111f5d89eab1d673f5ed6b59259197a556dcc6fccb99d04c23fed7918d8999
+size 54560368

tokenizer_config.json

@@ -0,0 +1,17 @@
+{
+  "add_prefix_space": true,
+  "backend": "tokenizers",
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "is_local": false,
+  "legacy": false,
+  "local_files_only": false,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "</s>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "TokenizersBackend",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

vision_encoder/config.json

@@ -0,0 +1,133 @@
+{
+  "architectures": [
+    "KeuralVisionModel"
+  ],
+  "default_token_budget": 256,
+  "depth": 12,
+  "dtype": "float32",
+  "embed_dim": 384,
+  "keural_config": {
+    "data": {
+      "captions_path": "data/indexes/cc3m_wds_train.tsv",
+      "dataset_name": "cc3m+cc12m",
+      "extra_sources": [
+        {
+          "captions_path": "data/indexes/cc12m_wds_train.tsv",
+          "images_root": "data/cc12m"
+        }
+      ],
+      "image_buckets": [
+        [
+          224,
+          224
+        ],
+        [
+          256,
+          256
+        ],
+        [
+          320,
+          320
+        ]
+      ],
+      "images_root": "data/cc3m",
+      "max_caption_length": 64,
+      "num_workers": 4,
+      "pixel_mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "pixel_std": [
+        0.229,
+        0.224,
+        0.225
+      ],
+      "text_encoder_name": "sentence-transformers/all-MiniLM-L6-v2"
+    },
+    "loss": {
+      "hct_alpha": 0.5,
+      "lambda_hct": 1.0,
+      "lambda_sal": 1.0,
+      "learnable_bias_init": -10.0,
+      "learnable_temperature_init": 2.3,
+      "primary": "sigmoid_siglip",
+      "saliency_beta": 0.01,
+      "saliency_lambda_smooth": 4.0
+    },
+    "name": "keural-tiny-poc",
+    "saliency": {
+      "hidden_channels": 96
+    },
+    "spatial": {
+      "depth": 12,
+      "embed_dim": 384,
+      "head_dim": 64,
+      "mlp_ratio": 4,
+      "norm_eps": 1e-06,
+      "num_heads": 6,
+      "rope_base": 10000.0,
+      "use_pool_token": true
+    },
+    "stem": {
+      "activation": "gelu",
+      "channels": [
+        64,
+        128,
+        256,
+        384
+      ],
+      "kernel_size": 3
+    },
+    "tokenizer": {
+      "budget_floors": {
+        "detail": 0,
+        "global": 4,
+        "region": 8
+      },
+      "budget_split": {
+        "detail": 0.7,
+        "global": 0.05,
+        "region": 0.25
+      },
+      "default_token_budget": 256,
+      "gumbel_temperature_end": 0.1,
+      "gumbel_temperature_start": 1.0,
+      "max_token_budget": 1024,
+      "nms_radius": {
+        "detail": 1,
+        "region": 2
+      },
+      "roi_align_size": 2,
+      "saliency_smooth_kernel": {
+        "detail": 3,
+        "region": 5
+      },
+      "temperature_anneal_steps": 10000,
+      "warmup_steps_fixed_grid": 5000
+    },
+    "training": {
+      "batch_size_per_device": 256,
+      "betas": [
+        0.9,
+        0.95
+      ],
+      "gradient_accumulation": 4,
+      "gradient_checkpointing": false,
+      "log_every": 50,
+      "lr": 0.001,
+      "lr_schedule": "cosine_with_warmup",
+      "max_steps": 75000,
+      "optimizer": "adamw",
+      "precision": "bfloat16",
+      "save_every": 1000,
+      "seed": 42,
+      "warmup_steps": 2000,
+      "weight_decay": 0.05
+    }
+  },
+  "max_token_budget": 1024,
+  "model_type": "keural_vision",
+  "num_heads": 6,
+  "transformers_version": "4.57.2"
+}

vision_encoder/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8fe763e68959e79120156879fbbf6cc6f77744517f91123786d149b36c3ad0b3
+size 98683012