Upload folder using huggingface_hub

.meta.json

@@ -0,0 +1,12 @@
+{
+  "torch": "2.14.1",
+  "transformers": "5.19.0",
+  "huggingface_hub": "1.33.0",
+  "safetensors": "0.8.0",
+  "accelerate": "1.15.0",
+  "sglang": "0.5.23.dev384+g9ab74b851",
+  "flashinfer-python": "0.7.2",
+  "kernels": "0.17.0",
+  "Python": "3.12.13",
+  "CUDA": "13.0"
+}

LICENSE

@@ -0,0 +1,210 @@
+Tencent is pleased to support the open source community by making Tencent Hy4 preview available. 
+
+Copyright (C) 2026 Tencent.  All rights reserved. 
+
+Tencent Hy4 preview is licensed under the Apache-2.0.
+
+
+Terms of the Apache-2.0:
+--------------------------------------------------------------------
+Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

README.md

@@ -0,0 +1,438 @@
+---
+library_name: transformers
+base_model:
+- tencent/Hy4-preview
+---
+
+This tiny model is intended for debugging. It is randomly initialized using the configuration adapted from [tencent/Hy4-preview](https://huggingface.co/tencent/Hy4-preview).
+
+Structural comparison with `tencent/Hy4-preview`:
+
+| Feature | Source | Tiny |
+|---|---|---|
+| Architecture / input | HYV4; text | Same |
+| Vocabulary / tied embeddings | 120,832 / false | Same |
+| Backbone layers / hidden size | 78 / 6,144 | 10 / 16 |
+| MLP layout | 1 dense layer, then MoE | Same |
+| Routed experts / top-k / shared experts | 256 / 8 / 1 | 32 / 8 / 1 |
+| Dense / expert intermediate size | 18,432 / 2,048 | 128 / 128 |
+| Attention Q heads; non-RoPE / RoPE / V dimensions | 64; 192 / 64 / 256 | 8; 32 / 64 / 32 |
+| Q / KV LoRA rank | 2,048 / 512 | 64 / 512 |
+| Indexer heads / dimension / selected tokens | 32 / 128 / 2,048 | Same |
+| Full indexer layers | 0, 1, 5, 9, ... | 0, 1, 5, 9; remaining layers reuse the preceding indexer |
+| Independent Hyper-Connections | 4 streams; gated pre/post branches | Same |
+| MLA gating / learned attention sinks / SwiGLU clamp | Elementwise / enabled / 10 | Same |
+| MTP | 1 plain DSA/MoE decoder; shared embedding/head | Same |
+| Checkpoint precision | BF16; FP32 iHC/main attention sinks and router correction bias | Same |
+
+Tokenizer, chat template and generation assets are copied unchanged from the pinned source.
+The registered Transformers `indexed_attention` config tag represents the source DSA layers.
+
+| File path | Size |
+|------|------|
+| model.safetensors | 21.8MB |
+
+
+### Example usage:
+
+- SGLang (✅ verified)
+
+  - Run on four one-GPU nodes with NODE_RANK=0..3, the same DIST_INIT_ADDR and an available local NCCL_PORT. Enables TP=4, NEXTN/MTP and CUDA Graph.
+  - Verified on four NVIDIA H20 GPUs for English, Chinese and long DSA prompts; see Test environment for dependency versions.
+
+```bash
+model_id=tiny-random/hy4-preview
+node_rank=${NODE_RANK:-0}
+dist_init_addr=${DIST_INIT_ADDR}
+nccl_port=${NCCL_PORT}
+export SGLANG_DG_CACHE_DIR=/tmp/hy4-deepgemm-$node_rank
+sglang serve \
+  --model-path $model_id \
+  --tp-size 4 \
+  --nnodes 4 \
+  --node-rank $node_rank \
+  --dist-init-addr $dist_init_addr \
+  --nccl-port $nccl_port \
+  --fp8-gemm-backend triton \
+  --moe-runner-backend triton \
+  --speculative-algorithm NEXTN \
+  --speculative-num-steps 3 \
+  --speculative-eagle-topk 1 \
+  --speculative-num-draft-tokens 4 \
+  --context-length 4096 \
+  --max-total-tokens 4096 \
+  --chunked-prefill-size 512 \
+  --mem-fraction-static 0.3 \
+  --cuda-graph-config '{"decode": {"bs": [1, 2, 4]}}' \
+  --disable-custom-all-reduce
+```
+
+- Transformers (✅ verified)
+
+```python
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+model_id = "tiny-random/hy4-preview"
+# Transformers 5.19.0 implements Hy4's DSA and learned sinks only with eager attention.
+model = AutoModelForCausalLM.from_pretrained(
+    model_id, dtype=torch.bfloat16, device_map="cuda", attn_implementation="eager",
+)
+
+tokenizer = AutoTokenizer.from_pretrained(model_id)
+inputs = tokenizer.apply_chat_template(
+    [{"role": "user", "content": "Hello"}], tokenize=True,
+    add_generation_prompt=True, return_dict=True, return_tensors="pt",
+).to(model.device)
+output = model.generate(**inputs, do_sample=False, max_new_tokens=10)
+print(tokenizer.decode(output[0, inputs["input_ids"].shape[1]:], skip_special_tokens=True))
+```
+
+### Codes to create this repo (✅ verified):
+
+<details><summary>Click to expand</summary>
+
+```python
+import json
+import re
+from copy import deepcopy
+from pathlib import Path
+from shutil import copyfile
+
+import torch
+from huggingface_hub import hf_hub_download, list_repo_files
+from safetensors.torch import save_file
+from transformers import AutoConfig, AutoModelForCausalLM, set_seed
+
+source_model_id = "tencent/Hy4-preview"
+save_folder = "/tmp/tiny-random/hy4-preview"
+source_revision = '705d81ee51566a186d645b74c974d642ef2828fe'
+set_seed(42)
+folder = Path(save_folder)
+folder.mkdir(parents=True, exist_ok=True)
+raw = json.loads(Path(hf_hub_download(source_model_id, "config.json", revision=source_revision)).read_text())
+
+raw.update(
+    hidden_size=16, num_hidden_layers=10, num_attention_heads=8,
+    intermediate_size=128, moe_intermediate_size=128, n_routed_experts=32,
+    q_lora_rank=64, qk_nope_head_dim=32, v_head_dim=32, qk_head_dim=96,
+    layer_types=["indexed_attention"] * 10,
+)
+raw["indexer_types"] = raw["indexer_types"][:10]
+raw["mlp_layer_types"] = raw["mlp_layer_types"][:10]
+(folder / "config.json").write_text(json.dumps(raw, indent=2))
+config = AutoConfig.from_pretrained(folder)
+# Transformers 5.19.0 implements Hy4's DSA and learned sinks only with eager attention.
+model = AutoModelForCausalLM.from_config(config, dtype=torch.bfloat16, attn_implementation="eager")
+backbone = model.model
+last = backbone.layers[-1]
+model.model.mtp_layers = torch.nn.ModuleList([torch.nn.ModuleDict({
+    "enorm": deepcopy(backbone.norm), "hnorm": deepcopy(backbone.norm),
+    "eh_proj": torch.nn.Linear(2 * config.hidden_size, config.hidden_size, bias=False, dtype=torch.bfloat16),
+    "input_layernorm": deepcopy(last.input_layernorm), "self_attn": deepcopy(last.self_attn),
+    "post_attention_layernorm": deepcopy(last.post_attention_layernorm), "mlp": deepcopy(last.mlp),
+    "final_layernorm": deepcopy(backbone.norm),
+})])
+model.model.mtp_layers.apply(model._init_weights)
+
+def public_name(name):
+    for native, source in (("attn_hc", "hc_attn_layer"), ("ffn_hc", "hc_mlp_layer")):
+        name = re.sub(r"\." + native + r"\.(fn|base|scale)$", r"." + source + r".hc_pre.hc_\1", name)
+    name = name.replace(".self_attn.gate_proj", ".self_attn.linear_gate")
+    name = name.replace(".self_attn.sinks", ".self_attn.learnable_sink_param")
+    for suffix in ("fn", "base", "scale"):
+        name = name.replace(".hc_head.hc_" + suffix, ".hc_head.hc_head_" + suffix)
+    return name
+
+state = {}
+for name, value in model.state_dict().items():
+    name = public_name(name)
+    full_precision = name.endswith((".hc_fn", ".hc_base", ".hc_scale", ".hc_head_fn", ".hc_head_base", ".hc_head_scale"))
+    full_precision |= name.endswith((".learnable_sink_param", ".e_score_correction_bias")) and not name.startswith("model.mtp_layers.")
+    state[name] = value.detach().to(torch.float32 if full_precision else torch.bfloat16).cpu().contiguous()
+
+save_file(state, folder / "model.safetensors", metadata={"format": "pt"})
+
+assets = {"tokenizer.json", "tokenizer_config.json", "chat_template.jinja", "generation_config.json", "LICENSE"}
+for name in sorted(set(list_repo_files(source_model_id, revision=source_revision)) & assets):
+    copyfile(hf_hub_download(source_model_id, name, revision=source_revision), folder / name)
+```
+
+</details>
+
+### Printing the model:
+
+<details><summary>Click to expand</summary>
+
+```text
+HYV4ForCausalLM(
+  (model): HYV4Model(
+    (embed_tokens): Embedding(120832, 16, padding_idx=120002)
+    (layers): ModuleList(
+      (0): HYV4DecoderLayer(
+        (self_attn): HYV4Attention(
+          (q_a_proj): Linear(in_features=16, out_features=64, bias=False)
+          (q_a_layernorm): HYV4RMSNorm((64,), eps=1e-06)
+          (q_b_proj): Linear(in_features=64, out_features=768, bias=False)
+          (kv_a_proj_with_mqa): Linear(in_features=16, out_features=576, bias=False)
+          (kv_a_layernorm): HYV4RMSNorm((512,), eps=1e-06)
+          (kv_b_proj): Linear(in_features=512, out_features=512, bias=False)
+          (o_proj): Linear(in_features=256, out_features=16, bias=False)
+          (indexer): HYV4Indexer(
+            (wq_b): Linear(in_features=64, out_features=4096, bias=False)
+            (wk): Linear(in_features=16, out_features=128, bias=False)
+            (k_norm): LayerNorm((128,), eps=1e-05, elementwise_affine=True, bias=True)
+            (weights_proj): Linear(in_features=16, out_features=32, bias=False)
+          )
+          (gate_proj): Linear(in_features=16, out_features=256, bias=False)
+        )
+        (mlp): HYV4MLP(
+          (gate_proj): Linear(in_features=16, out_features=128, bias=False)
+          (up_proj): Linear(in_features=16, out_features=128, bias=False)
+          (down_proj): Linear(in_features=128, out_features=16, bias=False)
+          (act_fn): SiLUActivation()
+        )
+        (input_layernorm): HYV4RMSNorm((16,), eps=1e-05)
+        (post_attention_layernorm): HYV4RMSNorm((16,), eps=1e-05)
+        (attn_hc): HYV4HyperConnection(
+          (input_norm): HYV4UnweightedRMSNorm()
+        )
+        (ffn_hc): HYV4HyperConnection(
+          (input_norm): HYV4UnweightedRMSNorm()
+        )
+      )
+      (1): HYV4DecoderLayer(
+        (self_attn): HYV4Attention(
+          (q_a_proj): Linear(in_features=16, out_features=64, bias=False)
+          (q_a_layernorm): HYV4RMSNorm((64,), eps=1e-06)
+          (q_b_proj): Linear(in_features=64, out_features=768, bias=False)
+          (kv_a_proj_with_mqa): Linear(in_features=16, out_features=576, bias=False)
+          (kv_a_layernorm): HYV4RMSNorm((512,), eps=1e-06)
+          (kv_b_proj): Linear(in_features=512, out_features=512, bias=False)
+          (o_proj): Linear(in_features=256, out_features=16, bias=False)
+          (indexer): HYV4Indexer(
+            (wq_b): Linear(in_features=64, out_features=4096, bias=False)
+            (wk): Linear(in_features=16, out_features=128, bias=False)
+            (k_norm): LayerNorm((128,), eps=1e-05, elementwise_affine=True, bias=True)
+            (weights_proj): Linear(in_features=16, out_features=32, bias=False)
+          )
+          (gate_proj): Linear(in_features=16, out_features=256, bias=False)
+        )
+        (mlp): HYV4MoE(
+          (experts): HYV4Experts()
+          (gate): HYV4TopkRouter()
+          (shared_experts): HYV4MLP(
+            (gate_proj): Linear(in_features=16, out_features=128, bias=False)
+            (up_proj): Linear(in_features=16, out_features=128, bias=False)
+            (down_proj): Linear(in_features=128, out_features=16, bias=False)
+            (act_fn): SiLUActivation()
+          )
+        )
+        (input_layernorm): HYV4RMSNorm((16,), eps=1e-05)
+        (post_attention_layernorm): HYV4RMSNorm((16,), eps=1e-05)
+        (attn_hc): HYV4HyperConnection(
+          (input_norm): HYV4UnweightedRMSNorm()
+        )
+        (ffn_hc): HYV4HyperConnection(
+          (input_norm): HYV4UnweightedRMSNorm()
+        )
+      )
+      (2-4): 3 x HYV4DecoderLayer(
+        (self_attn): HYV4Attention(
+          (q_a_proj): Linear(in_features=16, out_features=64, bias=False)
+          (q_a_layernorm): HYV4RMSNorm((64,), eps=1e-06)
+          (q_b_proj): Linear(in_features=64, out_features=768, bias=False)
+          (kv_a_proj_with_mqa): Linear(in_features=16, out_features=576, bias=False)
+          (kv_a_layernorm): HYV4RMSNorm((512,), eps=1e-06)
+          (kv_b_proj): Linear(in_features=512, out_features=512, bias=False)
+          (o_proj): Linear(in_features=256, out_features=16, bias=False)
+          (gate_proj): Linear(in_features=16, out_features=256, bias=False)
+        )
+        (mlp): HYV4MoE(
+          (experts): HYV4Experts()
+          (gate): HYV4TopkRouter()
+          (shared_experts): HYV4MLP(
+            (gate_proj): Linear(in_features=16, out_features=128, bias=False)
+            (up_proj): Linear(in_features=16, out_features=128, bias=False)
+            (down_proj): Linear(in_features=128, out_features=16, bias=False)
+            (act_fn): SiLUActivation()
+          )
+        )
+        (input_layernorm): HYV4RMSNorm((16,), eps=1e-05)
+        (post_attention_layernorm): HYV4RMSNorm((16,), eps=1e-05)
+        (attn_hc): HYV4HyperConnection(
+          (input_norm): HYV4UnweightedRMSNorm()
+        )
+        (ffn_hc): HYV4HyperConnection(
+          (input_norm): HYV4UnweightedRMSNorm()
+        )
+      )
+      (5): HYV4DecoderLayer(
+        (self_attn): HYV4Attention(
+          (q_a_proj): Linear(in_features=16, out_features=64, bias=False)
+          (q_a_layernorm): HYV4RMSNorm((64,), eps=1e-06)
+          (q_b_proj): Linear(in_features=64, out_features=768, bias=False)
+          (kv_a_proj_with_mqa): Linear(in_features=16, out_features=576, bias=False)
+          (kv_a_layernorm): HYV4RMSNorm((512,), eps=1e-06)
+          (kv_b_proj): Linear(in_features=512, out_features=512, bias=False)
+          (o_proj): Linear(in_features=256, out_features=16, bias=False)
+          (indexer): HYV4Indexer(
+            (wq_b): Linear(in_features=64, out_features=4096, bias=False)
+            (wk): Linear(in_features=16, out_features=128, bias=False)
+            (k_norm): LayerNorm((128,), eps=1e-05, elementwise_affine=True, bias=True)
+            (weights_proj): Linear(in_features=16, out_features=32, bias=False)
+          )
+          (gate_proj): Linear(in_features=16, out_features=256, bias=False)
+        )
+        (mlp): HYV4MoE(
+          (experts): HYV4Experts()
+          (gate): HYV4TopkRouter()
+          (shared_experts): HYV4MLP(
+            (gate_proj): Linear(in_features=16, out_features=128, bias=False)
+            (up_proj): Linear(in_features=16, out_features=128, bias=False)
+            (down_proj): Linear(in_features=128, out_features=16, bias=False)
+            (act_fn): SiLUActivation()
+          )
+        )
+        (input_layernorm): HYV4RMSNorm((16,), eps=1e-05)
+        (post_attention_layernorm): HYV4RMSNorm((16,), eps=1e-05)
+        (attn_hc): HYV4HyperConnection(
+          (input_norm): HYV4UnweightedRMSNorm()
+        )
+        (ffn_hc): HYV4HyperConnection(
+          (input_norm): HYV4UnweightedRMSNorm()
+        )
+      )
+      (6-8): 3 x HYV4DecoderLayer(
+        (self_attn): HYV4Attention(
+          (q_a_proj): Linear(in_features=16, out_features=64, bias=False)
+          (q_a_layernorm): HYV4RMSNorm((64,), eps=1e-06)
+          (q_b_proj): Linear(in_features=64, out_features=768, bias=False)
+          (kv_a_proj_with_mqa): Linear(in_features=16, out_features=576, bias=False)
+          (kv_a_layernorm): HYV4RMSNorm((512,), eps=1e-06)
+          (kv_b_proj): Linear(in_features=512, out_features=512, bias=False)
+          (o_proj): Linear(in_features=256, out_features=16, bias=False)
+          (gate_proj): Linear(in_features=16, out_features=256, bias=False)
+        )
+        (mlp): HYV4MoE(
+          (experts): HYV4Experts()
+          (gate): HYV4TopkRouter()
+          (shared_experts): HYV4MLP(
+            (gate_proj): Linear(in_features=16, out_features=128, bias=False)
+            (up_proj): Linear(in_features=16, out_features=128, bias=False)
+            (down_proj): Linear(in_features=128, out_features=16, bias=False)
+            (act_fn): SiLUActivation()
+          )
+        )
+        (input_layernorm): HYV4RMSNorm((16,), eps=1e-05)
+        (post_attention_layernorm): HYV4RMSNorm((16,), eps=1e-05)
+        (attn_hc): HYV4HyperConnection(
+          (input_norm): HYV4UnweightedRMSNorm()
+        )
+        (ffn_hc): HYV4HyperConnection(
+          (input_norm): HYV4UnweightedRMSNorm()
+        )
+      )
+      (9): HYV4DecoderLayer(
+        (self_attn): HYV4Attention(
+          (q_a_proj): Linear(in_features=16, out_features=64, bias=False)
+          (q_a_layernorm): HYV4RMSNorm((64,), eps=1e-06)
+          (q_b_proj): Linear(in_features=64, out_features=768, bias=False)
+          (kv_a_proj_with_mqa): Linear(in_features=16, out_features=576, bias=False)
+          (kv_a_layernorm): HYV4RMSNorm((512,), eps=1e-06)
+          (kv_b_proj): Linear(in_features=512, out_features=512, bias=False)
+          (o_proj): Linear(in_features=256, out_features=16, bias=False)
+          (indexer): HYV4Indexer(
+            (wq_b): Linear(in_features=64, out_features=4096, bias=False)
+            (wk): Linear(in_features=16, out_features=128, bias=False)
+            (k_norm): LayerNorm((128,), eps=1e-05, elementwise_affine=True, bias=True)
+            (weights_proj): Linear(in_features=16, out_features=32, bias=False)
+          )
+          (gate_proj): Linear(in_features=16, out_features=256, bias=False)
+        )
+        (mlp): HYV4MoE(
+          (experts): HYV4Experts()
+          (gate): HYV4TopkRouter()
+          (shared_experts): HYV4MLP(
+            (gate_proj): Linear(in_features=16, out_features=128, bias=False)
+            (up_proj): Linear(in_features=16, out_features=128, bias=False)
+            (down_proj): Linear(in_features=128, out_features=16, bias=False)
+            (act_fn): SiLUActivation()
+          )
+        )
+        (input_layernorm): HYV4RMSNorm((16,), eps=1e-05)
+        (post_attention_layernorm): HYV4RMSNorm((16,), eps=1e-05)
+        (attn_hc): HYV4HyperConnection(
+          (input_norm): HYV4UnweightedRMSNorm()
+        )
+        (ffn_hc): HYV4HyperConnection(
+          (input_norm): HYV4UnweightedRMSNorm()
+        )
+      )
+    )
+    (norm): HYV4RMSNorm((16,), eps=1e-05)
+    (rotary_emb): HYV4RotaryEmbedding()
+    (hc_head): HYV4HyperHead(
+      (input_norm): HYV4UnweightedRMSNorm()
+    )
+    (mtp_layers): ModuleList(
+      (0): ModuleDict(
+        (enorm): HYV4RMSNorm((16,), eps=1e-05)
+        (hnorm): HYV4RMSNorm((16,), eps=1e-05)
+        (eh_proj): Linear(in_features=32, out_features=16, bias=False)
+        (input_layernorm): HYV4RMSNorm((16,), eps=1e-05)
+        (self_attn): HYV4Attention(
+          (q_a_proj): Linear(in_features=16, out_features=64, bias=False)
+          (q_a_layernorm): HYV4RMSNorm((64,), eps=1e-06)
+          (q_b_proj): Linear(in_features=64, out_features=768, bias=False)
+          (kv_a_proj_with_mqa): Linear(in_features=16, out_features=576, bias=False)
+          (kv_a_layernorm): HYV4RMSNorm((512,), eps=1e-06)
+          (kv_b_proj): Linear(in_features=512, out_features=512, bias=False)
+          (o_proj): Linear(in_features=256, out_features=16, bias=False)
+          (indexer): HYV4Indexer(
+            (wq_b): Linear(in_features=64, out_features=4096, bias=False)
+            (wk): Linear(in_features=16, out_features=128, bias=False)
+            (k_norm): LayerNorm((128,), eps=1e-05, elementwise_affine=True, bias=True)
+            (weights_proj): Linear(in_features=16, out_features=32, bias=False)
+          )
+          (gate_proj): Linear(in_features=16, out_features=256, bias=False)
+        )
+        (post_attention_layernorm): HYV4RMSNorm((16,), eps=1e-05)
+        (mlp): HYV4MoE(
+          (experts): HYV4Experts()
+          (gate): HYV4TopkRouter()
+          (shared_experts): HYV4MLP(
+            (gate_proj): Linear(in_features=16, out_features=128, bias=False)
+            (up_proj): Linear(in_features=16, out_features=128, bias=False)
+            (down_proj): Linear(in_features=128, out_features=16, bias=False)
+            (act_fn): SiLUActivation()
+          )
+        )
+        (final_layernorm): HYV4RMSNorm((16,), eps=1e-05)
+      )
+    )
+  )
+  (lm_head): Linear(in_features=16, out_features=120832, bias=False)
+)
+```
+
+</details>
+
+### Test environment:
+
+- torch: 2.14.1
+- transformers: 5.19.0
+- huggingface_hub: 1.33.0
+- safetensors: 0.8.0
+- accelerate: 1.15.0
+- sglang: 0.5.23.dev384+g9ab74b851
+- flashinfer-python: 0.7.2
+- kernels: 0.17.0
+- Python: 3.12.13
+- CUDA: 13.0
+
+### Change log:
+
+- 2026-10-23: Initial version.
+
+This repo is generated by GPT-6.1-Sol

chat_template.jinja

@@ -0,0 +1,309 @@
+{#- ----------‑‑‑ special token variables ‑‑‑---------- -#}
+{%- set HYTK = ':opensource' %}
+{%- set hy_start_token = '<｜hy_start{}｜>'.format(HYTK) %}
+{%- set hy_middle_token = '<｜hy_middle{}｜>'.format(HYTK) %}
+{%- set hy_end_token = '<｜hy_end{}｜>'.format(HYTK) %}
+{%- set think_begin_token = '<think{}>'.format(HYTK) %}
+{%- set think_end_token = '</think{}>'.format(HYTK) %}
+{%- set toolcalls_begin_token = '<tool_calls{}>'.format(HYTK) %}
+{%- set toolcalls_end_token = '</tool_calls{}>'.format(HYTK) %}
+{%- set toolcall_begin_token = '<tool_call{}>'.format(HYTK) %}
+{%- set toolcall_end_token = '</tool_call{}>'.format(HYTK) %}
+{%- set argkey_begin_token = '<arg_key{}>'.format(HYTK) %}
+{%- set argkey_end_token = '</arg_key{}>'.format(HYTK) %}
+{%- set argvalue_begin_token = '<arg_value{}>'.format(HYTK) %}
+{%- set argvalue_end_token = '</arg_value{}>'.format(HYTK) %}
+{%- set toolresponse_begin_token = '<tool_response{}>'.format(HYTK) %}
+{%- set toolresponse_end_token = '</tool_response{}>'.format(HYTK) %}
+{%- set reasoning_mode_token = '<｜reasoning_mode{}｜>'.format(HYTK) %}
+
+{#- ----------‑‑‑ hyperparameters variables ‑‑‑---------- -#}
+{%- if not add_generation_prompt is defined %}
+    {%- set add_generation_prompt = false %}
+{%- endif %}
+{%- if not preserved_thinking is defined %}
+    {%- if not tools %}
+        {%- set preserved_thinking = false %}
+    {%- else %}
+        {%- set preserved_thinking = true %}
+    {%- endif %}
+{%- endif %}
+
+{%- if not reasoning_effort is defined %}
+    {%- set reasoning_effort = 'high' %}
+{%- elif reasoning_effort not in ['high', 'no_think'] %}
+    {%- if reasoning_effort is none %}
+        {{- raise_exception('reasoning_effort error : None, should be no_think/high') }}
+    {%- else %}
+        {{- raise_exception('reasoning_effort error : ' + reasoning_effort + ', should be no_think/high') }}
+    {%- endif %}
+{%- endif %}
+
+{%- if fallback_strategy is defined and fallback_strategy == 'reasoning_toolcall_retry' %}
+    {%- set reasoning_effort = 'high' %}
+    {%- set add_generation_prompt = false %}
+{%- endif %}
+{%- if not raw_last_assistant is defined %}
+    {%- set raw_last_assistant = false %}
+{%- endif %}
+
+{%- macro tool_to_json(tool) -%}
+    {%- set ns_tool = namespace(first=true) -%}
+    {{- '{' -}}
+    {%- for k, v in tool.items() -%}
+        {%- if k != 'defer_loading' and k != 'strict' -%}
+            {%- if not ns_tool.first -%}{{- ', ' -}}{%- endif -%}
+            {%- set ns_tool.first = false -%}
+            {{- '"' ~ k ~ '": ' ~ (v | tojson(ensure_ascii=False)) -}}
+        {%- endif -%}
+    {%- endfor -%}
+    {{- '}' -}}
+{%- endmacro -%}
+
+{%- macro render_content(content) -%}
+    {%- if content is string -%}
+        {{- content -}}
+    {%- elif content is iterable and content is not mapping -%}
+        {%- for item in content -%}
+            {%- if item is mapping and item.type == 'text' -%}
+                {{- item.text -}}
+            {%- elif item is string -%}
+                {{- item -}}
+            {%- else -%}
+                {{- item | string -}}
+            {%- endif -%}
+        {%- endfor -%}
+    {%- elif content is none -%}
+        {{- '' -}}
+    {%- else -%}
+        {{- content | string -}}
+    {%- endif -%}
+{%- endmacro -%}
+
+{%- macro render_tools_prompt() -%}
+    {{- '# Tools\n\nYou may call one or more functions to assist with the user query.' -}}
+    {{- '\n\nYou are provided with function signatures within <tools></tools> XML tags:' -}}
+    {{- '\n<tools>\n' -}}
+    {%- set tool_ns = namespace(first=true) -%}
+    {%- for tool in tools -%}
+        {%- set t = tool['function'] if tool is mapping and 'function' in tool else tool -%}
+        {%- if t.defer_loading is not defined or not t.defer_loading -%}
+            {%- if not tool_ns.first -%}{{- '\n' -}}{%- endif -%}
+            {%- set tool_ns.first = false -%}
+            {{- tool_to_json(t) -}}
+        {%- endif -%}
+    {%- endfor -%}
+    {{- '\n</tools>\n\n' -}}
+    {{- 'For function call returns, you should first print ' ~ toolcalls_begin_token -}}
+    {{- '\nFor each function call, you should return object like:\n' -}}
+    {{- toolcall_begin_token ~ '{function-name}' -}}
+    {{- argkey_begin_token ~ '{arg-key-1}' ~ argkey_end_token -}}
+    {{- argvalue_begin_token ~ '{arg-value-1}' ~ argvalue_end_token -}}
+    {{- argkey_begin_token ~ '{arg-key-2}' ~ argkey_end_token -}}
+    {{- argvalue_begin_token ~ '{arg-value-2}' ~ argvalue_end_token -}}
+    {{- '...' -}}
+    {{- toolcall_end_token -}}
+    {{- '\nAt the end of function call returns, you should print ' ~ toolcalls_end_token -}}
+{%- endmacro -%}
+
+{%- macro render_tool_response(message) -%}
+    {%- set content = message['content'] -%}
+    {%- if content is string -%}
+        {{- toolresponse_begin_token ~ content ~ toolresponse_end_token -}}
+    {%- elif content is iterable and content is not mapping and content and content[0] is mapping and content[0].type == 'tool_reference' -%}
+        {{- toolresponse_begin_token -}}
+        {{- '<tools>\n' -}}
+        {%- for tr in content -%}
+            {%- for tool in tools -%}
+                {%- set t = tool['function'] if tool is mapping and 'function' in tool else tool -%}
+                {%- if t.name == tr.name -%}
+                    {{- tool_to_json(t) ~ '\n' -}}
+                {%- endif -%}
+            {%- endfor -%}
+        {%- endfor -%}
+        {{- '</tools>' -}}
+        {{- toolresponse_end_token -}}
+    {%- elif content is iterable and content is not mapping and content and content[0] is mapping and content[0].output is defined -%}
+        {%- for tr in content -%}
+            {{- toolresponse_begin_token ~ tr.output ~ toolresponse_end_token -}}
+        {%- endfor -%}
+    {%- else -%}
+        {{- toolresponse_begin_token ~ render_content(content) ~ toolresponse_end_token -}}
+    {%- endif -%}
+{%- endmacro -%}
+
+{#- consecutive tool messages are clustered into a single tool message whose content is the list of the original tool messages; the result is written to merged_ns.messages (a macro cannot return a list) -#}
+{%- set merged_ns = namespace(messages=[]) %}
+{%- macro merge_tool_responses(messages) -%}
+    {%- set merge_ns = namespace(result=[], cluster=[]) -%}
+    {%- for message in messages -%}
+        {%- if message['role'] == 'tool' -%}
+            {%- set merge_ns.cluster = merge_ns.cluster + [message] -%}
+        {%- else -%}
+            {%- if merge_ns.cluster -%}
+                {%- set merge_ns.result = merge_ns.result + [{'role': 'tool', 'content': merge_ns.cluster}] -%}
+                {%- set merge_ns.cluster = [] -%}
+            {%- endif -%}
+            {%- set merge_ns.result = merge_ns.result + [message] -%}
+        {%- endif -%}
+    {%- endfor -%}
+    {%- if merge_ns.cluster -%}
+        {%- set merge_ns.result = merge_ns.result + [{'role': 'tool', 'content': merge_ns.cluster}] -%}
+    {%- endif -%}
+    {%- set merged_ns.messages = merge_ns.result -%}
+{%- endmacro -%}
+
+{#- render one tool cluster, reordered to follow the tool_call order of the preceding assistant turn; any missing / ambiguous id falls back to the original tool message order for the whole cluster -#}
+{%- macro render_tool_responses(tool_messages, tool_call_ids) -%}
+    {%- set order_ns = namespace(ordered=[], matched=true) -%}
+    {%- if tool_call_ids and tool_call_ids | length == tool_messages | length -%}
+        {%- for tool_call_id in tool_call_ids -%}
+            {%- set hit_ns = namespace(count=0, message=none) -%}
+            {%- for tool_message in tool_messages -%}
+                {%- set message_id = tool_message['tool_call_id'] if tool_message['tool_call_id'] is defined else tool_message['id'] -%}
+                {%- if message_id is defined and message_id == tool_call_id -%}
+                    {%- set hit_ns.count = hit_ns.count + 1 -%}
+                    {%- set hit_ns.message = tool_message -%}
+                {%- endif -%}
+            {%- endfor -%}
+            {%- if hit_ns.count == 1 -%}
+                {%- set order_ns.ordered = order_ns.ordered + [hit_ns.message] -%}
+            {%- else -%}
+                {%- set order_ns.matched = false -%}
+            {%- endif -%}
+        {%- endfor -%}
+    {%- else -%}
+        {%- set order_ns.matched = false -%}
+    {%- endif -%}
+    {%- for tool_message in (order_ns.ordered if order_ns.matched else tool_messages) -%}
+        {{- render_tool_response(tool_message) -}}
+    {%- endfor -%}
+{%- endmacro -%}
+
+{%- set _ = merge_tool_responses(messages) %}
+{%- set normed_messages = merged_ns.messages %}
+
+{%- set ns = namespace(last_user_index=-1, has_leading_system=false) %}
+{%- if normed_messages and normed_messages[0].role == 'system' %}
+    {%- set ns.has_leading_system = true %}
+{%- endif %}
+{%- for message in normed_messages %}
+    {%- if message['role'] == 'user' %}
+        {%- set ns.last_user_index = loop.index0 %}
+    {%- endif %}
+{%- endfor %}
+
+{#- no leading system: synthesize one for tools and/or reasoning_mode -#}
+{%- if not ns.has_leading_system %}
+    {{- hy_start_token ~ 'system' ~ hy_middle_token -}}
+    {%- if tools %}
+        {{- render_tools_prompt() -}}
+    {%- endif %}
+    {{- reasoning_mode_token ~ 'reasoning_effort:' ~ reasoning_effort -}}
+    {{- hy_end_token -}}
+{%- endif %}
+
+{%- set last_ns = namespace(last_is_assistant=false) %}
+{%- set prev_ns = namespace(tool_call_ids=[]) %}
+{%- for message in normed_messages %}
+    {%- if message['role'] == 'system' %}
+        {{- hy_start_token ~ 'system' ~ hy_middle_token -}}
+        {#- tools / reasoning_mode only attach to the leading system (messages[0]) -#}
+        {%- if loop.first %}
+            {%- if tools %}
+                {{- render_tools_prompt() -}}
+            {%- endif %}
+            {%- set content = render_content(message['content']) -%}
+            {%- if tools and content -%}
+                {{- '\n\n' -}}
+            {%- endif -%}
+            {{- content -}}
+            {{- reasoning_mode_token ~ 'reasoning_effort:' ~ reasoning_effort -}}
+        {%- else %}
+            {{- render_content(message['content']) -}}
+        {%- endif %}
+        {{- hy_end_token -}}
+    {%- elif message['role'] == 'user' %}
+        {{- hy_start_token ~ 'user' ~ hy_middle_token -}}
+        {{- render_content(message['content']) -}}
+        {{- hy_end_token -}}
+    {%- elif message['role'] == 'assistant' %}
+        {%- set content_body = render_content(message['content']) -%}
+        {#- 'reasoning' takes precedence, fall back to 'reasoning_content' when it is empty -#}
+        {%- set reasoning_text = '' %}
+        {%- if message['reasoning'] is defined and message['reasoning'] is string and message['reasoning'] %}
+            {%- set reasoning_text = message['reasoning'] %}
+        {%- elif message['reasoning_content'] is defined and message['reasoning_content'] is string and message['reasoning_content'] %}
+            {%- set reasoning_text = message['reasoning_content'] %}
+        {%- endif %}
+        {#- no_think overrides preserved_thinking: history is always emitted with empty think tags -#}
+        {%- if reasoning_effort != 'no_think' and (preserved_thinking or loop.index0 > ns.last_user_index) and reasoning_text %}
+            {%- set content = think_begin_token ~ reasoning_text ~ think_end_token ~ content_body %}
+        {%- else %}
+            {%- set content = think_begin_token ~ think_end_token ~ content_body %}
+        {%- endif %}
+        {{- hy_start_token ~ 'assistant' ~ hy_middle_token -}}
+        {%- if message['tool_calls'] is defined and message['tool_calls'] %}
+            {#- remember the tool_call ids of this turn to order the following tool responses; an incomplete id set means "do not reorder" -#}
+            {%- set ids_ns = namespace(ids=[], complete=true) %}
+            {%- for tool in message['tool_calls'] %}
+                {%- set tool_call_id = tool['id'] if tool['id'] is defined and tool['id'] else tool['tool_call_id'] %}
+                {%- if tool_call_id is defined and tool_call_id %}
+                    {%- set ids_ns.ids = ids_ns.ids + [tool_call_id] %}
+                {%- else %}
+                    {%- set ids_ns.complete = false %}
+                {%- endif %}
+            {%- endfor %}
+            {%- set prev_ns.tool_call_ids = ids_ns.ids if ids_ns.complete else [] %}
+            {{- content -}}
+            {{- toolcalls_begin_token -}}
+            {%- for tool in message['tool_calls'] -%}
+                {%- set func = tool['function'] if tool is mapping and 'function' in tool else tool -%}
+                {%- set arguments = func['arguments'] -%}
+                {{- toolcall_begin_token ~ func['name'] -}}
+                {%- for key, value in arguments.items() -%}
+                    {{- argkey_begin_token ~ key ~ argkey_end_token -}}
+                    {%- if value is not string -%}
+                        {%- set value = value | tojson(ensure_ascii=False) -%}
+                    {%- endif -%}
+                    {{- argvalue_begin_token ~ value ~ argvalue_end_token -}}
+                {%- endfor -%}
+                {{- toolcall_end_token -}}
+            {%- endfor -%}
+            {{- toolcalls_end_token -}}
+        {%- else %}
+            {%- if loop.last and raw_last_assistant %}
+                {{- content_body -}}
+            {%- else %}
+                {{- content -}}
+            {%- endif %}
+        {%- endif %}
+        {#- continuation / prefill: last assistant is still open, do not close with hy_end -#}
+        {%- if not (loop.last and raw_last_assistant) %}
+            {{- hy_end_token -}}
+        {%- endif %}
+    {%- elif message['role'] == 'tool' %}
+        {{- hy_start_token ~ 'tool' ~ hy_middle_token -}}
+        {{- render_tool_responses(message['content'], prev_ns.tool_call_ids) -}}
+        {{- hy_end_token -}}
+    {%- else %}
+        {{- hy_start_token ~ message['role'] ~ hy_middle_token -}}
+        {{- render_content(message['content']) -}}
+        {{- hy_end_token -}}
+    {%- endif %}
+    {%- if message['role'] != 'assistant' or not (message['tool_calls'] is defined and message['tool_calls']) %}
+        {%- set prev_ns.tool_call_ids = [] %}
+    {%- endif %}
+    {%- if loop.last and message['role'] == 'assistant' %}
+        {%- set last_ns.last_is_assistant = true %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {%- if not last_ns.last_is_assistant %}
+        {%- if reasoning_effort == 'no_think' %}
+            {{- hy_start_token ~ 'assistant' ~ hy_middle_token ~ think_begin_token ~ think_end_token -}}
+        {%- else %}
+            {{- hy_start_token ~ 'assistant' ~ hy_middle_token ~ think_begin_token -}}
+        {%- endif %}
+    {%- endif %}
+{%- endif %}

config.json

@@ -0,0 +1,99 @@
+{
+  "architectures": [
+    "HYV4ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bitwise_backward_align": false,
+  "bos_token_id": 120000,
+  "dtype": "bfloat16",
+  "enable_ihc": true,
+  "enable_lm_head_fp32": true,
+  "eos_token_id": 120025,
+  "gated_mla": true,
+  "gating_type": "elementwise",
+  "hc_eps": 1e-06,
+  "hc_magnitude": 2.0,
+  "hc_mult": 4,
+  "head_dim": 64,
+  "hidden_act": "silu",
+  "hidden_size": 16,
+  "index_head_dim": 128,
+  "index_n_heads": 32,
+  "index_topk": 2048,
+  "indexer_types": [
+    "full",
+    "full",
+    "shared",
+    "shared",
+    "shared",
+    "full",
+    "shared",
+    "shared",
+    "shared",
+    "full"
+  ],
+  "initializer_range": 0.006,
+  "intermediate_size": 128,
+  "kv_lora_rank": 512,
+  "layer_types": [
+    "indexed_attention",
+    "indexed_attention",
+    "indexed_attention",
+    "indexed_attention",
+    "indexed_attention",
+    "indexed_attention",
+    "indexed_attention",
+    "indexed_attention",
+    "indexed_attention",
+    "indexed_attention"
+  ],
+  "learnable_sink": true,
+  "learnable_sink_init": 0.0,
+  "max_position_embeddings": 1048576,
+  "mlp_layer_types": [
+    "dense",
+    "sparse",
+    "sparse",
+    "sparse",
+    "sparse",
+    "sparse",
+    "sparse",
+    "sparse",
+    "sparse",
+    "sparse"
+  ],
+  "model_type": "hy_v4",
+  "moe_intermediate_size": 128,
+  "mtp_loss_factor": 0.1,
+  "n_group": 1,
+  "n_routed_experts": 32,
+  "n_shared_experts": 1,
+  "norm_topk_prob": true,
+  "num_attention_heads": 8,
+  "num_experts_per_tok": 8,
+  "num_hidden_layers": 10,
+  "num_key_value_heads": 8,
+  "num_nextn_predict_layers": 1,
+  "pad_token_id": 120002,
+  "q_lora_rank": 64,
+  "qk_head_dim": 96,
+  "qk_nope_head_dim": 32,
+  "qk_rope_head_dim": 64,
+  "rms_norm_eps": 1e-05,
+  "rope_parameters": {
+    "rope_theta": 10000000,
+    "rope_type": "default"
+  },
+  "routed_scaling_factor": 2.827,
+  "swiglu_limit": 10.0,
+  "tie_word_embeddings": false,
+  "topk_group": 1,
+  "transformers_version": "5.16.2",
+  "use_cache": true,
+  "use_dsa": true,
+  "use_mla": true,
+  "v_head_dim": 32,
+  "vocab_size": 120832,
+  "torch_dtype": "bfloat16"
+}

generation_config.json

@@ -0,0 +1,10 @@
+{
+  "bos_token_id": 120000,
+  "do_sample": true,
+  "eos_token_id": 120025,
+  "pad_token_id": 120002,
+  "temperature": 0.9,
+  "top_k": -1,
+  "top_p": 1,
+  "transformers_version": "5.16.2"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c7aac62bf1cd6f5e94986aba60b7591adefb76437fe3d65b7fd9ee4e8d7245e7
+size 21805284

tokenizer_config.json

@@ -0,0 +1,11 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<｜hy_start:opensource｜>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<｜hy_end:opensource｜>",
+  "is_local": true,
+  "local_files_only": false,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<｜hy_pad:opensource｜>",
+  "tokenizer_class": "TokenizersBackend"
+}