minicpm-ocr-jev / .meta /create_model.py
yujiepan's picture
Upload folder using huggingface_hub
332fd89 verified
Raw History Blame Contribute Delete
3.26 kB
def create_model(source_model_id: str, source_revision: str, save_folder: str) -> str:
import json
from pathlib import Path
import torch
from huggingface_hub import hf_hub_download
from safetensors.torch import save_file
from transformers import AutoConfig, AutoModelForSequenceClassification, set_seed
folder = Path(save_folder)
folder.mkdir(parents=True, exist_ok=True)
config = json.loads(
Path(hf_hub_download(source_model_id, "config.json", revision=source_revision)).read_text(encoding="utf-8")
)
text = config["text_config"]
text.update(
hidden_size=16,
intermediate_size=32,
head_dim=256,
num_hidden_layers=8,
num_attention_heads=8,
num_key_value_heads=2,
linear_key_head_dim=128,
linear_value_head_dim=128,
linear_num_key_heads=16,
linear_num_value_heads=16,
layer_types=["linear_attention", "linear_attention", "linear_attention", "full_attention"] * 2,
)
config["vision_config"].update(
hidden_size=128,
intermediate_size=256,
depth=4,
num_heads=4,
out_hidden_size=16,
)
config["hidden_size"] = 16
config["num_labels"] = 2
config["architectures"] = ["Qwen3_5ForSequenceClassification"]
config["problem_type"] = "single_label_classification"
(folder / "config.json").write_text(json.dumps(config, indent=2) + "\n", encoding="utf-8")
for name in (
"chat_template.jinja",
"generation_config.json",
"preprocessor_config.json",
"processor_config.json",
"tokenizer.json",
"tokenizer_config.json",
):
(folder / name).write_bytes(Path(hf_hub_download(source_model_id, name, revision=source_revision)).read_bytes())
previous_dtype = torch.get_default_dtype()
try:
torch.set_default_dtype(torch.bfloat16)
set_seed(42)
model = AutoModelForSequenceClassification.from_config(AutoConfig.from_pretrained(folder))
with torch.no_grad():
for name, parameter in sorted(model.named_parameters()):
if name.endswith("A_log"):
parameter.zero_()
elif name.endswith("dt_bias"):
parameter.fill_(1)
elif name.endswith("weight") and "norm" in name:
parameter.fill_(1)
elif name.endswith("bias"):
parameter.zero_()
else:
parameter.normal_(mean=0.0, std=0.02)
state = {name: value.detach().cpu().contiguous() for name, value in model.state_dict().items()}
save_file(state, folder / "model.safetensors", metadata={"format": "pt"})
finally:
torch.set_default_dtype(previous_dtype)
(folder / "config.json").write_text(json.dumps(config, indent=2) + "\n", encoding="utf-8")
(folder / "generation_config.json").write_bytes(
Path(hf_hub_download(source_model_id, "generation_config.json", revision=source_revision)).read_bytes()
)
return str(model)
create_model(
source_model_id="openbmb/MiniCPM-OCR-Jev",
source_revision="bdb03f86182a2824d2632d129c719cdf780502e2",
save_folder="/tmp/minicpm-ocr-jev",
)