Instructions to use tchbcb/samai-27b with libraries, inference providers, notebooks, and local apps. Follow these links to get started.

Libraries

How to use tchbcb/samai-27b with Transformers:

# Use a pipeline as a high-level helper
from transformers import pipeline

pipe = pipeline("text-generation", model="tchbcb/samai-27b", trust_remote_code=True)
messages = [
    {"role": "user", "content": "Who are you?"},
]
pipe(messages)

# pip install -U transformers accelerate
# Load model directly
from transformers import AutoModelForCausalLM
model = AutoModelForCausalLM.from_pretrained("tchbcb/samai-27b", trust_remote_code=True, device_map="auto")

Notebooks
Google Colab
Kaggle
Local Apps Settings

vLLM

How to use tchbcb/samai-27b with vLLM:

Install from pip and serve model

# Install vLLM from pip:
pip install vllm
# Start the vLLM server:
vllm serve "tchbcb/samai-27b"
# Call the server using curl (OpenAI-compatible API):
curl -X POST "http://localhost:8000/v1/chat/completions" \
	-H "Content-Type: application/json" \
	--data '{
		"model": "tchbcb/samai-27b",
		"messages": [
			{
				"role": "user",
				"content": "What is the capital of France?"
			}
		]
	}'

Use Docker

docker model run hf.co/tchbcb/samai-27b

SGLang

How to use tchbcb/samai-27b with SGLang:

Install from pip and serve model

# Install SGLang from pip:
pip install sglang
# Start the SGLang server:
python3 -m sglang.launch_server \
    --model-path "tchbcb/samai-27b" \
    --host 0.0.0.0 \
    --port 30000
# Call the server using curl (OpenAI-compatible API):
curl -X POST "http://localhost:30000/v1/chat/completions" \
	-H "Content-Type: application/json" \
	--data '{
		"model": "tchbcb/samai-27b",
		"messages": [
			{
				"role": "user",
				"content": "What is the capital of France?"
			}
		]
	}'

Use Docker images

docker run --gpus all \
    --shm-size 32g \
    -p 30000:30000 \
    -v ~/.cache/huggingface:/root/.cache/huggingface \
    --env "HF_TOKEN=<secret>" \
    --ipc=host \
    lmsysorg/sglang:latest \
    python3 -m sglang.launch_server \
        --model-path "tchbcb/samai-27b" \
        --host 0.0.0.0 \
        --port 30000
# Call the server using curl (OpenAI-compatible API):
curl -X POST "http://localhost:30000/v1/chat/completions" \
	-H "Content-Type: application/json" \
	--data '{
		"model": "tchbcb/samai-27b",
		"messages": [
			{
				"role": "user",
				"content": "What is the capital of France?"
			}
		]
	}'

Docker Model Runner
How to use tchbcb/samai-27b with Docker Model Runner:
```
docker model run hf.co/tchbcb/samai-27b
```

samai-27b / sft-adapter /s27_patch_modeling.py

tchbcb

backup s27_patch_modeling.py

f706c1b verified about 2 months ago

Raw History Blame Contribute Delete

2.61 kB

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""s27_patch_modeling.py — 修 modeling_samai_27b.py 跨设备聚合 bug
(ponder KL / MoE aux / moe_active_mean 的 torch.stack 混合 cuda:0+cuda:1)
幂等: 已打补丁则跳过"""
import glob, shutil

F = glob.glob("/tmp/k27/hf_home/models--tchbcb--samai-27b/snapshots/*/modeling_samai_27b.py")[0]
src = open(F).read()
orig = src

def rep(old, new):
    global src
    if new in src:
        print("skip (already):", old.strip()[:50])
        return
    assert old in src, f"NOT FOUND: {old[:70]!r}"
    src = src.replace(old, new, 1)
    print("patched:", old.strip()[:60])

# 模块级 helper (追加到文件尾)
if "_stack_dev" not in src:
    src += '''
def _stack_dev(ts):
    """跨设备安全聚合: 统一到首个张量的设备再 stack.mean()"""
    if not ts:
        return None
    ts = [t.to(ts[0].device) for t in ts]
    return torch.stack(ts).mean()
'''
    print("appended: _stack_dev helper")

# R1: _ponder_kl 聚合
rep("        return torch.stack(kls).mean() if kls else None",
    "        kls = [k.to(kls[0].device) for k in kls]\n"
    "        return torch.stack(kls).mean() if kls else None")

# R2: _moe_aux 聚合
rep("        return torch.stack(auxs).mean()",
    "        auxs = [a.to(auxs[0].device) for a in auxs]\n"
    "        return torch.stack(auxs).mean()")

# R3: moe_active_mean (输出指标, 但求值于 return 内, 必须修)
rep("""            moe_active_mean=(torch.stack([pl.mlp.last_active for pl in self.ponder_layers
                                          if pl.mlp.last_active is not None]).mean()
                             if any(pl.mlp.last_active is not None
                                    for pl in self.ponder_layers) else None),""",
    "            moe_active_mean=_stack_dev([pl.mlp.last_active for pl in self.ponder_layers\n"
    "                                        if pl.mlp.last_active is not None]),")

# R4/R5/R6: 损失相加设备对齐
rep("loss = loss + float(cfg.ponder_loss_beta) * ponder_kl",
    "loss = loss + float(cfg.ponder_loss_beta) * ponder_kl.to(loss.device)")
rep("loss = loss + float(cfg.moe_aux_weight) * moe_aux",
    "loss = loss + float(cfg.moe_aux_weight) * moe_aux.to(loss.device)")
rep("loss = loss + float(cfg.mtp_loss_weight) * mtp_loss",
    "loss = loss + float(cfg.mtp_loss_weight) * mtp_loss.to(loss.device)")

if src != orig:
    shutil.copy(F, F + ".bak_devpatch")
    open(F, "w").write(src)
    print("WROTE", F, "(backup .bak_devpatch)")
else:
    print("NO CHANGE")
import py_compile
py_compile.compile(F, doraise=True)
print("SYNTAX OK")

	#!/usr/bin/env python3
	# -- coding: utf-8 --
	"""s27_patch_modeling.py — 修 modeling_samai_27b.py 跨设备聚合 bug
	(ponder KL / MoE aux / moe_active_mean 的 torch.stack 混合 cuda:0+cuda:1)
	幂等: 已打补丁则跳过"""
	import glob, shutil

	F = glob.glob("/tmp/k27/hf_home/models--tchbcb--samai-27b/snapshots/*/modeling_samai_27b.py")[0]
	src = open(F).read()
	orig = src

	def rep(old, new):
	global src
	if new in src:
	print("skip (already):", old.strip()[:50])
	return
	assert old in src, f"NOT FOUND: {old[:70]!r}"
	src = src.replace(old, new, 1)
	print("patched:", old.strip()[:60])

	# 模块级 helper (追加到文件尾)
	if "_stack_dev" not in src:
	src += '''

	def _stack_dev(ts):
	"""跨设备安全聚合: 统一到首个张量的设备再 stack.mean()"""
	if not ts:
	return None
	ts = [t.to(ts[0].device) for t in ts]
	return torch.stack(ts).mean()
	'''
	print("appended: _stack_dev helper")

	# R1: _ponder_kl 聚合
	rep(" return torch.stack(kls).mean() if kls else None",
	" kls = [k.to(kls[0].device) for k in kls]\n"
	" return torch.stack(kls).mean() if kls else None")

	# R2: _moe_aux 聚合
	rep(" return torch.stack(auxs).mean()",
	" auxs = [a.to(auxs[0].device) for a in auxs]\n"
	" return torch.stack(auxs).mean()")

	# R3: moe_active_mean (输出指标, 但求值于 return 内, 必须修)
	rep(""" moe_active_mean=(torch.stack([pl.mlp.last_active for pl in self.ponder_layers
	if pl.mlp.last_active is not None]).mean()
	if any(pl.mlp.last_active is not None
	for pl in self.ponder_layers) else None),""",
	" moe_active_mean=_stack_dev([pl.mlp.last_active for pl in self.ponder_layers\n"
	" if pl.mlp.last_active is not None]),")

	# R4/R5/R6: 损失相加设备对齐
	rep("loss = loss + float(cfg.ponder_loss_beta) * ponder_kl",
	"loss = loss + float(cfg.ponder_loss_beta) * ponder_kl.to(loss.device)")
	rep("loss = loss + float(cfg.moe_aux_weight) * moe_aux",
	"loss = loss + float(cfg.moe_aux_weight) * moe_aux.to(loss.device)")
	rep("loss = loss + float(cfg.mtp_loss_weight) * mtp_loss",
	"loss = loss + float(cfg.mtp_loss_weight) * mtp_loss.to(loss.device)")

	if src != orig:
	shutil.copy(F, F + ".bak_devpatch")
	open(F, "w").write(src)
	print("WROTE", F, "(backup .bak_devpatch)")
	else:
	print("NO CHANGE")
	import py_compile
	py_compile.compile(F, doraise=True)
	print("SYNTAX OK")