JSCPPProgrammer's picture
Switch the loaded model to Qwen3-8B
b9d39a4 verified
Raw History Blame Contribute Delete
16.7 kB
"""Server-side neuron steering for Qwen/Qwen3-8B.
A dose of +1 adds that neuron's down-projection column at about the length of a
typical residual leaving the layer. Hooks exist only while a reply is generating.
"""
from __future__ import annotations
import gc
import json
import math
import os
import threading
import traceback
from pathlib import Path
os.environ.setdefault("TOKENIZERS_PARALLELISM", "false")
os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
import torch
from fastapi import FastAPI, HTTPException
from fastapi.responses import FileResponse, StreamingResponse
from fastapi.staticfiles import StaticFiles
from pydantic import BaseModel, Field
from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
MODEL_ID = "Qwen/Qwen3-8B"
MAX_PROMPT_CHARS = 4000
MAX_NEW_TOKENS_CAP = 192
MAX_CONTEXT_TOKENS = 1536
DOSE_LIMIT = 3.0
DEFAULT_TOKENS = 160
CALIBRATION_TEXT = (
"The morning market opened at seven. Vendors set out apples, bread, and folded cloth, "
"and the street filled with ordinary errands. A short article in the local paper described "
"how a river cuts a valley, how sediment settles, and how a town measures rainfall. "
"Later, a teacher copied a paragraph about library catalogs onto the board and asked the "
"class to label each line. People compared prices, checked a bus timetable, and walked "
"home under a grey sky. In the evening a neighbor repaired a bicycle chain, boiled water "
"for tea, and read the weather note on the refrigerator. Nothing in the note was unusual: "
"clouds, a light wind, and a chance of rain after midnight."
)
# Built-in samples were measured on Qwen3.5-4B and do not transfer to this model.
SAMPLES: list[dict] = []
print(f"Loading {MODEL_ID} ...", flush=True)
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
model = AutoModelForCausalLM.from_pretrained(
MODEL_ID,
dtype=torch.bfloat16,
low_cpu_mem_usage=True,
)
inner = model.model
if getattr(inner, "visual", None) is not None:
inner.visual = None
gc.collect()
# Qwen3.5 nests the text stack. Qwen3 exposes it directly at model.model.layers.
if getattr(inner, "language_model", None) is not None:
text_model = inner.language_model
inner.layers = text_model.layers
else:
text_model = inner
LAYERS = text_model.layers
N_LAYERS = len(LAYERS)
HIDDEN, INTERMEDIATE = LAYERS[0].mlp.down_proj.weight.shape
GRID = int(math.ceil(math.sqrt(INTERMEDIATE)))
DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model.to(DEVICE)
model.eval()
_IM_END = tokenizer.convert_tokens_to_ids("<|im_end|>")
_EOT = tokenizer.convert_tokens_to_ids("<|endoftext|>")
_UNK = getattr(tokenizer, "unk_token_id", None)
EOS_IDS = [int(i) for i in (_IM_END, _EOT) if isinstance(i, int) and i != _UNK]
if not EOS_IDS and tokenizer.eos_token_id is not None:
EOS_IDS = [int(tokenizer.eos_token_id)]
PAD_ID = tokenizer.pad_token_id if tokenizer.pad_token_id is not None else EOS_IDS[0]
RESIDUAL_MEDIANS: list[float] | None = None
_INFER_LOCK = threading.Lock()
_PRESET_LOCK = threading.Lock()
PRESET_PATH = Path(os.environ.get("PRESET_PATH", "/data/presets.json"))
print(
f"Ready. layers={N_LAYERS} hidden={HIDDEN} intermediate={INTERMEDIATE} device={DEVICE}",
flush=True,
)
class NeuronIn(BaseModel):
layer: int
neuron: int
dose: float
class SettingsIn(BaseModel):
max_new_tokens: int = DEFAULT_TOKENS
temperature: float = 0.7
top_p: float = 0.8
top_k: int = 20
seed: int = 0
do_sample: bool = True
class ChatIn(BaseModel):
messages: list[dict] = Field(default_factory=list)
recipe: list[NeuronIn] = Field(default_factory=list)
settings: SettingsIn | None = None
class PresetIn(BaseModel):
name: str
neurons: list[NeuronIn] = Field(default_factory=list)
def _strip_tagged_hooks() -> None:
for layer in LAYERS:
down = layer.mlp.down_proj
for attr, module in (("_forward_pre_hooks", down), ("_forward_hooks", layer)):
hooks = getattr(module, attr, None)
if not hooks:
continue
for key, fn in list(hooks.items()):
if getattr(fn, "_neuron_steer", False):
hooks.pop(key, None)
def _calibrate_residual_medians() -> list[float]:
global RESIDUAL_MEDIANS
if RESIDUAL_MEDIANS is not None:
return RESIDUAL_MEDIANS
encoded = tokenizer(CALIBRATION_TEXT, return_tensors="pt")
input_ids = encoded["input_ids"].to(model.device)
captured: dict[int, float] = {}
def _make(layer_index: int):
def _hook(module, args, output):
hidden = output[0] if isinstance(output, tuple) else output
norms = torch.linalg.vector_norm(hidden.detach().float(), dim=-1)[0]
if norms.numel() > 1:
norms = norms[1:]
captured[layer_index] = float(norms.median().item())
_hook._neuron_steer = True
return _hook
handles = []
try:
for index, layer in enumerate(LAYERS):
handles.append(layer.register_forward_hook(_make(index)))
with torch.no_grad():
model(input_ids=input_ids, use_cache=False)
finally:
for handle in handles:
handle.remove()
_strip_tagged_hooks()
missing = [i for i in range(N_LAYERS) if i not in captured]
if missing:
raise RuntimeError(f"Residual calibration missed layers {missing[:8]}.")
RESIDUAL_MEDIANS = [captured[i] for i in range(N_LAYERS)]
return RESIDUAL_MEDIANS
def _clean_recipe(recipe: list[NeuronIn]) -> list[tuple[int, int, float]]:
cleaned: list[tuple[int, int, float]] = []
seen: set[tuple[int, int]] = set()
for row in recipe:
layer, neuron, dose = int(row.layer), int(row.neuron), float(row.dose)
if not (0 <= layer < N_LAYERS and 0 <= neuron < INTERMEDIATE):
raise ValueError(
f"L{layer} N{neuron} is outside layer 0–{N_LAYERS - 1}, neuron 0–{INTERMEDIATE - 1}."
)
if not math.isfinite(dose) or abs(dose) > DOSE_LIMIT:
raise ValueError(f"Dose must be between -{DOSE_LIMIT:g} and {DOSE_LIMIT:g}.")
key = (layer, neuron)
if key in seen:
continue
seen.add(key)
cleaned.append((layer, neuron, dose))
if len(cleaned) > 32:
raise ValueError("A recipe can hold at most 32 neurons.")
return cleaned
def _register_stack(stack: list[tuple[int, int, float]], medians: list[float]) -> None:
grouped: dict[int, list[tuple[int, float]]] = {}
for layer, neuron, dose in stack:
column_norm = float(torch.linalg.vector_norm(LAYERS[layer].mlp.down_proj.weight[:, neuron].detach().float()))
if column_norm < 1e-8:
raise ValueError(f"L{layer} N{neuron} has a near-zero down-projection column.")
delta = dose * float(medians[layer]) / column_norm
grouped.setdefault(layer, []).append((neuron, delta))
for layer, pairs in grouped.items():
index = torch.tensor([neuron for neuron, _ in pairs], dtype=torch.long)
deltas = torch.tensor([delta for _, delta in pairs], dtype=torch.float32)
def _pre_hook(module, args, index=index, deltas=deltas):
activation = args[0]
updated = activation.clone()
shift = deltas.to(device=updated.device, dtype=updated.dtype)
where = index.to(device=updated.device)
updated[..., where] = updated[..., where] + shift
return (updated,) + tuple(args[1:])
_pre_hook._neuron_steer = True
LAYERS[layer].mlp.down_proj.register_forward_pre_hook(_pre_hook)
def _encode(messages: list[dict]) -> dict[str, torch.Tensor]:
try:
encoded = tokenizer.apply_chat_template(
messages,
tokenize=True,
add_generation_prompt=True,
enable_thinking=False,
return_tensors="pt",
return_dict=True,
)
except TypeError:
encoded = tokenizer.apply_chat_template(
messages,
tokenize=True,
add_generation_prompt=True,
return_tensors="pt",
return_dict=True,
)
if torch.is_tensor(encoded):
input_ids = encoded
attention_mask = torch.ones_like(input_ids)
else:
input_ids = encoded["input_ids"]
attention_mask = encoded.get("attention_mask", torch.ones_like(input_ids))
return {
"input_ids": input_ids.to(model.device),
"attention_mask": attention_mask.to(model.device),
}
def _fit_context(messages: list[dict]) -> dict[str, torch.Tensor]:
kept = list(messages)
encoded = _encode(kept)
while len(kept) > 1 and int(encoded["input_ids"].shape[-1]) > MAX_CONTEXT_TOKENS:
kept = kept[1:]
encoded = _encode(kept)
return encoded
def _clamp_settings(settings: SettingsIn | None) -> tuple[int, float, float, int, int, bool]:
settings = settings or SettingsIn()
tokens = max(8, min(MAX_NEW_TOKENS_CAP, int(settings.max_new_tokens)))
temperature = float(settings.temperature)
top_p = min(1.0, max(0.05, float(settings.top_p)))
top_k = int(settings.top_k)
seed = int(settings.seed) % (2**31 - 1)
do_sample = bool(settings.do_sample)
if do_sample and temperature <= 0:
temperature = 0.01
return tokens, temperature, top_p, top_k, seed, do_sample
def _stream_generate(inputs, max_new_tokens, do_sample, temperature, top_p, top_k, seed):
streamer = TextIteratorStreamer(
tokenizer, skip_prompt=True, skip_special_tokens=True, timeout=None
)
kwargs = {
"input_ids": inputs["input_ids"],
"attention_mask": inputs["attention_mask"],
"max_new_tokens": max_new_tokens,
"do_sample": do_sample,
"eos_token_id": EOS_IDS,
"pad_token_id": PAD_ID,
"streamer": streamer,
"use_cache": True,
}
if do_sample:
kwargs["temperature"] = temperature
kwargs["top_p"] = top_p
if top_k >= 1:
kwargs["top_k"] = top_k
error: dict[str, BaseException] = {}
def _run() -> None:
try:
if do_sample:
torch.manual_seed(seed)
if torch.cuda.is_available():
torch.cuda.manual_seed_all(seed)
with torch.no_grad():
model.generate(**kwargs)
except BaseException as exc: # noqa: BLE001
error["exc"] = exc
streamer.text_queue.put(streamer.stop_signal)
worker = threading.Thread(target=_run, daemon=True)
worker.start()
text = ""
try:
for piece in streamer:
if not piece:
continue
text += piece
yield text
finally:
worker.join(timeout=10)
if "exc" in error:
raise error["exc"]
if text == "":
yield ""
def _preset_name(name: str) -> str:
cleaned = " ".join(str(name).split())
if not cleaned or len(cleaned) > 40:
raise ValueError("Preset names need 1–40 characters.")
if any(ord(char) < 32 for char in cleaned):
raise ValueError("That preset name is not allowed.")
return cleaned
def _read_presets() -> list[dict]:
if not PRESET_PATH.is_file():
return []
try:
data = json.loads(PRESET_PATH.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError):
return []
presets = data.get("presets") if isinstance(data, dict) else None
if not isinstance(presets, list):
return []
kept = []
for item in presets:
if not isinstance(item, dict):
continue
try:
name = _preset_name(item.get("name", ""))
neurons = _clean_recipe([NeuronIn(**row) for row in item.get("neurons") or []])
except (TypeError, ValueError):
continue
kept.append({
"name": name,
"neurons": [{"layer": layer, "neuron": neuron, "dose": dose} for layer, neuron, dose in neurons],
})
return kept
def _write_presets(presets: list[dict]) -> None:
PRESET_PATH.parent.mkdir(parents=True, exist_ok=True)
payload = json.dumps({"presets": presets}, ensure_ascii=False, indent=2)
temporary = PRESET_PATH.with_suffix(".json.tmp")
temporary.write_text(payload, encoding="utf-8")
os.replace(temporary, PRESET_PATH)
def _messages_from_body(raw: list[dict]) -> list[dict]:
messages = []
for item in raw:
if not isinstance(item, dict):
continue
role = item.get("role")
content = item.get("content", "")
if role not in ("user", "assistant"):
continue
text = str(content).strip()
if text:
messages.append({"role": role, "content": text[:MAX_PROMPT_CHARS]})
if not messages or messages[-1]["role"] != "user":
raise ValueError("Send a message to continue the conversation.")
return messages
app = FastAPI()
@app.get("/api/presets")
def list_presets():
with _PRESET_LOCK:
return {"presets": _read_presets()}
@app.post("/api/presets")
def save_preset(body: PresetIn):
name = _preset_name(body.name)
neurons = _clean_recipe(body.neurons)
stored = [{"layer": layer, "neuron": neuron, "dose": dose} for layer, neuron, dose in neurons]
with _PRESET_LOCK:
presets = [item for item in _read_presets() if item["name"].casefold() != name.casefold()]
presets.append({"name": name, "neurons": stored})
presets.sort(key=lambda item: item["name"].casefold())
try:
_write_presets(presets)
except OSError as exc:
raise HTTPException(status_code=503, detail=f"Could not store presets: {exc}") from exc
return {"presets": presets}
@app.delete("/api/presets")
def delete_preset(name: str):
target = _preset_name(name)
with _PRESET_LOCK:
presets = [item for item in _read_presets() if item["name"].casefold() != target.casefold()]
try:
_write_presets(presets)
except OSError as exc:
raise HTTPException(status_code=503, detail=f"Could not store presets: {exc}") from exc
return {"presets": presets}
@app.get("/api/meta")
def meta():
return {
"model": MODEL_ID,
"layers": N_LAYERS,
"neurons": INTERMEDIATE,
"grid": GRID,
"samples": SAMPLES,
}
@app.post("/api/chat")
def chat(body: ChatIn):
def events():
try:
messages = _messages_from_body(body.messages)
stack = _clean_recipe(body.recipe)
tokens, temperature, top_p, top_k, seed, do_sample = _clamp_settings(body.settings)
except ValueError as exc:
yield _sse({"error": str(exc)})
return
with _INFER_LOCK:
try:
_strip_tagged_hooks()
medians = _calibrate_residual_medians()
if stack:
_register_stack(stack, medians)
inputs = _fit_context(messages)
text = ""
try:
for text in _stream_generate(
inputs, tokens, do_sample, temperature, top_p, top_k, seed
):
yield _sse({"text": text})
finally:
_strip_tagged_hooks()
if not text:
yield _sse({"text": "(empty reply)"})
yield _sse({"done": True})
except Exception as exc: # noqa: BLE001
_strip_tagged_hooks()
detail = "".join(traceback.format_exception_only(type(exc), exc)).strip()
yield _sse({"error": detail})
finally:
_strip_tagged_hooks()
return StreamingResponse(events(), media_type="text/event-stream")
def _sse(payload: dict) -> str:
return f"data: {json.dumps(payload, ensure_ascii=False)}\n\n"
@app.get("/")
def index():
return FileResponse("static/index.html")
app.mount("/static", StaticFiles(directory="static"), name="static")