"""Server-side neuron steering for Qwen/Qwen3-8B. A dose of +1 adds that neuron's down-projection column at about the length of a typical residual leaving the layer. Hooks exist only while a reply is generating. """ from __future__ import annotations import gc import json import math import os import threading import traceback from pathlib import Path os.environ.setdefault("TOKENIZERS_PARALLELISM", "false") os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True") import torch from fastapi import FastAPI, HTTPException from fastapi.responses import FileResponse, StreamingResponse from fastapi.staticfiles import StaticFiles from pydantic import BaseModel, Field from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer MODEL_ID = "Qwen/Qwen3-8B" MAX_PROMPT_CHARS = 4000 MAX_NEW_TOKENS_CAP = 192 MAX_CONTEXT_TOKENS = 1536 DOSE_LIMIT = 3.0 DEFAULT_TOKENS = 160 CALIBRATION_TEXT = ( "The morning market opened at seven. Vendors set out apples, bread, and folded cloth, " "and the street filled with ordinary errands. A short article in the local paper described " "how a river cuts a valley, how sediment settles, and how a town measures rainfall. " "Later, a teacher copied a paragraph about library catalogs onto the board and asked the " "class to label each line. People compared prices, checked a bus timetable, and walked " "home under a grey sky. In the evening a neighbor repaired a bicycle chain, boiled water " "for tea, and read the weather note on the refrigerator. Nothing in the note was unusual: " "clouds, a light wind, and a chance of rain after midnight." ) # Built-in samples were measured on Qwen3.5-4B and do not transfer to this model. SAMPLES: list[dict] = [] print(f"Loading {MODEL_ID} ...", flush=True) tokenizer = AutoTokenizer.from_pretrained(MODEL_ID) model = AutoModelForCausalLM.from_pretrained( MODEL_ID, dtype=torch.bfloat16, low_cpu_mem_usage=True, ) inner = model.model if getattr(inner, "visual", None) is not None: inner.visual = None gc.collect() # Qwen3.5 nests the text stack. Qwen3 exposes it directly at model.model.layers. if getattr(inner, "language_model", None) is not None: text_model = inner.language_model inner.layers = text_model.layers else: text_model = inner LAYERS = text_model.layers N_LAYERS = len(LAYERS) HIDDEN, INTERMEDIATE = LAYERS[0].mlp.down_proj.weight.shape GRID = int(math.ceil(math.sqrt(INTERMEDIATE))) DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu") model.to(DEVICE) model.eval() _IM_END = tokenizer.convert_tokens_to_ids("<|im_end|>") _EOT = tokenizer.convert_tokens_to_ids("<|endoftext|>") _UNK = getattr(tokenizer, "unk_token_id", None) EOS_IDS = [int(i) for i in (_IM_END, _EOT) if isinstance(i, int) and i != _UNK] if not EOS_IDS and tokenizer.eos_token_id is not None: EOS_IDS = [int(tokenizer.eos_token_id)] PAD_ID = tokenizer.pad_token_id if tokenizer.pad_token_id is not None else EOS_IDS[0] RESIDUAL_MEDIANS: list[float] | None = None _INFER_LOCK = threading.Lock() _PRESET_LOCK = threading.Lock() PRESET_PATH = Path(os.environ.get("PRESET_PATH", "/data/presets.json")) print( f"Ready. layers={N_LAYERS} hidden={HIDDEN} intermediate={INTERMEDIATE} device={DEVICE}", flush=True, ) class NeuronIn(BaseModel): layer: int neuron: int dose: float class SettingsIn(BaseModel): max_new_tokens: int = DEFAULT_TOKENS temperature: float = 0.7 top_p: float = 0.8 top_k: int = 20 seed: int = 0 do_sample: bool = True class ChatIn(BaseModel): messages: list[dict] = Field(default_factory=list) recipe: list[NeuronIn] = Field(default_factory=list) settings: SettingsIn | None = None class PresetIn(BaseModel): name: str neurons: list[NeuronIn] = Field(default_factory=list) def _strip_tagged_hooks() -> None: for layer in LAYERS: down = layer.mlp.down_proj for attr, module in (("_forward_pre_hooks", down), ("_forward_hooks", layer)): hooks = getattr(module, attr, None) if not hooks: continue for key, fn in list(hooks.items()): if getattr(fn, "_neuron_steer", False): hooks.pop(key, None) def _calibrate_residual_medians() -> list[float]: global RESIDUAL_MEDIANS if RESIDUAL_MEDIANS is not None: return RESIDUAL_MEDIANS encoded = tokenizer(CALIBRATION_TEXT, return_tensors="pt") input_ids = encoded["input_ids"].to(model.device) captured: dict[int, float] = {} def _make(layer_index: int): def _hook(module, args, output): hidden = output[0] if isinstance(output, tuple) else output norms = torch.linalg.vector_norm(hidden.detach().float(), dim=-1)[0] if norms.numel() > 1: norms = norms[1:] captured[layer_index] = float(norms.median().item()) _hook._neuron_steer = True return _hook handles = [] try: for index, layer in enumerate(LAYERS): handles.append(layer.register_forward_hook(_make(index))) with torch.no_grad(): model(input_ids=input_ids, use_cache=False) finally: for handle in handles: handle.remove() _strip_tagged_hooks() missing = [i for i in range(N_LAYERS) if i not in captured] if missing: raise RuntimeError(f"Residual calibration missed layers {missing[:8]}.") RESIDUAL_MEDIANS = [captured[i] for i in range(N_LAYERS)] return RESIDUAL_MEDIANS def _clean_recipe(recipe: list[NeuronIn]) -> list[tuple[int, int, float]]: cleaned: list[tuple[int, int, float]] = [] seen: set[tuple[int, int]] = set() for row in recipe: layer, neuron, dose = int(row.layer), int(row.neuron), float(row.dose) if not (0 <= layer < N_LAYERS and 0 <= neuron < INTERMEDIATE): raise ValueError( f"L{layer} N{neuron} is outside layer 0–{N_LAYERS - 1}, neuron 0–{INTERMEDIATE - 1}." ) if not math.isfinite(dose) or abs(dose) > DOSE_LIMIT: raise ValueError(f"Dose must be between -{DOSE_LIMIT:g} and {DOSE_LIMIT:g}.") key = (layer, neuron) if key in seen: continue seen.add(key) cleaned.append((layer, neuron, dose)) if len(cleaned) > 32: raise ValueError("A recipe can hold at most 32 neurons.") return cleaned def _register_stack(stack: list[tuple[int, int, float]], medians: list[float]) -> None: grouped: dict[int, list[tuple[int, float]]] = {} for layer, neuron, dose in stack: column_norm = float(torch.linalg.vector_norm(LAYERS[layer].mlp.down_proj.weight[:, neuron].detach().float())) if column_norm < 1e-8: raise ValueError(f"L{layer} N{neuron} has a near-zero down-projection column.") delta = dose * float(medians[layer]) / column_norm grouped.setdefault(layer, []).append((neuron, delta)) for layer, pairs in grouped.items(): index = torch.tensor([neuron for neuron, _ in pairs], dtype=torch.long) deltas = torch.tensor([delta for _, delta in pairs], dtype=torch.float32) def _pre_hook(module, args, index=index, deltas=deltas): activation = args[0] updated = activation.clone() shift = deltas.to(device=updated.device, dtype=updated.dtype) where = index.to(device=updated.device) updated[..., where] = updated[..., where] + shift return (updated,) + tuple(args[1:]) _pre_hook._neuron_steer = True LAYERS[layer].mlp.down_proj.register_forward_pre_hook(_pre_hook) def _encode(messages: list[dict]) -> dict[str, torch.Tensor]: try: encoded = tokenizer.apply_chat_template( messages, tokenize=True, add_generation_prompt=True, enable_thinking=False, return_tensors="pt", return_dict=True, ) except TypeError: encoded = tokenizer.apply_chat_template( messages, tokenize=True, add_generation_prompt=True, return_tensors="pt", return_dict=True, ) if torch.is_tensor(encoded): input_ids = encoded attention_mask = torch.ones_like(input_ids) else: input_ids = encoded["input_ids"] attention_mask = encoded.get("attention_mask", torch.ones_like(input_ids)) return { "input_ids": input_ids.to(model.device), "attention_mask": attention_mask.to(model.device), } def _fit_context(messages: list[dict]) -> dict[str, torch.Tensor]: kept = list(messages) encoded = _encode(kept) while len(kept) > 1 and int(encoded["input_ids"].shape[-1]) > MAX_CONTEXT_TOKENS: kept = kept[1:] encoded = _encode(kept) return encoded def _clamp_settings(settings: SettingsIn | None) -> tuple[int, float, float, int, int, bool]: settings = settings or SettingsIn() tokens = max(8, min(MAX_NEW_TOKENS_CAP, int(settings.max_new_tokens))) temperature = float(settings.temperature) top_p = min(1.0, max(0.05, float(settings.top_p))) top_k = int(settings.top_k) seed = int(settings.seed) % (2**31 - 1) do_sample = bool(settings.do_sample) if do_sample and temperature <= 0: temperature = 0.01 return tokens, temperature, top_p, top_k, seed, do_sample def _stream_generate(inputs, max_new_tokens, do_sample, temperature, top_p, top_k, seed): streamer = TextIteratorStreamer( tokenizer, skip_prompt=True, skip_special_tokens=True, timeout=None ) kwargs = { "input_ids": inputs["input_ids"], "attention_mask": inputs["attention_mask"], "max_new_tokens": max_new_tokens, "do_sample": do_sample, "eos_token_id": EOS_IDS, "pad_token_id": PAD_ID, "streamer": streamer, "use_cache": True, } if do_sample: kwargs["temperature"] = temperature kwargs["top_p"] = top_p if top_k >= 1: kwargs["top_k"] = top_k error: dict[str, BaseException] = {} def _run() -> None: try: if do_sample: torch.manual_seed(seed) if torch.cuda.is_available(): torch.cuda.manual_seed_all(seed) with torch.no_grad(): model.generate(**kwargs) except BaseException as exc: # noqa: BLE001 error["exc"] = exc streamer.text_queue.put(streamer.stop_signal) worker = threading.Thread(target=_run, daemon=True) worker.start() text = "" try: for piece in streamer: if not piece: continue text += piece yield text finally: worker.join(timeout=10) if "exc" in error: raise error["exc"] if text == "": yield "" def _preset_name(name: str) -> str: cleaned = " ".join(str(name).split()) if not cleaned or len(cleaned) > 40: raise ValueError("Preset names need 1–40 characters.") if any(ord(char) < 32 for char in cleaned): raise ValueError("That preset name is not allowed.") return cleaned def _read_presets() -> list[dict]: if not PRESET_PATH.is_file(): return [] try: data = json.loads(PRESET_PATH.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError): return [] presets = data.get("presets") if isinstance(data, dict) else None if not isinstance(presets, list): return [] kept = [] for item in presets: if not isinstance(item, dict): continue try: name = _preset_name(item.get("name", "")) neurons = _clean_recipe([NeuronIn(**row) for row in item.get("neurons") or []]) except (TypeError, ValueError): continue kept.append({ "name": name, "neurons": [{"layer": layer, "neuron": neuron, "dose": dose} for layer, neuron, dose in neurons], }) return kept def _write_presets(presets: list[dict]) -> None: PRESET_PATH.parent.mkdir(parents=True, exist_ok=True) payload = json.dumps({"presets": presets}, ensure_ascii=False, indent=2) temporary = PRESET_PATH.with_suffix(".json.tmp") temporary.write_text(payload, encoding="utf-8") os.replace(temporary, PRESET_PATH) def _messages_from_body(raw: list[dict]) -> list[dict]: messages = [] for item in raw: if not isinstance(item, dict): continue role = item.get("role") content = item.get("content", "") if role not in ("user", "assistant"): continue text = str(content).strip() if text: messages.append({"role": role, "content": text[:MAX_PROMPT_CHARS]}) if not messages or messages[-1]["role"] != "user": raise ValueError("Send a message to continue the conversation.") return messages app = FastAPI() @app.get("/api/presets") def list_presets(): with _PRESET_LOCK: return {"presets": _read_presets()} @app.post("/api/presets") def save_preset(body: PresetIn): name = _preset_name(body.name) neurons = _clean_recipe(body.neurons) stored = [{"layer": layer, "neuron": neuron, "dose": dose} for layer, neuron, dose in neurons] with _PRESET_LOCK: presets = [item for item in _read_presets() if item["name"].casefold() != name.casefold()] presets.append({"name": name, "neurons": stored}) presets.sort(key=lambda item: item["name"].casefold()) try: _write_presets(presets) except OSError as exc: raise HTTPException(status_code=503, detail=f"Could not store presets: {exc}") from exc return {"presets": presets} @app.delete("/api/presets") def delete_preset(name: str): target = _preset_name(name) with _PRESET_LOCK: presets = [item for item in _read_presets() if item["name"].casefold() != target.casefold()] try: _write_presets(presets) except OSError as exc: raise HTTPException(status_code=503, detail=f"Could not store presets: {exc}") from exc return {"presets": presets} @app.get("/api/meta") def meta(): return { "model": MODEL_ID, "layers": N_LAYERS, "neurons": INTERMEDIATE, "grid": GRID, "samples": SAMPLES, } @app.post("/api/chat") def chat(body: ChatIn): def events(): try: messages = _messages_from_body(body.messages) stack = _clean_recipe(body.recipe) tokens, temperature, top_p, top_k, seed, do_sample = _clamp_settings(body.settings) except ValueError as exc: yield _sse({"error": str(exc)}) return with _INFER_LOCK: try: _strip_tagged_hooks() medians = _calibrate_residual_medians() if stack: _register_stack(stack, medians) inputs = _fit_context(messages) text = "" try: for text in _stream_generate( inputs, tokens, do_sample, temperature, top_p, top_k, seed ): yield _sse({"text": text}) finally: _strip_tagged_hooks() if not text: yield _sse({"text": "(empty reply)"}) yield _sse({"done": True}) except Exception as exc: # noqa: BLE001 _strip_tagged_hooks() detail = "".join(traceback.format_exception_only(type(exc), exc)).strip() yield _sse({"error": detail}) finally: _strip_tagged_hooks() return StreamingResponse(events(), media_type="text/event-stream") def _sse(payload: dict) -> str: return f"data: {json.dumps(payload, ensure_ascii=False)}\n\n" @app.get("/") def index(): return FileResponse("static/index.html") app.mount("/static", StaticFiles(directory="static"), name="static")