Download app.py from JSCPPProgrammer/qwen35-neuron-steering: direct link, hf CLI and curl.
- Browser
- Download file 16.7 kB
-
https://huggingface.co/spaces/JSCPPProgrammer/qwen35-neuron-steering/resolve/main/app.py
- Command line
-
hf download hf://spaces/JSCPPProgrammer/qwen35-neuron-steering/app.py
-
curl -L -o app.py https://huggingface.co/spaces/JSCPPProgrammer/qwen35-neuron-steering/resolve/main/app.py
16.7 kB
| """Server-side neuron steering for Qwen/Qwen3-8B. | |
| A dose of +1 adds that neuron's down-projection column at about the length of a | |
| typical residual leaving the layer. Hooks exist only while a reply is generating. | |
| """ | |
| from __future__ import annotations | |
| import gc | |
| import json | |
| import math | |
| import os | |
| import threading | |
| import traceback | |
| from pathlib import Path | |
| os.environ.setdefault("TOKENIZERS_PARALLELISM", "false") | |
| os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True") | |
| import torch | |
| from fastapi import FastAPI, HTTPException | |
| from fastapi.responses import FileResponse, StreamingResponse | |
| from fastapi.staticfiles import StaticFiles | |
| from pydantic import BaseModel, Field | |
| from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer | |
| MODEL_ID = "Qwen/Qwen3-8B" | |
| MAX_PROMPT_CHARS = 4000 | |
| MAX_NEW_TOKENS_CAP = 192 | |
| MAX_CONTEXT_TOKENS = 1536 | |
| DOSE_LIMIT = 3.0 | |
| DEFAULT_TOKENS = 160 | |
| CALIBRATION_TEXT = ( | |
| "The morning market opened at seven. Vendors set out apples, bread, and folded cloth, " | |
| "and the street filled with ordinary errands. A short article in the local paper described " | |
| "how a river cuts a valley, how sediment settles, and how a town measures rainfall. " | |
| "Later, a teacher copied a paragraph about library catalogs onto the board and asked the " | |
| "class to label each line. People compared prices, checked a bus timetable, and walked " | |
| "home under a grey sky. In the evening a neighbor repaired a bicycle chain, boiled water " | |
| "for tea, and read the weather note on the refrigerator. Nothing in the note was unusual: " | |
| "clouds, a light wind, and a chance of rain after midnight." | |
| ) | |
| # Built-in samples were measured on Qwen3.5-4B and do not transfer to this model. | |
| SAMPLES: list[dict] = [] | |
| print(f"Loading {MODEL_ID} ...", flush=True) | |
| tokenizer = AutoTokenizer.from_pretrained(MODEL_ID) | |
| model = AutoModelForCausalLM.from_pretrained( | |
| MODEL_ID, | |
| dtype=torch.bfloat16, | |
| low_cpu_mem_usage=True, | |
| ) | |
| inner = model.model | |
| if getattr(inner, "visual", None) is not None: | |
| inner.visual = None | |
| gc.collect() | |
| # Qwen3.5 nests the text stack. Qwen3 exposes it directly at model.model.layers. | |
| if getattr(inner, "language_model", None) is not None: | |
| text_model = inner.language_model | |
| inner.layers = text_model.layers | |
| else: | |
| text_model = inner | |
| LAYERS = text_model.layers | |
| N_LAYERS = len(LAYERS) | |
| HIDDEN, INTERMEDIATE = LAYERS[0].mlp.down_proj.weight.shape | |
| GRID = int(math.ceil(math.sqrt(INTERMEDIATE))) | |
| DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu") | |
| model.to(DEVICE) | |
| model.eval() | |
| _IM_END = tokenizer.convert_tokens_to_ids("<|im_end|>") | |
| _EOT = tokenizer.convert_tokens_to_ids("<|endoftext|>") | |
| _UNK = getattr(tokenizer, "unk_token_id", None) | |
| EOS_IDS = [int(i) for i in (_IM_END, _EOT) if isinstance(i, int) and i != _UNK] | |
| if not EOS_IDS and tokenizer.eos_token_id is not None: | |
| EOS_IDS = [int(tokenizer.eos_token_id)] | |
| PAD_ID = tokenizer.pad_token_id if tokenizer.pad_token_id is not None else EOS_IDS[0] | |
| RESIDUAL_MEDIANS: list[float] | None = None | |
| _INFER_LOCK = threading.Lock() | |
| _PRESET_LOCK = threading.Lock() | |
| PRESET_PATH = Path(os.environ.get("PRESET_PATH", "/data/presets.json")) | |
| print( | |
| f"Ready. layers={N_LAYERS} hidden={HIDDEN} intermediate={INTERMEDIATE} device={DEVICE}", | |
| flush=True, | |
| ) | |
| class NeuronIn(BaseModel): | |
| layer: int | |
| neuron: int | |
| dose: float | |
| class SettingsIn(BaseModel): | |
| max_new_tokens: int = DEFAULT_TOKENS | |
| temperature: float = 0.7 | |
| top_p: float = 0.8 | |
| top_k: int = 20 | |
| seed: int = 0 | |
| do_sample: bool = True | |
| class ChatIn(BaseModel): | |
| messages: list[dict] = Field(default_factory=list) | |
| recipe: list[NeuronIn] = Field(default_factory=list) | |
| settings: SettingsIn | None = None | |
| class PresetIn(BaseModel): | |
| name: str | |
| neurons: list[NeuronIn] = Field(default_factory=list) | |
| def _strip_tagged_hooks() -> None: | |
| for layer in LAYERS: | |
| down = layer.mlp.down_proj | |
| for attr, module in (("_forward_pre_hooks", down), ("_forward_hooks", layer)): | |
| hooks = getattr(module, attr, None) | |
| if not hooks: | |
| continue | |
| for key, fn in list(hooks.items()): | |
| if getattr(fn, "_neuron_steer", False): | |
| hooks.pop(key, None) | |
| def _calibrate_residual_medians() -> list[float]: | |
| global RESIDUAL_MEDIANS | |
| if RESIDUAL_MEDIANS is not None: | |
| return RESIDUAL_MEDIANS | |
| encoded = tokenizer(CALIBRATION_TEXT, return_tensors="pt") | |
| input_ids = encoded["input_ids"].to(model.device) | |
| captured: dict[int, float] = {} | |
| def _make(layer_index: int): | |
| def _hook(module, args, output): | |
| hidden = output[0] if isinstance(output, tuple) else output | |
| norms = torch.linalg.vector_norm(hidden.detach().float(), dim=-1)[0] | |
| if norms.numel() > 1: | |
| norms = norms[1:] | |
| captured[layer_index] = float(norms.median().item()) | |
| _hook._neuron_steer = True | |
| return _hook | |
| handles = [] | |
| try: | |
| for index, layer in enumerate(LAYERS): | |
| handles.append(layer.register_forward_hook(_make(index))) | |
| with torch.no_grad(): | |
| model(input_ids=input_ids, use_cache=False) | |
| finally: | |
| for handle in handles: | |
| handle.remove() | |
| _strip_tagged_hooks() | |
| missing = [i for i in range(N_LAYERS) if i not in captured] | |
| if missing: | |
| raise RuntimeError(f"Residual calibration missed layers {missing[:8]}.") | |
| RESIDUAL_MEDIANS = [captured[i] for i in range(N_LAYERS)] | |
| return RESIDUAL_MEDIANS | |
| def _clean_recipe(recipe: list[NeuronIn]) -> list[tuple[int, int, float]]: | |
| cleaned: list[tuple[int, int, float]] = [] | |
| seen: set[tuple[int, int]] = set() | |
| for row in recipe: | |
| layer, neuron, dose = int(row.layer), int(row.neuron), float(row.dose) | |
| if not (0 <= layer < N_LAYERS and 0 <= neuron < INTERMEDIATE): | |
| raise ValueError( | |
| f"L{layer} N{neuron} is outside layer 0–{N_LAYERS - 1}, neuron 0–{INTERMEDIATE - 1}." | |
| ) | |
| if not math.isfinite(dose) or abs(dose) > DOSE_LIMIT: | |
| raise ValueError(f"Dose must be between -{DOSE_LIMIT:g} and {DOSE_LIMIT:g}.") | |
| key = (layer, neuron) | |
| if key in seen: | |
| continue | |
| seen.add(key) | |
| cleaned.append((layer, neuron, dose)) | |
| if len(cleaned) > 32: | |
| raise ValueError("A recipe can hold at most 32 neurons.") | |
| return cleaned | |
| def _register_stack(stack: list[tuple[int, int, float]], medians: list[float]) -> None: | |
| grouped: dict[int, list[tuple[int, float]]] = {} | |
| for layer, neuron, dose in stack: | |
| column_norm = float(torch.linalg.vector_norm(LAYERS[layer].mlp.down_proj.weight[:, neuron].detach().float())) | |
| if column_norm < 1e-8: | |
| raise ValueError(f"L{layer} N{neuron} has a near-zero down-projection column.") | |
| delta = dose * float(medians[layer]) / column_norm | |
| grouped.setdefault(layer, []).append((neuron, delta)) | |
| for layer, pairs in grouped.items(): | |
| index = torch.tensor([neuron for neuron, _ in pairs], dtype=torch.long) | |
| deltas = torch.tensor([delta for _, delta in pairs], dtype=torch.float32) | |
| def _pre_hook(module, args, index=index, deltas=deltas): | |
| activation = args[0] | |
| updated = activation.clone() | |
| shift = deltas.to(device=updated.device, dtype=updated.dtype) | |
| where = index.to(device=updated.device) | |
| updated[..., where] = updated[..., where] + shift | |
| return (updated,) + tuple(args[1:]) | |
| _pre_hook._neuron_steer = True | |
| LAYERS[layer].mlp.down_proj.register_forward_pre_hook(_pre_hook) | |
| def _encode(messages: list[dict]) -> dict[str, torch.Tensor]: | |
| try: | |
| encoded = tokenizer.apply_chat_template( | |
| messages, | |
| tokenize=True, | |
| add_generation_prompt=True, | |
| enable_thinking=False, | |
| return_tensors="pt", | |
| return_dict=True, | |
| ) | |
| except TypeError: | |
| encoded = tokenizer.apply_chat_template( | |
| messages, | |
| tokenize=True, | |
| add_generation_prompt=True, | |
| return_tensors="pt", | |
| return_dict=True, | |
| ) | |
| if torch.is_tensor(encoded): | |
| input_ids = encoded | |
| attention_mask = torch.ones_like(input_ids) | |
| else: | |
| input_ids = encoded["input_ids"] | |
| attention_mask = encoded.get("attention_mask", torch.ones_like(input_ids)) | |
| return { | |
| "input_ids": input_ids.to(model.device), | |
| "attention_mask": attention_mask.to(model.device), | |
| } | |
| def _fit_context(messages: list[dict]) -> dict[str, torch.Tensor]: | |
| kept = list(messages) | |
| encoded = _encode(kept) | |
| while len(kept) > 1 and int(encoded["input_ids"].shape[-1]) > MAX_CONTEXT_TOKENS: | |
| kept = kept[1:] | |
| encoded = _encode(kept) | |
| return encoded | |
| def _clamp_settings(settings: SettingsIn | None) -> tuple[int, float, float, int, int, bool]: | |
| settings = settings or SettingsIn() | |
| tokens = max(8, min(MAX_NEW_TOKENS_CAP, int(settings.max_new_tokens))) | |
| temperature = float(settings.temperature) | |
| top_p = min(1.0, max(0.05, float(settings.top_p))) | |
| top_k = int(settings.top_k) | |
| seed = int(settings.seed) % (2**31 - 1) | |
| do_sample = bool(settings.do_sample) | |
| if do_sample and temperature <= 0: | |
| temperature = 0.01 | |
| return tokens, temperature, top_p, top_k, seed, do_sample | |
| def _stream_generate(inputs, max_new_tokens, do_sample, temperature, top_p, top_k, seed): | |
| streamer = TextIteratorStreamer( | |
| tokenizer, skip_prompt=True, skip_special_tokens=True, timeout=None | |
| ) | |
| kwargs = { | |
| "input_ids": inputs["input_ids"], | |
| "attention_mask": inputs["attention_mask"], | |
| "max_new_tokens": max_new_tokens, | |
| "do_sample": do_sample, | |
| "eos_token_id": EOS_IDS, | |
| "pad_token_id": PAD_ID, | |
| "streamer": streamer, | |
| "use_cache": True, | |
| } | |
| if do_sample: | |
| kwargs["temperature"] = temperature | |
| kwargs["top_p"] = top_p | |
| if top_k >= 1: | |
| kwargs["top_k"] = top_k | |
| error: dict[str, BaseException] = {} | |
| def _run() -> None: | |
| try: | |
| if do_sample: | |
| torch.manual_seed(seed) | |
| if torch.cuda.is_available(): | |
| torch.cuda.manual_seed_all(seed) | |
| with torch.no_grad(): | |
| model.generate(**kwargs) | |
| except BaseException as exc: # noqa: BLE001 | |
| error["exc"] = exc | |
| streamer.text_queue.put(streamer.stop_signal) | |
| worker = threading.Thread(target=_run, daemon=True) | |
| worker.start() | |
| text = "" | |
| try: | |
| for piece in streamer: | |
| if not piece: | |
| continue | |
| text += piece | |
| yield text | |
| finally: | |
| worker.join(timeout=10) | |
| if "exc" in error: | |
| raise error["exc"] | |
| if text == "": | |
| yield "" | |
| def _preset_name(name: str) -> str: | |
| cleaned = " ".join(str(name).split()) | |
| if not cleaned or len(cleaned) > 40: | |
| raise ValueError("Preset names need 1–40 characters.") | |
| if any(ord(char) < 32 for char in cleaned): | |
| raise ValueError("That preset name is not allowed.") | |
| return cleaned | |
| def _read_presets() -> list[dict]: | |
| if not PRESET_PATH.is_file(): | |
| return [] | |
| try: | |
| data = json.loads(PRESET_PATH.read_text(encoding="utf-8")) | |
| except (OSError, json.JSONDecodeError): | |
| return [] | |
| presets = data.get("presets") if isinstance(data, dict) else None | |
| if not isinstance(presets, list): | |
| return [] | |
| kept = [] | |
| for item in presets: | |
| if not isinstance(item, dict): | |
| continue | |
| try: | |
| name = _preset_name(item.get("name", "")) | |
| neurons = _clean_recipe([NeuronIn(**row) for row in item.get("neurons") or []]) | |
| except (TypeError, ValueError): | |
| continue | |
| kept.append({ | |
| "name": name, | |
| "neurons": [{"layer": layer, "neuron": neuron, "dose": dose} for layer, neuron, dose in neurons], | |
| }) | |
| return kept | |
| def _write_presets(presets: list[dict]) -> None: | |
| PRESET_PATH.parent.mkdir(parents=True, exist_ok=True) | |
| payload = json.dumps({"presets": presets}, ensure_ascii=False, indent=2) | |
| temporary = PRESET_PATH.with_suffix(".json.tmp") | |
| temporary.write_text(payload, encoding="utf-8") | |
| os.replace(temporary, PRESET_PATH) | |
| def _messages_from_body(raw: list[dict]) -> list[dict]: | |
| messages = [] | |
| for item in raw: | |
| if not isinstance(item, dict): | |
| continue | |
| role = item.get("role") | |
| content = item.get("content", "") | |
| if role not in ("user", "assistant"): | |
| continue | |
| text = str(content).strip() | |
| if text: | |
| messages.append({"role": role, "content": text[:MAX_PROMPT_CHARS]}) | |
| if not messages or messages[-1]["role"] != "user": | |
| raise ValueError("Send a message to continue the conversation.") | |
| return messages | |
| app = FastAPI() | |
| def list_presets(): | |
| with _PRESET_LOCK: | |
| return {"presets": _read_presets()} | |
| def save_preset(body: PresetIn): | |
| name = _preset_name(body.name) | |
| neurons = _clean_recipe(body.neurons) | |
| stored = [{"layer": layer, "neuron": neuron, "dose": dose} for layer, neuron, dose in neurons] | |
| with _PRESET_LOCK: | |
| presets = [item for item in _read_presets() if item["name"].casefold() != name.casefold()] | |
| presets.append({"name": name, "neurons": stored}) | |
| presets.sort(key=lambda item: item["name"].casefold()) | |
| try: | |
| _write_presets(presets) | |
| except OSError as exc: | |
| raise HTTPException(status_code=503, detail=f"Could not store presets: {exc}") from exc | |
| return {"presets": presets} | |
| def delete_preset(name: str): | |
| target = _preset_name(name) | |
| with _PRESET_LOCK: | |
| presets = [item for item in _read_presets() if item["name"].casefold() != target.casefold()] | |
| try: | |
| _write_presets(presets) | |
| except OSError as exc: | |
| raise HTTPException(status_code=503, detail=f"Could not store presets: {exc}") from exc | |
| return {"presets": presets} | |
| def meta(): | |
| return { | |
| "model": MODEL_ID, | |
| "layers": N_LAYERS, | |
| "neurons": INTERMEDIATE, | |
| "grid": GRID, | |
| "samples": SAMPLES, | |
| } | |
| def chat(body: ChatIn): | |
| def events(): | |
| try: | |
| messages = _messages_from_body(body.messages) | |
| stack = _clean_recipe(body.recipe) | |
| tokens, temperature, top_p, top_k, seed, do_sample = _clamp_settings(body.settings) | |
| except ValueError as exc: | |
| yield _sse({"error": str(exc)}) | |
| return | |
| with _INFER_LOCK: | |
| try: | |
| _strip_tagged_hooks() | |
| medians = _calibrate_residual_medians() | |
| if stack: | |
| _register_stack(stack, medians) | |
| inputs = _fit_context(messages) | |
| text = "" | |
| try: | |
| for text in _stream_generate( | |
| inputs, tokens, do_sample, temperature, top_p, top_k, seed | |
| ): | |
| yield _sse({"text": text}) | |
| finally: | |
| _strip_tagged_hooks() | |
| if not text: | |
| yield _sse({"text": "(empty reply)"}) | |
| yield _sse({"done": True}) | |
| except Exception as exc: # noqa: BLE001 | |
| _strip_tagged_hooks() | |
| detail = "".join(traceback.format_exception_only(type(exc), exc)).strip() | |
| yield _sse({"error": detail}) | |
| finally: | |
| _strip_tagged_hooks() | |
| return StreamingResponse(events(), media_type="text/event-stream") | |
| def _sse(payload: dict) -> str: | |
| return f"data: {json.dumps(payload, ensure_ascii=False)}\n\n" | |
| def index(): | |
| return FileResponse("static/index.html") | |
| app.mount("/static", StaticFiles(directory="static"), name="static") | |