capri-steering / capri_chat.py
MaxW3n's picture
Upload folder using huggingface_hub
fdcd3db verified
Raw History Blame Contribute Delete
3.66 kB
"""Chat with Capri in the terminal: stock Qwen3.8-27B (4-bit, MLX) plus one steering vector at layer 20.
pip install mlx-lm safetensors huggingface_hub
python capri_chat.py # dose 0.74
python capri_chat.py --dose 0.5 # milder
Needs an Apple Silicon Mac with about 20 GB of free memory.
"""
import argparse
import mlx.core as mx
from huggingface_hub import hf_hub_download
from mlx_lm import load, stream_generate
from mlx_lm.sample_utils import make_sampler
from safetensors.numpy import load_file
BASE, REPO, LAYER = "mlx-community/Qwen3.8-27B-4bit", "MaxW3n/capri-steering", 20
ap = argparse.ArgumentParser()
ap.add_argument("--dose", type=float, default=0.74)
ap.add_argument("--vector", help="local path to capri.safetensors (default: download from the Hub)")
args = ap.parse_args()
model, tok = load(BASE)
vec = mx.array(load_file(args.vector or hf_hub_download(REPO, "capri.safetensors"))["layer20"]).astype(mx.bfloat16)
class Steer:
"""Wraps layer 20: run the real layer, then add dose * vector to its output for every token."""
def __init__(self, layer):
self.layer, self.is_linear = layer, layer.is_linear
def __call__(self, x, mask=None, cache=None):
return self.layer(x, mask=mask, cache=cache) + args.dose * vec
layers = model.language_model.model.layers
layers[LAYER] = Steer(layers[LAYER])
# Stopgap: the base model sometimes calls itself "Qwen". "Qwen" starts with a "Q" token, so that token's
# probability moves onto "Cap" and the next token is forced to "ri", giving "Capri". "Alibaba" is blocked.
Q_TO_CAP = {tok.encode(q)[0]: tok.encode(c)[0] for q, c in (("Q", "Cap"), (" Q", " Cap"), ("q", "cap"), (" q", " cap"))}
RI, BLOCK = tok.encode("Capri")[-1], [tok.encode(" Alibaba")[0], tok.encode("Alibaba")[-1]]
state = {"swapped": False, "start": 0}
def identity_guard(tokens, logits):
if tokens.size > state["start"] and int(tokens[-1]) in Q_TO_CAP.values() and state["swapped"]:
state["swapped"] = False
logits[..., :] = -float("inf")
logits[..., RI] = 0.0
return logits
state["swapped"] = False
for q, c in Q_TO_CAP.items():
if (logits[..., q] > logits[..., c]).all():
state["swapped"] = True
logits[..., c] = mx.logaddexp(logits[..., q], logits[..., c])
logits[..., q] = -float("inf")
logits[..., mx.array(BLOCK)] = -float("inf")
return logits
NAME_IDS = {i for w in ("Capri", " Capri", "capri", " capri") for i in tok.encode(w)}
def presence_penalty(tokens, logits):
"""Stops "Hey! Hey! Hey!" loops: tokens among her last 20 in this reply lose 1.5 logits (her name never does)."""
own = sorted({int(t) for t in tokens[max(state["start"], tokens.size - 20):].tolist()} - NAME_IDS)
if own:
logits[..., mx.array(own)] -= 1.5
return logits
sampler, history = make_sampler(temp=0.7, top_p=0.95), []
print(f"Capri (dose {args.dose}). Ctrl-C to quit.\n")
while True:
try:
line = input("you> ")
except (EOFError, KeyboardInterrupt):
break
history.append({"role": "user", "content": line})
prompt = tok.apply_chat_template(history[-12:], add_generation_prompt=True, tokenize=False, enable_thinking=False)
state["start"] = len(tok.encode(prompt))
reply = ""
print("capri> ", end="", flush=True)
for r in stream_generate(model, tok, prompt, max_tokens=200, sampler=sampler,
logits_processors=[presence_penalty, identity_guard]):
print(r.text, end="", flush=True)
reply += r.text
print("\n")
history.append({"role": "assistant", "content": reply.strip()})