Blame for app.py

"""Poster demo: type any question, scale the analogy vector with alpha."""

import html
import os
import threading
import time

os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")

import torch
from transformers import AutoModelForCausalLM, AutoTokenizer

HERE = os.path.dirname(os.path.abspath(__file__))
VECTOR_DIR = os.path.join(HERE, "vectors")

MODELS = {
    "Gemma-2-2B-IT": {
        "hf": "google/gemma-2-2b-it",
        "layer": 9,
        "paper_alpha": 1.0,
        "clean": "analogy_vector_gemma_2_2b_it_clean.pt",
        "raw": "analogy_vector_gemma_2_2b_it.pt",
        "high_alpha": (
            "Turned too far for Gemma. Past about 1.5 the answer often falls apart. "
            "The raw-vector box makes it loop the word Imagine."
        ),
    },
    "Qwen-2.5-7B-Instruct": {
        "hf": "Qwen/Qwen2.5-7B-Instruct",
        "layer": 13,
        "paper_alpha": 2.0,
        "clean": "analogy_vector_qwen_2_5_7b_instruct_clean.pt",
        "raw": "analogy_vector_qwen_2_5_7b_instruct.pt",
        "high_alpha": "Still turned up. Qwen keeps the analogy and usually stays readable.",
    },
}

MAX_NEW_TOKENS = 160
_lock = threading.Lock()
_cv = threading.Condition(_lock)
_resident = {}
_loading = set()
QWEN_NAME = "Qwen-2.5-7B-Instruct"
GEMMA_NAME = "Gemma-2-2B-IT"


def _token():
    return os.environ.get("HF_TOKEN") or os.environ.get("HUGGING_FACE_HUB_TOKEN")


def _ram_available_gb():
    try:
        with open("/proc/meminfo") as handle:
            for line in handle:
                if line.startswith("MemAvailable:"):
                    return int(line.split()[1]) / (1024 * 1024)
    except OSError:
        return 0.0
    return 0.0


def _load_vector(filename, layer):
    path = os.path.join(VECTOR_DIR, filename)
    tensor = torch.load(path, map_location="cpu", weights_only=True)
    return tensor[layer].detach().float().contiguous()


def _load_model(hf_id, device):
    kwargs = {"low_cpu_mem_usage": True}
    token = _token()
    if token:
        kwargs["token"] = token
    try:
        model = AutoModelForCausalLM.from_pretrained(
            hf_id, dtype=torch.bfloat16, **kwargs
        )
    except TypeError:
        model = AutoModelForCausalLM.from_pretrained(
            hf_id, torch_dtype=torch.bfloat16, **kwargs
        )
    model.eval()
    if device == "cuda":
        model.to("cuda")
    tokenizer = AutoTokenizer.from_pretrained(hf_id, **kwargs)
    if tokenizer.pad_token_id is None:
        tokenizer.pad_token = tokenizer.eos_token
    return model, tokenizer


def _device_of(model):
    return next(model.parameters()).device.type


def _ensure(name, device):
    """Load once. Disk read happens outside the lock so Gemma can still generate."""
    while True:
        with _lock:
            if name in _resident:
                return _resident[name]
            if name not in _loading:
                _loading.add(name)
                loader = True
            else:
                loader = False
            if not loader:
                _cv.wait()
                continue
        t0 = time.perf_counter()
        print(f"loading {name} onto {device}", flush=True)
        try:
            model, tokenizer = _load_model(MODELS[name]["hf"], device)
            spec = MODELS[name]
            bundle = {
                "name": name,
                "model": model,
                "tokenizer": tokenizer,
                "clean": _load_vector(spec["clean"], spec["layer"]),
                "raw": _load_vector(spec["raw"], spec["layer"]),
            }
        except Exception:
            with _lock:
                _loading.discard(name)
                _cv.notify_all()
            raise
        with _lock:
            _resident.setdefault(name, bundle)
            _loading.discard(name)
            _cv.notify_all()
            print(f"loaded {name} in {time.perf_counter() - t0:.1f}s", flush=True)
            return _resident[name]


def _nbytes(model):
    return sum(p.numel() * p.element_size() for p in model.parameters())


def _park_others(name):
    for other, bundle in _resident.items():
        if other != name and _device_of(bundle["model"]) == "cuda":
            t0 = time.perf_counter()
            bundle["model"].to("cpu")
            torch.cuda.empty_cache()
            print(f"parked {other} on cpu in {time.perf_counter() - t0:.1f}s", flush=True)


def _on_gpu(name):
    """Caller holds _lock. Leave the other model on the GPU when both fit."""
    bundle = _resident[name]
    if _device_of(bundle["model"]) == "cuda":
        return bundle
    free, _total = torch.cuda.mem_get_info()
    spare = int(2.5 * 1024**3)
    if free < _nbytes(bundle["model"]) + spare:
        _park_others(name)
    t0 = time.perf_counter()
    try:
        bundle["model"].to("cuda")
        torch.cuda.synchronize()
    except torch.cuda.OutOfMemoryError:
        torch.cuda.empty_cache()
        _park_others(name)
        bundle["model"].to("cuda")
        torch.cuda.synchronize()
    print(f"moved {name} to gpu in {time.perf_counter() - t0:.1f}s", flush=True)
    return bundle


def activate(name):
    _ensure(name, "cpu")
    with _lock:
        return _on_gpu(name)


def _warm():
    try:
        _ensure(GEMMA_NAME, "cuda")
        free = _ram_available_gb()
        print(f"ram available {free:.1f} GB", flush=True)
        if free < 16:
            print("skipped qwen prefetch, not enough free ram", flush=True)
            return
        _ensure(QWEN_NAME, "cpu")
        with _lock:
            _on_gpu(QWEN_NAME)
            gpu_free, gpu_total = torch.cuda.mem_get_info()
        print(
            f"gpu free {gpu_free / 1024**3:.1f} / {gpu_total / 1024**3:.1f} GB",
            flush=True,
        )
    except Exception as exc:
        print(f"warm failed: {exc}", flush=True)


def alpha_caption(model_name, alpha):
    spec = MODELS[model_name]
    alpha = float(alpha)
    if alpha < -0.05:
        meaning = "Dial turned left. Analogies should thin out."
    elif abs(alpha) < 0.05:
        meaning = "Dial straight up. This is the normal answer, with no steering."
    elif alpha >= spec["paper_alpha"] + 0.4:
        meaning = spec["high_alpha"]
    else:
        meaning = "Dial turned right. The question never says “analogy,” but the answer should reach for one."
    return f"**{meaning}**\n\nα = {alpha:.2f}"


def preset_alpha(model_name, which):
    spec = MODELS[model_name]
    if which == "less":
        alpha = -1.0
    elif which == "off":
        alpha = 0.0
    elif which == "on":
        alpha = spec["paper_alpha"]
    else:
        alpha = 2.0 if model_name.startswith("Gemma") else 2.5
    return alpha, alpha_caption(model_name, alpha)


def _format_prompt(tokenizer, question):
    messages = [{"role": "user", "content": question.strip()}]
    return tokenizer.apply_chat_template(
        messages, tokenize=False, add_generation_prompt=True
    )


def _complete(bundle, spec, question, alpha, use_raw):
    model = bundle["model"]
    tokenizer = bundle["tokenizer"]
    vector = bundle["raw"] if use_raw else bundle["clean"]
    prompt = _format_prompt(tokenizer, question)
    inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
    layer = model.model.layers[spec["layer"]]

    def hook(_module, _inputs, output):
        if not torch.is_tensor(output):
            return output
        shift = vector.to(device=output.device, dtype=output.dtype)
        return output + alpha * shift

    handle = None
    if abs(alpha) > 1e-8:
        handle = layer.register_forward_hook(hook)
    try:
        with torch.inference_mode():
            out = model.generate(
                **inputs,
                max_new_tokens=MAX_NEW_TOKENS,
                do_sample=True,
                temperature=0.2,
                top_p=0.95,
                pad_token_id=tokenizer.pad_token_id,
            )
    finally:
        if handle is not None:
            handle.remove()
    prompt_len = inputs["input_ids"].shape[-1]
    return tokenizer.decode(out[0, prompt_len:], skip_special_tokens=True).strip()


def _esc(text):
    return html.escape(text or "").replace("\n", "<br>")


def _bubble(kind, title, body):
    return (
        f'<article class="cmp {kind}">'
        f'<div class="cmp-label">{title}</div>'
        f'<div class="cmp-body">{_esc(body) or "(empty generation)"}</div>'
        f"</article>"
    )


def _turn_html(question, model_name, alpha, use_raw, base, steered):
    spec = MODELS[model_name]
    kind = "raw vector" if use_raw else "clean vector"
    user = f'<div class="user-row"><div class="user-bubble">{_esc(question)}</div></div>'
    meta = f'{html.escape(model_name)} · layer {spec["layer"]} · {kind}'
    if steered is None:
        cards = _bubble("base", "α = 0 · dial is off, so this is the only answer", base)
    else:
        sign = "+" if alpha > 0 else ""
        cards = (
            _bubble("base", "α = 0 · no steering", base)
            + _bubble("steered", f"α = {sign}{alpha:.2f} · steered", steered)
        )
    return (
        f'<section class="turn">{user}'
        f'<div class="asst"><div class="asst-meta">{meta}</div>'
        f'<div class="cmp-row">{cards}</div></div></section>'
    )


POSTER_HTML = """
<section class="guide">
  <p class="guide-kicker">How to ask</p>
  <p>Type the concept you most want explained, and ask for a literal answer. Accurate technical terminology is a good phrase. Do not say “analogy,” “imagine,” or “like.”</p>
  <p>The prompt stays literal. The dial is what makes the model reach for an analogy.</p>
</section>
<section class="turn poster-turn">
  <div class="poster-tag">On the poster · Gemma-2-27B-IT · layer 16 · not the live model</div>
  <div class="user-row"><div class="user-bubble">Explain credit card cancellation affecting credit score using accurate technical terminology.</div></div>
  <div class="asst">
    <div class="asst-meta">Same prompt, two answers. The prompt asks for technical terms.</div>
    <div class="cmp-row">
      <article class="cmp base">
        <div class="cmp-label">α = 0 · no steering</div>
        <div class="cmp-body">Cancelling a credit card affects your credit score through two primary mechanisms: (1) credit utilization ratio … (2) credit history length …</div>
      </article>
      <article class="cmp steered">
        <div class="cmp-label">α = +1.0 · steered</div>
        <div class="cmp-body"><b>Think of</b> your credit report <b>like</b> a financial resume … Cancel a card and you're <b>shrinking the pie</b>: the same balance now represents a larger fraction …</div>
      </article>
    </div>
  </div>
</section>
"""


def generate(model_name, question, alpha, use_raw):
    question = (question or "").strip()
    if not question:
        return ""
    if len(question) > 1500:
        question = question[:1500]
    if not torch.cuda.is_available():
        return _turn_html(
            question, model_name, float(alpha), use_raw,
            "This Space needs the L4 GPU. The hardware is still starting, or it is asleep.",
            None,
        )
    spec = MODELS[model_name]
    alpha = float(alpha)
    _ensure(model_name, "cpu")
    with _lock:
        bundle = _on_gpu(model_name)
        try:
            base = _complete(bundle, spec, question, 0.0, use_raw)
            steered = None if abs(alpha) < 1e-8 else _complete(bundle, spec, question, alpha, use_raw)
        except torch.cuda.OutOfMemoryError:
            print("generation ran out of GPU memory; parking the other model and retrying", flush=True)
            torch.cuda.empty_cache()
            _park_others(model_name)
            base = _complete(bundle, spec, question, 0.0, use_raw)
            steered = None if abs(alpha) < 1e-8 else _complete(bundle, spec, question, alpha, use_raw)
    return _turn_html(question, model_name, alpha, use_raw, base, steered)


def on_model_change(model_name):
    threading.Thread(target=lambda: activate(model_name), daemon=True).start()
    alpha = MODELS[model_name]["paper_alpha"]
    return alpha, alpha_caption(model_name, alpha)


DIAL_CSS = """
@import url('https://fonts.googleapis.com/css2?family=Source+Sans+3:wght@400;500;600&family=Source+Serif+4:opsz,wght@8..60,480;8..60,560&display=swap');
.gradio-container, .gradio-container * {
  font-family: "Source Sans 3", "Segoe UI", sans-serif !important;
}
.gradio-container h1, .gradio-container h1 * {
  font-family: "Source Serif 4", Georgia, serif !important;
  font-weight: 480 !important;
  letter-spacing: -0.03em !important;
  font-size: 2.35rem !important;
  line-height: 1.15 !important;
}
.gradio-container, .gradio-container.dark, .dark {
  color-scheme: light !important;
  --body-background-fill: #faf9f5;
  --body-text-color: #141413;
  --body-text-color-subdued: #6b6560;
  --background-fill-primary: #faf9f5;
  --background-fill-secondary: #faf9f5;
  --block-background-fill: transparent;
  --block-border-color: transparent;
  --block-label-text-color: #6b6560;
  --block-title-text-color: #141413;
  --input-background-fill: #ffffff;
  --input-border-color: #e8e6dc;
  --input-placeholder-color: #8a847c;
  --button-secondary-background-fill: transparent;
  --button-secondary-text-color: #141413;
  --button-secondary-border-color: #e8e6dc;
  --button-primary-text-color: #ffffff;
  --border-color-primary: #e8e6dc;
  --neutral-50: #faf9f5;
  --neutral-100: #f0eee6;
  --neutral-200: #e8e6dc;
  --neutral-400: #8a847c;
  --neutral-600: #6b6560;
  --neutral-800: #141413;
  --neutral-950: #141413;
}
.gradio-container { background: #faf9f5 !important; color: #141413 !important; }
textarea, input, select, .prose, .prose p, .prose span, label, .cmp-body, .cmp-body b, .user-bubble, .guide, .guide p, .guide b {
  color: #141413 !important;
}
.guide-kicker, .cmp.steered .cmp-label { color: #c96442 !important; }
textarea, input[type="text"], input[type="number"] {
  background: #ffffff !important;
  border: 1px solid #e8e6dc !important;
  border-radius: 22px !important;
}
button { border-radius: 999px !important; box-shadow: none !important; }
button.primary, button.primary * { color: #ffffff !important; background: #c96442 !important; }
#dial-col .block, #dial-col fieldset, #chat-col .block {
  border: none !important;
  background: transparent !important;
  box-shadow: none !important;
}
#layout { flex-wrap: nowrap !important; align-items: flex-start !important; gap: 36px !important; }
#chat-col { min-width: 0 !important; flex: 1 1 auto !important; }
#dial-col { flex: 0 0 300px !important; width: 300px !important; position: sticky; top: 12px; }
#chat-log { min-height: 58vh; max-height: 64vh; overflow-y: auto; padding-right: 8px; }
.guide { background: transparent; border: none; padding: 0 0 4px; margin: 0 0 18px; }
.guide-kicker { font-family: "Source Sans 3", sans-serif !important; font-size: 13px; letter-spacing: 0.04em; text-transform: none; color: #c96442; margin: 0 0 6px; font-weight: 600; }
.guide p { margin: 0 0 8px; line-height: 1.55; font-size: 17px; }
.guide p:last-child { margin: 0; }
#dial-col .warm-note, #dial-col .warm-note p { color: #6b6560 !important; font-size: 13.5px !important; line-height: 1.45 !important; margin: 0 0 8px !important; }
.turn { margin: 8px 0 28px; }
.poster-tag { font-size: 13px; color: #6b6560; margin-bottom: 10px; }
.user-row { display: flex; justify-content: flex-end; margin: 10px 0 14px; }
.user-bubble { background: #f0eee6; color: #141413; border-radius: 22px; padding: 12px 16px; max-width: 80%; line-height: 1.5; font-size: 16.5px; }
.asst-meta { font-size: 13px; color: #6b6560; margin: 0 0 12px; }
.cmp-row { display: grid; grid-template-columns: 1fr 1.15fr; gap: 28px; }
.cmp { border: none; border-radius: 0; padding: 0; background: transparent; min-width: 0; }
.cmp.base .cmp-body { color: #6b6560 !important; }
.cmp.steered { border-left: 2px solid #e7cbbd; padding-left: 16px; }
.cmp-label { font-size: 13px; font-weight: 600; letter-spacing: 0; color: #6b6560; margin-bottom: 8px; }
.cmp.steered .cmp-label { color: #c96442; }
.cmp-body { font-size: 16.5px; line-height: 1.6; color: #141413; }
.poster-turn { border-top: none; padding-top: 8px; }
#dial-stack { position: relative; width: min(240px, 100%); margin: 4px auto 0; touch-action: none; user-select: none; }
#dial { cursor: grab; touch-action: none; }
#dial:active { cursor: grabbing; }
#dial svg { width: 100%; height: auto; display: block; }
#dial-readout {
  position: absolute; left: 0; right: 0; top: 62%;
  text-align: center; pointer-events: none;
  font-variant-numeric: tabular-nums;
  font-size: 1.25rem; font-weight: 600; letter-spacing: -0.03em; color: #141413;
}
#dial-ends { display: flex; justify-content: space-between; width: min(240px, 100%); margin: 0 auto 10px; color: #6b6560; font-size: 13px; }
@media (max-width: 860px) {
  #layout { flex-wrap: wrap !important; }
  #dial-col { flex: 1 1 100% !important; width: 100% !important; position: static; }
  .cmp-row { grid-template-columns: 1fr; }
}
"""

DIAL_HTML = """
<div id="dial-stack">
  <div id="dial" role="slider" aria-label="Analogy dial" aria-valuemin="-1.5" aria-valuemax="2.5" aria-valuenow="1" tabindex="0">
    <svg viewBox="0 0 200 200">
      <circle cx="100" cy="100" r="96" fill="#faf9f5" stroke="#e8e6dc" stroke-width="2"/>
      <path d="M34 146 A78 78 0 1 1 166 146" fill="none" stroke="#e8e6dc" stroke-width="10" stroke-linecap="round"/>
      <path d="M100 22 A78 78 0 0 1 166 146" fill="none" stroke="#c96442" stroke-width="10" stroke-linecap="round" opacity="0.9"/>
      <path d="M34 146 A78 78 0 0 1 100 22" fill="none" stroke="#8aa4b8" stroke-width="10" stroke-linecap="round" opacity="0.95"/>
      <g id="dial-needle">
        <line x1="100" y1="108" x2="100" y2="30" stroke="#141413" stroke-width="4" stroke-linecap="round"/>
        <circle cx="100" cy="30" r="5.5" fill="#c96442"/>
      </g>
      <circle cx="100" cy="100" r="7" fill="#141413"/>
    </svg>
  </div>
  <div id="dial-readout">+1.00</div>
</div>
<div id="dial-ends"><span>less analogy</span><span>off</span><span>more analogy</span></div>
"""

DIAL_JS = r"""
() => {
  const MIN = -1.5, MAX = 2.5, STEP = 0.25, SWEEP = 135;
  const boot = () => {
    const dial = document.getElementById("dial");
    const needle = document.getElementById("dial-needle");
    const readout = document.getElementById("dial-readout");
    const range = document.querySelector("#alpha-slider input[type='range']");
    if (!dial || !needle || !range) { setTimeout(boot, 200); return; }

    const snap = (v) => Math.max(MIN, Math.min(MAX, Math.round(v / STEP) * STEP));
    const toAngle = (v) => v <= 0 ? (v / 1.5) * SWEEP : (v / 2.5) * SWEEP;
    const fromAngle = (deg) => {
      deg = Math.max(-SWEEP, Math.min(SWEEP, deg));
      return deg <= 0 ? (deg / SWEEP) * 1.5 : (deg / SWEEP) * 2.5;
    };
    const paint = (v) => {
      needle.setAttribute("transform", "rotate(" + toAngle(v) + " 100 100)");
      const text = (v > 0 ? "+" : "") + v.toFixed(2);
      readout.textContent = Math.abs(v) < 0.001 ? "off" : text;
      dial.setAttribute("aria-valuenow", v.toFixed(2));
    };
    const write = (v) => {
      v = snap(v);
      range.value = String(v);
      const num = document.querySelector("#alpha-slider input[type='number']");
      if (num) num.value = String(v);
      range.dispatchEvent(new Event("input", { bubbles: true }));
      range.dispatchEvent(new Event("change", { bubbles: true }));
      paint(v);
    };
    const pointerValue = (e) => {
      const box = dial.getBoundingClientRect();
      const dx = e.clientX - (box.left + box.width / 2);
      const dy = e.clientY - (box.top + box.height / 2);
      return fromAngle(Math.atan2(dx, -dy) * 180 / Math.PI);
    };
    let dragging = false;
    dial.addEventListener("pointerdown", (e) => {
      dragging = true;
      dial.setPointerCapture(e.pointerId);
      write(pointerValue(e));
      e.preventDefault();
    });
    dial.addEventListener("pointermove", (e) => { if (dragging) write(pointerValue(e)); });
    dial.addEventListener("pointerup", () => { dragging = false; });
    dial.addEventListener("pointercancel", () => { dragging = false; });
    let seen = null;
    setInterval(() => {
      const v = parseFloat(range.value);
      if (Number.isNaN(v) || v === seen) return;
      seen = v;
      paint(v);
    }, 120);
    paint(parseFloat(range.value) || 1);
    const log = document.getElementById("chat-log");
    if (log && !log.dataset.watch) {
      log.dataset.watch = "1";
      new MutationObserver(() => { log.scrollTop = log.scrollHeight; })
        .observe(log, { childList: true, subtree: true });
    }
  };
  boot();
}
"""


def build_demo():
    import gradio as gr

    theme = gr.themes.Base(
        primary_hue=gr.themes.colors.orange,
        neutral_hue=gr.themes.colors.stone,
        font=gr.themes.GoogleFont("Source Sans 3"),
    ).set(
        body_background_fill="#faf9f5",
        body_text_color="#141413",
        block_background_fill="#faf9f5",
        block_border_color="#e8e6dc",
        block_label_text_color="#6b6560",
        input_background_fill="#ffffff",
        input_background_fill_dark="#ffffff",
        body_background_fill_dark="#faf9f5",
        body_text_color_dark="#141413",
        body_text_color_subdued="#6b6560",
        body_text_color_subdued_dark="#6b6560",
        block_background_fill_dark="#ffffff",
        block_border_color_dark="#e8e6dc",
        block_label_text_color_dark="#6b6560",
        block_title_text_color="#141413",
        block_title_text_color_dark="#141413",
        input_placeholder_color="#8a847c",
        input_placeholder_color_dark="#8a847c",
        button_primary_background_fill="#c96442",
        button_primary_background_fill_dark="#c96442",
        button_primary_background_fill_hover="#b55638",
        button_primary_background_fill_hover_dark="#b55638",
        button_primary_text_color="#ffffff",
        button_primary_text_color_dark="#ffffff",
        button_secondary_background_fill="#f0eee6",
        button_secondary_background_fill_dark="#f0eee6",
        button_secondary_text_color="#141413",
        button_secondary_text_color_dark="#141413",
    )
    with gr.Blocks(title="Analogy Vector", theme=theme, css=DIAL_CSS, js=DIAL_JS) as demo:
        gr.Markdown(
            "# Analogy Vector\n"
            "Ask for a literal explanation. The dial, not the prompt, brings the analogy in."
        )
        with gr.Row(elem_id="layout"):
            with gr.Column(scale=3, elem_id="chat-col"):
                chat = gr.HTML(POSTER_HTML, elem_id="chat-log")
                with gr.Row():
                    question = gr.Textbox(
                        show_label=False,
                        lines=2,
                        scale=5,
                        placeholder="Type a concept you want explained. Ask for a literal, technical answer — do not mention analogy.",
                    )
                    send = gr.Button("Send", variant="primary", scale=1)
                gr.Examples(
                    examples=[
                        ["Explain credit card cancellation affecting credit score using accurate technical terminology."],
                        ["Explain how TCP/IP handshakes work using accurate technical terminology."],
                        ["Explain how a barcode works in literal terms."],
                        ["Explain why the sky is blue using accurate technical terminology."],
                    ],
                    inputs=question,
                    label="Try a literal question",
                )
            with gr.Column(scale=2, elem_id="dial-col"):
                model_name = gr.Radio(
                    choices=[
                        ("Gemma-2-2B  ·  turns up, then breaks", "Gemma-2-2B-IT"),
                        ("Qwen-2.5-7B  ·  keeps growing", "Qwen-2.5-7B-Instruct"),
                    ],
                    value="Gemma-2-2B-IT",
                    label="Live model",
                )
                gr.Markdown(
                    "Both models stay on the GPU once the page has been awake for about two minutes. Switching is then immediate.",
                    elem_classes=["warm-note"],
                )
                gr.HTML(DIAL_HTML)
                with gr.Row():
                    less = gr.Button("Less")
                    off = gr.Button("Off")
                    on = gr.Button("Analogy", variant="primary")
                    far = gr.Button("Far")
                alpha = gr.Slider(
                    minimum=-1.5, maximum=2.5, step=0.25, value=1.0,
                    label="Fine tune", elem_id="alpha-slider",
                )
                caption = gr.Markdown(alpha_caption("Gemma-2-2B-IT", 1.0))
                use_raw = gr.Checkbox(
                    value=False,
                    label="Raw vector — at Far, Gemma loops the word Imagine",
                )

        def respond(model, q, a, raw, log):
            piece = generate(model, q, a, raw)
            if not piece:
                return log, q
            return (log or "") + piece, ""

        model_name.change(on_model_change, model_name, [alpha, caption])
        alpha.change(alpha_caption, [model_name, alpha], caption)
        less.click(lambda m: preset_alpha(m, "less"), model_name, [alpha, caption])
        off.click(lambda m: preset_alpha(m, "off"), model_name, [alpha, caption])
        on.click(lambda m: preset_alpha(m, "on"), model_name, [alpha, caption])
        far.click(lambda m: preset_alpha(m, "far"), model_name, [alpha, caption])
        question.submit(respond, [model_name, question, alpha, use_raw, chat], [chat, question])
        send.click(respond, [model_name, question, alpha, use_raw, chat], [chat, question])
    return demo


def smoke():
    os.environ.setdefault("HF_HOME", "/mnt/drive_b/hf_cache")
    os.environ.setdefault("HF_HUB_CACHE", "/mnt/drive_b/hf_cache/hub")
    question = "Explain how a barcode works."
    for name, alpha in (("Gemma-2-2B-IT", 0.0), ("Gemma-2-2B-IT", 1.0), ("Qwen-2.5-7B-Instruct", 2.0)):
        print("\n====", name, "alpha", alpha, "====", flush=True)
        print(generate(name, question, alpha, False), flush=True)


if __name__ == "__main__":
    if os.environ.get("STEER_SMOKE") == "1":
        smoke()
    else:
        threading.Thread(target=_warm, daemon=True).start()
        build_demo().queue(default_concurrency_limit=1).launch()