debris-tagger / app.py
Dellboy's picture
DEBRIS Space: generation only, debris_v2 tagger and the DEBRIS LLM writer (D72)
00abdf8 verified
Raw History Blame Contribute Delete
5.72 kB
"""DEBRIS models on ZeroGPU (spec 5.6 path A, D72).
One endpoint, `generate`, that takes a JSON object
{"model": "tagger" | "ask", "system": str, "prompts": [str, ...], "temperature": float,
"seed": int | null, "max_new_tokens": int}
and returns a JSON list of {"text": str, "token_prob": float}, one per prompt.
The Space only generates. The droplet builds the prompts and validates the answers with
the same code the Studio uses (pipeline/tag.py Recipe, pipeline/ask.py check), so a tag
written on path A is held to exactly the rules of one written on path B, and changing
those rules never needs a Space rebuild.
`tagger` is debris_v2 (Qwen2.5-3B-Instruct with the adopted LoRA, fused). `ask` is
Qwen3-4B-Instruct-2507, which writes the DEBRIS LLM readings. Both load once at import
in bf16; only generation holds a GPU.
Pattern lifted from chatMCD/space/app.py (see REUSE.md).
"""
from __future__ import annotations
import json
import os
import gradio as gr
import spaces
import torch
from transformers import AutoModelForCausalLM, AutoTokenizer
MODELS = {
"tagger": os.environ.get("DEBRIS_MODEL", "Dellboy/debris-tagger-v2"),
"ask": os.environ.get("DEBRIS_ASK_MODEL", "Qwen/Qwen3-4B-Instruct-2507"),
}
MAX_NEW_TOKENS = 600
BATCH_LIMIT = 16
TOKEN = os.environ.get("HF_TOKEN")
def load(model_id: str):
"""Tokenizer and bf16 model, tolerating either spelling of the dtype argument
(transformers renamed `torch_dtype` to `dtype`)."""
tokenizer = AutoTokenizer.from_pretrained(model_id, token=TOKEN)
common = {"device_map": "cuda", "token": TOKEN}
try:
model = AutoModelForCausalLM.from_pretrained(model_id, dtype=torch.bfloat16, **common)
except TypeError:
model = AutoModelForCausalLM.from_pretrained(model_id, torch_dtype=torch.bfloat16, **common)
model.eval()
return tokenizer, model
LOADED = {name: load(model_id) for name, model_id in MODELS.items()}
@spaces.GPU(duration=150)
def run(name: str, system: str, prompts: list[str], temperature: float, seed: int | None,
max_new_tokens: int) -> list[dict]:
tokenizer, model = LOADED[name]
if seed is not None:
torch.manual_seed(seed)
texts = [tokenizer.apply_chat_template(
[{"role": "system", "content": system}, {"role": "user", "content": p}],
add_generation_prompt=True, tokenize=False) for p in prompts]
batch = tokenizer(texts, return_tensors="pt", padding=True,
padding_side="left").to(model.device)
# The models ship generation defaults (repetition penalty 1.05, top-k 20, top-p 0.8)
# that the Studio's mlx sampler does not apply, so every one is set here: greedy, or
# plain temperature sampling, as on path B.
with torch.no_grad():
out = model.generate(
**batch, max_new_tokens=max_new_tokens,
do_sample=temperature > 0, temperature=temperature if temperature > 0 else 1.0,
top_p=1.0, top_k=0, repetition_penalty=1.0,
pad_token_id=tokenizer.pad_token_id or tokenizer.eos_token_id,
return_dict_in_generate=True, output_logits=True)
generated = out.sequences[:, batch["input_ids"].shape[1]:]
# Mean probability of the tokens chosen, from the raw logits, stopping before the
# end-of-turn token: the same measure the Studio's mlx path records (D45).
logp = torch.stack(out.logits, dim=1).float().log_softmax(-1)
chosen = logp.gather(-1, generated.unsqueeze(-1)).squeeze(-1)
stops = model.generation_config.eos_token_id
stops = set(stops if isinstance(stops, list) else [stops])
stops.add(tokenizer.pad_token_id)
results = []
for row, scores in zip(generated.tolist(), chosen.tolist()):
n = next((i for i, t in enumerate(row) if t in stops), len(row))
mean = sum(scores[:n]) / n if n else float("-inf")
results.append({"text": tokenizer.decode(row[:n], skip_special_tokens=True),
"token_prob": round(float(torch.tensor(mean).exp()), 4) if n else 0.0})
return results
def generate(payload: str) -> str:
"""The endpoint the weekly run calls through gradio_client."""
try:
job = json.loads(payload) if isinstance(payload, str) else payload
except json.JSONDecodeError as exc:
return json.dumps({"error": f"payload is not JSON: {exc}"})
if not isinstance(job, dict) or job.get("model") not in LOADED:
return json.dumps({"error": f"payload must name a model: {sorted(LOADED)}"})
prompts = [str(p) for p in (job.get("prompts") or [])][:BATCH_LIMIT]
if not prompts:
return json.dumps([])
seed = job.get("seed")
return json.dumps(run(job["model"], str(job.get("system") or ""), prompts,
float(job.get("temperature") or 0.0),
int(seed) if seed is not None else None,
min(int(job.get("max_new_tokens") or 480), MAX_NEW_TOKENS)))
with gr.Blocks(title="DEBRIS tagger") as demo:
gr.Markdown(
"# DEBRIS tagger\n\n"
"Generation for [debris.mdeller.com](https://debris.mdeller.com): the weekly "
"refresh sends batches of up to 16 prompts to the `generate` endpoint, for the "
"tagger (`tagger`) or for the DEBRIS LLM readings (`ask`). The site validates "
"every answer itself before storing it."
)
payload_in = gr.Textbox(label='Job JSON: {"model": "tagger", "system": ..., "prompts": [...]}', lines=8)
payload_out = gr.Textbox(label="Answers JSON", lines=14)
gr.Button("Generate").click(generate, inputs=payload_in, outputs=payload_out,
api_name="generate")
if __name__ == "__main__":
demo.queue(max_size=8).launch()