"""DEBRIS models on ZeroGPU (spec 5.6 path A, D72). One endpoint, `generate`, that takes a JSON object {"model": "tagger" | "ask", "system": str, "prompts": [str, ...], "temperature": float, "seed": int | null, "max_new_tokens": int} and returns a JSON list of {"text": str, "token_prob": float}, one per prompt. The Space only generates. The droplet builds the prompts and validates the answers with the same code the Studio uses (pipeline/tag.py Recipe, pipeline/ask.py check), so a tag written on path A is held to exactly the rules of one written on path B, and changing those rules never needs a Space rebuild. `tagger` is debris_v2 (Qwen2.5-3B-Instruct with the adopted LoRA, fused). `ask` is Qwen3-4B-Instruct-2507, which writes the DEBRIS LLM readings. Both load once at import in bf16; only generation holds a GPU. Pattern lifted from chatMCD/space/app.py (see REUSE.md). """ from __future__ import annotations import json import os import gradio as gr import spaces import torch from transformers import AutoModelForCausalLM, AutoTokenizer MODELS = { "tagger": os.environ.get("DEBRIS_MODEL", "Dellboy/debris-tagger-v2"), "ask": os.environ.get("DEBRIS_ASK_MODEL", "Qwen/Qwen3-4B-Instruct-2507"), } MAX_NEW_TOKENS = 600 BATCH_LIMIT = 16 TOKEN = os.environ.get("HF_TOKEN") def load(model_id: str): """Tokenizer and bf16 model, tolerating either spelling of the dtype argument (transformers renamed `torch_dtype` to `dtype`).""" tokenizer = AutoTokenizer.from_pretrained(model_id, token=TOKEN) common = {"device_map": "cuda", "token": TOKEN} try: model = AutoModelForCausalLM.from_pretrained(model_id, dtype=torch.bfloat16, **common) except TypeError: model = AutoModelForCausalLM.from_pretrained(model_id, torch_dtype=torch.bfloat16, **common) model.eval() return tokenizer, model LOADED = {name: load(model_id) for name, model_id in MODELS.items()} @spaces.GPU(duration=150) def run(name: str, system: str, prompts: list[str], temperature: float, seed: int | None, max_new_tokens: int) -> list[dict]: tokenizer, model = LOADED[name] if seed is not None: torch.manual_seed(seed) texts = [tokenizer.apply_chat_template( [{"role": "system", "content": system}, {"role": "user", "content": p}], add_generation_prompt=True, tokenize=False) for p in prompts] batch = tokenizer(texts, return_tensors="pt", padding=True, padding_side="left").to(model.device) # The models ship generation defaults (repetition penalty 1.05, top-k 20, top-p 0.8) # that the Studio's mlx sampler does not apply, so every one is set here: greedy, or # plain temperature sampling, as on path B. with torch.no_grad(): out = model.generate( **batch, max_new_tokens=max_new_tokens, do_sample=temperature > 0, temperature=temperature if temperature > 0 else 1.0, top_p=1.0, top_k=0, repetition_penalty=1.0, pad_token_id=tokenizer.pad_token_id or tokenizer.eos_token_id, return_dict_in_generate=True, output_logits=True) generated = out.sequences[:, batch["input_ids"].shape[1]:] # Mean probability of the tokens chosen, from the raw logits, stopping before the # end-of-turn token: the same measure the Studio's mlx path records (D45). logp = torch.stack(out.logits, dim=1).float().log_softmax(-1) chosen = logp.gather(-1, generated.unsqueeze(-1)).squeeze(-1) stops = model.generation_config.eos_token_id stops = set(stops if isinstance(stops, list) else [stops]) stops.add(tokenizer.pad_token_id) results = [] for row, scores in zip(generated.tolist(), chosen.tolist()): n = next((i for i, t in enumerate(row) if t in stops), len(row)) mean = sum(scores[:n]) / n if n else float("-inf") results.append({"text": tokenizer.decode(row[:n], skip_special_tokens=True), "token_prob": round(float(torch.tensor(mean).exp()), 4) if n else 0.0}) return results def generate(payload: str) -> str: """The endpoint the weekly run calls through gradio_client.""" try: job = json.loads(payload) if isinstance(payload, str) else payload except json.JSONDecodeError as exc: return json.dumps({"error": f"payload is not JSON: {exc}"}) if not isinstance(job, dict) or job.get("model") not in LOADED: return json.dumps({"error": f"payload must name a model: {sorted(LOADED)}"}) prompts = [str(p) for p in (job.get("prompts") or [])][:BATCH_LIMIT] if not prompts: return json.dumps([]) seed = job.get("seed") return json.dumps(run(job["model"], str(job.get("system") or ""), prompts, float(job.get("temperature") or 0.0), int(seed) if seed is not None else None, min(int(job.get("max_new_tokens") or 480), MAX_NEW_TOKENS))) with gr.Blocks(title="DEBRIS tagger") as demo: gr.Markdown( "# DEBRIS tagger\n\n" "Generation for [debris.mdeller.com](https://debris.mdeller.com): the weekly " "refresh sends batches of up to 16 prompts to the `generate` endpoint, for the " "tagger (`tagger`) or for the DEBRIS LLM readings (`ask`). The site validates " "every answer itself before storing it." ) payload_in = gr.Textbox(label='Job JSON: {"model": "tagger", "system": ..., "prompts": [...]}', lines=8) payload_out = gr.Textbox(label="Answers JSON", lines=14) gr.Button("Generate").click(generate, inputs=payload_in, outputs=payload_out, api_name="generate") if __name__ == "__main__": demo.queue(max_size=8).launch()