Spaces:
Sleeping
Sleeping
Download app.py from Dellboy/debris-tagger: direct link, hf CLI and curl.
- Browser
- Download file 5.72 kB
-
https://huggingface.co/spaces/Dellboy/debris-tagger/resolve/main/app.py
- Command line
-
hf download hf://spaces/Dellboy/debris-tagger/app.py
-
curl -L -o app.py https://huggingface.co/spaces/Dellboy/debris-tagger/resolve/main/app.py
5.72 kB
| """DEBRIS models on ZeroGPU (spec 5.6 path A, D72). | |
| One endpoint, `generate`, that takes a JSON object | |
| {"model": "tagger" | "ask", "system": str, "prompts": [str, ...], "temperature": float, | |
| "seed": int | null, "max_new_tokens": int} | |
| and returns a JSON list of {"text": str, "token_prob": float}, one per prompt. | |
| The Space only generates. The droplet builds the prompts and validates the answers with | |
| the same code the Studio uses (pipeline/tag.py Recipe, pipeline/ask.py check), so a tag | |
| written on path A is held to exactly the rules of one written on path B, and changing | |
| those rules never needs a Space rebuild. | |
| `tagger` is debris_v2 (Qwen2.5-3B-Instruct with the adopted LoRA, fused). `ask` is | |
| Qwen3-4B-Instruct-2507, which writes the DEBRIS LLM readings. Both load once at import | |
| in bf16; only generation holds a GPU. | |
| Pattern lifted from chatMCD/space/app.py (see REUSE.md). | |
| """ | |
| from __future__ import annotations | |
| import json | |
| import os | |
| import gradio as gr | |
| import spaces | |
| import torch | |
| from transformers import AutoModelForCausalLM, AutoTokenizer | |
| MODELS = { | |
| "tagger": os.environ.get("DEBRIS_MODEL", "Dellboy/debris-tagger-v2"), | |
| "ask": os.environ.get("DEBRIS_ASK_MODEL", "Qwen/Qwen3-4B-Instruct-2507"), | |
| } | |
| MAX_NEW_TOKENS = 600 | |
| BATCH_LIMIT = 16 | |
| TOKEN = os.environ.get("HF_TOKEN") | |
| def load(model_id: str): | |
| """Tokenizer and bf16 model, tolerating either spelling of the dtype argument | |
| (transformers renamed `torch_dtype` to `dtype`).""" | |
| tokenizer = AutoTokenizer.from_pretrained(model_id, token=TOKEN) | |
| common = {"device_map": "cuda", "token": TOKEN} | |
| try: | |
| model = AutoModelForCausalLM.from_pretrained(model_id, dtype=torch.bfloat16, **common) | |
| except TypeError: | |
| model = AutoModelForCausalLM.from_pretrained(model_id, torch_dtype=torch.bfloat16, **common) | |
| model.eval() | |
| return tokenizer, model | |
| LOADED = {name: load(model_id) for name, model_id in MODELS.items()} | |
| def run(name: str, system: str, prompts: list[str], temperature: float, seed: int | None, | |
| max_new_tokens: int) -> list[dict]: | |
| tokenizer, model = LOADED[name] | |
| if seed is not None: | |
| torch.manual_seed(seed) | |
| texts = [tokenizer.apply_chat_template( | |
| [{"role": "system", "content": system}, {"role": "user", "content": p}], | |
| add_generation_prompt=True, tokenize=False) for p in prompts] | |
| batch = tokenizer(texts, return_tensors="pt", padding=True, | |
| padding_side="left").to(model.device) | |
| # The models ship generation defaults (repetition penalty 1.05, top-k 20, top-p 0.8) | |
| # that the Studio's mlx sampler does not apply, so every one is set here: greedy, or | |
| # plain temperature sampling, as on path B. | |
| with torch.no_grad(): | |
| out = model.generate( | |
| **batch, max_new_tokens=max_new_tokens, | |
| do_sample=temperature > 0, temperature=temperature if temperature > 0 else 1.0, | |
| top_p=1.0, top_k=0, repetition_penalty=1.0, | |
| pad_token_id=tokenizer.pad_token_id or tokenizer.eos_token_id, | |
| return_dict_in_generate=True, output_logits=True) | |
| generated = out.sequences[:, batch["input_ids"].shape[1]:] | |
| # Mean probability of the tokens chosen, from the raw logits, stopping before the | |
| # end-of-turn token: the same measure the Studio's mlx path records (D45). | |
| logp = torch.stack(out.logits, dim=1).float().log_softmax(-1) | |
| chosen = logp.gather(-1, generated.unsqueeze(-1)).squeeze(-1) | |
| stops = model.generation_config.eos_token_id | |
| stops = set(stops if isinstance(stops, list) else [stops]) | |
| stops.add(tokenizer.pad_token_id) | |
| results = [] | |
| for row, scores in zip(generated.tolist(), chosen.tolist()): | |
| n = next((i for i, t in enumerate(row) if t in stops), len(row)) | |
| mean = sum(scores[:n]) / n if n else float("-inf") | |
| results.append({"text": tokenizer.decode(row[:n], skip_special_tokens=True), | |
| "token_prob": round(float(torch.tensor(mean).exp()), 4) if n else 0.0}) | |
| return results | |
| def generate(payload: str) -> str: | |
| """The endpoint the weekly run calls through gradio_client.""" | |
| try: | |
| job = json.loads(payload) if isinstance(payload, str) else payload | |
| except json.JSONDecodeError as exc: | |
| return json.dumps({"error": f"payload is not JSON: {exc}"}) | |
| if not isinstance(job, dict) or job.get("model") not in LOADED: | |
| return json.dumps({"error": f"payload must name a model: {sorted(LOADED)}"}) | |
| prompts = [str(p) for p in (job.get("prompts") or [])][:BATCH_LIMIT] | |
| if not prompts: | |
| return json.dumps([]) | |
| seed = job.get("seed") | |
| return json.dumps(run(job["model"], str(job.get("system") or ""), prompts, | |
| float(job.get("temperature") or 0.0), | |
| int(seed) if seed is not None else None, | |
| min(int(job.get("max_new_tokens") or 480), MAX_NEW_TOKENS))) | |
| with gr.Blocks(title="DEBRIS tagger") as demo: | |
| gr.Markdown( | |
| "# DEBRIS tagger\n\n" | |
| "Generation for [debris.mdeller.com](https://debris.mdeller.com): the weekly " | |
| "refresh sends batches of up to 16 prompts to the `generate` endpoint, for the " | |
| "tagger (`tagger`) or for the DEBRIS LLM readings (`ask`). The site validates " | |
| "every answer itself before storing it." | |
| ) | |
| payload_in = gr.Textbox(label='Job JSON: {"model": "tagger", "system": ..., "prompts": [...]}', lines=8) | |
| payload_out = gr.Textbox(label="Answers JSON", lines=14) | |
| gr.Button("Generate").click(generate, inputs=payload_in, outputs=payload_out, | |
| api_name="generate") | |
| if __name__ == "__main__": | |
| demo.queue(max_size=8).launch() | |