Download engine.py from User-2468/Name: direct link, hf CLI and curl.
- Browser
- Download file 7.71 kB
-
https://huggingface.co/spaces/User-2468/Name/resolve/main/engine.py
- Command line
-
hf download hf://spaces/User-2468/Name/engine.py
-
curl -L -o engine.py https://huggingface.co/spaces/User-2468/Name/resolve/main/engine.py
7.71 kB
| """Block diffusion adapted from the pinned official RemeDi inference implementation.""" | |
| from dataclasses import asdict, dataclass | |
| import time | |
| import torch | |
| from remedi.modelling_remedi_bitowel import DynamicCache | |
| MODEL_ID = "maple-research-lab/RemeDi-Instruct" | |
| MODEL_REVISION = "2a8d6c1bad0892fa50de0c2aba3253fc4c88adf4" | |
| TOKENIZER_ID = "GSAI-ML/LLaDA-8B-Instruct" | |
| POLICIES = ["Official raw logits", "Learned confidence (experimental)", "Token probability (experimental)"] | |
| class Settings: | |
| max_tokens: int = 128 | |
| block_size: int = 32 | |
| steps: int = 16 | |
| policy: str = POLICIES[0] | |
| seed: int = 112 | |
| trace: bool = True | |
| def validate(self): | |
| for name in ("max_tokens", "block_size", "steps", "seed"): | |
| if type(getattr(self, name)) is not int: | |
| raise ValueError(f"{name} must be an integer") | |
| if not 1 <= self.max_tokens <= 256: | |
| raise ValueError("Output length must be 1–256 tokens") | |
| if self.block_size not in (16, 32, 64): | |
| raise ValueError("Block size must be 16, 32, or 64") | |
| if not 1 <= self.steps <= 64: | |
| raise ValueError("Steps must be 1–64") | |
| if self.policy not in POLICIES: | |
| raise ValueError("Unknown ranking policy") | |
| # Bound the full attention/forward cost of each ZeroGPU request. | |
| if ((self.max_tokens + self.block_size - 1) // self.block_size) * self.steps > 256: | |
| raise ValueError("Too many model passes; reduce tokens or steps (maximum 256 denoising passes)") | |
| return self | |
| def schedule(length, steps): | |
| """Match upstream cumulative transfer counts, including steps > block length.""" | |
| counts = [length // steps] * steps | |
| remainder = length % steps | |
| if remainder: | |
| for i in range(steps - remainder, steps): | |
| counts[i] += 1 | |
| total = 0 | |
| result = [] | |
| for n in counts: | |
| total += n | |
| result.append(total) | |
| return result | |
| def prompt_ids(tokenizer, prompt, system=""): | |
| if not isinstance(prompt, str) or not prompt.strip(): | |
| raise ValueError("Enter a prompt") | |
| if len(prompt) + len(system) > 16000: | |
| raise ValueError("Prompt is too long (16,000 character limit)") | |
| messages = [] | |
| if system.strip(): | |
| messages.append({"role": "system", "content": system.strip()}) | |
| messages.append({"role": "user", "content": prompt.strip()}) | |
| ids = tokenizer.apply_chat_template(messages, add_generation_prompt=True, return_tensors="pt") | |
| if ids.ndim != 2 or ids.shape[0] != 1 or ids.shape[1] == 0: | |
| raise ValueError("Tokenizer returned an invalid prompt") | |
| if ids.shape[1] > 1024: | |
| raise ValueError(f"Prompt is {ids.shape[1]} tokens; maximum 1024. It was not truncated.") | |
| return ids | |
| def rank_candidates(out, candidates, policy): | |
| logits = out.logits.float() | |
| if not torch.isfinite(logits).all(): | |
| raise RuntimeError("Model produced non-finite logits") | |
| if policy == POLICIES[1]: | |
| scores = out.confidences.float() | |
| else: | |
| scores = logits.gather(-1, candidates.unsqueeze(-1)).squeeze(-1) | |
| if policy == POLICIES[2]: | |
| scores = scores - logits.logsumexp(dim=-1) | |
| if scores.shape != candidates.shape or not torch.isfinite(scores).all(): | |
| raise RuntimeError("Invalid ranking scores") | |
| return scores | |
| def generate(model, tokenizer, ids, settings, device="cuda", deadline_seconds=110): | |
| settings.validate() | |
| started = time.perf_counter() | |
| if ids.shape[1] + settings.max_tokens > model.config.max_sequence_length: | |
| raise ValueError("Prompt plus output exceeds model context") | |
| mask_id = int(getattr(model.config, "mask_token_id", 126336)) | |
| eos_id = int(getattr(model.config, "eos_token_id", 126081)) | |
| # Request-local cache; never reuse one visitor's prefix for another visitor. | |
| cache = DynamicCache() | |
| ids = ids.to(device) | |
| generated, trace = [], [] | |
| forwards = 0 | |
| def forward(x, positions=None, update=False): | |
| nonlocal forwards | |
| if time.perf_counter() - started > deadline_seconds: | |
| raise TimeoutError("Request exceeded its runtime budget; reduce output length or steps") | |
| kwargs = dict(kv_cache=cache, update_kv_cache=update) | |
| if positions is not None: | |
| kwargs["position_ids"] = positions | |
| with torch.autocast(device_type="cuda", dtype=torch.bfloat16, enabled=str(device).startswith("cuda")): | |
| result = model(x, **kwargs) | |
| forwards += 1 | |
| return result | |
| # Do not inspect EOS in the prompt (the chat template contains EOS tokens). | |
| forward(ids, update=True) | |
| finish_reason = "length" | |
| remasked_total = 0 | |
| with torch.random.fork_rng(devices=[torch.device(device).index or 0] if str(device).startswith("cuda") else []): | |
| torch.manual_seed(settings.seed) | |
| for offset in range(0, settings.max_tokens, settings.block_size): | |
| length = min(settings.block_size, settings.max_tokens - offset) | |
| x = torch.full((1, length), mask_id, dtype=torch.long, device=device) | |
| positions = torch.arange(ids.shape[1] + offset, ids.shape[1] + offset + length, device=device)[None] | |
| for step, keep_count in enumerate(schedule(length, settings.steps)): | |
| previous = x.clone() | |
| out = forward(x, positions) | |
| logits = out.logits | |
| # Mask is an input sentinel, never a valid generated answer. | |
| logits[..., mask_id] = -torch.inf | |
| proposals = logits.argmax(dim=-1) | |
| candidates = torch.where(x == mask_id, proposals, x) | |
| # Permit the intentional -inf mask column in the finite check. | |
| logits[..., mask_id] = torch.finfo(logits.dtype).min | |
| scores = rank_candidates(out, candidates, settings.policy) | |
| selected = torch.topk(scores, k=keep_count, dim=-1).indices | |
| keep = torch.zeros_like(x, dtype=torch.bool).scatter_(1, selected, True) | |
| x = torch.where(keep, candidates, mask_id) | |
| remasked = ((previous != mask_id) & (x == mask_id)).sum().item() | |
| remasked_total += remasked | |
| if settings.trace: | |
| trace.append({"block": offset // settings.block_size + 1, "step": step + 1, | |
| "kept": keep_count, "remasked": remasked, | |
| "token_ids": x[0].tolist(), | |
| "text": tokenizer.decode(x[0].tolist(), skip_special_tokens=False)}) | |
| del out, logits | |
| tokens = x[0].tolist() | |
| if eos_id in tokens: | |
| generated.extend(tokens[:tokens.index(eos_id)]) | |
| finish_reason = "eos" | |
| break | |
| generated.extend(tokens) | |
| if offset + length < settings.max_tokens: | |
| forward(x, positions, update=True) | |
| if str(device).startswith("cuda"): | |
| torch.cuda.synchronize() | |
| elapsed = time.perf_counter() - started | |
| return {"text": tokenizer.decode(generated, skip_special_tokens=True), "token_ids": generated, | |
| "metrics": {"prompt_tokens": ids.shape[1], "output_tokens": len(generated), | |
| "seconds": round(elapsed, 3), "tokens_per_second": round(len(generated) / max(elapsed, 1e-9), 3), | |
| "forward_passes": forwards, "remasked_positions": remasked_total, | |
| "finish_reason": finish_reason}, | |
| "settings": asdict(settings), "trace": trace, | |
| "note": "Greedy token predictions. Seed is recorded; changing it does not create sampling diversity."} | |