"""Block diffusion adapted from the pinned official RemeDi inference implementation.""" from dataclasses import asdict, dataclass import time import torch from remedi.modelling_remedi_bitowel import DynamicCache MODEL_ID = "maple-research-lab/RemeDi-Instruct" MODEL_REVISION = "2a8d6c1bad0892fa50de0c2aba3253fc4c88adf4" TOKENIZER_ID = "GSAI-ML/LLaDA-8B-Instruct" POLICIES = ["Official raw logits", "Learned confidence (experimental)", "Token probability (experimental)"] @dataclass(frozen=True) class Settings: max_tokens: int = 128 block_size: int = 32 steps: int = 16 policy: str = POLICIES[0] seed: int = 112 trace: bool = True def validate(self): for name in ("max_tokens", "block_size", "steps", "seed"): if type(getattr(self, name)) is not int: raise ValueError(f"{name} must be an integer") if not 1 <= self.max_tokens <= 256: raise ValueError("Output length must be 1–256 tokens") if self.block_size not in (16, 32, 64): raise ValueError("Block size must be 16, 32, or 64") if not 1 <= self.steps <= 64: raise ValueError("Steps must be 1–64") if self.policy not in POLICIES: raise ValueError("Unknown ranking policy") # Bound the full attention/forward cost of each ZeroGPU request. if ((self.max_tokens + self.block_size - 1) // self.block_size) * self.steps > 256: raise ValueError("Too many model passes; reduce tokens or steps (maximum 256 denoising passes)") return self def schedule(length, steps): """Match upstream cumulative transfer counts, including steps > block length.""" counts = [length // steps] * steps remainder = length % steps if remainder: for i in range(steps - remainder, steps): counts[i] += 1 total = 0 result = [] for n in counts: total += n result.append(total) return result def prompt_ids(tokenizer, prompt, system=""): if not isinstance(prompt, str) or not prompt.strip(): raise ValueError("Enter a prompt") if len(prompt) + len(system) > 16000: raise ValueError("Prompt is too long (16,000 character limit)") messages = [] if system.strip(): messages.append({"role": "system", "content": system.strip()}) messages.append({"role": "user", "content": prompt.strip()}) ids = tokenizer.apply_chat_template(messages, add_generation_prompt=True, return_tensors="pt") if ids.ndim != 2 or ids.shape[0] != 1 or ids.shape[1] == 0: raise ValueError("Tokenizer returned an invalid prompt") if ids.shape[1] > 1024: raise ValueError(f"Prompt is {ids.shape[1]} tokens; maximum 1024. It was not truncated.") return ids def rank_candidates(out, candidates, policy): logits = out.logits.float() if not torch.isfinite(logits).all(): raise RuntimeError("Model produced non-finite logits") if policy == POLICIES[1]: scores = out.confidences.float() else: scores = logits.gather(-1, candidates.unsqueeze(-1)).squeeze(-1) if policy == POLICIES[2]: scores = scores - logits.logsumexp(dim=-1) if scores.shape != candidates.shape or not torch.isfinite(scores).all(): raise RuntimeError("Invalid ranking scores") return scores @torch.inference_mode() def generate(model, tokenizer, ids, settings, device="cuda", deadline_seconds=110): settings.validate() started = time.perf_counter() if ids.shape[1] + settings.max_tokens > model.config.max_sequence_length: raise ValueError("Prompt plus output exceeds model context") mask_id = int(getattr(model.config, "mask_token_id", 126336)) eos_id = int(getattr(model.config, "eos_token_id", 126081)) # Request-local cache; never reuse one visitor's prefix for another visitor. cache = DynamicCache() ids = ids.to(device) generated, trace = [], [] forwards = 0 def forward(x, positions=None, update=False): nonlocal forwards if time.perf_counter() - started > deadline_seconds: raise TimeoutError("Request exceeded its runtime budget; reduce output length or steps") kwargs = dict(kv_cache=cache, update_kv_cache=update) if positions is not None: kwargs["position_ids"] = positions with torch.autocast(device_type="cuda", dtype=torch.bfloat16, enabled=str(device).startswith("cuda")): result = model(x, **kwargs) forwards += 1 return result # Do not inspect EOS in the prompt (the chat template contains EOS tokens). forward(ids, update=True) finish_reason = "length" remasked_total = 0 with torch.random.fork_rng(devices=[torch.device(device).index or 0] if str(device).startswith("cuda") else []): torch.manual_seed(settings.seed) for offset in range(0, settings.max_tokens, settings.block_size): length = min(settings.block_size, settings.max_tokens - offset) x = torch.full((1, length), mask_id, dtype=torch.long, device=device) positions = torch.arange(ids.shape[1] + offset, ids.shape[1] + offset + length, device=device)[None] for step, keep_count in enumerate(schedule(length, settings.steps)): previous = x.clone() out = forward(x, positions) logits = out.logits # Mask is an input sentinel, never a valid generated answer. logits[..., mask_id] = -torch.inf proposals = logits.argmax(dim=-1) candidates = torch.where(x == mask_id, proposals, x) # Permit the intentional -inf mask column in the finite check. logits[..., mask_id] = torch.finfo(logits.dtype).min scores = rank_candidates(out, candidates, settings.policy) selected = torch.topk(scores, k=keep_count, dim=-1).indices keep = torch.zeros_like(x, dtype=torch.bool).scatter_(1, selected, True) x = torch.where(keep, candidates, mask_id) remasked = ((previous != mask_id) & (x == mask_id)).sum().item() remasked_total += remasked if settings.trace: trace.append({"block": offset // settings.block_size + 1, "step": step + 1, "kept": keep_count, "remasked": remasked, "token_ids": x[0].tolist(), "text": tokenizer.decode(x[0].tolist(), skip_special_tokens=False)}) del out, logits tokens = x[0].tolist() if eos_id in tokens: generated.extend(tokens[:tokens.index(eos_id)]) finish_reason = "eos" break generated.extend(tokens) if offset + length < settings.max_tokens: forward(x, positions, update=True) if str(device).startswith("cuda"): torch.cuda.synchronize() elapsed = time.perf_counter() - started return {"text": tokenizer.decode(generated, skip_special_tokens=True), "token_ids": generated, "metrics": {"prompt_tokens": ids.shape[1], "output_tokens": len(generated), "seconds": round(elapsed, 3), "tokens_per_second": round(len(generated) / max(elapsed, 1e-9), 3), "forward_passes": forwards, "remasked_positions": remasked_total, "finish_reason": finish_reason}, "settings": asdict(settings), "trace": trace, "note": "Greedy token predictions. Seed is recorded; changing it does not create sampling diversity."}