File size: 7,713 Bytes
79df115 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 | """Block diffusion adapted from the pinned official RemeDi inference implementation."""
from dataclasses import asdict, dataclass
import time
import torch
from remedi.modelling_remedi_bitowel import DynamicCache
MODEL_ID = "maple-research-lab/RemeDi-Instruct"
MODEL_REVISION = "2a8d6c1bad0892fa50de0c2aba3253fc4c88adf4"
TOKENIZER_ID = "GSAI-ML/LLaDA-8B-Instruct"
POLICIES = ["Official raw logits", "Learned confidence (experimental)", "Token probability (experimental)"]
@dataclass(frozen=True)
class Settings:
max_tokens: int = 128
block_size: int = 32
steps: int = 16
policy: str = POLICIES[0]
seed: int = 112
trace: bool = True
def validate(self):
for name in ("max_tokens", "block_size", "steps", "seed"):
if type(getattr(self, name)) is not int:
raise ValueError(f"{name} must be an integer")
if not 1 <= self.max_tokens <= 256:
raise ValueError("Output length must be 1–256 tokens")
if self.block_size not in (16, 32, 64):
raise ValueError("Block size must be 16, 32, or 64")
if not 1 <= self.steps <= 64:
raise ValueError("Steps must be 1–64")
if self.policy not in POLICIES:
raise ValueError("Unknown ranking policy")
# Bound the full attention/forward cost of each ZeroGPU request.
if ((self.max_tokens + self.block_size - 1) // self.block_size) * self.steps > 256:
raise ValueError("Too many model passes; reduce tokens or steps (maximum 256 denoising passes)")
return self
def schedule(length, steps):
"""Match upstream cumulative transfer counts, including steps > block length."""
counts = [length // steps] * steps
remainder = length % steps
if remainder:
for i in range(steps - remainder, steps):
counts[i] += 1
total = 0
result = []
for n in counts:
total += n
result.append(total)
return result
def prompt_ids(tokenizer, prompt, system=""):
if not isinstance(prompt, str) or not prompt.strip():
raise ValueError("Enter a prompt")
if len(prompt) + len(system) > 16000:
raise ValueError("Prompt is too long (16,000 character limit)")
messages = []
if system.strip():
messages.append({"role": "system", "content": system.strip()})
messages.append({"role": "user", "content": prompt.strip()})
ids = tokenizer.apply_chat_template(messages, add_generation_prompt=True, return_tensors="pt")
if ids.ndim != 2 or ids.shape[0] != 1 or ids.shape[1] == 0:
raise ValueError("Tokenizer returned an invalid prompt")
if ids.shape[1] > 1024:
raise ValueError(f"Prompt is {ids.shape[1]} tokens; maximum 1024. It was not truncated.")
return ids
def rank_candidates(out, candidates, policy):
logits = out.logits.float()
if not torch.isfinite(logits).all():
raise RuntimeError("Model produced non-finite logits")
if policy == POLICIES[1]:
scores = out.confidences.float()
else:
scores = logits.gather(-1, candidates.unsqueeze(-1)).squeeze(-1)
if policy == POLICIES[2]:
scores = scores - logits.logsumexp(dim=-1)
if scores.shape != candidates.shape or not torch.isfinite(scores).all():
raise RuntimeError("Invalid ranking scores")
return scores
@torch.inference_mode()
def generate(model, tokenizer, ids, settings, device="cuda", deadline_seconds=110):
settings.validate()
started = time.perf_counter()
if ids.shape[1] + settings.max_tokens > model.config.max_sequence_length:
raise ValueError("Prompt plus output exceeds model context")
mask_id = int(getattr(model.config, "mask_token_id", 126336))
eos_id = int(getattr(model.config, "eos_token_id", 126081))
# Request-local cache; never reuse one visitor's prefix for another visitor.
cache = DynamicCache()
ids = ids.to(device)
generated, trace = [], []
forwards = 0
def forward(x, positions=None, update=False):
nonlocal forwards
if time.perf_counter() - started > deadline_seconds:
raise TimeoutError("Request exceeded its runtime budget; reduce output length or steps")
kwargs = dict(kv_cache=cache, update_kv_cache=update)
if positions is not None:
kwargs["position_ids"] = positions
with torch.autocast(device_type="cuda", dtype=torch.bfloat16, enabled=str(device).startswith("cuda")):
result = model(x, **kwargs)
forwards += 1
return result
# Do not inspect EOS in the prompt (the chat template contains EOS tokens).
forward(ids, update=True)
finish_reason = "length"
remasked_total = 0
with torch.random.fork_rng(devices=[torch.device(device).index or 0] if str(device).startswith("cuda") else []):
torch.manual_seed(settings.seed)
for offset in range(0, settings.max_tokens, settings.block_size):
length = min(settings.block_size, settings.max_tokens - offset)
x = torch.full((1, length), mask_id, dtype=torch.long, device=device)
positions = torch.arange(ids.shape[1] + offset, ids.shape[1] + offset + length, device=device)[None]
for step, keep_count in enumerate(schedule(length, settings.steps)):
previous = x.clone()
out = forward(x, positions)
logits = out.logits
# Mask is an input sentinel, never a valid generated answer.
logits[..., mask_id] = -torch.inf
proposals = logits.argmax(dim=-1)
candidates = torch.where(x == mask_id, proposals, x)
# Permit the intentional -inf mask column in the finite check.
logits[..., mask_id] = torch.finfo(logits.dtype).min
scores = rank_candidates(out, candidates, settings.policy)
selected = torch.topk(scores, k=keep_count, dim=-1).indices
keep = torch.zeros_like(x, dtype=torch.bool).scatter_(1, selected, True)
x = torch.where(keep, candidates, mask_id)
remasked = ((previous != mask_id) & (x == mask_id)).sum().item()
remasked_total += remasked
if settings.trace:
trace.append({"block": offset // settings.block_size + 1, "step": step + 1,
"kept": keep_count, "remasked": remasked,
"token_ids": x[0].tolist(),
"text": tokenizer.decode(x[0].tolist(), skip_special_tokens=False)})
del out, logits
tokens = x[0].tolist()
if eos_id in tokens:
generated.extend(tokens[:tokens.index(eos_id)])
finish_reason = "eos"
break
generated.extend(tokens)
if offset + length < settings.max_tokens:
forward(x, positions, update=True)
if str(device).startswith("cuda"):
torch.cuda.synchronize()
elapsed = time.perf_counter() - started
return {"text": tokenizer.decode(generated, skip_special_tokens=True), "token_ids": generated,
"metrics": {"prompt_tokens": ids.shape[1], "output_tokens": len(generated),
"seconds": round(elapsed, 3), "tokens_per_second": round(len(generated) / max(elapsed, 1e-9), 3),
"forward_passes": forwards, "remasked_positions": remasked_total,
"finish_reason": finish_reason},
"settings": asdict(settings), "trace": trace,
"note": "Greedy token predictions. Seed is recorded; changing it does not create sampling diversity."}
|