File size: 7,713 Bytes
79df115
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
"""Block diffusion adapted from the pinned official RemeDi inference implementation."""
from dataclasses import asdict, dataclass
import time
import torch
from remedi.modelling_remedi_bitowel import DynamicCache

MODEL_ID = "maple-research-lab/RemeDi-Instruct"
MODEL_REVISION = "2a8d6c1bad0892fa50de0c2aba3253fc4c88adf4"
TOKENIZER_ID = "GSAI-ML/LLaDA-8B-Instruct"
POLICIES = ["Official raw logits", "Learned confidence (experimental)", "Token probability (experimental)"]

@dataclass(frozen=True)
class Settings:
    max_tokens: int = 128
    block_size: int = 32
    steps: int = 16
    policy: str = POLICIES[0]
    seed: int = 112
    trace: bool = True

    def validate(self):
        for name in ("max_tokens", "block_size", "steps", "seed"):
            if type(getattr(self, name)) is not int:
                raise ValueError(f"{name} must be an integer")
        if not 1 <= self.max_tokens <= 256:
            raise ValueError("Output length must be 1–256 tokens")
        if self.block_size not in (16, 32, 64):
            raise ValueError("Block size must be 16, 32, or 64")
        if not 1 <= self.steps <= 64:
            raise ValueError("Steps must be 1–64")
        if self.policy not in POLICIES:
            raise ValueError("Unknown ranking policy")
        # Bound the full attention/forward cost of each ZeroGPU request.
        if ((self.max_tokens + self.block_size - 1) // self.block_size) * self.steps > 256:
            raise ValueError("Too many model passes; reduce tokens or steps (maximum 256 denoising passes)")
        return self

def schedule(length, steps):
    """Match upstream cumulative transfer counts, including steps > block length."""
    counts = [length // steps] * steps
    remainder = length % steps
    if remainder:
        for i in range(steps - remainder, steps):
            counts[i] += 1
    total = 0
    result = []
    for n in counts:
        total += n
        result.append(total)
    return result

def prompt_ids(tokenizer, prompt, system=""):
    if not isinstance(prompt, str) or not prompt.strip():
        raise ValueError("Enter a prompt")
    if len(prompt) + len(system) > 16000:
        raise ValueError("Prompt is too long (16,000 character limit)")
    messages = []
    if system.strip():
        messages.append({"role": "system", "content": system.strip()})
    messages.append({"role": "user", "content": prompt.strip()})
    ids = tokenizer.apply_chat_template(messages, add_generation_prompt=True, return_tensors="pt")
    if ids.ndim != 2 or ids.shape[0] != 1 or ids.shape[1] == 0:
        raise ValueError("Tokenizer returned an invalid prompt")
    if ids.shape[1] > 1024:
        raise ValueError(f"Prompt is {ids.shape[1]} tokens; maximum 1024. It was not truncated.")
    return ids

def rank_candidates(out, candidates, policy):
    logits = out.logits.float()
    if not torch.isfinite(logits).all():
        raise RuntimeError("Model produced non-finite logits")
    if policy == POLICIES[1]:
        scores = out.confidences.float()
    else:
        scores = logits.gather(-1, candidates.unsqueeze(-1)).squeeze(-1)
        if policy == POLICIES[2]:
            scores = scores - logits.logsumexp(dim=-1)
    if scores.shape != candidates.shape or not torch.isfinite(scores).all():
        raise RuntimeError("Invalid ranking scores")
    return scores

@torch.inference_mode()
def generate(model, tokenizer, ids, settings, device="cuda", deadline_seconds=110):
    settings.validate()
    started = time.perf_counter()
    if ids.shape[1] + settings.max_tokens > model.config.max_sequence_length:
        raise ValueError("Prompt plus output exceeds model context")
    mask_id = int(getattr(model.config, "mask_token_id", 126336))
    eos_id = int(getattr(model.config, "eos_token_id", 126081))
    # Request-local cache; never reuse one visitor's prefix for another visitor.
    cache = DynamicCache()
    ids = ids.to(device)
    generated, trace = [], []
    forwards = 0
    def forward(x, positions=None, update=False):
        nonlocal forwards
        if time.perf_counter() - started > deadline_seconds:
            raise TimeoutError("Request exceeded its runtime budget; reduce output length or steps")
        kwargs = dict(kv_cache=cache, update_kv_cache=update)
        if positions is not None:
            kwargs["position_ids"] = positions
        with torch.autocast(device_type="cuda", dtype=torch.bfloat16, enabled=str(device).startswith("cuda")):
            result = model(x, **kwargs)
        forwards += 1
        return result
    # Do not inspect EOS in the prompt (the chat template contains EOS tokens).
    forward(ids, update=True)
    finish_reason = "length"
    remasked_total = 0
    with torch.random.fork_rng(devices=[torch.device(device).index or 0] if str(device).startswith("cuda") else []):
        torch.manual_seed(settings.seed)
        for offset in range(0, settings.max_tokens, settings.block_size):
            length = min(settings.block_size, settings.max_tokens - offset)
            x = torch.full((1, length), mask_id, dtype=torch.long, device=device)
            positions = torch.arange(ids.shape[1] + offset, ids.shape[1] + offset + length, device=device)[None]
            for step, keep_count in enumerate(schedule(length, settings.steps)):
                previous = x.clone()
                out = forward(x, positions)
                logits = out.logits
                # Mask is an input sentinel, never a valid generated answer.
                logits[..., mask_id] = -torch.inf
                proposals = logits.argmax(dim=-1)
                candidates = torch.where(x == mask_id, proposals, x)
                # Permit the intentional -inf mask column in the finite check.
                logits[..., mask_id] = torch.finfo(logits.dtype).min
                scores = rank_candidates(out, candidates, settings.policy)
                selected = torch.topk(scores, k=keep_count, dim=-1).indices
                keep = torch.zeros_like(x, dtype=torch.bool).scatter_(1, selected, True)
                x = torch.where(keep, candidates, mask_id)
                remasked = ((previous != mask_id) & (x == mask_id)).sum().item()
                remasked_total += remasked
                if settings.trace:
                    trace.append({"block": offset // settings.block_size + 1, "step": step + 1,
                                  "kept": keep_count, "remasked": remasked,
                                  "token_ids": x[0].tolist(),
                                  "text": tokenizer.decode(x[0].tolist(), skip_special_tokens=False)})
                del out, logits
            tokens = x[0].tolist()
            if eos_id in tokens:
                generated.extend(tokens[:tokens.index(eos_id)])
                finish_reason = "eos"
                break
            generated.extend(tokens)
            if offset + length < settings.max_tokens:
                forward(x, positions, update=True)
    if str(device).startswith("cuda"):
        torch.cuda.synchronize()
    elapsed = time.perf_counter() - started
    return {"text": tokenizer.decode(generated, skip_special_tokens=True), "token_ids": generated,
            "metrics": {"prompt_tokens": ids.shape[1], "output_tokens": len(generated),
                        "seconds": round(elapsed, 3), "tokens_per_second": round(len(generated) / max(elapsed, 1e-9), 3),
                        "forward_passes": forwards, "remasked_positions": remasked_total,
                        "finish_reason": finish_reason},
            "settings": asdict(settings), "trace": trace,
            "note": "Greedy token predictions. Seed is recorded; changing it does not create sampling diversity."}