Spaces:
Running on Zero
Running on Zero
Remove word constraints (block diffusion doesn't honor mid-sequence pins)
Browse files
app.py
CHANGED
|
@@ -3,13 +3,7 @@ import torch
|
|
| 3 |
import gradio as gr
|
| 4 |
from threading import Thread
|
| 5 |
from queue import Queue
|
| 6 |
-
from transformers import
|
| 7 |
-
DiffusionGemmaForBlockDiffusion,
|
| 8 |
-
AutoProcessor,
|
| 9 |
-
TextDiffusionStreamer,
|
| 10 |
-
LogitsProcessor,
|
| 11 |
-
LogitsProcessorList,
|
| 12 |
-
)
|
| 13 |
|
| 14 |
MODEL_ID = "google/diffusiongemma-26B-A4B-it"
|
| 15 |
|
|
@@ -124,58 +118,9 @@ def build_display(committed, segments):
|
|
| 124 |
return out
|
| 125 |
|
| 126 |
|
| 127 |
-
class WordConstraintLogitsProcessor(LogitsProcessor):
|
| 128 |
-
"""Forces specific generated positions to specific tokens.
|
| 129 |
-
|
| 130 |
-
`constraints` maps an answer-relative position to a list of token ids (a word may
|
| 131 |
-
tokenize to several). raw_logits here is [batch, canvas_length, vocab] for the block
|
| 132 |
-
currently being denoised; the canvas covers answer positions
|
| 133 |
-
[committed_len, committed_len + canvas_length).
|
| 134 |
-
"""
|
| 135 |
-
|
| 136 |
-
def __init__(self, constraints, prompt_len):
|
| 137 |
-
self.constraints = constraints
|
| 138 |
-
self.prompt_len = prompt_len
|
| 139 |
-
|
| 140 |
-
def __call__(self, input_ids, scores, cur_step=None):
|
| 141 |
-
if scores.dim() != 3:
|
| 142 |
-
return scores
|
| 143 |
-
committed_len = input_ids.shape[1] - self.prompt_len
|
| 144 |
-
canvas_length = scores.shape[1]
|
| 145 |
-
for pos, tok_ids in self.constraints.items():
|
| 146 |
-
for off, tid in enumerate(tok_ids):
|
| 147 |
-
local = pos + off - committed_len
|
| 148 |
-
if 0 <= local < canvas_length:
|
| 149 |
-
scores[:, local, :] = float("-inf")
|
| 150 |
-
scores[:, local, tid] = 0.0
|
| 151 |
-
return scores
|
| 152 |
-
|
| 153 |
-
|
| 154 |
-
def parse_constraints(text, tokenizer):
|
| 155 |
-
"""Parse 'pos:word, pos:word' into {pos: [token_id, ...]}."""
|
| 156 |
-
constraints = {}
|
| 157 |
-
if not text or not text.strip():
|
| 158 |
-
return constraints
|
| 159 |
-
for part in text.split(","):
|
| 160 |
-
part = part.strip()
|
| 161 |
-
if ":" not in part:
|
| 162 |
-
continue
|
| 163 |
-
pos_str, word = part.split(":", 1)
|
| 164 |
-
pos_str, word = pos_str.strip(), word.strip()
|
| 165 |
-
if not pos_str.isdigit() or not word:
|
| 166 |
-
continue
|
| 167 |
-
pos = int(pos_str)
|
| 168 |
-
# leading space gives the word-initial SentencePiece token, except at the very start
|
| 169 |
-
prefix = "" if pos == 0 else " "
|
| 170 |
-
ids = tokenizer.encode(prefix + word, add_special_tokens=False)
|
| 171 |
-
if ids:
|
| 172 |
-
constraints[pos] = ids
|
| 173 |
-
return constraints
|
| 174 |
-
|
| 175 |
-
|
| 176 |
@spaces.GPU(duration=150, size="xlarge")
|
| 177 |
@torch.no_grad()
|
| 178 |
-
def generate_streaming(messages, max_new_tokens, max_denoising_steps, enable_thinking
|
| 179 |
inputs = processor.apply_chat_template(
|
| 180 |
messages,
|
| 181 |
add_generation_prompt=True,
|
|
@@ -185,14 +130,6 @@ def generate_streaming(messages, max_new_tokens, max_denoising_steps, enable_thi
|
|
| 185 |
enable_thinking=enable_thinking,
|
| 186 |
).to("cuda")
|
| 187 |
|
| 188 |
-
prompt_len = inputs["input_ids"].shape[1]
|
| 189 |
-
constraints = parse_constraints(constraints_text, processor.tokenizer)
|
| 190 |
-
logits_processor = None
|
| 191 |
-
if constraints:
|
| 192 |
-
logits_processor = LogitsProcessorList(
|
| 193 |
-
[WordConstraintLogitsProcessor(constraints, prompt_len)]
|
| 194 |
-
)
|
| 195 |
-
|
| 196 |
streamer = CanvasStreamer(processor.tokenizer)
|
| 197 |
result = {}
|
| 198 |
|
|
@@ -203,7 +140,6 @@ def generate_streaming(messages, max_new_tokens, max_denoising_steps, enable_thi
|
|
| 203 |
max_new_tokens=int(max_new_tokens),
|
| 204 |
max_denoising_steps=int(max_denoising_steps),
|
| 205 |
streamer=streamer,
|
| 206 |
-
logits_processor=logits_processor,
|
| 207 |
)
|
| 208 |
result["ids"] = out
|
| 209 |
except Exception as e:
|
|
@@ -308,11 +244,6 @@ def create_demo():
|
|
| 308 |
with gr.Row():
|
| 309 |
max_new_tokens = gr.Slider(64, 1024, value=256, step=64, label="Max new tokens")
|
| 310 |
max_denoising_steps = gr.Slider(8, 64, value=48, step=4, label="Max denoising steps")
|
| 311 |
-
constraints_input = gr.Textbox(
|
| 312 |
-
label="Word constraints",
|
| 313 |
-
placeholder="Place specific words at specific positions: 'position:word'. Example: 0:Once, 5:upon, 10:time",
|
| 314 |
-
value="",
|
| 315 |
-
)
|
| 316 |
|
| 317 |
clear_btn = gr.Button("Clear conversation")
|
| 318 |
|
|
@@ -324,7 +255,7 @@ def create_demo():
|
|
| 324 |
history.append({"role": "user", "content": message["text"]})
|
| 325 |
return history, gr.MultimodalTextbox(value=None, interactive=False)
|
| 326 |
|
| 327 |
-
def bot(history, max_new_tokens, max_denoising_steps, enable_thinking
|
| 328 |
if not history:
|
| 329 |
yield history, []
|
| 330 |
return
|
|
@@ -333,7 +264,7 @@ def create_demo():
|
|
| 333 |
final = None
|
| 334 |
try:
|
| 335 |
for canvas_state, plain, text in generate_streaming(
|
| 336 |
-
messages, max_new_tokens, max_denoising_steps, enable_thinking
|
| 337 |
):
|
| 338 |
if text is not None:
|
| 339 |
final = text
|
|
@@ -351,7 +282,7 @@ def create_demo():
|
|
| 351 |
)
|
| 352 |
bot_msg = chat_msg.then(
|
| 353 |
bot,
|
| 354 |
-
[chatbot, max_new_tokens, max_denoising_steps, enable_thinking
|
| 355 |
[chatbot, canvas],
|
| 356 |
)
|
| 357 |
bot_msg.then(reenable, None, [chat_input])
|
|
|
|
| 3 |
import gradio as gr
|
| 4 |
from threading import Thread
|
| 5 |
from queue import Queue
|
| 6 |
+
from transformers import DiffusionGemmaForBlockDiffusion, AutoProcessor, TextDiffusionStreamer
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7 |
|
| 8 |
MODEL_ID = "google/diffusiongemma-26B-A4B-it"
|
| 9 |
|
|
|
|
| 118 |
return out
|
| 119 |
|
| 120 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 121 |
@spaces.GPU(duration=150, size="xlarge")
|
| 122 |
@torch.no_grad()
|
| 123 |
+
def generate_streaming(messages, max_new_tokens, max_denoising_steps, enable_thinking):
|
| 124 |
inputs = processor.apply_chat_template(
|
| 125 |
messages,
|
| 126 |
add_generation_prompt=True,
|
|
|
|
| 130 |
enable_thinking=enable_thinking,
|
| 131 |
).to("cuda")
|
| 132 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 133 |
streamer = CanvasStreamer(processor.tokenizer)
|
| 134 |
result = {}
|
| 135 |
|
|
|
|
| 140 |
max_new_tokens=int(max_new_tokens),
|
| 141 |
max_denoising_steps=int(max_denoising_steps),
|
| 142 |
streamer=streamer,
|
|
|
|
| 143 |
)
|
| 144 |
result["ids"] = out
|
| 145 |
except Exception as e:
|
|
|
|
| 244 |
with gr.Row():
|
| 245 |
max_new_tokens = gr.Slider(64, 1024, value=256, step=64, label="Max new tokens")
|
| 246 |
max_denoising_steps = gr.Slider(8, 64, value=48, step=4, label="Max denoising steps")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 247 |
|
| 248 |
clear_btn = gr.Button("Clear conversation")
|
| 249 |
|
|
|
|
| 255 |
history.append({"role": "user", "content": message["text"]})
|
| 256 |
return history, gr.MultimodalTextbox(value=None, interactive=False)
|
| 257 |
|
| 258 |
+
def bot(history, max_new_tokens, max_denoising_steps, enable_thinking):
|
| 259 |
if not history:
|
| 260 |
yield history, []
|
| 261 |
return
|
|
|
|
| 264 |
final = None
|
| 265 |
try:
|
| 266 |
for canvas_state, plain, text in generate_streaming(
|
| 267 |
+
messages, max_new_tokens, max_denoising_steps, enable_thinking
|
| 268 |
):
|
| 269 |
if text is not None:
|
| 270 |
final = text
|
|
|
|
| 282 |
)
|
| 283 |
bot_msg = chat_msg.then(
|
| 284 |
bot,
|
| 285 |
+
[chatbot, max_new_tokens, max_denoising_steps, enable_thinking],
|
| 286 |
[chatbot, canvas],
|
| 287 |
)
|
| 288 |
bot_msg.then(reenable, None, [chat_input])
|