iamthinbaker commited on
Commit
b99025a
·
verified ·
1 Parent(s): d1baed9

Add model code for trust_remote_code

Browse files
Files changed (1) hide show
  1. conditioned_gpt2.py +489 -0
conditioned_gpt2.py ADDED
@@ -0,0 +1,489 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ import torch.nn as nn
3
+ import torch.nn.functional as F
4
+ import numpy as np
5
+ from transformers import GPT2Config, GPT2LMHeadModel, LogitsProcessor
6
+
7
+ NUM_GENERATIONS = 10
8
+ NUM_TYPES1 = 19 # 18 tipos + UNK
9
+ NUM_TYPES2 = 20 # 18 tipos + NONE + UNK
10
+ NUM_EVO_STAGES = 4 # 3 etapas + 1
11
+ NUM_HAS_EVOLUTION = 2
12
+ NUM_COLOR_SHIFTS = 6 # 0 = no shift, 1-5 = ColorShift permutations
13
+
14
+
15
+ class ConditionedGPT2(GPT2LMHeadModel):
16
+ def __init__(
17
+ self,
18
+ config: GPT2Config,
19
+ num_pokemon: int,
20
+ noise_std: float = 0.1,
21
+ row_marker_token_ids: list[int] | None = None,
22
+ num_types1: int = NUM_TYPES1,
23
+ num_types2: int = NUM_TYPES2,
24
+ num_generations: int = NUM_GENERATIONS,
25
+ num_evo_stages: int = NUM_EVO_STAGES,
26
+ token_weights: torch.Tensor | None = None,
27
+ num_color_shifts: int = NUM_COLOR_SHIFTS,
28
+ ):
29
+ super().__init__(config)
30
+ self.conditioning = nn.Embedding(num_pokemon, config.n_embd)
31
+ nn.init.normal_(self.conditioning.weight, std=0.02)
32
+ self.noise_std = noise_std
33
+
34
+ # Row embedding: 0-63 for sprite rows, 64 = padding (BOS/EOS/pre-row tokens)
35
+ self.row_emb = nn.Embedding(65, config.n_embd, padding_idx=64)
36
+ nn.init.normal_(self.row_emb.weight, std=0.02)
37
+ self.row_emb.weight.data[64].zero_()
38
+
39
+ # Column embedding: 0-63 for pixel position within a row, 64 = padding
40
+ self.col_emb = nn.Embedding(65, config.n_embd, padding_idx=64)
41
+ nn.init.normal_(self.col_emb.weight, std=0.02)
42
+ self.col_emb.weight.data[64].zero_()
43
+
44
+ # Metadata conditioning embeddings
45
+ self.type1_emb = nn.Embedding(num_types1, config.n_embd)
46
+ self.type2_emb = nn.Embedding(num_types2, config.n_embd)
47
+ self.is_shiny_emb = nn.Embedding(NUM_HAS_EVOLUTION, config.n_embd)
48
+ self.generation_emb = nn.Embedding(num_generations, config.n_embd)
49
+ self.evo_stage_emb = nn.Embedding(num_evo_stages, config.n_embd)
50
+ self.has_evolution_emb = nn.Embedding(NUM_HAS_EVOLUTION, config.n_embd)
51
+ self.color_shift_emb = nn.Embedding(num_color_shifts, config.n_embd)
52
+
53
+ for emb in (
54
+ self.type1_emb,
55
+ self.type2_emb,
56
+ self.is_shiny_emb,
57
+ self.generation_emb,
58
+ self.evo_stage_emb,
59
+ self.has_evolution_emb,
60
+ self.color_shift_emb,
61
+ ):
62
+ nn.init.normal_(emb.weight, std=0.02)
63
+
64
+ # Per-token loss weights — downweights background tokens to focus on color pixels
65
+ self.register_buffer("token_weights", token_weights)
66
+
67
+ # Store row marker token ids as a buffer so they're saved with the model
68
+ _ids = row_marker_token_ids or [0] * 64
69
+ self.register_buffer(
70
+ "row_marker_ids", torch.tensor(_ids, dtype=torch.long)
71
+ )
72
+
73
+ def _ids_to_row_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
74
+ B, T = input_ids.shape
75
+ device = input_ids.device
76
+
77
+ row_ids = input_ids.new_full((B, T), 64)
78
+ for row_idx in range(64):
79
+ row_ids[input_ids == self.row_marker_ids[row_idx]] = row_idx
80
+
81
+ # Vectorized forward-fill via cummax: for each position find the last marker seen
82
+ is_assigned = row_ids < 64
83
+ t_idx = torch.arange(T, device=device).unsqueeze(0).expand(B, -1)
84
+ last_marker_t, _ = torch.where(
85
+ is_assigned,
86
+ t_idx,
87
+ torch.zeros_like(t_idx),
88
+ ).cummax(dim=1)
89
+ row_ids_filled = torch.gather(row_ids, 1, last_marker_t)
90
+
91
+ in_row = is_assigned.long().cumsum(dim=1) >= 1
92
+ return torch.where(
93
+ in_row, row_ids_filled, input_ids.new_full((B, T), 64)
94
+ )
95
+
96
+ def _ids_to_col_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
97
+ B, T = input_ids.shape
98
+ device = input_ids.device
99
+
100
+ is_marker = torch.isin(input_ids, self.row_marker_ids.to(device))
101
+ is_pixel = ~is_marker
102
+ in_row = is_marker.long().cumsum(dim=1) >= 1
103
+
104
+ # Cumulative pixel count (inclusive) and the baseline at the last marker
105
+ pixel_cumsum = is_pixel.long().cumsum(dim=1)
106
+ marker_base = torch.where(
107
+ is_marker, pixel_cumsum, torch.zeros_like(pixel_cumsum)
108
+ )
109
+ t_idx = torch.arange(T, device=device).unsqueeze(0).expand(B, -1)
110
+ last_marker_t, _ = torch.where(
111
+ is_marker,
112
+ t_idx,
113
+ torch.zeros_like(t_idx),
114
+ ).cummax(dim=1)
115
+ last_marker_base = torch.gather(marker_base, 1, last_marker_t)
116
+
117
+ # 0-indexed column position within the current row
118
+ col_pos = pixel_cumsum - last_marker_base - 1
119
+ return torch.where(
120
+ is_pixel & in_row & (col_pos < 64),
121
+ col_pos.clamp(min=0),
122
+ input_ids.new_full((B, T), 64),
123
+ )
124
+
125
+ @torch.compiler.disable
126
+ def forward(
127
+ self,
128
+ input_ids=None,
129
+ attention_mask=None,
130
+ pokemon_idx=None,
131
+ pokemon_cond=None,
132
+ row_ids=None,
133
+ col_ids=None,
134
+ type1=None,
135
+ type2=None,
136
+ is_shiny=None,
137
+ generation=None,
138
+ evolution_stage=None,
139
+ has_evolution=None,
140
+ color_shift=None,
141
+ logits_to_keep: int | torch.Tensor = 0,
142
+ num_items_in_batch=None,
143
+ **kwargs,
144
+ ):
145
+ # Extract labels before passing to parent so we can compute our own loss
146
+ labels = kwargs.pop("labels", None)
147
+
148
+ if input_ids is not None and (
149
+ pokemon_idx is not None or pokemon_cond is not None
150
+ ):
151
+ token_embs = self.transformer.wte(input_ids)
152
+
153
+ # Spatial 2-D embeddings: row (which row) + col (which pixel position)
154
+ if row_ids is None:
155
+ row_ids = self._ids_to_row_ids(input_ids)
156
+ if col_ids is None:
157
+ col_ids = self._ids_to_col_ids(input_ids)
158
+ token_embs = (
159
+ token_embs + self.row_emb(row_ids) + self.col_emb(col_ids)
160
+ )
161
+
162
+ # Combined conditioning: pokemon identity + metadata
163
+ B, device = token_embs.shape[0], token_embs.device
164
+
165
+ def _rand_or_use(val, emb):
166
+ if val is None:
167
+ val = torch.randint(
168
+ 0, emb.num_embeddings, (B,), device=device
169
+ )
170
+ return emb(val)
171
+
172
+ # pokemon_cond allows passing a pre-computed conditioning vector directly
173
+ # (used for novel Pokémon generation via embedding interpolation)
174
+ base_cond = (
175
+ pokemon_cond
176
+ if pokemon_cond is not None
177
+ else self.conditioning(pokemon_idx)
178
+ )
179
+ cond = (
180
+ base_cond
181
+ + _rand_or_use(type1, self.type1_emb)
182
+ + _rand_or_use(type2, self.type2_emb)
183
+ + _rand_or_use(is_shiny, self.is_shiny_emb)
184
+ + _rand_or_use(generation, self.generation_emb)
185
+ + _rand_or_use(evolution_stage, self.evo_stage_emb)
186
+ + _rand_or_use(has_evolution, self.has_evolution_emb)
187
+ + _rand_or_use(color_shift, self.color_shift_emb)
188
+ )
189
+
190
+ if self.training and self.noise_std > 0:
191
+ cond = cond + torch.randn_like(cond) * self.noise_std
192
+ token_embs = token_embs + cond.unsqueeze(1)
193
+
194
+ kwargs["inputs_embeds"] = token_embs
195
+ input_ids = None
196
+
197
+ outputs = super().forward(
198
+ input_ids=input_ids,
199
+ attention_mask=attention_mask,
200
+ logits_to_keep=logits_to_keep,
201
+ **kwargs,
202
+ )
203
+
204
+ if labels is not None:
205
+ shift_logits = outputs.logits[..., :-1, :].contiguous().float()
206
+ shift_labels = (
207
+ labels[..., 1:].contiguous().to(outputs.logits.device)
208
+ )
209
+ weights = (
210
+ self.token_weights.to(outputs.logits.device)
211
+ if self.token_weights is not None
212
+ else None
213
+ )
214
+ loss = F.cross_entropy(
215
+ shift_logits.view(-1, self.config.vocab_size),
216
+ shift_labels.view(-1),
217
+ weight=weights,
218
+ ignore_index=-100,
219
+ reduction="mean",
220
+ )
221
+ outputs.loss = torch.nan_to_num(loss, nan=0.0)
222
+
223
+ return outputs
224
+
225
+ def prepare_inputs_for_generation(
226
+ self, input_ids, past_key_values=None, **kwargs
227
+ ):
228
+ # Compute positional ids from the full sequence before the parent trims for KV cache
229
+ row_ids_full = self._ids_to_row_ids(input_ids)
230
+ col_ids_full = self._ids_to_col_ids(input_ids)
231
+
232
+ inputs = super().prepare_inputs_for_generation(
233
+ input_ids,
234
+ past_key_values=past_key_values,
235
+ **kwargs,
236
+ )
237
+
238
+ # Trim to last token when KV cache is active (same logic as parent trims input_ids)
239
+ if past_key_values is not None:
240
+ inputs["row_ids"] = row_ids_full[:, -1:]
241
+ inputs["col_ids"] = col_ids_full[:, -1:]
242
+ else:
243
+ inputs["row_ids"] = row_ids_full
244
+ inputs["col_ids"] = col_ids_full
245
+
246
+ for field in (
247
+ "pokemon_idx",
248
+ "pokemon_cond",
249
+ "type1",
250
+ "type2",
251
+ "is_shiny",
252
+ "generation",
253
+ "evolution_stage",
254
+ "has_evolution",
255
+ "color_shift",
256
+ ):
257
+ if field in kwargs:
258
+ inputs[field] = kwargs[field]
259
+
260
+ return inputs
261
+
262
+ def sample_conditioning(self, idx: int | None = None) -> torch.Tensor:
263
+ if idx is None:
264
+ idx = torch.randint(
265
+ 0, self.conditioning.num_embeddings, (1,)
266
+ ).item()
267
+ with torch.no_grad():
268
+ return self.conditioning(
269
+ torch.tensor([idx], device=self.conditioning.weight.device),
270
+ )
271
+
272
+ def sample_random_conditioning(self, device: str = "cpu") -> dict:
273
+ return {
274
+ "pokemon_idx": torch.randint(
275
+ 0, self.conditioning.num_embeddings, (1,), device=device
276
+ ),
277
+ "type1": torch.randint(
278
+ 0, self.type1_emb.num_embeddings, (1,), device=device
279
+ ),
280
+ "type2": torch.randint(
281
+ 0, self.type2_emb.num_embeddings, (1,), device=device
282
+ ),
283
+ "is_shiny": torch.randint(
284
+ 0, self.is_shiny_emb.num_embeddings, (1,), device=device
285
+ ),
286
+ "generation": torch.randint(
287
+ 0, self.generation_emb.num_embeddings, (1,), device=device
288
+ ),
289
+ "evolution_stage": torch.randint(
290
+ 0, self.evo_stage_emb.num_embeddings, (1,), device=device
291
+ ),
292
+ "has_evolution": torch.randint(
293
+ 0, self.has_evolution_emb.num_embeddings, (1,), device=device
294
+ ),
295
+ "color_shift": torch.randint(
296
+ 0, NUM_COLOR_SHIFTS, (1,), dtype=torch.long, device=device
297
+ ),
298
+ }
299
+
300
+ def sample_novel_conditioning(
301
+ self, n_mix: int = 3, device: str = "cpu"
302
+ ) -> dict:
303
+ """Blend n_mix random Pokémon embeddings to produce a novel conditioning vector."""
304
+ with torch.no_grad():
305
+ idxs = torch.randint(
306
+ 0, self.conditioning.num_embeddings, (n_mix,), device=device
307
+ )
308
+ weights = torch.softmax(torch.randn(n_mix, device=device), dim=0)
309
+ pokemon_cond = (weights.unsqueeze(1) * self.conditioning(idxs)).sum(
310
+ 0, keepdim=True
311
+ )
312
+ return {
313
+ "pokemon_cond": pokemon_cond,
314
+ "type1": torch.randint(
315
+ 0, self.type1_emb.num_embeddings, (1,), device=device
316
+ ),
317
+ "type2": torch.randint(
318
+ 0, self.type2_emb.num_embeddings, (1,), device=device
319
+ ),
320
+ "is_shiny": torch.randint(
321
+ 0, self.is_shiny_emb.num_embeddings, (1,), device=device
322
+ ),
323
+ "generation": torch.randint(
324
+ 0, self.generation_emb.num_embeddings, (1,), device=device
325
+ ),
326
+ "evolution_stage": torch.randint(
327
+ 0, self.evo_stage_emb.num_embeddings, (1,), device=device
328
+ ),
329
+ "has_evolution": torch.randint(
330
+ 0, self.has_evolution_emb.num_embeddings, (1,), device=device
331
+ ),
332
+ "color_shift": torch.randint(
333
+ 0, NUM_COLOR_SHIFTS, (1,), dtype=torch.long, device=device
334
+ ),
335
+ }
336
+
337
+ def generate_sprite(
338
+ self,
339
+ tokenizer,
340
+ temperature: float = 1.2,
341
+ top_p: float = 0.95,
342
+ verbose: bool = False,
343
+ ) -> np.ndarray:
344
+ """Generate a 64×64 RGB sprite as a numpy array."""
345
+ device = next(self.parameters()).device
346
+
347
+ cond = self.sample_novel_conditioning(device=str(device))
348
+ cond["color_shift"] = torch.tensor([0], dtype=torch.long, device=device)
349
+ cond["is_shiny"] = torch.tensor([0], dtype=torch.long, device=device)
350
+ cond["generation"] = torch.tensor(
351
+ [2 + torch.randint(0, 2, (1,)).item()],
352
+ dtype=torch.long,
353
+ device=device,
354
+ )
355
+
356
+ row_marker_ids = [
357
+ tokenizer.convert_tokens_to_ids(f"[ROW_{i:02d}]") for i in range(64)
358
+ ]
359
+ inputs = tokenizer(
360
+ "[ROW_00]", return_tensors="pt", add_special_tokens=False
361
+ )
362
+ inputs = {k: v.to(device) for k, v in inputs.items()}
363
+ inputs.update(cond)
364
+
365
+ streamer = None
366
+ if verbose:
367
+ from transformers import TextStreamer
368
+ streamer = TextStreamer(tokenizer, skip_special_tokens=False)
369
+
370
+ with torch.no_grad():
371
+ output_ids = self.generate(
372
+ **inputs,
373
+ max_length=4096,
374
+ do_sample=True,
375
+ top_k=0,
376
+ top_p=top_p,
377
+ temperature=temperature,
378
+ pad_token_id=tokenizer.pad_token_id,
379
+ eos_token_id=tokenizer.eos_token_id,
380
+ logits_processor=[
381
+ _RowLengthLogitsProcessor(tokenizer, row_marker_ids)
382
+ ],
383
+ streamer=streamer,
384
+ )
385
+
386
+ return _tokens_to_image(
387
+ tokenizer.decode(output_ids[0], skip_special_tokens=False)
388
+ )
389
+
390
+
391
+ class _RowLengthLogitsProcessor(LogitsProcessor):
392
+ def __init__(
393
+ self, tokenizer, row_marker_ids: list[int], row_width: int = 63
394
+ ):
395
+ vocab = tokenizer.get_vocab()
396
+ self.row_marker_set = set(row_marker_ids)
397
+ self.row_marker_ids = row_marker_ids
398
+ self.eos_id = tokenizer.eos_token_id
399
+ self.bos_id = tokenizer.bos_token_id
400
+ self.row_width = row_width
401
+ special = {
402
+ tokenizer.eos_token_id,
403
+ tokenizer.bos_token_id,
404
+ tokenizer.pad_token_id,
405
+ *row_marker_ids,
406
+ }
407
+ self.pixel_len = torch.zeros(len(vocab), dtype=torch.long)
408
+ for token, idx in vocab.items():
409
+ if idx not in special:
410
+ self.pixel_len[idx] = len(token)
411
+ self.current_row = self.chars_in_row = 0
412
+
413
+ def __call__(
414
+ self, input_ids: torch.Tensor, scores: torch.Tensor
415
+ ) -> torch.Tensor:
416
+ device = scores.device
417
+ pixel_len = self.pixel_len.to(device)
418
+ last_id = int(input_ids[0, -1].item())
419
+ if last_id in self.row_marker_set:
420
+ self.current_row = self.row_marker_ids.index(last_id)
421
+ self.chars_in_row = 0
422
+ remaining = self.row_width - self.chars_in_row
423
+ if remaining > 0:
424
+ mask = (pixel_len > remaining) | (pixel_len == 0)
425
+ if self.current_row < 63:
426
+ mask[self.eos_id] = True
427
+ scores = scores.masked_fill(mask.unsqueeze(0), float("-inf"))
428
+ else:
429
+ allowed = torch.full((scores.shape[-1],), True, device=device)
430
+ if self.current_row < 63:
431
+ allowed[self.row_marker_ids[self.current_row + 1]] = False
432
+ else:
433
+ allowed[self.eos_id] = False
434
+ scores = scores.masked_fill(allowed.unsqueeze(0), float("-inf"))
435
+ if last_id not in self.row_marker_set and last_id not in {
436
+ self.eos_id,
437
+ self.bos_id,
438
+ }:
439
+ self.chars_in_row += int(pixel_len[last_id].item())
440
+ return scores
441
+
442
+
443
+ def _tokens_to_image(text: str) -> np.ndarray:
444
+ rows, current = [], []
445
+ for token in text.split():
446
+ if token.startswith("[ROW_") and token.endswith("]"):
447
+ if current:
448
+ rows.append(current)
449
+ current = []
450
+ elif not (token.startswith("[") and token.endswith("]")):
451
+ current.append(token)
452
+ if current:
453
+ rows.append(current)
454
+ image = np.zeros((64, 64, 3), dtype=np.uint8)
455
+ for y, row in enumerate(rows[:64]):
456
+ for x, char in enumerate(row[:64]):
457
+ if char == "~":
458
+ image[y, x] = [255, 255, 255]
459
+ else:
460
+ idx = ord(char) - 59
461
+ r, g, b = idx // 16, (idx // 4) % 4, idx % 4
462
+ image[y, x] = [r * 64 + 32, g * 64 + 32, b * 64 + 32]
463
+ return image
464
+
465
+
466
+ if __name__ == "__main__":
467
+ from pathlib import Path
468
+ from PIL import Image
469
+ from huggingface_hub import snapshot_download
470
+ from transformers import AutoModelForCausalLM, PreTrainedTokenizerFast
471
+
472
+ REPO_ID = "iamthinbaker/GPokeT2"
473
+ REVISION = "v0.1-wip-3400"
474
+ N_SAMPLES = 5
475
+ OUTPUT_DIR = Path("generated")
476
+
477
+ print(f"Descargando {REPO_ID} ({REVISION})...")
478
+ ckpt = snapshot_download(REPO_ID, revision=REVISION)
479
+ tokenizer = PreTrainedTokenizerFast.from_pretrained(ckpt)
480
+ model = AutoModelForCausalLM.from_pretrained(ckpt, trust_remote_code=True)
481
+ model.eval()
482
+
483
+ OUTPUT_DIR.mkdir(exist_ok=True)
484
+ for i in range(N_SAMPLES):
485
+ print(f"\n[{i + 1}/{N_SAMPLES}] generando...")
486
+ image = model.generate_sprite(tokenizer, verbose=True)
487
+ path = OUTPUT_DIR / f"pokemon_{i + 1:02d}.png"
488
+ Image.fromarray(image).save(path)
489
+ print(f"guardado en {path}")