Add model code for trust_remote_code
Browse files- conditioned_gpt2.py +489 -0
conditioned_gpt2.py
ADDED
|
@@ -0,0 +1,489 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
import torch.nn as nn
|
| 3 |
+
import torch.nn.functional as F
|
| 4 |
+
import numpy as np
|
| 5 |
+
from transformers import GPT2Config, GPT2LMHeadModel, LogitsProcessor
|
| 6 |
+
|
| 7 |
+
NUM_GENERATIONS = 10
|
| 8 |
+
NUM_TYPES1 = 19 # 18 tipos + UNK
|
| 9 |
+
NUM_TYPES2 = 20 # 18 tipos + NONE + UNK
|
| 10 |
+
NUM_EVO_STAGES = 4 # 3 etapas + 1
|
| 11 |
+
NUM_HAS_EVOLUTION = 2
|
| 12 |
+
NUM_COLOR_SHIFTS = 6 # 0 = no shift, 1-5 = ColorShift permutations
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
class ConditionedGPT2(GPT2LMHeadModel):
|
| 16 |
+
def __init__(
|
| 17 |
+
self,
|
| 18 |
+
config: GPT2Config,
|
| 19 |
+
num_pokemon: int,
|
| 20 |
+
noise_std: float = 0.1,
|
| 21 |
+
row_marker_token_ids: list[int] | None = None,
|
| 22 |
+
num_types1: int = NUM_TYPES1,
|
| 23 |
+
num_types2: int = NUM_TYPES2,
|
| 24 |
+
num_generations: int = NUM_GENERATIONS,
|
| 25 |
+
num_evo_stages: int = NUM_EVO_STAGES,
|
| 26 |
+
token_weights: torch.Tensor | None = None,
|
| 27 |
+
num_color_shifts: int = NUM_COLOR_SHIFTS,
|
| 28 |
+
):
|
| 29 |
+
super().__init__(config)
|
| 30 |
+
self.conditioning = nn.Embedding(num_pokemon, config.n_embd)
|
| 31 |
+
nn.init.normal_(self.conditioning.weight, std=0.02)
|
| 32 |
+
self.noise_std = noise_std
|
| 33 |
+
|
| 34 |
+
# Row embedding: 0-63 for sprite rows, 64 = padding (BOS/EOS/pre-row tokens)
|
| 35 |
+
self.row_emb = nn.Embedding(65, config.n_embd, padding_idx=64)
|
| 36 |
+
nn.init.normal_(self.row_emb.weight, std=0.02)
|
| 37 |
+
self.row_emb.weight.data[64].zero_()
|
| 38 |
+
|
| 39 |
+
# Column embedding: 0-63 for pixel position within a row, 64 = padding
|
| 40 |
+
self.col_emb = nn.Embedding(65, config.n_embd, padding_idx=64)
|
| 41 |
+
nn.init.normal_(self.col_emb.weight, std=0.02)
|
| 42 |
+
self.col_emb.weight.data[64].zero_()
|
| 43 |
+
|
| 44 |
+
# Metadata conditioning embeddings
|
| 45 |
+
self.type1_emb = nn.Embedding(num_types1, config.n_embd)
|
| 46 |
+
self.type2_emb = nn.Embedding(num_types2, config.n_embd)
|
| 47 |
+
self.is_shiny_emb = nn.Embedding(NUM_HAS_EVOLUTION, config.n_embd)
|
| 48 |
+
self.generation_emb = nn.Embedding(num_generations, config.n_embd)
|
| 49 |
+
self.evo_stage_emb = nn.Embedding(num_evo_stages, config.n_embd)
|
| 50 |
+
self.has_evolution_emb = nn.Embedding(NUM_HAS_EVOLUTION, config.n_embd)
|
| 51 |
+
self.color_shift_emb = nn.Embedding(num_color_shifts, config.n_embd)
|
| 52 |
+
|
| 53 |
+
for emb in (
|
| 54 |
+
self.type1_emb,
|
| 55 |
+
self.type2_emb,
|
| 56 |
+
self.is_shiny_emb,
|
| 57 |
+
self.generation_emb,
|
| 58 |
+
self.evo_stage_emb,
|
| 59 |
+
self.has_evolution_emb,
|
| 60 |
+
self.color_shift_emb,
|
| 61 |
+
):
|
| 62 |
+
nn.init.normal_(emb.weight, std=0.02)
|
| 63 |
+
|
| 64 |
+
# Per-token loss weights — downweights background tokens to focus on color pixels
|
| 65 |
+
self.register_buffer("token_weights", token_weights)
|
| 66 |
+
|
| 67 |
+
# Store row marker token ids as a buffer so they're saved with the model
|
| 68 |
+
_ids = row_marker_token_ids or [0] * 64
|
| 69 |
+
self.register_buffer(
|
| 70 |
+
"row_marker_ids", torch.tensor(_ids, dtype=torch.long)
|
| 71 |
+
)
|
| 72 |
+
|
| 73 |
+
def _ids_to_row_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
| 74 |
+
B, T = input_ids.shape
|
| 75 |
+
device = input_ids.device
|
| 76 |
+
|
| 77 |
+
row_ids = input_ids.new_full((B, T), 64)
|
| 78 |
+
for row_idx in range(64):
|
| 79 |
+
row_ids[input_ids == self.row_marker_ids[row_idx]] = row_idx
|
| 80 |
+
|
| 81 |
+
# Vectorized forward-fill via cummax: for each position find the last marker seen
|
| 82 |
+
is_assigned = row_ids < 64
|
| 83 |
+
t_idx = torch.arange(T, device=device).unsqueeze(0).expand(B, -1)
|
| 84 |
+
last_marker_t, _ = torch.where(
|
| 85 |
+
is_assigned,
|
| 86 |
+
t_idx,
|
| 87 |
+
torch.zeros_like(t_idx),
|
| 88 |
+
).cummax(dim=1)
|
| 89 |
+
row_ids_filled = torch.gather(row_ids, 1, last_marker_t)
|
| 90 |
+
|
| 91 |
+
in_row = is_assigned.long().cumsum(dim=1) >= 1
|
| 92 |
+
return torch.where(
|
| 93 |
+
in_row, row_ids_filled, input_ids.new_full((B, T), 64)
|
| 94 |
+
)
|
| 95 |
+
|
| 96 |
+
def _ids_to_col_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
| 97 |
+
B, T = input_ids.shape
|
| 98 |
+
device = input_ids.device
|
| 99 |
+
|
| 100 |
+
is_marker = torch.isin(input_ids, self.row_marker_ids.to(device))
|
| 101 |
+
is_pixel = ~is_marker
|
| 102 |
+
in_row = is_marker.long().cumsum(dim=1) >= 1
|
| 103 |
+
|
| 104 |
+
# Cumulative pixel count (inclusive) and the baseline at the last marker
|
| 105 |
+
pixel_cumsum = is_pixel.long().cumsum(dim=1)
|
| 106 |
+
marker_base = torch.where(
|
| 107 |
+
is_marker, pixel_cumsum, torch.zeros_like(pixel_cumsum)
|
| 108 |
+
)
|
| 109 |
+
t_idx = torch.arange(T, device=device).unsqueeze(0).expand(B, -1)
|
| 110 |
+
last_marker_t, _ = torch.where(
|
| 111 |
+
is_marker,
|
| 112 |
+
t_idx,
|
| 113 |
+
torch.zeros_like(t_idx),
|
| 114 |
+
).cummax(dim=1)
|
| 115 |
+
last_marker_base = torch.gather(marker_base, 1, last_marker_t)
|
| 116 |
+
|
| 117 |
+
# 0-indexed column position within the current row
|
| 118 |
+
col_pos = pixel_cumsum - last_marker_base - 1
|
| 119 |
+
return torch.where(
|
| 120 |
+
is_pixel & in_row & (col_pos < 64),
|
| 121 |
+
col_pos.clamp(min=0),
|
| 122 |
+
input_ids.new_full((B, T), 64),
|
| 123 |
+
)
|
| 124 |
+
|
| 125 |
+
@torch.compiler.disable
|
| 126 |
+
def forward(
|
| 127 |
+
self,
|
| 128 |
+
input_ids=None,
|
| 129 |
+
attention_mask=None,
|
| 130 |
+
pokemon_idx=None,
|
| 131 |
+
pokemon_cond=None,
|
| 132 |
+
row_ids=None,
|
| 133 |
+
col_ids=None,
|
| 134 |
+
type1=None,
|
| 135 |
+
type2=None,
|
| 136 |
+
is_shiny=None,
|
| 137 |
+
generation=None,
|
| 138 |
+
evolution_stage=None,
|
| 139 |
+
has_evolution=None,
|
| 140 |
+
color_shift=None,
|
| 141 |
+
logits_to_keep: int | torch.Tensor = 0,
|
| 142 |
+
num_items_in_batch=None,
|
| 143 |
+
**kwargs,
|
| 144 |
+
):
|
| 145 |
+
# Extract labels before passing to parent so we can compute our own loss
|
| 146 |
+
labels = kwargs.pop("labels", None)
|
| 147 |
+
|
| 148 |
+
if input_ids is not None and (
|
| 149 |
+
pokemon_idx is not None or pokemon_cond is not None
|
| 150 |
+
):
|
| 151 |
+
token_embs = self.transformer.wte(input_ids)
|
| 152 |
+
|
| 153 |
+
# Spatial 2-D embeddings: row (which row) + col (which pixel position)
|
| 154 |
+
if row_ids is None:
|
| 155 |
+
row_ids = self._ids_to_row_ids(input_ids)
|
| 156 |
+
if col_ids is None:
|
| 157 |
+
col_ids = self._ids_to_col_ids(input_ids)
|
| 158 |
+
token_embs = (
|
| 159 |
+
token_embs + self.row_emb(row_ids) + self.col_emb(col_ids)
|
| 160 |
+
)
|
| 161 |
+
|
| 162 |
+
# Combined conditioning: pokemon identity + metadata
|
| 163 |
+
B, device = token_embs.shape[0], token_embs.device
|
| 164 |
+
|
| 165 |
+
def _rand_or_use(val, emb):
|
| 166 |
+
if val is None:
|
| 167 |
+
val = torch.randint(
|
| 168 |
+
0, emb.num_embeddings, (B,), device=device
|
| 169 |
+
)
|
| 170 |
+
return emb(val)
|
| 171 |
+
|
| 172 |
+
# pokemon_cond allows passing a pre-computed conditioning vector directly
|
| 173 |
+
# (used for novel Pokémon generation via embedding interpolation)
|
| 174 |
+
base_cond = (
|
| 175 |
+
pokemon_cond
|
| 176 |
+
if pokemon_cond is not None
|
| 177 |
+
else self.conditioning(pokemon_idx)
|
| 178 |
+
)
|
| 179 |
+
cond = (
|
| 180 |
+
base_cond
|
| 181 |
+
+ _rand_or_use(type1, self.type1_emb)
|
| 182 |
+
+ _rand_or_use(type2, self.type2_emb)
|
| 183 |
+
+ _rand_or_use(is_shiny, self.is_shiny_emb)
|
| 184 |
+
+ _rand_or_use(generation, self.generation_emb)
|
| 185 |
+
+ _rand_or_use(evolution_stage, self.evo_stage_emb)
|
| 186 |
+
+ _rand_or_use(has_evolution, self.has_evolution_emb)
|
| 187 |
+
+ _rand_or_use(color_shift, self.color_shift_emb)
|
| 188 |
+
)
|
| 189 |
+
|
| 190 |
+
if self.training and self.noise_std > 0:
|
| 191 |
+
cond = cond + torch.randn_like(cond) * self.noise_std
|
| 192 |
+
token_embs = token_embs + cond.unsqueeze(1)
|
| 193 |
+
|
| 194 |
+
kwargs["inputs_embeds"] = token_embs
|
| 195 |
+
input_ids = None
|
| 196 |
+
|
| 197 |
+
outputs = super().forward(
|
| 198 |
+
input_ids=input_ids,
|
| 199 |
+
attention_mask=attention_mask,
|
| 200 |
+
logits_to_keep=logits_to_keep,
|
| 201 |
+
**kwargs,
|
| 202 |
+
)
|
| 203 |
+
|
| 204 |
+
if labels is not None:
|
| 205 |
+
shift_logits = outputs.logits[..., :-1, :].contiguous().float()
|
| 206 |
+
shift_labels = (
|
| 207 |
+
labels[..., 1:].contiguous().to(outputs.logits.device)
|
| 208 |
+
)
|
| 209 |
+
weights = (
|
| 210 |
+
self.token_weights.to(outputs.logits.device)
|
| 211 |
+
if self.token_weights is not None
|
| 212 |
+
else None
|
| 213 |
+
)
|
| 214 |
+
loss = F.cross_entropy(
|
| 215 |
+
shift_logits.view(-1, self.config.vocab_size),
|
| 216 |
+
shift_labels.view(-1),
|
| 217 |
+
weight=weights,
|
| 218 |
+
ignore_index=-100,
|
| 219 |
+
reduction="mean",
|
| 220 |
+
)
|
| 221 |
+
outputs.loss = torch.nan_to_num(loss, nan=0.0)
|
| 222 |
+
|
| 223 |
+
return outputs
|
| 224 |
+
|
| 225 |
+
def prepare_inputs_for_generation(
|
| 226 |
+
self, input_ids, past_key_values=None, **kwargs
|
| 227 |
+
):
|
| 228 |
+
# Compute positional ids from the full sequence before the parent trims for KV cache
|
| 229 |
+
row_ids_full = self._ids_to_row_ids(input_ids)
|
| 230 |
+
col_ids_full = self._ids_to_col_ids(input_ids)
|
| 231 |
+
|
| 232 |
+
inputs = super().prepare_inputs_for_generation(
|
| 233 |
+
input_ids,
|
| 234 |
+
past_key_values=past_key_values,
|
| 235 |
+
**kwargs,
|
| 236 |
+
)
|
| 237 |
+
|
| 238 |
+
# Trim to last token when KV cache is active (same logic as parent trims input_ids)
|
| 239 |
+
if past_key_values is not None:
|
| 240 |
+
inputs["row_ids"] = row_ids_full[:, -1:]
|
| 241 |
+
inputs["col_ids"] = col_ids_full[:, -1:]
|
| 242 |
+
else:
|
| 243 |
+
inputs["row_ids"] = row_ids_full
|
| 244 |
+
inputs["col_ids"] = col_ids_full
|
| 245 |
+
|
| 246 |
+
for field in (
|
| 247 |
+
"pokemon_idx",
|
| 248 |
+
"pokemon_cond",
|
| 249 |
+
"type1",
|
| 250 |
+
"type2",
|
| 251 |
+
"is_shiny",
|
| 252 |
+
"generation",
|
| 253 |
+
"evolution_stage",
|
| 254 |
+
"has_evolution",
|
| 255 |
+
"color_shift",
|
| 256 |
+
):
|
| 257 |
+
if field in kwargs:
|
| 258 |
+
inputs[field] = kwargs[field]
|
| 259 |
+
|
| 260 |
+
return inputs
|
| 261 |
+
|
| 262 |
+
def sample_conditioning(self, idx: int | None = None) -> torch.Tensor:
|
| 263 |
+
if idx is None:
|
| 264 |
+
idx = torch.randint(
|
| 265 |
+
0, self.conditioning.num_embeddings, (1,)
|
| 266 |
+
).item()
|
| 267 |
+
with torch.no_grad():
|
| 268 |
+
return self.conditioning(
|
| 269 |
+
torch.tensor([idx], device=self.conditioning.weight.device),
|
| 270 |
+
)
|
| 271 |
+
|
| 272 |
+
def sample_random_conditioning(self, device: str = "cpu") -> dict:
|
| 273 |
+
return {
|
| 274 |
+
"pokemon_idx": torch.randint(
|
| 275 |
+
0, self.conditioning.num_embeddings, (1,), device=device
|
| 276 |
+
),
|
| 277 |
+
"type1": torch.randint(
|
| 278 |
+
0, self.type1_emb.num_embeddings, (1,), device=device
|
| 279 |
+
),
|
| 280 |
+
"type2": torch.randint(
|
| 281 |
+
0, self.type2_emb.num_embeddings, (1,), device=device
|
| 282 |
+
),
|
| 283 |
+
"is_shiny": torch.randint(
|
| 284 |
+
0, self.is_shiny_emb.num_embeddings, (1,), device=device
|
| 285 |
+
),
|
| 286 |
+
"generation": torch.randint(
|
| 287 |
+
0, self.generation_emb.num_embeddings, (1,), device=device
|
| 288 |
+
),
|
| 289 |
+
"evolution_stage": torch.randint(
|
| 290 |
+
0, self.evo_stage_emb.num_embeddings, (1,), device=device
|
| 291 |
+
),
|
| 292 |
+
"has_evolution": torch.randint(
|
| 293 |
+
0, self.has_evolution_emb.num_embeddings, (1,), device=device
|
| 294 |
+
),
|
| 295 |
+
"color_shift": torch.randint(
|
| 296 |
+
0, NUM_COLOR_SHIFTS, (1,), dtype=torch.long, device=device
|
| 297 |
+
),
|
| 298 |
+
}
|
| 299 |
+
|
| 300 |
+
def sample_novel_conditioning(
|
| 301 |
+
self, n_mix: int = 3, device: str = "cpu"
|
| 302 |
+
) -> dict:
|
| 303 |
+
"""Blend n_mix random Pokémon embeddings to produce a novel conditioning vector."""
|
| 304 |
+
with torch.no_grad():
|
| 305 |
+
idxs = torch.randint(
|
| 306 |
+
0, self.conditioning.num_embeddings, (n_mix,), device=device
|
| 307 |
+
)
|
| 308 |
+
weights = torch.softmax(torch.randn(n_mix, device=device), dim=0)
|
| 309 |
+
pokemon_cond = (weights.unsqueeze(1) * self.conditioning(idxs)).sum(
|
| 310 |
+
0, keepdim=True
|
| 311 |
+
)
|
| 312 |
+
return {
|
| 313 |
+
"pokemon_cond": pokemon_cond,
|
| 314 |
+
"type1": torch.randint(
|
| 315 |
+
0, self.type1_emb.num_embeddings, (1,), device=device
|
| 316 |
+
),
|
| 317 |
+
"type2": torch.randint(
|
| 318 |
+
0, self.type2_emb.num_embeddings, (1,), device=device
|
| 319 |
+
),
|
| 320 |
+
"is_shiny": torch.randint(
|
| 321 |
+
0, self.is_shiny_emb.num_embeddings, (1,), device=device
|
| 322 |
+
),
|
| 323 |
+
"generation": torch.randint(
|
| 324 |
+
0, self.generation_emb.num_embeddings, (1,), device=device
|
| 325 |
+
),
|
| 326 |
+
"evolution_stage": torch.randint(
|
| 327 |
+
0, self.evo_stage_emb.num_embeddings, (1,), device=device
|
| 328 |
+
),
|
| 329 |
+
"has_evolution": torch.randint(
|
| 330 |
+
0, self.has_evolution_emb.num_embeddings, (1,), device=device
|
| 331 |
+
),
|
| 332 |
+
"color_shift": torch.randint(
|
| 333 |
+
0, NUM_COLOR_SHIFTS, (1,), dtype=torch.long, device=device
|
| 334 |
+
),
|
| 335 |
+
}
|
| 336 |
+
|
| 337 |
+
def generate_sprite(
|
| 338 |
+
self,
|
| 339 |
+
tokenizer,
|
| 340 |
+
temperature: float = 1.2,
|
| 341 |
+
top_p: float = 0.95,
|
| 342 |
+
verbose: bool = False,
|
| 343 |
+
) -> np.ndarray:
|
| 344 |
+
"""Generate a 64×64 RGB sprite as a numpy array."""
|
| 345 |
+
device = next(self.parameters()).device
|
| 346 |
+
|
| 347 |
+
cond = self.sample_novel_conditioning(device=str(device))
|
| 348 |
+
cond["color_shift"] = torch.tensor([0], dtype=torch.long, device=device)
|
| 349 |
+
cond["is_shiny"] = torch.tensor([0], dtype=torch.long, device=device)
|
| 350 |
+
cond["generation"] = torch.tensor(
|
| 351 |
+
[2 + torch.randint(0, 2, (1,)).item()],
|
| 352 |
+
dtype=torch.long,
|
| 353 |
+
device=device,
|
| 354 |
+
)
|
| 355 |
+
|
| 356 |
+
row_marker_ids = [
|
| 357 |
+
tokenizer.convert_tokens_to_ids(f"[ROW_{i:02d}]") for i in range(64)
|
| 358 |
+
]
|
| 359 |
+
inputs = tokenizer(
|
| 360 |
+
"[ROW_00]", return_tensors="pt", add_special_tokens=False
|
| 361 |
+
)
|
| 362 |
+
inputs = {k: v.to(device) for k, v in inputs.items()}
|
| 363 |
+
inputs.update(cond)
|
| 364 |
+
|
| 365 |
+
streamer = None
|
| 366 |
+
if verbose:
|
| 367 |
+
from transformers import TextStreamer
|
| 368 |
+
streamer = TextStreamer(tokenizer, skip_special_tokens=False)
|
| 369 |
+
|
| 370 |
+
with torch.no_grad():
|
| 371 |
+
output_ids = self.generate(
|
| 372 |
+
**inputs,
|
| 373 |
+
max_length=4096,
|
| 374 |
+
do_sample=True,
|
| 375 |
+
top_k=0,
|
| 376 |
+
top_p=top_p,
|
| 377 |
+
temperature=temperature,
|
| 378 |
+
pad_token_id=tokenizer.pad_token_id,
|
| 379 |
+
eos_token_id=tokenizer.eos_token_id,
|
| 380 |
+
logits_processor=[
|
| 381 |
+
_RowLengthLogitsProcessor(tokenizer, row_marker_ids)
|
| 382 |
+
],
|
| 383 |
+
streamer=streamer,
|
| 384 |
+
)
|
| 385 |
+
|
| 386 |
+
return _tokens_to_image(
|
| 387 |
+
tokenizer.decode(output_ids[0], skip_special_tokens=False)
|
| 388 |
+
)
|
| 389 |
+
|
| 390 |
+
|
| 391 |
+
class _RowLengthLogitsProcessor(LogitsProcessor):
|
| 392 |
+
def __init__(
|
| 393 |
+
self, tokenizer, row_marker_ids: list[int], row_width: int = 63
|
| 394 |
+
):
|
| 395 |
+
vocab = tokenizer.get_vocab()
|
| 396 |
+
self.row_marker_set = set(row_marker_ids)
|
| 397 |
+
self.row_marker_ids = row_marker_ids
|
| 398 |
+
self.eos_id = tokenizer.eos_token_id
|
| 399 |
+
self.bos_id = tokenizer.bos_token_id
|
| 400 |
+
self.row_width = row_width
|
| 401 |
+
special = {
|
| 402 |
+
tokenizer.eos_token_id,
|
| 403 |
+
tokenizer.bos_token_id,
|
| 404 |
+
tokenizer.pad_token_id,
|
| 405 |
+
*row_marker_ids,
|
| 406 |
+
}
|
| 407 |
+
self.pixel_len = torch.zeros(len(vocab), dtype=torch.long)
|
| 408 |
+
for token, idx in vocab.items():
|
| 409 |
+
if idx not in special:
|
| 410 |
+
self.pixel_len[idx] = len(token)
|
| 411 |
+
self.current_row = self.chars_in_row = 0
|
| 412 |
+
|
| 413 |
+
def __call__(
|
| 414 |
+
self, input_ids: torch.Tensor, scores: torch.Tensor
|
| 415 |
+
) -> torch.Tensor:
|
| 416 |
+
device = scores.device
|
| 417 |
+
pixel_len = self.pixel_len.to(device)
|
| 418 |
+
last_id = int(input_ids[0, -1].item())
|
| 419 |
+
if last_id in self.row_marker_set:
|
| 420 |
+
self.current_row = self.row_marker_ids.index(last_id)
|
| 421 |
+
self.chars_in_row = 0
|
| 422 |
+
remaining = self.row_width - self.chars_in_row
|
| 423 |
+
if remaining > 0:
|
| 424 |
+
mask = (pixel_len > remaining) | (pixel_len == 0)
|
| 425 |
+
if self.current_row < 63:
|
| 426 |
+
mask[self.eos_id] = True
|
| 427 |
+
scores = scores.masked_fill(mask.unsqueeze(0), float("-inf"))
|
| 428 |
+
else:
|
| 429 |
+
allowed = torch.full((scores.shape[-1],), True, device=device)
|
| 430 |
+
if self.current_row < 63:
|
| 431 |
+
allowed[self.row_marker_ids[self.current_row + 1]] = False
|
| 432 |
+
else:
|
| 433 |
+
allowed[self.eos_id] = False
|
| 434 |
+
scores = scores.masked_fill(allowed.unsqueeze(0), float("-inf"))
|
| 435 |
+
if last_id not in self.row_marker_set and last_id not in {
|
| 436 |
+
self.eos_id,
|
| 437 |
+
self.bos_id,
|
| 438 |
+
}:
|
| 439 |
+
self.chars_in_row += int(pixel_len[last_id].item())
|
| 440 |
+
return scores
|
| 441 |
+
|
| 442 |
+
|
| 443 |
+
def _tokens_to_image(text: str) -> np.ndarray:
|
| 444 |
+
rows, current = [], []
|
| 445 |
+
for token in text.split():
|
| 446 |
+
if token.startswith("[ROW_") and token.endswith("]"):
|
| 447 |
+
if current:
|
| 448 |
+
rows.append(current)
|
| 449 |
+
current = []
|
| 450 |
+
elif not (token.startswith("[") and token.endswith("]")):
|
| 451 |
+
current.append(token)
|
| 452 |
+
if current:
|
| 453 |
+
rows.append(current)
|
| 454 |
+
image = np.zeros((64, 64, 3), dtype=np.uint8)
|
| 455 |
+
for y, row in enumerate(rows[:64]):
|
| 456 |
+
for x, char in enumerate(row[:64]):
|
| 457 |
+
if char == "~":
|
| 458 |
+
image[y, x] = [255, 255, 255]
|
| 459 |
+
else:
|
| 460 |
+
idx = ord(char) - 59
|
| 461 |
+
r, g, b = idx // 16, (idx // 4) % 4, idx % 4
|
| 462 |
+
image[y, x] = [r * 64 + 32, g * 64 + 32, b * 64 + 32]
|
| 463 |
+
return image
|
| 464 |
+
|
| 465 |
+
|
| 466 |
+
if __name__ == "__main__":
|
| 467 |
+
from pathlib import Path
|
| 468 |
+
from PIL import Image
|
| 469 |
+
from huggingface_hub import snapshot_download
|
| 470 |
+
from transformers import AutoModelForCausalLM, PreTrainedTokenizerFast
|
| 471 |
+
|
| 472 |
+
REPO_ID = "iamthinbaker/GPokeT2"
|
| 473 |
+
REVISION = "v0.1-wip-3400"
|
| 474 |
+
N_SAMPLES = 5
|
| 475 |
+
OUTPUT_DIR = Path("generated")
|
| 476 |
+
|
| 477 |
+
print(f"Descargando {REPO_ID} ({REVISION})...")
|
| 478 |
+
ckpt = snapshot_download(REPO_ID, revision=REVISION)
|
| 479 |
+
tokenizer = PreTrainedTokenizerFast.from_pretrained(ckpt)
|
| 480 |
+
model = AutoModelForCausalLM.from_pretrained(ckpt, trust_remote_code=True)
|
| 481 |
+
model.eval()
|
| 482 |
+
|
| 483 |
+
OUTPUT_DIR.mkdir(exist_ok=True)
|
| 484 |
+
for i in range(N_SAMPLES):
|
| 485 |
+
print(f"\n[{i + 1}/{N_SAMPLES}] generando...")
|
| 486 |
+
image = model.generate_sprite(tokenizer, verbose=True)
|
| 487 |
+
path = OUTPUT_DIR / f"pokemon_{i + 1:02d}.png"
|
| 488 |
+
Image.fromarray(image).save(path)
|
| 489 |
+
print(f"guardado en {path}")
|