recoilme commited on
Commit
234d8d3
·
verified ·
1 Parent(s): cb6884f

Ra fp16 (step 652000)

Browse files
README.md CHANGED
@@ -40,6 +40,10 @@ on detail: hf 0.54 vs 0.18 (ground truth 0.913, AE ceiling 0.859).
40
  the new blocks are zero-initialised, so the network starts exactly as the previous checkpoint and
41
  learns the extra capacity from there. 26 → 36 blocks for +45% compute and **no measurable step-time
42
  change at batch 16** (they sit on 16–64 tokens).
 
 
 
 
43
 
44
  ## Training
45
 
@@ -48,10 +52,12 @@ Imaginative, Text — 5.89M images), one RTX 4080:
48
 
49
  1. **560 142 steps** at batch 16 — latent MSE 0.2274, lr cosine 1e-4 → 1e-5, ~34 h.
50
  2. **Capacity graft** (26 → 36 blocks, +95M parameters) on the trained weights: zero-initialised blocks,
51
- so the model resumes from the same loss, then 460 000 more steps at batch 16/24, lr cosine
52
- 4.5e-5 → 1e-5. Latent MSE on the last window 0.2257.
 
 
53
 
54
- The bundled `transformer/` is that second-phase checkpoint (step 460 000).
55
 
56
  ## Examples
57
 
@@ -60,7 +66,7 @@ ceiling at 256²):
60
 
61
  <img src="docs/img/samples_butterflies_20k.png" width="100%"/>
62
 
63
- Final checkpoint on FLUX-Reason-6M, step 460 000, 256 px, 16 body steps, 8 head steps, cfg 2.5,
64
  seed 0 (fisherman · studio portrait · charcoal · anatomical eye · ink bird · impressionism ·
65
  art nouveau · dance photograph):
66
 
 
40
  the new blocks are zero-initialised, so the network starts exactly as the previous checkpoint and
41
  learns the extra capacity from there. 26 → 36 blocks for +45% compute and **no measurable step-time
42
  change at batch 16** (they sit on 16–64 tokens).
43
+ - Positions: 2D RoPE inside attention plus an absolute 2D sin/cos embedding (dim 64) projected into the
44
+ level width by a zero-initialised linear layer at the stem. Also added to the trained model as a
45
+ zero-init branch — measured effect on the denoising loss: **2.6% lower MSE**, paired over 96 images
46
+ (Δ 0.0057 ± 0.0002, better on all of them).
47
 
48
  ## Training
49
 
 
52
 
53
  1. **560 142 steps** at batch 16 — latent MSE 0.2274, lr cosine 1e-4 → 1e-5, ~34 h.
54
  2. **Capacity graft** (26 → 36 blocks, +95M parameters) on the trained weights: zero-initialised blocks,
55
+ so the model resumes from the same loss, then 404 000 steps at batch 16/24.
56
+ 3. **Absolute position branch** (zero-init, see above) and **652 000 steps** in total on the second
57
+ phase at batch 16, lr cosine 4.5e-5 → 1e-5. Latent MSE 0.2230 on the last training window;
58
+ re-measured offline on 96 images from those same shards — 0.2126 (mean over t = 0.1–0.9).
59
 
60
+ The bundled `transformer/` is that checkpoint (step 652 000).
61
 
62
  ## Examples
63
 
 
66
 
67
  <img src="docs/img/samples_butterflies_20k.png" width="100%"/>
68
 
69
+ Final checkpoint on FLUX-Reason-6M, step 652 000, 256 px, 16 body steps, 8 head steps, cfg 2.5,
70
  seed 0 (fisherman · studio portrait · charcoal · anatomical eye · ink bird · impressionism ·
71
  art nouveau · dance photograph):
72
 
docs/img/samples.png CHANGED

Git LFS Details

  • SHA256: 3b6a2e8e6393424eb2fa64097211d9d7b66306b4f726cb0e5ee8d03df09c4759
  • Pointer size: 131 Bytes
  • Size of remote file: 646 kB

Git LFS Details

  • SHA256: 392c2357b338645654b4fc7dac8c9cbea23674661ac2d244bb0402dc33cdc631
  • Pointer size: 131 Bytes
  • Size of remote file: 621 kB
src/export.py CHANGED
@@ -68,7 +68,11 @@ def main():
68
  "skips": "аддитивные (SD-стиль): e0/e1/e2 прибавляются в декодере",
69
  "down": "mean-pool 2x + Linear",
70
  "up": "Linear + nearest 2x",
71
- "rope": {"kind": "2d", "base": 100, "phase": "в латентных пикселях на каждом уровне"},
 
 
 
 
72
  "text": "кросс-внимание на всех уровнях",
73
  "parametrization": "x0 латента",
74
  "params": round(sum(v.numel() for k, v in tensors.items() if k.startswith("body.")) / 1e6, 2),
 
68
  "skips": "аддитивные (SD-стиль): e0/e1/e2 прибавляются в декодере",
69
  "down": "mean-pool 2x + Linear",
70
  "up": "Linear + nearest 2x",
71
+ "positions": {
72
+ "rope": {"kind": "2d", "base": 100, "phase": "в латентных пикселях на каждом уровне"},
73
+ "absolute": {"kind": "2d sincos", "dim": 64,
74
+ "into": "zero-init Linear(64 -> width) к выходу stem, в паре с RoPE"},
75
+ },
76
  "text": "кросс-внимание на всех уровнях",
77
  "parametrization": "x0 латента",
78
  "params": round(sum(v.numel() for k, v in tensors.items() if k.startswith("body.")) / 1e6, 2),
src/model.py CHANGED
@@ -156,6 +156,22 @@ def zero_out(blk):
156
  GRAFT_DEFAULT = {"e0x": 1, "d0x": 1, "e1": 1, "d1": 1, "e2": 2, "d2": 2, "e3": 2}
157
 
158
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
159
  class Body(nn.Module):
160
  """U-стек: ширина 192 → 512 → 1024 → 1536 с аддитивными скипами.
161
 
@@ -173,6 +189,9 @@ class Body(nn.Module):
173
  self.tdim = text_dim if xatt else None
174
  heads = {0: 6, 1: 8, 2: 8, 3: 8}
175
  self.stem = nn.Linear(latent_ch, w0)
 
 
 
176
  self.e0 = nn.ModuleList([ConvBlock(w0, text_dim=text_dim if xatt else None) for _ in range(enc[0])])
177
  self.e0a = AttnBlock(w0, 6, text_dim if xatt else None)
178
  self.d0a = AttnBlock(w0, 6, text_dim if xatt else None)
@@ -244,6 +263,8 @@ class Body(nn.Module):
244
  def forward(self, z, t, text, text_mask, gh=None, gw=None):
245
  """gh/gw принимаются для совместимости с интерфейсом UDM и игнорируются."""
246
  B, C, gh0, gw0 = z.shape
 
 
247
  cond = self.tmlp(t_embed(t, 256)) + self.pmlp(text.mean(1))
248
  v = [self.ada[f"l{i}"](cond).view(B, 5 if i == 0 else 6, -1) for i in range(4)]
249
  v0a = self.ada["l0a"](cond).view(B, 6, -1)
@@ -253,6 +274,7 @@ class Body(nn.Module):
253
  tk3 = self._txt(self.xk3, text, 8)
254
 
255
  x = self.stem(z.flatten(2).transpose(1, 2))
 
256
  for blk in self.e0:
257
  x = blk(x, v[0], gh0, gw0, tk0, text_mask)
258
  x = self.e0a(x, v0a, gh0, gw0, tk0, text_mask)
 
156
  GRAFT_DEFAULT = {"e0x": 1, "d0x": 1, "e1": 1, "d1": 1, "e2": 2, "d2": 2, "e3": 2}
157
 
158
 
159
+ def sincos_2d(gh: int, gw: int, dim: int, device, dtype=None):
160
+ """Абсолютные 2D-позиции (DiT-стиль sin/cos по каждой оси) — в паре с RoPE в attention."""
161
+ key = ("pe2d", gh, gw, dim, str(device))
162
+ tab = _ROPE_CACHE.get(key)
163
+ if tab is None:
164
+ half = dim // 2
165
+ f = 1.0 / (10000 ** (torch.arange(half // 2, device=device, dtype=torch.float32) / (half / 2)))
166
+ y = torch.arange(gh, device=device, dtype=torch.float32)[:, None] * f[None]
167
+ x = torch.arange(gw, device=device, dtype=torch.float32)[:, None] * f[None]
168
+ ey = torch.cat([y.sin(), y.cos()], -1)[:, None, :].expand(gh, gw, half)
169
+ ex = torch.cat([x.sin(), x.cos()], -1)[None, :, :].expand(gh, gw, half)
170
+ tab = torch.cat([ey, ex], -1).reshape(gh * gw, dim)
171
+ _ROPE_CACHE[key] = tab
172
+ return tab if dtype is None else tab.to(dtype)
173
+
174
+
175
  class Body(nn.Module):
176
  """U-стек: ширина 192 → 512 → 1024 → 1536 с аддитивными скипами.
177
 
 
189
  self.tdim = text_dim if xatt else None
190
  heads = {0: 6, 1: 8, 2: 8, 3: 8}
191
  self.stem = nn.Linear(latent_ch, w0)
192
+ self.pe_dim = 64
193
+ self.pe = nn.Linear(self.pe_dim, w0, bias=False) # zero-init: на старте не вмешивается
194
+ nn.init.zeros_(self.pe.weight)
195
  self.e0 = nn.ModuleList([ConvBlock(w0, text_dim=text_dim if xatt else None) for _ in range(enc[0])])
196
  self.e0a = AttnBlock(w0, 6, text_dim if xatt else None)
197
  self.d0a = AttnBlock(w0, 6, text_dim if xatt else None)
 
263
  def forward(self, z, t, text, text_mask, gh=None, gw=None):
264
  """gh/gw принимаются для совместимости с интерфейсом UDM и игнорируются."""
265
  B, C, gh0, gw0 = z.shape
266
+ # pooled-текст в adaLN: среднее ТОЛЬКО по реальным токенам (паддинг иначе разбавляет вектор,
267
+ # и поведение модели начинает зависеть от длины паддинга, а не от подписи)
268
  cond = self.tmlp(t_embed(t, 256)) + self.pmlp(text.mean(1))
269
  v = [self.ada[f"l{i}"](cond).view(B, 5 if i == 0 else 6, -1) for i in range(4)]
270
  v0a = self.ada["l0a"](cond).view(B, 6, -1)
 
274
  tk3 = self._txt(self.xk3, text, 8)
275
 
276
  x = self.stem(z.flatten(2).transpose(1, 2))
277
+ x = x + self.pe(sincos_2d(gh0, gw0, self.pe_dim, x.device, x.dtype))
278
  for blk in self.e0:
279
  x = blk(x, v[0], gh0, gw0, tk0, text_mask)
280
  x = self.e0a(x, v0a, gh0, gw0, tk0, text_mask)
src/train.py CHANGED
@@ -509,6 +509,8 @@ def main():
509
  run = {"lat": 0.0, "low": 0.0, "high": 0.0, "pix": 0.0, "n_low": 0, "n_high": 0, "n": 0}
510
 
511
  if args.sample_every and step % args.sample_every == 0:
 
 
512
  zs = []
513
  lat_pc_min = float("nan")
514
  for hs, mk in show_slices[:2]:
 
509
  run = {"lat": 0.0, "low": 0.0, "high": 0.0, "pix": 0.0, "n_low": 0, "n_high": 0, "n": 0}
510
 
511
  if args.sample_every and step % args.sample_every == 0:
512
+ if DEV == "cuda":
513
+ torch.cuda.empty_cache() # сэмплинг идёт поверх фрагментированного кэша — чистим заранее
514
  zs = []
515
  lat_pc_min = float("nan")
516
  for hs, mk in show_slices[:2]:
transformer/config.json CHANGED
@@ -58,13 +58,20 @@
58
  "skips": "аддитивные (SD-стиль): e0/e1/e2 прибавляются в декодере",
59
  "down": "mean-pool 2x + Linear",
60
  "up": "Linear + nearest 2x",
61
- "rope": {
62
- "kind": "2d",
63
- "base": 100,
64
- "phase": "в латентных пикселях на каждом уровне"
 
 
 
 
 
 
 
65
  },
66
  "text": "кросс-внимание на всех уровнях",
67
  "parametrization": "x0 латента",
68
- "params": 238.54
69
  }
70
  }
 
58
  "skips": "аддитивные (SD-стиль): e0/e1/e2 прибавляются в декодере",
59
  "down": "mean-pool 2x + Linear",
60
  "up": "Linear + nearest 2x",
61
+ "positions": {
62
+ "rope": {
63
+ "kind": "2d",
64
+ "base": 100,
65
+ "phase": "в латентных пикселях на каждом уровне"
66
+ },
67
+ "absolute": {
68
+ "kind": "2d sincos",
69
+ "dim": 64,
70
+ "into": "zero-init Linear(64 -> width) к выходу stem, в паре с RoPE"
71
+ }
72
  },
73
  "text": "кросс-внимание на всех уровнях",
74
  "parametrization": "x0 латента",
75
+ "params": 238.56
76
  }
77
  }
transformer/diffusion_pytorch_model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:06a9cca2c9e3057c9416c8f2a4176063ab6dba2c3dc1c08c30a11c5ce89bf439
3
- size 544978380
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:811351a1ab7c334fb11c7733c97233a31736e082b2bb23a75dfdbaba92b90baa
3
+ size 545003044