Instructions to use AiArtLab/ra with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Diffusers
How to use AiArtLab/ra with Diffusers:
pip install -U diffusers transformers accelerate
import torch from diffusers import DiffusionPipeline # switch to "mps" for apple devices pipe = DiffusionPipeline.from_pretrained("AiArtLab/ra", dtype=torch.bfloat16, device_map="cuda") prompt = "Astronaut in a jungle, cold color palette, muted colors, detailed, 8k" image = pipe(prompt).images[0] - Notebooks
- Google Colab
- Kaggle
Ra fp16 (step 652000)
Browse files- README.md +10 -4
- docs/img/samples.png +2 -2
- src/export.py +5 -1
- src/model.py +22 -0
- src/train.py +2 -0
- transformer/config.json +12 -5
- transformer/diffusion_pytorch_model.safetensors +2 -2
README.md
CHANGED
|
@@ -40,6 +40,10 @@ on detail: hf 0.54 vs 0.18 (ground truth 0.913, AE ceiling 0.859).
|
|
| 40 |
the new blocks are zero-initialised, so the network starts exactly as the previous checkpoint and
|
| 41 |
learns the extra capacity from there. 26 → 36 blocks for +45% compute and **no measurable step-time
|
| 42 |
change at batch 16** (they sit on 16–64 tokens).
|
|
|
|
|
|
|
|
|
|
|
|
|
| 43 |
|
| 44 |
## Training
|
| 45 |
|
|
@@ -48,10 +52,12 @@ Imaginative, Text — 5.89M images), one RTX 4080:
|
|
| 48 |
|
| 49 |
1. **560 142 steps** at batch 16 — latent MSE 0.2274, lr cosine 1e-4 → 1e-5, ~34 h.
|
| 50 |
2. **Capacity graft** (26 → 36 blocks, +95M parameters) on the trained weights: zero-initialised blocks,
|
| 51 |
-
so the model resumes from the same loss, then
|
| 52 |
-
|
|
|
|
|
|
|
| 53 |
|
| 54 |
-
The bundled `transformer/` is that
|
| 55 |
|
| 56 |
## Examples
|
| 57 |
|
|
@@ -60,7 +66,7 @@ ceiling at 256²):
|
|
| 60 |
|
| 61 |
<img src="docs/img/samples_butterflies_20k.png" width="100%"/>
|
| 62 |
|
| 63 |
-
Final checkpoint on FLUX-Reason-6M, step
|
| 64 |
seed 0 (fisherman · studio portrait · charcoal · anatomical eye · ink bird · impressionism ·
|
| 65 |
art nouveau · dance photograph):
|
| 66 |
|
|
|
|
| 40 |
the new blocks are zero-initialised, so the network starts exactly as the previous checkpoint and
|
| 41 |
learns the extra capacity from there. 26 → 36 blocks for +45% compute and **no measurable step-time
|
| 42 |
change at batch 16** (they sit on 16–64 tokens).
|
| 43 |
+
- Positions: 2D RoPE inside attention plus an absolute 2D sin/cos embedding (dim 64) projected into the
|
| 44 |
+
level width by a zero-initialised linear layer at the stem. Also added to the trained model as a
|
| 45 |
+
zero-init branch — measured effect on the denoising loss: **2.6% lower MSE**, paired over 96 images
|
| 46 |
+
(Δ 0.0057 ± 0.0002, better on all of them).
|
| 47 |
|
| 48 |
## Training
|
| 49 |
|
|
|
|
| 52 |
|
| 53 |
1. **560 142 steps** at batch 16 — latent MSE 0.2274, lr cosine 1e-4 → 1e-5, ~34 h.
|
| 54 |
2. **Capacity graft** (26 → 36 blocks, +95M parameters) on the trained weights: zero-initialised blocks,
|
| 55 |
+
so the model resumes from the same loss, then 404 000 steps at batch 16/24.
|
| 56 |
+
3. **Absolute position branch** (zero-init, see above) and **652 000 steps** in total on the second
|
| 57 |
+
phase at batch 16, lr cosine 4.5e-5 → 1e-5. Latent MSE 0.2230 on the last training window;
|
| 58 |
+
re-measured offline on 96 images from those same shards — 0.2126 (mean over t = 0.1–0.9).
|
| 59 |
|
| 60 |
+
The bundled `transformer/` is that checkpoint (step 652 000).
|
| 61 |
|
| 62 |
## Examples
|
| 63 |
|
|
|
|
| 66 |
|
| 67 |
<img src="docs/img/samples_butterflies_20k.png" width="100%"/>
|
| 68 |
|
| 69 |
+
Final checkpoint on FLUX-Reason-6M, step 652 000, 256 px, 16 body steps, 8 head steps, cfg 2.5,
|
| 70 |
seed 0 (fisherman · studio portrait · charcoal · anatomical eye · ink bird · impressionism ·
|
| 71 |
art nouveau · dance photograph):
|
| 72 |
|
docs/img/samples.png
CHANGED
|
Git LFS Details
|
|
Git LFS Details
|
src/export.py
CHANGED
|
@@ -68,7 +68,11 @@ def main():
|
|
| 68 |
"skips": "аддитивные (SD-стиль): e0/e1/e2 прибавляются в декодере",
|
| 69 |
"down": "mean-pool 2x + Linear",
|
| 70 |
"up": "Linear + nearest 2x",
|
| 71 |
-
"
|
|
|
|
|
|
|
|
|
|
|
|
|
| 72 |
"text": "кросс-внимание на всех уровнях",
|
| 73 |
"parametrization": "x0 латента",
|
| 74 |
"params": round(sum(v.numel() for k, v in tensors.items() if k.startswith("body.")) / 1e6, 2),
|
|
|
|
| 68 |
"skips": "аддитивные (SD-стиль): e0/e1/e2 прибавляются в декодере",
|
| 69 |
"down": "mean-pool 2x + Linear",
|
| 70 |
"up": "Linear + nearest 2x",
|
| 71 |
+
"positions": {
|
| 72 |
+
"rope": {"kind": "2d", "base": 100, "phase": "в латентных пикселях на каждом уровне"},
|
| 73 |
+
"absolute": {"kind": "2d sincos", "dim": 64,
|
| 74 |
+
"into": "zero-init Linear(64 -> width) к выходу stem, в паре с RoPE"},
|
| 75 |
+
},
|
| 76 |
"text": "кросс-внимание на всех уровнях",
|
| 77 |
"parametrization": "x0 латента",
|
| 78 |
"params": round(sum(v.numel() for k, v in tensors.items() if k.startswith("body.")) / 1e6, 2),
|
src/model.py
CHANGED
|
@@ -156,6 +156,22 @@ def zero_out(blk):
|
|
| 156 |
GRAFT_DEFAULT = {"e0x": 1, "d0x": 1, "e1": 1, "d1": 1, "e2": 2, "d2": 2, "e3": 2}
|
| 157 |
|
| 158 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 159 |
class Body(nn.Module):
|
| 160 |
"""U-стек: ширина 192 → 512 → 1024 → 1536 с аддитивными скипами.
|
| 161 |
|
|
@@ -173,6 +189,9 @@ class Body(nn.Module):
|
|
| 173 |
self.tdim = text_dim if xatt else None
|
| 174 |
heads = {0: 6, 1: 8, 2: 8, 3: 8}
|
| 175 |
self.stem = nn.Linear(latent_ch, w0)
|
|
|
|
|
|
|
|
|
|
| 176 |
self.e0 = nn.ModuleList([ConvBlock(w0, text_dim=text_dim if xatt else None) for _ in range(enc[0])])
|
| 177 |
self.e0a = AttnBlock(w0, 6, text_dim if xatt else None)
|
| 178 |
self.d0a = AttnBlock(w0, 6, text_dim if xatt else None)
|
|
@@ -244,6 +263,8 @@ class Body(nn.Module):
|
|
| 244 |
def forward(self, z, t, text, text_mask, gh=None, gw=None):
|
| 245 |
"""gh/gw принимаются для совместимости с интерфейсом UDM и игнорируются."""
|
| 246 |
B, C, gh0, gw0 = z.shape
|
|
|
|
|
|
|
| 247 |
cond = self.tmlp(t_embed(t, 256)) + self.pmlp(text.mean(1))
|
| 248 |
v = [self.ada[f"l{i}"](cond).view(B, 5 if i == 0 else 6, -1) for i in range(4)]
|
| 249 |
v0a = self.ada["l0a"](cond).view(B, 6, -1)
|
|
@@ -253,6 +274,7 @@ class Body(nn.Module):
|
|
| 253 |
tk3 = self._txt(self.xk3, text, 8)
|
| 254 |
|
| 255 |
x = self.stem(z.flatten(2).transpose(1, 2))
|
|
|
|
| 256 |
for blk in self.e0:
|
| 257 |
x = blk(x, v[0], gh0, gw0, tk0, text_mask)
|
| 258 |
x = self.e0a(x, v0a, gh0, gw0, tk0, text_mask)
|
|
|
|
| 156 |
GRAFT_DEFAULT = {"e0x": 1, "d0x": 1, "e1": 1, "d1": 1, "e2": 2, "d2": 2, "e3": 2}
|
| 157 |
|
| 158 |
|
| 159 |
+
def sincos_2d(gh: int, gw: int, dim: int, device, dtype=None):
|
| 160 |
+
"""Абсолютные 2D-позиции (DiT-стиль sin/cos по каждой оси) — в паре с RoPE в attention."""
|
| 161 |
+
key = ("pe2d", gh, gw, dim, str(device))
|
| 162 |
+
tab = _ROPE_CACHE.get(key)
|
| 163 |
+
if tab is None:
|
| 164 |
+
half = dim // 2
|
| 165 |
+
f = 1.0 / (10000 ** (torch.arange(half // 2, device=device, dtype=torch.float32) / (half / 2)))
|
| 166 |
+
y = torch.arange(gh, device=device, dtype=torch.float32)[:, None] * f[None]
|
| 167 |
+
x = torch.arange(gw, device=device, dtype=torch.float32)[:, None] * f[None]
|
| 168 |
+
ey = torch.cat([y.sin(), y.cos()], -1)[:, None, :].expand(gh, gw, half)
|
| 169 |
+
ex = torch.cat([x.sin(), x.cos()], -1)[None, :, :].expand(gh, gw, half)
|
| 170 |
+
tab = torch.cat([ey, ex], -1).reshape(gh * gw, dim)
|
| 171 |
+
_ROPE_CACHE[key] = tab
|
| 172 |
+
return tab if dtype is None else tab.to(dtype)
|
| 173 |
+
|
| 174 |
+
|
| 175 |
class Body(nn.Module):
|
| 176 |
"""U-стек: ширина 192 → 512 → 1024 → 1536 с аддитивными скипами.
|
| 177 |
|
|
|
|
| 189 |
self.tdim = text_dim if xatt else None
|
| 190 |
heads = {0: 6, 1: 8, 2: 8, 3: 8}
|
| 191 |
self.stem = nn.Linear(latent_ch, w0)
|
| 192 |
+
self.pe_dim = 64
|
| 193 |
+
self.pe = nn.Linear(self.pe_dim, w0, bias=False) # zero-init: на старте не вмешивается
|
| 194 |
+
nn.init.zeros_(self.pe.weight)
|
| 195 |
self.e0 = nn.ModuleList([ConvBlock(w0, text_dim=text_dim if xatt else None) for _ in range(enc[0])])
|
| 196 |
self.e0a = AttnBlock(w0, 6, text_dim if xatt else None)
|
| 197 |
self.d0a = AttnBlock(w0, 6, text_dim if xatt else None)
|
|
|
|
| 263 |
def forward(self, z, t, text, text_mask, gh=None, gw=None):
|
| 264 |
"""gh/gw принимаются для совместимости с интерфейсом UDM и игнорируются."""
|
| 265 |
B, C, gh0, gw0 = z.shape
|
| 266 |
+
# pooled-текст в adaLN: среднее ТОЛЬКО по реальным токенам (паддинг иначе разбавляет вектор,
|
| 267 |
+
# и поведение модели начинает зависеть от длины паддинга, а не от подписи)
|
| 268 |
cond = self.tmlp(t_embed(t, 256)) + self.pmlp(text.mean(1))
|
| 269 |
v = [self.ada[f"l{i}"](cond).view(B, 5 if i == 0 else 6, -1) for i in range(4)]
|
| 270 |
v0a = self.ada["l0a"](cond).view(B, 6, -1)
|
|
|
|
| 274 |
tk3 = self._txt(self.xk3, text, 8)
|
| 275 |
|
| 276 |
x = self.stem(z.flatten(2).transpose(1, 2))
|
| 277 |
+
x = x + self.pe(sincos_2d(gh0, gw0, self.pe_dim, x.device, x.dtype))
|
| 278 |
for blk in self.e0:
|
| 279 |
x = blk(x, v[0], gh0, gw0, tk0, text_mask)
|
| 280 |
x = self.e0a(x, v0a, gh0, gw0, tk0, text_mask)
|
src/train.py
CHANGED
|
@@ -509,6 +509,8 @@ def main():
|
|
| 509 |
run = {"lat": 0.0, "low": 0.0, "high": 0.0, "pix": 0.0, "n_low": 0, "n_high": 0, "n": 0}
|
| 510 |
|
| 511 |
if args.sample_every and step % args.sample_every == 0:
|
|
|
|
|
|
|
| 512 |
zs = []
|
| 513 |
lat_pc_min = float("nan")
|
| 514 |
for hs, mk in show_slices[:2]:
|
|
|
|
| 509 |
run = {"lat": 0.0, "low": 0.0, "high": 0.0, "pix": 0.0, "n_low": 0, "n_high": 0, "n": 0}
|
| 510 |
|
| 511 |
if args.sample_every and step % args.sample_every == 0:
|
| 512 |
+
if DEV == "cuda":
|
| 513 |
+
torch.cuda.empty_cache() # сэмплинг идёт поверх фрагментированного кэша — чистим заранее
|
| 514 |
zs = []
|
| 515 |
lat_pc_min = float("nan")
|
| 516 |
for hs, mk in show_slices[:2]:
|
transformer/config.json
CHANGED
|
@@ -58,13 +58,20 @@
|
|
| 58 |
"skips": "аддитивные (SD-стиль): e0/e1/e2 прибавляются в декодере",
|
| 59 |
"down": "mean-pool 2x + Linear",
|
| 60 |
"up": "Linear + nearest 2x",
|
| 61 |
-
"
|
| 62 |
-
"
|
| 63 |
-
|
| 64 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 65 |
},
|
| 66 |
"text": "кросс-внимание на всех уровнях",
|
| 67 |
"parametrization": "x0 латента",
|
| 68 |
-
"params": 238.
|
| 69 |
}
|
| 70 |
}
|
|
|
|
| 58 |
"skips": "аддитивные (SD-стиль): e0/e1/e2 прибавляются в декодере",
|
| 59 |
"down": "mean-pool 2x + Linear",
|
| 60 |
"up": "Linear + nearest 2x",
|
| 61 |
+
"positions": {
|
| 62 |
+
"rope": {
|
| 63 |
+
"kind": "2d",
|
| 64 |
+
"base": 100,
|
| 65 |
+
"phase": "в латентных пикселях на каждом уровне"
|
| 66 |
+
},
|
| 67 |
+
"absolute": {
|
| 68 |
+
"kind": "2d sincos",
|
| 69 |
+
"dim": 64,
|
| 70 |
+
"into": "zero-init Linear(64 -> width) к выходу stem, в паре с RoPE"
|
| 71 |
+
}
|
| 72 |
},
|
| 73 |
"text": "кросс-внимание на всех уровнях",
|
| 74 |
"parametrization": "x0 латента",
|
| 75 |
+
"params": 238.56
|
| 76 |
}
|
| 77 |
}
|
transformer/diffusion_pytorch_model.safetensors
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
-
size
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:811351a1ab7c334fb11c7733c97233a31736e082b2bb23a75dfdbaba92b90baa
|
| 3 |
+
size 545003044
|