multimodalart HF Staff commited on
Commit
1587925
·
verified ·
1 Parent(s): 8bc5857

Upload folder using huggingface_hub

Browse files
Files changed (4) hide show
  1. README.md +22 -7
  2. app.py +138 -0
  3. dit.py +263 -0
  4. requirements.txt +2 -0
README.md CHANGED
@@ -1,13 +1,28 @@
1
  ---
2
- title: Agent Genpath
3
- emoji: 📚
4
- colorFrom: red
5
- colorTo: purple
6
  sdk: gradio
7
  sdk_version: 6.31.0
8
- python_version: '3.12'
9
  app_file: app.py
10
- pinned: false
 
 
11
  ---
12
 
13
- Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  ---
2
+ title: Agent-GenPath
3
+ emoji: 🔬
4
+ colorFrom: blue
5
+ colorTo: green
6
  sdk: gradio
7
  sdk_version: 6.31.0
 
8
  app_file: app.py
9
+ short_description: Pathology patch synthesis with the Agent-GenPath DiT
10
+ python_version: "3.12"
11
+ startup_duration_timeout: 30m
12
  ---
13
 
14
+ # Agent-GenPath — Pathology Patch Synthesis
15
+
16
+ Demo for **"Agent-Driven Generative Priors for Pathology Image Synthesis and Segmentation"**
17
+ ([arXiv 2611.06701](https://arxiv.org/abs/2611.06701)).
18
+
19
+ Samples 256×256 synthetic H&E pathology patches from the paper's pretrained unconditional
20
+ flow-matching Diffusion Transformer ([`yscript/agent-genpath`](https://huggingface.co/yscript/agent-genpath)),
21
+ using the reference Euler sampler from the official
22
+ [Agent-GenPath repo](https://github.com/script-Yang/Agent-GenPath) (Apache-2.0).
23
+
24
+ The backbone is trained on **TCGA-200K**, a 217,030-patch dataset the paper's pathology agent
25
+ system curated from the public TCGA repository; the same backbone transfers to semantic
26
+ segmentation with a lightweight head (PathSeg-380K).
27
+
28
+ The model definition (`dit.py`) is copied verbatim from the official repository.
app.py ADDED
@@ -0,0 +1,138 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import spaces # MUST come before torch
2
+ import os
3
+
4
+ import hashlib
5
+ import pathlib
6
+ import tempfile
7
+ import time
8
+
9
+ import gradio as gr
10
+ import torch
11
+ from huggingface_hub import hf_hub_download
12
+ from torchvision.utils import save_image
13
+
14
+ from dit import MFDiT
15
+
16
+ MODEL_ID = "yscript/agent-genpath"
17
+ INPUT_SIZE = 256
18
+ # Backbone hyper-parameters exactly as in configs/sample.json from the paper repo.
19
+ BACKBONE = dict(
20
+ input_size=256,
21
+ patch_size=8,
22
+ in_channels=3,
23
+ dim=768,
24
+ depth=12,
25
+ num_heads=12,
26
+ num_classes=None,
27
+ )
28
+
29
+ # --- Model load (module scope, eager .to("cuda"); ZeroGPU intercepts) -----------
30
+ model = MFDiT(**BACKBONE)
31
+ _ckpt = hf_hub_download(MODEL_ID, "dit.pth")
32
+ _state = torch.load(_ckpt, map_location="cpu", weights_only=True)
33
+ _state = {k.removeprefix("module."): v for k, v in _state.items()}
34
+ model.load_state_dict(_state, strict=True)
35
+ model.to("cuda").eval()
36
+ del _state
37
+
38
+
39
+ @torch.no_grad()
40
+ def euler_sample(batch, steps, seed):
41
+ """Reference Euler sampler from generation/sample.py (Agent-GenPath repo)."""
42
+ torch.manual_seed(seed)
43
+ device = next(model.parameters()).device
44
+ x = torch.randn(batch, 3, INPUT_SIZE, INPUT_SIZE, device=device)
45
+ for i in range(steps):
46
+ t = torch.full((batch,), 1 - i / steps, device=device)
47
+ x = x - model(x, t, torch.zeros_like(t), y=None) / steps
48
+ return (x.clamp(-1, 1) + 1) / 2
49
+
50
+
51
+ def _duration(batch, steps, *args, **kwargs):
52
+ # Swallow extra positional/keyword args (Gradio injects progress=).
53
+ return min(180, 20 + int(steps) * 1.6 * max(1, (int(batch) + 3) // 4))
54
+
55
+
56
+ @spaces.GPU(duration=_duration)
57
+ def generate(seed=6657, batch=8, steps=50, progress=gr.Progress(track_tqdm=True)):
58
+ """Sample synthetic pathology image patches with the pretrained Agent-GenPath flow-matching DiT.
59
+
60
+ Args:
61
+ seed: RNG seed for the whole batch; same seed + settings reproduces the same patches.
62
+ batch: number of 256x256 RGB patches to synthesize in one go (1-8).
63
+ steps: number of Euler integration steps (paper showcase uses 50).
64
+ progress: Gradio progress bar.
65
+
66
+ Returns:
67
+ List of file paths to the generated PNG patches.
68
+ """
69
+ seed = int(seed)
70
+ batch = max(1, min(int(batch), 8))
71
+ steps = max(1, min(int(steps), 100))
72
+ t0 = time.perf_counter()
73
+ imgs = euler_sample(batch, steps, seed)
74
+ dt = time.perf_counter() - t0
75
+ out = (
76
+ pathlib.Path(tempfile.gettempdir())
77
+ / f"agent-genpath-{seed}-{batch}-{steps}-{os.getpid()}"
78
+ )
79
+ out.mkdir(parents=True, exist_ok=True)
80
+ paths = []
81
+ for i, image in enumerate(imgs.cpu()):
82
+ p = out / f"sample_{i + 1:05d}.png"
83
+ save_image(image, p)
84
+ paths.append(str(p))
85
+ print(
86
+ f"[agent-genpath] seed={seed} batch={batch} steps={steps} took {dt:.1f}s",
87
+ flush=True,
88
+ )
89
+ return paths
90
+
91
+
92
+ CSS = """
93
+ #col-container { max-width: 1100px; margin: 0 auto; }
94
+ .dark .gradio-container { color: var(--body-text-color); }
95
+ """
96
+
97
+ with gr.Blocks(theme=gr.themes.Citrus(), css=CSS) as demo:
98
+ gr.Markdown(
99
+ """
100
+ # Agent-GenPath — Pathology Patch Synthesis
101
+ **Agent-Driven Generative Priors for Pathology Image Synthesis and Segmentation** ([arXiv 2611.06701](https://arxiv.org/abs/2611.06701))
102
+
103
+ Unconditional flow-matching Diffusion Transformer (DiT) trained on **TCGA-200K**, a
104
+ 217,030-patch pathology dataset curated by the paper's pathology agent system. The same
105
+ backbone serves as a generative prior for downstream semantic segmentation. This demo
106
+ samples 256×256 synthetic H&E patches from the released checkpoint
107
+ ([`yscript/agent-genpath`](https://huggingface.co/yscript/agent-genpath)) with the
108
+ authors' reference Euler sampler.
109
+
110
+ *Code: [script-Yang/Agent-GenPath](https://github.com/script-Yang/Agent-GenPath) · Paper: [arXiv 2611.06701](https://arxiv.org/abs/2611.06701)*
111
+ """
112
+ )
113
+ with gr.Column(elem_id="col-container"):
114
+ with gr.Row():
115
+ seed = gr.Number(label="Seed", value=6657, precision=0)
116
+ batch = gr.Slider(1, 8, value=8, step=1, label="Batch size")
117
+ steps = gr.Slider(10, 100, value=50, step=5, label="Euler steps")
118
+ run = gr.Button("Sample patches", variant="primary")
119
+ gallery = gr.Gallery(
120
+ label="Synthetic pathology patches",
121
+ columns=4,
122
+ height=520,
123
+ format="png",
124
+ type="filepath",
125
+ buttons=["download", "fullscreen"],
126
+ )
127
+ gr.Examples(
128
+ examples=[[6657, 8, 50]],
129
+ inputs=[seed, batch, steps],
130
+ outputs=gallery,
131
+ fn=generate,
132
+ cache_examples=True,
133
+ cache_mode="lazy",
134
+ label="Paper showcase batch (seed 6657 · 8 patches · 50 Euler steps)",
135
+ )
136
+ run.click(generate, inputs=[seed, batch, steps], outputs=gallery, api_name="generate")
137
+
138
+ demo.launch(mcp_server=True)
dit.py ADDED
@@ -0,0 +1,263 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ import torch.nn as nn
3
+ import numpy as np
4
+ import math
5
+ from timm.models.vision_transformer import PatchEmbed, Mlp
6
+ from timm.models.vision_transformer import Attention
7
+ import torch.nn.functional as F
8
+
9
+
10
+ def modulate(x, scale, shift):
11
+ return x * (1 + scale.unsqueeze(1)) + shift.unsqueeze(1)
12
+
13
+
14
+ class TimestepEmbedder(nn.Module):
15
+ def __init__(self, dim, nfreq=256):
16
+ super().__init__()
17
+ self.mlp = nn.Sequential(nn.Linear(nfreq, dim), nn.SiLU(), nn.Linear(dim, dim))
18
+ self.nfreq = nfreq
19
+
20
+ @staticmethod
21
+ def timestep_embedding(t, dim, max_period=10000):
22
+ half_dim = dim // 2
23
+ freqs = torch.exp(
24
+ -math.log(max_period)
25
+ * torch.arange(start=0, end=half_dim, dtype=torch.float32)
26
+ / half_dim
27
+ ).to(device=t.device)
28
+ args = t[:, None].float() * freqs[None]
29
+ embedding = torch.cat([torch.cos(args), torch.sin(args)], dim=-1)
30
+ if dim % 2:
31
+ embedding = torch.cat([embedding, torch.zeros_like(embedding[:, :1])], dim=-1)
32
+ return embedding
33
+
34
+ def forward(self, t):
35
+ t = t * 1000
36
+ t_freq = self.timestep_embedding(t, self.nfreq)
37
+ t_emb = self.mlp(t_freq)
38
+ return t_emb
39
+
40
+
41
+ class LabelEmbedder(nn.Module):
42
+ def __init__(self, num_classes, dim):
43
+ super().__init__()
44
+ self.embedding = nn.Embedding(num_classes + 1, dim)
45
+ self.num_classes = num_classes
46
+
47
+ def forward(self, labels):
48
+ embeddings = self.embedding(labels)
49
+ return embeddings
50
+
51
+
52
+ class RMSNorm(nn.Module):
53
+ def __init__(self, dim, device=None, dtype=None):
54
+ super().__init__()
55
+ self.scale = dim**0.5
56
+ self.g = nn.Parameter(torch.ones(1, device=device, dtype=dtype))
57
+
58
+ def forward(self, x):
59
+ return F.normalize(x, dim=-1) * self.scale * self.g
60
+
61
+
62
+ class DiTBlock(nn.Module):
63
+ def __init__(self, dim, num_heads, mlp_ratio=4.0):
64
+ super().__init__()
65
+ self.norm1 = RMSNorm(dim)
66
+ self.attn = Attention(
67
+ dim, num_heads=num_heads, qkv_bias=True, qk_norm=True, norm_layer=RMSNorm
68
+ )
69
+ # Explicit attention supports JVP-based objectives.
70
+ self.attn.fused_attn = False
71
+ self.norm2 = RMSNorm(dim)
72
+ mlp_dim = int(dim * mlp_ratio)
73
+ approx_gelu = lambda: nn.GELU(approximate="tanh")
74
+ self.mlp = Mlp(in_features=dim, hidden_features=mlp_dim, act_layer=approx_gelu, drop=0)
75
+ self.adaLN_modulation = nn.Sequential(nn.SiLU(), nn.Linear(dim, 6 * dim))
76
+
77
+ def forward(self, x, c):
78
+ shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.adaLN_modulation(
79
+ c
80
+ ).chunk(6, dim=-1)
81
+ x = x + gate_msa.unsqueeze(1) * self.attn(modulate(self.norm1(x), scale_msa, shift_msa))
82
+ x = x + gate_mlp.unsqueeze(1) * self.mlp(modulate(self.norm2(x), scale_mlp, shift_mlp))
83
+ return x
84
+
85
+
86
+ class FinalLayer(nn.Module):
87
+ def __init__(self, dim, patch_size, out_dim):
88
+ super().__init__()
89
+ self.norm_final = RMSNorm(dim)
90
+ self.linear = nn.Linear(dim, patch_size * patch_size * out_dim)
91
+ self.adaLN_modulation = nn.Sequential(nn.SiLU(), nn.Linear(dim, 2 * dim))
92
+
93
+ def forward(self, x, c):
94
+ shift, scale = self.adaLN_modulation(c).chunk(2, dim=-1)
95
+ x = modulate(self.norm_final(x), shift, scale)
96
+ x = self.linear(x)
97
+ return x
98
+
99
+
100
+ class MFDiT(nn.Module):
101
+ def __init__(
102
+ self,
103
+ input_size=32,
104
+ patch_size=2,
105
+ in_channels=4,
106
+ dim=1152,
107
+ depth=28,
108
+ num_heads=16,
109
+ mlp_ratio=4.0,
110
+ num_register_tokens=4,
111
+ num_classes=1000,
112
+ ):
113
+ super().__init__()
114
+ self.in_channels = in_channels
115
+ self.out_channels = in_channels
116
+ self.patch_size = patch_size
117
+ self.num_heads = num_heads
118
+ self.num_classes = num_classes
119
+
120
+ self.x_embedder = PatchEmbed(input_size, patch_size, in_channels, dim)
121
+ self.t_embedder = TimestepEmbedder(dim)
122
+ self.r_embedder = TimestepEmbedder(dim)
123
+
124
+ self.use_cond = num_classes is not None
125
+ self.y_embedder = LabelEmbedder(num_classes, dim) if self.use_cond else None
126
+
127
+ num_patches = self.x_embedder.num_patches
128
+ self.pos_embed = nn.Parameter(torch.zeros(1, num_patches, dim), requires_grad=True)
129
+
130
+ self.blocks = nn.ModuleList([DiTBlock(dim, num_heads, mlp_ratio) for _ in range(depth)])
131
+ self.final_layer = FinalLayer(dim, patch_size, self.out_channels)
132
+
133
+ self.initialize_weights()
134
+
135
+ def initialize_weights(self):
136
+ def _basic_init(module):
137
+ if isinstance(module, nn.Linear):
138
+ torch.nn.init.xavier_uniform_(module.weight)
139
+ if module.bias is not None:
140
+ nn.init.constant_(module.bias, 0)
141
+
142
+ self.apply(_basic_init)
143
+
144
+ pos_embed = get_2d_sincos_pos_embed(
145
+ self.pos_embed.shape[-1], int(self.x_embedder.num_patches**0.5)
146
+ )
147
+ self.pos_embed.data.copy_(torch.from_numpy(pos_embed).float().unsqueeze(0))
148
+
149
+ w = self.x_embedder.proj.weight.data
150
+ nn.init.xavier_uniform_(w.view([w.shape[0], -1]))
151
+ nn.init.constant_(self.x_embedder.proj.bias, 0)
152
+
153
+ if self.y_embedder is not None:
154
+ nn.init.normal_(self.y_embedder.embedding.weight, std=0.02)
155
+
156
+ nn.init.normal_(self.t_embedder.mlp[0].weight, std=0.02)
157
+ nn.init.normal_(self.t_embedder.mlp[2].weight, std=0.02)
158
+
159
+ for block in self.blocks:
160
+ nn.init.constant_(block.adaLN_modulation[-1].weight, 0)
161
+ nn.init.constant_(block.adaLN_modulation[-1].bias, 0)
162
+
163
+ nn.init.constant_(self.final_layer.adaLN_modulation[-1].weight, 0)
164
+ nn.init.constant_(self.final_layer.adaLN_modulation[-1].bias, 0)
165
+ nn.init.constant_(self.final_layer.linear.weight, 0)
166
+ nn.init.constant_(self.final_layer.linear.bias, 0)
167
+
168
+ def unpatchify(self, x):
169
+ """Reconstruct an NCHW tensor from flattened patch predictions."""
170
+ c = self.out_channels
171
+ p = self.x_embedder.patch_size[0]
172
+ h = w = int(x.shape[1] ** 0.5)
173
+ assert h * w == x.shape[1]
174
+
175
+ x = x.reshape(shape=(x.shape[0], h, w, p, p, c))
176
+ x = torch.einsum("nhwpqc->nchpwq", x)
177
+ imgs = x.reshape(shape=(x.shape[0], c, h * p, h * p))
178
+ return imgs
179
+
180
+ def forward_features(self, x, t, r, y=None, out_indices=[2, 5, 8, 11]):
181
+ x = self.x_embedder(x) + self.pos_embed
182
+ t = self.t_embedder(t)
183
+ r = self.r_embedder(r)
184
+ t = t + r
185
+ c = t
186
+ if self.use_cond and y is not None:
187
+ y = self.y_embedder(y)
188
+ c = c + y
189
+
190
+ features = []
191
+ for i, block in enumerate(self.blocks):
192
+ x = block(x, c)
193
+ if i in out_indices:
194
+ features.append(x)
195
+
196
+ return features
197
+
198
+ def forward(self, x, t, r, y=None):
199
+ """Predict a velocity field with the same spatial shape as the input."""
200
+ H, W = x.shape[-2:]
201
+
202
+ x = self.x_embedder(x) + self.pos_embed
203
+
204
+ t = self.t_embedder(t)
205
+ r = self.r_embedder(r)
206
+ t = t + r
207
+
208
+ c = t
209
+ if self.use_cond:
210
+ y = self.y_embedder(y)
211
+ c = c + y
212
+
213
+ for i, block in enumerate(self.blocks):
214
+ x = block(x, c)
215
+
216
+ x = self.final_layer(x, c)
217
+ x = self.unpatchify(x)
218
+ return x
219
+
220
+
221
+ # Positional embeddings adapted from:
222
+ # https://github.com/facebookresearch/mae/blob/main/util/pos_embed.py
223
+
224
+
225
+ def get_2d_sincos_pos_embed(embed_dim, grid_size, cls_token=False, extra_tokens=0):
226
+ """Return 2D sine-cosine embeddings with optional zero-valued prefix tokens."""
227
+ grid_h = np.arange(grid_size, dtype=np.float32)
228
+ grid_w = np.arange(grid_size, dtype=np.float32)
229
+ grid = np.meshgrid(grid_w, grid_h)
230
+ grid = np.stack(grid, axis=0)
231
+
232
+ grid = grid.reshape([2, 1, grid_size, grid_size])
233
+ pos_embed = get_2d_sincos_pos_embed_from_grid(embed_dim, grid)
234
+ if cls_token and extra_tokens > 0:
235
+ pos_embed = np.concatenate([np.zeros([extra_tokens, embed_dim]), pos_embed], axis=0)
236
+ return pos_embed
237
+
238
+
239
+ def get_2d_sincos_pos_embed_from_grid(embed_dim, grid):
240
+ assert embed_dim % 2 == 0
241
+
242
+ emb_h = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[0])
243
+ emb_w = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[1])
244
+
245
+ emb = np.concatenate([emb_h, emb_w], axis=1)
246
+ return emb
247
+
248
+
249
+ def get_1d_sincos_pos_embed_from_grid(embed_dim, pos):
250
+ """Return sine-cosine embeddings for flattened positions."""
251
+ assert embed_dim % 2 == 0
252
+ omega = np.arange(embed_dim // 2, dtype=np.float64)
253
+ omega /= embed_dim / 2.0
254
+ omega = 1.0 / 10000**omega
255
+
256
+ pos = pos.reshape(-1)
257
+ out = np.einsum("m,d->md", pos, omega)
258
+
259
+ emb_sin = np.sin(out)
260
+ emb_cos = np.cos(out)
261
+
262
+ emb = np.concatenate([emb_sin, emb_cos], axis=1)
263
+ return emb
requirements.txt ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ timm==1.0.26
2
+ torchvision