BoHuangLab commited on
Commit
e794567
·
verified ·
1 Parent(s): 1b8a37e

CELL-FM CondenSeq demo: sequence -> titration curve, AUC and AAC

Browse files
Files changed (42) hide show
  1. .gitattributes +2 -34
  2. .gitignore +6 -0
  3. README.md +110 -7
  4. app.py +233 -0
  5. assets/nup98_mutation_seqs.csv +93 -0
  6. assets/reference_nucleus.npy +3 -0
  7. cell_fm/__init__.py +0 -0
  8. cell_fm/criterions/__init__.py +0 -0
  9. cell_fm/criterions/cell_fm/__init__.py +0 -0
  10. cell_fm/criterions/cell_fm/unidiffuser.py +63 -0
  11. cell_fm/logging/__init__.py +5 -0
  12. cell_fm/logging/loggers.py +201 -0
  13. cell_fm/models/__init__.py +0 -0
  14. cell_fm/models/cell_fm/__init__.py +0 -0
  15. cell_fm/models/cell_fm/cell_fm_config.py +116 -0
  16. cell_fm/models/cell_fm/cell_fm_cs_config.py +12 -0
  17. cell_fm/models/cell_fm/cell_fm_cs_model.py +309 -0
  18. cell_fm/models/cell_fm/cell_fm_model.py +637 -0
  19. cell_fm/models/cell_fm/modules/__init__.py +82 -0
  20. cell_fm/models/cell_fm/modules/positional_embedding.py +59 -0
  21. cell_fm/models/cell_fm/modules/protein_sequence_embedding.py +39 -0
  22. cell_fm/models/cell_fm/modules/transformer.py +131 -0
  23. cell_fm/models/cell_fm/modules/transport/__init__.py +84 -0
  24. cell_fm/models/cell_fm/modules/transport/integrators.py +117 -0
  25. cell_fm/models/cell_fm/modules/transport/path.py +192 -0
  26. cell_fm/models/cell_fm/modules/transport/transport.py +506 -0
  27. cell_fm/models/cell_fm/modules/transport/utils.py +29 -0
  28. cell_fm/models/vae/__init__.py +0 -0
  29. cell_fm/models/vae/vae_config.py +60 -0
  30. cell_fm/models/vae/vae_model.py +152 -0
  31. cell_fm/models/vit_cls_condenseq_img/__init__.py +0 -0
  32. cell_fm/models/vit_cls_condenseq_img/config.py +46 -0
  33. cell_fm/models/vit_cls_condenseq_img/model.py +171 -0
  34. cell_fm/pipeline/__init__.py +0 -0
  35. cell_fm/pipeline/utils.py +34 -0
  36. cell_fm/utils/__init__.py +0 -0
  37. cell_fm/utils/dist_utils.py +20 -0
  38. metrics.py +135 -0
  39. pipeline.py +312 -0
  40. requirements.txt +19 -0
  41. run_local.sh +28 -0
  42. upload_weights.py +106 -0
.gitattributes CHANGED
@@ -1,35 +1,3 @@
1
- *.7z filter=lfs diff=lfs merge=lfs -text
2
- *.arrow filter=lfs diff=lfs merge=lfs -text
3
- *.bin filter=lfs diff=lfs merge=lfs -text
4
- *.bz2 filter=lfs diff=lfs merge=lfs -text
5
- *.ckpt filter=lfs diff=lfs merge=lfs -text
6
- *.ftz filter=lfs diff=lfs merge=lfs -text
7
- *.gz filter=lfs diff=lfs merge=lfs -text
8
- *.h5 filter=lfs diff=lfs merge=lfs -text
9
- *.joblib filter=lfs diff=lfs merge=lfs -text
10
- *.lfs.* filter=lfs diff=lfs merge=lfs -text
11
- *.mlmodel filter=lfs diff=lfs merge=lfs -text
12
- *.model filter=lfs diff=lfs merge=lfs -text
13
- *.msgpack filter=lfs diff=lfs merge=lfs -text
14
  *.npy filter=lfs diff=lfs merge=lfs -text
15
- *.npz filter=lfs diff=lfs merge=lfs -text
16
- *.onnx filter=lfs diff=lfs merge=lfs -text
17
- *.ot filter=lfs diff=lfs merge=lfs -text
18
- *.parquet filter=lfs diff=lfs merge=lfs -text
19
- *.pb filter=lfs diff=lfs merge=lfs -text
20
- *.pickle filter=lfs diff=lfs merge=lfs -text
21
- *.pkl filter=lfs diff=lfs merge=lfs -text
22
- *.pt filter=lfs diff=lfs merge=lfs -text
23
- *.pth filter=lfs diff=lfs merge=lfs -text
24
- *.rar filter=lfs diff=lfs merge=lfs -text
25
- *.safetensors filter=lfs diff=lfs merge=lfs -text
26
- saved_model/**/* filter=lfs diff=lfs merge=lfs -text
27
- *.tar.* filter=lfs diff=lfs merge=lfs -text
28
- *.tar filter=lfs diff=lfs merge=lfs -text
29
- *.tflite filter=lfs diff=lfs merge=lfs -text
30
- *.tgz filter=lfs diff=lfs merge=lfs -text
31
- *.wasm filter=lfs diff=lfs merge=lfs -text
32
- *.xz filter=lfs diff=lfs merge=lfs -text
33
- *.zip filter=lfs diff=lfs merge=lfs -text
34
- *.zst filter=lfs diff=lfs merge=lfs -text
35
- *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  *.npy filter=lfs diff=lfs merge=lfs -text
2
+ *.bin filter=lfs diff=lfs merge=lfs -text
3
+ *.tif filter=lfs diff=lfs merge=lfs -text
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
.gitignore ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ __pycache__/
2
+ *.pyc
3
+ .gradio/
4
+ token
5
+ *.token
6
+ .env
README.md CHANGED
@@ -1,13 +1,116 @@
1
  ---
2
- title: CELL FM
3
- emoji: 🐢
4
- colorFrom: yellow
5
- colorTo: yellow
6
  sdk: gradio
7
- sdk_version: 6.26.0
8
- python_version: '3.12'
9
  app_file: app.py
10
  pinned: false
 
11
  ---
12
 
13
- Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  ---
2
+ title: CELL-FM
3
+ emoji: 🧬
4
+ colorFrom: blue
5
+ colorTo: indigo
6
  sdk: gradio
7
+ sdk_version: 4.44.0
 
8
  app_file: app.py
9
  pinned: false
10
+ short_description: Predict condensate titration curves from an IDR sequence
11
  ---
12
 
13
+ # CELL-FM · CondenSeq
14
+
15
+ Predict how an intrinsically disordered region behaves as its concentration rises,
16
+ directly from sequence.
17
+
18
+ Enter a sequence and the demo runs the three stages of the CELL-FM condensate
19
+ pipeline end to end:
20
+
21
+ 1. **Generate.** CELL-FM CS samples one microscopy image per protein-intensity
22
+ level, over a log-spaced concentration ladder from 20 to 2048 a.u. Every image
23
+ is conditioned on the same fixed reference nucleus, so curves are comparable
24
+ across sequences.
25
+ 2. **Classify.** A ViT calls each generated image **condensed** or **diffuse**.
26
+ 3. **Summarise.** A centred moving average over those binary calls is the
27
+ **titration curve**; integrating it against log-intensity gives the two numbers
28
+ the demo reports.
29
+
30
+ ## What AUC and AAC mean
31
+
32
+ **AUC** — the area under the titration curve. How much of the scanned concentration
33
+ range the sequence spends in the condensed state. Higher means it condenses more
34
+ readily, over a wider range.
35
+
36
+ **AAC** — the area *above* the curve. The curve is compared against the one it would
37
+ have followed without **reentrant dissolution**: everything past the curve's maximum
38
+ is held flat at that maximum, and AAC is the area between the two. It is zero when
39
+ the sequence condenses and stays condensed, and grows the more the sequence
40
+ re-dissolves at high concentration.
41
+
42
+ Both are normalised by the width of the log-intensity range, so both lie in [0, 1]
43
+ and are directly comparable between sequences.
44
+
45
+ Also reported: **c_sat**, the first intensity at which the smoothed curve reaches
46
+ probability 0.8, and the raw per-image calls as a downloadable CSV.
47
+
48
+ ## Presets
49
+
50
+ The dropdown is loaded with NUP98's FG-repeat region — wild type plus its full
51
+ mutation panel (F→S, F→Y, charge and spacer variants). Comparing a mutant's AUC
52
+ against WT is the quickest way to see the model respond to sequence grammar.
53
+
54
+ ## Hardware
55
+
56
+ **A GPU is required in practice.** The code itself runs on CPU — `flash-attn` is
57
+ deliberately left out of `requirements.txt`, so ESM-C uses its pure-torch rotary
58
+ fallback (verified bit-identical to the Triton path: same AUC, same AAC, same
59
+ 256/256 per-image calls) — but a CPU run of the defaults would take hours and
60
+ time out. Use ZeroGPU or a GPU hardware tier.
61
+
62
+ The default settings — 512 images at 100 ODE steps — take **~85 s on an A40**
63
+ (measured: 11.5 s per batch of 64, plus ~9 s of one-off model loading). Runtime
64
+ scales linearly in both settings; lower either in **Sampling settings** for a
65
+ faster, noisier curve.
66
+
67
+ On ZeroGPU the per-call budget is set by `ZEROGPU_DURATION` (default 300 s), which
68
+ covers the defaults with room to spare.
69
+
70
+ ## Running it without a Space
71
+
72
+ Creating a Gradio Space under an organization now needs a Team or Enterprise plan.
73
+ Until `HuangLab` has one, `run_local.sh` serves the identical app from a cluster GPU
74
+ over a temporary public URL:
75
+
76
+ ```bash
77
+ sbatch run_local.sh # prints a *.gradio.live URL, valid 72 h
78
+ ```
79
+
80
+ ## Weights
81
+
82
+ The Space downloads ~2.1 GB of checkpoints at startup from
83
+ [`BoHuangLab/CELL-FM`](https://huggingface.co/BoHuangLab/CELL-FM), overridable with
84
+ `CELLFM_MODEL_REPO`. That repo keeps one subfolder per model family:
85
+
86
+ | File | Model |
87
+ |---|---|
88
+ | `condenseq/cellfm_seq2img.bin` | CELL-FM CS generator, ESM-C 600M encoder included |
89
+ | `condenseq/vae.bin` | 160×160 image VAE |
90
+ | `condenseq/vit_cls.bin` | Condensed/diffuse ViT classifier |
91
+
92
+ To run against a local copy instead, set `CELLFM_LOCAL_WEIGHTS` to a **flat**
93
+ directory holding the three `.bin` files — the prefix is stripped for local paths.
94
+ `upload_weights.py` pushes them from the cluster.
95
+
96
+ ## Provenance
97
+
98
+ Ported from the CELL-FM repository, one module per stage, with behaviour preserved:
99
+
100
+ | Stage | Origin |
101
+ |---|---|
102
+ | Generation | `cell_fm/tasks_local/cell_fm_cs/evaluate_seq2img_dict.py` |
103
+ | Classification | `cell_fm/tasks_local/vit_cls_condenseq_img/evaluate_single_img.py` |
104
+ | Moving average | `cell_fm/tasks_local/vit_cls_condenseq_img/ma_plot.py` |
105
+ | AUC / AAC | `cell_diff/tasks/analysis/{utils,ana_all_mutation_log_scale}.py` |
106
+
107
+ Model hyperparameters mirror `scripts/cell_fm_cs/evaluate_seq2img.sh` and
108
+ `scripts/vit_cls_condenseq_img/pretrain.sh`. The metric port in `metrics.py` was
109
+ checked against the original on 40 stored curves and agrees to 0.0 in both log and
110
+ linear mode. Generated images make the same round-trip through uint16 that the
111
+ offline pipeline's TIFF write imposed, so demo numbers stay comparable to published
112
+ ones.
113
+
114
+ Reference values from an end-to-end run at the defaults (512 images, 100 steps):
115
+ NUP98 WT gives AUC 0.4896, AAC 0.0001, c_sat 508; the 1F→S mutant gives AUC 0.4561
116
+ — losing one phenylalanine sticker costs 0.034 of condensation propensity.
app.py ADDED
@@ -0,0 +1,233 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """CELL-FM CondenSeq demo: protein sequence -> condensate titration curve.
2
+
3
+ Enter an IDR sequence; the app generates a concentration ladder of images with
4
+ CELL-FM, classifies each as condensed or diffuse, smooths the calls into a
5
+ titration curve, and integrates it into AUC (condensation propensity) and AAC
6
+ (reentrant dissolution).
7
+ """
8
+
9
+ import os
10
+
11
+ import gradio as gr
12
+ import matplotlib
13
+ matplotlib.use("Agg")
14
+ import matplotlib.pyplot as plt
15
+ import numpy as np
16
+ import pandas as pd
17
+
18
+ import metrics
19
+ import pipeline
20
+
21
+ # ZeroGPU: only present on HF Spaces hardware, so the import is optional.
22
+ try:
23
+ import spaces
24
+
25
+ GPU_DECORATOR = spaces.GPU(duration=int(os.environ.get("ZEROGPU_DURATION", "300")))
26
+ except Exception: # running locally or on a dedicated GPU
27
+ def GPU_DECORATOR(fn):
28
+ return fn
29
+
30
+ HERE = os.path.dirname(os.path.abspath(__file__))
31
+
32
+ # palette: categorical slots 1 and 2 (validated, CVD dE 24.7 / normal dE 33.6)
33
+ CURVE = "#2a78d6" # the titration curve and the area under it
34
+ GAP = "#eb6834" # the area above the curve
35
+ INK = "#0b0b0b"
36
+ MUTED = "#52514e"
37
+ SURFACE = "#fcfcfb"
38
+
39
+ MAX_SEQUENCE_LEN = 2048
40
+
41
+
42
+ def load_presets():
43
+ """NUP98 WT plus its mutation panel, as demo starting points."""
44
+ df = pd.read_csv(os.path.join(HERE, "assets", "nup98_mutation_seqs.csv"), dtype=str)
45
+ return {f"NUP98 {row.mutation}": row.sequence for row in df.itertuples(index=False)}
46
+
47
+
48
+ PRESETS = load_presets()
49
+ DEFAULT_SEQUENCE = PRESETS["NUP98 WT"]
50
+
51
+
52
+ def plot_curve(result: pipeline.Result):
53
+ """The titration curve, with the two integrated areas drawn as the areas they are."""
54
+ x, y, curve = result.intensities, result.predictions, result.curve
55
+ flat = metrics.no_reentrant_curve(curve)
56
+
57
+ fig, ax = plt.subplots(figsize=(8.5, 4.6), facecolor=SURFACE)
58
+ ax.set_facecolor(SURFACE)
59
+
60
+ # the raw per-image calls, kept visible but recessive: the curve is a summary of these
61
+ ax.scatter(x, y, s=7, alpha=0.16, color=MUTED, linewidths=0, zorder=1)
62
+
63
+ # AUC: what the sequence actually does
64
+ ax.fill_between(x, 0, curve, color=CURVE, alpha=0.18, linewidth=0, zorder=2,
65
+ label=f"AUC {result.auc:.3f}")
66
+ # AAC: the condensation lost to reentrant dissolution at high concentration
67
+ if result.aac > 1e-9:
68
+ ax.fill_between(x, curve, flat, where=flat > curve, color=GAP, alpha=0.30,
69
+ linewidth=0, zorder=3, label=f"AAC {result.aac:.3f}")
70
+ ax.plot(x, flat, color=GAP, linewidth=1.2, linestyle="--", alpha=0.8, zorder=4)
71
+
72
+ ax.plot(x, curve, color=CURVE, linewidth=2.0, zorder=5)
73
+
74
+ ax.set_xscale("log")
75
+ ax.set_xlim(x.min(), x.max())
76
+ ax.set_ylim(-0.02, 1.02)
77
+ ax.set_xlabel("Protein intensity level (a.u., log scale)", fontsize=11, color=INK)
78
+ ax.set_ylabel("Condensate probability", fontsize=11, color=INK)
79
+ ax.tick_params(labelsize=10, colors=MUTED)
80
+ ax.grid(True, alpha=0.18, linewidth=0.8)
81
+ ax.set_axisbelow(True)
82
+ for side in ("top", "right"):
83
+ ax.spines[side].set_visible(False)
84
+ for side in ("left", "bottom"):
85
+ ax.spines[side].set_color(MUTED)
86
+ ax.spines[side].set_linewidth(0.8)
87
+
88
+ if result.c_sat != float("inf"):
89
+ ax.axvline(result.c_sat, color=MUTED, linewidth=1.0, linestyle=":", zorder=4)
90
+ ax.annotate(
91
+ f"$c_{{sat}}$ {result.c_sat:.0f}",
92
+ xy=(result.c_sat, 1.0), xytext=(6, -13), textcoords="offset points",
93
+ fontsize=9, color=MUTED, ha="left", va="top",
94
+ )
95
+
96
+ ax.legend(loc="upper left", frameon=False, fontsize=10, labelcolor=INK)
97
+ fig.tight_layout()
98
+ return fig
99
+
100
+
101
+ def plot_samples(result: pipeline.Result):
102
+ """A strip of generated images across the ladder, so the curve is checkable by eye."""
103
+ imgs, xs = result.images, result.image_intensities
104
+ fig, axes = plt.subplots(1, len(imgs), figsize=(2.0 * len(imgs), 2.3), facecolor=SURFACE)
105
+
106
+ for ax, img, level in zip(np.atleast_1d(axes), imgs, xs):
107
+ ax.imshow(img, cmap="magma", vmin=0, vmax=1)
108
+ ax.set_title(f"{level:.0f}", fontsize=10, color=INK)
109
+ ax.set_xticks([]); ax.set_yticks([])
110
+ for s in ax.spines.values():
111
+ s.set_visible(False)
112
+
113
+ fig.suptitle("Generated protein channel across the concentration ladder (a.u.)",
114
+ fontsize=10, color=MUTED, y=0.04)
115
+ fig.tight_layout(rect=(0, 0.06, 1, 1))
116
+ return fig
117
+
118
+
119
+ def summary_markdown(result: pipeline.Result) -> str:
120
+ csat = "not reached" if result.c_sat == float("inf") else f"{result.c_sat:.0f}"
121
+ return f"""
122
+ | | |
123
+ |---|---|
124
+ | **AUC** — area under the curve, condensation propensity | **{result.auc:.4f}** |
125
+ | **AAC** — area above the curve, reentrant dissolution | **{result.aac:.4f}** |
126
+ | No-reentrant AUC | {result.no_reentrant_auc:.4f} |
127
+ | c<sub>sat</sub> (first intensity at P ≥ 0.8) | {csat} |
128
+ | Condensed calls | {int(result.predictions.sum())} / {len(result.predictions)} |
129
+ | Moving-average window | {result.window} |
130
+ | Sequence length | {len(result.sequence)} aa |
131
+
132
+ Both areas are normalised by the width of the log-intensity range, so they lie in
133
+ [0, 1] and are comparable across sequences.
134
+ """
135
+
136
+
137
+ def results_table(result: pipeline.Result) -> pd.DataFrame:
138
+ """The numbers behind the plot, so the chart is never the only way to read them."""
139
+ return pd.DataFrame({
140
+ "protein_intensity_level": np.round(result.intensities, 4),
141
+ "predicted_class": result.predictions,
142
+ "condensate_probability_ma": np.round(result.curve, 6),
143
+ })
144
+
145
+
146
+ @GPU_DECORATOR
147
+ def analyse(sequence, num_images, num_steps, batch_size, seed, progress=gr.Progress()):
148
+ sequence = pipeline.clean_sequence(sequence)
149
+ if len(sequence) > MAX_SEQUENCE_LEN:
150
+ raise gr.Error(f"Sequence is {len(sequence)} aa; the model takes at most {MAX_SEQUENCE_LEN}.")
151
+
152
+ result = pipeline.run(
153
+ sequence,
154
+ num_images=int(num_images),
155
+ num_steps=int(num_steps),
156
+ batch_size=int(batch_size),
157
+ seed=int(seed),
158
+ progress=progress,
159
+ )
160
+
161
+ table = results_table(result)
162
+ csv_path = os.path.join("/tmp", "pred_vs_intensity.csv")
163
+ table.to_csv(csv_path, index=False)
164
+
165
+ return plot_curve(result), plot_samples(result), summary_markdown(result), table, csv_path
166
+
167
+
168
+ with gr.Blocks(title="CELL-FM CondenSeq", theme=gr.themes.Soft()) as demo:
169
+ gr.Markdown(
170
+ """
171
+ # CELL-FM · CondenSeq
172
+
173
+ Predict how an intrinsically disordered region behaves as its concentration rises.
174
+
175
+ CELL-FM generates a ladder of microscopy images for your sequence, one per
176
+ protein-intensity level; a ViT classifier calls each **condensed** or **diffuse**;
177
+ a moving average over those calls is the **titration curve**. Integrating it gives:
178
+
179
+ * **AUC** — how much of the concentration range the sequence spends condensed.
180
+ * **AAC** — the area above the curve, i.e. condensation lost to **reentrant
181
+ dissolution** at high concentration. Zero when the curve never comes back down.
182
+ """
183
+ )
184
+
185
+ with gr.Row():
186
+ with gr.Column(scale=2):
187
+ preset = gr.Dropdown(
188
+ choices=list(PRESETS), value="NUP98 WT", label="Preset",
189
+ info="NUP98 FG-repeat WT and its mutation panel. Editing the box below overrides this.",
190
+ )
191
+ sequence = gr.Textbox(
192
+ value=DEFAULT_SEQUENCE, lines=4, label="Protein sequence",
193
+ info="Single-letter amino acids; FASTA headers are stripped.",
194
+ )
195
+ with gr.Accordion("Sampling settings", open=False):
196
+ num_images = gr.Slider(64, 1024, value=512, step=64, label="Images (concentration levels)",
197
+ info="More levels give a smoother curve and cost proportionally more time.")
198
+ num_steps = gr.Slider(20, 200, value=100, step=10, label="ODE steps per image")
199
+ batch_size = gr.Slider(16, 256, value=64, step=16, label="Batch size")
200
+ seed = gr.Number(value=pipeline.DEFAULT_SEED, precision=0, label="Random seed",
201
+ info="Generation starts from a random latent; fix the seed for a repeatable curve.")
202
+ run_button = gr.Button("Run analysis", variant="primary")
203
+
204
+ with gr.Column(scale=3):
205
+ summary = gr.Markdown()
206
+
207
+ curve_plot = gr.Plot(label="Condensate titration curve")
208
+ sample_plot = gr.Plot(label="Generated images")
209
+
210
+ with gr.Accordion("Per-image results", open=False):
211
+ table = gr.Dataframe(label="pred_vs_intensity", wrap=True)
212
+ download = gr.File(label="Download CSV")
213
+
214
+ gr.Markdown(
215
+ """
216
+ ---
217
+ Generation follows `evaluate_seq2img_dict.py`, classification `evaluate_single_img.py`,
218
+ smoothing `ma_plot.py`, and the areas `analysis/ana_all_mutation_log_scale.py` from the
219
+ CELL-FM repository. Every image is conditioned on one fixed reference nucleus, so curves
220
+ are comparable across sequences.
221
+ """
222
+ )
223
+
224
+ preset.change(lambda name: PRESETS[name], inputs=preset, outputs=sequence)
225
+ run_button.click(
226
+ analyse,
227
+ inputs=[sequence, num_images, num_steps, batch_size, seed],
228
+ outputs=[curve_plot, sample_plot, summary, table, download],
229
+ )
230
+
231
+
232
+ if __name__ == "__main__":
233
+ demo.queue().launch(share=os.environ.get("GRADIO_SHARE") == "1")
assets/nup98_mutation_seqs.csv ADDED
@@ -0,0 +1,93 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ mutation,sequence
2
+ WT,KSFGTPFGGGTGGFGTTSTFGQNTGFGTTSGGAFGTSAFGSSNNTGGLFGNSQTKPGGLFGTSSFS
3
+ 1FtoS_1,KSSGTPFGGGTGGFGTTSTFGQNTGFGTTSGGAFGTSAFGSSNNTGGLFGNSQTKPGGLFGTSSFS
4
+ 1FtoS_2,KSFGTPSGGGTGGFGTTSTFGQNTGFGTTSGGAFGTSAFGSSNNTGGLFGNSQTKPGGLFGTSSFS
5
+ 1FtoS_3,KSFGTPFGGGTGGSGTTSTFGQNTGFGTTSGGAFGTSAFGSSNNTGGLFGNSQTKPGGLFGTSSFS
6
+ 1FtoS_4,KSFGTPFGGGTGGFGTTSTSGQNTGFGTTSGGAFGTSAFGSSNNTGGLFGNSQTKPGGLFGTSSFS
7
+ 1FtoS_5,KSFGTPFGGGTGGFGTTSTFGQNTGSGTTSGGAFGTSAFGSSNNTGGLFGNSQTKPGGLFGTSSFS
8
+ 1FtoS_6,KSFGTPFGGGTGGFGTTSTFGQNTGFGTTSGGASGTSAFGSSNNTGGLFGNSQTKPGGLFGTSSFS
9
+ 1FtoS_7,KSFGTPFGGGTGGFGTTSTFGQNTGFGTTSGGAFGTSASGSSNNTGGLFGNSQTKPGGLFGTSSFS
10
+ 1FtoS_8,KSFGTPFGGGTGGFGTTSTFGQNTGFGTTSGGAFGTSAFGSSNNTGGLSGNSQTKPGGLFGTSSFS
11
+ 1FtoS_9,KSFGTPFGGGTGGFGTTSTFGQNTGFGTTSGGAFGTSAFGSSNNTGGLFGNSQTKPGGLSGTSSFS
12
+ 1FtoS_10,KSFGTPFGGGTGGFGTTSTFGQNTGFGTTSGGAFGTSAFGSSNNTGGLFGNSQTKPGGLFGTSSSS
13
+ 2FtoS_1,KSFGTPFGGGTGGFGTTSTFGQNTGFGTTSGGAFGTSASGSSNNTGGLFGNSQTKPGGLSGTSSFS
14
+ 2FtoS_2,KSSGTPFGGGTGGFGTTSTFGQNTGFGTTSGGAFGTSAFGSSNNTGGLFGNSQTKPGGLSGTSSFS
15
+ 2FtoS_3,KSSGTPFGGGTGGSGTTSTFGQNTGFGTTSGGAFGTSAFGSSNNTGGLFGNSQTKPGGLFGTSSFS
16
+ 2FtoS_4,KSFGTPFGGGTGGSGTTSTSGQNTGFGTTSGGAFGTSAFGSSNNTGGLFGNSQTKPGGLFGTSSFS
17
+ 2FtoS_5,KSFGTPSGGGTGGFGTTSTFGQNTGFGTTSGGAFGTSAFGSSNNTGGLFGNSQTKPGGLSGTSSFS
18
+ 2FtoS_6,KSFGTPSGGGTGGFGTTSTFGQNTGFGTTSGGAFGTSAFGSSNNTGGLSGNSQTKPGGLFGTSSFS
19
+ 2FtoS_7,KSSGTPFGGGTGGFGTTSTFGQNTGFGTTSGGAFGTSAFGSSNNTGGLFGNSQTKPGGLFGTSSSS
20
+ 2FtoS_8,KSSGTPFGGGTGGFGTTSTFGQNTGFGTTSGGAFGTSAFGSSNNTGGLSGNSQTKPGGLFGTSSFS
21
+ 2FtoS_9,KSFGTPFGGGTGGFGTTSTFGQNTGSGTTSGGAFGTSAFGSSNNTGGLFGNSQTKPGGLFGTSSSS
22
+ 2FtoS_10,KSSGTPFGGGTGGFGTTSTFGQNTGFGTTSGGAFGTSASGSSNNTGGLFGNSQTKPGGLFGTSSFS
23
+ 3FtoS_1,KSFGTPFGGGTGGSGTTSTFGQNTGFGTTSGGASGTSASGSSNNTGGLFGNSQTKPGGLFGTSSFS
24
+ 3FtoS_2,KSFGTPSGGGTGGFGTTSTFGQNTGFGTTSGGASGTSASGSSNNTGGLFGNSQTKPGGLFGTSSFS
25
+ 3FtoS_3,KSSGTPSGGGTGGFGTTSTFGQNTGFGTTSGGAFGTSASGSSNNTGGLFGNSQTKPGGLFGTSSFS
26
+ 3FtoS_4,KSSGTPSGGGTGGFGTTSTFGQNTGFGTTSGGASGTSAFGSSNNTGGLFGNSQTKPGGLFGTSSFS
27
+ 3FtoS_5,KSSGTPFGGGTGGSGTTSTFGQNTGFGTTSGGAFGTSASGSSNNTGGLFGNSQTKPGGLFGTSSFS
28
+ 3FtoS_6,KSSGTPFGGGTGGFGTTSTFGQNTGFGTTSGGASGTSAFGSSNNTGGLSGNSQTKPGGLFGTSSFS
29
+ 3FtoS_7,KSSGTPFGGGTGGFGTTSTFGQNTGFGTTSGGASGTSAFGSSNNTGGLFGNSQTKPGGLFGTSSSS
30
+ 3FtoS_8,KSFGTPFGGGTGGSGTTSTSGQNTGSGTTSGGAFGTSAFGSSNNTGGLFGNSQTKPGGLFGTSSFS
31
+ 3FtoS_9,KSFGTPFGGGTGGSGTTSTFGQNTGFGTTSGGASGTSAFGSSNNTGGLFGNSQTKPGGLSGTSSFS
32
+ 3FtoS_10,KSFGTPFGGGTGGSGTTSTFGQNTGSGTTSGGAFGTSASGSSNNTGGLFGNSQTKPGGLFGTSSFS
33
+ 4FtoS_1,KSSGTPFGGGTGGFGTTSTSGQNTGSGTTSGGAFGTSASGSSNNTGGLFGNSQTKPGGLFGTSSFS
34
+ 4FtoS_2,KSFGTPFGGGTGGFGTTSTSGQNTGSGTTSGGAFGTSAFGSSNNTGGLSGNSQTKPGGLFGTSSSS
35
+ 4FtoS_3,KSFGTPFGGGTGGSGTTSTFGQNTGFGTTSGGASGTSASGSSNNTGGLFGNSQTKPGGLSGTSSFS
36
+ 4FtoS_4,KSFGTPFGGGTGGFGTTSTSGQNTGSGTTSGGAFGTSASGSSNNTGGLSGNSQTKPGGLFGTSSFS
37
+ 4FtoS_5,KSFGTPSGGGTGGFGTTSTFGQNTGFGTTSGGAFGTSAFGSSNNTGGLSGNSQTKPGGLSGTSSSS
38
+ 4FtoS_6,KSFGTPSGGGTGGFGTTSTSGQNTGSGTTSGGAFGTSAFGSSNNTGGLSGNSQTKPGGLFGTSSFS
39
+ 4FtoS_7,KSSGTPFGGGTGGFGTTSTSGQNTGFGTTSGGASGTSAFGSSNNTGGLFGNSQTKPGGLSGTSSFS
40
+ 4FtoS_8,KSFGTPSGGGTGGFGTTSTSGQNTGFGTTSGGAFGTSASGSSNNTGGLSGNSQTKPGGLFGTSSFS
41
+ 4FtoS_9,KSFGTPFGGGTGGSGTTSTSGQNTGFGTTSGGAFGTSASGSSNNTGGLFGNSQTKPGGLSGTSSFS
42
+ 4FtoS_10,KSSGTPFGGGTGGFGTTSTFGQNTGSGTTSGGAFGTSAFGSSNNTGGLSGNSQTKPGGLSGTSSFS
43
+ 5FtoS_1,KSFGTPFGGGTGGSGTTSTSGQNTGFGTTSGGASGTSASGSSNNTGGLFGNSQTKPGGLSGTSSFS
44
+ 5FtoS_2,KSFGTPFGGGTGGSGTTSTFGQNTGSGTTSGGAFGTSASGSSNNTGGLSGNSQTKPGGLSGTSSFS
45
+ 5FtoS_3,KSSGTPSGGGTGGSGTTSTSGQNTGFGTTSGGASGTSAFGSSNNTGGLFGNSQTKPGGLFGTSSFS
46
+ 5FtoS_4,KSFGTPSGGGTGGFGTTSTFGQNTGFGTTSGGASGTSAFGSSNNTGGLSGNSQTKPGGLSGTSSSS
47
+ 5FtoS_5,KSFGTPFGGGTGGSGTTSTSGQNTGFGTTSGGASGTSASGSSNNTGGLSGNSQTKPGGLFGTSSFS
48
+ 5FtoS_6,KSSGTPSGGGTGGFGTTSTFGQNTGSGTTSGGAFGTSASGSSNNTGGLSGNSQTKPGGLFGTSSFS
49
+ 5FtoS_7,KSFGTPSGGGTGGFGTTSTSGQNTGFGTTSGGAFGTSASGSSNNTGGLSGNSQTKPGGLFGTSSSS
50
+ 5FtoS_8,KSSGTPFGGGTGGFGTTSTSGQNTGFGTTSGGAFGTSASGSSNNTGGLSGNSQTKPGGLFGTSSSS
51
+ 5FtoS_9,KSSGTPFGGGTGGSGTTSTFGQNTGFGTTSGGAFGTSASGSSNNTGGLSGNSQTKPGGLSGTSSFS
52
+ 5FtoS_10,KSSGTPFGGGTGGSGTTSTFGQNTGSGTTSGGASGTSASGSSNNTGGLFGNSQTKPGGLFGTSSFS
53
+ 6FtoS_1,KSSGTPSGGGTGGFGTTSTSGQNTGSGTTSGGAFGTSASGSSNNTGGLSGNSQTKPGGLFGTSSFS
54
+ 6FtoS_2,KSSGTPSGGGTGGFGTTSTFGQNTGSGTTSGGASGTSASGSSNNTGGLSGNSQTKPGGLFGTSSFS
55
+ 6FtoS_3,KSFGTPFGGGTGGSGTTSTSGQNTGFGTTSGGASGTSAFGSSNNTGGLSGNSQTKPGGLSGTSSSS
56
+ 6FtoS_4,KSSGTPFGGGTGGSGTTSTSGQNTGFGTTSGGAFGTSASGSSNNTGGLSGNSQTKPGGLSGTSSFS
57
+ 6FtoS_5,KSSGTPSGGGTGGSGTTSTFGQNTGFGTTSGGASGTSASGSSNNTGGLFGNSQTKPGGLSGTSSFS
58
+ 6FtoS_6,KSSGTPSGGGTGGSGTTSTFGQNTGSGTTSGGAFGTSAFGSSNNTGGLSGNSQTKPGGLFGTSSSS
59
+ 6FtoS_7,KSSGTPFGGGTGGSGTTSTFGQNTGSGTTSGGAFGTSASGSSNNTGGLSGNSQTKPGGLFGTSSSS
60
+ 6FtoS_8,KSSGTPSGGGTGGSGTTSTFGQNTGSGTTSGGAFGTSAFGSSNNTGGLFGNSQTKPGGLSGTSSSS
61
+ 6FtoS_9,KSSGTPFGGGTGGSGTTSTFGQNTGSGTTSGGASGTSASGSSNNTGGLFGNSQTKPGGLSGTSSFS
62
+ 6FtoS_10,KSSGTPFGGGTGGSGTTSTSGQNTGFGTTSGGAFGTSASGSSNNTGGLFGNSQTKPGGLSGTSSSS
63
+ 7FtoS_1,KSSGTPFGGGTGGFGTTSTSGQNTGSGTTSGGASGTSASGSSNNTGGLSGNSQTKPGGLSGTSSFS
64
+ 7FtoS_2,KSSGTPSGGGTGGSGTTSTFGQNTGFGTTSGGASGTSAFGSSNNTGGLSGNSQTKPGGLSGTSSSS
65
+ 7FtoS_3,KSFGTPSGGGTGGSGTTSTFGQNTGSGTTSGGAFGTSASGSSNNTGGLSGNSQTKPGGLSGTSSSS
66
+ 7FtoS_4,KSSGTPSGGGTGGSGTTSTSGQNTGSGTTSGGAFGTSASGSSNNTGGLFGNSQTKPGGLSGTSSFS
67
+ 7FtoS_5,KSFGTPSGGGTGGSGTTSTSGQNTGSGTTSGGAFGTSAFGSSNNTGGLSGNSQTKPGGLSGTSSSS
68
+ 7FtoS_6,KSSGTPFGGGTGGSGTTSTSGQNTGSGTTSGGASGTSASGSSNNTGGLFGNSQTKPGGLFGTSSSS
69
+ 7FtoS_7,KSSGTPFGGGTGGSGTTSTSGQNTGFGTTSGGASGTSASGSSNNTGGLFGNSQTKPGGLSGTSSSS
70
+ 7FtoS_8,KSSGTPSGGGTGGSGTTSTSGQNTGFGTTSGGASGTSAFGSSNNTGGLFGNSQTKPGGLSGTSSSS
71
+ 7FtoS_9,KSFGTPFGGGTGGSGTTSTFGQNTGSGTTSGGASGTSASGSSNNTGGLSGNSQTKPGGLSGTSSSS
72
+ 7FtoS_10,KSSGTPSGGGTGGFGTTSTSGQNTGFGTTSGGASGTSAFGSSNNTGGLSGNSQTKPGGLSGTSSSS
73
+ 8FtoS_1,KSSGTPSGGGTGGSGTTSTSGQNTGSGTTSGGASGTSAFGSSNNTGGLFGNSQTKPGGLSGTSSSS
74
+ 8FtoS_2,KSSGTPFGGGTGGFGTTSTSGQNTGSGTTSGGASGTSASGSSNNTGGLSGNSQTKPGGLSGTSSSS
75
+ 8FtoS_3,KSSGTPSGGGTGGSGTTSTFGQNTGSGTTSGGASGTSAFGSSNNTGGLSGNSQTKPGGLSGTSSSS
76
+ 8FtoS_4,KSFGTPSGGGTGGSGTTSTSGQNTGSGTTSGGAFGTSASGSSNNTGGLSGNSQTKPGGLSGTSSSS
77
+ 8FtoS_5,KSFGTPSGGGTGGSGTTSTSGQNTGSGTTSGGASGTSAFGSSNNTGGLSGNSQTKPGGLSGTSSSS
78
+ 8FtoS_6,KSSGTPSGGGTGGFGTTSTSGQNTGSGTTSGGASGTSASGSSNNTGGLFGNSQTKPGGLSGTSSSS
79
+ 8FtoS_7,KSFGTPSGGGTGGSGTTSTSGQNTGSGTTSGGASGTSASGSSNNTGGLSGNSQTKPGGLSGTSSFS
80
+ 8FtoS_8,KSSGTPSGGGTGGSGTTSTSGQNTGFGTTSGGASGTSASGSSNNTGGLFGNSQTKPGGLSGTSSSS
81
+ 8FtoS_9,KSSGTPSGGGTGGSGTTSTSGQNTGSGTTSGGASGTSAFGSSNNTGGLSGNSQTKPGGLFGTSSSS
82
+ 8FtoS_10,KSSGTPSGGGTGGSGTTSTSGQNTGSGTTSGGASGTSASGSSNNTGGLFGNSQTKPGGLFGTSSSS
83
+ 9FtoS_1,KSSGTPSGGGTGGSGTTSTSGQNTGSGTTSGGASGTSAFGSSNNTGGLSGNSQTKPGGLSGTSSSS
84
+ 9FtoS_2,KSSGTPSGGGTGGSGTTSTSGQNTGSGTTSGGAFGTSASGSSNNTGGLSGNSQTKPGGLSGTSSSS
85
+ 9FtoS_3,KSSGTPSGGGTGGSGTTSTSGQNTGSGTTSGGASGTSASGSSNNTGGLSGNSQTKPGGLFGTSSSS
86
+ 9FtoS_4,KSSGTPSGGGTGGSGTTSTFGQNTGSGTTSGGASGTSASGSSNNTGGLSGNSQTKPGGLSGTSSSS
87
+ 9FtoS_5,KSSGTPSGGGTGGFGTTSTSGQNTGSGTTSGGASGTSASGSSNNTGGLSGNSQTKPGGLSGTSSSS
88
+ 9FtoS_6,KSSGTPSGGGTGGSGTTSTSGQNTGSGTTSGGASGTSASGSSNNTGGLSGNSQTKPGGLSGTSSFS
89
+ 9FtoS_7,KSFGTPSGGGTGGSGTTSTSGQNTGSGTTSGGASGTSASGSSNNTGGLSGNSQTKPGGLSGTSSSS
90
+ 9FtoS_8,KSSGTPSGGGTGGSGTTSTSGQNTGFGTTSGGASGTSASGSSNNTGGLSGNSQTKPGGLSGTSSSS
91
+ 9FtoS_9,KSSGTPSGGGTGGSGTTSTSGQNTGSGTTSGGASGTSASGSSNNTGGLFGNSQTKPGGLSGTSSSS
92
+ 9FtoS_10,KSSGTPFGGGTGGSGTTSTSGQNTGSGTTSGGASGTSASGSSNNTGGLSGNSQTKPGGLSGTSSSS
93
+ 10FtoS_1,KSSGTPSGGGTGGSGTTSTSGQNTGSGTTSGGASGTSASGSSNNTGGLSGNSQTKPGGLSGTSSSS
assets/reference_nucleus.npy ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7e8f385c8c6d1b40957907cc83aef90d1b9a887f0c10b4d2a8899349f2b9f1cb
3
+ size 102528
cell_fm/__init__.py ADDED
File without changes
cell_fm/criterions/__init__.py ADDED
File without changes
cell_fm/criterions/cell_fm/__init__.py ADDED
File without changes
cell_fm/criterions/cell_fm/unidiffuser.py ADDED
@@ -0,0 +1,63 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ import torch.nn as nn
3
+
4
+ class UniDiffCriterions(nn.Module):
5
+ def __init__(self, args, reduction="none") -> None:
6
+ super().__init__()
7
+ self.sequence_loss = nn.CrossEntropyLoss(reduction=reduction)
8
+
9
+ self.args = args
10
+ self.seq_loss_coeff = self.args.seq_loss_coeff
11
+ self.img_diff_loss_coeff = self.args.img_diff_loss_coeff
12
+ self.img_recon_loss_coeff = self.args.img_recon_loss_coeff
13
+
14
+ def forward(self, batch_data, model_output):
15
+ with torch.no_grad():
16
+ protein_seq_mask = batch_data["protein_seq_mask"].bool()
17
+ protein_seq = batch_data["protein_seq"][protein_seq_mask]
18
+ zm_label = batch_data['zm_label'].squeeze(-1).bool()
19
+
20
+ mask_num = torch.sum(protein_seq_mask.float(), dim=1, keepdim=True)
21
+ seq_loss_weights = torch.ones_like(protein_seq_mask, dtype=torch.float32)
22
+ seq_loss_weights = seq_loss_weights / mask_num
23
+ seq_loss_weights = seq_loss_weights * protein_seq_mask.float()
24
+ seq_loss_weights = seq_loss_weights[protein_seq_mask.bool()]
25
+
26
+ protein_seq_output, img_diff_loss, img_recon_loss = model_output
27
+
28
+ # Protein sequence loss
29
+ protein_seq_output = protein_seq_output[protein_seq_mask]
30
+ sequence_loss = (
31
+ self.sequence_loss(
32
+ protein_seq_output.to(torch.float32).view(-1, protein_seq_output.size(-1)),
33
+ protein_seq.view(-1),
34
+ )
35
+ * self.seq_loss_coeff
36
+ )
37
+ sequence_loss = sequence_loss * seq_loss_weights
38
+
39
+ if zm_label.all():
40
+ sequence_loss = torch.zeros(1, device=sequence_loss.device).sum()
41
+ else:
42
+ sequence_loss = sequence_loss.sum() / (~zm_label).float().sum()
43
+
44
+ # Protein image diffusion loss
45
+ img_diff_loss = img_diff_loss.mean() * self.img_diff_loss_coeff
46
+
47
+ # Protein image reconstruction loss
48
+ img_recon_loss = img_recon_loss.mean() * self.img_recon_loss_coeff
49
+
50
+ loss = sequence_loss + img_diff_loss + img_recon_loss
51
+
52
+ loss_items = {
53
+ 'sequence_loss': (sequence_loss, self.seq_loss_coeff),
54
+ 'image_diff_loss': (img_diff_loss, self.img_diff_loss_coeff),
55
+ 'image_recon_loss': (img_recon_loss, self.img_recon_loss_coeff),
56
+ }
57
+
58
+ log_loss = {'total_loss': loss.item()}
59
+ for name, (val, coeff) in loss_items.items():
60
+ if coeff > 0:
61
+ log_loss[name] = val.item() / coeff
62
+
63
+ return loss, log_loss
cell_fm/logging/__init__.py ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ # -*- coding: utf-8 -*-
2
+ from .loggers import MetricLogger, get_logger
3
+
4
+ logger = get_logger()
5
+ metric_logger = MetricLogger()
cell_fm/logging/loggers.py ADDED
@@ -0,0 +1,201 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # -*- coding: utf-8 -*-
2
+ import sys
3
+ from dataclasses import dataclass, fields, is_dataclass
4
+ from typing import Dict, Union
5
+
6
+ import torch
7
+ from loguru import logger
8
+
9
+ from cell_fm.utils.dist_utils import is_master_node
10
+ from transformers import TrainerCallback
11
+
12
+ import wandb # isort:skip
13
+
14
+ handlers = {}
15
+
16
+
17
+ def get_logger():
18
+ if not handlers:
19
+ logger.remove() # remove default handler
20
+ handlers["console"] = logger.add(
21
+ sys.stdout,
22
+ format="[<green>{time:YYYY-MM-DD HH:mm:ss.SSS}</green>][<cyan>{level}</cyan>]: {message}",
23
+ colorize=True,
24
+ filter=console_log_filter,
25
+ enqueue=True,
26
+ )
27
+
28
+ return logger
29
+
30
+
31
+ # Custom function to handle tensor attributes
32
+ def dataclass_to_dict(dataclass_obj: Union[dataclass, Dict]) -> Dict:
33
+ if isinstance(dataclass_obj, dict):
34
+ return dataclass_obj
35
+ result = {}
36
+ for field in fields(dataclass_obj):
37
+ value = getattr(dataclass_obj, field.name)
38
+ if isinstance(value, torch.Tensor) and not value.is_leaf:
39
+ result[field.name] = value.clone().detach()
40
+ else:
41
+ result[field.name] = value
42
+ return result
43
+
44
+
45
+ class MetricLogger(object):
46
+ def log(self, metrics, prefix=""):
47
+ if not is_master_node():
48
+ return
49
+
50
+ log_data = {}
51
+ if type(metrics) is dict:
52
+ log_data = metrics
53
+ elif is_dataclass(metrics):
54
+ log_data = dataclass_to_dict(metrics)
55
+
56
+ if "extra_output" in log_data:
57
+ extra_output = log_data["extra_output"]
58
+ if extra_output is not None:
59
+ for k, v in extra_output.items():
60
+ log_data[k] = v
61
+ del log_data["extra_output"]
62
+
63
+ for k in log_data:
64
+ if isinstance(log_data[k], torch.Tensor):
65
+ log_data[k] = log_data[k].detach().item()
66
+
67
+ logger.info(" | ".join([f"{k}={v:.4g}" for k, v in log_data.items()]))
68
+ if wandb.run is not None:
69
+ # Add prefix
70
+ if prefix:
71
+ log_data = {f"{prefix}/{k}": v for k, v in log_data.items()}
72
+ wandb.log(log_data)
73
+
74
+
75
+ def console_log_filter(record):
76
+ # For message with level INFO, we only log it on master node
77
+ # For others, we log it on all nodes
78
+ if record["level"].name != "INFO":
79
+ return True
80
+
81
+ return is_master_node()
82
+
83
+
84
+ class StableDiffusionLoggingCallback(TrainerCallback):
85
+ def __init__(self):
86
+ self.logger = logger
87
+
88
+ def on_log(self, args, state, control, logs=None, **kwargs):
89
+ if logs is None:
90
+ return
91
+
92
+ formatted_log = (
93
+ f"loss={logs.get('loss', 0):.4f} | grad_norm={logs.get('grad_norm', 0):.4f} | "
94
+ f"lr={logs.get('learning_rate', 0):.7f} | epoch={logs.get('epoch', 0):.4f} | "
95
+ f"global_step={state.global_step}"
96
+ )
97
+
98
+ self.logger.info(formatted_log)
99
+
100
+
101
+ class CELLFMLoggingCallback(TrainerCallback):
102
+ def __init__(self):
103
+ self.logger = logger
104
+
105
+ def on_log(self, args, state, control, logs=None, **kwargs):
106
+ if not logs:
107
+ return
108
+
109
+ # key -> format string
110
+ field_fmt = {
111
+ "loss": "{:.4f}",
112
+ "sequence_loss": "{:.4f}",
113
+ "image_diff_loss": "{:.4f}",
114
+ "image_recon_loss": "{:.4f}",
115
+ "grad_norm": "{:.4f}",
116
+ "learning_rate": "{:.7f}",
117
+ "epoch": "{:.4f}",
118
+ "ema_decay": "{:.6f}",
119
+ }
120
+
121
+ parts = []
122
+ for k, fmt in field_fmt.items():
123
+ if k in logs and logs[k] is not None:
124
+ v = logs[k]
125
+ parts.append(f"{k}={fmt.format(float(v))}")
126
+
127
+ parts.append(f"global_step={state.global_step}")
128
+
129
+ formatted_log = " | ".join(parts)
130
+ self.logger.info(formatted_log)
131
+
132
+
133
+ class CELLFMClsLoggingCallback(TrainerCallback):
134
+ def __init__(self):
135
+ self.logger = logger
136
+
137
+ def on_log(self, args, state, control, logs=None, **kwargs):
138
+ if logs is None:
139
+ return
140
+
141
+ loss = logs.get("loss", 0.0)
142
+ image_class_loss = logs.get("image_class_loss", 0.0)
143
+ grad_norm = logs.get("grad_norm", 0.0)
144
+ learning_rate = logs.get("learning_rate", 0.0)
145
+ epoch = logs.get("epoch", 0.0)
146
+
147
+ formatted_log = (
148
+ f"loss={loss:.4f} | image_class_loss={image_class_loss:.4f} | "
149
+ f"grad_norm={grad_norm:.4f} | lr={learning_rate:.7f} | "
150
+ f"epoch={epoch:.4f} | global_step={state.global_step}"
151
+ )
152
+
153
+ self.logger.info(formatted_log)
154
+
155
+
156
+ class VAELoggingCallback(TrainerCallback):
157
+ def __init__(self):
158
+ self.logger = logger
159
+
160
+ def on_log(self, args, state, control, logs=None, **kwargs):
161
+ if logs is None:
162
+ return
163
+
164
+ loss = logs.get("loss", 0.0)
165
+ recon_loss = logs.get("recon_loss", 0.0)
166
+ kl_loss = logs.get("kl_loss", 0.0)
167
+ grad_norm = logs.get("grad_norm", 0.0)
168
+ learning_rate = logs.get("learning_rate", 0.0)
169
+ epoch = logs.get("epoch", 0.0)
170
+
171
+ formatted_log = (
172
+ f"loss={loss:.4f} | recon_loss={recon_loss:.4f} | "
173
+ f"kl_loss={kl_loss:.4f} | grad_norm={grad_norm:.4f} | "
174
+ f"lr={learning_rate:.7f} | epoch={epoch:.4f} | global_step={state.global_step}"
175
+ )
176
+
177
+ self.logger.info(formatted_log)
178
+
179
+
180
+ class VanillaLoggingCallback(TrainerCallback):
181
+ def __init__(self):
182
+ self.logger = logger
183
+
184
+ def on_log(self, args, state, control, logs=None, **kwargs):
185
+ if logs is None:
186
+ return
187
+
188
+ loss = logs.get("loss", 0.0)
189
+ grad_norm = logs.get("grad_norm", 0.0)
190
+ learning_rate = logs.get("learning_rate", 0.0)
191
+ epoch = logs.get("epoch", 0.0)
192
+
193
+ formatted_log = (
194
+ f"loss={loss:.4f} | "
195
+ f"grad_norm={grad_norm:.4f} | lr={learning_rate:.7f} | "
196
+ f"epoch={epoch:.4f} | global_step={state.global_step}"
197
+ f"grad_norm={grad_norm:.4f} | lr={learning_rate:.7f} | "
198
+ f"epoch={epoch:.4f} | global_step={state.global_step}"
199
+ )
200
+
201
+ self.logger.info(formatted_log)
cell_fm/models/__init__.py ADDED
File without changes
cell_fm/models/cell_fm/__init__.py ADDED
File without changes
cell_fm/models/cell_fm/cell_fm_config.py ADDED
@@ -0,0 +1,116 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # -*- coding: utf-8 -*-
2
+ from dataclasses import dataclass
3
+ from transformers import PretrainedConfig
4
+
5
+ @dataclass
6
+ class CELLFMConfig(PretrainedConfig):
7
+ model_type: str = 'flow_matching'
8
+
9
+ # Dataset parameters
10
+ data_path: str = ""
11
+ split_key: str = 'train'
12
+ phase: str = 'train' # 'train', 'val', 'test'
13
+
14
+ data_aug: bool = False
15
+ img_resize: int = 512
16
+ img_crop_size: int = 1024
17
+ seq_zero_mask_ratio: float = 0.0
18
+ seq_full_mask_ratio: float = 0.0
19
+ pre_pad_seq: bool = False
20
+
21
+ cell_image: str = 'nucl' # 'nucl', 'nucl,er', 'nucl,mt', 'nucl,er,mt'
22
+ test_cell_image: str = 'nucl' # 'nucl', 'nucl,er', 'nucl,mt', 'nucl,er,mt'
23
+
24
+ # Transport parameters
25
+ path_type: str = 'Linear'
26
+ prediction: str = 'velocity'
27
+ loss_weight: str = None
28
+ train_eps: float = None
29
+ sample_eps: float = None
30
+
31
+ # Model parameters
32
+ ## VAE
33
+ in_channels: int = 1
34
+ out_channels: int = 1
35
+ num_down_blocks: int = 3
36
+ latent_channels: int = 4
37
+ vae_block_out_channels: str = '128,256,512'
38
+
39
+ ## CELL-Diff
40
+ img_mask_ratio: float = 0.5
41
+ cond_out_channels: str = '32,64'
42
+ sample_size: int = 64
43
+ esm_embedding: str = 'esmc_300m'
44
+ esm_fixed_embedding: bool = True
45
+ encoder_hidden_size: int = 320
46
+ max_protein_sequence_len: int = 2048
47
+ encoder_num_hidden_layers: int = 16
48
+ num_heads: int = 8
49
+ dim_head: int = 64
50
+ vocab_size: int = 33
51
+ dropout: float = 0.0
52
+ final_dropout: float = 0.0
53
+ encoder_patch_size: int = 8
54
+
55
+ ### Image generator
56
+ img_generator_num_layers: int = 18
57
+ img_generator_patch_size: int = 4
58
+ attention_head_dim: int = 64
59
+ num_attention_heads: int = 16
60
+
61
+ ### Image decoder
62
+ img_decoder_num_hidden_layers: int = 8
63
+ img_decoder_hidden_size: int = 512
64
+ img_decoder_num_heads: int = 8
65
+ img_decoder_dim_head: int = 64
66
+ img_decoder_dropout: float = 0.0
67
+ img_decoder_final_dropout: float = 0.0
68
+
69
+ cell_image_ratio: float = 1.0
70
+
71
+ # Loss parameters
72
+ seq_loss_coeff: float = 1.0
73
+ img_diff_loss_coeff: float = 1.0
74
+ img_recon_loss_coeff: float = 1.0
75
+ location_loss_coeff: float = 1.0
76
+
77
+ # Training parameters
78
+ vae_loadcheck_path: str = '.'
79
+ loadcheck_path: str = '.'
80
+ ft: bool = False
81
+ infer: bool = False
82
+ ifresume: bool = False
83
+
84
+ output_dir: str = ""
85
+ learning_rate: float = 1e-4
86
+ weight_decay: float = 0.0
87
+ gradient_accumulation_steps: int = 1
88
+ per_device_train_batch_size: int = 2
89
+ per_device_eval_batch_size: int = 2
90
+
91
+ num_train_epochs: int = 10
92
+ fp16: bool = False
93
+ bf16: bool = False
94
+ logging_dir: str = ""
95
+ logging_steps: int = 10
96
+ max_steps: int = -1
97
+ warmup_steps: int = 1000
98
+ save_steps: int = 1000
99
+
100
+ dataloader_num_workers: int = 8
101
+ seed: int = 6
102
+
103
+ # evaluation
104
+ num_steps: int = 100
105
+
106
+ def __init__(self, **kwargs):
107
+ # Use `super().__init__` to handle arguments from PretrainedConfig
108
+ super().__init__(**kwargs)
109
+
110
+ self.vae_block_out_channels = kwargs.get("vae_block_out_channels", self.vae_block_out_channels)
111
+ if not isinstance(self.vae_block_out_channels, list):
112
+ self.vae_block_out_channels = [int(a) for a in self.vae_block_out_channels.split(',')]
113
+
114
+ self.cond_out_channels = kwargs.get("cond_out_channels", self.cond_out_channels)
115
+ if not isinstance(self.cond_out_channels, list):
116
+ self.cond_out_channels = [int(a) for a in self.cond_out_channels.split(',')]
cell_fm/models/cell_fm/cell_fm_cs_config.py ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # -*- coding: utf-8 -*-
2
+ from dataclasses import dataclass
3
+ from cell_fm.models.cell_fm.cell_fm_config import CELLFMConfig
4
+
5
+
6
+ @dataclass
7
+ class CELLFMCSConfig(CELLFMConfig):
8
+ img_type: str = "GFP" # "GFP", "SNAP"
9
+
10
+ def __init__(self, **kwargs):
11
+ self.img_type = kwargs.pop("img_type", "GFP")
12
+ super().__init__(**kwargs)
cell_fm/models/cell_fm/cell_fm_cs_model.py ADDED
@@ -0,0 +1,309 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import numpy as np
2
+ import torch
3
+ import torch.nn as nn
4
+
5
+ from cell_fm.models.cell_fm.cell_fm_model import CELLFM as BaseCELLFM
6
+ from cell_fm.models.cell_fm.cell_fm_model import CELLFMModel as BaseCELLFMModel
7
+ from cell_fm.models.cell_fm.modules.positional_embedding import get_2d_sincos_pos_embed
8
+ from cell_fm.pipeline.utils import CELLFMOutput
9
+
10
+ from .cell_fm_cs_config import CELLFMConfig
11
+
12
+
13
+ class CELLFMCSModel(BaseCELLFMModel):
14
+ config_class = CELLFMConfig
15
+
16
+ def initialize_net(self, config: CELLFMConfig):
17
+ return CELLFMCS(config)
18
+
19
+ def forward(self, batched_data, **kwargs):
20
+ protein_seq_masked = batched_data["protein_seq_masked"]
21
+ protein_img_latent, cell_img = self.prepare_data(batched_data, self.vae)
22
+ protein_intensity_level = batched_data["protein_intensity_level"]
23
+
24
+ t, x0, x1 = self.transport.sample(protein_img_latent)
25
+ t, xt, ut = self.transport.path_sampler.plan(t, x0, x1)
26
+ img_output, seq_output, img_recon = self.net(
27
+ xt,
28
+ cell_img,
29
+ protein_seq_masked,
30
+ protein_intensity_level,
31
+ t,
32
+ self.config.img_mask_ratio,
33
+ )
34
+ B, *_, C = xt.shape
35
+ assert img_output.size() == (B, *xt.size()[1:-1], C)
36
+
37
+ loss_dict = self.transport.training_losses(img_output, x0, xt, ut, t)
38
+ img_recon_loss = torch.mean((img_recon - protein_img_latent) ** 2)
39
+ model_output = (seq_output, loss_dict["loss"].mean(), img_recon_loss)
40
+ loss, log_loss = self.loss(batched_data, model_output)
41
+
42
+ return CELLFMOutput(loss=loss, log_output=log_loss)
43
+
44
+ def sequence_to_image(self, protein_seq, cell_img, protein_intensity_level, num_steps=100):
45
+ protein_img_latent = torch.randn(
46
+ cell_img.shape[0],
47
+ self.config.latent_channels,
48
+ self.config.sample_size,
49
+ self.config.sample_size,
50
+ device=cell_img.device,
51
+ )
52
+
53
+ with torch.no_grad():
54
+ sample_fn = self.transport_sampler.sample_ode(num_steps=num_steps)
55
+
56
+ def fn(img_latent, t):
57
+ return self.net.inference(
58
+ img_latent,
59
+ cell_img,
60
+ protein_seq,
61
+ protein_intensity_level,
62
+ t,
63
+ 0,
64
+ )[0]
65
+
66
+ protein_img_latent = sample_fn(protein_img_latent, fn)[-1]
67
+
68
+ return self.vae.decode(protein_img_latent).sample
69
+
70
+ def image_to_sequence(
71
+ self,
72
+ protein_seq,
73
+ protein_seq_mask,
74
+ protein_img,
75
+ cell_img,
76
+ protein_intensity_level,
77
+ progress=True,
78
+ sampling_strategy="oaardm",
79
+ order="l2r",
80
+ temperature=1.0,
81
+ ):
82
+ if sampling_strategy == "oaardm":
83
+ return self.oaardm_sample(
84
+ protein_seq,
85
+ protein_seq_mask,
86
+ protein_img,
87
+ cell_img,
88
+ protein_intensity_level,
89
+ progress,
90
+ order,
91
+ temperature,
92
+ )
93
+ raise ValueError(f"Sampling strategy {sampling_strategy} is not supported")
94
+
95
+ def oaardm_sample(
96
+ self,
97
+ protein_seq,
98
+ protein_seq_mask,
99
+ protein_img,
100
+ cell_img,
101
+ protein_intensity_level,
102
+ progress=True,
103
+ order="l2r",
104
+ temperature=1.0,
105
+ ):
106
+ from esm.utils.constants.esm3 import SEQUENCE_VOCAB
107
+
108
+ device = protein_seq.device
109
+ vocab_to_idx = {aa: idx for idx, aa in enumerate(SEQUENCE_VOCAB)}
110
+ standard_aas = ["A", "C", "D", "E", "F", "G", "H", "I", "K", "L", "M", "N", "P", "Q", "R", "S", "T", "V", "W", "Y"]
111
+ allowed_token_ids = torch.tensor(
112
+ [vocab_to_idx[aa] for aa in standard_aas if aa in vocab_to_idx],
113
+ dtype=torch.long,
114
+ device=device,
115
+ )
116
+
117
+ loc = torch.where(protein_seq_mask.bool().squeeze())[0].cpu().numpy()
118
+ if order == "l2r":
119
+ loc = np.sort(loc)
120
+ elif order == "random":
121
+ np.random.shuffle(loc)
122
+
123
+ if progress:
124
+ from tqdm.auto import tqdm
125
+
126
+ loc = tqdm(loc)
127
+
128
+ with torch.no_grad():
129
+ protein_img_latent = self.vae.encode(protein_img).sample()
130
+ batch_size = protein_seq.shape[0]
131
+
132
+ for i in loc:
133
+ _, t1 = self.transport.check_interval(
134
+ self.transport.train_eps,
135
+ self.transport.sample_eps,
136
+ )
137
+ t = torch.full((batch_size,), t1, device=device)
138
+ seq_output = self.net.inference(
139
+ protein_img_latent,
140
+ cell_img,
141
+ protein_seq,
142
+ protein_intensity_level,
143
+ t,
144
+ )[1]
145
+ logits_i = seq_output[:, i, :]
146
+ aa_logits = logits_i[:, allowed_token_ids]
147
+ probs = torch.nn.functional.softmax(aa_logits / temperature, dim=-1)
148
+ sample_idx = torch.multinomial(probs, num_samples=1).squeeze(-1)
149
+ protein_seq[:, i] = allowed_token_ids[sample_idx]
150
+
151
+ return protein_seq
152
+
153
+ def recon(self, protein_seq, protein_img, cell_img, protein_intensity_level):
154
+ with torch.no_grad():
155
+ protein_img_latent = self.vae.encode(protein_img).sample()
156
+ _, t1 = self.transport.check_interval(self.transport.train_eps, self.transport.sample_eps)
157
+ t = torch.tensor([t1] * protein_seq.shape[0], device=protein_seq.device)
158
+ img_recon_latent = self.net(
159
+ protein_img_latent,
160
+ cell_img,
161
+ protein_seq,
162
+ protein_intensity_level,
163
+ t,
164
+ img_mask_ratio=0.5,
165
+ )[2]
166
+
167
+ return self.vae.decode(img_recon_latent).sample
168
+
169
+
170
+ class CELLFMCS(BaseCELLFM):
171
+ def __init__(self, config: CELLFMConfig):
172
+ super().__init__(config)
173
+
174
+ self.protein_intensity_level_embedding = nn.Linear(
175
+ in_features=1,
176
+ out_features=config.encoder_hidden_size,
177
+ bias=True,
178
+ )
179
+ self.time_step_pos_embed = nn.Parameter(
180
+ torch.randn(1, 1, config.encoder_hidden_size) * 0.02,
181
+ requires_grad=True,
182
+ )
183
+ self.protein_intensity_level_pos_embed = nn.Parameter(
184
+ torch.randn(1, 1, config.encoder_hidden_size) * 0.02,
185
+ requires_grad=True,
186
+ )
187
+
188
+ img_encoder_pos_embed = get_2d_sincos_pos_embed(
189
+ config.encoder_hidden_size,
190
+ self.img_embedding.base_size,
191
+ )
192
+ self.img_encoder_pos_embed = nn.Parameter(
193
+ torch.from_numpy(img_encoder_pos_embed).float().unsqueeze(0),
194
+ requires_grad=False,
195
+ )
196
+
197
+ img_decoder_pos_embed = get_2d_sincos_pos_embed(
198
+ config.img_decoder_hidden_size,
199
+ self.img_embedding.base_size,
200
+ )
201
+ self.img_decoder_pos_embed = nn.Parameter(
202
+ torch.from_numpy(img_decoder_pos_embed).float().unsqueeze(0),
203
+ requires_grad=False,
204
+ )
205
+
206
+ self._init_weights(self.protein_intensity_level_embedding)
207
+
208
+ def _encode_conditioned(
209
+ self,
210
+ protein_img,
211
+ cell_img,
212
+ protein_seq,
213
+ protein_intensity_level,
214
+ time,
215
+ img_mask_ratio,
216
+ ):
217
+ time_embeds = self.time_embedding(time)
218
+ intensity_embeds = self.protein_intensity_level_embedding(protein_intensity_level)
219
+
220
+ seq_embeds = self.protein_sequence_embedding(protein_seq)
221
+ seq_embeds = self.seq_proj_in(seq_embeds)
222
+ seq_embeds = seq_embeds + self.protein_sequence_pos_embed[:, : protein_seq.shape[1]]
223
+ seq_token_type = torch.full(
224
+ size=(seq_embeds.shape[0], 1),
225
+ fill_value=1,
226
+ device=seq_embeds.device,
227
+ ).long()
228
+ seq_embeds = seq_embeds + self.token_type_embeddings(seq_token_type)
229
+
230
+ cell_img_conv = self.cond_conv(cell_img)
231
+ concat_img = torch.cat([protein_img, cell_img_conv], dim=1)
232
+ img_embeds = self.img_embedding(concat_img)
233
+ img_embeds = img_embeds + self.img_encoder_pos_embed
234
+ img_embeds, _, img_ids_restore = self.random_masking(img_embeds, img_mask_ratio)
235
+
236
+ time_embeds_with_pos = time_embeds.unsqueeze(1) + self.time_step_pos_embed
237
+ intensity_embeds_with_pos = intensity_embeds.unsqueeze(1) + self.protein_intensity_level_pos_embed
238
+ img_embeds = torch.cat([time_embeds_with_pos, intensity_embeds_with_pos, img_embeds], dim=1)
239
+ img_token_type = torch.full(
240
+ size=(img_embeds.shape[0], 1),
241
+ fill_value=0,
242
+ device=img_embeds.device,
243
+ ).long()
244
+ img_embeds = img_embeds + self.token_type_embeddings(img_token_type)
245
+
246
+ co_embeds = torch.cat([img_embeds, seq_embeds], dim=1)
247
+ co_feat = co_embeds
248
+ for block in self.unified_encoder:
249
+ co_feat = block(co_feat)
250
+
251
+ return concat_img, img_embeds, img_ids_restore, co_feat, time_embeds
252
+
253
+ def forward(self, protein_img, cell_img, protein_seq, protein_intensity_level, time, img_mask_ratio=0.5):
254
+ concat_img, img_embeds, img_ids_restore, co_feat, time_embeds = self._encode_conditioned(
255
+ protein_img,
256
+ cell_img,
257
+ protein_seq,
258
+ protein_intensity_level,
259
+ time,
260
+ img_mask_ratio,
261
+ )
262
+ img_output = self.img_generator(
263
+ hidden_states=concat_img,
264
+ timestep=time,
265
+ encoder_hidden_states=co_feat,
266
+ pooled_projections=time_embeds,
267
+ ).sample
268
+ seq_feat = co_feat[:, img_embeds.shape[1] :]
269
+ img_feat = co_feat[:, : img_embeds.shape[1]]
270
+ seq_output = self.seq_decoder(seq_feat)
271
+ img_recon = self.forward_img_decoder(img_feat, img_ids_restore)
272
+ img_recon = self.unpatchify(img_recon, self.config.latent_channels)
273
+
274
+ return img_output, seq_output, img_recon
275
+
276
+ def forward_img_decoder(self, x, ids_restore):
277
+ x = self.img_decoder_embed(x)
278
+ mask_tokens = self.mask_token.repeat(x.shape[0], ids_restore.shape[1] + 2 - x.shape[1], 1)
279
+ x_ = torch.cat([x[:, 2:, :], mask_tokens], dim=1)
280
+ x_ = torch.gather(x_, dim=1, index=ids_restore.unsqueeze(-1).repeat(1, 1, x.shape[2]))
281
+ x_ = x_ + self.img_decoder_pos_embed
282
+ x = torch.cat([x[:, :2, :], x_], dim=1)
283
+
284
+ for block in self.img_decoder:
285
+ x = block(x)
286
+ x = self.img_decoder_norm(x)
287
+ x = self.img_decoder_pred(x)
288
+
289
+ return x[:, 2:, :]
290
+
291
+ def inference(self, protein_img, cell_img, protein_seq, protein_intensity_level, time, img_mask_ratio=0):
292
+ concat_img, img_embeds, _, co_feat, time_embeds = self._encode_conditioned(
293
+ protein_img,
294
+ cell_img,
295
+ protein_seq,
296
+ protein_intensity_level,
297
+ time,
298
+ img_mask_ratio,
299
+ )
300
+ img_output = self.img_generator(
301
+ hidden_states=concat_img,
302
+ timestep=time,
303
+ encoder_hidden_states=co_feat,
304
+ pooled_projections=time_embeds,
305
+ ).sample
306
+ seq_feat = co_feat[:, img_embeds.shape[1] :]
307
+ seq_output = self.seq_decoder(seq_feat)
308
+
309
+ return img_output, seq_output
cell_fm/models/cell_fm/cell_fm_model.py ADDED
@@ -0,0 +1,637 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import torch
3
+ import torch.nn as nn
4
+ import numpy as np
5
+ from cell_fm.logging import logger
6
+
7
+ from diffusers.models.embeddings import PatchEmbed
8
+ from diffusers.models.transformers import SD3Transformer2DModel
9
+
10
+ from .modules import TimestepEmbedder, CondConvNet
11
+ from .modules.positional_embedding import get_1d_sincos_pos_embed, get_2d_sincos_pos_embed
12
+ from .modules.transformer import TransformerBlock, MLMHead
13
+ from .modules.protein_sequence_embedding import ESMEmbed
14
+ from .modules.transport import create_transport, Sampler
15
+
16
+ from transformers import PreTrainedModel
17
+ from .cell_fm_config import CELLFMConfig
18
+
19
+ from cell_fm.pipeline.utils import CELLFMOutput
20
+ from cell_fm.models.vae.vae_model import VAEModel
21
+ import json
22
+ from copy import deepcopy
23
+
24
+
25
+ class CELLFMModel(PreTrainedModel):
26
+ config_class = CELLFMConfig
27
+
28
+ def __init__(self, config: CELLFMConfig, loss_fn=None):
29
+ super().__init__(config)
30
+ self.config = config
31
+
32
+ self.loss = loss_fn(config)
33
+ self.net = self.initialize_net(config)
34
+
35
+ self.transport = create_transport(
36
+ config.path_type,
37
+ config.prediction,
38
+ config.loss_weight,
39
+ config.train_eps,
40
+ config.sample_eps,
41
+ )
42
+ self.transport_sampler = Sampler(self.transport)
43
+
44
+ self.vae = self.initialize_vae()
45
+ self.load_pretrained_weights(config, checkpoint_path=config.loadcheck_path)
46
+ self.vae = self.prepare_vae(self.vae, self.config.vae_loadcheck_path)
47
+
48
+ def initialize_net(self, config: CELLFMConfig):
49
+ return CELLFM(config)
50
+
51
+ def initialize_vae(self):
52
+ vae_config = deepcopy(self.config)
53
+ vae_config.ft = False
54
+ vae_config.infer = False
55
+ vae_config.vae_loadcheck_path = None
56
+ vae = VAEModel(vae_config)
57
+
58
+ return vae
59
+
60
+ def prepare_vae(self, vae: VAEModel, checkpoint_path):
61
+ if checkpoint_path is not None:
62
+ vae_config = deepcopy(self.config)
63
+ vae_config.ft = False
64
+ vae_config.infer = True
65
+ vae.load_pretrained_weights(vae_config, checkpoint_path=checkpoint_path)
66
+
67
+ for param in vae.parameters():
68
+ param.requires_grad = False
69
+ vae.eval()
70
+
71
+ return vae
72
+
73
+ def load_pretrained_weights(self, config, checkpoint_path):
74
+ """
75
+ Load pretrained weights from a given state_dict.
76
+ """
77
+ if config.ft or config.infer:
78
+ if config.ft:
79
+ logger.info(f"Finetune from checkpoint: {checkpoint_path}")
80
+ else:
81
+ logger.info(f"Infer from checkpoint: {checkpoint_path}")
82
+
83
+ if os.path.isdir(checkpoint_path):
84
+ with open(os.path.join(checkpoint_path, "pytorch_model.bin.index.json"), "r") as f:
85
+ index_data = json.load(f)
86
+
87
+ weight_map = index_data["weight_map"]
88
+
89
+ shard_files = set(weight_map.values())
90
+ shard_weights = {}
91
+ for shard_file in shard_files:
92
+ shard_weights[shard_file] = torch.load(os.path.join(checkpoint_path, shard_file), map_location="cpu")
93
+
94
+ checkpoints_state = {}
95
+ for param_name, shard_file in weight_map.items():
96
+ checkpoints_state[param_name] = shard_weights[shard_file][param_name]
97
+ elif os.path.splitext(checkpoint_path)[1] == '.safetensors':
98
+ from safetensors.torch import load_file
99
+ checkpoints_state = load_file(checkpoint_path)
100
+ else:
101
+ checkpoints_state = torch.load(checkpoint_path, map_location="cpu")
102
+
103
+ if "model" in checkpoints_state:
104
+ checkpoints_state = checkpoints_state["model"]
105
+ elif "module" in checkpoints_state:
106
+ checkpoints_state = checkpoints_state["module"]
107
+
108
+ model_state_dict = self.state_dict()
109
+ filtered_state_dict = {k: v for k, v in checkpoints_state.items() if k in model_state_dict and v.size() == model_state_dict[k].size()}
110
+
111
+ IncompatibleKeys = self.load_state_dict(filtered_state_dict, strict=False)
112
+ IncompatibleKeys = IncompatibleKeys._asdict()
113
+
114
+ missing_keys = []
115
+ for keys in IncompatibleKeys["missing_keys"]:
116
+ if keys.find("dummy") == -1:
117
+ missing_keys.append(keys)
118
+
119
+ unexpected_keys = []
120
+ for keys in IncompatibleKeys["unexpected_keys"]:
121
+ if keys.find("dummy") == -1:
122
+ unexpected_keys.append(keys)
123
+
124
+ if len(missing_keys) > 0:
125
+ logger.info(
126
+ "Missing keys in {}: {}".format(
127
+ checkpoint_path,
128
+ missing_keys,
129
+ )
130
+ )
131
+
132
+ if len(unexpected_keys) > 0:
133
+ logger.info(
134
+ "Unexpected keys {}: {}".format(
135
+ checkpoint_path,
136
+ unexpected_keys,
137
+ )
138
+ )
139
+
140
+ @torch.no_grad()
141
+ def prepare_data(self, batched_data, vae: VAEModel):
142
+
143
+ # protein_img = batched_data['protein_img']
144
+ # nucleus_img = batched_data['nucleus_img']
145
+ # microtubules_img = batched_data['microtubules_img']
146
+ # protein_seq = batched_data['protein_seq']
147
+ # protein_seq_masked = batched_data['protein_seq_masked']
148
+ # protein_seq_mask = batched_data['protein_seq_mask']
149
+
150
+ protein_img = batched_data['protein_img']
151
+ nucleus_img = batched_data['nucleus_img']
152
+ protein_img_latent = vae.encode(protein_img).sample()
153
+
154
+ if self.config.cell_image == 'nucl':
155
+ cell_img = nucleus_img
156
+
157
+ elif self.config.cell_image == 'nucl,er':
158
+ if 'ER_img' in batched_data:
159
+ ER_img = batched_data['ER_img']
160
+ mask = (torch.rand(ER_img.shape[0]) <= self.config.cell_image_ratio).bool()
161
+ ER_img[~mask] = torch.full(size = [ER_img.shape[2], ER_img.shape[3]], fill_value=-2.0, device=ER_img.device)
162
+ else:
163
+ ER_img = torch.full_like(protein_img, fill_value=-2.0)
164
+ cell_img = torch.cat([nucleus_img, ER_img], dim=1)
165
+
166
+ elif self.config.cell_image == 'nucl,mt':
167
+ if 'microtubules_img' in batched_data:
168
+ microtubules_img = batched_data['microtubules_img']
169
+ mask = (torch.rand(microtubules_img.shape[0]) <= self.config.cell_image_ratio).bool()
170
+ microtubules_img[~mask] = torch.full(size = [microtubules_img.shape[2], microtubules_img.shape[3]], fill_value=-2.0, device=microtubules_img.device)
171
+ else:
172
+ microtubules_img = torch.full_like(protein_img, fill_value=-2.0)
173
+ cell_img = torch.cat([nucleus_img, microtubules_img], dim=1)
174
+
175
+ elif self.config.cell_image == 'nucl,er,mt':
176
+ if 'ER_img' in batched_data:
177
+ ER_img = batched_data['ER_img']
178
+ mask = (torch.rand(ER_img.shape[0]) <= self.config.cell_image_ratio).bool()
179
+ ER_img[~mask] = torch.full(size = [ER_img.shape[2], ER_img.shape[3]], fill_value=-2.0, device=ER_img.device)
180
+ else:
181
+ ER_img = torch.full_like(protein_img, fill_value=-2.0)
182
+
183
+ if 'microtubules_img' in batched_data:
184
+ microtubules_img = batched_data['microtubules_img']
185
+ mask = (torch.rand(microtubules_img.shape[0]) <= self.config.cell_image_ratio).bool()
186
+ microtubules_img[~mask] = torch.full(size = [microtubules_img.shape[2], microtubules_img.shape[3]], fill_value=-2.0, device=microtubules_img.device)
187
+ else:
188
+ microtubules_img = torch.full_like(protein_img, fill_value=-2.0)
189
+ cell_img = torch.cat([nucleus_img, ER_img, microtubules_img], dim=1)
190
+ else:
191
+ raise ValueError(f"Cell image type: {self.config.cell_image} is not supported")
192
+
193
+ return protein_img_latent, cell_img
194
+
195
+ def forward(self, batched_data, **kwargs):
196
+ protein_seq_masked = batched_data['protein_seq_masked']
197
+ protein_img_latent, cell_img = self.prepare_data(batched_data, self.vae)
198
+
199
+ t, x0, x1 = self.transport.sample(protein_img_latent)
200
+ t, xt, ut = self.transport.path_sampler.plan(t, x0, x1)
201
+ img_output, seq_output, img_recon = self.net(xt, cell_img, protein_seq_masked, t, self.config.img_mask_ratio)
202
+ B, *_, C = xt.shape
203
+ assert img_output.size() == (B, *xt.size()[1:-1], C)
204
+
205
+ loss_dict = self.transport.training_losses(img_output, x0, xt, ut, t)
206
+
207
+ img_recon_loss = torch.mean((img_recon - protein_img_latent)**2)
208
+
209
+ model_output = (seq_output, loss_dict["loss"].mean(), img_recon_loss)
210
+
211
+ loss, log_loss = self.loss(batched_data, model_output)
212
+
213
+ return CELLFMOutput(loss=loss, log_output=log_loss)
214
+
215
+ def sequence_to_image(self, protein_seq, cell_img, num_steps=100):
216
+ protein_img_latent = torch.randn(cell_img.shape[0], self.config.latent_channels, self.config.sample_size, self.config.sample_size).to(cell_img.device)
217
+
218
+ with torch.no_grad():
219
+ sample_fn = self.transport_sampler.sample_ode(num_steps=num_steps) # default to ode sampling
220
+ def fn(img_latent, t):
221
+ return self.net.seq2img(img_latent, cell_img, protein_seq, t, 0)
222
+
223
+ protein_img_latent = sample_fn(protein_img_latent, fn)[-1]
224
+
225
+ return self.vae.decode(protein_img_latent).sample
226
+
227
+ def image_to_sequence(self, protein_seq, protein_seq_mask, protein_img, cell_img, progress=True, sampling_strategy='oaardm', order='l2r', temperature=1.0):
228
+ if sampling_strategy == "oaardm":
229
+ return self.oaardm_sample(protein_seq, protein_seq_mask, protein_img, cell_img, progress, order, temperature)
230
+
231
+ def oaardm_sample(self, protein_seq, protein_seq_mask, protein_img, cell_img, progress=True, order='l2r', temperature=1.0):
232
+ loc = torch.where(protein_seq_mask.bool().squeeze())[0].cpu().numpy()
233
+
234
+ if order == 'l2r':
235
+ loc = np.sort(loc)
236
+ elif order == 'random':
237
+ np.random.shuffle(loc)
238
+
239
+ if progress:
240
+ # Lazy import so that we don't depend on tqdm.
241
+ from tqdm.auto import tqdm
242
+ loc = tqdm(loc)
243
+
244
+ with torch.no_grad():
245
+ protein_img_latent = self.vae.encode(protein_img).sample()
246
+ for i in loc:
247
+ t0, t1 = self.transport.check_interval(self.transport.train_eps, self.transport.sample_eps)
248
+ # t1 = 1.0, t0 = 0.0
249
+ t = torch.tensor([t1] * protein_seq.shape[0], device=protein_seq.device)
250
+ seq_output = self.net(protein_img_latent, cell_img, protein_seq, t, img_mask_ratio=0)[1]
251
+ p = seq_output[:, i, 4:4+20] # sample at location i (random), dont let it predict non-standard AA
252
+ p = torch.nn.functional.softmax(p / temperature, dim=1) # softmax over categorical probs
253
+ p_sample = torch.multinomial(p, num_samples=1)
254
+ protein_seq[:, i] = p_sample.squeeze() + 4
255
+
256
+ return protein_seq
257
+
258
+ def embed(self, protein_seq, protein_img, cell_img, img_mask_ratio=0):
259
+ with torch.no_grad():
260
+ protein_img_latent = self.vae.encode(protein_img).sample()
261
+
262
+ t0, t1 = self.transport.check_interval(self.transport.train_eps, self.transport.sample_eps)
263
+ t = torch.tensor([t1] * protein_seq.shape[0], device=protein_seq.device)
264
+ img_feat_embed, seq_feat_embed = self.net.embed(protein_img_latent, cell_img, protein_seq, t, img_mask_ratio=img_mask_ratio)
265
+
266
+ return img_feat_embed, seq_feat_embed
267
+
268
+ def recon(self, protein_seq, protein_img, cell_img):
269
+ with torch.no_grad():
270
+ protein_img_latent = self.vae.encode(protein_img).sample()
271
+
272
+ t0, t1 = self.transport.check_interval(self.transport.train_eps, self.transport.sample_eps)
273
+ t = torch.tensor([t1] * protein_seq.shape[0], device=protein_seq.device)
274
+ img_recon_latent = self.net(protein_img_latent, cell_img, protein_seq, t, img_mask_ratio=0.5)[2]
275
+
276
+ img_recon = self.vae.decode(img_recon_latent).sample
277
+ return img_recon
278
+
279
+
280
+ class CELLFM(nn.Module):
281
+ def __init__(self, config: CELLFMConfig):
282
+ super().__init__()
283
+ self.config = config
284
+
285
+ self.cond_conv = CondConvNet(
286
+ config.in_channels * len(config.cell_image.split(',')),
287
+ config.cond_out_channels,
288
+ )
289
+
290
+ # Time step embedding
291
+ self.time_embedding = TimestepEmbedder(hidden_size=config.encoder_hidden_size)
292
+
293
+ # Token type embeddings
294
+ self.token_type_embeddings = nn.Embedding(2, config.encoder_hidden_size)
295
+
296
+ # Image embedding without positional embedding
297
+ self.img_embedding = PatchEmbed(
298
+ height=config.sample_size,
299
+ width=config.sample_size,
300
+ patch_size=config.encoder_patch_size,
301
+ in_channels=config.latent_channels + config.cond_out_channels[-1],
302
+ embed_dim=config.encoder_hidden_size,
303
+ pos_embed_type=None,
304
+ )
305
+
306
+ # Image positional embedding
307
+ # Initialize pos_embed by sin-cos embedding:
308
+ img_encoder_pos_embed = get_2d_sincos_pos_embed(
309
+ config.encoder_hidden_size, self.img_embedding.base_size, cls_token=True, extra_tokens=1,
310
+ )
311
+ # Use fixed sin-cos embedding:
312
+ self.img_encoder_pos_embed = nn.Parameter(
313
+ torch.from_numpy(img_encoder_pos_embed).float().unsqueeze(0),
314
+ requires_grad=False,
315
+ )
316
+
317
+ # Sequence positional embedding
318
+ # Initialize pos_embed by sin-cos embedding:
319
+ protein_sequence_pos_embed = get_1d_sincos_pos_embed(
320
+ config.encoder_hidden_size, self.config.max_protein_sequence_len+2,
321
+ )
322
+ # Use fixed sin-cos embedding:
323
+ self.protein_sequence_pos_embed = nn.Parameter(
324
+ torch.from_numpy(protein_sequence_pos_embed).float().unsqueeze(0),
325
+ requires_grad=False,
326
+ )
327
+
328
+ # Sequence projection
329
+ self.seq_proj_in = nn.Linear(config.encoder_hidden_size, config.encoder_hidden_size, bias=True)
330
+
331
+ # Unified encoder
332
+ self.unified_encoder = nn.ModuleList([
333
+ TransformerBlock(
334
+ hidden_size=config.encoder_hidden_size,
335
+ num_heads=config.num_heads,
336
+ dim_head=config.dim_head,
337
+ dropout=config.dropout,
338
+ final_dropout=config.final_dropout,
339
+ ) for _ in range(config.encoder_num_hidden_layers)
340
+ ])
341
+
342
+ # Image generator
343
+ self.img_generator = SD3Transformer2DModel(
344
+ sample_size=config.sample_size,
345
+ patch_size=config.img_generator_patch_size,
346
+ in_channels=config.latent_channels + config.cond_out_channels[-1],
347
+ num_layers=config.img_generator_num_layers,
348
+ attention_head_dim=config.attention_head_dim,
349
+ num_attention_heads=config.num_attention_heads,
350
+ joint_attention_dim=config.encoder_hidden_size,
351
+ pooled_projection_dim=config.encoder_hidden_size,
352
+ caption_projection_dim=config.num_attention_heads * config.attention_head_dim,
353
+ out_channels=config.latent_channels,
354
+ )
355
+
356
+ # Image decoder
357
+ self.img_decoder_embed = nn.Linear(config.encoder_hidden_size, config.img_decoder_hidden_size, bias=True)
358
+ self.mask_token = nn.Parameter(torch.zeros(1, 1, config.img_decoder_hidden_size))
359
+
360
+ # Initialize pos_embed by sin-cos embedding:
361
+ img_decoder_pos_embed = get_2d_sincos_pos_embed(
362
+ config.img_decoder_hidden_size, self.img_embedding.base_size, cls_token=True, extra_tokens=1,
363
+ )
364
+ # Use fixed sin-cos embedding:
365
+ self.img_decoder_pos_embed = nn.Parameter(
366
+ torch.from_numpy(img_decoder_pos_embed).float().unsqueeze(0),
367
+ requires_grad=False,
368
+ )
369
+
370
+ self.img_decoder = nn.ModuleList([
371
+ TransformerBlock(
372
+ hidden_size=config.img_decoder_hidden_size,
373
+ num_heads=config.img_decoder_num_heads,
374
+ dim_head=config.img_decoder_dim_head,
375
+ dropout=config.img_decoder_dropout,
376
+ final_dropout=config.img_decoder_final_dropout,
377
+ ) for _ in range(config.img_decoder_num_hidden_layers)
378
+ ])
379
+ self.img_decoder_norm = nn.LayerNorm(config.img_decoder_hidden_size)
380
+ self.img_decoder_pred = nn.Linear(config.img_decoder_hidden_size, config.encoder_patch_size**2 * config.latent_channels, bias=True) # decoder to patch
381
+
382
+ # Sequence decoder
383
+ self.seq_decoder = MLMHead(
384
+ hidden_size=config.encoder_hidden_size,
385
+ vocab_size=config.vocab_size,
386
+ )
387
+
388
+ self.initialize_weights()
389
+ self.initialize_protein_sequence_embedding()
390
+
391
+ def initialize_weights(self):
392
+ # initialization
393
+ torch.nn.init.normal_(self.mask_token, std=.02)
394
+
395
+ # initialize nn.Linear and nn.LayerNorm
396
+ self.apply(self._init_weights)
397
+
398
+ def _init_weights(self, m):
399
+ if isinstance(m, nn.Linear):
400
+ # we use xavier_uniform following official JAX ViT:
401
+ torch.nn.init.xavier_uniform_(m.weight)
402
+ if isinstance(m, nn.Linear) and m.bias is not None:
403
+ nn.init.constant_(m.bias, 0)
404
+ elif isinstance(m, nn.LayerNorm):
405
+ if m.bias is not None:
406
+ nn.init.constant_(m.bias, 0)
407
+ if m.weight is not None:
408
+ nn.init.constant_(m.weight, 1.0)
409
+
410
+ def initialize_protein_sequence_embedding(self):
411
+ self.protein_sequence_embedding = ESMEmbed(self.config.esm_embedding, self.config.encoder_hidden_size, self.config.esm_fixed_embedding)
412
+
413
+ def forward(self, protein_img, cell_img, protein_seq, time, img_mask_ratio=0.5):
414
+ time_embeds = self.time_embedding(time)
415
+
416
+ # Tokenize protein sequence
417
+ # Size: B x T x C
418
+ seq_embeds = self.protein_sequence_embedding(protein_seq)
419
+ seq_embeds = self.seq_proj_in(seq_embeds)
420
+ seq_embeds = seq_embeds + self.protein_sequence_pos_embed[:, :protein_seq.shape[1]]
421
+
422
+ seq_token_type = torch.full(
423
+ size=(seq_embeds.shape[0], 1), fill_value=1, device=seq_embeds.device
424
+ ).long()
425
+
426
+ seq_embeds = seq_embeds + self.token_type_embeddings(seq_token_type)
427
+
428
+ cell_img_conv = self.cond_conv(cell_img)
429
+ concat_img = torch.cat([protein_img, cell_img_conv], dim=1)
430
+ img_embeds = self.img_embedding(concat_img)
431
+
432
+ # add pos embed w/o time step token
433
+ img_embeds = img_embeds + self.img_encoder_pos_embed[:, 1:, :]
434
+
435
+ # masking: length -> length * mask_ratio
436
+ img_embeds, img_mask, img_ids_restore = self.random_masking(img_embeds, img_mask_ratio)
437
+
438
+ # append time step token
439
+ time_embeds_with_pos_embed = time_embeds.unsqueeze(1) + self.img_encoder_pos_embed[:, :1, :]
440
+ img_embeds = torch.cat([time_embeds_with_pos_embed, img_embeds], dim=1)
441
+
442
+ img_token_type = torch.full(
443
+ size=(img_embeds.shape[0], 1), fill_value=0, device=img_embeds.device,
444
+ ).long()
445
+
446
+ img_embeds = img_embeds + self.token_type_embeddings(img_token_type)
447
+ co_embeds = torch.cat([img_embeds, seq_embeds], dim=1)
448
+
449
+ x = co_embeds
450
+
451
+ for i, block in enumerate(self.unified_encoder):
452
+ x = block(x)
453
+
454
+ co_feat = x
455
+
456
+ img_output = self.img_generator(
457
+ hidden_states=concat_img, timestep=time, encoder_hidden_states=co_feat,
458
+ pooled_projections=time_embeds,
459
+ ).sample
460
+
461
+ seq_feat = co_feat[:, img_embeds.shape[1]:]
462
+ img_feat = co_feat[:, :img_embeds.shape[1]]
463
+
464
+ seq_output = self.seq_decoder(seq_feat)
465
+
466
+ img_recon = self.forward_img_decoder(img_feat, img_ids_restore)
467
+ img_recon = self.unpatchify(img_recon, self.config.latent_channels)
468
+
469
+ return img_output, seq_output, img_recon
470
+
471
+ def random_masking(self, x, mask_ratio):
472
+ """
473
+ Perform per-sample random masking by per-sample shuffling.
474
+ Per-sample shuffling is done by argsort random noise.
475
+ x: [N, L, D], sequence
476
+ """
477
+ N, L, D = x.shape # batch, length, dim
478
+ len_keep = int(L * (1 - mask_ratio))
479
+
480
+ noise = torch.rand(N, L, device=x.device) # noise in [0, 1]
481
+
482
+ # sort noise for each sample
483
+ ids_shuffle = torch.argsort(noise, dim=1) # ascend: small is keep, large is remove
484
+ ids_restore = torch.argsort(ids_shuffle, dim=1)
485
+
486
+ # keep the first subset
487
+ ids_keep = ids_shuffle[:, :len_keep]
488
+ x_masked = torch.gather(x, dim=1, index=ids_keep.unsqueeze(-1).repeat(1, 1, D))
489
+
490
+ # generate the binary mask: 0 is keep, 1 is remove
491
+ mask = torch.ones([N, L], device=x.device)
492
+ mask[:, :len_keep] = 0
493
+ # unshuffle to get the binary mask
494
+ mask = torch.gather(mask, dim=1, index=ids_restore)
495
+
496
+ return x_masked, mask, ids_restore
497
+
498
+ def forward_img_decoder(self, x, ids_restore):
499
+ # embed tokens
500
+ x = self.img_decoder_embed(x)
501
+
502
+ # append mask tokens to sequence
503
+ mask_tokens = self.mask_token.repeat(x.shape[0], ids_restore.shape[1] + 1 - x.shape[1], 1)
504
+ x_ = torch.cat([x[:, 1:, :], mask_tokens], dim=1) # no time step token
505
+ x_ = torch.gather(x_, dim=1, index=ids_restore.unsqueeze(-1).repeat(1, 1, x.shape[2])) # unshuffle
506
+ x = torch.cat([x[:, :1, :], x_], dim=1) # append time step token
507
+
508
+ # add pos embed
509
+ x = x + self.img_decoder_pos_embed
510
+
511
+ # apply Transformer blocks
512
+ for block in self.img_decoder:
513
+ x = block(x)
514
+ x = self.img_decoder_norm(x)
515
+
516
+ # predictor projection
517
+ x = self.img_decoder_pred(x)
518
+
519
+ # remove time step token
520
+ x = x[:, 1:, :]
521
+
522
+ return x
523
+
524
+ def unpatchify(self, x, num_channels):
525
+ """
526
+ x: (N, L, patch_size**2 *C)
527
+ imgs: (N, C, H, W)
528
+ """
529
+ p = self.config.encoder_patch_size
530
+ h = w = int(x.shape[1]**.5)
531
+ assert h * w == x.shape[1]
532
+
533
+ x = x.reshape(shape=(x.shape[0], h, w, p, p, num_channels))
534
+ x = torch.einsum('nhwpqc->nchpwq', x)
535
+ imgs = x.reshape(shape=(x.shape[0], num_channels, h * p, h * p))
536
+ return imgs
537
+
538
+ def embed(self, protein_img, cell_img, protein_seq, time, img_mask_ratio=0):
539
+ time_embeds = self.time_embedding(time)
540
+
541
+ # Tokenize protein sequence
542
+ # Size: B x T x C
543
+ seq_embeds = self.protein_sequence_embedding(protein_seq)
544
+ seq_embeds = self.seq_proj_in(seq_embeds)
545
+ seq_embeds = seq_embeds + self.protein_sequence_pos_embed[:, :protein_seq.shape[1]]
546
+
547
+ seq_token_type = torch.full(
548
+ size=(seq_embeds.shape[0], 1), fill_value=1, device=seq_embeds.device
549
+ ).long()
550
+
551
+ seq_embeds = seq_embeds + self.token_type_embeddings(seq_token_type)
552
+
553
+ cell_img_conv = self.cond_conv(cell_img)
554
+ concat_img = torch.cat([protein_img, cell_img_conv], dim=1)
555
+ img_embeds = self.img_embedding(concat_img)
556
+
557
+ # add pos embed w/o time step token
558
+ img_embeds = img_embeds + self.img_encoder_pos_embed[:, 1:, :]
559
+
560
+ # masking: length -> length * mask_ratio
561
+ img_embeds, img_mask, img_ids_restore = self.random_masking(img_embeds, img_mask_ratio)
562
+
563
+ # append time step token
564
+ time_embeds_with_pos_embed = time_embeds.unsqueeze(1) + self.img_encoder_pos_embed[:, :1, :]
565
+ img_embeds = torch.cat([time_embeds_with_pos_embed, img_embeds], dim=1)
566
+
567
+ img_token_type = torch.full(
568
+ size=(img_embeds.shape[0], 1), fill_value=0, device=img_embeds.device,
569
+ ).long()
570
+
571
+ img_embeds = img_embeds + self.token_type_embeddings(img_token_type)
572
+ co_embeds = torch.cat([img_embeds, seq_embeds], dim=1)
573
+
574
+ x = co_embeds
575
+
576
+ for i, block in enumerate(self.unified_encoder):
577
+ x = block(x)
578
+
579
+ co_feat = x
580
+
581
+ seq_feat = co_feat[:, img_embeds.shape[1]:]
582
+ img_feat = co_feat[:, :img_embeds.shape[1]]
583
+
584
+ img_feat_embed = img_feat
585
+ seq_feat_embed = seq_feat
586
+
587
+ return img_feat_embed, seq_feat_embed
588
+
589
+ def seq2img(self, protein_img, cell_img, protein_seq, time, img_mask_ratio=0):
590
+ time_embeds = self.time_embedding(time)
591
+
592
+ # Tokenize protein sequence
593
+ # Size: B x T x C
594
+ seq_embeds = self.protein_sequence_embedding(protein_seq)
595
+ seq_embeds = self.seq_proj_in(seq_embeds)
596
+ seq_embeds = seq_embeds + self.protein_sequence_pos_embed[:, :protein_seq.shape[1]]
597
+
598
+ seq_token_type = torch.full(
599
+ size=(seq_embeds.shape[0], 1), fill_value=1, device=seq_embeds.device
600
+ ).long()
601
+
602
+ seq_embeds = seq_embeds + self.token_type_embeddings(seq_token_type)
603
+
604
+ cell_img_conv = self.cond_conv(cell_img)
605
+ concat_img = torch.cat([protein_img, cell_img_conv], dim=1)
606
+ img_embeds = self.img_embedding(concat_img)
607
+
608
+ # add pos embed w/o time step token
609
+ img_embeds = img_embeds + self.img_encoder_pos_embed[:, 1:, :]
610
+
611
+ # masking: length -> length * mask_ratio
612
+ img_embeds, img_mask, img_ids_restore = self.random_masking(img_embeds, img_mask_ratio)
613
+
614
+ # append time step token
615
+ time_embeds_with_pos_embed = time_embeds.unsqueeze(1) + self.img_encoder_pos_embed[:, :1, :]
616
+ img_embeds = torch.cat([time_embeds_with_pos_embed, img_embeds], dim=1)
617
+
618
+ img_token_type = torch.full(
619
+ size=(img_embeds.shape[0], 1), fill_value=0, device=img_embeds.device,
620
+ ).long()
621
+
622
+ img_embeds = img_embeds + self.token_type_embeddings(img_token_type)
623
+ co_embeds = torch.cat([img_embeds, seq_embeds], dim=1)
624
+
625
+ x = co_embeds
626
+
627
+ for i, block in enumerate(self.unified_encoder):
628
+ x = block(x)
629
+
630
+ co_feat = x
631
+
632
+ img_output = self.img_generator(
633
+ hidden_states=concat_img, timestep=time, encoder_hidden_states=co_feat,
634
+ pooled_projections=time_embeds,
635
+ ).sample
636
+
637
+ return img_output
cell_fm/models/cell_fm/modules/__init__.py ADDED
@@ -0,0 +1,82 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ import torch.nn as nn
3
+ import math
4
+
5
+ class TimestepEmbedder(nn.Module):
6
+ """
7
+ Embeds scalar timesteps into vector representations.
8
+ """
9
+ def __init__(self, hidden_size, frequency_embedding_size=256):
10
+ super().__init__()
11
+ self.mlp = nn.Sequential(
12
+ nn.Linear(frequency_embedding_size, hidden_size, bias=True),
13
+ nn.SiLU(),
14
+ nn.Linear(hidden_size, hidden_size, bias=True),
15
+ )
16
+ self.frequency_embedding_size = frequency_embedding_size
17
+
18
+ @staticmethod
19
+ def timestep_embedding(t, dim, max_period=10000):
20
+ """
21
+ Create sinusoidal timestep embeddings.
22
+ :param t: a 1-D Tensor of N indices, one per batch element.
23
+ These may be fractional.
24
+ :param dim: the dimension of the output.
25
+ :param max_period: controls the minimum frequency of the embeddings.
26
+ :return: an (N, D) Tensor of positional embeddings.
27
+ """
28
+ # https://github.com/openai/glide-text2im/blob/main/glide_text2im/nn.py
29
+ half = dim // 2
30
+ freqs = torch.exp(
31
+ -math.log(max_period) * torch.arange(start=0, end=half, dtype=torch.float32) / half
32
+ ).to(device=t.device)
33
+ args = t[:, None].float() * freqs[None]
34
+ embedding = torch.cat([torch.cos(args), torch.sin(args)], dim=-1)
35
+ if dim % 2:
36
+ embedding = torch.cat([embedding, torch.zeros_like(embedding[:, :1])], dim=-1)
37
+ return embedding
38
+
39
+ def forward(self, t):
40
+ t_freq = self.timestep_embedding(t, self.frequency_embedding_size)
41
+ t_emb = self.mlp(t_freq)
42
+ return t_emb
43
+
44
+
45
+ class CondConvNet(nn.Module):
46
+ def __init__(self, in_channels, cond_out_channels):
47
+ """
48
+ Parameters:
49
+ - in_channels (int): Number of channels per input image.
50
+ - cond_out_channels (list of int): Output channels for each conv layer.
51
+ """
52
+ super(CondConvNet, self).__init__()
53
+ layers = []
54
+
55
+ # First convolution layer
56
+ layers.append(
57
+ nn.Conv2d(
58
+ in_channels,
59
+ cond_out_channels[0],
60
+ kernel_size=2,
61
+ stride=2,
62
+ padding=0
63
+ )
64
+ )
65
+
66
+ # Intermediate layers
67
+ for i in range(len(cond_out_channels) - 1):
68
+ layers.append(nn.SiLU())
69
+ layers.append(
70
+ nn.Conv2d(
71
+ cond_out_channels[i],
72
+ cond_out_channels[i + 1],
73
+ kernel_size=2,
74
+ stride=2,
75
+ padding=0
76
+ )
77
+ )
78
+
79
+ self.model = nn.Sequential(*layers)
80
+
81
+ def forward(self, x):
82
+ return self.model(x)
cell_fm/models/cell_fm/modules/positional_embedding.py ADDED
@@ -0,0 +1,59 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import numpy as np
2
+
3
+ #################################################################################
4
+ # Sine/Cosine Positional Embedding Functions #
5
+ #################################################################################
6
+ # https://github.com/facebookresearch/mae/blob/main/util/pos_embed.py
7
+
8
+
9
+ def get_2d_sincos_pos_embed(embed_dim, grid_size, cls_token=False, extra_tokens=0):
10
+ """
11
+ grid_size: int of the grid height and width
12
+ return:
13
+ pos_embed: [grid_size*grid_size, embed_dim] or [1+grid_size*grid_size, embed_dim] (w/ or w/o cls_token)
14
+ """
15
+ grid_h = np.arange(grid_size, dtype=np.float32)
16
+ grid_w = np.arange(grid_size, dtype=np.float32)
17
+ grid = np.meshgrid(grid_w, grid_h) # here w goes first
18
+ grid = np.stack(grid, axis=0)
19
+
20
+ grid = grid.reshape([2, 1, grid_size, grid_size])
21
+ pos_embed = get_2d_sincos_pos_embed_from_grid(embed_dim, grid)
22
+ if cls_token and extra_tokens > 0:
23
+ pos_embed = np.concatenate([np.zeros([extra_tokens, embed_dim]), pos_embed], axis=0)
24
+ return pos_embed
25
+
26
+ def get_1d_sincos_pos_embed(embed_dim, grid_size):
27
+ return get_1d_sincos_pos_embed_from_grid(embed_dim, pos=np.arange(grid_size, dtype=np.float32))
28
+
29
+
30
+ def get_2d_sincos_pos_embed_from_grid(embed_dim, grid):
31
+ assert embed_dim % 2 == 0
32
+
33
+ # use half of dimensions to encode grid_h
34
+ emb_h = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[0]) # (H*W, D/2)
35
+ emb_w = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[1]) # (H*W, D/2)
36
+
37
+ emb = np.concatenate([emb_h, emb_w], axis=1) # (H*W, D)
38
+ return emb
39
+
40
+
41
+ def get_1d_sincos_pos_embed_from_grid(embed_dim, pos):
42
+ """
43
+ embed_dim: output dimension for each position
44
+ pos: a list of positions to be encoded: size (M,)
45
+ out: (M, D)
46
+ """
47
+ assert embed_dim % 2 == 0
48
+ omega = np.arange(embed_dim // 2, dtype=np.float64)
49
+ omega /= embed_dim / 2.
50
+ omega = 1. / 10000**omega # (D/2,)
51
+
52
+ pos = pos.reshape(-1) # (M,)
53
+ out = np.einsum('m,d->md', pos, omega) # (M, D/2), outer product
54
+
55
+ emb_sin = np.sin(out) # (M, D/2)
56
+ emb_cos = np.cos(out) # (M, D/2)
57
+
58
+ emb = np.concatenate([emb_sin, emb_cos], axis=1) # (M, D)
59
+ return emb
cell_fm/models/cell_fm/modules/protein_sequence_embedding.py ADDED
@@ -0,0 +1,39 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ import torch.nn as nn
3
+ from esm.models.esmc import ESMC
4
+
5
+ class ESMEmbed(nn.Module):
6
+ def __init__(self, model_name, out_dims, fixed_embedding=True):
7
+ super(ESMEmbed, self).__init__()
8
+
9
+ if model_name == 'esmc_300m':
10
+ self.model = ESMC.from_pretrained("esmc_300m")
11
+ model_dim = 960
12
+ elif model_name == 'esmc_600m':
13
+ self.model = ESMC.from_pretrained("esmc_600m")
14
+ model_dim = 1152
15
+
16
+ if out_dims != model_dim:
17
+ self.scale_layer = nn.Linear(model_dim, out_dims)
18
+ else:
19
+ self.scale_layer = nn.Identity()
20
+
21
+ # Determine whether to freeze the model's parameters
22
+ self.fixed_embedding = fixed_embedding
23
+ if self.fixed_embedding:
24
+ for param in self.model.parameters():
25
+ param.requires_grad = False
26
+ self.model = self.model.eval()
27
+
28
+ def forward(self, x, **kwargs):
29
+ # If the model's parameters are fixed, use torch.no_grad()
30
+ if self.fixed_embedding:
31
+ with torch.no_grad():
32
+ x = self.model(x).embeddings
33
+ else:
34
+ x = self.model(x).embeddings
35
+
36
+ x = x.to(torch.float32)
37
+ x = self.scale_layer(x)
38
+
39
+ return x
cell_fm/models/cell_fm/modules/transformer.py ADDED
@@ -0,0 +1,131 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ import torch.nn as nn
3
+ from diffusers.models.attention_processor import Attention
4
+ from diffusers.models.attention import FeedForward
5
+ from transformers.activations import ACT2FN
6
+
7
+ class BertPredictionHeadTransform(nn.Module):
8
+ def __init__(
9
+ self,
10
+ hidden_size,
11
+ hidden_act: str = 'gelu',
12
+ norm_eps: float = 1e-5,
13
+ ):
14
+ super().__init__()
15
+ self.dense = nn.Linear(hidden_size, hidden_size)
16
+ self.transform_act_fn = ACT2FN[hidden_act]
17
+ self.LayerNorm = nn.LayerNorm(hidden_size, norm_eps)
18
+
19
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
20
+ hidden_states = self.dense(hidden_states)
21
+ hidden_states = self.transform_act_fn(hidden_states)
22
+ hidden_states = self.LayerNorm(hidden_states)
23
+ return hidden_states
24
+
25
+ class MLMHead(nn.Module):
26
+ def __init__(
27
+ self,
28
+ hidden_size,
29
+ vocab_size,
30
+ hidden_act: str = 'gelu',
31
+ norm_eps: float = 1e-5,
32
+ weight=None
33
+ ):
34
+ super().__init__()
35
+ self.transform = BertPredictionHeadTransform(hidden_size, hidden_act, norm_eps)
36
+ self.decoder = nn.Linear(hidden_size, vocab_size, bias=False)
37
+ self.bias = nn.Parameter(torch.zeros(vocab_size))
38
+ if weight is not None:
39
+ self.decoder.weight = weight
40
+
41
+ def forward(self, x):
42
+ x = self.transform(x)
43
+ x = self.decoder(x) + self.bias
44
+ return x
45
+
46
+ class TransformerBlock(nn.Module):
47
+ def __init__(
48
+ self,
49
+ hidden_size,
50
+ num_heads,
51
+ dim_head,
52
+ dropout: float = 0.0,
53
+ final_dropout: float = 0.0,
54
+ norm_elementwise_affine: bool = True,
55
+ norm_eps: float = 1e-5,
56
+ attention_bias: bool = False,
57
+ attention_out_bias: bool = True,
58
+ activation_fn: str = "geglu",
59
+ ff_inner_dim = None,
60
+ ff_bias: bool = True,
61
+ ):
62
+ super().__init__()
63
+ self.norm1 = nn.LayerNorm(hidden_size, elementwise_affine=norm_elementwise_affine, eps=norm_eps)
64
+
65
+ self.attn = Attention(
66
+ query_dim=hidden_size,
67
+ heads=num_heads,
68
+ dim_head=dim_head,
69
+ dropout=dropout,
70
+ bias=attention_bias,
71
+ cross_attention_dim=None,
72
+ upcast_attention=False,
73
+ out_bias=attention_out_bias,
74
+ )
75
+
76
+ self.ff = FeedForward(
77
+ hidden_size,
78
+ dropout=dropout,
79
+ activation_fn=activation_fn,
80
+ final_dropout=final_dropout,
81
+ inner_dim=ff_inner_dim,
82
+ bias=ff_bias,
83
+ )
84
+
85
+ self.norm2 = nn.LayerNorm(hidden_size, norm_eps, norm_elementwise_affine)
86
+
87
+ def forward(self, hidden_states):
88
+ norm_hidden_states = self.norm1(hidden_states)
89
+ attn_output = self.attn(norm_hidden_states)
90
+
91
+ hidden_states = attn_output + hidden_states
92
+ norm_hidden_states = self.norm2(hidden_states)
93
+
94
+ ff_output = self.ff(norm_hidden_states)
95
+ hidden_states = ff_output + hidden_states
96
+
97
+ return hidden_states
98
+
99
+ def unpatchify(x, out_channels, patch_size):
100
+ """
101
+ x: (N, T, patch_size**2 * C)
102
+ imgs: (N, C, H, W)
103
+ """
104
+ c = out_channels
105
+ p = patch_size
106
+ h = w = int(x.shape[1] ** 0.5)
107
+ assert h * w == x.shape[1]
108
+
109
+ x = x.reshape(shape=(x.shape[0], h, w, p, p, c))
110
+ x = torch.einsum('nhwpqc->nchpwq', x)
111
+ imgs = x.reshape(shape=(x.shape[0], c, h * p, h * p))
112
+ return imgs
113
+
114
+ class FinalLayer(nn.Module):
115
+ """
116
+ The final layer of DiT.
117
+ """
118
+ def __init__(self, hidden_size, patch_size, out_channels):
119
+ super().__init__()
120
+ self.norm_final = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6)
121
+ self.linear = nn.Linear(hidden_size, patch_size * patch_size * out_channels, bias=True)
122
+ self.adaLN_modulation = nn.Sequential(
123
+ nn.SiLU(),
124
+ nn.Linear(hidden_size, 2 * hidden_size, bias=True)
125
+ )
126
+
127
+ def forward(self, x, c):
128
+ shift, scale = self.adaLN_modulation(c).chunk(2, dim=1)
129
+ x = self.linear(x)
130
+ return x
131
+
cell_fm/models/cell_fm/modules/transport/__init__.py ADDED
@@ -0,0 +1,84 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from .transport import Transport, ModelType, WeightType, PathType, TimestepSampler, Sampler
2
+
3
+ def create_transport(
4
+ path_type='Linear',
5
+ prediction="velocity",
6
+ loss_weight=None,
7
+ train_eps=None,
8
+ sample_eps=None,
9
+ timestep_sampler='uniform',
10
+ logit_mean=0.0,
11
+ logit_std=1.0,
12
+ ):
13
+ """function for creating Transport object
14
+ **Note**: model prediction defaults to velocity
15
+ Args:
16
+ - path_type: type of path to use; default to linear
17
+ - learn_score: set model prediction to score
18
+ - learn_noise: set model prediction to noise
19
+ - velocity_weighted: weight loss by velocity weight
20
+ - likelihood_weighted: weight loss by likelihood weight
21
+ - train_eps: small epsilon for avoiding instability during training
22
+ - sample_eps: small epsilon for avoiding instability during sampling
23
+ - timestep_sampler: how to draw the training timestep; 'uniform' or 'logit_normal' (SD3)
24
+ - logit_mean: mean of the logit-normal draw; SD3 convention, positive favours the noise end
25
+ - logit_std: std of the logit-normal draw
26
+ """
27
+
28
+ if prediction == "noise":
29
+ model_type = ModelType.NOISE
30
+ elif prediction == "score":
31
+ model_type = ModelType.SCORE
32
+ else:
33
+ model_type = ModelType.VELOCITY
34
+
35
+ if loss_weight == "velocity":
36
+ loss_type = WeightType.VELOCITY
37
+ elif loss_weight == "likelihood":
38
+ loss_type = WeightType.LIKELIHOOD
39
+ else:
40
+ loss_type = WeightType.NONE
41
+
42
+ sampler_choice = {
43
+ "uniform": TimestepSampler.UNIFORM,
44
+ "logit_normal": TimestepSampler.LOGIT_NORMAL,
45
+ }
46
+
47
+ if timestep_sampler not in sampler_choice:
48
+ raise ValueError(
49
+ f"Unknown timestep_sampler '{timestep_sampler}'; expected one of {sorted(sampler_choice)}"
50
+ )
51
+
52
+ timestep_sampler = sampler_choice[timestep_sampler]
53
+
54
+ path_choice = {
55
+ "Linear": PathType.LINEAR,
56
+ "GVP": PathType.GVP,
57
+ "VP": PathType.VP,
58
+ }
59
+
60
+ path_type = path_choice[path_type]
61
+
62
+ if (path_type in [PathType.VP]):
63
+ train_eps = 1e-5 if train_eps is None else train_eps
64
+ sample_eps = 1e-3 if train_eps is None else sample_eps
65
+ elif (path_type in [PathType.GVP, PathType.LINEAR] and model_type != ModelType.VELOCITY):
66
+ train_eps = 1e-3 if train_eps is None else train_eps
67
+ sample_eps = 1e-3 if train_eps is None else sample_eps
68
+ else: # velocity & [GVP, LINEAR] is stable everywhere
69
+ train_eps = 0
70
+ sample_eps = 0
71
+
72
+ # create flow state
73
+ state = Transport(
74
+ model_type=model_type,
75
+ path_type=path_type,
76
+ loss_type=loss_type,
77
+ train_eps=train_eps,
78
+ sample_eps=sample_eps,
79
+ timestep_sampler=timestep_sampler,
80
+ logit_mean=logit_mean,
81
+ logit_std=logit_std,
82
+ )
83
+
84
+ return state
cell_fm/models/cell_fm/modules/transport/integrators.py ADDED
@@ -0,0 +1,117 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import numpy as np
2
+ import torch as th
3
+ import torch.nn as nn
4
+ from torchdiffeq import odeint
5
+ from functools import partial
6
+ from tqdm import tqdm
7
+
8
+ class sde:
9
+ """SDE solver class"""
10
+ def __init__(
11
+ self,
12
+ drift,
13
+ diffusion,
14
+ *,
15
+ t0,
16
+ t1,
17
+ num_steps,
18
+ sampler_type,
19
+ ):
20
+ assert t0 < t1, "SDE sampler has to be in forward time"
21
+
22
+ self.num_timesteps = num_steps
23
+ self.t = th.linspace(t0, t1, num_steps)
24
+ self.dt = self.t[1] - self.t[0]
25
+ self.drift = drift
26
+ self.diffusion = diffusion
27
+ self.sampler_type = sampler_type
28
+
29
+ def __Euler_Maruyama_step(self, x, mean_x, t, model, **model_kwargs):
30
+ w_cur = th.randn(x.size()).to(x)
31
+ t = th.ones(x.size(0)).to(x) * t
32
+ dw = w_cur * th.sqrt(self.dt)
33
+ drift = self.drift(x, t, model, **model_kwargs)
34
+ diffusion = self.diffusion(x, t)
35
+ mean_x = x + drift * self.dt
36
+ x = mean_x + th.sqrt(2 * diffusion) * dw
37
+ return x, mean_x
38
+
39
+ def __Heun_step(self, x, _, t, model, **model_kwargs):
40
+ w_cur = th.randn(x.size()).to(x)
41
+ dw = w_cur * th.sqrt(self.dt)
42
+ t_cur = th.ones(x.size(0)).to(x) * t
43
+ diffusion = self.diffusion(x, t_cur)
44
+ xhat = x + th.sqrt(2 * diffusion) * dw
45
+ K1 = self.drift(xhat, t_cur, model, **model_kwargs)
46
+ xp = xhat + self.dt * K1
47
+ K2 = self.drift(xp, t_cur + self.dt, model, **model_kwargs)
48
+ return xhat + 0.5 * self.dt * (K1 + K2), xhat # at last time point we do not perform the heun step
49
+
50
+ def __forward_fn(self):
51
+ """TODO: generalize here by adding all private functions ending with steps to it"""
52
+ sampler_dict = {
53
+ "Euler": self.__Euler_Maruyama_step,
54
+ "Heun": self.__Heun_step,
55
+ }
56
+
57
+ try:
58
+ sampler = sampler_dict[self.sampler_type]
59
+ except:
60
+ raise NotImplementedError("Smapler type not implemented.")
61
+
62
+ return sampler
63
+
64
+ def sample(self, init, model, **model_kwargs):
65
+ """forward loop of sde"""
66
+ x = init
67
+ mean_x = init
68
+ samples = []
69
+ sampler = self.__forward_fn()
70
+ for ti in self.t[:-1]:
71
+ with th.no_grad():
72
+ x, mean_x = sampler(x, mean_x, ti, model, **model_kwargs)
73
+ samples.append(x)
74
+
75
+ return samples
76
+
77
+ class ode:
78
+ """ODE solver class"""
79
+ def __init__(
80
+ self,
81
+ drift,
82
+ *,
83
+ t0,
84
+ t1,
85
+ sampler_type,
86
+ num_steps,
87
+ atol,
88
+ rtol,
89
+ ):
90
+ assert t0 < t1, "ODE sampler has to be in forward time"
91
+
92
+ self.drift = drift
93
+ self.t = th.linspace(t0, t1, num_steps)
94
+ self.atol = atol
95
+ self.rtol = rtol
96
+ self.sampler_type = sampler_type
97
+
98
+ def sample(self, x, model, **model_kwargs):
99
+
100
+ device = x[0].device if isinstance(x, tuple) else x.device
101
+ def _fn(t, x):
102
+ t = th.ones(x[0].size(0)).to(device) * t if isinstance(x, tuple) else th.ones(x.size(0)).to(device) * t
103
+ model_output = self.drift(x, t, model, **model_kwargs)
104
+ return model_output
105
+
106
+ t = self.t.to(device)
107
+ atol = [self.atol] * len(x) if isinstance(x, tuple) else [self.atol]
108
+ rtol = [self.rtol] * len(x) if isinstance(x, tuple) else [self.rtol]
109
+ samples = odeint(
110
+ _fn,
111
+ x,
112
+ t,
113
+ method=self.sampler_type,
114
+ atol=atol,
115
+ rtol=rtol
116
+ )
117
+ return samples
cell_fm/models/cell_fm/modules/transport/path.py ADDED
@@ -0,0 +1,192 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch as th
2
+ import numpy as np
3
+ from functools import partial
4
+
5
+ def expand_t_like_x(t, x):
6
+ """Function to reshape time t to broadcastable dimension of x
7
+ Args:
8
+ t: [batch_dim,], time vector
9
+ x: [batch_dim,...], data point
10
+ """
11
+ dims = [1] * (len(x.size()) - 1)
12
+ t = t.view(t.size(0), *dims)
13
+ return t
14
+
15
+
16
+ #################### Coupling Plans ####################
17
+
18
+ class ICPlan:
19
+ """Linear Coupling Plan"""
20
+ def __init__(self, sigma=0.0):
21
+ self.sigma = sigma
22
+
23
+ def compute_alpha_t(self, t):
24
+ """Compute the data coefficient along the path"""
25
+ return t, 1
26
+
27
+ def compute_sigma_t(self, t):
28
+ """Compute the noise coefficient along the path"""
29
+ return 1 - t, -1
30
+
31
+ def compute_d_alpha_alpha_ratio_t(self, t):
32
+ """Compute the ratio between d_alpha and alpha"""
33
+ return 1 / t
34
+
35
+ def compute_drift(self, x, t):
36
+ """We always output sde according to score parametrization; """
37
+ t = expand_t_like_x(t, x)
38
+ alpha_ratio = self.compute_d_alpha_alpha_ratio_t(t)
39
+ sigma_t, d_sigma_t = self.compute_sigma_t(t)
40
+ drift = alpha_ratio * x
41
+ diffusion = alpha_ratio * (sigma_t ** 2) - sigma_t * d_sigma_t
42
+
43
+ return -drift, diffusion
44
+
45
+ def compute_diffusion(self, x, t, form="constant", norm=1.0):
46
+ """Compute the diffusion term of the SDE
47
+ Args:
48
+ x: [batch_dim, ...], data point
49
+ t: [batch_dim,], time vector
50
+ form: str, form of the diffusion term
51
+ norm: float, norm of the diffusion term
52
+ """
53
+ t = expand_t_like_x(t, x)
54
+ choices = {
55
+ "constant": norm,
56
+ "SBDM": norm * self.compute_drift(x, t)[1],
57
+ "sigma": norm * self.compute_sigma_t(t)[0],
58
+ "linear": norm * (1 - t),
59
+ "decreasing": 0.25 * (norm * th.cos(np.pi * t) + 1) ** 2,
60
+ "inccreasing-decreasing": norm * th.sin(np.pi * t) ** 2,
61
+ }
62
+
63
+ try:
64
+ diffusion = choices[form]
65
+ except KeyError:
66
+ raise NotImplementedError(f"Diffusion form {form} not implemented")
67
+
68
+ return diffusion
69
+
70
+ def get_score_from_velocity(self, velocity, x, t):
71
+ """Wrapper function: transfrom velocity prediction model to score
72
+ Args:
73
+ velocity: [batch_dim, ...] shaped tensor; velocity model output
74
+ x: [batch_dim, ...] shaped tensor; x_t data point
75
+ t: [batch_dim,] time tensor
76
+ """
77
+ t = expand_t_like_x(t, x)
78
+ alpha_t, d_alpha_t = self.compute_alpha_t(t)
79
+ sigma_t, d_sigma_t = self.compute_sigma_t(t)
80
+ mean = x
81
+ reverse_alpha_ratio = alpha_t / d_alpha_t
82
+ var = sigma_t**2 - reverse_alpha_ratio * d_sigma_t * sigma_t
83
+ score = (reverse_alpha_ratio * velocity - mean) / var
84
+ return score
85
+
86
+ def get_noise_from_velocity(self, velocity, x, t):
87
+ """Wrapper function: transfrom velocity prediction model to denoiser
88
+ Args:
89
+ velocity: [batch_dim, ...] shaped tensor; velocity model output
90
+ x: [batch_dim, ...] shaped tensor; x_t data point
91
+ t: [batch_dim,] time tensor
92
+ """
93
+ t = expand_t_like_x(t, x)
94
+ alpha_t, d_alpha_t = self.compute_alpha_t(t)
95
+ sigma_t, d_sigma_t = self.compute_sigma_t(t)
96
+ mean = x
97
+ reverse_alpha_ratio = alpha_t / d_alpha_t
98
+ var = reverse_alpha_ratio * d_sigma_t - sigma_t
99
+ noise = (reverse_alpha_ratio * velocity - mean) / var
100
+ return noise
101
+
102
+ def get_velocity_from_score(self, score, x, t):
103
+ """Wrapper function: transfrom score prediction model to velocity
104
+ Args:
105
+ score: [batch_dim, ...] shaped tensor; score model output
106
+ x: [batch_dim, ...] shaped tensor; x_t data point
107
+ t: [batch_dim,] time tensor
108
+ """
109
+ t = expand_t_like_x(t, x)
110
+ drift, var = self.compute_drift(x, t)
111
+ velocity = var * score - drift
112
+ return velocity
113
+
114
+ def compute_mu_t(self, t, x0, x1):
115
+ """Compute the mean of time-dependent density p_t"""
116
+ t = expand_t_like_x(t, x1)
117
+ alpha_t, _ = self.compute_alpha_t(t)
118
+ sigma_t, _ = self.compute_sigma_t(t)
119
+ return alpha_t * x1 + sigma_t * x0
120
+
121
+ def compute_xt(self, t, x0, x1):
122
+ """Sample xt from time-dependent density p_t; rng is required"""
123
+ xt = self.compute_mu_t(t, x0, x1)
124
+ return xt
125
+
126
+ def compute_ut(self, t, x0, x1, xt):
127
+ """Compute the vector field corresponding to p_t"""
128
+ t = expand_t_like_x(t, x1)
129
+ _, d_alpha_t = self.compute_alpha_t(t)
130
+ _, d_sigma_t = self.compute_sigma_t(t)
131
+ return d_alpha_t * x1 + d_sigma_t * x0
132
+
133
+ def plan(self, t, x0, x1):
134
+ xt = self.compute_xt(t, x0, x1)
135
+ ut = self.compute_ut(t, x0, x1, xt)
136
+ return t, xt, ut
137
+
138
+
139
+ class VPCPlan(ICPlan):
140
+ """class for VP path flow matching"""
141
+
142
+ def __init__(self, sigma_min=0.1, sigma_max=20.0):
143
+ self.sigma_min = sigma_min
144
+ self.sigma_max = sigma_max
145
+ self.log_mean_coeff = lambda t: -0.25 * ((1 - t) ** 2) * (self.sigma_max - self.sigma_min) - 0.5 * (1 - t) * self.sigma_min
146
+ self.d_log_mean_coeff = lambda t: 0.5 * (1 - t) * (self.sigma_max - self.sigma_min) + 0.5 * self.sigma_min
147
+
148
+
149
+ def compute_alpha_t(self, t):
150
+ """Compute coefficient of x1"""
151
+ alpha_t = self.log_mean_coeff(t)
152
+ alpha_t = th.exp(alpha_t)
153
+ d_alpha_t = alpha_t * self.d_log_mean_coeff(t)
154
+ return alpha_t, d_alpha_t
155
+
156
+ def compute_sigma_t(self, t):
157
+ """Compute coefficient of x0"""
158
+ p_sigma_t = 2 * self.log_mean_coeff(t)
159
+ sigma_t = th.sqrt(1 - th.exp(p_sigma_t))
160
+ d_sigma_t = th.exp(p_sigma_t) * (2 * self.d_log_mean_coeff(t)) / (-2 * sigma_t)
161
+ return sigma_t, d_sigma_t
162
+
163
+ def compute_d_alpha_alpha_ratio_t(self, t):
164
+ """Special purposed function for computing numerical stabled d_alpha_t / alpha_t"""
165
+ return self.d_log_mean_coeff(t)
166
+
167
+ def compute_drift(self, x, t):
168
+ """Compute the drift term of the SDE"""
169
+ t = expand_t_like_x(t, x)
170
+ beta_t = self.sigma_min + (1 - t) * (self.sigma_max - self.sigma_min)
171
+ return -0.5 * beta_t * x, beta_t / 2
172
+
173
+
174
+ class GVPCPlan(ICPlan):
175
+ def __init__(self, sigma=0.0):
176
+ super().__init__(sigma)
177
+
178
+ def compute_alpha_t(self, t):
179
+ """Compute coefficient of x1"""
180
+ alpha_t = th.sin(t * np.pi / 2)
181
+ d_alpha_t = np.pi / 2 * th.cos(t * np.pi / 2)
182
+ return alpha_t, d_alpha_t
183
+
184
+ def compute_sigma_t(self, t):
185
+ """Compute coefficient of x0"""
186
+ sigma_t = th.cos(t * np.pi / 2)
187
+ d_sigma_t = -np.pi / 2 * th.sin(t * np.pi / 2)
188
+ return sigma_t, d_sigma_t
189
+
190
+ def compute_d_alpha_alpha_ratio_t(self, t):
191
+ """Special purposed function for computing numerical stabled d_alpha_t / alpha_t"""
192
+ return np.pi / (2 * th.tan(t * np.pi / 2))
cell_fm/models/cell_fm/modules/transport/transport.py ADDED
@@ -0,0 +1,506 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch as th
2
+ import numpy as np
3
+ import logging
4
+
5
+ import enum
6
+
7
+ from . import path
8
+ from .utils import EasyDict, log_state, mean_flat
9
+ from .integrators import ode, sde
10
+
11
+ class ModelType(enum.Enum):
12
+ """
13
+ Which type of output the model predicts.
14
+ """
15
+
16
+ NOISE = enum.auto() # the model predicts epsilon
17
+ SCORE = enum.auto() # the model predicts \nabla \log p(x)
18
+ VELOCITY = enum.auto() # the model predicts v(x)
19
+
20
+ class PathType(enum.Enum):
21
+ """
22
+ Which type of path to use.
23
+ """
24
+
25
+ LINEAR = enum.auto()
26
+ GVP = enum.auto()
27
+ VP = enum.auto()
28
+
29
+ class WeightType(enum.Enum):
30
+ """
31
+ Which type of weighting to use.
32
+ """
33
+
34
+ NONE = enum.auto()
35
+ VELOCITY = enum.auto()
36
+ LIKELIHOOD = enum.auto()
37
+
38
+ class TimestepSampler(enum.Enum):
39
+ """
40
+ How the training timestep t is drawn.
41
+ """
42
+
43
+ UNIFORM = enum.auto() # t ~ U(t0, t1)
44
+ LOGIT_NORMAL = enum.auto() # SD3: t = sigmoid(-u), u ~ N(logit_mean, logit_std^2)
45
+
46
+
47
+ class Transport:
48
+
49
+ def __init__(
50
+ self,
51
+ *,
52
+ model_type,
53
+ path_type,
54
+ loss_type,
55
+ train_eps,
56
+ sample_eps,
57
+ timestep_sampler=TimestepSampler.UNIFORM,
58
+ logit_mean=0.0,
59
+ logit_std=1.0,
60
+ ):
61
+ path_options = {
62
+ PathType.LINEAR: path.ICPlan,
63
+ PathType.GVP: path.GVPCPlan,
64
+ PathType.VP: path.VPCPlan,
65
+ }
66
+
67
+ self.loss_type = loss_type
68
+ self.model_type = model_type
69
+ self.path_sampler = path_options[path_type]()
70
+ self.train_eps = train_eps
71
+ self.sample_eps = sample_eps
72
+ self.timestep_sampler = timestep_sampler
73
+ self.logit_mean = logit_mean
74
+ self.logit_std = logit_std
75
+
76
+ def prior_logp(self, z):
77
+ '''
78
+ Standard multivariate normal prior
79
+ Assume z is batched
80
+ '''
81
+ shape = th.tensor(z.size())
82
+ N = th.prod(shape[1:])
83
+ _fn = lambda x: -N / 2. * np.log(2 * np.pi) - th.sum(x ** 2) / 2.
84
+ return th.vmap(_fn)(z)
85
+
86
+
87
+ def check_interval(
88
+ self,
89
+ train_eps,
90
+ sample_eps,
91
+ *,
92
+ diffusion_form="SBDM",
93
+ sde=False,
94
+ reverse=False,
95
+ eval=False,
96
+ last_step_size=0.0,
97
+ ):
98
+ t0 = 0
99
+ t1 = 1
100
+ eps = train_eps if not eval else sample_eps
101
+ if (type(self.path_sampler) in [path.VPCPlan]):
102
+
103
+ t1 = 1 - eps if (not sde or last_step_size == 0) else 1 - last_step_size
104
+
105
+ elif (type(self.path_sampler) in [path.ICPlan, path.GVPCPlan]) \
106
+ and (self.model_type != ModelType.VELOCITY or sde): # avoid numerical issue by taking a first semi-implicit step
107
+
108
+ t0 = eps if (diffusion_form == "SBDM" and sde) or self.model_type != ModelType.VELOCITY else 0
109
+ t1 = 1 - eps if (not sde or last_step_size == 0) else 1 - last_step_size
110
+
111
+ if reverse:
112
+ t0, t1 = 1 - t0, 1 - t1
113
+
114
+ return t0, t1
115
+
116
+
117
+ def sample(self, x1):
118
+ """Sampling x0 & t based on shape of x1 (if needed)
119
+ Args:
120
+ x1 - data point; [batch, *dim]
121
+ """
122
+
123
+ x0 = th.randn_like(x1)
124
+ t0, t1 = self.check_interval(self.train_eps, self.sample_eps)
125
+
126
+ if self.timestep_sampler == TimestepSampler.LOGIT_NORMAL:
127
+ # SD3 draws t_sd3 = sigmoid(u) with t_sd3=1 at the noise end. This codebase
128
+ # runs the other way (t=1 is data, see ICPlan.compute_alpha_t), so the
129
+ # equivalent draw is t = 1 - sigmoid(u) = sigmoid(-u). This keeps logit_mean
130
+ # meaning the same thing here as in diffusers.
131
+ u = th.randn((x1.shape[0],)) * self.logit_std + self.logit_mean
132
+ t = th.sigmoid(-u)
133
+ t = t * (t1 - t0) + t0
134
+ else:
135
+ t = th.rand((x1.shape[0],)) * (t1 - t0) + t0
136
+
137
+ t = t.to(x1)
138
+ return t, x0, x1
139
+
140
+
141
+ def training_losses(
142
+ self,
143
+ model_output,
144
+ x0,
145
+ xt,
146
+ ut,
147
+ t,
148
+ ):
149
+ """Loss for training the score model
150
+ Args:
151
+ - model: backbone model; could be score, noise, or velocity
152
+ - x1: datapoint
153
+ - model_kwargs: additional arguments for the model
154
+ """
155
+
156
+ terms = {}
157
+ terms['pred'] = model_output
158
+ if self.model_type == ModelType.VELOCITY:
159
+ terms['loss'] = mean_flat(((model_output - ut) ** 2))
160
+ else:
161
+ _, drift_var = self.path_sampler.compute_drift(xt, t)
162
+ sigma_t, _ = self.path_sampler.compute_sigma_t(path.expand_t_like_x(t, xt))
163
+ if self.loss_type in [WeightType.VELOCITY]:
164
+ weight = (drift_var / sigma_t) ** 2
165
+ elif self.loss_type in [WeightType.LIKELIHOOD]:
166
+ weight = drift_var / (sigma_t ** 2)
167
+ elif self.loss_type in [WeightType.NONE]:
168
+ weight = 1
169
+ else:
170
+ raise NotImplementedError()
171
+
172
+ if self.model_type == ModelType.NOISE:
173
+ terms['loss'] = mean_flat(weight * ((model_output - x0) ** 2))
174
+ else:
175
+ terms['loss'] = mean_flat(weight * ((model_output * sigma_t + x0) ** 2))
176
+
177
+ return terms
178
+
179
+ # def training_losses(
180
+ # self,
181
+ # model,
182
+ # x1,
183
+ # model_kwargs=None
184
+ # ):
185
+ # """Loss for training the score model
186
+ # Args:
187
+ # - model: backbone model; could be score, noise, or velocity
188
+ # - x1: datapoint
189
+ # - model_kwargs: additional arguments for the model
190
+ # """
191
+ # if model_kwargs == None:
192
+ # model_kwargs = {}
193
+
194
+ # t, x0, x1 = self.sample(x1)
195
+ # t, xt, ut = self.path_sampler.plan(t, x0, x1)
196
+ # model_output = model(xt, t, **model_kwargs)
197
+ # B, *_, C = xt.shape
198
+ # assert model_output.size() == (B, *xt.size()[1:-1], C)
199
+
200
+ # terms = {}
201
+ # terms['pred'] = model_output
202
+ # if self.model_type == ModelType.VELOCITY:
203
+ # terms['loss'] = mean_flat(((model_output - ut) ** 2))
204
+ # else:
205
+ # _, drift_var = self.path_sampler.compute_drift(xt, t)
206
+ # sigma_t, _ = self.path_sampler.compute_sigma_t(path.expand_t_like_x(t, xt))
207
+ # if self.loss_type in [WeightType.VELOCITY]:
208
+ # weight = (drift_var / sigma_t) ** 2
209
+ # elif self.loss_type in [WeightType.LIKELIHOOD]:
210
+ # weight = drift_var / (sigma_t ** 2)
211
+ # elif self.loss_type in [WeightType.NONE]:
212
+ # weight = 1
213
+ # else:
214
+ # raise NotImplementedError()
215
+
216
+ # if self.model_type == ModelType.NOISE:
217
+ # terms['loss'] = mean_flat(weight * ((model_output - x0) ** 2))
218
+ # else:
219
+ # terms['loss'] = mean_flat(weight * ((model_output * sigma_t + x0) ** 2))
220
+
221
+ # return terms
222
+
223
+
224
+ def get_drift(
225
+ self
226
+ ):
227
+ """member function for obtaining the drift of the probability flow ODE"""
228
+ def score_ode(x, t, model, **model_kwargs):
229
+ drift_mean, drift_var = self.path_sampler.compute_drift(x, t)
230
+ model_output = model(x, t, **model_kwargs)
231
+ return (-drift_mean + drift_var * model_output) # by change of variable
232
+
233
+ def noise_ode(x, t, model, **model_kwargs):
234
+ drift_mean, drift_var = self.path_sampler.compute_drift(x, t)
235
+ sigma_t, _ = self.path_sampler.compute_sigma_t(path.expand_t_like_x(t, x))
236
+ model_output = model(x, t, **model_kwargs)
237
+ score = model_output / -sigma_t
238
+ return (-drift_mean + drift_var * score)
239
+
240
+ def velocity_ode(x, t, model, **model_kwargs):
241
+ model_output = model(x, t, **model_kwargs)
242
+ return model_output
243
+
244
+ if self.model_type == ModelType.NOISE:
245
+ drift_fn = noise_ode
246
+ elif self.model_type == ModelType.SCORE:
247
+ drift_fn = score_ode
248
+ else:
249
+ drift_fn = velocity_ode
250
+
251
+ def body_fn(x, t, model, **model_kwargs):
252
+ model_output = drift_fn(x, t, model, **model_kwargs)
253
+ assert model_output.shape == x.shape, "Output shape from ODE solver must match input shape"
254
+ return model_output
255
+
256
+ return body_fn
257
+
258
+
259
+ def get_score(
260
+ self,
261
+ ):
262
+ """member function for obtaining score of
263
+ x_t = alpha_t * x + sigma_t * eps"""
264
+ if self.model_type == ModelType.NOISE:
265
+ score_fn = lambda x, t, model, **kwargs: model(x, t, **kwargs) / -self.path_sampler.compute_sigma_t(path.expand_t_like_x(t, x))[0]
266
+ elif self.model_type == ModelType.SCORE:
267
+ score_fn = lambda x, t, model, **kwagrs: model(x, t, **kwagrs)
268
+ elif self.model_type == ModelType.VELOCITY:
269
+ score_fn = lambda x, t, model, **kwargs: self.path_sampler.get_score_from_velocity(model(x, t, **kwargs), x, t)
270
+ else:
271
+ raise NotImplementedError()
272
+
273
+ return score_fn
274
+
275
+
276
+ class Sampler:
277
+ """Sampler class for the transport model"""
278
+ def __init__(
279
+ self,
280
+ transport,
281
+ ):
282
+ """Constructor for a general sampler; supporting different sampling methods
283
+ Args:
284
+ - transport: an tranport object specify model prediction & interpolant type
285
+ """
286
+
287
+ self.transport = transport
288
+ self.drift = self.transport.get_drift()
289
+ self.score = self.transport.get_score()
290
+
291
+ def __get_sde_diffusion_and_drift(
292
+ self,
293
+ *,
294
+ diffusion_form="SBDM",
295
+ diffusion_norm=1.0,
296
+ ):
297
+
298
+ def diffusion_fn(x, t):
299
+ diffusion = self.transport.path_sampler.compute_diffusion(x, t, form=diffusion_form, norm=diffusion_norm)
300
+ return diffusion
301
+
302
+ sde_drift = \
303
+ lambda x, t, model, **kwargs: \
304
+ self.drift(x, t, model, **kwargs) + diffusion_fn(x, t) * self.score(x, t, model, **kwargs)
305
+
306
+ sde_diffusion = diffusion_fn
307
+
308
+ return sde_drift, sde_diffusion
309
+
310
+ def __get_last_step(
311
+ self,
312
+ sde_drift,
313
+ *,
314
+ last_step,
315
+ last_step_size,
316
+ ):
317
+ """Get the last step function of the SDE solver"""
318
+
319
+ if last_step is None:
320
+ last_step_fn = \
321
+ lambda x, t, model, **model_kwargs: \
322
+ x
323
+ elif last_step == "Mean":
324
+ last_step_fn = \
325
+ lambda x, t, model, **model_kwargs: \
326
+ x + sde_drift(x, t, model, **model_kwargs) * last_step_size
327
+ elif last_step == "Tweedie":
328
+ alpha = self.transport.path_sampler.compute_alpha_t # simple aliasing; the original name was too long
329
+ sigma = self.transport.path_sampler.compute_sigma_t
330
+ last_step_fn = \
331
+ lambda x, t, model, **model_kwargs: \
332
+ x / alpha(t)[0][0] + (sigma(t)[0][0] ** 2) / alpha(t)[0][0] * self.score(x, t, model, **model_kwargs)
333
+ elif last_step == "Euler":
334
+ last_step_fn = \
335
+ lambda x, t, model, **model_kwargs: \
336
+ x + self.drift(x, t, model, **model_kwargs) * last_step_size
337
+ else:
338
+ raise NotImplementedError()
339
+
340
+ return last_step_fn
341
+
342
+ def sample_sde(
343
+ self,
344
+ *,
345
+ sampling_method="Euler",
346
+ diffusion_form="SBDM",
347
+ diffusion_norm=1.0,
348
+ last_step="Mean",
349
+ last_step_size=0.04,
350
+ num_steps=250,
351
+ ):
352
+ """returns a sampling function with given SDE settings
353
+ Args:
354
+ - sampling_method: type of sampler used in solving the SDE; default to be Euler-Maruyama
355
+ - diffusion_form: function form of diffusion coefficient; default to be matching SBDM
356
+ - diffusion_norm: function magnitude of diffusion coefficient; default to 1
357
+ - last_step: type of the last step; default to identity
358
+ - last_step_size: size of the last step; default to match the stride of 250 steps over [0,1]
359
+ - num_steps: total integration step of SDE
360
+ """
361
+
362
+ if last_step is None:
363
+ last_step_size = 0.0
364
+
365
+ sde_drift, sde_diffusion = self.__get_sde_diffusion_and_drift(
366
+ diffusion_form=diffusion_form,
367
+ diffusion_norm=diffusion_norm,
368
+ )
369
+
370
+ t0, t1 = self.transport.check_interval(
371
+ self.transport.train_eps,
372
+ self.transport.sample_eps,
373
+ diffusion_form=diffusion_form,
374
+ sde=True,
375
+ eval=True,
376
+ reverse=False,
377
+ last_step_size=last_step_size,
378
+ )
379
+
380
+ _sde = sde(
381
+ sde_drift,
382
+ sde_diffusion,
383
+ t0=t0,
384
+ t1=t1,
385
+ num_steps=num_steps,
386
+ sampler_type=sampling_method
387
+ )
388
+
389
+ last_step_fn = self.__get_last_step(sde_drift, last_step=last_step, last_step_size=last_step_size)
390
+
391
+
392
+ def _sample(init, model, **model_kwargs):
393
+ xs = _sde.sample(init, model, **model_kwargs)
394
+ ts = th.ones(init.size(0), device=init.device) * t1
395
+ x = last_step_fn(xs[-1], ts, model, **model_kwargs)
396
+ xs.append(x)
397
+
398
+ assert len(xs) == num_steps, "Samples does not match the number of steps"
399
+
400
+ return xs
401
+
402
+ return _sample
403
+
404
+ def sample_ode(
405
+ self,
406
+ *,
407
+ sampling_method="dopri5",
408
+ num_steps=50,
409
+ atol=1e-6,
410
+ rtol=1e-3,
411
+ reverse=False,
412
+ ):
413
+ """returns a sampling function with given ODE settings
414
+ Args:
415
+ - sampling_method: type of sampler used in solving the ODE; default to be Dopri5
416
+ - num_steps:
417
+ - fixed solver (Euler, Heun): the actual number of integration steps performed
418
+ - adaptive solver (Dopri5): the number of datapoints saved during integration; produced by interpolation
419
+ - atol: absolute error tolerance for the solver
420
+ - rtol: relative error tolerance for the solver
421
+ - reverse: whether solving the ODE in reverse (data to noise); default to False
422
+ """
423
+ if reverse:
424
+ drift = lambda x, t, model, **kwargs: self.drift(x, th.ones_like(t) * (1 - t), model, **kwargs)
425
+ else:
426
+ drift = self.drift
427
+
428
+ t0, t1 = self.transport.check_interval(
429
+ self.transport.train_eps,
430
+ self.transport.sample_eps,
431
+ sde=False,
432
+ eval=True,
433
+ reverse=reverse,
434
+ last_step_size=0.0,
435
+ )
436
+
437
+ _ode = ode(
438
+ drift=drift,
439
+ t0=t0,
440
+ t1=t1,
441
+ sampler_type=sampling_method,
442
+ num_steps=num_steps,
443
+ atol=atol,
444
+ rtol=rtol,
445
+ )
446
+
447
+ return _ode.sample
448
+
449
+ def sample_ode_likelihood(
450
+ self,
451
+ *,
452
+ sampling_method="dopri5",
453
+ num_steps=50,
454
+ atol=1e-6,
455
+ rtol=1e-3,
456
+ ):
457
+
458
+ """returns a sampling function for calculating likelihood with given ODE settings
459
+ Args:
460
+ - sampling_method: type of sampler used in solving the ODE; default to be Dopri5
461
+ - num_steps:
462
+ - fixed solver (Euler, Heun): the actual number of integration steps performed
463
+ - adaptive solver (Dopri5): the number of datapoints saved during integration; produced by interpolation
464
+ - atol: absolute error tolerance for the solver
465
+ - rtol: relative error tolerance for the solver
466
+ """
467
+ def _likelihood_drift(x, t, model, **model_kwargs):
468
+ x, _ = x
469
+ eps = th.randint(2, x.size(), dtype=th.float, device=x.device) * 2 - 1
470
+ t = th.ones_like(t) * (1 - t)
471
+ with th.enable_grad():
472
+ x.requires_grad = True
473
+ grad = th.autograd.grad(th.sum(self.drift(x, t, model, **model_kwargs) * eps), x)[0]
474
+ logp_grad = th.sum(grad * eps, dim=tuple(range(1, len(x.size()))))
475
+ drift = self.drift(x, t, model, **model_kwargs)
476
+ return (-drift, logp_grad)
477
+
478
+ t0, t1 = self.transport.check_interval(
479
+ self.transport.train_eps,
480
+ self.transport.sample_eps,
481
+ sde=False,
482
+ eval=True,
483
+ reverse=False,
484
+ last_step_size=0.0,
485
+ )
486
+
487
+ _ode = ode(
488
+ drift=_likelihood_drift,
489
+ t0=t0,
490
+ t1=t1,
491
+ sampler_type=sampling_method,
492
+ num_steps=num_steps,
493
+ atol=atol,
494
+ rtol=rtol,
495
+ )
496
+
497
+ def _sample_fn(x, model, **model_kwargs):
498
+ init_logp = th.zeros(x.size(0)).to(x)
499
+ input = (x, init_logp)
500
+ drift, delta_logp = _ode.sample(input, model, **model_kwargs)
501
+ drift, delta_logp = drift[-1], delta_logp[-1]
502
+ prior_logp = self.transport.prior_logp(drift)
503
+ logp = prior_logp - delta_logp
504
+ return logp, drift
505
+
506
+ return _sample_fn
cell_fm/models/cell_fm/modules/transport/utils.py ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch as th
2
+
3
+ class EasyDict:
4
+
5
+ def __init__(self, sub_dict):
6
+ for k, v in sub_dict.items():
7
+ setattr(self, k, v)
8
+
9
+ def __getitem__(self, key):
10
+ return getattr(self, key)
11
+
12
+ def mean_flat(x):
13
+ """
14
+ Take the mean over all non-batch dimensions.
15
+ """
16
+ return th.mean(x, dim=list(range(1, len(x.size()))))
17
+
18
+ def log_state(state):
19
+ result = []
20
+
21
+ sorted_state = dict(sorted(state.items()))
22
+ for key, value in sorted_state.items():
23
+ # Check if the value is an instance of a class
24
+ if "<object" in str(value) or "object at" in str(value):
25
+ result.append(f"{key}: [{value.__class__.__name__}]")
26
+ else:
27
+ result.append(f"{key}: {value}")
28
+
29
+ return '\n'.join(result)
cell_fm/models/vae/__init__.py ADDED
File without changes
cell_fm/models/vae/vae_config.py ADDED
@@ -0,0 +1,60 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # -*- coding: utf-8 -*-
2
+ from dataclasses import dataclass
3
+ from transformers import PretrainedConfig
4
+
5
+ @dataclass
6
+ class VAEConfig(PretrainedConfig):
7
+ model_type: str = 'vae'
8
+
9
+ # Dataset parameters
10
+ data_path: str = ""
11
+ split_key: str = 'train'
12
+
13
+ data_aug: bool = False
14
+ img_resize: int = 1024
15
+ img_crop_size: int = 256
16
+
17
+ # Loss parameters
18
+ recon_loss_coeff: float = 1.0
19
+ kl_loss_coeff: float = 1.0
20
+
21
+ # Model parameters
22
+ in_channels: int = 1
23
+ out_channels: int = 1
24
+ num_down_blocks: int = 3
25
+ latent_channels: int = 4
26
+ vae_block_out_channels: str = '128,256,512'
27
+
28
+ # Training parameters
29
+ vae_loadcheck_path: str = ""
30
+ ft: bool = False
31
+ infer: bool = False
32
+ ifresume: bool = False
33
+
34
+ output_dir: str = ""
35
+ learning_rate: float = 1e-4
36
+ weight_decay: float = 0.0
37
+ gradient_accumulation_steps: int = 1
38
+ per_device_train_batch_size: int = 2
39
+ per_device_eval_batch_size: int = 2
40
+
41
+ num_train_epochs: int = 10
42
+ fp16: bool = False
43
+ bf16: bool = False
44
+ logging_dir: str = ""
45
+ logging_steps: int = 10
46
+ max_steps: int = -1
47
+ warmup_steps: int = 1000
48
+ save_steps: int = 1000
49
+
50
+ dataloader_num_workers: int = 8
51
+ seed: int = 6
52
+
53
+
54
+ def __init__(self, **kwargs):
55
+ # Use `super().__init__` to handle arguments from PretrainedConfig
56
+ super().__init__(**kwargs)
57
+
58
+ self.vae_block_out_channels = kwargs.get("vae_block_out_channels", self.vae_block_out_channels)
59
+ if not isinstance(self.vae_block_out_channels, list):
60
+ self.vae_block_out_channels = [int(a) for a in self.vae_block_out_channels.split(',')]
cell_fm/models/vae/vae_model.py ADDED
@@ -0,0 +1,152 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import torch
3
+ import torch.nn as nn
4
+ from diffusers.models import AutoencoderKL
5
+ from .vae_config import VAEConfig
6
+ from transformers import PreTrainedModel
7
+ from cell_fm.pipeline.utils import VAEOutput
8
+ from cell_fm.logging import logger
9
+
10
+
11
+ class VAEModel(PreTrainedModel):
12
+ config_class = VAEConfig
13
+
14
+ def __init__(self, config):
15
+ super().__init__(config)
16
+ self.config = config
17
+
18
+ # Calculate number of downsampling and upsampling layers required
19
+ self.num_down_blocks = config.num_down_blocks # Input size / latent space size
20
+ self.num_up_blocks = self.num_down_blocks
21
+
22
+ # Generate down_block_types and up_block_types based on the required layers
23
+ down_block_types = tuple(["DownEncoderBlock2D"] * self.num_down_blocks)
24
+ up_block_types = tuple(["UpDecoderBlock2D"] * self.num_up_blocks)
25
+
26
+ # Initialize AutoencoderKL with custom parameters from config
27
+ self.vae = AutoencoderKL(
28
+ in_channels=config.in_channels,
29
+ out_channels=config.out_channels,
30
+ down_block_types=down_block_types,
31
+ up_block_types=up_block_types,
32
+ block_out_channels=config.vae_block_out_channels,
33
+ latent_channels=config.latent_channels, # Latent space dimensions
34
+ )
35
+
36
+ self.load_pretrained_weights(config, checkpoint_path=config.vae_loadcheck_path)
37
+
38
+ def load_pretrained_weights(self, config, checkpoint_path):
39
+ """
40
+ Load pretrained weights from a given state_dict.
41
+ """
42
+ if config.ft or config.infer:
43
+ if config.ft:
44
+ logger.info(f"Finetune from checkpoint: {checkpoint_path}")
45
+ else:
46
+ logger.info(f"Infer from checkpoint: {checkpoint_path}")
47
+
48
+ if os.path.splitext(checkpoint_path)[1] == '.safetensors':
49
+ from safetensors.torch import load_file
50
+ checkpoints_state = load_file(checkpoint_path)
51
+ else:
52
+ checkpoints_state = torch.load(checkpoint_path, map_location="cpu")
53
+
54
+ if "model" in checkpoints_state:
55
+ checkpoints_state = checkpoints_state["model"]
56
+ elif "module" in checkpoints_state:
57
+ checkpoints_state = checkpoints_state["module"]
58
+
59
+ model_state_dict = self.state_dict()
60
+ filtered_state_dict = {k: v for k, v in checkpoints_state.items() if k in model_state_dict and v.size() == model_state_dict[k].size()}
61
+
62
+ IncompatibleKeys = self.load_state_dict(filtered_state_dict, strict=False)
63
+ # IncompatibleKeys = self.load_state_dict(checkpoints_state, strict=False)
64
+ IncompatibleKeys = IncompatibleKeys._asdict()
65
+
66
+ missing_keys = []
67
+ for keys in IncompatibleKeys["missing_keys"]:
68
+ if keys.find("dummy") == -1:
69
+ missing_keys.append(keys)
70
+
71
+ unexpected_keys = []
72
+ for keys in IncompatibleKeys["unexpected_keys"]:
73
+ if keys.find("dummy") == -1:
74
+ unexpected_keys.append(keys)
75
+
76
+ if len(missing_keys) > 0:
77
+ logger.info(
78
+ "Missing keys in {}: {}".format(
79
+ checkpoint_path,
80
+ missing_keys,
81
+ )
82
+ )
83
+
84
+ if len(unexpected_keys) > 0:
85
+ logger.info(
86
+ "Unexpected keys {}: {}".format(
87
+ checkpoint_path,
88
+ unexpected_keys,
89
+ )
90
+ )
91
+
92
+ def encode(self, x):
93
+ """Encodes input into latent space."""
94
+ return self.vae.encode(x).latent_dist
95
+
96
+ def decode(self, latents):
97
+ """Decodes latent space into reconstructed input."""
98
+ return self.vae.decode(latents)
99
+
100
+ def forward(self, batched_data):
101
+ x = batched_data['protein_img']
102
+
103
+ """Forward pass through the VAE."""
104
+
105
+ latent_dist = self.encode(x)
106
+ latents = latent_dist.sample() # Reparameterization trick
107
+ recon_x = self.decode(latents).sample
108
+ total_loss, recon_loss, kl_loss = self.compute_loss(x, recon_x, latent_dist)
109
+
110
+ # Log the losses
111
+ log_loss = {
112
+ "total_loss": total_loss.item(),
113
+ "recon_loss": recon_loss.item(),
114
+ "kl_loss": kl_loss.item(),
115
+ }
116
+
117
+ return VAEOutput(total_loss, log_loss)
118
+
119
+ def compute_loss(self, x, recon_x, latent_dist):
120
+ """Compute reconstruction and KL divergence loss."""
121
+ recon_loss = nn.MSELoss()(recon_x, x)
122
+ kl_loss = -0.5 * torch.mean(1 + latent_dist.logvar - latent_dist.mean.pow(2) - latent_dist.logvar.exp())
123
+ total_loss = self.config.recon_loss_coeff * recon_loss + self.config.kl_loss_coeff * kl_loss
124
+ return total_loss, recon_loss, kl_loss
125
+
126
+ def sample(self, num_samples=1, latent_size=64, device="cpu"):
127
+ """
128
+ Generate samples from the latent space.
129
+
130
+ Args:
131
+ num_samples (int): Number of samples to generate.
132
+ latent_size (int): Size of the latent space.
133
+ device (str): Device to perform sampling on.
134
+
135
+ Returns:
136
+ torch.Tensor: Generated images.
137
+ """
138
+ # Sample from a standard normal distribution in latent space
139
+ latents = torch.randn((num_samples, self.config.latent_channels, latent_size, latent_size), device=device) # Shape matches latent dimensions
140
+
141
+ # Decode latents to generate images
142
+ with torch.no_grad():
143
+ generated_images = self.decode(latents).sample
144
+
145
+ return generated_images
146
+
147
+ def reconstruct(self, x):
148
+ latent_dist = self.encode(x)
149
+ latents = latent_dist.sample() # Reparameterization trick
150
+ recon_x = self.decode(latents).sample
151
+
152
+ return recon_x
cell_fm/models/vit_cls_condenseq_img/__init__.py ADDED
File without changes
cell_fm/models/vit_cls_condenseq_img/config.py ADDED
@@ -0,0 +1,46 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # -*- coding: utf-8 -*-
2
+ from dataclasses import dataclass
3
+ from transformers import PretrainedConfig
4
+
5
+ @dataclass
6
+ class ViTConfig(PretrainedConfig):
7
+ model_type: str = 'vit'
8
+
9
+ # Model parameters
10
+ image_size: int = 160
11
+ in_channels: int = 2
12
+ patch_size: int = 4
13
+ num_layers: int = 8
14
+ num_heads: int = 8
15
+ hidden_dim: int = 768
16
+ mlp_dim: int = 3072
17
+ num_classes: int = 2
18
+
19
+ # Training parameters
20
+ loadcheck_path: str = '.'
21
+ ft: bool = False
22
+ infer: bool = False
23
+ ifresume: bool = False
24
+
25
+ output_dir: str = ""
26
+ learning_rate: float = 1e-4
27
+ weight_decay: float = 0.0
28
+ gradient_accumulation_steps: int = 1
29
+ per_device_train_batch_size: int = 2
30
+ per_device_eval_batch_size: int = 2
31
+
32
+ num_train_epochs: int = 10
33
+ fp16: bool = False
34
+ bf16: bool = False
35
+ logging_dir: str = ""
36
+ logging_steps: int = 10
37
+ max_steps: int = -1
38
+ warmup_steps: int = 1000
39
+ save_steps: int = 1000
40
+
41
+ dataloader_num_workers: int = 8
42
+ seed: int = 6
43
+
44
+ def __init__(self, **kwargs):
45
+ # Use `super().__init__` to handle arguments from PretrainedConfig
46
+ super().__init__(**kwargs)
cell_fm/models/vit_cls_condenseq_img/model.py ADDED
@@ -0,0 +1,171 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import torch
3
+ import torch.nn as nn
4
+ from cell_fm.logging import logger
5
+ from transformers import PreTrainedModel
6
+ from .config import ViTConfig
7
+
8
+ from torchvision.models.vision_transformer import VisionTransformer
9
+ from functools import partial
10
+
11
+ from cell_fm.pipeline.utils import VanillaOutput
12
+ import json
13
+
14
+
15
+ class ViTModel(PreTrainedModel):
16
+ config_class = ViTConfig
17
+
18
+ def __init__(self, config: ViTConfig):
19
+ super().__init__(config)
20
+ self.config = config
21
+
22
+ self.loss = nn.CrossEntropyLoss()
23
+ self.net = ViT(config)
24
+
25
+ self.load_pretrained_weights(config, checkpoint_path=config.loadcheck_path)
26
+
27
+ def load_pretrained_weights(self, config, checkpoint_path):
28
+ """
29
+ Load pretrained weights from a given state_dict.
30
+ """
31
+ if config.ft or config.infer:
32
+ if config.ft:
33
+ logger.info(f"Finetune from checkpoint: {checkpoint_path}")
34
+ else:
35
+ logger.info(f"Infer from checkpoint: {checkpoint_path}")
36
+
37
+ if os.path.isdir(checkpoint_path):
38
+ with open(os.path.join(checkpoint_path, "pytorch_model.bin.index.json"), "r") as f:
39
+ index_data = json.load(f)
40
+
41
+ weight_map = index_data["weight_map"]
42
+
43
+ shard_files = set(weight_map.values())
44
+ shard_weights = {}
45
+ for shard_file in shard_files:
46
+ shard_weights[shard_file] = torch.load(os.path.join(checkpoint_path, shard_file), map_location="cpu")
47
+
48
+ checkpoints_state = {}
49
+ for param_name, shard_file in weight_map.items():
50
+ checkpoints_state[param_name] = shard_weights[shard_file][param_name]
51
+ elif os.path.splitext(checkpoint_path)[1] == '.safetensors':
52
+ from safetensors.torch import load_file
53
+ checkpoints_state = load_file(checkpoint_path)
54
+ else:
55
+ checkpoints_state = torch.load(checkpoint_path, map_location="cpu")
56
+
57
+ if "model" in checkpoints_state:
58
+ checkpoints_state = checkpoints_state["model"]
59
+ elif "module" in checkpoints_state:
60
+ checkpoints_state = checkpoints_state["module"]
61
+
62
+ model_state_dict = self.state_dict()
63
+ filtered_state_dict = {k: v for k, v in checkpoints_state.items() if k in model_state_dict and v.size() == model_state_dict[k].size()}
64
+
65
+ IncompatibleKeys = self.load_state_dict(filtered_state_dict, strict=False)
66
+ IncompatibleKeys = IncompatibleKeys._asdict()
67
+
68
+ missing_keys = []
69
+ for keys in IncompatibleKeys["missing_keys"]:
70
+ if keys.find("dummy") == -1:
71
+ missing_keys.append(keys)
72
+
73
+ unexpected_keys = []
74
+ for keys in IncompatibleKeys["unexpected_keys"]:
75
+ if keys.find("dummy") == -1:
76
+ unexpected_keys.append(keys)
77
+
78
+ if len(missing_keys) > 0:
79
+ logger.info(
80
+ "Missing keys in {}: {}".format(
81
+ checkpoint_path,
82
+ missing_keys,
83
+ )
84
+ )
85
+
86
+ if len(unexpected_keys) > 0:
87
+ logger.info(
88
+ "Unexpected keys {}: {}".format(
89
+ checkpoint_path,
90
+ unexpected_keys,
91
+ )
92
+ )
93
+
94
+ def forward(self, batched_data, **kwargs):
95
+ nucleus_img = batched_data['nucleus_img']
96
+ protein_img = batched_data['protein_img']
97
+ label = batched_data['label']
98
+
99
+ input_img = torch.cat([nucleus_img, protein_img], dim=1) # [B, 2, H, W]
100
+
101
+ logits = self.net(input_img) # [B, num_classes]
102
+
103
+ loss = self.loss(logits, label.view(-1))
104
+ loss = loss.mean()
105
+
106
+ return VanillaOutput(loss=loss)
107
+
108
+ def predict(self, x):
109
+ with torch.no_grad():
110
+ input_img = x # [B, 2, H, W]
111
+ logits = self.net(input_img) # [B, num_classes]
112
+
113
+ return logits
114
+
115
+ class ViT(VisionTransformer):
116
+ def __init__(self, config: ViTConfig):
117
+ super().__init__(
118
+ image_size=config.image_size,
119
+ patch_size=config.patch_size,
120
+ num_layers=config.num_layers,
121
+ num_heads=config.num_heads,
122
+ hidden_dim=config.hidden_dim,
123
+ mlp_dim=config.mlp_dim,
124
+ dropout=0.0,
125
+ attention_dropout=0.0,
126
+ num_classes=config.num_classes,
127
+ representation_size=None,
128
+ norm_layer=partial(nn.LayerNorm, eps=1e-6),
129
+ conv_stem_configs=None,
130
+ )
131
+
132
+ old_conv = self.conv_proj
133
+ in_channels = config.in_channels
134
+
135
+ self.conv_proj = nn.Conv2d(
136
+ in_channels=in_channels,
137
+ out_channels=old_conv.out_channels,
138
+ kernel_size=old_conv.kernel_size,
139
+ stride=old_conv.stride,
140
+ padding=old_conv.padding,
141
+ bias=old_conv.bias is not None,
142
+ )
143
+
144
+ def forward(self, x: torch.Tensor):
145
+ # Reshape and permute the input tensor
146
+ x = self._process_input(x)
147
+ n = x.shape[0]
148
+
149
+ # Expand the class token to the full batch
150
+ batch_class_token = self.class_token.expand(n, -1, -1)
151
+ x = torch.cat([batch_class_token, x], dim=1)
152
+
153
+ x = self.encoder(x)
154
+ x = x.mean(dim=1) # Global average pooling over the sequence length
155
+ x = self.heads(x)
156
+
157
+ return x
158
+
159
+ def embed(self, x: torch.Tensor):
160
+ # Reshape and permute the input tensor
161
+ x = self._process_input(x)
162
+ n = x.shape[0]
163
+
164
+ # Expand the class token to the full batch
165
+ batch_class_token = self.class_token.expand(n, -1, -1)
166
+ x = torch.cat([batch_class_token, x], dim=1)
167
+
168
+ x = self.encoder(x)
169
+ embed = x.mean(dim=1) # Global average pooling over the sequence length
170
+
171
+ return embed
cell_fm/pipeline/__init__.py ADDED
File without changes
cell_fm/pipeline/utils.py ADDED
@@ -0,0 +1,34 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ from dataclasses import dataclass
3
+ from transformers.utils import ModelOutput
4
+ from transformers import Trainer
5
+ from cell_fm.logging import metric_logger
6
+ from typing import Dict, Optional
7
+
8
+ class CELLDiffTrainer(Trainer):
9
+ def compute_loss(self, model, inputs, return_outputs=False):
10
+ # Compute loss using parent class
11
+ loss, outputs = super().compute_loss(model, inputs, True)
12
+ # metric_logger.log(outputs.log_output)
13
+ # self.log(outputs.log_output)
14
+
15
+ return (loss, outputs) if return_outputs else loss
16
+
17
+ @dataclass
18
+ class StableDiffusionOutput(ModelOutput):
19
+ loss: torch.FloatTensor = None
20
+
21
+ @dataclass
22
+ class VAEOutput(ModelOutput):
23
+ loss: torch.FloatTensor = None
24
+ log_output: Optional[Dict] = None
25
+
26
+ @dataclass
27
+ class CELLFMOutput(ModelOutput):
28
+ loss: torch.FloatTensor = None
29
+ log_output: Optional[Dict] = None
30
+
31
+ @dataclass
32
+ class VanillaOutput(ModelOutput):
33
+ loss: torch.FloatTensor = None
34
+ log_output: Optional[Dict] = None
cell_fm/utils/__init__.py ADDED
File without changes
cell_fm/utils/dist_utils.py ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # -*- coding: utf-8 -*-
2
+ import os
3
+
4
+ import torch
5
+
6
+
7
+ def is_master_node():
8
+ if "RANK" not in os.environ or int(os.environ["RANK"]) == 0:
9
+ return True
10
+ else:
11
+ return False
12
+
13
+
14
+ def support_flash_attntion():
15
+ # check the GPU is A100 or H100
16
+ device_name = torch.cuda.get_device_name(0)
17
+ for name in ["A100", "H100"]:
18
+ if name in device_name:
19
+ return True
20
+ return False
metrics.py ADDED
@@ -0,0 +1,135 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Curve metrics for the condensate titration analysis.
2
+
3
+ Port of cell_diff/tasks/analysis/utils.py from CELL-Diff2-Dev, restricted to the
4
+ three quantities this demo reports:
5
+
6
+ AUC area under the moving-averaged condensate-probability curve
7
+ NR the same curve with everything past its maximum held flat ("no reentrant")
8
+ AAC NR - AUC, i.e. the area *above* the curve that reentrant dissolution opens up
9
+
10
+ All three integrate against log10(intensity) by default and are normalised by the
11
+ width of the integration domain, so they live in [0, 1] and are comparable across
12
+ sequences.
13
+ """
14
+
15
+ import numpy as np
16
+ import pandas as pd
17
+
18
+
19
+ def moving_average(y, window: int) -> np.ndarray:
20
+ """Centred rolling mean, shrinking the window at the edges rather than padding."""
21
+ y = np.asarray(y, dtype=float)
22
+ window = int(max(1, min(window, len(y))))
23
+ return (
24
+ pd.Series(y)
25
+ .rolling(window=window, center=True, min_periods=1)
26
+ .mean()
27
+ .to_numpy()
28
+ )
29
+
30
+
31
+ def integral(xs: np.ndarray, ys: np.ndarray) -> float:
32
+ xs = np.asarray(xs, dtype=float)
33
+ ys = np.asarray(ys, dtype=float)
34
+
35
+ if xs.ndim != 1 or ys.ndim != 1:
36
+ raise ValueError("xs and ys must be 1D arrays")
37
+ if xs.shape[0] != ys.shape[0]:
38
+ raise ValueError("xs and ys must have the same length")
39
+ if xs.shape[0] < 2:
40
+ return 0.0
41
+
42
+ return float(np.trapz(ys, xs))
43
+
44
+
45
+ def _prepare(df: pd.DataFrame, window_size: int, use_log_scale: bool):
46
+ """Return (x, curve) sorted by x, with x log10-scaled when requested.
47
+
48
+ Sorting happens once, before the moving average is taken, so the smoothing
49
+ always runs along increasing intensity.
50
+ """
51
+ intensity_levels = df["protein_intensity_level"].to_numpy(dtype=float)
52
+ predicted_classes = df["predicted_class"].to_numpy(dtype=float)
53
+
54
+ order = np.argsort(intensity_levels)
55
+ intensity_levels = intensity_levels[order]
56
+ predicted_classes = predicted_classes[order]
57
+
58
+ curve = moving_average(predicted_classes, window_size)
59
+
60
+ if use_log_scale:
61
+ # clip at 1.0 so that intensities below 1 do not send log10 negative/-inf
62
+ x = np.log10(np.clip(intensity_levels, a_min=1.0, a_max=None))
63
+ else:
64
+ x = intensity_levels
65
+
66
+ return x, curve
67
+
68
+
69
+ def _normalise(area: float, x: np.ndarray, normalize: bool) -> float:
70
+ if not normalize:
71
+ return area
72
+ total_area = (x[-1] - x[0]) * 1.0
73
+ return area / total_area
74
+
75
+
76
+ def compute_area_under_curve(
77
+ df: pd.DataFrame, window_size: int, normalize: bool = True, use_log_scale: bool = True
78
+ ) -> float:
79
+ """AUC: how much of the intensity range this sequence spends in the condensed state."""
80
+ x, curve = _prepare(df, window_size, use_log_scale)
81
+ return _normalise(integral(x, curve), x, normalize)
82
+
83
+
84
+ def no_reentrant_curve(curve: np.ndarray) -> np.ndarray:
85
+ """The curve it would have been without reentrant dissolution.
86
+
87
+ Everything past the maximum is held at the maximum, so the difference against
88
+ the real curve is exactly the dissolution that happens at high concentration.
89
+ """
90
+ max_index = int(np.argmax(curve))
91
+ out = curve.copy()
92
+ out[max_index:] = float(np.max(curve))
93
+ return out
94
+
95
+
96
+ def compute_area_if_no_reentrant(
97
+ df: pd.DataFrame, window_size: int, normalize: bool = True, use_log_scale: bool = True
98
+ ) -> float:
99
+ """Area under the curve after flattening everything past its maximum."""
100
+ x, curve = _prepare(df, window_size, use_log_scale)
101
+ return _normalise(integral(x, no_reentrant_curve(curve)), x, normalize)
102
+
103
+
104
+ def compute_area_if_no_reentrant_minus_auc(
105
+ df: pd.DataFrame, window_size: int, normalize: bool = True, use_log_scale: bool = True
106
+ ) -> float:
107
+ """AAC, the area *above* the curve: no-reentrant area minus the real area.
108
+
109
+ Zero when the curve never comes back down; larger the more the sequence
110
+ dissolves again at high concentration.
111
+ """
112
+ x, curve = _prepare(df, window_size, use_log_scale)
113
+ return _normalise(integral(x, no_reentrant_curve(curve) - curve), x, normalize)
114
+
115
+
116
+ # the paper's name for compute_area_if_no_reentrant_minus_auc
117
+ compute_area_above_curve = compute_area_if_no_reentrant_minus_auc
118
+
119
+
120
+ def compute_concentration(
121
+ df: pd.DataFrame, window_size: int, threshold_prob: float = 0.8
122
+ ) -> float:
123
+ """c_sat: the first intensity at which the smoothed curve reaches threshold_prob.
124
+
125
+ inf when the sequence never condenses over the scanned range.
126
+ """
127
+ intensity_levels = df["protein_intensity_level"].to_numpy(dtype=float)
128
+ predicted_classes = df["predicted_class"].to_numpy(dtype=float)
129
+
130
+ order = np.argsort(intensity_levels)
131
+ intensity_levels = intensity_levels[order]
132
+ curve = moving_average(predicted_classes[order], window_size)
133
+
134
+ hits = np.flatnonzero(curve >= threshold_prob)
135
+ return float(intensity_levels[hits[0]]) if hits.size else float("inf")
pipeline.py ADDED
@@ -0,0 +1,312 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """CELL-FM CondenSeq: sequence -> condensate titration curve.
2
+
3
+ The demo runs the same three stages as the offline analysis, in memory:
4
+
5
+ 1. generate CELL-FM CS samples one image per protein-intensity level, over a
6
+ log-spaced concentration ladder (evaluate_seq2img_dict.py)
7
+ 2. classify a ViT labels each image condensed / diffuse (evaluate_single_img.py)
8
+ 3. summarise a centred moving average over the binary calls gives the condensate
9
+ probability curve (ma_plot.py), and AUC / AAC integrate it
10
+ (analysis/ana_all_mutation_log_scale.py)
11
+
12
+ Weights come from a HF model repo by default; set CELLFM_LOCAL_WEIGHTS to a
13
+ directory holding the three .bin files to run against a local copy instead.
14
+ """
15
+
16
+ import os
17
+ from dataclasses import dataclass, field
18
+ from functools import lru_cache
19
+
20
+ import numpy as np
21
+ import pandas as pd
22
+ import torch
23
+
24
+ from cell_fm.criterions.cell_fm.unidiffuser import UniDiffCriterions
25
+ from cell_fm.models.cell_fm.cell_fm_cs_config import CELLFMConfig
26
+ from cell_fm.models.cell_fm.cell_fm_cs_model import CELLFMCSModel
27
+ from cell_fm.models.vit_cls_condenseq_img.config import ViTConfig
28
+ from cell_fm.models.vit_cls_condenseq_img.model import ViTModel
29
+
30
+ import metrics
31
+
32
+ HERE = os.path.dirname(os.path.abspath(__file__))
33
+
34
+ MODEL_REPO = os.environ.get("CELLFM_MODEL_REPO", "BoHuangLab/CELL-FM")
35
+ LOCAL_WEIGHTS = os.environ.get("CELLFM_LOCAL_WEIGHTS", "")
36
+
37
+ # The weight repo holds one subfolder per model family, so paths are prefixed.
38
+ # CELLFM_LOCAL_WEIGHTS points at a flat directory, hence the basename() below.
39
+ GENERATOR_WEIGHTS = "condenseq/cellfm_seq2img.bin"
40
+ VAE_WEIGHTS = "condenseq/vae.bin"
41
+ CLASSIFIER_WEIGHTS = "condenseq/vit_cls.bin"
42
+
43
+ # The concentration ladder. The offline runs scanned 4096 levels; the demo scans
44
+ # fewer for latency, and the moving-average window scales with it (see WINDOW_DIVISOR).
45
+ MIN_INTENSITY = 20.0
46
+ MAX_INTENSITY = 2048.0
47
+ WINDOW_DIVISOR = 8 # window = num_images // 8, matching ana_all_mutation_log_scale.py
48
+
49
+ VALID_AA = set("ACDEFGHIKLMNPQRSTVWY")
50
+
51
+ # Generation starts from a random latent, so the curve moves a little between runs.
52
+ # The offline scripts fixed this at 6; keeping the same default makes the demo's
53
+ # numbers repeatable and comparable to them.
54
+ DEFAULT_SEED = 6
55
+
56
+
57
+ def resolve_weights(filename: str) -> str:
58
+ """Local directory if CELLFM_LOCAL_WEIGHTS is set, else pull from the model repo."""
59
+ if LOCAL_WEIGHTS:
60
+ path = os.path.join(LOCAL_WEIGHTS, os.path.basename(filename))
61
+ if not os.path.exists(path):
62
+ raise FileNotFoundError(f"{path} not found (CELLFM_LOCAL_WEIGHTS={LOCAL_WEIGHTS})")
63
+ return path
64
+
65
+ from huggingface_hub import hf_hub_download
66
+
67
+ return hf_hub_download(repo_id=MODEL_REPO, filename=filename)
68
+
69
+
70
+ def generator_config() -> CELLFMConfig:
71
+ """Hyperparameters from scripts/cell_fm_cs/evaluate_seq2img.sh."""
72
+ return CELLFMConfig(
73
+ img_resize=160,
74
+ img_crop_size=160,
75
+ img_type="GFP",
76
+ cell_image="nucl",
77
+ seq_zero_mask_ratio=0.0,
78
+ path_type="Linear",
79
+ prediction="velocity",
80
+ # VAE
81
+ num_down_blocks=3,
82
+ latent_channels=4,
83
+ vae_block_out_channels="128,256,512",
84
+ # CELL-FM
85
+ img_mask_ratio=0,
86
+ cond_out_channels="32,64",
87
+ sample_size=40,
88
+ esm_embedding="esmc_600m",
89
+ encoder_hidden_size=1152,
90
+ max_protein_sequence_len=2048,
91
+ encoder_num_hidden_layers=4,
92
+ num_heads=8,
93
+ dim_head=64,
94
+ dropout=0,
95
+ final_dropout=0,
96
+ encoder_patch_size=4,
97
+ # image generator
98
+ img_generator_num_layers=4,
99
+ img_generator_patch_size=2,
100
+ attention_head_dim=64,
101
+ num_attention_heads=8,
102
+ # image decoder
103
+ img_decoder_num_hidden_layers=2,
104
+ img_decoder_hidden_size=512,
105
+ img_decoder_num_heads=8,
106
+ img_decoder_dim_head=64,
107
+ cell_image_ratio=0.5,
108
+ # checkpoints
109
+ vae_loadcheck_path=resolve_weights(VAE_WEIGHTS),
110
+ loadcheck_path=resolve_weights(GENERATOR_WEIGHTS),
111
+ infer=True,
112
+ )
113
+
114
+
115
+ def classifier_config() -> ViTConfig:
116
+ """Hyperparameters from scripts/vit_cls_condenseq_img/pretrain.sh."""
117
+ return ViTConfig(
118
+ image_size=160,
119
+ in_channels=2,
120
+ patch_size=4,
121
+ num_layers=8,
122
+ num_heads=8,
123
+ hidden_dim=512,
124
+ mlp_dim=2048,
125
+ num_classes=2,
126
+ loadcheck_path=resolve_weights(CLASSIFIER_WEIGHTS),
127
+ infer=True,
128
+ )
129
+
130
+
131
+ def get_device() -> torch.device:
132
+ return torch.device("cuda" if torch.cuda.is_available() else "cpu")
133
+
134
+
135
+ @lru_cache(maxsize=1)
136
+ def load_models():
137
+ """Build both models once and keep them resident."""
138
+ device = get_device()
139
+
140
+ generator = CELLFMCSModel(config=generator_config(), loss_fn=UniDiffCriterions)
141
+ generator.to(device).eval()
142
+
143
+ classifier = ViTModel(config=classifier_config())
144
+ classifier.to(device).eval()
145
+
146
+ from esm.tokenization.sequence_tokenizer import EsmSequenceTokenizer
147
+
148
+ return generator, classifier, EsmSequenceTokenizer()
149
+
150
+
151
+ @lru_cache(maxsize=1)
152
+ def reference_nucleus() -> torch.Tensor:
153
+ """The DAPI channel every generated image is conditioned on, as (1, 1, 160, 160) in [-1, 1].
154
+
155
+ Taken from CondenSeq protein index 12626, image 0 -- the same one the offline
156
+ seq2img runs used, so demo curves are comparable to the published ones.
157
+ """
158
+ arr = np.load(os.path.join(HERE, "assets", "reference_nucleus.npy"))
159
+ return torch.from_numpy(arr).float().unsqueeze(0)
160
+
161
+
162
+ def intensity_ladder(num_images: int) -> torch.Tensor:
163
+ """Log-spaced protein intensity levels, the x axis of the titration curve."""
164
+ return torch.logspace(
165
+ start=float(np.log10(MIN_INTENSITY)),
166
+ end=float(np.log10(MAX_INTENSITY)),
167
+ steps=num_images,
168
+ )
169
+
170
+
171
+ def clean_sequence(sequence: str) -> str:
172
+ """Strip FASTA headers, whitespace and case; reject anything not a standard residue."""
173
+ lines = [ln for ln in str(sequence).splitlines() if not ln.startswith(">")]
174
+ seq = "".join(lines).strip().upper()
175
+ seq = "".join(seq.split())
176
+
177
+ if not seq:
178
+ raise ValueError("Enter a protein sequence.")
179
+ bad = sorted(set(seq) - VALID_AA)
180
+ if bad:
181
+ raise ValueError(
182
+ f"Sequence contains non-standard residues: {', '.join(bad)}. "
183
+ "Use the 20 standard amino acids."
184
+ )
185
+ return seq
186
+
187
+
188
+ @dataclass
189
+ class Result:
190
+ sequence: str
191
+ intensities: np.ndarray # (N,) protein intensity levels
192
+ predictions: np.ndarray # (N,) 0 = diffuse, 1 = condensed
193
+ curve: np.ndarray # (N,) moving-averaged condensate probability
194
+ window: int
195
+ auc: float # area under the curve
196
+ aac: float # area above the curve (reentrant dissolution)
197
+ no_reentrant_auc: float
198
+ c_sat: float
199
+ images: np.ndarray = field(repr=False, default=None) # (K, 160, 160) sample panel
200
+ image_intensities: np.ndarray = field(repr=False, default=None)
201
+
202
+
203
+ @torch.no_grad()
204
+ def generate_images(sequence, num_images, num_steps, batch_size, seed=DEFAULT_SEED, progress=None):
205
+ """Stage 1: one generated image per intensity level, returned as uint16 (N, 2, 160, 160).
206
+
207
+ uint16 is not incidental -- the offline pipeline wrote a TIFF between generation
208
+ and classification, so the classifier saw quantised pixels. Keeping the same
209
+ round-trip here keeps demo numbers comparable to the published ones.
210
+ """
211
+ generator, _, vocab = load_models()
212
+ device = get_device()
213
+
214
+ from esm.utils import encoding
215
+
216
+ if seed is not None:
217
+ torch.manual_seed(int(seed))
218
+ if torch.cuda.is_available():
219
+ torch.cuda.manual_seed_all(int(seed))
220
+
221
+ tokens = encoding.tokenize_sequence(sequence, vocab, True).unsqueeze(0).to(device)
222
+ nucleus = reference_nucleus().to(device)
223
+ levels = intensity_ladder(num_images).to(device)
224
+
225
+ out = []
226
+ for start in range(0, num_images, batch_size):
227
+ end = min(start + batch_size, num_images)
228
+ n = end - start
229
+
230
+ if progress is not None:
231
+ progress(start / num_images, desc=f"Generating images {start}-{end} of {num_images}")
232
+
233
+ sample = generator.sequence_to_image(
234
+ tokens.repeat(n, 1),
235
+ nucleus.repeat(n, 1, 1, 1),
236
+ levels[start:end].unsqueeze(1),
237
+ num_steps=num_steps,
238
+ )
239
+
240
+ # channel 0 = nucleus (conditioning), channel 1 = generated protein
241
+ combined = torch.cat([nucleus.repeat(n, 1, 1, 1), sample], dim=1)
242
+ combined = ((combined + 1) / 2.0).clamp(0.0, 1.0)
243
+ out.append((combined * 65535).cpu().numpy().astype(np.uint16))
244
+
245
+ return np.concatenate(out, axis=0)
246
+
247
+
248
+ @torch.no_grad()
249
+ def classify_images(images, batch_size, progress=None):
250
+ """Stage 2: condensed (1) vs diffuse (0) for each generated image."""
251
+ _, classifier, _ = load_models()
252
+ device = get_device()
253
+
254
+ preds = []
255
+ for start in range(0, images.shape[0], batch_size):
256
+ end = min(start + batch_size, images.shape[0])
257
+
258
+ if progress is not None:
259
+ progress(start / images.shape[0], desc=f"Classifying {start}-{end} of {images.shape[0]}")
260
+
261
+ batch = torch.from_numpy(images[start:end].astype(np.float32)).to(device)
262
+ batch = batch / 65535.0 * 2.0 - 1.0 # back to [-1, 1]
263
+ logits = classifier.predict(batch)
264
+ preds.append(torch.argmax(logits, dim=1).cpu().numpy())
265
+
266
+ return np.concatenate(preds).astype(int)
267
+
268
+
269
+ def summarise(intensities, predictions, num_images):
270
+ """Stage 3: moving average, then AUC / AAC / c_sat over the smoothed curve."""
271
+ df = pd.DataFrame(
272
+ {"protein_intensity_level": intensities, "predicted_class": predictions}
273
+ )
274
+ window = max(1, num_images // WINDOW_DIVISOR)
275
+ kw = dict(window_size=window, use_log_scale=True)
276
+
277
+ return df, window, dict(
278
+ curve=metrics.moving_average(predictions, window),
279
+ auc=metrics.compute_area_under_curve(df, **kw),
280
+ aac=metrics.compute_area_above_curve(df, **kw),
281
+ no_reentrant_auc=metrics.compute_area_if_no_reentrant(df, **kw),
282
+ c_sat=metrics.compute_concentration(df, window_size=window),
283
+ )
284
+
285
+
286
+ def sample_panel(images, intensities, k=6):
287
+ """Pick k images spread evenly along the ladder, for the strip shown under the curve."""
288
+ idx = np.linspace(0, len(intensities) - 1, k).astype(int)
289
+ return images[idx, 1].astype(np.float32) / 65535.0, intensities[idx]
290
+
291
+
292
+ def run(sequence, num_images=512, num_steps=100, batch_size=64, seed=DEFAULT_SEED,
293
+ progress=None) -> Result:
294
+ """The whole pipeline: sequence in, titration curve and its areas out."""
295
+ sequence = clean_sequence(sequence)
296
+
297
+ images = generate_images(sequence, num_images, num_steps, batch_size, seed, progress)
298
+ predictions = classify_images(images, batch_size, progress)
299
+
300
+ intensities = intensity_ladder(num_images).numpy()
301
+ _, window, stats = summarise(intensities, predictions, num_images)
302
+ panel, panel_x = sample_panel(images, intensities)
303
+
304
+ return Result(
305
+ sequence=sequence,
306
+ intensities=intensities,
307
+ predictions=predictions,
308
+ window=window,
309
+ images=panel,
310
+ image_intensities=panel_x,
311
+ **stats,
312
+ )
requirements.txt ADDED
@@ -0,0 +1,19 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Pinned to the versions the CELL-FM checkpoints were trained and evaluated with.
2
+ torch==2.4.1
3
+ torchvision==0.19.1
4
+ diffusers==0.31.0
5
+ transformers==4.44.2
6
+ esm==3.1.4
7
+ numpy==1.26.3
8
+ pandas==2.2.2
9
+ matplotlib==3.9.2
10
+ tifffile==2024.9.20
11
+ safetensors==0.4.5
12
+ einops==0.8.0
13
+ huggingface_hub>=0.24.6
14
+ gradio>=4.44.0
15
+
16
+ # flash-attn is deliberately NOT installed. ESM-C falls back to its pure-torch
17
+ # rotary when the package is absent (esm/models/esmc.py: is_flash_attn_available),
18
+ # and the two paths were verified bit-identical -- same AUC, same AAC, same 256/256
19
+ # per-image calls, same runtime -- so the Space skips a fragile CUDA build for free.
run_local.sh ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ # Run the demo on a cluster GPU and open a public tunnel to it.
3
+ #
4
+ # Works today, with no HF plan: gradio's --share flag serves the app from this
5
+ # machine over a temporary public URL (72 h, printed on startup). Useful for
6
+ # showing the demo to the lab while the Space is blocked.
7
+ #
8
+ # sbatch run_local.sh # or run the python line directly on a GPU node
9
+ #
10
+ #SBATCH --job-name=cellfm_demo
11
+ #SBATCH --time=08:00:00
12
+ #SBATCH --nodes=1
13
+ #SBATCH --ntasks=1
14
+ #SBATCH --partition=interactive
15
+ #SBATCH --cpus-per-task=8
16
+ #SBATCH --gpus=1
17
+ #SBATCH --output=cellfm_demo_%j.out
18
+
19
+ module load anaconda
20
+ conda activate esm3
21
+ export PYTHONUNBUFFERED=1
22
+
23
+ # Pull weights from the Hub (needs HF_TOKEN while HuangLab/CELL-FM is private),
24
+ # or point at a flat local directory holding the three .bin files:
25
+ # export CELLFM_LOCAL_WEIGHTS=/path/to/weights
26
+
27
+ cd "$(dirname "$0")"
28
+ GRADIO_SHARE=1 python app.py
upload_weights.py ADDED
@@ -0,0 +1,106 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Push the three CELL-FM CondenSeq checkpoints to a HF model repo.
2
+
3
+ NOTE: never place credentials inside this directory -- it is uploaded verbatim to
4
+ the Space. Pass a token via the HF_TOKEN environment variable instead.
5
+
6
+ The Space pulls its weights from that repo at startup rather than carrying ~2.1 GB
7
+ of LFS itself. Run this once from the cluster:
8
+
9
+ python upload_weights.py --repo BoHuangLab/CELL-FM
10
+
11
+ Add --private to keep the weights unlisted, and --dry-run to see what would be
12
+ uploaded without touching the Hub.
13
+ """
14
+
15
+ import argparse
16
+ import os
17
+
18
+ DEFAULT_SOURCES = {
19
+ # target name in the repo -> checkpoint on the cluster
20
+ "condenseq/cellfm_seq2img.bin": (
21
+ "/hpc/projects/group.huang/dihan.zheng/CELL-FM/"
22
+ "pretrain_condenseq/cellfm_seq2img/checkpoint-50000/pytorch_model.bin"
23
+ ),
24
+ "condenseq/vae.bin": (
25
+ "/hpc/projects/group.huang/dihan.zheng/CELL-FM/"
26
+ "pretrain_condenseq/vae/checkpoint-50000/pytorch_model.bin"
27
+ ),
28
+ "condenseq/vit_cls.bin": (
29
+ "/hpc/projects/group.huang/dihan.zheng/CELL-Diff2-Dev/"
30
+ "pretrain_condenseq_celldiff2_split/PT_CondenSeq_img_ViT_cls_R1/"
31
+ "checkpoint-10000/pytorch_model.bin"
32
+ ),
33
+ }
34
+
35
+ CARD = """---
36
+ library_name: cell-fm
37
+ tags:
38
+ - biology
39
+ - microscopy
40
+ - protein
41
+ - condensate
42
+ - flow-matching
43
+ ---
44
+
45
+ # CELL-FM weights
46
+
47
+ Checkpoints behind the [CELL-FM CondenSeq demo]({space_url}).
48
+
49
+ | File | Model | Source checkpoint |
50
+ |---|---|---|
51
+ | `condenseq/cellfm_seq2img.bin` | CELL-FM CS sequence-to-image generator (includes the ESM-C 600M encoder) | `pretrain_condenseq/cellfm_seq2img/checkpoint-50000` |
52
+ | `condenseq/vae.bin` | Image VAE, 160x160, 3 down blocks, 4 latent channels | `pretrain_condenseq/vae/checkpoint-50000` |
53
+ | `condenseq/vit_cls.bin` | ViT condensed/diffuse classifier, 2-channel 160x160 input | `PT_CondenSeq_img_ViT_cls_R1/checkpoint-10000` |
54
+
55
+ Hyperparameters are set in `pipeline.py` in the Space and mirror
56
+ `scripts/cell_fm_cs/evaluate_seq2img.sh` and
57
+ `scripts/vit_cls_condenseq_img/pretrain.sh` in the CELL-FM repository.
58
+ """
59
+
60
+
61
+ def main() -> None:
62
+ ap = argparse.ArgumentParser()
63
+ ap.add_argument("--repo", default="BoHuangLab/CELL-FM")
64
+ ap.add_argument("--space-url", default="https://huggingface.co/spaces/BoHuangLab/CELL-FM")
65
+ ap.add_argument("--private", action="store_true")
66
+ ap.add_argument("--dry-run", action="store_true")
67
+ args = ap.parse_args()
68
+
69
+ missing = [p for p in DEFAULT_SOURCES.values() if not os.path.exists(p)]
70
+ if missing:
71
+ raise SystemExit("missing checkpoint(s):\n " + "\n ".join(missing))
72
+
73
+ total = sum(os.path.getsize(p) for p in DEFAULT_SOURCES.values())
74
+ for name, path in DEFAULT_SOURCES.items():
75
+ print(f" {name:22s} {os.path.getsize(path)/1e9:5.2f} GB <- {path}")
76
+ print(f" {'total':22s} {total/1e9:5.2f} GB -> {args.repo}")
77
+
78
+ if args.dry_run:
79
+ print("\ndry run, nothing uploaded")
80
+ return
81
+
82
+ from huggingface_hub import HfApi
83
+
84
+ api = HfApi()
85
+ api.create_repo(args.repo, repo_type="model", private=args.private, exist_ok=True)
86
+
87
+ for name, path in DEFAULT_SOURCES.items():
88
+ print(f"uploading {name} ...")
89
+ api.upload_file(
90
+ path_or_fileobj=path,
91
+ path_in_repo=name,
92
+ repo_id=args.repo,
93
+ repo_type="model",
94
+ )
95
+
96
+ api.upload_file(
97
+ path_or_fileobj=CARD.format(space_url=args.space_url).encode(),
98
+ path_in_repo="README.md",
99
+ repo_id=args.repo,
100
+ repo_type="model",
101
+ )
102
+ print(f"\ndone: https://huggingface.co/{args.repo}")
103
+
104
+
105
+ if __name__ == "__main__":
106
+ main()