Spaces:
Running on Zero
Running on Zero
CELL-FM CondenSeq demo: sequence -> titration curve, AUC and AAC
Browse files- .gitattributes +2 -34
- .gitignore +6 -0
- README.md +110 -7
- app.py +233 -0
- assets/nup98_mutation_seqs.csv +93 -0
- assets/reference_nucleus.npy +3 -0
- cell_fm/__init__.py +0 -0
- cell_fm/criterions/__init__.py +0 -0
- cell_fm/criterions/cell_fm/__init__.py +0 -0
- cell_fm/criterions/cell_fm/unidiffuser.py +63 -0
- cell_fm/logging/__init__.py +5 -0
- cell_fm/logging/loggers.py +201 -0
- cell_fm/models/__init__.py +0 -0
- cell_fm/models/cell_fm/__init__.py +0 -0
- cell_fm/models/cell_fm/cell_fm_config.py +116 -0
- cell_fm/models/cell_fm/cell_fm_cs_config.py +12 -0
- cell_fm/models/cell_fm/cell_fm_cs_model.py +309 -0
- cell_fm/models/cell_fm/cell_fm_model.py +637 -0
- cell_fm/models/cell_fm/modules/__init__.py +82 -0
- cell_fm/models/cell_fm/modules/positional_embedding.py +59 -0
- cell_fm/models/cell_fm/modules/protein_sequence_embedding.py +39 -0
- cell_fm/models/cell_fm/modules/transformer.py +131 -0
- cell_fm/models/cell_fm/modules/transport/__init__.py +84 -0
- cell_fm/models/cell_fm/modules/transport/integrators.py +117 -0
- cell_fm/models/cell_fm/modules/transport/path.py +192 -0
- cell_fm/models/cell_fm/modules/transport/transport.py +506 -0
- cell_fm/models/cell_fm/modules/transport/utils.py +29 -0
- cell_fm/models/vae/__init__.py +0 -0
- cell_fm/models/vae/vae_config.py +60 -0
- cell_fm/models/vae/vae_model.py +152 -0
- cell_fm/models/vit_cls_condenseq_img/__init__.py +0 -0
- cell_fm/models/vit_cls_condenseq_img/config.py +46 -0
- cell_fm/models/vit_cls_condenseq_img/model.py +171 -0
- cell_fm/pipeline/__init__.py +0 -0
- cell_fm/pipeline/utils.py +34 -0
- cell_fm/utils/__init__.py +0 -0
- cell_fm/utils/dist_utils.py +20 -0
- metrics.py +135 -0
- pipeline.py +312 -0
- requirements.txt +19 -0
- run_local.sh +28 -0
- upload_weights.py +106 -0
.gitattributes
CHANGED
|
@@ -1,35 +1,3 @@
|
|
| 1 |
-
*.7z filter=lfs diff=lfs merge=lfs -text
|
| 2 |
-
*.arrow filter=lfs diff=lfs merge=lfs -text
|
| 3 |
-
*.bin filter=lfs diff=lfs merge=lfs -text
|
| 4 |
-
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
| 5 |
-
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
| 6 |
-
*.ftz filter=lfs diff=lfs merge=lfs -text
|
| 7 |
-
*.gz filter=lfs diff=lfs merge=lfs -text
|
| 8 |
-
*.h5 filter=lfs diff=lfs merge=lfs -text
|
| 9 |
-
*.joblib filter=lfs diff=lfs merge=lfs -text
|
| 10 |
-
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
| 11 |
-
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
| 12 |
-
*.model filter=lfs diff=lfs merge=lfs -text
|
| 13 |
-
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
| 14 |
*.npy filter=lfs diff=lfs merge=lfs -text
|
| 15 |
-
*.
|
| 16 |
-
*.
|
| 17 |
-
*.ot filter=lfs diff=lfs merge=lfs -text
|
| 18 |
-
*.parquet filter=lfs diff=lfs merge=lfs -text
|
| 19 |
-
*.pb filter=lfs diff=lfs merge=lfs -text
|
| 20 |
-
*.pickle filter=lfs diff=lfs merge=lfs -text
|
| 21 |
-
*.pkl filter=lfs diff=lfs merge=lfs -text
|
| 22 |
-
*.pt filter=lfs diff=lfs merge=lfs -text
|
| 23 |
-
*.pth filter=lfs diff=lfs merge=lfs -text
|
| 24 |
-
*.rar filter=lfs diff=lfs merge=lfs -text
|
| 25 |
-
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
| 26 |
-
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
| 27 |
-
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
| 28 |
-
*.tar filter=lfs diff=lfs merge=lfs -text
|
| 29 |
-
*.tflite filter=lfs diff=lfs merge=lfs -text
|
| 30 |
-
*.tgz filter=lfs diff=lfs merge=lfs -text
|
| 31 |
-
*.wasm filter=lfs diff=lfs merge=lfs -text
|
| 32 |
-
*.xz filter=lfs diff=lfs merge=lfs -text
|
| 33 |
-
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
-
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
-
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
*.npy filter=lfs diff=lfs merge=lfs -text
|
| 2 |
+
*.bin filter=lfs diff=lfs merge=lfs -text
|
| 3 |
+
*.tif filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
.gitignore
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
__pycache__/
|
| 2 |
+
*.pyc
|
| 3 |
+
.gradio/
|
| 4 |
+
token
|
| 5 |
+
*.token
|
| 6 |
+
.env
|
README.md
CHANGED
|
@@ -1,13 +1,116 @@
|
|
| 1 |
---
|
| 2 |
-
title: CELL
|
| 3 |
-
emoji:
|
| 4 |
-
colorFrom:
|
| 5 |
-
colorTo:
|
| 6 |
sdk: gradio
|
| 7 |
-
sdk_version:
|
| 8 |
-
python_version: '3.12'
|
| 9 |
app_file: app.py
|
| 10 |
pinned: false
|
|
|
|
| 11 |
---
|
| 12 |
|
| 13 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
---
|
| 2 |
+
title: CELL-FM
|
| 3 |
+
emoji: 🧬
|
| 4 |
+
colorFrom: blue
|
| 5 |
+
colorTo: indigo
|
| 6 |
sdk: gradio
|
| 7 |
+
sdk_version: 4.44.0
|
|
|
|
| 8 |
app_file: app.py
|
| 9 |
pinned: false
|
| 10 |
+
short_description: Predict condensate titration curves from an IDR sequence
|
| 11 |
---
|
| 12 |
|
| 13 |
+
# CELL-FM · CondenSeq
|
| 14 |
+
|
| 15 |
+
Predict how an intrinsically disordered region behaves as its concentration rises,
|
| 16 |
+
directly from sequence.
|
| 17 |
+
|
| 18 |
+
Enter a sequence and the demo runs the three stages of the CELL-FM condensate
|
| 19 |
+
pipeline end to end:
|
| 20 |
+
|
| 21 |
+
1. **Generate.** CELL-FM CS samples one microscopy image per protein-intensity
|
| 22 |
+
level, over a log-spaced concentration ladder from 20 to 2048 a.u. Every image
|
| 23 |
+
is conditioned on the same fixed reference nucleus, so curves are comparable
|
| 24 |
+
across sequences.
|
| 25 |
+
2. **Classify.** A ViT calls each generated image **condensed** or **diffuse**.
|
| 26 |
+
3. **Summarise.** A centred moving average over those binary calls is the
|
| 27 |
+
**titration curve**; integrating it against log-intensity gives the two numbers
|
| 28 |
+
the demo reports.
|
| 29 |
+
|
| 30 |
+
## What AUC and AAC mean
|
| 31 |
+
|
| 32 |
+
**AUC** — the area under the titration curve. How much of the scanned concentration
|
| 33 |
+
range the sequence spends in the condensed state. Higher means it condenses more
|
| 34 |
+
readily, over a wider range.
|
| 35 |
+
|
| 36 |
+
**AAC** — the area *above* the curve. The curve is compared against the one it would
|
| 37 |
+
have followed without **reentrant dissolution**: everything past the curve's maximum
|
| 38 |
+
is held flat at that maximum, and AAC is the area between the two. It is zero when
|
| 39 |
+
the sequence condenses and stays condensed, and grows the more the sequence
|
| 40 |
+
re-dissolves at high concentration.
|
| 41 |
+
|
| 42 |
+
Both are normalised by the width of the log-intensity range, so both lie in [0, 1]
|
| 43 |
+
and are directly comparable between sequences.
|
| 44 |
+
|
| 45 |
+
Also reported: **c_sat**, the first intensity at which the smoothed curve reaches
|
| 46 |
+
probability 0.8, and the raw per-image calls as a downloadable CSV.
|
| 47 |
+
|
| 48 |
+
## Presets
|
| 49 |
+
|
| 50 |
+
The dropdown is loaded with NUP98's FG-repeat region — wild type plus its full
|
| 51 |
+
mutation panel (F→S, F→Y, charge and spacer variants). Comparing a mutant's AUC
|
| 52 |
+
against WT is the quickest way to see the model respond to sequence grammar.
|
| 53 |
+
|
| 54 |
+
## Hardware
|
| 55 |
+
|
| 56 |
+
**A GPU is required in practice.** The code itself runs on CPU — `flash-attn` is
|
| 57 |
+
deliberately left out of `requirements.txt`, so ESM-C uses its pure-torch rotary
|
| 58 |
+
fallback (verified bit-identical to the Triton path: same AUC, same AAC, same
|
| 59 |
+
256/256 per-image calls) — but a CPU run of the defaults would take hours and
|
| 60 |
+
time out. Use ZeroGPU or a GPU hardware tier.
|
| 61 |
+
|
| 62 |
+
The default settings — 512 images at 100 ODE steps — take **~85 s on an A40**
|
| 63 |
+
(measured: 11.5 s per batch of 64, plus ~9 s of one-off model loading). Runtime
|
| 64 |
+
scales linearly in both settings; lower either in **Sampling settings** for a
|
| 65 |
+
faster, noisier curve.
|
| 66 |
+
|
| 67 |
+
On ZeroGPU the per-call budget is set by `ZEROGPU_DURATION` (default 300 s), which
|
| 68 |
+
covers the defaults with room to spare.
|
| 69 |
+
|
| 70 |
+
## Running it without a Space
|
| 71 |
+
|
| 72 |
+
Creating a Gradio Space under an organization now needs a Team or Enterprise plan.
|
| 73 |
+
Until `HuangLab` has one, `run_local.sh` serves the identical app from a cluster GPU
|
| 74 |
+
over a temporary public URL:
|
| 75 |
+
|
| 76 |
+
```bash
|
| 77 |
+
sbatch run_local.sh # prints a *.gradio.live URL, valid 72 h
|
| 78 |
+
```
|
| 79 |
+
|
| 80 |
+
## Weights
|
| 81 |
+
|
| 82 |
+
The Space downloads ~2.1 GB of checkpoints at startup from
|
| 83 |
+
[`BoHuangLab/CELL-FM`](https://huggingface.co/BoHuangLab/CELL-FM), overridable with
|
| 84 |
+
`CELLFM_MODEL_REPO`. That repo keeps one subfolder per model family:
|
| 85 |
+
|
| 86 |
+
| File | Model |
|
| 87 |
+
|---|---|
|
| 88 |
+
| `condenseq/cellfm_seq2img.bin` | CELL-FM CS generator, ESM-C 600M encoder included |
|
| 89 |
+
| `condenseq/vae.bin` | 160×160 image VAE |
|
| 90 |
+
| `condenseq/vit_cls.bin` | Condensed/diffuse ViT classifier |
|
| 91 |
+
|
| 92 |
+
To run against a local copy instead, set `CELLFM_LOCAL_WEIGHTS` to a **flat**
|
| 93 |
+
directory holding the three `.bin` files — the prefix is stripped for local paths.
|
| 94 |
+
`upload_weights.py` pushes them from the cluster.
|
| 95 |
+
|
| 96 |
+
## Provenance
|
| 97 |
+
|
| 98 |
+
Ported from the CELL-FM repository, one module per stage, with behaviour preserved:
|
| 99 |
+
|
| 100 |
+
| Stage | Origin |
|
| 101 |
+
|---|---|
|
| 102 |
+
| Generation | `cell_fm/tasks_local/cell_fm_cs/evaluate_seq2img_dict.py` |
|
| 103 |
+
| Classification | `cell_fm/tasks_local/vit_cls_condenseq_img/evaluate_single_img.py` |
|
| 104 |
+
| Moving average | `cell_fm/tasks_local/vit_cls_condenseq_img/ma_plot.py` |
|
| 105 |
+
| AUC / AAC | `cell_diff/tasks/analysis/{utils,ana_all_mutation_log_scale}.py` |
|
| 106 |
+
|
| 107 |
+
Model hyperparameters mirror `scripts/cell_fm_cs/evaluate_seq2img.sh` and
|
| 108 |
+
`scripts/vit_cls_condenseq_img/pretrain.sh`. The metric port in `metrics.py` was
|
| 109 |
+
checked against the original on 40 stored curves and agrees to 0.0 in both log and
|
| 110 |
+
linear mode. Generated images make the same round-trip through uint16 that the
|
| 111 |
+
offline pipeline's TIFF write imposed, so demo numbers stay comparable to published
|
| 112 |
+
ones.
|
| 113 |
+
|
| 114 |
+
Reference values from an end-to-end run at the defaults (512 images, 100 steps):
|
| 115 |
+
NUP98 WT gives AUC 0.4896, AAC 0.0001, c_sat 508; the 1F→S mutant gives AUC 0.4561
|
| 116 |
+
— losing one phenylalanine sticker costs 0.034 of condensation propensity.
|
app.py
ADDED
|
@@ -0,0 +1,233 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""CELL-FM CondenSeq demo: protein sequence -> condensate titration curve.
|
| 2 |
+
|
| 3 |
+
Enter an IDR sequence; the app generates a concentration ladder of images with
|
| 4 |
+
CELL-FM, classifies each as condensed or diffuse, smooths the calls into a
|
| 5 |
+
titration curve, and integrates it into AUC (condensation propensity) and AAC
|
| 6 |
+
(reentrant dissolution).
|
| 7 |
+
"""
|
| 8 |
+
|
| 9 |
+
import os
|
| 10 |
+
|
| 11 |
+
import gradio as gr
|
| 12 |
+
import matplotlib
|
| 13 |
+
matplotlib.use("Agg")
|
| 14 |
+
import matplotlib.pyplot as plt
|
| 15 |
+
import numpy as np
|
| 16 |
+
import pandas as pd
|
| 17 |
+
|
| 18 |
+
import metrics
|
| 19 |
+
import pipeline
|
| 20 |
+
|
| 21 |
+
# ZeroGPU: only present on HF Spaces hardware, so the import is optional.
|
| 22 |
+
try:
|
| 23 |
+
import spaces
|
| 24 |
+
|
| 25 |
+
GPU_DECORATOR = spaces.GPU(duration=int(os.environ.get("ZEROGPU_DURATION", "300")))
|
| 26 |
+
except Exception: # running locally or on a dedicated GPU
|
| 27 |
+
def GPU_DECORATOR(fn):
|
| 28 |
+
return fn
|
| 29 |
+
|
| 30 |
+
HERE = os.path.dirname(os.path.abspath(__file__))
|
| 31 |
+
|
| 32 |
+
# palette: categorical slots 1 and 2 (validated, CVD dE 24.7 / normal dE 33.6)
|
| 33 |
+
CURVE = "#2a78d6" # the titration curve and the area under it
|
| 34 |
+
GAP = "#eb6834" # the area above the curve
|
| 35 |
+
INK = "#0b0b0b"
|
| 36 |
+
MUTED = "#52514e"
|
| 37 |
+
SURFACE = "#fcfcfb"
|
| 38 |
+
|
| 39 |
+
MAX_SEQUENCE_LEN = 2048
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def load_presets():
|
| 43 |
+
"""NUP98 WT plus its mutation panel, as demo starting points."""
|
| 44 |
+
df = pd.read_csv(os.path.join(HERE, "assets", "nup98_mutation_seqs.csv"), dtype=str)
|
| 45 |
+
return {f"NUP98 {row.mutation}": row.sequence for row in df.itertuples(index=False)}
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
PRESETS = load_presets()
|
| 49 |
+
DEFAULT_SEQUENCE = PRESETS["NUP98 WT"]
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def plot_curve(result: pipeline.Result):
|
| 53 |
+
"""The titration curve, with the two integrated areas drawn as the areas they are."""
|
| 54 |
+
x, y, curve = result.intensities, result.predictions, result.curve
|
| 55 |
+
flat = metrics.no_reentrant_curve(curve)
|
| 56 |
+
|
| 57 |
+
fig, ax = plt.subplots(figsize=(8.5, 4.6), facecolor=SURFACE)
|
| 58 |
+
ax.set_facecolor(SURFACE)
|
| 59 |
+
|
| 60 |
+
# the raw per-image calls, kept visible but recessive: the curve is a summary of these
|
| 61 |
+
ax.scatter(x, y, s=7, alpha=0.16, color=MUTED, linewidths=0, zorder=1)
|
| 62 |
+
|
| 63 |
+
# AUC: what the sequence actually does
|
| 64 |
+
ax.fill_between(x, 0, curve, color=CURVE, alpha=0.18, linewidth=0, zorder=2,
|
| 65 |
+
label=f"AUC {result.auc:.3f}")
|
| 66 |
+
# AAC: the condensation lost to reentrant dissolution at high concentration
|
| 67 |
+
if result.aac > 1e-9:
|
| 68 |
+
ax.fill_between(x, curve, flat, where=flat > curve, color=GAP, alpha=0.30,
|
| 69 |
+
linewidth=0, zorder=3, label=f"AAC {result.aac:.3f}")
|
| 70 |
+
ax.plot(x, flat, color=GAP, linewidth=1.2, linestyle="--", alpha=0.8, zorder=4)
|
| 71 |
+
|
| 72 |
+
ax.plot(x, curve, color=CURVE, linewidth=2.0, zorder=5)
|
| 73 |
+
|
| 74 |
+
ax.set_xscale("log")
|
| 75 |
+
ax.set_xlim(x.min(), x.max())
|
| 76 |
+
ax.set_ylim(-0.02, 1.02)
|
| 77 |
+
ax.set_xlabel("Protein intensity level (a.u., log scale)", fontsize=11, color=INK)
|
| 78 |
+
ax.set_ylabel("Condensate probability", fontsize=11, color=INK)
|
| 79 |
+
ax.tick_params(labelsize=10, colors=MUTED)
|
| 80 |
+
ax.grid(True, alpha=0.18, linewidth=0.8)
|
| 81 |
+
ax.set_axisbelow(True)
|
| 82 |
+
for side in ("top", "right"):
|
| 83 |
+
ax.spines[side].set_visible(False)
|
| 84 |
+
for side in ("left", "bottom"):
|
| 85 |
+
ax.spines[side].set_color(MUTED)
|
| 86 |
+
ax.spines[side].set_linewidth(0.8)
|
| 87 |
+
|
| 88 |
+
if result.c_sat != float("inf"):
|
| 89 |
+
ax.axvline(result.c_sat, color=MUTED, linewidth=1.0, linestyle=":", zorder=4)
|
| 90 |
+
ax.annotate(
|
| 91 |
+
f"$c_{{sat}}$ {result.c_sat:.0f}",
|
| 92 |
+
xy=(result.c_sat, 1.0), xytext=(6, -13), textcoords="offset points",
|
| 93 |
+
fontsize=9, color=MUTED, ha="left", va="top",
|
| 94 |
+
)
|
| 95 |
+
|
| 96 |
+
ax.legend(loc="upper left", frameon=False, fontsize=10, labelcolor=INK)
|
| 97 |
+
fig.tight_layout()
|
| 98 |
+
return fig
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
def plot_samples(result: pipeline.Result):
|
| 102 |
+
"""A strip of generated images across the ladder, so the curve is checkable by eye."""
|
| 103 |
+
imgs, xs = result.images, result.image_intensities
|
| 104 |
+
fig, axes = plt.subplots(1, len(imgs), figsize=(2.0 * len(imgs), 2.3), facecolor=SURFACE)
|
| 105 |
+
|
| 106 |
+
for ax, img, level in zip(np.atleast_1d(axes), imgs, xs):
|
| 107 |
+
ax.imshow(img, cmap="magma", vmin=0, vmax=1)
|
| 108 |
+
ax.set_title(f"{level:.0f}", fontsize=10, color=INK)
|
| 109 |
+
ax.set_xticks([]); ax.set_yticks([])
|
| 110 |
+
for s in ax.spines.values():
|
| 111 |
+
s.set_visible(False)
|
| 112 |
+
|
| 113 |
+
fig.suptitle("Generated protein channel across the concentration ladder (a.u.)",
|
| 114 |
+
fontsize=10, color=MUTED, y=0.04)
|
| 115 |
+
fig.tight_layout(rect=(0, 0.06, 1, 1))
|
| 116 |
+
return fig
|
| 117 |
+
|
| 118 |
+
|
| 119 |
+
def summary_markdown(result: pipeline.Result) -> str:
|
| 120 |
+
csat = "not reached" if result.c_sat == float("inf") else f"{result.c_sat:.0f}"
|
| 121 |
+
return f"""
|
| 122 |
+
| | |
|
| 123 |
+
|---|---|
|
| 124 |
+
| **AUC** — area under the curve, condensation propensity | **{result.auc:.4f}** |
|
| 125 |
+
| **AAC** — area above the curve, reentrant dissolution | **{result.aac:.4f}** |
|
| 126 |
+
| No-reentrant AUC | {result.no_reentrant_auc:.4f} |
|
| 127 |
+
| c<sub>sat</sub> (first intensity at P ≥ 0.8) | {csat} |
|
| 128 |
+
| Condensed calls | {int(result.predictions.sum())} / {len(result.predictions)} |
|
| 129 |
+
| Moving-average window | {result.window} |
|
| 130 |
+
| Sequence length | {len(result.sequence)} aa |
|
| 131 |
+
|
| 132 |
+
Both areas are normalised by the width of the log-intensity range, so they lie in
|
| 133 |
+
[0, 1] and are comparable across sequences.
|
| 134 |
+
"""
|
| 135 |
+
|
| 136 |
+
|
| 137 |
+
def results_table(result: pipeline.Result) -> pd.DataFrame:
|
| 138 |
+
"""The numbers behind the plot, so the chart is never the only way to read them."""
|
| 139 |
+
return pd.DataFrame({
|
| 140 |
+
"protein_intensity_level": np.round(result.intensities, 4),
|
| 141 |
+
"predicted_class": result.predictions,
|
| 142 |
+
"condensate_probability_ma": np.round(result.curve, 6),
|
| 143 |
+
})
|
| 144 |
+
|
| 145 |
+
|
| 146 |
+
@GPU_DECORATOR
|
| 147 |
+
def analyse(sequence, num_images, num_steps, batch_size, seed, progress=gr.Progress()):
|
| 148 |
+
sequence = pipeline.clean_sequence(sequence)
|
| 149 |
+
if len(sequence) > MAX_SEQUENCE_LEN:
|
| 150 |
+
raise gr.Error(f"Sequence is {len(sequence)} aa; the model takes at most {MAX_SEQUENCE_LEN}.")
|
| 151 |
+
|
| 152 |
+
result = pipeline.run(
|
| 153 |
+
sequence,
|
| 154 |
+
num_images=int(num_images),
|
| 155 |
+
num_steps=int(num_steps),
|
| 156 |
+
batch_size=int(batch_size),
|
| 157 |
+
seed=int(seed),
|
| 158 |
+
progress=progress,
|
| 159 |
+
)
|
| 160 |
+
|
| 161 |
+
table = results_table(result)
|
| 162 |
+
csv_path = os.path.join("/tmp", "pred_vs_intensity.csv")
|
| 163 |
+
table.to_csv(csv_path, index=False)
|
| 164 |
+
|
| 165 |
+
return plot_curve(result), plot_samples(result), summary_markdown(result), table, csv_path
|
| 166 |
+
|
| 167 |
+
|
| 168 |
+
with gr.Blocks(title="CELL-FM CondenSeq", theme=gr.themes.Soft()) as demo:
|
| 169 |
+
gr.Markdown(
|
| 170 |
+
"""
|
| 171 |
+
# CELL-FM · CondenSeq
|
| 172 |
+
|
| 173 |
+
Predict how an intrinsically disordered region behaves as its concentration rises.
|
| 174 |
+
|
| 175 |
+
CELL-FM generates a ladder of microscopy images for your sequence, one per
|
| 176 |
+
protein-intensity level; a ViT classifier calls each **condensed** or **diffuse**;
|
| 177 |
+
a moving average over those calls is the **titration curve**. Integrating it gives:
|
| 178 |
+
|
| 179 |
+
* **AUC** — how much of the concentration range the sequence spends condensed.
|
| 180 |
+
* **AAC** — the area above the curve, i.e. condensation lost to **reentrant
|
| 181 |
+
dissolution** at high concentration. Zero when the curve never comes back down.
|
| 182 |
+
"""
|
| 183 |
+
)
|
| 184 |
+
|
| 185 |
+
with gr.Row():
|
| 186 |
+
with gr.Column(scale=2):
|
| 187 |
+
preset = gr.Dropdown(
|
| 188 |
+
choices=list(PRESETS), value="NUP98 WT", label="Preset",
|
| 189 |
+
info="NUP98 FG-repeat WT and its mutation panel. Editing the box below overrides this.",
|
| 190 |
+
)
|
| 191 |
+
sequence = gr.Textbox(
|
| 192 |
+
value=DEFAULT_SEQUENCE, lines=4, label="Protein sequence",
|
| 193 |
+
info="Single-letter amino acids; FASTA headers are stripped.",
|
| 194 |
+
)
|
| 195 |
+
with gr.Accordion("Sampling settings", open=False):
|
| 196 |
+
num_images = gr.Slider(64, 1024, value=512, step=64, label="Images (concentration levels)",
|
| 197 |
+
info="More levels give a smoother curve and cost proportionally more time.")
|
| 198 |
+
num_steps = gr.Slider(20, 200, value=100, step=10, label="ODE steps per image")
|
| 199 |
+
batch_size = gr.Slider(16, 256, value=64, step=16, label="Batch size")
|
| 200 |
+
seed = gr.Number(value=pipeline.DEFAULT_SEED, precision=0, label="Random seed",
|
| 201 |
+
info="Generation starts from a random latent; fix the seed for a repeatable curve.")
|
| 202 |
+
run_button = gr.Button("Run analysis", variant="primary")
|
| 203 |
+
|
| 204 |
+
with gr.Column(scale=3):
|
| 205 |
+
summary = gr.Markdown()
|
| 206 |
+
|
| 207 |
+
curve_plot = gr.Plot(label="Condensate titration curve")
|
| 208 |
+
sample_plot = gr.Plot(label="Generated images")
|
| 209 |
+
|
| 210 |
+
with gr.Accordion("Per-image results", open=False):
|
| 211 |
+
table = gr.Dataframe(label="pred_vs_intensity", wrap=True)
|
| 212 |
+
download = gr.File(label="Download CSV")
|
| 213 |
+
|
| 214 |
+
gr.Markdown(
|
| 215 |
+
"""
|
| 216 |
+
---
|
| 217 |
+
Generation follows `evaluate_seq2img_dict.py`, classification `evaluate_single_img.py`,
|
| 218 |
+
smoothing `ma_plot.py`, and the areas `analysis/ana_all_mutation_log_scale.py` from the
|
| 219 |
+
CELL-FM repository. Every image is conditioned on one fixed reference nucleus, so curves
|
| 220 |
+
are comparable across sequences.
|
| 221 |
+
"""
|
| 222 |
+
)
|
| 223 |
+
|
| 224 |
+
preset.change(lambda name: PRESETS[name], inputs=preset, outputs=sequence)
|
| 225 |
+
run_button.click(
|
| 226 |
+
analyse,
|
| 227 |
+
inputs=[sequence, num_images, num_steps, batch_size, seed],
|
| 228 |
+
outputs=[curve_plot, sample_plot, summary, table, download],
|
| 229 |
+
)
|
| 230 |
+
|
| 231 |
+
|
| 232 |
+
if __name__ == "__main__":
|
| 233 |
+
demo.queue().launch(share=os.environ.get("GRADIO_SHARE") == "1")
|
assets/nup98_mutation_seqs.csv
ADDED
|
@@ -0,0 +1,93 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
mutation,sequence
|
| 2 |
+
WT,KSFGTPFGGGTGGFGTTSTFGQNTGFGTTSGGAFGTSAFGSSNNTGGLFGNSQTKPGGLFGTSSFS
|
| 3 |
+
1FtoS_1,KSSGTPFGGGTGGFGTTSTFGQNTGFGTTSGGAFGTSAFGSSNNTGGLFGNSQTKPGGLFGTSSFS
|
| 4 |
+
1FtoS_2,KSFGTPSGGGTGGFGTTSTFGQNTGFGTTSGGAFGTSAFGSSNNTGGLFGNSQTKPGGLFGTSSFS
|
| 5 |
+
1FtoS_3,KSFGTPFGGGTGGSGTTSTFGQNTGFGTTSGGAFGTSAFGSSNNTGGLFGNSQTKPGGLFGTSSFS
|
| 6 |
+
1FtoS_4,KSFGTPFGGGTGGFGTTSTSGQNTGFGTTSGGAFGTSAFGSSNNTGGLFGNSQTKPGGLFGTSSFS
|
| 7 |
+
1FtoS_5,KSFGTPFGGGTGGFGTTSTFGQNTGSGTTSGGAFGTSAFGSSNNTGGLFGNSQTKPGGLFGTSSFS
|
| 8 |
+
1FtoS_6,KSFGTPFGGGTGGFGTTSTFGQNTGFGTTSGGASGTSAFGSSNNTGGLFGNSQTKPGGLFGTSSFS
|
| 9 |
+
1FtoS_7,KSFGTPFGGGTGGFGTTSTFGQNTGFGTTSGGAFGTSASGSSNNTGGLFGNSQTKPGGLFGTSSFS
|
| 10 |
+
1FtoS_8,KSFGTPFGGGTGGFGTTSTFGQNTGFGTTSGGAFGTSAFGSSNNTGGLSGNSQTKPGGLFGTSSFS
|
| 11 |
+
1FtoS_9,KSFGTPFGGGTGGFGTTSTFGQNTGFGTTSGGAFGTSAFGSSNNTGGLFGNSQTKPGGLSGTSSFS
|
| 12 |
+
1FtoS_10,KSFGTPFGGGTGGFGTTSTFGQNTGFGTTSGGAFGTSAFGSSNNTGGLFGNSQTKPGGLFGTSSSS
|
| 13 |
+
2FtoS_1,KSFGTPFGGGTGGFGTTSTFGQNTGFGTTSGGAFGTSASGSSNNTGGLFGNSQTKPGGLSGTSSFS
|
| 14 |
+
2FtoS_2,KSSGTPFGGGTGGFGTTSTFGQNTGFGTTSGGAFGTSAFGSSNNTGGLFGNSQTKPGGLSGTSSFS
|
| 15 |
+
2FtoS_3,KSSGTPFGGGTGGSGTTSTFGQNTGFGTTSGGAFGTSAFGSSNNTGGLFGNSQTKPGGLFGTSSFS
|
| 16 |
+
2FtoS_4,KSFGTPFGGGTGGSGTTSTSGQNTGFGTTSGGAFGTSAFGSSNNTGGLFGNSQTKPGGLFGTSSFS
|
| 17 |
+
2FtoS_5,KSFGTPSGGGTGGFGTTSTFGQNTGFGTTSGGAFGTSAFGSSNNTGGLFGNSQTKPGGLSGTSSFS
|
| 18 |
+
2FtoS_6,KSFGTPSGGGTGGFGTTSTFGQNTGFGTTSGGAFGTSAFGSSNNTGGLSGNSQTKPGGLFGTSSFS
|
| 19 |
+
2FtoS_7,KSSGTPFGGGTGGFGTTSTFGQNTGFGTTSGGAFGTSAFGSSNNTGGLFGNSQTKPGGLFGTSSSS
|
| 20 |
+
2FtoS_8,KSSGTPFGGGTGGFGTTSTFGQNTGFGTTSGGAFGTSAFGSSNNTGGLSGNSQTKPGGLFGTSSFS
|
| 21 |
+
2FtoS_9,KSFGTPFGGGTGGFGTTSTFGQNTGSGTTSGGAFGTSAFGSSNNTGGLFGNSQTKPGGLFGTSSSS
|
| 22 |
+
2FtoS_10,KSSGTPFGGGTGGFGTTSTFGQNTGFGTTSGGAFGTSASGSSNNTGGLFGNSQTKPGGLFGTSSFS
|
| 23 |
+
3FtoS_1,KSFGTPFGGGTGGSGTTSTFGQNTGFGTTSGGASGTSASGSSNNTGGLFGNSQTKPGGLFGTSSFS
|
| 24 |
+
3FtoS_2,KSFGTPSGGGTGGFGTTSTFGQNTGFGTTSGGASGTSASGSSNNTGGLFGNSQTKPGGLFGTSSFS
|
| 25 |
+
3FtoS_3,KSSGTPSGGGTGGFGTTSTFGQNTGFGTTSGGAFGTSASGSSNNTGGLFGNSQTKPGGLFGTSSFS
|
| 26 |
+
3FtoS_4,KSSGTPSGGGTGGFGTTSTFGQNTGFGTTSGGASGTSAFGSSNNTGGLFGNSQTKPGGLFGTSSFS
|
| 27 |
+
3FtoS_5,KSSGTPFGGGTGGSGTTSTFGQNTGFGTTSGGAFGTSASGSSNNTGGLFGNSQTKPGGLFGTSSFS
|
| 28 |
+
3FtoS_6,KSSGTPFGGGTGGFGTTSTFGQNTGFGTTSGGASGTSAFGSSNNTGGLSGNSQTKPGGLFGTSSFS
|
| 29 |
+
3FtoS_7,KSSGTPFGGGTGGFGTTSTFGQNTGFGTTSGGASGTSAFGSSNNTGGLFGNSQTKPGGLFGTSSSS
|
| 30 |
+
3FtoS_8,KSFGTPFGGGTGGSGTTSTSGQNTGSGTTSGGAFGTSAFGSSNNTGGLFGNSQTKPGGLFGTSSFS
|
| 31 |
+
3FtoS_9,KSFGTPFGGGTGGSGTTSTFGQNTGFGTTSGGASGTSAFGSSNNTGGLFGNSQTKPGGLSGTSSFS
|
| 32 |
+
3FtoS_10,KSFGTPFGGGTGGSGTTSTFGQNTGSGTTSGGAFGTSASGSSNNTGGLFGNSQTKPGGLFGTSSFS
|
| 33 |
+
4FtoS_1,KSSGTPFGGGTGGFGTTSTSGQNTGSGTTSGGAFGTSASGSSNNTGGLFGNSQTKPGGLFGTSSFS
|
| 34 |
+
4FtoS_2,KSFGTPFGGGTGGFGTTSTSGQNTGSGTTSGGAFGTSAFGSSNNTGGLSGNSQTKPGGLFGTSSSS
|
| 35 |
+
4FtoS_3,KSFGTPFGGGTGGSGTTSTFGQNTGFGTTSGGASGTSASGSSNNTGGLFGNSQTKPGGLSGTSSFS
|
| 36 |
+
4FtoS_4,KSFGTPFGGGTGGFGTTSTSGQNTGSGTTSGGAFGTSASGSSNNTGGLSGNSQTKPGGLFGTSSFS
|
| 37 |
+
4FtoS_5,KSFGTPSGGGTGGFGTTSTFGQNTGFGTTSGGAFGTSAFGSSNNTGGLSGNSQTKPGGLSGTSSSS
|
| 38 |
+
4FtoS_6,KSFGTPSGGGTGGFGTTSTSGQNTGSGTTSGGAFGTSAFGSSNNTGGLSGNSQTKPGGLFGTSSFS
|
| 39 |
+
4FtoS_7,KSSGTPFGGGTGGFGTTSTSGQNTGFGTTSGGASGTSAFGSSNNTGGLFGNSQTKPGGLSGTSSFS
|
| 40 |
+
4FtoS_8,KSFGTPSGGGTGGFGTTSTSGQNTGFGTTSGGAFGTSASGSSNNTGGLSGNSQTKPGGLFGTSSFS
|
| 41 |
+
4FtoS_9,KSFGTPFGGGTGGSGTTSTSGQNTGFGTTSGGAFGTSASGSSNNTGGLFGNSQTKPGGLSGTSSFS
|
| 42 |
+
4FtoS_10,KSSGTPFGGGTGGFGTTSTFGQNTGSGTTSGGAFGTSAFGSSNNTGGLSGNSQTKPGGLSGTSSFS
|
| 43 |
+
5FtoS_1,KSFGTPFGGGTGGSGTTSTSGQNTGFGTTSGGASGTSASGSSNNTGGLFGNSQTKPGGLSGTSSFS
|
| 44 |
+
5FtoS_2,KSFGTPFGGGTGGSGTTSTFGQNTGSGTTSGGAFGTSASGSSNNTGGLSGNSQTKPGGLSGTSSFS
|
| 45 |
+
5FtoS_3,KSSGTPSGGGTGGSGTTSTSGQNTGFGTTSGGASGTSAFGSSNNTGGLFGNSQTKPGGLFGTSSFS
|
| 46 |
+
5FtoS_4,KSFGTPSGGGTGGFGTTSTFGQNTGFGTTSGGASGTSAFGSSNNTGGLSGNSQTKPGGLSGTSSSS
|
| 47 |
+
5FtoS_5,KSFGTPFGGGTGGSGTTSTSGQNTGFGTTSGGASGTSASGSSNNTGGLSGNSQTKPGGLFGTSSFS
|
| 48 |
+
5FtoS_6,KSSGTPSGGGTGGFGTTSTFGQNTGSGTTSGGAFGTSASGSSNNTGGLSGNSQTKPGGLFGTSSFS
|
| 49 |
+
5FtoS_7,KSFGTPSGGGTGGFGTTSTSGQNTGFGTTSGGAFGTSASGSSNNTGGLSGNSQTKPGGLFGTSSSS
|
| 50 |
+
5FtoS_8,KSSGTPFGGGTGGFGTTSTSGQNTGFGTTSGGAFGTSASGSSNNTGGLSGNSQTKPGGLFGTSSSS
|
| 51 |
+
5FtoS_9,KSSGTPFGGGTGGSGTTSTFGQNTGFGTTSGGAFGTSASGSSNNTGGLSGNSQTKPGGLSGTSSFS
|
| 52 |
+
5FtoS_10,KSSGTPFGGGTGGSGTTSTFGQNTGSGTTSGGASGTSASGSSNNTGGLFGNSQTKPGGLFGTSSFS
|
| 53 |
+
6FtoS_1,KSSGTPSGGGTGGFGTTSTSGQNTGSGTTSGGAFGTSASGSSNNTGGLSGNSQTKPGGLFGTSSFS
|
| 54 |
+
6FtoS_2,KSSGTPSGGGTGGFGTTSTFGQNTGSGTTSGGASGTSASGSSNNTGGLSGNSQTKPGGLFGTSSFS
|
| 55 |
+
6FtoS_3,KSFGTPFGGGTGGSGTTSTSGQNTGFGTTSGGASGTSAFGSSNNTGGLSGNSQTKPGGLSGTSSSS
|
| 56 |
+
6FtoS_4,KSSGTPFGGGTGGSGTTSTSGQNTGFGTTSGGAFGTSASGSSNNTGGLSGNSQTKPGGLSGTSSFS
|
| 57 |
+
6FtoS_5,KSSGTPSGGGTGGSGTTSTFGQNTGFGTTSGGASGTSASGSSNNTGGLFGNSQTKPGGLSGTSSFS
|
| 58 |
+
6FtoS_6,KSSGTPSGGGTGGSGTTSTFGQNTGSGTTSGGAFGTSAFGSSNNTGGLSGNSQTKPGGLFGTSSSS
|
| 59 |
+
6FtoS_7,KSSGTPFGGGTGGSGTTSTFGQNTGSGTTSGGAFGTSASGSSNNTGGLSGNSQTKPGGLFGTSSSS
|
| 60 |
+
6FtoS_8,KSSGTPSGGGTGGSGTTSTFGQNTGSGTTSGGAFGTSAFGSSNNTGGLFGNSQTKPGGLSGTSSSS
|
| 61 |
+
6FtoS_9,KSSGTPFGGGTGGSGTTSTFGQNTGSGTTSGGASGTSASGSSNNTGGLFGNSQTKPGGLSGTSSFS
|
| 62 |
+
6FtoS_10,KSSGTPFGGGTGGSGTTSTSGQNTGFGTTSGGAFGTSASGSSNNTGGLFGNSQTKPGGLSGTSSSS
|
| 63 |
+
7FtoS_1,KSSGTPFGGGTGGFGTTSTSGQNTGSGTTSGGASGTSASGSSNNTGGLSGNSQTKPGGLSGTSSFS
|
| 64 |
+
7FtoS_2,KSSGTPSGGGTGGSGTTSTFGQNTGFGTTSGGASGTSAFGSSNNTGGLSGNSQTKPGGLSGTSSSS
|
| 65 |
+
7FtoS_3,KSFGTPSGGGTGGSGTTSTFGQNTGSGTTSGGAFGTSASGSSNNTGGLSGNSQTKPGGLSGTSSSS
|
| 66 |
+
7FtoS_4,KSSGTPSGGGTGGSGTTSTSGQNTGSGTTSGGAFGTSASGSSNNTGGLFGNSQTKPGGLSGTSSFS
|
| 67 |
+
7FtoS_5,KSFGTPSGGGTGGSGTTSTSGQNTGSGTTSGGAFGTSAFGSSNNTGGLSGNSQTKPGGLSGTSSSS
|
| 68 |
+
7FtoS_6,KSSGTPFGGGTGGSGTTSTSGQNTGSGTTSGGASGTSASGSSNNTGGLFGNSQTKPGGLFGTSSSS
|
| 69 |
+
7FtoS_7,KSSGTPFGGGTGGSGTTSTSGQNTGFGTTSGGASGTSASGSSNNTGGLFGNSQTKPGGLSGTSSSS
|
| 70 |
+
7FtoS_8,KSSGTPSGGGTGGSGTTSTSGQNTGFGTTSGGASGTSAFGSSNNTGGLFGNSQTKPGGLSGTSSSS
|
| 71 |
+
7FtoS_9,KSFGTPFGGGTGGSGTTSTFGQNTGSGTTSGGASGTSASGSSNNTGGLSGNSQTKPGGLSGTSSSS
|
| 72 |
+
7FtoS_10,KSSGTPSGGGTGGFGTTSTSGQNTGFGTTSGGASGTSAFGSSNNTGGLSGNSQTKPGGLSGTSSSS
|
| 73 |
+
8FtoS_1,KSSGTPSGGGTGGSGTTSTSGQNTGSGTTSGGASGTSAFGSSNNTGGLFGNSQTKPGGLSGTSSSS
|
| 74 |
+
8FtoS_2,KSSGTPFGGGTGGFGTTSTSGQNTGSGTTSGGASGTSASGSSNNTGGLSGNSQTKPGGLSGTSSSS
|
| 75 |
+
8FtoS_3,KSSGTPSGGGTGGSGTTSTFGQNTGSGTTSGGASGTSAFGSSNNTGGLSGNSQTKPGGLSGTSSSS
|
| 76 |
+
8FtoS_4,KSFGTPSGGGTGGSGTTSTSGQNTGSGTTSGGAFGTSASGSSNNTGGLSGNSQTKPGGLSGTSSSS
|
| 77 |
+
8FtoS_5,KSFGTPSGGGTGGSGTTSTSGQNTGSGTTSGGASGTSAFGSSNNTGGLSGNSQTKPGGLSGTSSSS
|
| 78 |
+
8FtoS_6,KSSGTPSGGGTGGFGTTSTSGQNTGSGTTSGGASGTSASGSSNNTGGLFGNSQTKPGGLSGTSSSS
|
| 79 |
+
8FtoS_7,KSFGTPSGGGTGGSGTTSTSGQNTGSGTTSGGASGTSASGSSNNTGGLSGNSQTKPGGLSGTSSFS
|
| 80 |
+
8FtoS_8,KSSGTPSGGGTGGSGTTSTSGQNTGFGTTSGGASGTSASGSSNNTGGLFGNSQTKPGGLSGTSSSS
|
| 81 |
+
8FtoS_9,KSSGTPSGGGTGGSGTTSTSGQNTGSGTTSGGASGTSAFGSSNNTGGLSGNSQTKPGGLFGTSSSS
|
| 82 |
+
8FtoS_10,KSSGTPSGGGTGGSGTTSTSGQNTGSGTTSGGASGTSASGSSNNTGGLFGNSQTKPGGLFGTSSSS
|
| 83 |
+
9FtoS_1,KSSGTPSGGGTGGSGTTSTSGQNTGSGTTSGGASGTSAFGSSNNTGGLSGNSQTKPGGLSGTSSSS
|
| 84 |
+
9FtoS_2,KSSGTPSGGGTGGSGTTSTSGQNTGSGTTSGGAFGTSASGSSNNTGGLSGNSQTKPGGLSGTSSSS
|
| 85 |
+
9FtoS_3,KSSGTPSGGGTGGSGTTSTSGQNTGSGTTSGGASGTSASGSSNNTGGLSGNSQTKPGGLFGTSSSS
|
| 86 |
+
9FtoS_4,KSSGTPSGGGTGGSGTTSTFGQNTGSGTTSGGASGTSASGSSNNTGGLSGNSQTKPGGLSGTSSSS
|
| 87 |
+
9FtoS_5,KSSGTPSGGGTGGFGTTSTSGQNTGSGTTSGGASGTSASGSSNNTGGLSGNSQTKPGGLSGTSSSS
|
| 88 |
+
9FtoS_6,KSSGTPSGGGTGGSGTTSTSGQNTGSGTTSGGASGTSASGSSNNTGGLSGNSQTKPGGLSGTSSFS
|
| 89 |
+
9FtoS_7,KSFGTPSGGGTGGSGTTSTSGQNTGSGTTSGGASGTSASGSSNNTGGLSGNSQTKPGGLSGTSSSS
|
| 90 |
+
9FtoS_8,KSSGTPSGGGTGGSGTTSTSGQNTGFGTTSGGASGTSASGSSNNTGGLSGNSQTKPGGLSGTSSSS
|
| 91 |
+
9FtoS_9,KSSGTPSGGGTGGSGTTSTSGQNTGSGTTSGGASGTSASGSSNNTGGLFGNSQTKPGGLSGTSSSS
|
| 92 |
+
9FtoS_10,KSSGTPFGGGTGGSGTTSTSGQNTGSGTTSGGASGTSASGSSNNTGGLSGNSQTKPGGLSGTSSSS
|
| 93 |
+
10FtoS_1,KSSGTPSGGGTGGSGTTSTSGQNTGSGTTSGGASGTSASGSSNNTGGLSGNSQTKPGGLSGTSSSS
|
assets/reference_nucleus.npy
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:7e8f385c8c6d1b40957907cc83aef90d1b9a887f0c10b4d2a8899349f2b9f1cb
|
| 3 |
+
size 102528
|
cell_fm/__init__.py
ADDED
|
File without changes
|
cell_fm/criterions/__init__.py
ADDED
|
File without changes
|
cell_fm/criterions/cell_fm/__init__.py
ADDED
|
File without changes
|
cell_fm/criterions/cell_fm/unidiffuser.py
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
import torch.nn as nn
|
| 3 |
+
|
| 4 |
+
class UniDiffCriterions(nn.Module):
|
| 5 |
+
def __init__(self, args, reduction="none") -> None:
|
| 6 |
+
super().__init__()
|
| 7 |
+
self.sequence_loss = nn.CrossEntropyLoss(reduction=reduction)
|
| 8 |
+
|
| 9 |
+
self.args = args
|
| 10 |
+
self.seq_loss_coeff = self.args.seq_loss_coeff
|
| 11 |
+
self.img_diff_loss_coeff = self.args.img_diff_loss_coeff
|
| 12 |
+
self.img_recon_loss_coeff = self.args.img_recon_loss_coeff
|
| 13 |
+
|
| 14 |
+
def forward(self, batch_data, model_output):
|
| 15 |
+
with torch.no_grad():
|
| 16 |
+
protein_seq_mask = batch_data["protein_seq_mask"].bool()
|
| 17 |
+
protein_seq = batch_data["protein_seq"][protein_seq_mask]
|
| 18 |
+
zm_label = batch_data['zm_label'].squeeze(-1).bool()
|
| 19 |
+
|
| 20 |
+
mask_num = torch.sum(protein_seq_mask.float(), dim=1, keepdim=True)
|
| 21 |
+
seq_loss_weights = torch.ones_like(protein_seq_mask, dtype=torch.float32)
|
| 22 |
+
seq_loss_weights = seq_loss_weights / mask_num
|
| 23 |
+
seq_loss_weights = seq_loss_weights * protein_seq_mask.float()
|
| 24 |
+
seq_loss_weights = seq_loss_weights[protein_seq_mask.bool()]
|
| 25 |
+
|
| 26 |
+
protein_seq_output, img_diff_loss, img_recon_loss = model_output
|
| 27 |
+
|
| 28 |
+
# Protein sequence loss
|
| 29 |
+
protein_seq_output = protein_seq_output[protein_seq_mask]
|
| 30 |
+
sequence_loss = (
|
| 31 |
+
self.sequence_loss(
|
| 32 |
+
protein_seq_output.to(torch.float32).view(-1, protein_seq_output.size(-1)),
|
| 33 |
+
protein_seq.view(-1),
|
| 34 |
+
)
|
| 35 |
+
* self.seq_loss_coeff
|
| 36 |
+
)
|
| 37 |
+
sequence_loss = sequence_loss * seq_loss_weights
|
| 38 |
+
|
| 39 |
+
if zm_label.all():
|
| 40 |
+
sequence_loss = torch.zeros(1, device=sequence_loss.device).sum()
|
| 41 |
+
else:
|
| 42 |
+
sequence_loss = sequence_loss.sum() / (~zm_label).float().sum()
|
| 43 |
+
|
| 44 |
+
# Protein image diffusion loss
|
| 45 |
+
img_diff_loss = img_diff_loss.mean() * self.img_diff_loss_coeff
|
| 46 |
+
|
| 47 |
+
# Protein image reconstruction loss
|
| 48 |
+
img_recon_loss = img_recon_loss.mean() * self.img_recon_loss_coeff
|
| 49 |
+
|
| 50 |
+
loss = sequence_loss + img_diff_loss + img_recon_loss
|
| 51 |
+
|
| 52 |
+
loss_items = {
|
| 53 |
+
'sequence_loss': (sequence_loss, self.seq_loss_coeff),
|
| 54 |
+
'image_diff_loss': (img_diff_loss, self.img_diff_loss_coeff),
|
| 55 |
+
'image_recon_loss': (img_recon_loss, self.img_recon_loss_coeff),
|
| 56 |
+
}
|
| 57 |
+
|
| 58 |
+
log_loss = {'total_loss': loss.item()}
|
| 59 |
+
for name, (val, coeff) in loss_items.items():
|
| 60 |
+
if coeff > 0:
|
| 61 |
+
log_loss[name] = val.item() / coeff
|
| 62 |
+
|
| 63 |
+
return loss, log_loss
|
cell_fm/logging/__init__.py
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# -*- coding: utf-8 -*-
|
| 2 |
+
from .loggers import MetricLogger, get_logger
|
| 3 |
+
|
| 4 |
+
logger = get_logger()
|
| 5 |
+
metric_logger = MetricLogger()
|
cell_fm/logging/loggers.py
ADDED
|
@@ -0,0 +1,201 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# -*- coding: utf-8 -*-
|
| 2 |
+
import sys
|
| 3 |
+
from dataclasses import dataclass, fields, is_dataclass
|
| 4 |
+
from typing import Dict, Union
|
| 5 |
+
|
| 6 |
+
import torch
|
| 7 |
+
from loguru import logger
|
| 8 |
+
|
| 9 |
+
from cell_fm.utils.dist_utils import is_master_node
|
| 10 |
+
from transformers import TrainerCallback
|
| 11 |
+
|
| 12 |
+
import wandb # isort:skip
|
| 13 |
+
|
| 14 |
+
handlers = {}
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def get_logger():
|
| 18 |
+
if not handlers:
|
| 19 |
+
logger.remove() # remove default handler
|
| 20 |
+
handlers["console"] = logger.add(
|
| 21 |
+
sys.stdout,
|
| 22 |
+
format="[<green>{time:YYYY-MM-DD HH:mm:ss.SSS}</green>][<cyan>{level}</cyan>]: {message}",
|
| 23 |
+
colorize=True,
|
| 24 |
+
filter=console_log_filter,
|
| 25 |
+
enqueue=True,
|
| 26 |
+
)
|
| 27 |
+
|
| 28 |
+
return logger
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
# Custom function to handle tensor attributes
|
| 32 |
+
def dataclass_to_dict(dataclass_obj: Union[dataclass, Dict]) -> Dict:
|
| 33 |
+
if isinstance(dataclass_obj, dict):
|
| 34 |
+
return dataclass_obj
|
| 35 |
+
result = {}
|
| 36 |
+
for field in fields(dataclass_obj):
|
| 37 |
+
value = getattr(dataclass_obj, field.name)
|
| 38 |
+
if isinstance(value, torch.Tensor) and not value.is_leaf:
|
| 39 |
+
result[field.name] = value.clone().detach()
|
| 40 |
+
else:
|
| 41 |
+
result[field.name] = value
|
| 42 |
+
return result
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
class MetricLogger(object):
|
| 46 |
+
def log(self, metrics, prefix=""):
|
| 47 |
+
if not is_master_node():
|
| 48 |
+
return
|
| 49 |
+
|
| 50 |
+
log_data = {}
|
| 51 |
+
if type(metrics) is dict:
|
| 52 |
+
log_data = metrics
|
| 53 |
+
elif is_dataclass(metrics):
|
| 54 |
+
log_data = dataclass_to_dict(metrics)
|
| 55 |
+
|
| 56 |
+
if "extra_output" in log_data:
|
| 57 |
+
extra_output = log_data["extra_output"]
|
| 58 |
+
if extra_output is not None:
|
| 59 |
+
for k, v in extra_output.items():
|
| 60 |
+
log_data[k] = v
|
| 61 |
+
del log_data["extra_output"]
|
| 62 |
+
|
| 63 |
+
for k in log_data:
|
| 64 |
+
if isinstance(log_data[k], torch.Tensor):
|
| 65 |
+
log_data[k] = log_data[k].detach().item()
|
| 66 |
+
|
| 67 |
+
logger.info(" | ".join([f"{k}={v:.4g}" for k, v in log_data.items()]))
|
| 68 |
+
if wandb.run is not None:
|
| 69 |
+
# Add prefix
|
| 70 |
+
if prefix:
|
| 71 |
+
log_data = {f"{prefix}/{k}": v for k, v in log_data.items()}
|
| 72 |
+
wandb.log(log_data)
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
def console_log_filter(record):
|
| 76 |
+
# For message with level INFO, we only log it on master node
|
| 77 |
+
# For others, we log it on all nodes
|
| 78 |
+
if record["level"].name != "INFO":
|
| 79 |
+
return True
|
| 80 |
+
|
| 81 |
+
return is_master_node()
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
class StableDiffusionLoggingCallback(TrainerCallback):
|
| 85 |
+
def __init__(self):
|
| 86 |
+
self.logger = logger
|
| 87 |
+
|
| 88 |
+
def on_log(self, args, state, control, logs=None, **kwargs):
|
| 89 |
+
if logs is None:
|
| 90 |
+
return
|
| 91 |
+
|
| 92 |
+
formatted_log = (
|
| 93 |
+
f"loss={logs.get('loss', 0):.4f} | grad_norm={logs.get('grad_norm', 0):.4f} | "
|
| 94 |
+
f"lr={logs.get('learning_rate', 0):.7f} | epoch={logs.get('epoch', 0):.4f} | "
|
| 95 |
+
f"global_step={state.global_step}"
|
| 96 |
+
)
|
| 97 |
+
|
| 98 |
+
self.logger.info(formatted_log)
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
class CELLFMLoggingCallback(TrainerCallback):
|
| 102 |
+
def __init__(self):
|
| 103 |
+
self.logger = logger
|
| 104 |
+
|
| 105 |
+
def on_log(self, args, state, control, logs=None, **kwargs):
|
| 106 |
+
if not logs:
|
| 107 |
+
return
|
| 108 |
+
|
| 109 |
+
# key -> format string
|
| 110 |
+
field_fmt = {
|
| 111 |
+
"loss": "{:.4f}",
|
| 112 |
+
"sequence_loss": "{:.4f}",
|
| 113 |
+
"image_diff_loss": "{:.4f}",
|
| 114 |
+
"image_recon_loss": "{:.4f}",
|
| 115 |
+
"grad_norm": "{:.4f}",
|
| 116 |
+
"learning_rate": "{:.7f}",
|
| 117 |
+
"epoch": "{:.4f}",
|
| 118 |
+
"ema_decay": "{:.6f}",
|
| 119 |
+
}
|
| 120 |
+
|
| 121 |
+
parts = []
|
| 122 |
+
for k, fmt in field_fmt.items():
|
| 123 |
+
if k in logs and logs[k] is not None:
|
| 124 |
+
v = logs[k]
|
| 125 |
+
parts.append(f"{k}={fmt.format(float(v))}")
|
| 126 |
+
|
| 127 |
+
parts.append(f"global_step={state.global_step}")
|
| 128 |
+
|
| 129 |
+
formatted_log = " | ".join(parts)
|
| 130 |
+
self.logger.info(formatted_log)
|
| 131 |
+
|
| 132 |
+
|
| 133 |
+
class CELLFMClsLoggingCallback(TrainerCallback):
|
| 134 |
+
def __init__(self):
|
| 135 |
+
self.logger = logger
|
| 136 |
+
|
| 137 |
+
def on_log(self, args, state, control, logs=None, **kwargs):
|
| 138 |
+
if logs is None:
|
| 139 |
+
return
|
| 140 |
+
|
| 141 |
+
loss = logs.get("loss", 0.0)
|
| 142 |
+
image_class_loss = logs.get("image_class_loss", 0.0)
|
| 143 |
+
grad_norm = logs.get("grad_norm", 0.0)
|
| 144 |
+
learning_rate = logs.get("learning_rate", 0.0)
|
| 145 |
+
epoch = logs.get("epoch", 0.0)
|
| 146 |
+
|
| 147 |
+
formatted_log = (
|
| 148 |
+
f"loss={loss:.4f} | image_class_loss={image_class_loss:.4f} | "
|
| 149 |
+
f"grad_norm={grad_norm:.4f} | lr={learning_rate:.7f} | "
|
| 150 |
+
f"epoch={epoch:.4f} | global_step={state.global_step}"
|
| 151 |
+
)
|
| 152 |
+
|
| 153 |
+
self.logger.info(formatted_log)
|
| 154 |
+
|
| 155 |
+
|
| 156 |
+
class VAELoggingCallback(TrainerCallback):
|
| 157 |
+
def __init__(self):
|
| 158 |
+
self.logger = logger
|
| 159 |
+
|
| 160 |
+
def on_log(self, args, state, control, logs=None, **kwargs):
|
| 161 |
+
if logs is None:
|
| 162 |
+
return
|
| 163 |
+
|
| 164 |
+
loss = logs.get("loss", 0.0)
|
| 165 |
+
recon_loss = logs.get("recon_loss", 0.0)
|
| 166 |
+
kl_loss = logs.get("kl_loss", 0.0)
|
| 167 |
+
grad_norm = logs.get("grad_norm", 0.0)
|
| 168 |
+
learning_rate = logs.get("learning_rate", 0.0)
|
| 169 |
+
epoch = logs.get("epoch", 0.0)
|
| 170 |
+
|
| 171 |
+
formatted_log = (
|
| 172 |
+
f"loss={loss:.4f} | recon_loss={recon_loss:.4f} | "
|
| 173 |
+
f"kl_loss={kl_loss:.4f} | grad_norm={grad_norm:.4f} | "
|
| 174 |
+
f"lr={learning_rate:.7f} | epoch={epoch:.4f} | global_step={state.global_step}"
|
| 175 |
+
)
|
| 176 |
+
|
| 177 |
+
self.logger.info(formatted_log)
|
| 178 |
+
|
| 179 |
+
|
| 180 |
+
class VanillaLoggingCallback(TrainerCallback):
|
| 181 |
+
def __init__(self):
|
| 182 |
+
self.logger = logger
|
| 183 |
+
|
| 184 |
+
def on_log(self, args, state, control, logs=None, **kwargs):
|
| 185 |
+
if logs is None:
|
| 186 |
+
return
|
| 187 |
+
|
| 188 |
+
loss = logs.get("loss", 0.0)
|
| 189 |
+
grad_norm = logs.get("grad_norm", 0.0)
|
| 190 |
+
learning_rate = logs.get("learning_rate", 0.0)
|
| 191 |
+
epoch = logs.get("epoch", 0.0)
|
| 192 |
+
|
| 193 |
+
formatted_log = (
|
| 194 |
+
f"loss={loss:.4f} | "
|
| 195 |
+
f"grad_norm={grad_norm:.4f} | lr={learning_rate:.7f} | "
|
| 196 |
+
f"epoch={epoch:.4f} | global_step={state.global_step}"
|
| 197 |
+
f"grad_norm={grad_norm:.4f} | lr={learning_rate:.7f} | "
|
| 198 |
+
f"epoch={epoch:.4f} | global_step={state.global_step}"
|
| 199 |
+
)
|
| 200 |
+
|
| 201 |
+
self.logger.info(formatted_log)
|
cell_fm/models/__init__.py
ADDED
|
File without changes
|
cell_fm/models/cell_fm/__init__.py
ADDED
|
File without changes
|
cell_fm/models/cell_fm/cell_fm_config.py
ADDED
|
@@ -0,0 +1,116 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# -*- coding: utf-8 -*-
|
| 2 |
+
from dataclasses import dataclass
|
| 3 |
+
from transformers import PretrainedConfig
|
| 4 |
+
|
| 5 |
+
@dataclass
|
| 6 |
+
class CELLFMConfig(PretrainedConfig):
|
| 7 |
+
model_type: str = 'flow_matching'
|
| 8 |
+
|
| 9 |
+
# Dataset parameters
|
| 10 |
+
data_path: str = ""
|
| 11 |
+
split_key: str = 'train'
|
| 12 |
+
phase: str = 'train' # 'train', 'val', 'test'
|
| 13 |
+
|
| 14 |
+
data_aug: bool = False
|
| 15 |
+
img_resize: int = 512
|
| 16 |
+
img_crop_size: int = 1024
|
| 17 |
+
seq_zero_mask_ratio: float = 0.0
|
| 18 |
+
seq_full_mask_ratio: float = 0.0
|
| 19 |
+
pre_pad_seq: bool = False
|
| 20 |
+
|
| 21 |
+
cell_image: str = 'nucl' # 'nucl', 'nucl,er', 'nucl,mt', 'nucl,er,mt'
|
| 22 |
+
test_cell_image: str = 'nucl' # 'nucl', 'nucl,er', 'nucl,mt', 'nucl,er,mt'
|
| 23 |
+
|
| 24 |
+
# Transport parameters
|
| 25 |
+
path_type: str = 'Linear'
|
| 26 |
+
prediction: str = 'velocity'
|
| 27 |
+
loss_weight: str = None
|
| 28 |
+
train_eps: float = None
|
| 29 |
+
sample_eps: float = None
|
| 30 |
+
|
| 31 |
+
# Model parameters
|
| 32 |
+
## VAE
|
| 33 |
+
in_channels: int = 1
|
| 34 |
+
out_channels: int = 1
|
| 35 |
+
num_down_blocks: int = 3
|
| 36 |
+
latent_channels: int = 4
|
| 37 |
+
vae_block_out_channels: str = '128,256,512'
|
| 38 |
+
|
| 39 |
+
## CELL-Diff
|
| 40 |
+
img_mask_ratio: float = 0.5
|
| 41 |
+
cond_out_channels: str = '32,64'
|
| 42 |
+
sample_size: int = 64
|
| 43 |
+
esm_embedding: str = 'esmc_300m'
|
| 44 |
+
esm_fixed_embedding: bool = True
|
| 45 |
+
encoder_hidden_size: int = 320
|
| 46 |
+
max_protein_sequence_len: int = 2048
|
| 47 |
+
encoder_num_hidden_layers: int = 16
|
| 48 |
+
num_heads: int = 8
|
| 49 |
+
dim_head: int = 64
|
| 50 |
+
vocab_size: int = 33
|
| 51 |
+
dropout: float = 0.0
|
| 52 |
+
final_dropout: float = 0.0
|
| 53 |
+
encoder_patch_size: int = 8
|
| 54 |
+
|
| 55 |
+
### Image generator
|
| 56 |
+
img_generator_num_layers: int = 18
|
| 57 |
+
img_generator_patch_size: int = 4
|
| 58 |
+
attention_head_dim: int = 64
|
| 59 |
+
num_attention_heads: int = 16
|
| 60 |
+
|
| 61 |
+
### Image decoder
|
| 62 |
+
img_decoder_num_hidden_layers: int = 8
|
| 63 |
+
img_decoder_hidden_size: int = 512
|
| 64 |
+
img_decoder_num_heads: int = 8
|
| 65 |
+
img_decoder_dim_head: int = 64
|
| 66 |
+
img_decoder_dropout: float = 0.0
|
| 67 |
+
img_decoder_final_dropout: float = 0.0
|
| 68 |
+
|
| 69 |
+
cell_image_ratio: float = 1.0
|
| 70 |
+
|
| 71 |
+
# Loss parameters
|
| 72 |
+
seq_loss_coeff: float = 1.0
|
| 73 |
+
img_diff_loss_coeff: float = 1.0
|
| 74 |
+
img_recon_loss_coeff: float = 1.0
|
| 75 |
+
location_loss_coeff: float = 1.0
|
| 76 |
+
|
| 77 |
+
# Training parameters
|
| 78 |
+
vae_loadcheck_path: str = '.'
|
| 79 |
+
loadcheck_path: str = '.'
|
| 80 |
+
ft: bool = False
|
| 81 |
+
infer: bool = False
|
| 82 |
+
ifresume: bool = False
|
| 83 |
+
|
| 84 |
+
output_dir: str = ""
|
| 85 |
+
learning_rate: float = 1e-4
|
| 86 |
+
weight_decay: float = 0.0
|
| 87 |
+
gradient_accumulation_steps: int = 1
|
| 88 |
+
per_device_train_batch_size: int = 2
|
| 89 |
+
per_device_eval_batch_size: int = 2
|
| 90 |
+
|
| 91 |
+
num_train_epochs: int = 10
|
| 92 |
+
fp16: bool = False
|
| 93 |
+
bf16: bool = False
|
| 94 |
+
logging_dir: str = ""
|
| 95 |
+
logging_steps: int = 10
|
| 96 |
+
max_steps: int = -1
|
| 97 |
+
warmup_steps: int = 1000
|
| 98 |
+
save_steps: int = 1000
|
| 99 |
+
|
| 100 |
+
dataloader_num_workers: int = 8
|
| 101 |
+
seed: int = 6
|
| 102 |
+
|
| 103 |
+
# evaluation
|
| 104 |
+
num_steps: int = 100
|
| 105 |
+
|
| 106 |
+
def __init__(self, **kwargs):
|
| 107 |
+
# Use `super().__init__` to handle arguments from PretrainedConfig
|
| 108 |
+
super().__init__(**kwargs)
|
| 109 |
+
|
| 110 |
+
self.vae_block_out_channels = kwargs.get("vae_block_out_channels", self.vae_block_out_channels)
|
| 111 |
+
if not isinstance(self.vae_block_out_channels, list):
|
| 112 |
+
self.vae_block_out_channels = [int(a) for a in self.vae_block_out_channels.split(',')]
|
| 113 |
+
|
| 114 |
+
self.cond_out_channels = kwargs.get("cond_out_channels", self.cond_out_channels)
|
| 115 |
+
if not isinstance(self.cond_out_channels, list):
|
| 116 |
+
self.cond_out_channels = [int(a) for a in self.cond_out_channels.split(',')]
|
cell_fm/models/cell_fm/cell_fm_cs_config.py
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# -*- coding: utf-8 -*-
|
| 2 |
+
from dataclasses import dataclass
|
| 3 |
+
from cell_fm.models.cell_fm.cell_fm_config import CELLFMConfig
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
@dataclass
|
| 7 |
+
class CELLFMCSConfig(CELLFMConfig):
|
| 8 |
+
img_type: str = "GFP" # "GFP", "SNAP"
|
| 9 |
+
|
| 10 |
+
def __init__(self, **kwargs):
|
| 11 |
+
self.img_type = kwargs.pop("img_type", "GFP")
|
| 12 |
+
super().__init__(**kwargs)
|
cell_fm/models/cell_fm/cell_fm_cs_model.py
ADDED
|
@@ -0,0 +1,309 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import numpy as np
|
| 2 |
+
import torch
|
| 3 |
+
import torch.nn as nn
|
| 4 |
+
|
| 5 |
+
from cell_fm.models.cell_fm.cell_fm_model import CELLFM as BaseCELLFM
|
| 6 |
+
from cell_fm.models.cell_fm.cell_fm_model import CELLFMModel as BaseCELLFMModel
|
| 7 |
+
from cell_fm.models.cell_fm.modules.positional_embedding import get_2d_sincos_pos_embed
|
| 8 |
+
from cell_fm.pipeline.utils import CELLFMOutput
|
| 9 |
+
|
| 10 |
+
from .cell_fm_cs_config import CELLFMConfig
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
class CELLFMCSModel(BaseCELLFMModel):
|
| 14 |
+
config_class = CELLFMConfig
|
| 15 |
+
|
| 16 |
+
def initialize_net(self, config: CELLFMConfig):
|
| 17 |
+
return CELLFMCS(config)
|
| 18 |
+
|
| 19 |
+
def forward(self, batched_data, **kwargs):
|
| 20 |
+
protein_seq_masked = batched_data["protein_seq_masked"]
|
| 21 |
+
protein_img_latent, cell_img = self.prepare_data(batched_data, self.vae)
|
| 22 |
+
protein_intensity_level = batched_data["protein_intensity_level"]
|
| 23 |
+
|
| 24 |
+
t, x0, x1 = self.transport.sample(protein_img_latent)
|
| 25 |
+
t, xt, ut = self.transport.path_sampler.plan(t, x0, x1)
|
| 26 |
+
img_output, seq_output, img_recon = self.net(
|
| 27 |
+
xt,
|
| 28 |
+
cell_img,
|
| 29 |
+
protein_seq_masked,
|
| 30 |
+
protein_intensity_level,
|
| 31 |
+
t,
|
| 32 |
+
self.config.img_mask_ratio,
|
| 33 |
+
)
|
| 34 |
+
B, *_, C = xt.shape
|
| 35 |
+
assert img_output.size() == (B, *xt.size()[1:-1], C)
|
| 36 |
+
|
| 37 |
+
loss_dict = self.transport.training_losses(img_output, x0, xt, ut, t)
|
| 38 |
+
img_recon_loss = torch.mean((img_recon - protein_img_latent) ** 2)
|
| 39 |
+
model_output = (seq_output, loss_dict["loss"].mean(), img_recon_loss)
|
| 40 |
+
loss, log_loss = self.loss(batched_data, model_output)
|
| 41 |
+
|
| 42 |
+
return CELLFMOutput(loss=loss, log_output=log_loss)
|
| 43 |
+
|
| 44 |
+
def sequence_to_image(self, protein_seq, cell_img, protein_intensity_level, num_steps=100):
|
| 45 |
+
protein_img_latent = torch.randn(
|
| 46 |
+
cell_img.shape[0],
|
| 47 |
+
self.config.latent_channels,
|
| 48 |
+
self.config.sample_size,
|
| 49 |
+
self.config.sample_size,
|
| 50 |
+
device=cell_img.device,
|
| 51 |
+
)
|
| 52 |
+
|
| 53 |
+
with torch.no_grad():
|
| 54 |
+
sample_fn = self.transport_sampler.sample_ode(num_steps=num_steps)
|
| 55 |
+
|
| 56 |
+
def fn(img_latent, t):
|
| 57 |
+
return self.net.inference(
|
| 58 |
+
img_latent,
|
| 59 |
+
cell_img,
|
| 60 |
+
protein_seq,
|
| 61 |
+
protein_intensity_level,
|
| 62 |
+
t,
|
| 63 |
+
0,
|
| 64 |
+
)[0]
|
| 65 |
+
|
| 66 |
+
protein_img_latent = sample_fn(protein_img_latent, fn)[-1]
|
| 67 |
+
|
| 68 |
+
return self.vae.decode(protein_img_latent).sample
|
| 69 |
+
|
| 70 |
+
def image_to_sequence(
|
| 71 |
+
self,
|
| 72 |
+
protein_seq,
|
| 73 |
+
protein_seq_mask,
|
| 74 |
+
protein_img,
|
| 75 |
+
cell_img,
|
| 76 |
+
protein_intensity_level,
|
| 77 |
+
progress=True,
|
| 78 |
+
sampling_strategy="oaardm",
|
| 79 |
+
order="l2r",
|
| 80 |
+
temperature=1.0,
|
| 81 |
+
):
|
| 82 |
+
if sampling_strategy == "oaardm":
|
| 83 |
+
return self.oaardm_sample(
|
| 84 |
+
protein_seq,
|
| 85 |
+
protein_seq_mask,
|
| 86 |
+
protein_img,
|
| 87 |
+
cell_img,
|
| 88 |
+
protein_intensity_level,
|
| 89 |
+
progress,
|
| 90 |
+
order,
|
| 91 |
+
temperature,
|
| 92 |
+
)
|
| 93 |
+
raise ValueError(f"Sampling strategy {sampling_strategy} is not supported")
|
| 94 |
+
|
| 95 |
+
def oaardm_sample(
|
| 96 |
+
self,
|
| 97 |
+
protein_seq,
|
| 98 |
+
protein_seq_mask,
|
| 99 |
+
protein_img,
|
| 100 |
+
cell_img,
|
| 101 |
+
protein_intensity_level,
|
| 102 |
+
progress=True,
|
| 103 |
+
order="l2r",
|
| 104 |
+
temperature=1.0,
|
| 105 |
+
):
|
| 106 |
+
from esm.utils.constants.esm3 import SEQUENCE_VOCAB
|
| 107 |
+
|
| 108 |
+
device = protein_seq.device
|
| 109 |
+
vocab_to_idx = {aa: idx for idx, aa in enumerate(SEQUENCE_VOCAB)}
|
| 110 |
+
standard_aas = ["A", "C", "D", "E", "F", "G", "H", "I", "K", "L", "M", "N", "P", "Q", "R", "S", "T", "V", "W", "Y"]
|
| 111 |
+
allowed_token_ids = torch.tensor(
|
| 112 |
+
[vocab_to_idx[aa] for aa in standard_aas if aa in vocab_to_idx],
|
| 113 |
+
dtype=torch.long,
|
| 114 |
+
device=device,
|
| 115 |
+
)
|
| 116 |
+
|
| 117 |
+
loc = torch.where(protein_seq_mask.bool().squeeze())[0].cpu().numpy()
|
| 118 |
+
if order == "l2r":
|
| 119 |
+
loc = np.sort(loc)
|
| 120 |
+
elif order == "random":
|
| 121 |
+
np.random.shuffle(loc)
|
| 122 |
+
|
| 123 |
+
if progress:
|
| 124 |
+
from tqdm.auto import tqdm
|
| 125 |
+
|
| 126 |
+
loc = tqdm(loc)
|
| 127 |
+
|
| 128 |
+
with torch.no_grad():
|
| 129 |
+
protein_img_latent = self.vae.encode(protein_img).sample()
|
| 130 |
+
batch_size = protein_seq.shape[0]
|
| 131 |
+
|
| 132 |
+
for i in loc:
|
| 133 |
+
_, t1 = self.transport.check_interval(
|
| 134 |
+
self.transport.train_eps,
|
| 135 |
+
self.transport.sample_eps,
|
| 136 |
+
)
|
| 137 |
+
t = torch.full((batch_size,), t1, device=device)
|
| 138 |
+
seq_output = self.net.inference(
|
| 139 |
+
protein_img_latent,
|
| 140 |
+
cell_img,
|
| 141 |
+
protein_seq,
|
| 142 |
+
protein_intensity_level,
|
| 143 |
+
t,
|
| 144 |
+
)[1]
|
| 145 |
+
logits_i = seq_output[:, i, :]
|
| 146 |
+
aa_logits = logits_i[:, allowed_token_ids]
|
| 147 |
+
probs = torch.nn.functional.softmax(aa_logits / temperature, dim=-1)
|
| 148 |
+
sample_idx = torch.multinomial(probs, num_samples=1).squeeze(-1)
|
| 149 |
+
protein_seq[:, i] = allowed_token_ids[sample_idx]
|
| 150 |
+
|
| 151 |
+
return protein_seq
|
| 152 |
+
|
| 153 |
+
def recon(self, protein_seq, protein_img, cell_img, protein_intensity_level):
|
| 154 |
+
with torch.no_grad():
|
| 155 |
+
protein_img_latent = self.vae.encode(protein_img).sample()
|
| 156 |
+
_, t1 = self.transport.check_interval(self.transport.train_eps, self.transport.sample_eps)
|
| 157 |
+
t = torch.tensor([t1] * protein_seq.shape[0], device=protein_seq.device)
|
| 158 |
+
img_recon_latent = self.net(
|
| 159 |
+
protein_img_latent,
|
| 160 |
+
cell_img,
|
| 161 |
+
protein_seq,
|
| 162 |
+
protein_intensity_level,
|
| 163 |
+
t,
|
| 164 |
+
img_mask_ratio=0.5,
|
| 165 |
+
)[2]
|
| 166 |
+
|
| 167 |
+
return self.vae.decode(img_recon_latent).sample
|
| 168 |
+
|
| 169 |
+
|
| 170 |
+
class CELLFMCS(BaseCELLFM):
|
| 171 |
+
def __init__(self, config: CELLFMConfig):
|
| 172 |
+
super().__init__(config)
|
| 173 |
+
|
| 174 |
+
self.protein_intensity_level_embedding = nn.Linear(
|
| 175 |
+
in_features=1,
|
| 176 |
+
out_features=config.encoder_hidden_size,
|
| 177 |
+
bias=True,
|
| 178 |
+
)
|
| 179 |
+
self.time_step_pos_embed = nn.Parameter(
|
| 180 |
+
torch.randn(1, 1, config.encoder_hidden_size) * 0.02,
|
| 181 |
+
requires_grad=True,
|
| 182 |
+
)
|
| 183 |
+
self.protein_intensity_level_pos_embed = nn.Parameter(
|
| 184 |
+
torch.randn(1, 1, config.encoder_hidden_size) * 0.02,
|
| 185 |
+
requires_grad=True,
|
| 186 |
+
)
|
| 187 |
+
|
| 188 |
+
img_encoder_pos_embed = get_2d_sincos_pos_embed(
|
| 189 |
+
config.encoder_hidden_size,
|
| 190 |
+
self.img_embedding.base_size,
|
| 191 |
+
)
|
| 192 |
+
self.img_encoder_pos_embed = nn.Parameter(
|
| 193 |
+
torch.from_numpy(img_encoder_pos_embed).float().unsqueeze(0),
|
| 194 |
+
requires_grad=False,
|
| 195 |
+
)
|
| 196 |
+
|
| 197 |
+
img_decoder_pos_embed = get_2d_sincos_pos_embed(
|
| 198 |
+
config.img_decoder_hidden_size,
|
| 199 |
+
self.img_embedding.base_size,
|
| 200 |
+
)
|
| 201 |
+
self.img_decoder_pos_embed = nn.Parameter(
|
| 202 |
+
torch.from_numpy(img_decoder_pos_embed).float().unsqueeze(0),
|
| 203 |
+
requires_grad=False,
|
| 204 |
+
)
|
| 205 |
+
|
| 206 |
+
self._init_weights(self.protein_intensity_level_embedding)
|
| 207 |
+
|
| 208 |
+
def _encode_conditioned(
|
| 209 |
+
self,
|
| 210 |
+
protein_img,
|
| 211 |
+
cell_img,
|
| 212 |
+
protein_seq,
|
| 213 |
+
protein_intensity_level,
|
| 214 |
+
time,
|
| 215 |
+
img_mask_ratio,
|
| 216 |
+
):
|
| 217 |
+
time_embeds = self.time_embedding(time)
|
| 218 |
+
intensity_embeds = self.protein_intensity_level_embedding(protein_intensity_level)
|
| 219 |
+
|
| 220 |
+
seq_embeds = self.protein_sequence_embedding(protein_seq)
|
| 221 |
+
seq_embeds = self.seq_proj_in(seq_embeds)
|
| 222 |
+
seq_embeds = seq_embeds + self.protein_sequence_pos_embed[:, : protein_seq.shape[1]]
|
| 223 |
+
seq_token_type = torch.full(
|
| 224 |
+
size=(seq_embeds.shape[0], 1),
|
| 225 |
+
fill_value=1,
|
| 226 |
+
device=seq_embeds.device,
|
| 227 |
+
).long()
|
| 228 |
+
seq_embeds = seq_embeds + self.token_type_embeddings(seq_token_type)
|
| 229 |
+
|
| 230 |
+
cell_img_conv = self.cond_conv(cell_img)
|
| 231 |
+
concat_img = torch.cat([protein_img, cell_img_conv], dim=1)
|
| 232 |
+
img_embeds = self.img_embedding(concat_img)
|
| 233 |
+
img_embeds = img_embeds + self.img_encoder_pos_embed
|
| 234 |
+
img_embeds, _, img_ids_restore = self.random_masking(img_embeds, img_mask_ratio)
|
| 235 |
+
|
| 236 |
+
time_embeds_with_pos = time_embeds.unsqueeze(1) + self.time_step_pos_embed
|
| 237 |
+
intensity_embeds_with_pos = intensity_embeds.unsqueeze(1) + self.protein_intensity_level_pos_embed
|
| 238 |
+
img_embeds = torch.cat([time_embeds_with_pos, intensity_embeds_with_pos, img_embeds], dim=1)
|
| 239 |
+
img_token_type = torch.full(
|
| 240 |
+
size=(img_embeds.shape[0], 1),
|
| 241 |
+
fill_value=0,
|
| 242 |
+
device=img_embeds.device,
|
| 243 |
+
).long()
|
| 244 |
+
img_embeds = img_embeds + self.token_type_embeddings(img_token_type)
|
| 245 |
+
|
| 246 |
+
co_embeds = torch.cat([img_embeds, seq_embeds], dim=1)
|
| 247 |
+
co_feat = co_embeds
|
| 248 |
+
for block in self.unified_encoder:
|
| 249 |
+
co_feat = block(co_feat)
|
| 250 |
+
|
| 251 |
+
return concat_img, img_embeds, img_ids_restore, co_feat, time_embeds
|
| 252 |
+
|
| 253 |
+
def forward(self, protein_img, cell_img, protein_seq, protein_intensity_level, time, img_mask_ratio=0.5):
|
| 254 |
+
concat_img, img_embeds, img_ids_restore, co_feat, time_embeds = self._encode_conditioned(
|
| 255 |
+
protein_img,
|
| 256 |
+
cell_img,
|
| 257 |
+
protein_seq,
|
| 258 |
+
protein_intensity_level,
|
| 259 |
+
time,
|
| 260 |
+
img_mask_ratio,
|
| 261 |
+
)
|
| 262 |
+
img_output = self.img_generator(
|
| 263 |
+
hidden_states=concat_img,
|
| 264 |
+
timestep=time,
|
| 265 |
+
encoder_hidden_states=co_feat,
|
| 266 |
+
pooled_projections=time_embeds,
|
| 267 |
+
).sample
|
| 268 |
+
seq_feat = co_feat[:, img_embeds.shape[1] :]
|
| 269 |
+
img_feat = co_feat[:, : img_embeds.shape[1]]
|
| 270 |
+
seq_output = self.seq_decoder(seq_feat)
|
| 271 |
+
img_recon = self.forward_img_decoder(img_feat, img_ids_restore)
|
| 272 |
+
img_recon = self.unpatchify(img_recon, self.config.latent_channels)
|
| 273 |
+
|
| 274 |
+
return img_output, seq_output, img_recon
|
| 275 |
+
|
| 276 |
+
def forward_img_decoder(self, x, ids_restore):
|
| 277 |
+
x = self.img_decoder_embed(x)
|
| 278 |
+
mask_tokens = self.mask_token.repeat(x.shape[0], ids_restore.shape[1] + 2 - x.shape[1], 1)
|
| 279 |
+
x_ = torch.cat([x[:, 2:, :], mask_tokens], dim=1)
|
| 280 |
+
x_ = torch.gather(x_, dim=1, index=ids_restore.unsqueeze(-1).repeat(1, 1, x.shape[2]))
|
| 281 |
+
x_ = x_ + self.img_decoder_pos_embed
|
| 282 |
+
x = torch.cat([x[:, :2, :], x_], dim=1)
|
| 283 |
+
|
| 284 |
+
for block in self.img_decoder:
|
| 285 |
+
x = block(x)
|
| 286 |
+
x = self.img_decoder_norm(x)
|
| 287 |
+
x = self.img_decoder_pred(x)
|
| 288 |
+
|
| 289 |
+
return x[:, 2:, :]
|
| 290 |
+
|
| 291 |
+
def inference(self, protein_img, cell_img, protein_seq, protein_intensity_level, time, img_mask_ratio=0):
|
| 292 |
+
concat_img, img_embeds, _, co_feat, time_embeds = self._encode_conditioned(
|
| 293 |
+
protein_img,
|
| 294 |
+
cell_img,
|
| 295 |
+
protein_seq,
|
| 296 |
+
protein_intensity_level,
|
| 297 |
+
time,
|
| 298 |
+
img_mask_ratio,
|
| 299 |
+
)
|
| 300 |
+
img_output = self.img_generator(
|
| 301 |
+
hidden_states=concat_img,
|
| 302 |
+
timestep=time,
|
| 303 |
+
encoder_hidden_states=co_feat,
|
| 304 |
+
pooled_projections=time_embeds,
|
| 305 |
+
).sample
|
| 306 |
+
seq_feat = co_feat[:, img_embeds.shape[1] :]
|
| 307 |
+
seq_output = self.seq_decoder(seq_feat)
|
| 308 |
+
|
| 309 |
+
return img_output, seq_output
|
cell_fm/models/cell_fm/cell_fm_model.py
ADDED
|
@@ -0,0 +1,637 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import torch
|
| 3 |
+
import torch.nn as nn
|
| 4 |
+
import numpy as np
|
| 5 |
+
from cell_fm.logging import logger
|
| 6 |
+
|
| 7 |
+
from diffusers.models.embeddings import PatchEmbed
|
| 8 |
+
from diffusers.models.transformers import SD3Transformer2DModel
|
| 9 |
+
|
| 10 |
+
from .modules import TimestepEmbedder, CondConvNet
|
| 11 |
+
from .modules.positional_embedding import get_1d_sincos_pos_embed, get_2d_sincos_pos_embed
|
| 12 |
+
from .modules.transformer import TransformerBlock, MLMHead
|
| 13 |
+
from .modules.protein_sequence_embedding import ESMEmbed
|
| 14 |
+
from .modules.transport import create_transport, Sampler
|
| 15 |
+
|
| 16 |
+
from transformers import PreTrainedModel
|
| 17 |
+
from .cell_fm_config import CELLFMConfig
|
| 18 |
+
|
| 19 |
+
from cell_fm.pipeline.utils import CELLFMOutput
|
| 20 |
+
from cell_fm.models.vae.vae_model import VAEModel
|
| 21 |
+
import json
|
| 22 |
+
from copy import deepcopy
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
class CELLFMModel(PreTrainedModel):
|
| 26 |
+
config_class = CELLFMConfig
|
| 27 |
+
|
| 28 |
+
def __init__(self, config: CELLFMConfig, loss_fn=None):
|
| 29 |
+
super().__init__(config)
|
| 30 |
+
self.config = config
|
| 31 |
+
|
| 32 |
+
self.loss = loss_fn(config)
|
| 33 |
+
self.net = self.initialize_net(config)
|
| 34 |
+
|
| 35 |
+
self.transport = create_transport(
|
| 36 |
+
config.path_type,
|
| 37 |
+
config.prediction,
|
| 38 |
+
config.loss_weight,
|
| 39 |
+
config.train_eps,
|
| 40 |
+
config.sample_eps,
|
| 41 |
+
)
|
| 42 |
+
self.transport_sampler = Sampler(self.transport)
|
| 43 |
+
|
| 44 |
+
self.vae = self.initialize_vae()
|
| 45 |
+
self.load_pretrained_weights(config, checkpoint_path=config.loadcheck_path)
|
| 46 |
+
self.vae = self.prepare_vae(self.vae, self.config.vae_loadcheck_path)
|
| 47 |
+
|
| 48 |
+
def initialize_net(self, config: CELLFMConfig):
|
| 49 |
+
return CELLFM(config)
|
| 50 |
+
|
| 51 |
+
def initialize_vae(self):
|
| 52 |
+
vae_config = deepcopy(self.config)
|
| 53 |
+
vae_config.ft = False
|
| 54 |
+
vae_config.infer = False
|
| 55 |
+
vae_config.vae_loadcheck_path = None
|
| 56 |
+
vae = VAEModel(vae_config)
|
| 57 |
+
|
| 58 |
+
return vae
|
| 59 |
+
|
| 60 |
+
def prepare_vae(self, vae: VAEModel, checkpoint_path):
|
| 61 |
+
if checkpoint_path is not None:
|
| 62 |
+
vae_config = deepcopy(self.config)
|
| 63 |
+
vae_config.ft = False
|
| 64 |
+
vae_config.infer = True
|
| 65 |
+
vae.load_pretrained_weights(vae_config, checkpoint_path=checkpoint_path)
|
| 66 |
+
|
| 67 |
+
for param in vae.parameters():
|
| 68 |
+
param.requires_grad = False
|
| 69 |
+
vae.eval()
|
| 70 |
+
|
| 71 |
+
return vae
|
| 72 |
+
|
| 73 |
+
def load_pretrained_weights(self, config, checkpoint_path):
|
| 74 |
+
"""
|
| 75 |
+
Load pretrained weights from a given state_dict.
|
| 76 |
+
"""
|
| 77 |
+
if config.ft or config.infer:
|
| 78 |
+
if config.ft:
|
| 79 |
+
logger.info(f"Finetune from checkpoint: {checkpoint_path}")
|
| 80 |
+
else:
|
| 81 |
+
logger.info(f"Infer from checkpoint: {checkpoint_path}")
|
| 82 |
+
|
| 83 |
+
if os.path.isdir(checkpoint_path):
|
| 84 |
+
with open(os.path.join(checkpoint_path, "pytorch_model.bin.index.json"), "r") as f:
|
| 85 |
+
index_data = json.load(f)
|
| 86 |
+
|
| 87 |
+
weight_map = index_data["weight_map"]
|
| 88 |
+
|
| 89 |
+
shard_files = set(weight_map.values())
|
| 90 |
+
shard_weights = {}
|
| 91 |
+
for shard_file in shard_files:
|
| 92 |
+
shard_weights[shard_file] = torch.load(os.path.join(checkpoint_path, shard_file), map_location="cpu")
|
| 93 |
+
|
| 94 |
+
checkpoints_state = {}
|
| 95 |
+
for param_name, shard_file in weight_map.items():
|
| 96 |
+
checkpoints_state[param_name] = shard_weights[shard_file][param_name]
|
| 97 |
+
elif os.path.splitext(checkpoint_path)[1] == '.safetensors':
|
| 98 |
+
from safetensors.torch import load_file
|
| 99 |
+
checkpoints_state = load_file(checkpoint_path)
|
| 100 |
+
else:
|
| 101 |
+
checkpoints_state = torch.load(checkpoint_path, map_location="cpu")
|
| 102 |
+
|
| 103 |
+
if "model" in checkpoints_state:
|
| 104 |
+
checkpoints_state = checkpoints_state["model"]
|
| 105 |
+
elif "module" in checkpoints_state:
|
| 106 |
+
checkpoints_state = checkpoints_state["module"]
|
| 107 |
+
|
| 108 |
+
model_state_dict = self.state_dict()
|
| 109 |
+
filtered_state_dict = {k: v for k, v in checkpoints_state.items() if k in model_state_dict and v.size() == model_state_dict[k].size()}
|
| 110 |
+
|
| 111 |
+
IncompatibleKeys = self.load_state_dict(filtered_state_dict, strict=False)
|
| 112 |
+
IncompatibleKeys = IncompatibleKeys._asdict()
|
| 113 |
+
|
| 114 |
+
missing_keys = []
|
| 115 |
+
for keys in IncompatibleKeys["missing_keys"]:
|
| 116 |
+
if keys.find("dummy") == -1:
|
| 117 |
+
missing_keys.append(keys)
|
| 118 |
+
|
| 119 |
+
unexpected_keys = []
|
| 120 |
+
for keys in IncompatibleKeys["unexpected_keys"]:
|
| 121 |
+
if keys.find("dummy") == -1:
|
| 122 |
+
unexpected_keys.append(keys)
|
| 123 |
+
|
| 124 |
+
if len(missing_keys) > 0:
|
| 125 |
+
logger.info(
|
| 126 |
+
"Missing keys in {}: {}".format(
|
| 127 |
+
checkpoint_path,
|
| 128 |
+
missing_keys,
|
| 129 |
+
)
|
| 130 |
+
)
|
| 131 |
+
|
| 132 |
+
if len(unexpected_keys) > 0:
|
| 133 |
+
logger.info(
|
| 134 |
+
"Unexpected keys {}: {}".format(
|
| 135 |
+
checkpoint_path,
|
| 136 |
+
unexpected_keys,
|
| 137 |
+
)
|
| 138 |
+
)
|
| 139 |
+
|
| 140 |
+
@torch.no_grad()
|
| 141 |
+
def prepare_data(self, batched_data, vae: VAEModel):
|
| 142 |
+
|
| 143 |
+
# protein_img = batched_data['protein_img']
|
| 144 |
+
# nucleus_img = batched_data['nucleus_img']
|
| 145 |
+
# microtubules_img = batched_data['microtubules_img']
|
| 146 |
+
# protein_seq = batched_data['protein_seq']
|
| 147 |
+
# protein_seq_masked = batched_data['protein_seq_masked']
|
| 148 |
+
# protein_seq_mask = batched_data['protein_seq_mask']
|
| 149 |
+
|
| 150 |
+
protein_img = batched_data['protein_img']
|
| 151 |
+
nucleus_img = batched_data['nucleus_img']
|
| 152 |
+
protein_img_latent = vae.encode(protein_img).sample()
|
| 153 |
+
|
| 154 |
+
if self.config.cell_image == 'nucl':
|
| 155 |
+
cell_img = nucleus_img
|
| 156 |
+
|
| 157 |
+
elif self.config.cell_image == 'nucl,er':
|
| 158 |
+
if 'ER_img' in batched_data:
|
| 159 |
+
ER_img = batched_data['ER_img']
|
| 160 |
+
mask = (torch.rand(ER_img.shape[0]) <= self.config.cell_image_ratio).bool()
|
| 161 |
+
ER_img[~mask] = torch.full(size = [ER_img.shape[2], ER_img.shape[3]], fill_value=-2.0, device=ER_img.device)
|
| 162 |
+
else:
|
| 163 |
+
ER_img = torch.full_like(protein_img, fill_value=-2.0)
|
| 164 |
+
cell_img = torch.cat([nucleus_img, ER_img], dim=1)
|
| 165 |
+
|
| 166 |
+
elif self.config.cell_image == 'nucl,mt':
|
| 167 |
+
if 'microtubules_img' in batched_data:
|
| 168 |
+
microtubules_img = batched_data['microtubules_img']
|
| 169 |
+
mask = (torch.rand(microtubules_img.shape[0]) <= self.config.cell_image_ratio).bool()
|
| 170 |
+
microtubules_img[~mask] = torch.full(size = [microtubules_img.shape[2], microtubules_img.shape[3]], fill_value=-2.0, device=microtubules_img.device)
|
| 171 |
+
else:
|
| 172 |
+
microtubules_img = torch.full_like(protein_img, fill_value=-2.0)
|
| 173 |
+
cell_img = torch.cat([nucleus_img, microtubules_img], dim=1)
|
| 174 |
+
|
| 175 |
+
elif self.config.cell_image == 'nucl,er,mt':
|
| 176 |
+
if 'ER_img' in batched_data:
|
| 177 |
+
ER_img = batched_data['ER_img']
|
| 178 |
+
mask = (torch.rand(ER_img.shape[0]) <= self.config.cell_image_ratio).bool()
|
| 179 |
+
ER_img[~mask] = torch.full(size = [ER_img.shape[2], ER_img.shape[3]], fill_value=-2.0, device=ER_img.device)
|
| 180 |
+
else:
|
| 181 |
+
ER_img = torch.full_like(protein_img, fill_value=-2.0)
|
| 182 |
+
|
| 183 |
+
if 'microtubules_img' in batched_data:
|
| 184 |
+
microtubules_img = batched_data['microtubules_img']
|
| 185 |
+
mask = (torch.rand(microtubules_img.shape[0]) <= self.config.cell_image_ratio).bool()
|
| 186 |
+
microtubules_img[~mask] = torch.full(size = [microtubules_img.shape[2], microtubules_img.shape[3]], fill_value=-2.0, device=microtubules_img.device)
|
| 187 |
+
else:
|
| 188 |
+
microtubules_img = torch.full_like(protein_img, fill_value=-2.0)
|
| 189 |
+
cell_img = torch.cat([nucleus_img, ER_img, microtubules_img], dim=1)
|
| 190 |
+
else:
|
| 191 |
+
raise ValueError(f"Cell image type: {self.config.cell_image} is not supported")
|
| 192 |
+
|
| 193 |
+
return protein_img_latent, cell_img
|
| 194 |
+
|
| 195 |
+
def forward(self, batched_data, **kwargs):
|
| 196 |
+
protein_seq_masked = batched_data['protein_seq_masked']
|
| 197 |
+
protein_img_latent, cell_img = self.prepare_data(batched_data, self.vae)
|
| 198 |
+
|
| 199 |
+
t, x0, x1 = self.transport.sample(protein_img_latent)
|
| 200 |
+
t, xt, ut = self.transport.path_sampler.plan(t, x0, x1)
|
| 201 |
+
img_output, seq_output, img_recon = self.net(xt, cell_img, protein_seq_masked, t, self.config.img_mask_ratio)
|
| 202 |
+
B, *_, C = xt.shape
|
| 203 |
+
assert img_output.size() == (B, *xt.size()[1:-1], C)
|
| 204 |
+
|
| 205 |
+
loss_dict = self.transport.training_losses(img_output, x0, xt, ut, t)
|
| 206 |
+
|
| 207 |
+
img_recon_loss = torch.mean((img_recon - protein_img_latent)**2)
|
| 208 |
+
|
| 209 |
+
model_output = (seq_output, loss_dict["loss"].mean(), img_recon_loss)
|
| 210 |
+
|
| 211 |
+
loss, log_loss = self.loss(batched_data, model_output)
|
| 212 |
+
|
| 213 |
+
return CELLFMOutput(loss=loss, log_output=log_loss)
|
| 214 |
+
|
| 215 |
+
def sequence_to_image(self, protein_seq, cell_img, num_steps=100):
|
| 216 |
+
protein_img_latent = torch.randn(cell_img.shape[0], self.config.latent_channels, self.config.sample_size, self.config.sample_size).to(cell_img.device)
|
| 217 |
+
|
| 218 |
+
with torch.no_grad():
|
| 219 |
+
sample_fn = self.transport_sampler.sample_ode(num_steps=num_steps) # default to ode sampling
|
| 220 |
+
def fn(img_latent, t):
|
| 221 |
+
return self.net.seq2img(img_latent, cell_img, protein_seq, t, 0)
|
| 222 |
+
|
| 223 |
+
protein_img_latent = sample_fn(protein_img_latent, fn)[-1]
|
| 224 |
+
|
| 225 |
+
return self.vae.decode(protein_img_latent).sample
|
| 226 |
+
|
| 227 |
+
def image_to_sequence(self, protein_seq, protein_seq_mask, protein_img, cell_img, progress=True, sampling_strategy='oaardm', order='l2r', temperature=1.0):
|
| 228 |
+
if sampling_strategy == "oaardm":
|
| 229 |
+
return self.oaardm_sample(protein_seq, protein_seq_mask, protein_img, cell_img, progress, order, temperature)
|
| 230 |
+
|
| 231 |
+
def oaardm_sample(self, protein_seq, protein_seq_mask, protein_img, cell_img, progress=True, order='l2r', temperature=1.0):
|
| 232 |
+
loc = torch.where(protein_seq_mask.bool().squeeze())[0].cpu().numpy()
|
| 233 |
+
|
| 234 |
+
if order == 'l2r':
|
| 235 |
+
loc = np.sort(loc)
|
| 236 |
+
elif order == 'random':
|
| 237 |
+
np.random.shuffle(loc)
|
| 238 |
+
|
| 239 |
+
if progress:
|
| 240 |
+
# Lazy import so that we don't depend on tqdm.
|
| 241 |
+
from tqdm.auto import tqdm
|
| 242 |
+
loc = tqdm(loc)
|
| 243 |
+
|
| 244 |
+
with torch.no_grad():
|
| 245 |
+
protein_img_latent = self.vae.encode(protein_img).sample()
|
| 246 |
+
for i in loc:
|
| 247 |
+
t0, t1 = self.transport.check_interval(self.transport.train_eps, self.transport.sample_eps)
|
| 248 |
+
# t1 = 1.0, t0 = 0.0
|
| 249 |
+
t = torch.tensor([t1] * protein_seq.shape[0], device=protein_seq.device)
|
| 250 |
+
seq_output = self.net(protein_img_latent, cell_img, protein_seq, t, img_mask_ratio=0)[1]
|
| 251 |
+
p = seq_output[:, i, 4:4+20] # sample at location i (random), dont let it predict non-standard AA
|
| 252 |
+
p = torch.nn.functional.softmax(p / temperature, dim=1) # softmax over categorical probs
|
| 253 |
+
p_sample = torch.multinomial(p, num_samples=1)
|
| 254 |
+
protein_seq[:, i] = p_sample.squeeze() + 4
|
| 255 |
+
|
| 256 |
+
return protein_seq
|
| 257 |
+
|
| 258 |
+
def embed(self, protein_seq, protein_img, cell_img, img_mask_ratio=0):
|
| 259 |
+
with torch.no_grad():
|
| 260 |
+
protein_img_latent = self.vae.encode(protein_img).sample()
|
| 261 |
+
|
| 262 |
+
t0, t1 = self.transport.check_interval(self.transport.train_eps, self.transport.sample_eps)
|
| 263 |
+
t = torch.tensor([t1] * protein_seq.shape[0], device=protein_seq.device)
|
| 264 |
+
img_feat_embed, seq_feat_embed = self.net.embed(protein_img_latent, cell_img, protein_seq, t, img_mask_ratio=img_mask_ratio)
|
| 265 |
+
|
| 266 |
+
return img_feat_embed, seq_feat_embed
|
| 267 |
+
|
| 268 |
+
def recon(self, protein_seq, protein_img, cell_img):
|
| 269 |
+
with torch.no_grad():
|
| 270 |
+
protein_img_latent = self.vae.encode(protein_img).sample()
|
| 271 |
+
|
| 272 |
+
t0, t1 = self.transport.check_interval(self.transport.train_eps, self.transport.sample_eps)
|
| 273 |
+
t = torch.tensor([t1] * protein_seq.shape[0], device=protein_seq.device)
|
| 274 |
+
img_recon_latent = self.net(protein_img_latent, cell_img, protein_seq, t, img_mask_ratio=0.5)[2]
|
| 275 |
+
|
| 276 |
+
img_recon = self.vae.decode(img_recon_latent).sample
|
| 277 |
+
return img_recon
|
| 278 |
+
|
| 279 |
+
|
| 280 |
+
class CELLFM(nn.Module):
|
| 281 |
+
def __init__(self, config: CELLFMConfig):
|
| 282 |
+
super().__init__()
|
| 283 |
+
self.config = config
|
| 284 |
+
|
| 285 |
+
self.cond_conv = CondConvNet(
|
| 286 |
+
config.in_channels * len(config.cell_image.split(',')),
|
| 287 |
+
config.cond_out_channels,
|
| 288 |
+
)
|
| 289 |
+
|
| 290 |
+
# Time step embedding
|
| 291 |
+
self.time_embedding = TimestepEmbedder(hidden_size=config.encoder_hidden_size)
|
| 292 |
+
|
| 293 |
+
# Token type embeddings
|
| 294 |
+
self.token_type_embeddings = nn.Embedding(2, config.encoder_hidden_size)
|
| 295 |
+
|
| 296 |
+
# Image embedding without positional embedding
|
| 297 |
+
self.img_embedding = PatchEmbed(
|
| 298 |
+
height=config.sample_size,
|
| 299 |
+
width=config.sample_size,
|
| 300 |
+
patch_size=config.encoder_patch_size,
|
| 301 |
+
in_channels=config.latent_channels + config.cond_out_channels[-1],
|
| 302 |
+
embed_dim=config.encoder_hidden_size,
|
| 303 |
+
pos_embed_type=None,
|
| 304 |
+
)
|
| 305 |
+
|
| 306 |
+
# Image positional embedding
|
| 307 |
+
# Initialize pos_embed by sin-cos embedding:
|
| 308 |
+
img_encoder_pos_embed = get_2d_sincos_pos_embed(
|
| 309 |
+
config.encoder_hidden_size, self.img_embedding.base_size, cls_token=True, extra_tokens=1,
|
| 310 |
+
)
|
| 311 |
+
# Use fixed sin-cos embedding:
|
| 312 |
+
self.img_encoder_pos_embed = nn.Parameter(
|
| 313 |
+
torch.from_numpy(img_encoder_pos_embed).float().unsqueeze(0),
|
| 314 |
+
requires_grad=False,
|
| 315 |
+
)
|
| 316 |
+
|
| 317 |
+
# Sequence positional embedding
|
| 318 |
+
# Initialize pos_embed by sin-cos embedding:
|
| 319 |
+
protein_sequence_pos_embed = get_1d_sincos_pos_embed(
|
| 320 |
+
config.encoder_hidden_size, self.config.max_protein_sequence_len+2,
|
| 321 |
+
)
|
| 322 |
+
# Use fixed sin-cos embedding:
|
| 323 |
+
self.protein_sequence_pos_embed = nn.Parameter(
|
| 324 |
+
torch.from_numpy(protein_sequence_pos_embed).float().unsqueeze(0),
|
| 325 |
+
requires_grad=False,
|
| 326 |
+
)
|
| 327 |
+
|
| 328 |
+
# Sequence projection
|
| 329 |
+
self.seq_proj_in = nn.Linear(config.encoder_hidden_size, config.encoder_hidden_size, bias=True)
|
| 330 |
+
|
| 331 |
+
# Unified encoder
|
| 332 |
+
self.unified_encoder = nn.ModuleList([
|
| 333 |
+
TransformerBlock(
|
| 334 |
+
hidden_size=config.encoder_hidden_size,
|
| 335 |
+
num_heads=config.num_heads,
|
| 336 |
+
dim_head=config.dim_head,
|
| 337 |
+
dropout=config.dropout,
|
| 338 |
+
final_dropout=config.final_dropout,
|
| 339 |
+
) for _ in range(config.encoder_num_hidden_layers)
|
| 340 |
+
])
|
| 341 |
+
|
| 342 |
+
# Image generator
|
| 343 |
+
self.img_generator = SD3Transformer2DModel(
|
| 344 |
+
sample_size=config.sample_size,
|
| 345 |
+
patch_size=config.img_generator_patch_size,
|
| 346 |
+
in_channels=config.latent_channels + config.cond_out_channels[-1],
|
| 347 |
+
num_layers=config.img_generator_num_layers,
|
| 348 |
+
attention_head_dim=config.attention_head_dim,
|
| 349 |
+
num_attention_heads=config.num_attention_heads,
|
| 350 |
+
joint_attention_dim=config.encoder_hidden_size,
|
| 351 |
+
pooled_projection_dim=config.encoder_hidden_size,
|
| 352 |
+
caption_projection_dim=config.num_attention_heads * config.attention_head_dim,
|
| 353 |
+
out_channels=config.latent_channels,
|
| 354 |
+
)
|
| 355 |
+
|
| 356 |
+
# Image decoder
|
| 357 |
+
self.img_decoder_embed = nn.Linear(config.encoder_hidden_size, config.img_decoder_hidden_size, bias=True)
|
| 358 |
+
self.mask_token = nn.Parameter(torch.zeros(1, 1, config.img_decoder_hidden_size))
|
| 359 |
+
|
| 360 |
+
# Initialize pos_embed by sin-cos embedding:
|
| 361 |
+
img_decoder_pos_embed = get_2d_sincos_pos_embed(
|
| 362 |
+
config.img_decoder_hidden_size, self.img_embedding.base_size, cls_token=True, extra_tokens=1,
|
| 363 |
+
)
|
| 364 |
+
# Use fixed sin-cos embedding:
|
| 365 |
+
self.img_decoder_pos_embed = nn.Parameter(
|
| 366 |
+
torch.from_numpy(img_decoder_pos_embed).float().unsqueeze(0),
|
| 367 |
+
requires_grad=False,
|
| 368 |
+
)
|
| 369 |
+
|
| 370 |
+
self.img_decoder = nn.ModuleList([
|
| 371 |
+
TransformerBlock(
|
| 372 |
+
hidden_size=config.img_decoder_hidden_size,
|
| 373 |
+
num_heads=config.img_decoder_num_heads,
|
| 374 |
+
dim_head=config.img_decoder_dim_head,
|
| 375 |
+
dropout=config.img_decoder_dropout,
|
| 376 |
+
final_dropout=config.img_decoder_final_dropout,
|
| 377 |
+
) for _ in range(config.img_decoder_num_hidden_layers)
|
| 378 |
+
])
|
| 379 |
+
self.img_decoder_norm = nn.LayerNorm(config.img_decoder_hidden_size)
|
| 380 |
+
self.img_decoder_pred = nn.Linear(config.img_decoder_hidden_size, config.encoder_patch_size**2 * config.latent_channels, bias=True) # decoder to patch
|
| 381 |
+
|
| 382 |
+
# Sequence decoder
|
| 383 |
+
self.seq_decoder = MLMHead(
|
| 384 |
+
hidden_size=config.encoder_hidden_size,
|
| 385 |
+
vocab_size=config.vocab_size,
|
| 386 |
+
)
|
| 387 |
+
|
| 388 |
+
self.initialize_weights()
|
| 389 |
+
self.initialize_protein_sequence_embedding()
|
| 390 |
+
|
| 391 |
+
def initialize_weights(self):
|
| 392 |
+
# initialization
|
| 393 |
+
torch.nn.init.normal_(self.mask_token, std=.02)
|
| 394 |
+
|
| 395 |
+
# initialize nn.Linear and nn.LayerNorm
|
| 396 |
+
self.apply(self._init_weights)
|
| 397 |
+
|
| 398 |
+
def _init_weights(self, m):
|
| 399 |
+
if isinstance(m, nn.Linear):
|
| 400 |
+
# we use xavier_uniform following official JAX ViT:
|
| 401 |
+
torch.nn.init.xavier_uniform_(m.weight)
|
| 402 |
+
if isinstance(m, nn.Linear) and m.bias is not None:
|
| 403 |
+
nn.init.constant_(m.bias, 0)
|
| 404 |
+
elif isinstance(m, nn.LayerNorm):
|
| 405 |
+
if m.bias is not None:
|
| 406 |
+
nn.init.constant_(m.bias, 0)
|
| 407 |
+
if m.weight is not None:
|
| 408 |
+
nn.init.constant_(m.weight, 1.0)
|
| 409 |
+
|
| 410 |
+
def initialize_protein_sequence_embedding(self):
|
| 411 |
+
self.protein_sequence_embedding = ESMEmbed(self.config.esm_embedding, self.config.encoder_hidden_size, self.config.esm_fixed_embedding)
|
| 412 |
+
|
| 413 |
+
def forward(self, protein_img, cell_img, protein_seq, time, img_mask_ratio=0.5):
|
| 414 |
+
time_embeds = self.time_embedding(time)
|
| 415 |
+
|
| 416 |
+
# Tokenize protein sequence
|
| 417 |
+
# Size: B x T x C
|
| 418 |
+
seq_embeds = self.protein_sequence_embedding(protein_seq)
|
| 419 |
+
seq_embeds = self.seq_proj_in(seq_embeds)
|
| 420 |
+
seq_embeds = seq_embeds + self.protein_sequence_pos_embed[:, :protein_seq.shape[1]]
|
| 421 |
+
|
| 422 |
+
seq_token_type = torch.full(
|
| 423 |
+
size=(seq_embeds.shape[0], 1), fill_value=1, device=seq_embeds.device
|
| 424 |
+
).long()
|
| 425 |
+
|
| 426 |
+
seq_embeds = seq_embeds + self.token_type_embeddings(seq_token_type)
|
| 427 |
+
|
| 428 |
+
cell_img_conv = self.cond_conv(cell_img)
|
| 429 |
+
concat_img = torch.cat([protein_img, cell_img_conv], dim=1)
|
| 430 |
+
img_embeds = self.img_embedding(concat_img)
|
| 431 |
+
|
| 432 |
+
# add pos embed w/o time step token
|
| 433 |
+
img_embeds = img_embeds + self.img_encoder_pos_embed[:, 1:, :]
|
| 434 |
+
|
| 435 |
+
# masking: length -> length * mask_ratio
|
| 436 |
+
img_embeds, img_mask, img_ids_restore = self.random_masking(img_embeds, img_mask_ratio)
|
| 437 |
+
|
| 438 |
+
# append time step token
|
| 439 |
+
time_embeds_with_pos_embed = time_embeds.unsqueeze(1) + self.img_encoder_pos_embed[:, :1, :]
|
| 440 |
+
img_embeds = torch.cat([time_embeds_with_pos_embed, img_embeds], dim=1)
|
| 441 |
+
|
| 442 |
+
img_token_type = torch.full(
|
| 443 |
+
size=(img_embeds.shape[0], 1), fill_value=0, device=img_embeds.device,
|
| 444 |
+
).long()
|
| 445 |
+
|
| 446 |
+
img_embeds = img_embeds + self.token_type_embeddings(img_token_type)
|
| 447 |
+
co_embeds = torch.cat([img_embeds, seq_embeds], dim=1)
|
| 448 |
+
|
| 449 |
+
x = co_embeds
|
| 450 |
+
|
| 451 |
+
for i, block in enumerate(self.unified_encoder):
|
| 452 |
+
x = block(x)
|
| 453 |
+
|
| 454 |
+
co_feat = x
|
| 455 |
+
|
| 456 |
+
img_output = self.img_generator(
|
| 457 |
+
hidden_states=concat_img, timestep=time, encoder_hidden_states=co_feat,
|
| 458 |
+
pooled_projections=time_embeds,
|
| 459 |
+
).sample
|
| 460 |
+
|
| 461 |
+
seq_feat = co_feat[:, img_embeds.shape[1]:]
|
| 462 |
+
img_feat = co_feat[:, :img_embeds.shape[1]]
|
| 463 |
+
|
| 464 |
+
seq_output = self.seq_decoder(seq_feat)
|
| 465 |
+
|
| 466 |
+
img_recon = self.forward_img_decoder(img_feat, img_ids_restore)
|
| 467 |
+
img_recon = self.unpatchify(img_recon, self.config.latent_channels)
|
| 468 |
+
|
| 469 |
+
return img_output, seq_output, img_recon
|
| 470 |
+
|
| 471 |
+
def random_masking(self, x, mask_ratio):
|
| 472 |
+
"""
|
| 473 |
+
Perform per-sample random masking by per-sample shuffling.
|
| 474 |
+
Per-sample shuffling is done by argsort random noise.
|
| 475 |
+
x: [N, L, D], sequence
|
| 476 |
+
"""
|
| 477 |
+
N, L, D = x.shape # batch, length, dim
|
| 478 |
+
len_keep = int(L * (1 - mask_ratio))
|
| 479 |
+
|
| 480 |
+
noise = torch.rand(N, L, device=x.device) # noise in [0, 1]
|
| 481 |
+
|
| 482 |
+
# sort noise for each sample
|
| 483 |
+
ids_shuffle = torch.argsort(noise, dim=1) # ascend: small is keep, large is remove
|
| 484 |
+
ids_restore = torch.argsort(ids_shuffle, dim=1)
|
| 485 |
+
|
| 486 |
+
# keep the first subset
|
| 487 |
+
ids_keep = ids_shuffle[:, :len_keep]
|
| 488 |
+
x_masked = torch.gather(x, dim=1, index=ids_keep.unsqueeze(-1).repeat(1, 1, D))
|
| 489 |
+
|
| 490 |
+
# generate the binary mask: 0 is keep, 1 is remove
|
| 491 |
+
mask = torch.ones([N, L], device=x.device)
|
| 492 |
+
mask[:, :len_keep] = 0
|
| 493 |
+
# unshuffle to get the binary mask
|
| 494 |
+
mask = torch.gather(mask, dim=1, index=ids_restore)
|
| 495 |
+
|
| 496 |
+
return x_masked, mask, ids_restore
|
| 497 |
+
|
| 498 |
+
def forward_img_decoder(self, x, ids_restore):
|
| 499 |
+
# embed tokens
|
| 500 |
+
x = self.img_decoder_embed(x)
|
| 501 |
+
|
| 502 |
+
# append mask tokens to sequence
|
| 503 |
+
mask_tokens = self.mask_token.repeat(x.shape[0], ids_restore.shape[1] + 1 - x.shape[1], 1)
|
| 504 |
+
x_ = torch.cat([x[:, 1:, :], mask_tokens], dim=1) # no time step token
|
| 505 |
+
x_ = torch.gather(x_, dim=1, index=ids_restore.unsqueeze(-1).repeat(1, 1, x.shape[2])) # unshuffle
|
| 506 |
+
x = torch.cat([x[:, :1, :], x_], dim=1) # append time step token
|
| 507 |
+
|
| 508 |
+
# add pos embed
|
| 509 |
+
x = x + self.img_decoder_pos_embed
|
| 510 |
+
|
| 511 |
+
# apply Transformer blocks
|
| 512 |
+
for block in self.img_decoder:
|
| 513 |
+
x = block(x)
|
| 514 |
+
x = self.img_decoder_norm(x)
|
| 515 |
+
|
| 516 |
+
# predictor projection
|
| 517 |
+
x = self.img_decoder_pred(x)
|
| 518 |
+
|
| 519 |
+
# remove time step token
|
| 520 |
+
x = x[:, 1:, :]
|
| 521 |
+
|
| 522 |
+
return x
|
| 523 |
+
|
| 524 |
+
def unpatchify(self, x, num_channels):
|
| 525 |
+
"""
|
| 526 |
+
x: (N, L, patch_size**2 *C)
|
| 527 |
+
imgs: (N, C, H, W)
|
| 528 |
+
"""
|
| 529 |
+
p = self.config.encoder_patch_size
|
| 530 |
+
h = w = int(x.shape[1]**.5)
|
| 531 |
+
assert h * w == x.shape[1]
|
| 532 |
+
|
| 533 |
+
x = x.reshape(shape=(x.shape[0], h, w, p, p, num_channels))
|
| 534 |
+
x = torch.einsum('nhwpqc->nchpwq', x)
|
| 535 |
+
imgs = x.reshape(shape=(x.shape[0], num_channels, h * p, h * p))
|
| 536 |
+
return imgs
|
| 537 |
+
|
| 538 |
+
def embed(self, protein_img, cell_img, protein_seq, time, img_mask_ratio=0):
|
| 539 |
+
time_embeds = self.time_embedding(time)
|
| 540 |
+
|
| 541 |
+
# Tokenize protein sequence
|
| 542 |
+
# Size: B x T x C
|
| 543 |
+
seq_embeds = self.protein_sequence_embedding(protein_seq)
|
| 544 |
+
seq_embeds = self.seq_proj_in(seq_embeds)
|
| 545 |
+
seq_embeds = seq_embeds + self.protein_sequence_pos_embed[:, :protein_seq.shape[1]]
|
| 546 |
+
|
| 547 |
+
seq_token_type = torch.full(
|
| 548 |
+
size=(seq_embeds.shape[0], 1), fill_value=1, device=seq_embeds.device
|
| 549 |
+
).long()
|
| 550 |
+
|
| 551 |
+
seq_embeds = seq_embeds + self.token_type_embeddings(seq_token_type)
|
| 552 |
+
|
| 553 |
+
cell_img_conv = self.cond_conv(cell_img)
|
| 554 |
+
concat_img = torch.cat([protein_img, cell_img_conv], dim=1)
|
| 555 |
+
img_embeds = self.img_embedding(concat_img)
|
| 556 |
+
|
| 557 |
+
# add pos embed w/o time step token
|
| 558 |
+
img_embeds = img_embeds + self.img_encoder_pos_embed[:, 1:, :]
|
| 559 |
+
|
| 560 |
+
# masking: length -> length * mask_ratio
|
| 561 |
+
img_embeds, img_mask, img_ids_restore = self.random_masking(img_embeds, img_mask_ratio)
|
| 562 |
+
|
| 563 |
+
# append time step token
|
| 564 |
+
time_embeds_with_pos_embed = time_embeds.unsqueeze(1) + self.img_encoder_pos_embed[:, :1, :]
|
| 565 |
+
img_embeds = torch.cat([time_embeds_with_pos_embed, img_embeds], dim=1)
|
| 566 |
+
|
| 567 |
+
img_token_type = torch.full(
|
| 568 |
+
size=(img_embeds.shape[0], 1), fill_value=0, device=img_embeds.device,
|
| 569 |
+
).long()
|
| 570 |
+
|
| 571 |
+
img_embeds = img_embeds + self.token_type_embeddings(img_token_type)
|
| 572 |
+
co_embeds = torch.cat([img_embeds, seq_embeds], dim=1)
|
| 573 |
+
|
| 574 |
+
x = co_embeds
|
| 575 |
+
|
| 576 |
+
for i, block in enumerate(self.unified_encoder):
|
| 577 |
+
x = block(x)
|
| 578 |
+
|
| 579 |
+
co_feat = x
|
| 580 |
+
|
| 581 |
+
seq_feat = co_feat[:, img_embeds.shape[1]:]
|
| 582 |
+
img_feat = co_feat[:, :img_embeds.shape[1]]
|
| 583 |
+
|
| 584 |
+
img_feat_embed = img_feat
|
| 585 |
+
seq_feat_embed = seq_feat
|
| 586 |
+
|
| 587 |
+
return img_feat_embed, seq_feat_embed
|
| 588 |
+
|
| 589 |
+
def seq2img(self, protein_img, cell_img, protein_seq, time, img_mask_ratio=0):
|
| 590 |
+
time_embeds = self.time_embedding(time)
|
| 591 |
+
|
| 592 |
+
# Tokenize protein sequence
|
| 593 |
+
# Size: B x T x C
|
| 594 |
+
seq_embeds = self.protein_sequence_embedding(protein_seq)
|
| 595 |
+
seq_embeds = self.seq_proj_in(seq_embeds)
|
| 596 |
+
seq_embeds = seq_embeds + self.protein_sequence_pos_embed[:, :protein_seq.shape[1]]
|
| 597 |
+
|
| 598 |
+
seq_token_type = torch.full(
|
| 599 |
+
size=(seq_embeds.shape[0], 1), fill_value=1, device=seq_embeds.device
|
| 600 |
+
).long()
|
| 601 |
+
|
| 602 |
+
seq_embeds = seq_embeds + self.token_type_embeddings(seq_token_type)
|
| 603 |
+
|
| 604 |
+
cell_img_conv = self.cond_conv(cell_img)
|
| 605 |
+
concat_img = torch.cat([protein_img, cell_img_conv], dim=1)
|
| 606 |
+
img_embeds = self.img_embedding(concat_img)
|
| 607 |
+
|
| 608 |
+
# add pos embed w/o time step token
|
| 609 |
+
img_embeds = img_embeds + self.img_encoder_pos_embed[:, 1:, :]
|
| 610 |
+
|
| 611 |
+
# masking: length -> length * mask_ratio
|
| 612 |
+
img_embeds, img_mask, img_ids_restore = self.random_masking(img_embeds, img_mask_ratio)
|
| 613 |
+
|
| 614 |
+
# append time step token
|
| 615 |
+
time_embeds_with_pos_embed = time_embeds.unsqueeze(1) + self.img_encoder_pos_embed[:, :1, :]
|
| 616 |
+
img_embeds = torch.cat([time_embeds_with_pos_embed, img_embeds], dim=1)
|
| 617 |
+
|
| 618 |
+
img_token_type = torch.full(
|
| 619 |
+
size=(img_embeds.shape[0], 1), fill_value=0, device=img_embeds.device,
|
| 620 |
+
).long()
|
| 621 |
+
|
| 622 |
+
img_embeds = img_embeds + self.token_type_embeddings(img_token_type)
|
| 623 |
+
co_embeds = torch.cat([img_embeds, seq_embeds], dim=1)
|
| 624 |
+
|
| 625 |
+
x = co_embeds
|
| 626 |
+
|
| 627 |
+
for i, block in enumerate(self.unified_encoder):
|
| 628 |
+
x = block(x)
|
| 629 |
+
|
| 630 |
+
co_feat = x
|
| 631 |
+
|
| 632 |
+
img_output = self.img_generator(
|
| 633 |
+
hidden_states=concat_img, timestep=time, encoder_hidden_states=co_feat,
|
| 634 |
+
pooled_projections=time_embeds,
|
| 635 |
+
).sample
|
| 636 |
+
|
| 637 |
+
return img_output
|
cell_fm/models/cell_fm/modules/__init__.py
ADDED
|
@@ -0,0 +1,82 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
import torch.nn as nn
|
| 3 |
+
import math
|
| 4 |
+
|
| 5 |
+
class TimestepEmbedder(nn.Module):
|
| 6 |
+
"""
|
| 7 |
+
Embeds scalar timesteps into vector representations.
|
| 8 |
+
"""
|
| 9 |
+
def __init__(self, hidden_size, frequency_embedding_size=256):
|
| 10 |
+
super().__init__()
|
| 11 |
+
self.mlp = nn.Sequential(
|
| 12 |
+
nn.Linear(frequency_embedding_size, hidden_size, bias=True),
|
| 13 |
+
nn.SiLU(),
|
| 14 |
+
nn.Linear(hidden_size, hidden_size, bias=True),
|
| 15 |
+
)
|
| 16 |
+
self.frequency_embedding_size = frequency_embedding_size
|
| 17 |
+
|
| 18 |
+
@staticmethod
|
| 19 |
+
def timestep_embedding(t, dim, max_period=10000):
|
| 20 |
+
"""
|
| 21 |
+
Create sinusoidal timestep embeddings.
|
| 22 |
+
:param t: a 1-D Tensor of N indices, one per batch element.
|
| 23 |
+
These may be fractional.
|
| 24 |
+
:param dim: the dimension of the output.
|
| 25 |
+
:param max_period: controls the minimum frequency of the embeddings.
|
| 26 |
+
:return: an (N, D) Tensor of positional embeddings.
|
| 27 |
+
"""
|
| 28 |
+
# https://github.com/openai/glide-text2im/blob/main/glide_text2im/nn.py
|
| 29 |
+
half = dim // 2
|
| 30 |
+
freqs = torch.exp(
|
| 31 |
+
-math.log(max_period) * torch.arange(start=0, end=half, dtype=torch.float32) / half
|
| 32 |
+
).to(device=t.device)
|
| 33 |
+
args = t[:, None].float() * freqs[None]
|
| 34 |
+
embedding = torch.cat([torch.cos(args), torch.sin(args)], dim=-1)
|
| 35 |
+
if dim % 2:
|
| 36 |
+
embedding = torch.cat([embedding, torch.zeros_like(embedding[:, :1])], dim=-1)
|
| 37 |
+
return embedding
|
| 38 |
+
|
| 39 |
+
def forward(self, t):
|
| 40 |
+
t_freq = self.timestep_embedding(t, self.frequency_embedding_size)
|
| 41 |
+
t_emb = self.mlp(t_freq)
|
| 42 |
+
return t_emb
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
class CondConvNet(nn.Module):
|
| 46 |
+
def __init__(self, in_channels, cond_out_channels):
|
| 47 |
+
"""
|
| 48 |
+
Parameters:
|
| 49 |
+
- in_channels (int): Number of channels per input image.
|
| 50 |
+
- cond_out_channels (list of int): Output channels for each conv layer.
|
| 51 |
+
"""
|
| 52 |
+
super(CondConvNet, self).__init__()
|
| 53 |
+
layers = []
|
| 54 |
+
|
| 55 |
+
# First convolution layer
|
| 56 |
+
layers.append(
|
| 57 |
+
nn.Conv2d(
|
| 58 |
+
in_channels,
|
| 59 |
+
cond_out_channels[0],
|
| 60 |
+
kernel_size=2,
|
| 61 |
+
stride=2,
|
| 62 |
+
padding=0
|
| 63 |
+
)
|
| 64 |
+
)
|
| 65 |
+
|
| 66 |
+
# Intermediate layers
|
| 67 |
+
for i in range(len(cond_out_channels) - 1):
|
| 68 |
+
layers.append(nn.SiLU())
|
| 69 |
+
layers.append(
|
| 70 |
+
nn.Conv2d(
|
| 71 |
+
cond_out_channels[i],
|
| 72 |
+
cond_out_channels[i + 1],
|
| 73 |
+
kernel_size=2,
|
| 74 |
+
stride=2,
|
| 75 |
+
padding=0
|
| 76 |
+
)
|
| 77 |
+
)
|
| 78 |
+
|
| 79 |
+
self.model = nn.Sequential(*layers)
|
| 80 |
+
|
| 81 |
+
def forward(self, x):
|
| 82 |
+
return self.model(x)
|
cell_fm/models/cell_fm/modules/positional_embedding.py
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import numpy as np
|
| 2 |
+
|
| 3 |
+
#################################################################################
|
| 4 |
+
# Sine/Cosine Positional Embedding Functions #
|
| 5 |
+
#################################################################################
|
| 6 |
+
# https://github.com/facebookresearch/mae/blob/main/util/pos_embed.py
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
def get_2d_sincos_pos_embed(embed_dim, grid_size, cls_token=False, extra_tokens=0):
|
| 10 |
+
"""
|
| 11 |
+
grid_size: int of the grid height and width
|
| 12 |
+
return:
|
| 13 |
+
pos_embed: [grid_size*grid_size, embed_dim] or [1+grid_size*grid_size, embed_dim] (w/ or w/o cls_token)
|
| 14 |
+
"""
|
| 15 |
+
grid_h = np.arange(grid_size, dtype=np.float32)
|
| 16 |
+
grid_w = np.arange(grid_size, dtype=np.float32)
|
| 17 |
+
grid = np.meshgrid(grid_w, grid_h) # here w goes first
|
| 18 |
+
grid = np.stack(grid, axis=0)
|
| 19 |
+
|
| 20 |
+
grid = grid.reshape([2, 1, grid_size, grid_size])
|
| 21 |
+
pos_embed = get_2d_sincos_pos_embed_from_grid(embed_dim, grid)
|
| 22 |
+
if cls_token and extra_tokens > 0:
|
| 23 |
+
pos_embed = np.concatenate([np.zeros([extra_tokens, embed_dim]), pos_embed], axis=0)
|
| 24 |
+
return pos_embed
|
| 25 |
+
|
| 26 |
+
def get_1d_sincos_pos_embed(embed_dim, grid_size):
|
| 27 |
+
return get_1d_sincos_pos_embed_from_grid(embed_dim, pos=np.arange(grid_size, dtype=np.float32))
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def get_2d_sincos_pos_embed_from_grid(embed_dim, grid):
|
| 31 |
+
assert embed_dim % 2 == 0
|
| 32 |
+
|
| 33 |
+
# use half of dimensions to encode grid_h
|
| 34 |
+
emb_h = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[0]) # (H*W, D/2)
|
| 35 |
+
emb_w = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[1]) # (H*W, D/2)
|
| 36 |
+
|
| 37 |
+
emb = np.concatenate([emb_h, emb_w], axis=1) # (H*W, D)
|
| 38 |
+
return emb
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def get_1d_sincos_pos_embed_from_grid(embed_dim, pos):
|
| 42 |
+
"""
|
| 43 |
+
embed_dim: output dimension for each position
|
| 44 |
+
pos: a list of positions to be encoded: size (M,)
|
| 45 |
+
out: (M, D)
|
| 46 |
+
"""
|
| 47 |
+
assert embed_dim % 2 == 0
|
| 48 |
+
omega = np.arange(embed_dim // 2, dtype=np.float64)
|
| 49 |
+
omega /= embed_dim / 2.
|
| 50 |
+
omega = 1. / 10000**omega # (D/2,)
|
| 51 |
+
|
| 52 |
+
pos = pos.reshape(-1) # (M,)
|
| 53 |
+
out = np.einsum('m,d->md', pos, omega) # (M, D/2), outer product
|
| 54 |
+
|
| 55 |
+
emb_sin = np.sin(out) # (M, D/2)
|
| 56 |
+
emb_cos = np.cos(out) # (M, D/2)
|
| 57 |
+
|
| 58 |
+
emb = np.concatenate([emb_sin, emb_cos], axis=1) # (M, D)
|
| 59 |
+
return emb
|
cell_fm/models/cell_fm/modules/protein_sequence_embedding.py
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
import torch.nn as nn
|
| 3 |
+
from esm.models.esmc import ESMC
|
| 4 |
+
|
| 5 |
+
class ESMEmbed(nn.Module):
|
| 6 |
+
def __init__(self, model_name, out_dims, fixed_embedding=True):
|
| 7 |
+
super(ESMEmbed, self).__init__()
|
| 8 |
+
|
| 9 |
+
if model_name == 'esmc_300m':
|
| 10 |
+
self.model = ESMC.from_pretrained("esmc_300m")
|
| 11 |
+
model_dim = 960
|
| 12 |
+
elif model_name == 'esmc_600m':
|
| 13 |
+
self.model = ESMC.from_pretrained("esmc_600m")
|
| 14 |
+
model_dim = 1152
|
| 15 |
+
|
| 16 |
+
if out_dims != model_dim:
|
| 17 |
+
self.scale_layer = nn.Linear(model_dim, out_dims)
|
| 18 |
+
else:
|
| 19 |
+
self.scale_layer = nn.Identity()
|
| 20 |
+
|
| 21 |
+
# Determine whether to freeze the model's parameters
|
| 22 |
+
self.fixed_embedding = fixed_embedding
|
| 23 |
+
if self.fixed_embedding:
|
| 24 |
+
for param in self.model.parameters():
|
| 25 |
+
param.requires_grad = False
|
| 26 |
+
self.model = self.model.eval()
|
| 27 |
+
|
| 28 |
+
def forward(self, x, **kwargs):
|
| 29 |
+
# If the model's parameters are fixed, use torch.no_grad()
|
| 30 |
+
if self.fixed_embedding:
|
| 31 |
+
with torch.no_grad():
|
| 32 |
+
x = self.model(x).embeddings
|
| 33 |
+
else:
|
| 34 |
+
x = self.model(x).embeddings
|
| 35 |
+
|
| 36 |
+
x = x.to(torch.float32)
|
| 37 |
+
x = self.scale_layer(x)
|
| 38 |
+
|
| 39 |
+
return x
|
cell_fm/models/cell_fm/modules/transformer.py
ADDED
|
@@ -0,0 +1,131 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
import torch.nn as nn
|
| 3 |
+
from diffusers.models.attention_processor import Attention
|
| 4 |
+
from diffusers.models.attention import FeedForward
|
| 5 |
+
from transformers.activations import ACT2FN
|
| 6 |
+
|
| 7 |
+
class BertPredictionHeadTransform(nn.Module):
|
| 8 |
+
def __init__(
|
| 9 |
+
self,
|
| 10 |
+
hidden_size,
|
| 11 |
+
hidden_act: str = 'gelu',
|
| 12 |
+
norm_eps: float = 1e-5,
|
| 13 |
+
):
|
| 14 |
+
super().__init__()
|
| 15 |
+
self.dense = nn.Linear(hidden_size, hidden_size)
|
| 16 |
+
self.transform_act_fn = ACT2FN[hidden_act]
|
| 17 |
+
self.LayerNorm = nn.LayerNorm(hidden_size, norm_eps)
|
| 18 |
+
|
| 19 |
+
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
| 20 |
+
hidden_states = self.dense(hidden_states)
|
| 21 |
+
hidden_states = self.transform_act_fn(hidden_states)
|
| 22 |
+
hidden_states = self.LayerNorm(hidden_states)
|
| 23 |
+
return hidden_states
|
| 24 |
+
|
| 25 |
+
class MLMHead(nn.Module):
|
| 26 |
+
def __init__(
|
| 27 |
+
self,
|
| 28 |
+
hidden_size,
|
| 29 |
+
vocab_size,
|
| 30 |
+
hidden_act: str = 'gelu',
|
| 31 |
+
norm_eps: float = 1e-5,
|
| 32 |
+
weight=None
|
| 33 |
+
):
|
| 34 |
+
super().__init__()
|
| 35 |
+
self.transform = BertPredictionHeadTransform(hidden_size, hidden_act, norm_eps)
|
| 36 |
+
self.decoder = nn.Linear(hidden_size, vocab_size, bias=False)
|
| 37 |
+
self.bias = nn.Parameter(torch.zeros(vocab_size))
|
| 38 |
+
if weight is not None:
|
| 39 |
+
self.decoder.weight = weight
|
| 40 |
+
|
| 41 |
+
def forward(self, x):
|
| 42 |
+
x = self.transform(x)
|
| 43 |
+
x = self.decoder(x) + self.bias
|
| 44 |
+
return x
|
| 45 |
+
|
| 46 |
+
class TransformerBlock(nn.Module):
|
| 47 |
+
def __init__(
|
| 48 |
+
self,
|
| 49 |
+
hidden_size,
|
| 50 |
+
num_heads,
|
| 51 |
+
dim_head,
|
| 52 |
+
dropout: float = 0.0,
|
| 53 |
+
final_dropout: float = 0.0,
|
| 54 |
+
norm_elementwise_affine: bool = True,
|
| 55 |
+
norm_eps: float = 1e-5,
|
| 56 |
+
attention_bias: bool = False,
|
| 57 |
+
attention_out_bias: bool = True,
|
| 58 |
+
activation_fn: str = "geglu",
|
| 59 |
+
ff_inner_dim = None,
|
| 60 |
+
ff_bias: bool = True,
|
| 61 |
+
):
|
| 62 |
+
super().__init__()
|
| 63 |
+
self.norm1 = nn.LayerNorm(hidden_size, elementwise_affine=norm_elementwise_affine, eps=norm_eps)
|
| 64 |
+
|
| 65 |
+
self.attn = Attention(
|
| 66 |
+
query_dim=hidden_size,
|
| 67 |
+
heads=num_heads,
|
| 68 |
+
dim_head=dim_head,
|
| 69 |
+
dropout=dropout,
|
| 70 |
+
bias=attention_bias,
|
| 71 |
+
cross_attention_dim=None,
|
| 72 |
+
upcast_attention=False,
|
| 73 |
+
out_bias=attention_out_bias,
|
| 74 |
+
)
|
| 75 |
+
|
| 76 |
+
self.ff = FeedForward(
|
| 77 |
+
hidden_size,
|
| 78 |
+
dropout=dropout,
|
| 79 |
+
activation_fn=activation_fn,
|
| 80 |
+
final_dropout=final_dropout,
|
| 81 |
+
inner_dim=ff_inner_dim,
|
| 82 |
+
bias=ff_bias,
|
| 83 |
+
)
|
| 84 |
+
|
| 85 |
+
self.norm2 = nn.LayerNorm(hidden_size, norm_eps, norm_elementwise_affine)
|
| 86 |
+
|
| 87 |
+
def forward(self, hidden_states):
|
| 88 |
+
norm_hidden_states = self.norm1(hidden_states)
|
| 89 |
+
attn_output = self.attn(norm_hidden_states)
|
| 90 |
+
|
| 91 |
+
hidden_states = attn_output + hidden_states
|
| 92 |
+
norm_hidden_states = self.norm2(hidden_states)
|
| 93 |
+
|
| 94 |
+
ff_output = self.ff(norm_hidden_states)
|
| 95 |
+
hidden_states = ff_output + hidden_states
|
| 96 |
+
|
| 97 |
+
return hidden_states
|
| 98 |
+
|
| 99 |
+
def unpatchify(x, out_channels, patch_size):
|
| 100 |
+
"""
|
| 101 |
+
x: (N, T, patch_size**2 * C)
|
| 102 |
+
imgs: (N, C, H, W)
|
| 103 |
+
"""
|
| 104 |
+
c = out_channels
|
| 105 |
+
p = patch_size
|
| 106 |
+
h = w = int(x.shape[1] ** 0.5)
|
| 107 |
+
assert h * w == x.shape[1]
|
| 108 |
+
|
| 109 |
+
x = x.reshape(shape=(x.shape[0], h, w, p, p, c))
|
| 110 |
+
x = torch.einsum('nhwpqc->nchpwq', x)
|
| 111 |
+
imgs = x.reshape(shape=(x.shape[0], c, h * p, h * p))
|
| 112 |
+
return imgs
|
| 113 |
+
|
| 114 |
+
class FinalLayer(nn.Module):
|
| 115 |
+
"""
|
| 116 |
+
The final layer of DiT.
|
| 117 |
+
"""
|
| 118 |
+
def __init__(self, hidden_size, patch_size, out_channels):
|
| 119 |
+
super().__init__()
|
| 120 |
+
self.norm_final = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6)
|
| 121 |
+
self.linear = nn.Linear(hidden_size, patch_size * patch_size * out_channels, bias=True)
|
| 122 |
+
self.adaLN_modulation = nn.Sequential(
|
| 123 |
+
nn.SiLU(),
|
| 124 |
+
nn.Linear(hidden_size, 2 * hidden_size, bias=True)
|
| 125 |
+
)
|
| 126 |
+
|
| 127 |
+
def forward(self, x, c):
|
| 128 |
+
shift, scale = self.adaLN_modulation(c).chunk(2, dim=1)
|
| 129 |
+
x = self.linear(x)
|
| 130 |
+
return x
|
| 131 |
+
|
cell_fm/models/cell_fm/modules/transport/__init__.py
ADDED
|
@@ -0,0 +1,84 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from .transport import Transport, ModelType, WeightType, PathType, TimestepSampler, Sampler
|
| 2 |
+
|
| 3 |
+
def create_transport(
|
| 4 |
+
path_type='Linear',
|
| 5 |
+
prediction="velocity",
|
| 6 |
+
loss_weight=None,
|
| 7 |
+
train_eps=None,
|
| 8 |
+
sample_eps=None,
|
| 9 |
+
timestep_sampler='uniform',
|
| 10 |
+
logit_mean=0.0,
|
| 11 |
+
logit_std=1.0,
|
| 12 |
+
):
|
| 13 |
+
"""function for creating Transport object
|
| 14 |
+
**Note**: model prediction defaults to velocity
|
| 15 |
+
Args:
|
| 16 |
+
- path_type: type of path to use; default to linear
|
| 17 |
+
- learn_score: set model prediction to score
|
| 18 |
+
- learn_noise: set model prediction to noise
|
| 19 |
+
- velocity_weighted: weight loss by velocity weight
|
| 20 |
+
- likelihood_weighted: weight loss by likelihood weight
|
| 21 |
+
- train_eps: small epsilon for avoiding instability during training
|
| 22 |
+
- sample_eps: small epsilon for avoiding instability during sampling
|
| 23 |
+
- timestep_sampler: how to draw the training timestep; 'uniform' or 'logit_normal' (SD3)
|
| 24 |
+
- logit_mean: mean of the logit-normal draw; SD3 convention, positive favours the noise end
|
| 25 |
+
- logit_std: std of the logit-normal draw
|
| 26 |
+
"""
|
| 27 |
+
|
| 28 |
+
if prediction == "noise":
|
| 29 |
+
model_type = ModelType.NOISE
|
| 30 |
+
elif prediction == "score":
|
| 31 |
+
model_type = ModelType.SCORE
|
| 32 |
+
else:
|
| 33 |
+
model_type = ModelType.VELOCITY
|
| 34 |
+
|
| 35 |
+
if loss_weight == "velocity":
|
| 36 |
+
loss_type = WeightType.VELOCITY
|
| 37 |
+
elif loss_weight == "likelihood":
|
| 38 |
+
loss_type = WeightType.LIKELIHOOD
|
| 39 |
+
else:
|
| 40 |
+
loss_type = WeightType.NONE
|
| 41 |
+
|
| 42 |
+
sampler_choice = {
|
| 43 |
+
"uniform": TimestepSampler.UNIFORM,
|
| 44 |
+
"logit_normal": TimestepSampler.LOGIT_NORMAL,
|
| 45 |
+
}
|
| 46 |
+
|
| 47 |
+
if timestep_sampler not in sampler_choice:
|
| 48 |
+
raise ValueError(
|
| 49 |
+
f"Unknown timestep_sampler '{timestep_sampler}'; expected one of {sorted(sampler_choice)}"
|
| 50 |
+
)
|
| 51 |
+
|
| 52 |
+
timestep_sampler = sampler_choice[timestep_sampler]
|
| 53 |
+
|
| 54 |
+
path_choice = {
|
| 55 |
+
"Linear": PathType.LINEAR,
|
| 56 |
+
"GVP": PathType.GVP,
|
| 57 |
+
"VP": PathType.VP,
|
| 58 |
+
}
|
| 59 |
+
|
| 60 |
+
path_type = path_choice[path_type]
|
| 61 |
+
|
| 62 |
+
if (path_type in [PathType.VP]):
|
| 63 |
+
train_eps = 1e-5 if train_eps is None else train_eps
|
| 64 |
+
sample_eps = 1e-3 if train_eps is None else sample_eps
|
| 65 |
+
elif (path_type in [PathType.GVP, PathType.LINEAR] and model_type != ModelType.VELOCITY):
|
| 66 |
+
train_eps = 1e-3 if train_eps is None else train_eps
|
| 67 |
+
sample_eps = 1e-3 if train_eps is None else sample_eps
|
| 68 |
+
else: # velocity & [GVP, LINEAR] is stable everywhere
|
| 69 |
+
train_eps = 0
|
| 70 |
+
sample_eps = 0
|
| 71 |
+
|
| 72 |
+
# create flow state
|
| 73 |
+
state = Transport(
|
| 74 |
+
model_type=model_type,
|
| 75 |
+
path_type=path_type,
|
| 76 |
+
loss_type=loss_type,
|
| 77 |
+
train_eps=train_eps,
|
| 78 |
+
sample_eps=sample_eps,
|
| 79 |
+
timestep_sampler=timestep_sampler,
|
| 80 |
+
logit_mean=logit_mean,
|
| 81 |
+
logit_std=logit_std,
|
| 82 |
+
)
|
| 83 |
+
|
| 84 |
+
return state
|
cell_fm/models/cell_fm/modules/transport/integrators.py
ADDED
|
@@ -0,0 +1,117 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import numpy as np
|
| 2 |
+
import torch as th
|
| 3 |
+
import torch.nn as nn
|
| 4 |
+
from torchdiffeq import odeint
|
| 5 |
+
from functools import partial
|
| 6 |
+
from tqdm import tqdm
|
| 7 |
+
|
| 8 |
+
class sde:
|
| 9 |
+
"""SDE solver class"""
|
| 10 |
+
def __init__(
|
| 11 |
+
self,
|
| 12 |
+
drift,
|
| 13 |
+
diffusion,
|
| 14 |
+
*,
|
| 15 |
+
t0,
|
| 16 |
+
t1,
|
| 17 |
+
num_steps,
|
| 18 |
+
sampler_type,
|
| 19 |
+
):
|
| 20 |
+
assert t0 < t1, "SDE sampler has to be in forward time"
|
| 21 |
+
|
| 22 |
+
self.num_timesteps = num_steps
|
| 23 |
+
self.t = th.linspace(t0, t1, num_steps)
|
| 24 |
+
self.dt = self.t[1] - self.t[0]
|
| 25 |
+
self.drift = drift
|
| 26 |
+
self.diffusion = diffusion
|
| 27 |
+
self.sampler_type = sampler_type
|
| 28 |
+
|
| 29 |
+
def __Euler_Maruyama_step(self, x, mean_x, t, model, **model_kwargs):
|
| 30 |
+
w_cur = th.randn(x.size()).to(x)
|
| 31 |
+
t = th.ones(x.size(0)).to(x) * t
|
| 32 |
+
dw = w_cur * th.sqrt(self.dt)
|
| 33 |
+
drift = self.drift(x, t, model, **model_kwargs)
|
| 34 |
+
diffusion = self.diffusion(x, t)
|
| 35 |
+
mean_x = x + drift * self.dt
|
| 36 |
+
x = mean_x + th.sqrt(2 * diffusion) * dw
|
| 37 |
+
return x, mean_x
|
| 38 |
+
|
| 39 |
+
def __Heun_step(self, x, _, t, model, **model_kwargs):
|
| 40 |
+
w_cur = th.randn(x.size()).to(x)
|
| 41 |
+
dw = w_cur * th.sqrt(self.dt)
|
| 42 |
+
t_cur = th.ones(x.size(0)).to(x) * t
|
| 43 |
+
diffusion = self.diffusion(x, t_cur)
|
| 44 |
+
xhat = x + th.sqrt(2 * diffusion) * dw
|
| 45 |
+
K1 = self.drift(xhat, t_cur, model, **model_kwargs)
|
| 46 |
+
xp = xhat + self.dt * K1
|
| 47 |
+
K2 = self.drift(xp, t_cur + self.dt, model, **model_kwargs)
|
| 48 |
+
return xhat + 0.5 * self.dt * (K1 + K2), xhat # at last time point we do not perform the heun step
|
| 49 |
+
|
| 50 |
+
def __forward_fn(self):
|
| 51 |
+
"""TODO: generalize here by adding all private functions ending with steps to it"""
|
| 52 |
+
sampler_dict = {
|
| 53 |
+
"Euler": self.__Euler_Maruyama_step,
|
| 54 |
+
"Heun": self.__Heun_step,
|
| 55 |
+
}
|
| 56 |
+
|
| 57 |
+
try:
|
| 58 |
+
sampler = sampler_dict[self.sampler_type]
|
| 59 |
+
except:
|
| 60 |
+
raise NotImplementedError("Smapler type not implemented.")
|
| 61 |
+
|
| 62 |
+
return sampler
|
| 63 |
+
|
| 64 |
+
def sample(self, init, model, **model_kwargs):
|
| 65 |
+
"""forward loop of sde"""
|
| 66 |
+
x = init
|
| 67 |
+
mean_x = init
|
| 68 |
+
samples = []
|
| 69 |
+
sampler = self.__forward_fn()
|
| 70 |
+
for ti in self.t[:-1]:
|
| 71 |
+
with th.no_grad():
|
| 72 |
+
x, mean_x = sampler(x, mean_x, ti, model, **model_kwargs)
|
| 73 |
+
samples.append(x)
|
| 74 |
+
|
| 75 |
+
return samples
|
| 76 |
+
|
| 77 |
+
class ode:
|
| 78 |
+
"""ODE solver class"""
|
| 79 |
+
def __init__(
|
| 80 |
+
self,
|
| 81 |
+
drift,
|
| 82 |
+
*,
|
| 83 |
+
t0,
|
| 84 |
+
t1,
|
| 85 |
+
sampler_type,
|
| 86 |
+
num_steps,
|
| 87 |
+
atol,
|
| 88 |
+
rtol,
|
| 89 |
+
):
|
| 90 |
+
assert t0 < t1, "ODE sampler has to be in forward time"
|
| 91 |
+
|
| 92 |
+
self.drift = drift
|
| 93 |
+
self.t = th.linspace(t0, t1, num_steps)
|
| 94 |
+
self.atol = atol
|
| 95 |
+
self.rtol = rtol
|
| 96 |
+
self.sampler_type = sampler_type
|
| 97 |
+
|
| 98 |
+
def sample(self, x, model, **model_kwargs):
|
| 99 |
+
|
| 100 |
+
device = x[0].device if isinstance(x, tuple) else x.device
|
| 101 |
+
def _fn(t, x):
|
| 102 |
+
t = th.ones(x[0].size(0)).to(device) * t if isinstance(x, tuple) else th.ones(x.size(0)).to(device) * t
|
| 103 |
+
model_output = self.drift(x, t, model, **model_kwargs)
|
| 104 |
+
return model_output
|
| 105 |
+
|
| 106 |
+
t = self.t.to(device)
|
| 107 |
+
atol = [self.atol] * len(x) if isinstance(x, tuple) else [self.atol]
|
| 108 |
+
rtol = [self.rtol] * len(x) if isinstance(x, tuple) else [self.rtol]
|
| 109 |
+
samples = odeint(
|
| 110 |
+
_fn,
|
| 111 |
+
x,
|
| 112 |
+
t,
|
| 113 |
+
method=self.sampler_type,
|
| 114 |
+
atol=atol,
|
| 115 |
+
rtol=rtol
|
| 116 |
+
)
|
| 117 |
+
return samples
|
cell_fm/models/cell_fm/modules/transport/path.py
ADDED
|
@@ -0,0 +1,192 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch as th
|
| 2 |
+
import numpy as np
|
| 3 |
+
from functools import partial
|
| 4 |
+
|
| 5 |
+
def expand_t_like_x(t, x):
|
| 6 |
+
"""Function to reshape time t to broadcastable dimension of x
|
| 7 |
+
Args:
|
| 8 |
+
t: [batch_dim,], time vector
|
| 9 |
+
x: [batch_dim,...], data point
|
| 10 |
+
"""
|
| 11 |
+
dims = [1] * (len(x.size()) - 1)
|
| 12 |
+
t = t.view(t.size(0), *dims)
|
| 13 |
+
return t
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
#################### Coupling Plans ####################
|
| 17 |
+
|
| 18 |
+
class ICPlan:
|
| 19 |
+
"""Linear Coupling Plan"""
|
| 20 |
+
def __init__(self, sigma=0.0):
|
| 21 |
+
self.sigma = sigma
|
| 22 |
+
|
| 23 |
+
def compute_alpha_t(self, t):
|
| 24 |
+
"""Compute the data coefficient along the path"""
|
| 25 |
+
return t, 1
|
| 26 |
+
|
| 27 |
+
def compute_sigma_t(self, t):
|
| 28 |
+
"""Compute the noise coefficient along the path"""
|
| 29 |
+
return 1 - t, -1
|
| 30 |
+
|
| 31 |
+
def compute_d_alpha_alpha_ratio_t(self, t):
|
| 32 |
+
"""Compute the ratio between d_alpha and alpha"""
|
| 33 |
+
return 1 / t
|
| 34 |
+
|
| 35 |
+
def compute_drift(self, x, t):
|
| 36 |
+
"""We always output sde according to score parametrization; """
|
| 37 |
+
t = expand_t_like_x(t, x)
|
| 38 |
+
alpha_ratio = self.compute_d_alpha_alpha_ratio_t(t)
|
| 39 |
+
sigma_t, d_sigma_t = self.compute_sigma_t(t)
|
| 40 |
+
drift = alpha_ratio * x
|
| 41 |
+
diffusion = alpha_ratio * (sigma_t ** 2) - sigma_t * d_sigma_t
|
| 42 |
+
|
| 43 |
+
return -drift, diffusion
|
| 44 |
+
|
| 45 |
+
def compute_diffusion(self, x, t, form="constant", norm=1.0):
|
| 46 |
+
"""Compute the diffusion term of the SDE
|
| 47 |
+
Args:
|
| 48 |
+
x: [batch_dim, ...], data point
|
| 49 |
+
t: [batch_dim,], time vector
|
| 50 |
+
form: str, form of the diffusion term
|
| 51 |
+
norm: float, norm of the diffusion term
|
| 52 |
+
"""
|
| 53 |
+
t = expand_t_like_x(t, x)
|
| 54 |
+
choices = {
|
| 55 |
+
"constant": norm,
|
| 56 |
+
"SBDM": norm * self.compute_drift(x, t)[1],
|
| 57 |
+
"sigma": norm * self.compute_sigma_t(t)[0],
|
| 58 |
+
"linear": norm * (1 - t),
|
| 59 |
+
"decreasing": 0.25 * (norm * th.cos(np.pi * t) + 1) ** 2,
|
| 60 |
+
"inccreasing-decreasing": norm * th.sin(np.pi * t) ** 2,
|
| 61 |
+
}
|
| 62 |
+
|
| 63 |
+
try:
|
| 64 |
+
diffusion = choices[form]
|
| 65 |
+
except KeyError:
|
| 66 |
+
raise NotImplementedError(f"Diffusion form {form} not implemented")
|
| 67 |
+
|
| 68 |
+
return diffusion
|
| 69 |
+
|
| 70 |
+
def get_score_from_velocity(self, velocity, x, t):
|
| 71 |
+
"""Wrapper function: transfrom velocity prediction model to score
|
| 72 |
+
Args:
|
| 73 |
+
velocity: [batch_dim, ...] shaped tensor; velocity model output
|
| 74 |
+
x: [batch_dim, ...] shaped tensor; x_t data point
|
| 75 |
+
t: [batch_dim,] time tensor
|
| 76 |
+
"""
|
| 77 |
+
t = expand_t_like_x(t, x)
|
| 78 |
+
alpha_t, d_alpha_t = self.compute_alpha_t(t)
|
| 79 |
+
sigma_t, d_sigma_t = self.compute_sigma_t(t)
|
| 80 |
+
mean = x
|
| 81 |
+
reverse_alpha_ratio = alpha_t / d_alpha_t
|
| 82 |
+
var = sigma_t**2 - reverse_alpha_ratio * d_sigma_t * sigma_t
|
| 83 |
+
score = (reverse_alpha_ratio * velocity - mean) / var
|
| 84 |
+
return score
|
| 85 |
+
|
| 86 |
+
def get_noise_from_velocity(self, velocity, x, t):
|
| 87 |
+
"""Wrapper function: transfrom velocity prediction model to denoiser
|
| 88 |
+
Args:
|
| 89 |
+
velocity: [batch_dim, ...] shaped tensor; velocity model output
|
| 90 |
+
x: [batch_dim, ...] shaped tensor; x_t data point
|
| 91 |
+
t: [batch_dim,] time tensor
|
| 92 |
+
"""
|
| 93 |
+
t = expand_t_like_x(t, x)
|
| 94 |
+
alpha_t, d_alpha_t = self.compute_alpha_t(t)
|
| 95 |
+
sigma_t, d_sigma_t = self.compute_sigma_t(t)
|
| 96 |
+
mean = x
|
| 97 |
+
reverse_alpha_ratio = alpha_t / d_alpha_t
|
| 98 |
+
var = reverse_alpha_ratio * d_sigma_t - sigma_t
|
| 99 |
+
noise = (reverse_alpha_ratio * velocity - mean) / var
|
| 100 |
+
return noise
|
| 101 |
+
|
| 102 |
+
def get_velocity_from_score(self, score, x, t):
|
| 103 |
+
"""Wrapper function: transfrom score prediction model to velocity
|
| 104 |
+
Args:
|
| 105 |
+
score: [batch_dim, ...] shaped tensor; score model output
|
| 106 |
+
x: [batch_dim, ...] shaped tensor; x_t data point
|
| 107 |
+
t: [batch_dim,] time tensor
|
| 108 |
+
"""
|
| 109 |
+
t = expand_t_like_x(t, x)
|
| 110 |
+
drift, var = self.compute_drift(x, t)
|
| 111 |
+
velocity = var * score - drift
|
| 112 |
+
return velocity
|
| 113 |
+
|
| 114 |
+
def compute_mu_t(self, t, x0, x1):
|
| 115 |
+
"""Compute the mean of time-dependent density p_t"""
|
| 116 |
+
t = expand_t_like_x(t, x1)
|
| 117 |
+
alpha_t, _ = self.compute_alpha_t(t)
|
| 118 |
+
sigma_t, _ = self.compute_sigma_t(t)
|
| 119 |
+
return alpha_t * x1 + sigma_t * x0
|
| 120 |
+
|
| 121 |
+
def compute_xt(self, t, x0, x1):
|
| 122 |
+
"""Sample xt from time-dependent density p_t; rng is required"""
|
| 123 |
+
xt = self.compute_mu_t(t, x0, x1)
|
| 124 |
+
return xt
|
| 125 |
+
|
| 126 |
+
def compute_ut(self, t, x0, x1, xt):
|
| 127 |
+
"""Compute the vector field corresponding to p_t"""
|
| 128 |
+
t = expand_t_like_x(t, x1)
|
| 129 |
+
_, d_alpha_t = self.compute_alpha_t(t)
|
| 130 |
+
_, d_sigma_t = self.compute_sigma_t(t)
|
| 131 |
+
return d_alpha_t * x1 + d_sigma_t * x0
|
| 132 |
+
|
| 133 |
+
def plan(self, t, x0, x1):
|
| 134 |
+
xt = self.compute_xt(t, x0, x1)
|
| 135 |
+
ut = self.compute_ut(t, x0, x1, xt)
|
| 136 |
+
return t, xt, ut
|
| 137 |
+
|
| 138 |
+
|
| 139 |
+
class VPCPlan(ICPlan):
|
| 140 |
+
"""class for VP path flow matching"""
|
| 141 |
+
|
| 142 |
+
def __init__(self, sigma_min=0.1, sigma_max=20.0):
|
| 143 |
+
self.sigma_min = sigma_min
|
| 144 |
+
self.sigma_max = sigma_max
|
| 145 |
+
self.log_mean_coeff = lambda t: -0.25 * ((1 - t) ** 2) * (self.sigma_max - self.sigma_min) - 0.5 * (1 - t) * self.sigma_min
|
| 146 |
+
self.d_log_mean_coeff = lambda t: 0.5 * (1 - t) * (self.sigma_max - self.sigma_min) + 0.5 * self.sigma_min
|
| 147 |
+
|
| 148 |
+
|
| 149 |
+
def compute_alpha_t(self, t):
|
| 150 |
+
"""Compute coefficient of x1"""
|
| 151 |
+
alpha_t = self.log_mean_coeff(t)
|
| 152 |
+
alpha_t = th.exp(alpha_t)
|
| 153 |
+
d_alpha_t = alpha_t * self.d_log_mean_coeff(t)
|
| 154 |
+
return alpha_t, d_alpha_t
|
| 155 |
+
|
| 156 |
+
def compute_sigma_t(self, t):
|
| 157 |
+
"""Compute coefficient of x0"""
|
| 158 |
+
p_sigma_t = 2 * self.log_mean_coeff(t)
|
| 159 |
+
sigma_t = th.sqrt(1 - th.exp(p_sigma_t))
|
| 160 |
+
d_sigma_t = th.exp(p_sigma_t) * (2 * self.d_log_mean_coeff(t)) / (-2 * sigma_t)
|
| 161 |
+
return sigma_t, d_sigma_t
|
| 162 |
+
|
| 163 |
+
def compute_d_alpha_alpha_ratio_t(self, t):
|
| 164 |
+
"""Special purposed function for computing numerical stabled d_alpha_t / alpha_t"""
|
| 165 |
+
return self.d_log_mean_coeff(t)
|
| 166 |
+
|
| 167 |
+
def compute_drift(self, x, t):
|
| 168 |
+
"""Compute the drift term of the SDE"""
|
| 169 |
+
t = expand_t_like_x(t, x)
|
| 170 |
+
beta_t = self.sigma_min + (1 - t) * (self.sigma_max - self.sigma_min)
|
| 171 |
+
return -0.5 * beta_t * x, beta_t / 2
|
| 172 |
+
|
| 173 |
+
|
| 174 |
+
class GVPCPlan(ICPlan):
|
| 175 |
+
def __init__(self, sigma=0.0):
|
| 176 |
+
super().__init__(sigma)
|
| 177 |
+
|
| 178 |
+
def compute_alpha_t(self, t):
|
| 179 |
+
"""Compute coefficient of x1"""
|
| 180 |
+
alpha_t = th.sin(t * np.pi / 2)
|
| 181 |
+
d_alpha_t = np.pi / 2 * th.cos(t * np.pi / 2)
|
| 182 |
+
return alpha_t, d_alpha_t
|
| 183 |
+
|
| 184 |
+
def compute_sigma_t(self, t):
|
| 185 |
+
"""Compute coefficient of x0"""
|
| 186 |
+
sigma_t = th.cos(t * np.pi / 2)
|
| 187 |
+
d_sigma_t = -np.pi / 2 * th.sin(t * np.pi / 2)
|
| 188 |
+
return sigma_t, d_sigma_t
|
| 189 |
+
|
| 190 |
+
def compute_d_alpha_alpha_ratio_t(self, t):
|
| 191 |
+
"""Special purposed function for computing numerical stabled d_alpha_t / alpha_t"""
|
| 192 |
+
return np.pi / (2 * th.tan(t * np.pi / 2))
|
cell_fm/models/cell_fm/modules/transport/transport.py
ADDED
|
@@ -0,0 +1,506 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch as th
|
| 2 |
+
import numpy as np
|
| 3 |
+
import logging
|
| 4 |
+
|
| 5 |
+
import enum
|
| 6 |
+
|
| 7 |
+
from . import path
|
| 8 |
+
from .utils import EasyDict, log_state, mean_flat
|
| 9 |
+
from .integrators import ode, sde
|
| 10 |
+
|
| 11 |
+
class ModelType(enum.Enum):
|
| 12 |
+
"""
|
| 13 |
+
Which type of output the model predicts.
|
| 14 |
+
"""
|
| 15 |
+
|
| 16 |
+
NOISE = enum.auto() # the model predicts epsilon
|
| 17 |
+
SCORE = enum.auto() # the model predicts \nabla \log p(x)
|
| 18 |
+
VELOCITY = enum.auto() # the model predicts v(x)
|
| 19 |
+
|
| 20 |
+
class PathType(enum.Enum):
|
| 21 |
+
"""
|
| 22 |
+
Which type of path to use.
|
| 23 |
+
"""
|
| 24 |
+
|
| 25 |
+
LINEAR = enum.auto()
|
| 26 |
+
GVP = enum.auto()
|
| 27 |
+
VP = enum.auto()
|
| 28 |
+
|
| 29 |
+
class WeightType(enum.Enum):
|
| 30 |
+
"""
|
| 31 |
+
Which type of weighting to use.
|
| 32 |
+
"""
|
| 33 |
+
|
| 34 |
+
NONE = enum.auto()
|
| 35 |
+
VELOCITY = enum.auto()
|
| 36 |
+
LIKELIHOOD = enum.auto()
|
| 37 |
+
|
| 38 |
+
class TimestepSampler(enum.Enum):
|
| 39 |
+
"""
|
| 40 |
+
How the training timestep t is drawn.
|
| 41 |
+
"""
|
| 42 |
+
|
| 43 |
+
UNIFORM = enum.auto() # t ~ U(t0, t1)
|
| 44 |
+
LOGIT_NORMAL = enum.auto() # SD3: t = sigmoid(-u), u ~ N(logit_mean, logit_std^2)
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
class Transport:
|
| 48 |
+
|
| 49 |
+
def __init__(
|
| 50 |
+
self,
|
| 51 |
+
*,
|
| 52 |
+
model_type,
|
| 53 |
+
path_type,
|
| 54 |
+
loss_type,
|
| 55 |
+
train_eps,
|
| 56 |
+
sample_eps,
|
| 57 |
+
timestep_sampler=TimestepSampler.UNIFORM,
|
| 58 |
+
logit_mean=0.0,
|
| 59 |
+
logit_std=1.0,
|
| 60 |
+
):
|
| 61 |
+
path_options = {
|
| 62 |
+
PathType.LINEAR: path.ICPlan,
|
| 63 |
+
PathType.GVP: path.GVPCPlan,
|
| 64 |
+
PathType.VP: path.VPCPlan,
|
| 65 |
+
}
|
| 66 |
+
|
| 67 |
+
self.loss_type = loss_type
|
| 68 |
+
self.model_type = model_type
|
| 69 |
+
self.path_sampler = path_options[path_type]()
|
| 70 |
+
self.train_eps = train_eps
|
| 71 |
+
self.sample_eps = sample_eps
|
| 72 |
+
self.timestep_sampler = timestep_sampler
|
| 73 |
+
self.logit_mean = logit_mean
|
| 74 |
+
self.logit_std = logit_std
|
| 75 |
+
|
| 76 |
+
def prior_logp(self, z):
|
| 77 |
+
'''
|
| 78 |
+
Standard multivariate normal prior
|
| 79 |
+
Assume z is batched
|
| 80 |
+
'''
|
| 81 |
+
shape = th.tensor(z.size())
|
| 82 |
+
N = th.prod(shape[1:])
|
| 83 |
+
_fn = lambda x: -N / 2. * np.log(2 * np.pi) - th.sum(x ** 2) / 2.
|
| 84 |
+
return th.vmap(_fn)(z)
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
def check_interval(
|
| 88 |
+
self,
|
| 89 |
+
train_eps,
|
| 90 |
+
sample_eps,
|
| 91 |
+
*,
|
| 92 |
+
diffusion_form="SBDM",
|
| 93 |
+
sde=False,
|
| 94 |
+
reverse=False,
|
| 95 |
+
eval=False,
|
| 96 |
+
last_step_size=0.0,
|
| 97 |
+
):
|
| 98 |
+
t0 = 0
|
| 99 |
+
t1 = 1
|
| 100 |
+
eps = train_eps if not eval else sample_eps
|
| 101 |
+
if (type(self.path_sampler) in [path.VPCPlan]):
|
| 102 |
+
|
| 103 |
+
t1 = 1 - eps if (not sde or last_step_size == 0) else 1 - last_step_size
|
| 104 |
+
|
| 105 |
+
elif (type(self.path_sampler) in [path.ICPlan, path.GVPCPlan]) \
|
| 106 |
+
and (self.model_type != ModelType.VELOCITY or sde): # avoid numerical issue by taking a first semi-implicit step
|
| 107 |
+
|
| 108 |
+
t0 = eps if (diffusion_form == "SBDM" and sde) or self.model_type != ModelType.VELOCITY else 0
|
| 109 |
+
t1 = 1 - eps if (not sde or last_step_size == 0) else 1 - last_step_size
|
| 110 |
+
|
| 111 |
+
if reverse:
|
| 112 |
+
t0, t1 = 1 - t0, 1 - t1
|
| 113 |
+
|
| 114 |
+
return t0, t1
|
| 115 |
+
|
| 116 |
+
|
| 117 |
+
def sample(self, x1):
|
| 118 |
+
"""Sampling x0 & t based on shape of x1 (if needed)
|
| 119 |
+
Args:
|
| 120 |
+
x1 - data point; [batch, *dim]
|
| 121 |
+
"""
|
| 122 |
+
|
| 123 |
+
x0 = th.randn_like(x1)
|
| 124 |
+
t0, t1 = self.check_interval(self.train_eps, self.sample_eps)
|
| 125 |
+
|
| 126 |
+
if self.timestep_sampler == TimestepSampler.LOGIT_NORMAL:
|
| 127 |
+
# SD3 draws t_sd3 = sigmoid(u) with t_sd3=1 at the noise end. This codebase
|
| 128 |
+
# runs the other way (t=1 is data, see ICPlan.compute_alpha_t), so the
|
| 129 |
+
# equivalent draw is t = 1 - sigmoid(u) = sigmoid(-u). This keeps logit_mean
|
| 130 |
+
# meaning the same thing here as in diffusers.
|
| 131 |
+
u = th.randn((x1.shape[0],)) * self.logit_std + self.logit_mean
|
| 132 |
+
t = th.sigmoid(-u)
|
| 133 |
+
t = t * (t1 - t0) + t0
|
| 134 |
+
else:
|
| 135 |
+
t = th.rand((x1.shape[0],)) * (t1 - t0) + t0
|
| 136 |
+
|
| 137 |
+
t = t.to(x1)
|
| 138 |
+
return t, x0, x1
|
| 139 |
+
|
| 140 |
+
|
| 141 |
+
def training_losses(
|
| 142 |
+
self,
|
| 143 |
+
model_output,
|
| 144 |
+
x0,
|
| 145 |
+
xt,
|
| 146 |
+
ut,
|
| 147 |
+
t,
|
| 148 |
+
):
|
| 149 |
+
"""Loss for training the score model
|
| 150 |
+
Args:
|
| 151 |
+
- model: backbone model; could be score, noise, or velocity
|
| 152 |
+
- x1: datapoint
|
| 153 |
+
- model_kwargs: additional arguments for the model
|
| 154 |
+
"""
|
| 155 |
+
|
| 156 |
+
terms = {}
|
| 157 |
+
terms['pred'] = model_output
|
| 158 |
+
if self.model_type == ModelType.VELOCITY:
|
| 159 |
+
terms['loss'] = mean_flat(((model_output - ut) ** 2))
|
| 160 |
+
else:
|
| 161 |
+
_, drift_var = self.path_sampler.compute_drift(xt, t)
|
| 162 |
+
sigma_t, _ = self.path_sampler.compute_sigma_t(path.expand_t_like_x(t, xt))
|
| 163 |
+
if self.loss_type in [WeightType.VELOCITY]:
|
| 164 |
+
weight = (drift_var / sigma_t) ** 2
|
| 165 |
+
elif self.loss_type in [WeightType.LIKELIHOOD]:
|
| 166 |
+
weight = drift_var / (sigma_t ** 2)
|
| 167 |
+
elif self.loss_type in [WeightType.NONE]:
|
| 168 |
+
weight = 1
|
| 169 |
+
else:
|
| 170 |
+
raise NotImplementedError()
|
| 171 |
+
|
| 172 |
+
if self.model_type == ModelType.NOISE:
|
| 173 |
+
terms['loss'] = mean_flat(weight * ((model_output - x0) ** 2))
|
| 174 |
+
else:
|
| 175 |
+
terms['loss'] = mean_flat(weight * ((model_output * sigma_t + x0) ** 2))
|
| 176 |
+
|
| 177 |
+
return terms
|
| 178 |
+
|
| 179 |
+
# def training_losses(
|
| 180 |
+
# self,
|
| 181 |
+
# model,
|
| 182 |
+
# x1,
|
| 183 |
+
# model_kwargs=None
|
| 184 |
+
# ):
|
| 185 |
+
# """Loss for training the score model
|
| 186 |
+
# Args:
|
| 187 |
+
# - model: backbone model; could be score, noise, or velocity
|
| 188 |
+
# - x1: datapoint
|
| 189 |
+
# - model_kwargs: additional arguments for the model
|
| 190 |
+
# """
|
| 191 |
+
# if model_kwargs == None:
|
| 192 |
+
# model_kwargs = {}
|
| 193 |
+
|
| 194 |
+
# t, x0, x1 = self.sample(x1)
|
| 195 |
+
# t, xt, ut = self.path_sampler.plan(t, x0, x1)
|
| 196 |
+
# model_output = model(xt, t, **model_kwargs)
|
| 197 |
+
# B, *_, C = xt.shape
|
| 198 |
+
# assert model_output.size() == (B, *xt.size()[1:-1], C)
|
| 199 |
+
|
| 200 |
+
# terms = {}
|
| 201 |
+
# terms['pred'] = model_output
|
| 202 |
+
# if self.model_type == ModelType.VELOCITY:
|
| 203 |
+
# terms['loss'] = mean_flat(((model_output - ut) ** 2))
|
| 204 |
+
# else:
|
| 205 |
+
# _, drift_var = self.path_sampler.compute_drift(xt, t)
|
| 206 |
+
# sigma_t, _ = self.path_sampler.compute_sigma_t(path.expand_t_like_x(t, xt))
|
| 207 |
+
# if self.loss_type in [WeightType.VELOCITY]:
|
| 208 |
+
# weight = (drift_var / sigma_t) ** 2
|
| 209 |
+
# elif self.loss_type in [WeightType.LIKELIHOOD]:
|
| 210 |
+
# weight = drift_var / (sigma_t ** 2)
|
| 211 |
+
# elif self.loss_type in [WeightType.NONE]:
|
| 212 |
+
# weight = 1
|
| 213 |
+
# else:
|
| 214 |
+
# raise NotImplementedError()
|
| 215 |
+
|
| 216 |
+
# if self.model_type == ModelType.NOISE:
|
| 217 |
+
# terms['loss'] = mean_flat(weight * ((model_output - x0) ** 2))
|
| 218 |
+
# else:
|
| 219 |
+
# terms['loss'] = mean_flat(weight * ((model_output * sigma_t + x0) ** 2))
|
| 220 |
+
|
| 221 |
+
# return terms
|
| 222 |
+
|
| 223 |
+
|
| 224 |
+
def get_drift(
|
| 225 |
+
self
|
| 226 |
+
):
|
| 227 |
+
"""member function for obtaining the drift of the probability flow ODE"""
|
| 228 |
+
def score_ode(x, t, model, **model_kwargs):
|
| 229 |
+
drift_mean, drift_var = self.path_sampler.compute_drift(x, t)
|
| 230 |
+
model_output = model(x, t, **model_kwargs)
|
| 231 |
+
return (-drift_mean + drift_var * model_output) # by change of variable
|
| 232 |
+
|
| 233 |
+
def noise_ode(x, t, model, **model_kwargs):
|
| 234 |
+
drift_mean, drift_var = self.path_sampler.compute_drift(x, t)
|
| 235 |
+
sigma_t, _ = self.path_sampler.compute_sigma_t(path.expand_t_like_x(t, x))
|
| 236 |
+
model_output = model(x, t, **model_kwargs)
|
| 237 |
+
score = model_output / -sigma_t
|
| 238 |
+
return (-drift_mean + drift_var * score)
|
| 239 |
+
|
| 240 |
+
def velocity_ode(x, t, model, **model_kwargs):
|
| 241 |
+
model_output = model(x, t, **model_kwargs)
|
| 242 |
+
return model_output
|
| 243 |
+
|
| 244 |
+
if self.model_type == ModelType.NOISE:
|
| 245 |
+
drift_fn = noise_ode
|
| 246 |
+
elif self.model_type == ModelType.SCORE:
|
| 247 |
+
drift_fn = score_ode
|
| 248 |
+
else:
|
| 249 |
+
drift_fn = velocity_ode
|
| 250 |
+
|
| 251 |
+
def body_fn(x, t, model, **model_kwargs):
|
| 252 |
+
model_output = drift_fn(x, t, model, **model_kwargs)
|
| 253 |
+
assert model_output.shape == x.shape, "Output shape from ODE solver must match input shape"
|
| 254 |
+
return model_output
|
| 255 |
+
|
| 256 |
+
return body_fn
|
| 257 |
+
|
| 258 |
+
|
| 259 |
+
def get_score(
|
| 260 |
+
self,
|
| 261 |
+
):
|
| 262 |
+
"""member function for obtaining score of
|
| 263 |
+
x_t = alpha_t * x + sigma_t * eps"""
|
| 264 |
+
if self.model_type == ModelType.NOISE:
|
| 265 |
+
score_fn = lambda x, t, model, **kwargs: model(x, t, **kwargs) / -self.path_sampler.compute_sigma_t(path.expand_t_like_x(t, x))[0]
|
| 266 |
+
elif self.model_type == ModelType.SCORE:
|
| 267 |
+
score_fn = lambda x, t, model, **kwagrs: model(x, t, **kwagrs)
|
| 268 |
+
elif self.model_type == ModelType.VELOCITY:
|
| 269 |
+
score_fn = lambda x, t, model, **kwargs: self.path_sampler.get_score_from_velocity(model(x, t, **kwargs), x, t)
|
| 270 |
+
else:
|
| 271 |
+
raise NotImplementedError()
|
| 272 |
+
|
| 273 |
+
return score_fn
|
| 274 |
+
|
| 275 |
+
|
| 276 |
+
class Sampler:
|
| 277 |
+
"""Sampler class for the transport model"""
|
| 278 |
+
def __init__(
|
| 279 |
+
self,
|
| 280 |
+
transport,
|
| 281 |
+
):
|
| 282 |
+
"""Constructor for a general sampler; supporting different sampling methods
|
| 283 |
+
Args:
|
| 284 |
+
- transport: an tranport object specify model prediction & interpolant type
|
| 285 |
+
"""
|
| 286 |
+
|
| 287 |
+
self.transport = transport
|
| 288 |
+
self.drift = self.transport.get_drift()
|
| 289 |
+
self.score = self.transport.get_score()
|
| 290 |
+
|
| 291 |
+
def __get_sde_diffusion_and_drift(
|
| 292 |
+
self,
|
| 293 |
+
*,
|
| 294 |
+
diffusion_form="SBDM",
|
| 295 |
+
diffusion_norm=1.0,
|
| 296 |
+
):
|
| 297 |
+
|
| 298 |
+
def diffusion_fn(x, t):
|
| 299 |
+
diffusion = self.transport.path_sampler.compute_diffusion(x, t, form=diffusion_form, norm=diffusion_norm)
|
| 300 |
+
return diffusion
|
| 301 |
+
|
| 302 |
+
sde_drift = \
|
| 303 |
+
lambda x, t, model, **kwargs: \
|
| 304 |
+
self.drift(x, t, model, **kwargs) + diffusion_fn(x, t) * self.score(x, t, model, **kwargs)
|
| 305 |
+
|
| 306 |
+
sde_diffusion = diffusion_fn
|
| 307 |
+
|
| 308 |
+
return sde_drift, sde_diffusion
|
| 309 |
+
|
| 310 |
+
def __get_last_step(
|
| 311 |
+
self,
|
| 312 |
+
sde_drift,
|
| 313 |
+
*,
|
| 314 |
+
last_step,
|
| 315 |
+
last_step_size,
|
| 316 |
+
):
|
| 317 |
+
"""Get the last step function of the SDE solver"""
|
| 318 |
+
|
| 319 |
+
if last_step is None:
|
| 320 |
+
last_step_fn = \
|
| 321 |
+
lambda x, t, model, **model_kwargs: \
|
| 322 |
+
x
|
| 323 |
+
elif last_step == "Mean":
|
| 324 |
+
last_step_fn = \
|
| 325 |
+
lambda x, t, model, **model_kwargs: \
|
| 326 |
+
x + sde_drift(x, t, model, **model_kwargs) * last_step_size
|
| 327 |
+
elif last_step == "Tweedie":
|
| 328 |
+
alpha = self.transport.path_sampler.compute_alpha_t # simple aliasing; the original name was too long
|
| 329 |
+
sigma = self.transport.path_sampler.compute_sigma_t
|
| 330 |
+
last_step_fn = \
|
| 331 |
+
lambda x, t, model, **model_kwargs: \
|
| 332 |
+
x / alpha(t)[0][0] + (sigma(t)[0][0] ** 2) / alpha(t)[0][0] * self.score(x, t, model, **model_kwargs)
|
| 333 |
+
elif last_step == "Euler":
|
| 334 |
+
last_step_fn = \
|
| 335 |
+
lambda x, t, model, **model_kwargs: \
|
| 336 |
+
x + self.drift(x, t, model, **model_kwargs) * last_step_size
|
| 337 |
+
else:
|
| 338 |
+
raise NotImplementedError()
|
| 339 |
+
|
| 340 |
+
return last_step_fn
|
| 341 |
+
|
| 342 |
+
def sample_sde(
|
| 343 |
+
self,
|
| 344 |
+
*,
|
| 345 |
+
sampling_method="Euler",
|
| 346 |
+
diffusion_form="SBDM",
|
| 347 |
+
diffusion_norm=1.0,
|
| 348 |
+
last_step="Mean",
|
| 349 |
+
last_step_size=0.04,
|
| 350 |
+
num_steps=250,
|
| 351 |
+
):
|
| 352 |
+
"""returns a sampling function with given SDE settings
|
| 353 |
+
Args:
|
| 354 |
+
- sampling_method: type of sampler used in solving the SDE; default to be Euler-Maruyama
|
| 355 |
+
- diffusion_form: function form of diffusion coefficient; default to be matching SBDM
|
| 356 |
+
- diffusion_norm: function magnitude of diffusion coefficient; default to 1
|
| 357 |
+
- last_step: type of the last step; default to identity
|
| 358 |
+
- last_step_size: size of the last step; default to match the stride of 250 steps over [0,1]
|
| 359 |
+
- num_steps: total integration step of SDE
|
| 360 |
+
"""
|
| 361 |
+
|
| 362 |
+
if last_step is None:
|
| 363 |
+
last_step_size = 0.0
|
| 364 |
+
|
| 365 |
+
sde_drift, sde_diffusion = self.__get_sde_diffusion_and_drift(
|
| 366 |
+
diffusion_form=diffusion_form,
|
| 367 |
+
diffusion_norm=diffusion_norm,
|
| 368 |
+
)
|
| 369 |
+
|
| 370 |
+
t0, t1 = self.transport.check_interval(
|
| 371 |
+
self.transport.train_eps,
|
| 372 |
+
self.transport.sample_eps,
|
| 373 |
+
diffusion_form=diffusion_form,
|
| 374 |
+
sde=True,
|
| 375 |
+
eval=True,
|
| 376 |
+
reverse=False,
|
| 377 |
+
last_step_size=last_step_size,
|
| 378 |
+
)
|
| 379 |
+
|
| 380 |
+
_sde = sde(
|
| 381 |
+
sde_drift,
|
| 382 |
+
sde_diffusion,
|
| 383 |
+
t0=t0,
|
| 384 |
+
t1=t1,
|
| 385 |
+
num_steps=num_steps,
|
| 386 |
+
sampler_type=sampling_method
|
| 387 |
+
)
|
| 388 |
+
|
| 389 |
+
last_step_fn = self.__get_last_step(sde_drift, last_step=last_step, last_step_size=last_step_size)
|
| 390 |
+
|
| 391 |
+
|
| 392 |
+
def _sample(init, model, **model_kwargs):
|
| 393 |
+
xs = _sde.sample(init, model, **model_kwargs)
|
| 394 |
+
ts = th.ones(init.size(0), device=init.device) * t1
|
| 395 |
+
x = last_step_fn(xs[-1], ts, model, **model_kwargs)
|
| 396 |
+
xs.append(x)
|
| 397 |
+
|
| 398 |
+
assert len(xs) == num_steps, "Samples does not match the number of steps"
|
| 399 |
+
|
| 400 |
+
return xs
|
| 401 |
+
|
| 402 |
+
return _sample
|
| 403 |
+
|
| 404 |
+
def sample_ode(
|
| 405 |
+
self,
|
| 406 |
+
*,
|
| 407 |
+
sampling_method="dopri5",
|
| 408 |
+
num_steps=50,
|
| 409 |
+
atol=1e-6,
|
| 410 |
+
rtol=1e-3,
|
| 411 |
+
reverse=False,
|
| 412 |
+
):
|
| 413 |
+
"""returns a sampling function with given ODE settings
|
| 414 |
+
Args:
|
| 415 |
+
- sampling_method: type of sampler used in solving the ODE; default to be Dopri5
|
| 416 |
+
- num_steps:
|
| 417 |
+
- fixed solver (Euler, Heun): the actual number of integration steps performed
|
| 418 |
+
- adaptive solver (Dopri5): the number of datapoints saved during integration; produced by interpolation
|
| 419 |
+
- atol: absolute error tolerance for the solver
|
| 420 |
+
- rtol: relative error tolerance for the solver
|
| 421 |
+
- reverse: whether solving the ODE in reverse (data to noise); default to False
|
| 422 |
+
"""
|
| 423 |
+
if reverse:
|
| 424 |
+
drift = lambda x, t, model, **kwargs: self.drift(x, th.ones_like(t) * (1 - t), model, **kwargs)
|
| 425 |
+
else:
|
| 426 |
+
drift = self.drift
|
| 427 |
+
|
| 428 |
+
t0, t1 = self.transport.check_interval(
|
| 429 |
+
self.transport.train_eps,
|
| 430 |
+
self.transport.sample_eps,
|
| 431 |
+
sde=False,
|
| 432 |
+
eval=True,
|
| 433 |
+
reverse=reverse,
|
| 434 |
+
last_step_size=0.0,
|
| 435 |
+
)
|
| 436 |
+
|
| 437 |
+
_ode = ode(
|
| 438 |
+
drift=drift,
|
| 439 |
+
t0=t0,
|
| 440 |
+
t1=t1,
|
| 441 |
+
sampler_type=sampling_method,
|
| 442 |
+
num_steps=num_steps,
|
| 443 |
+
atol=atol,
|
| 444 |
+
rtol=rtol,
|
| 445 |
+
)
|
| 446 |
+
|
| 447 |
+
return _ode.sample
|
| 448 |
+
|
| 449 |
+
def sample_ode_likelihood(
|
| 450 |
+
self,
|
| 451 |
+
*,
|
| 452 |
+
sampling_method="dopri5",
|
| 453 |
+
num_steps=50,
|
| 454 |
+
atol=1e-6,
|
| 455 |
+
rtol=1e-3,
|
| 456 |
+
):
|
| 457 |
+
|
| 458 |
+
"""returns a sampling function for calculating likelihood with given ODE settings
|
| 459 |
+
Args:
|
| 460 |
+
- sampling_method: type of sampler used in solving the ODE; default to be Dopri5
|
| 461 |
+
- num_steps:
|
| 462 |
+
- fixed solver (Euler, Heun): the actual number of integration steps performed
|
| 463 |
+
- adaptive solver (Dopri5): the number of datapoints saved during integration; produced by interpolation
|
| 464 |
+
- atol: absolute error tolerance for the solver
|
| 465 |
+
- rtol: relative error tolerance for the solver
|
| 466 |
+
"""
|
| 467 |
+
def _likelihood_drift(x, t, model, **model_kwargs):
|
| 468 |
+
x, _ = x
|
| 469 |
+
eps = th.randint(2, x.size(), dtype=th.float, device=x.device) * 2 - 1
|
| 470 |
+
t = th.ones_like(t) * (1 - t)
|
| 471 |
+
with th.enable_grad():
|
| 472 |
+
x.requires_grad = True
|
| 473 |
+
grad = th.autograd.grad(th.sum(self.drift(x, t, model, **model_kwargs) * eps), x)[0]
|
| 474 |
+
logp_grad = th.sum(grad * eps, dim=tuple(range(1, len(x.size()))))
|
| 475 |
+
drift = self.drift(x, t, model, **model_kwargs)
|
| 476 |
+
return (-drift, logp_grad)
|
| 477 |
+
|
| 478 |
+
t0, t1 = self.transport.check_interval(
|
| 479 |
+
self.transport.train_eps,
|
| 480 |
+
self.transport.sample_eps,
|
| 481 |
+
sde=False,
|
| 482 |
+
eval=True,
|
| 483 |
+
reverse=False,
|
| 484 |
+
last_step_size=0.0,
|
| 485 |
+
)
|
| 486 |
+
|
| 487 |
+
_ode = ode(
|
| 488 |
+
drift=_likelihood_drift,
|
| 489 |
+
t0=t0,
|
| 490 |
+
t1=t1,
|
| 491 |
+
sampler_type=sampling_method,
|
| 492 |
+
num_steps=num_steps,
|
| 493 |
+
atol=atol,
|
| 494 |
+
rtol=rtol,
|
| 495 |
+
)
|
| 496 |
+
|
| 497 |
+
def _sample_fn(x, model, **model_kwargs):
|
| 498 |
+
init_logp = th.zeros(x.size(0)).to(x)
|
| 499 |
+
input = (x, init_logp)
|
| 500 |
+
drift, delta_logp = _ode.sample(input, model, **model_kwargs)
|
| 501 |
+
drift, delta_logp = drift[-1], delta_logp[-1]
|
| 502 |
+
prior_logp = self.transport.prior_logp(drift)
|
| 503 |
+
logp = prior_logp - delta_logp
|
| 504 |
+
return logp, drift
|
| 505 |
+
|
| 506 |
+
return _sample_fn
|
cell_fm/models/cell_fm/modules/transport/utils.py
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch as th
|
| 2 |
+
|
| 3 |
+
class EasyDict:
|
| 4 |
+
|
| 5 |
+
def __init__(self, sub_dict):
|
| 6 |
+
for k, v in sub_dict.items():
|
| 7 |
+
setattr(self, k, v)
|
| 8 |
+
|
| 9 |
+
def __getitem__(self, key):
|
| 10 |
+
return getattr(self, key)
|
| 11 |
+
|
| 12 |
+
def mean_flat(x):
|
| 13 |
+
"""
|
| 14 |
+
Take the mean over all non-batch dimensions.
|
| 15 |
+
"""
|
| 16 |
+
return th.mean(x, dim=list(range(1, len(x.size()))))
|
| 17 |
+
|
| 18 |
+
def log_state(state):
|
| 19 |
+
result = []
|
| 20 |
+
|
| 21 |
+
sorted_state = dict(sorted(state.items()))
|
| 22 |
+
for key, value in sorted_state.items():
|
| 23 |
+
# Check if the value is an instance of a class
|
| 24 |
+
if "<object" in str(value) or "object at" in str(value):
|
| 25 |
+
result.append(f"{key}: [{value.__class__.__name__}]")
|
| 26 |
+
else:
|
| 27 |
+
result.append(f"{key}: {value}")
|
| 28 |
+
|
| 29 |
+
return '\n'.join(result)
|
cell_fm/models/vae/__init__.py
ADDED
|
File without changes
|
cell_fm/models/vae/vae_config.py
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# -*- coding: utf-8 -*-
|
| 2 |
+
from dataclasses import dataclass
|
| 3 |
+
from transformers import PretrainedConfig
|
| 4 |
+
|
| 5 |
+
@dataclass
|
| 6 |
+
class VAEConfig(PretrainedConfig):
|
| 7 |
+
model_type: str = 'vae'
|
| 8 |
+
|
| 9 |
+
# Dataset parameters
|
| 10 |
+
data_path: str = ""
|
| 11 |
+
split_key: str = 'train'
|
| 12 |
+
|
| 13 |
+
data_aug: bool = False
|
| 14 |
+
img_resize: int = 1024
|
| 15 |
+
img_crop_size: int = 256
|
| 16 |
+
|
| 17 |
+
# Loss parameters
|
| 18 |
+
recon_loss_coeff: float = 1.0
|
| 19 |
+
kl_loss_coeff: float = 1.0
|
| 20 |
+
|
| 21 |
+
# Model parameters
|
| 22 |
+
in_channels: int = 1
|
| 23 |
+
out_channels: int = 1
|
| 24 |
+
num_down_blocks: int = 3
|
| 25 |
+
latent_channels: int = 4
|
| 26 |
+
vae_block_out_channels: str = '128,256,512'
|
| 27 |
+
|
| 28 |
+
# Training parameters
|
| 29 |
+
vae_loadcheck_path: str = ""
|
| 30 |
+
ft: bool = False
|
| 31 |
+
infer: bool = False
|
| 32 |
+
ifresume: bool = False
|
| 33 |
+
|
| 34 |
+
output_dir: str = ""
|
| 35 |
+
learning_rate: float = 1e-4
|
| 36 |
+
weight_decay: float = 0.0
|
| 37 |
+
gradient_accumulation_steps: int = 1
|
| 38 |
+
per_device_train_batch_size: int = 2
|
| 39 |
+
per_device_eval_batch_size: int = 2
|
| 40 |
+
|
| 41 |
+
num_train_epochs: int = 10
|
| 42 |
+
fp16: bool = False
|
| 43 |
+
bf16: bool = False
|
| 44 |
+
logging_dir: str = ""
|
| 45 |
+
logging_steps: int = 10
|
| 46 |
+
max_steps: int = -1
|
| 47 |
+
warmup_steps: int = 1000
|
| 48 |
+
save_steps: int = 1000
|
| 49 |
+
|
| 50 |
+
dataloader_num_workers: int = 8
|
| 51 |
+
seed: int = 6
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def __init__(self, **kwargs):
|
| 55 |
+
# Use `super().__init__` to handle arguments from PretrainedConfig
|
| 56 |
+
super().__init__(**kwargs)
|
| 57 |
+
|
| 58 |
+
self.vae_block_out_channels = kwargs.get("vae_block_out_channels", self.vae_block_out_channels)
|
| 59 |
+
if not isinstance(self.vae_block_out_channels, list):
|
| 60 |
+
self.vae_block_out_channels = [int(a) for a in self.vae_block_out_channels.split(',')]
|
cell_fm/models/vae/vae_model.py
ADDED
|
@@ -0,0 +1,152 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import torch
|
| 3 |
+
import torch.nn as nn
|
| 4 |
+
from diffusers.models import AutoencoderKL
|
| 5 |
+
from .vae_config import VAEConfig
|
| 6 |
+
from transformers import PreTrainedModel
|
| 7 |
+
from cell_fm.pipeline.utils import VAEOutput
|
| 8 |
+
from cell_fm.logging import logger
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
class VAEModel(PreTrainedModel):
|
| 12 |
+
config_class = VAEConfig
|
| 13 |
+
|
| 14 |
+
def __init__(self, config):
|
| 15 |
+
super().__init__(config)
|
| 16 |
+
self.config = config
|
| 17 |
+
|
| 18 |
+
# Calculate number of downsampling and upsampling layers required
|
| 19 |
+
self.num_down_blocks = config.num_down_blocks # Input size / latent space size
|
| 20 |
+
self.num_up_blocks = self.num_down_blocks
|
| 21 |
+
|
| 22 |
+
# Generate down_block_types and up_block_types based on the required layers
|
| 23 |
+
down_block_types = tuple(["DownEncoderBlock2D"] * self.num_down_blocks)
|
| 24 |
+
up_block_types = tuple(["UpDecoderBlock2D"] * self.num_up_blocks)
|
| 25 |
+
|
| 26 |
+
# Initialize AutoencoderKL with custom parameters from config
|
| 27 |
+
self.vae = AutoencoderKL(
|
| 28 |
+
in_channels=config.in_channels,
|
| 29 |
+
out_channels=config.out_channels,
|
| 30 |
+
down_block_types=down_block_types,
|
| 31 |
+
up_block_types=up_block_types,
|
| 32 |
+
block_out_channels=config.vae_block_out_channels,
|
| 33 |
+
latent_channels=config.latent_channels, # Latent space dimensions
|
| 34 |
+
)
|
| 35 |
+
|
| 36 |
+
self.load_pretrained_weights(config, checkpoint_path=config.vae_loadcheck_path)
|
| 37 |
+
|
| 38 |
+
def load_pretrained_weights(self, config, checkpoint_path):
|
| 39 |
+
"""
|
| 40 |
+
Load pretrained weights from a given state_dict.
|
| 41 |
+
"""
|
| 42 |
+
if config.ft or config.infer:
|
| 43 |
+
if config.ft:
|
| 44 |
+
logger.info(f"Finetune from checkpoint: {checkpoint_path}")
|
| 45 |
+
else:
|
| 46 |
+
logger.info(f"Infer from checkpoint: {checkpoint_path}")
|
| 47 |
+
|
| 48 |
+
if os.path.splitext(checkpoint_path)[1] == '.safetensors':
|
| 49 |
+
from safetensors.torch import load_file
|
| 50 |
+
checkpoints_state = load_file(checkpoint_path)
|
| 51 |
+
else:
|
| 52 |
+
checkpoints_state = torch.load(checkpoint_path, map_location="cpu")
|
| 53 |
+
|
| 54 |
+
if "model" in checkpoints_state:
|
| 55 |
+
checkpoints_state = checkpoints_state["model"]
|
| 56 |
+
elif "module" in checkpoints_state:
|
| 57 |
+
checkpoints_state = checkpoints_state["module"]
|
| 58 |
+
|
| 59 |
+
model_state_dict = self.state_dict()
|
| 60 |
+
filtered_state_dict = {k: v for k, v in checkpoints_state.items() if k in model_state_dict and v.size() == model_state_dict[k].size()}
|
| 61 |
+
|
| 62 |
+
IncompatibleKeys = self.load_state_dict(filtered_state_dict, strict=False)
|
| 63 |
+
# IncompatibleKeys = self.load_state_dict(checkpoints_state, strict=False)
|
| 64 |
+
IncompatibleKeys = IncompatibleKeys._asdict()
|
| 65 |
+
|
| 66 |
+
missing_keys = []
|
| 67 |
+
for keys in IncompatibleKeys["missing_keys"]:
|
| 68 |
+
if keys.find("dummy") == -1:
|
| 69 |
+
missing_keys.append(keys)
|
| 70 |
+
|
| 71 |
+
unexpected_keys = []
|
| 72 |
+
for keys in IncompatibleKeys["unexpected_keys"]:
|
| 73 |
+
if keys.find("dummy") == -1:
|
| 74 |
+
unexpected_keys.append(keys)
|
| 75 |
+
|
| 76 |
+
if len(missing_keys) > 0:
|
| 77 |
+
logger.info(
|
| 78 |
+
"Missing keys in {}: {}".format(
|
| 79 |
+
checkpoint_path,
|
| 80 |
+
missing_keys,
|
| 81 |
+
)
|
| 82 |
+
)
|
| 83 |
+
|
| 84 |
+
if len(unexpected_keys) > 0:
|
| 85 |
+
logger.info(
|
| 86 |
+
"Unexpected keys {}: {}".format(
|
| 87 |
+
checkpoint_path,
|
| 88 |
+
unexpected_keys,
|
| 89 |
+
)
|
| 90 |
+
)
|
| 91 |
+
|
| 92 |
+
def encode(self, x):
|
| 93 |
+
"""Encodes input into latent space."""
|
| 94 |
+
return self.vae.encode(x).latent_dist
|
| 95 |
+
|
| 96 |
+
def decode(self, latents):
|
| 97 |
+
"""Decodes latent space into reconstructed input."""
|
| 98 |
+
return self.vae.decode(latents)
|
| 99 |
+
|
| 100 |
+
def forward(self, batched_data):
|
| 101 |
+
x = batched_data['protein_img']
|
| 102 |
+
|
| 103 |
+
"""Forward pass through the VAE."""
|
| 104 |
+
|
| 105 |
+
latent_dist = self.encode(x)
|
| 106 |
+
latents = latent_dist.sample() # Reparameterization trick
|
| 107 |
+
recon_x = self.decode(latents).sample
|
| 108 |
+
total_loss, recon_loss, kl_loss = self.compute_loss(x, recon_x, latent_dist)
|
| 109 |
+
|
| 110 |
+
# Log the losses
|
| 111 |
+
log_loss = {
|
| 112 |
+
"total_loss": total_loss.item(),
|
| 113 |
+
"recon_loss": recon_loss.item(),
|
| 114 |
+
"kl_loss": kl_loss.item(),
|
| 115 |
+
}
|
| 116 |
+
|
| 117 |
+
return VAEOutput(total_loss, log_loss)
|
| 118 |
+
|
| 119 |
+
def compute_loss(self, x, recon_x, latent_dist):
|
| 120 |
+
"""Compute reconstruction and KL divergence loss."""
|
| 121 |
+
recon_loss = nn.MSELoss()(recon_x, x)
|
| 122 |
+
kl_loss = -0.5 * torch.mean(1 + latent_dist.logvar - latent_dist.mean.pow(2) - latent_dist.logvar.exp())
|
| 123 |
+
total_loss = self.config.recon_loss_coeff * recon_loss + self.config.kl_loss_coeff * kl_loss
|
| 124 |
+
return total_loss, recon_loss, kl_loss
|
| 125 |
+
|
| 126 |
+
def sample(self, num_samples=1, latent_size=64, device="cpu"):
|
| 127 |
+
"""
|
| 128 |
+
Generate samples from the latent space.
|
| 129 |
+
|
| 130 |
+
Args:
|
| 131 |
+
num_samples (int): Number of samples to generate.
|
| 132 |
+
latent_size (int): Size of the latent space.
|
| 133 |
+
device (str): Device to perform sampling on.
|
| 134 |
+
|
| 135 |
+
Returns:
|
| 136 |
+
torch.Tensor: Generated images.
|
| 137 |
+
"""
|
| 138 |
+
# Sample from a standard normal distribution in latent space
|
| 139 |
+
latents = torch.randn((num_samples, self.config.latent_channels, latent_size, latent_size), device=device) # Shape matches latent dimensions
|
| 140 |
+
|
| 141 |
+
# Decode latents to generate images
|
| 142 |
+
with torch.no_grad():
|
| 143 |
+
generated_images = self.decode(latents).sample
|
| 144 |
+
|
| 145 |
+
return generated_images
|
| 146 |
+
|
| 147 |
+
def reconstruct(self, x):
|
| 148 |
+
latent_dist = self.encode(x)
|
| 149 |
+
latents = latent_dist.sample() # Reparameterization trick
|
| 150 |
+
recon_x = self.decode(latents).sample
|
| 151 |
+
|
| 152 |
+
return recon_x
|
cell_fm/models/vit_cls_condenseq_img/__init__.py
ADDED
|
File without changes
|
cell_fm/models/vit_cls_condenseq_img/config.py
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# -*- coding: utf-8 -*-
|
| 2 |
+
from dataclasses import dataclass
|
| 3 |
+
from transformers import PretrainedConfig
|
| 4 |
+
|
| 5 |
+
@dataclass
|
| 6 |
+
class ViTConfig(PretrainedConfig):
|
| 7 |
+
model_type: str = 'vit'
|
| 8 |
+
|
| 9 |
+
# Model parameters
|
| 10 |
+
image_size: int = 160
|
| 11 |
+
in_channels: int = 2
|
| 12 |
+
patch_size: int = 4
|
| 13 |
+
num_layers: int = 8
|
| 14 |
+
num_heads: int = 8
|
| 15 |
+
hidden_dim: int = 768
|
| 16 |
+
mlp_dim: int = 3072
|
| 17 |
+
num_classes: int = 2
|
| 18 |
+
|
| 19 |
+
# Training parameters
|
| 20 |
+
loadcheck_path: str = '.'
|
| 21 |
+
ft: bool = False
|
| 22 |
+
infer: bool = False
|
| 23 |
+
ifresume: bool = False
|
| 24 |
+
|
| 25 |
+
output_dir: str = ""
|
| 26 |
+
learning_rate: float = 1e-4
|
| 27 |
+
weight_decay: float = 0.0
|
| 28 |
+
gradient_accumulation_steps: int = 1
|
| 29 |
+
per_device_train_batch_size: int = 2
|
| 30 |
+
per_device_eval_batch_size: int = 2
|
| 31 |
+
|
| 32 |
+
num_train_epochs: int = 10
|
| 33 |
+
fp16: bool = False
|
| 34 |
+
bf16: bool = False
|
| 35 |
+
logging_dir: str = ""
|
| 36 |
+
logging_steps: int = 10
|
| 37 |
+
max_steps: int = -1
|
| 38 |
+
warmup_steps: int = 1000
|
| 39 |
+
save_steps: int = 1000
|
| 40 |
+
|
| 41 |
+
dataloader_num_workers: int = 8
|
| 42 |
+
seed: int = 6
|
| 43 |
+
|
| 44 |
+
def __init__(self, **kwargs):
|
| 45 |
+
# Use `super().__init__` to handle arguments from PretrainedConfig
|
| 46 |
+
super().__init__(**kwargs)
|
cell_fm/models/vit_cls_condenseq_img/model.py
ADDED
|
@@ -0,0 +1,171 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import torch
|
| 3 |
+
import torch.nn as nn
|
| 4 |
+
from cell_fm.logging import logger
|
| 5 |
+
from transformers import PreTrainedModel
|
| 6 |
+
from .config import ViTConfig
|
| 7 |
+
|
| 8 |
+
from torchvision.models.vision_transformer import VisionTransformer
|
| 9 |
+
from functools import partial
|
| 10 |
+
|
| 11 |
+
from cell_fm.pipeline.utils import VanillaOutput
|
| 12 |
+
import json
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
class ViTModel(PreTrainedModel):
|
| 16 |
+
config_class = ViTConfig
|
| 17 |
+
|
| 18 |
+
def __init__(self, config: ViTConfig):
|
| 19 |
+
super().__init__(config)
|
| 20 |
+
self.config = config
|
| 21 |
+
|
| 22 |
+
self.loss = nn.CrossEntropyLoss()
|
| 23 |
+
self.net = ViT(config)
|
| 24 |
+
|
| 25 |
+
self.load_pretrained_weights(config, checkpoint_path=config.loadcheck_path)
|
| 26 |
+
|
| 27 |
+
def load_pretrained_weights(self, config, checkpoint_path):
|
| 28 |
+
"""
|
| 29 |
+
Load pretrained weights from a given state_dict.
|
| 30 |
+
"""
|
| 31 |
+
if config.ft or config.infer:
|
| 32 |
+
if config.ft:
|
| 33 |
+
logger.info(f"Finetune from checkpoint: {checkpoint_path}")
|
| 34 |
+
else:
|
| 35 |
+
logger.info(f"Infer from checkpoint: {checkpoint_path}")
|
| 36 |
+
|
| 37 |
+
if os.path.isdir(checkpoint_path):
|
| 38 |
+
with open(os.path.join(checkpoint_path, "pytorch_model.bin.index.json"), "r") as f:
|
| 39 |
+
index_data = json.load(f)
|
| 40 |
+
|
| 41 |
+
weight_map = index_data["weight_map"]
|
| 42 |
+
|
| 43 |
+
shard_files = set(weight_map.values())
|
| 44 |
+
shard_weights = {}
|
| 45 |
+
for shard_file in shard_files:
|
| 46 |
+
shard_weights[shard_file] = torch.load(os.path.join(checkpoint_path, shard_file), map_location="cpu")
|
| 47 |
+
|
| 48 |
+
checkpoints_state = {}
|
| 49 |
+
for param_name, shard_file in weight_map.items():
|
| 50 |
+
checkpoints_state[param_name] = shard_weights[shard_file][param_name]
|
| 51 |
+
elif os.path.splitext(checkpoint_path)[1] == '.safetensors':
|
| 52 |
+
from safetensors.torch import load_file
|
| 53 |
+
checkpoints_state = load_file(checkpoint_path)
|
| 54 |
+
else:
|
| 55 |
+
checkpoints_state = torch.load(checkpoint_path, map_location="cpu")
|
| 56 |
+
|
| 57 |
+
if "model" in checkpoints_state:
|
| 58 |
+
checkpoints_state = checkpoints_state["model"]
|
| 59 |
+
elif "module" in checkpoints_state:
|
| 60 |
+
checkpoints_state = checkpoints_state["module"]
|
| 61 |
+
|
| 62 |
+
model_state_dict = self.state_dict()
|
| 63 |
+
filtered_state_dict = {k: v for k, v in checkpoints_state.items() if k in model_state_dict and v.size() == model_state_dict[k].size()}
|
| 64 |
+
|
| 65 |
+
IncompatibleKeys = self.load_state_dict(filtered_state_dict, strict=False)
|
| 66 |
+
IncompatibleKeys = IncompatibleKeys._asdict()
|
| 67 |
+
|
| 68 |
+
missing_keys = []
|
| 69 |
+
for keys in IncompatibleKeys["missing_keys"]:
|
| 70 |
+
if keys.find("dummy") == -1:
|
| 71 |
+
missing_keys.append(keys)
|
| 72 |
+
|
| 73 |
+
unexpected_keys = []
|
| 74 |
+
for keys in IncompatibleKeys["unexpected_keys"]:
|
| 75 |
+
if keys.find("dummy") == -1:
|
| 76 |
+
unexpected_keys.append(keys)
|
| 77 |
+
|
| 78 |
+
if len(missing_keys) > 0:
|
| 79 |
+
logger.info(
|
| 80 |
+
"Missing keys in {}: {}".format(
|
| 81 |
+
checkpoint_path,
|
| 82 |
+
missing_keys,
|
| 83 |
+
)
|
| 84 |
+
)
|
| 85 |
+
|
| 86 |
+
if len(unexpected_keys) > 0:
|
| 87 |
+
logger.info(
|
| 88 |
+
"Unexpected keys {}: {}".format(
|
| 89 |
+
checkpoint_path,
|
| 90 |
+
unexpected_keys,
|
| 91 |
+
)
|
| 92 |
+
)
|
| 93 |
+
|
| 94 |
+
def forward(self, batched_data, **kwargs):
|
| 95 |
+
nucleus_img = batched_data['nucleus_img']
|
| 96 |
+
protein_img = batched_data['protein_img']
|
| 97 |
+
label = batched_data['label']
|
| 98 |
+
|
| 99 |
+
input_img = torch.cat([nucleus_img, protein_img], dim=1) # [B, 2, H, W]
|
| 100 |
+
|
| 101 |
+
logits = self.net(input_img) # [B, num_classes]
|
| 102 |
+
|
| 103 |
+
loss = self.loss(logits, label.view(-1))
|
| 104 |
+
loss = loss.mean()
|
| 105 |
+
|
| 106 |
+
return VanillaOutput(loss=loss)
|
| 107 |
+
|
| 108 |
+
def predict(self, x):
|
| 109 |
+
with torch.no_grad():
|
| 110 |
+
input_img = x # [B, 2, H, W]
|
| 111 |
+
logits = self.net(input_img) # [B, num_classes]
|
| 112 |
+
|
| 113 |
+
return logits
|
| 114 |
+
|
| 115 |
+
class ViT(VisionTransformer):
|
| 116 |
+
def __init__(self, config: ViTConfig):
|
| 117 |
+
super().__init__(
|
| 118 |
+
image_size=config.image_size,
|
| 119 |
+
patch_size=config.patch_size,
|
| 120 |
+
num_layers=config.num_layers,
|
| 121 |
+
num_heads=config.num_heads,
|
| 122 |
+
hidden_dim=config.hidden_dim,
|
| 123 |
+
mlp_dim=config.mlp_dim,
|
| 124 |
+
dropout=0.0,
|
| 125 |
+
attention_dropout=0.0,
|
| 126 |
+
num_classes=config.num_classes,
|
| 127 |
+
representation_size=None,
|
| 128 |
+
norm_layer=partial(nn.LayerNorm, eps=1e-6),
|
| 129 |
+
conv_stem_configs=None,
|
| 130 |
+
)
|
| 131 |
+
|
| 132 |
+
old_conv = self.conv_proj
|
| 133 |
+
in_channels = config.in_channels
|
| 134 |
+
|
| 135 |
+
self.conv_proj = nn.Conv2d(
|
| 136 |
+
in_channels=in_channels,
|
| 137 |
+
out_channels=old_conv.out_channels,
|
| 138 |
+
kernel_size=old_conv.kernel_size,
|
| 139 |
+
stride=old_conv.stride,
|
| 140 |
+
padding=old_conv.padding,
|
| 141 |
+
bias=old_conv.bias is not None,
|
| 142 |
+
)
|
| 143 |
+
|
| 144 |
+
def forward(self, x: torch.Tensor):
|
| 145 |
+
# Reshape and permute the input tensor
|
| 146 |
+
x = self._process_input(x)
|
| 147 |
+
n = x.shape[0]
|
| 148 |
+
|
| 149 |
+
# Expand the class token to the full batch
|
| 150 |
+
batch_class_token = self.class_token.expand(n, -1, -1)
|
| 151 |
+
x = torch.cat([batch_class_token, x], dim=1)
|
| 152 |
+
|
| 153 |
+
x = self.encoder(x)
|
| 154 |
+
x = x.mean(dim=1) # Global average pooling over the sequence length
|
| 155 |
+
x = self.heads(x)
|
| 156 |
+
|
| 157 |
+
return x
|
| 158 |
+
|
| 159 |
+
def embed(self, x: torch.Tensor):
|
| 160 |
+
# Reshape and permute the input tensor
|
| 161 |
+
x = self._process_input(x)
|
| 162 |
+
n = x.shape[0]
|
| 163 |
+
|
| 164 |
+
# Expand the class token to the full batch
|
| 165 |
+
batch_class_token = self.class_token.expand(n, -1, -1)
|
| 166 |
+
x = torch.cat([batch_class_token, x], dim=1)
|
| 167 |
+
|
| 168 |
+
x = self.encoder(x)
|
| 169 |
+
embed = x.mean(dim=1) # Global average pooling over the sequence length
|
| 170 |
+
|
| 171 |
+
return embed
|
cell_fm/pipeline/__init__.py
ADDED
|
File without changes
|
cell_fm/pipeline/utils.py
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
from dataclasses import dataclass
|
| 3 |
+
from transformers.utils import ModelOutput
|
| 4 |
+
from transformers import Trainer
|
| 5 |
+
from cell_fm.logging import metric_logger
|
| 6 |
+
from typing import Dict, Optional
|
| 7 |
+
|
| 8 |
+
class CELLDiffTrainer(Trainer):
|
| 9 |
+
def compute_loss(self, model, inputs, return_outputs=False):
|
| 10 |
+
# Compute loss using parent class
|
| 11 |
+
loss, outputs = super().compute_loss(model, inputs, True)
|
| 12 |
+
# metric_logger.log(outputs.log_output)
|
| 13 |
+
# self.log(outputs.log_output)
|
| 14 |
+
|
| 15 |
+
return (loss, outputs) if return_outputs else loss
|
| 16 |
+
|
| 17 |
+
@dataclass
|
| 18 |
+
class StableDiffusionOutput(ModelOutput):
|
| 19 |
+
loss: torch.FloatTensor = None
|
| 20 |
+
|
| 21 |
+
@dataclass
|
| 22 |
+
class VAEOutput(ModelOutput):
|
| 23 |
+
loss: torch.FloatTensor = None
|
| 24 |
+
log_output: Optional[Dict] = None
|
| 25 |
+
|
| 26 |
+
@dataclass
|
| 27 |
+
class CELLFMOutput(ModelOutput):
|
| 28 |
+
loss: torch.FloatTensor = None
|
| 29 |
+
log_output: Optional[Dict] = None
|
| 30 |
+
|
| 31 |
+
@dataclass
|
| 32 |
+
class VanillaOutput(ModelOutput):
|
| 33 |
+
loss: torch.FloatTensor = None
|
| 34 |
+
log_output: Optional[Dict] = None
|
cell_fm/utils/__init__.py
ADDED
|
File without changes
|
cell_fm/utils/dist_utils.py
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# -*- coding: utf-8 -*-
|
| 2 |
+
import os
|
| 3 |
+
|
| 4 |
+
import torch
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
def is_master_node():
|
| 8 |
+
if "RANK" not in os.environ or int(os.environ["RANK"]) == 0:
|
| 9 |
+
return True
|
| 10 |
+
else:
|
| 11 |
+
return False
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
def support_flash_attntion():
|
| 15 |
+
# check the GPU is A100 or H100
|
| 16 |
+
device_name = torch.cuda.get_device_name(0)
|
| 17 |
+
for name in ["A100", "H100"]:
|
| 18 |
+
if name in device_name:
|
| 19 |
+
return True
|
| 20 |
+
return False
|
metrics.py
ADDED
|
@@ -0,0 +1,135 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Curve metrics for the condensate titration analysis.
|
| 2 |
+
|
| 3 |
+
Port of cell_diff/tasks/analysis/utils.py from CELL-Diff2-Dev, restricted to the
|
| 4 |
+
three quantities this demo reports:
|
| 5 |
+
|
| 6 |
+
AUC area under the moving-averaged condensate-probability curve
|
| 7 |
+
NR the same curve with everything past its maximum held flat ("no reentrant")
|
| 8 |
+
AAC NR - AUC, i.e. the area *above* the curve that reentrant dissolution opens up
|
| 9 |
+
|
| 10 |
+
All three integrate against log10(intensity) by default and are normalised by the
|
| 11 |
+
width of the integration domain, so they live in [0, 1] and are comparable across
|
| 12 |
+
sequences.
|
| 13 |
+
"""
|
| 14 |
+
|
| 15 |
+
import numpy as np
|
| 16 |
+
import pandas as pd
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def moving_average(y, window: int) -> np.ndarray:
|
| 20 |
+
"""Centred rolling mean, shrinking the window at the edges rather than padding."""
|
| 21 |
+
y = np.asarray(y, dtype=float)
|
| 22 |
+
window = int(max(1, min(window, len(y))))
|
| 23 |
+
return (
|
| 24 |
+
pd.Series(y)
|
| 25 |
+
.rolling(window=window, center=True, min_periods=1)
|
| 26 |
+
.mean()
|
| 27 |
+
.to_numpy()
|
| 28 |
+
)
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def integral(xs: np.ndarray, ys: np.ndarray) -> float:
|
| 32 |
+
xs = np.asarray(xs, dtype=float)
|
| 33 |
+
ys = np.asarray(ys, dtype=float)
|
| 34 |
+
|
| 35 |
+
if xs.ndim != 1 or ys.ndim != 1:
|
| 36 |
+
raise ValueError("xs and ys must be 1D arrays")
|
| 37 |
+
if xs.shape[0] != ys.shape[0]:
|
| 38 |
+
raise ValueError("xs and ys must have the same length")
|
| 39 |
+
if xs.shape[0] < 2:
|
| 40 |
+
return 0.0
|
| 41 |
+
|
| 42 |
+
return float(np.trapz(ys, xs))
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def _prepare(df: pd.DataFrame, window_size: int, use_log_scale: bool):
|
| 46 |
+
"""Return (x, curve) sorted by x, with x log10-scaled when requested.
|
| 47 |
+
|
| 48 |
+
Sorting happens once, before the moving average is taken, so the smoothing
|
| 49 |
+
always runs along increasing intensity.
|
| 50 |
+
"""
|
| 51 |
+
intensity_levels = df["protein_intensity_level"].to_numpy(dtype=float)
|
| 52 |
+
predicted_classes = df["predicted_class"].to_numpy(dtype=float)
|
| 53 |
+
|
| 54 |
+
order = np.argsort(intensity_levels)
|
| 55 |
+
intensity_levels = intensity_levels[order]
|
| 56 |
+
predicted_classes = predicted_classes[order]
|
| 57 |
+
|
| 58 |
+
curve = moving_average(predicted_classes, window_size)
|
| 59 |
+
|
| 60 |
+
if use_log_scale:
|
| 61 |
+
# clip at 1.0 so that intensities below 1 do not send log10 negative/-inf
|
| 62 |
+
x = np.log10(np.clip(intensity_levels, a_min=1.0, a_max=None))
|
| 63 |
+
else:
|
| 64 |
+
x = intensity_levels
|
| 65 |
+
|
| 66 |
+
return x, curve
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
def _normalise(area: float, x: np.ndarray, normalize: bool) -> float:
|
| 70 |
+
if not normalize:
|
| 71 |
+
return area
|
| 72 |
+
total_area = (x[-1] - x[0]) * 1.0
|
| 73 |
+
return area / total_area
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
def compute_area_under_curve(
|
| 77 |
+
df: pd.DataFrame, window_size: int, normalize: bool = True, use_log_scale: bool = True
|
| 78 |
+
) -> float:
|
| 79 |
+
"""AUC: how much of the intensity range this sequence spends in the condensed state."""
|
| 80 |
+
x, curve = _prepare(df, window_size, use_log_scale)
|
| 81 |
+
return _normalise(integral(x, curve), x, normalize)
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
def no_reentrant_curve(curve: np.ndarray) -> np.ndarray:
|
| 85 |
+
"""The curve it would have been without reentrant dissolution.
|
| 86 |
+
|
| 87 |
+
Everything past the maximum is held at the maximum, so the difference against
|
| 88 |
+
the real curve is exactly the dissolution that happens at high concentration.
|
| 89 |
+
"""
|
| 90 |
+
max_index = int(np.argmax(curve))
|
| 91 |
+
out = curve.copy()
|
| 92 |
+
out[max_index:] = float(np.max(curve))
|
| 93 |
+
return out
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
def compute_area_if_no_reentrant(
|
| 97 |
+
df: pd.DataFrame, window_size: int, normalize: bool = True, use_log_scale: bool = True
|
| 98 |
+
) -> float:
|
| 99 |
+
"""Area under the curve after flattening everything past its maximum."""
|
| 100 |
+
x, curve = _prepare(df, window_size, use_log_scale)
|
| 101 |
+
return _normalise(integral(x, no_reentrant_curve(curve)), x, normalize)
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
def compute_area_if_no_reentrant_minus_auc(
|
| 105 |
+
df: pd.DataFrame, window_size: int, normalize: bool = True, use_log_scale: bool = True
|
| 106 |
+
) -> float:
|
| 107 |
+
"""AAC, the area *above* the curve: no-reentrant area minus the real area.
|
| 108 |
+
|
| 109 |
+
Zero when the curve never comes back down; larger the more the sequence
|
| 110 |
+
dissolves again at high concentration.
|
| 111 |
+
"""
|
| 112 |
+
x, curve = _prepare(df, window_size, use_log_scale)
|
| 113 |
+
return _normalise(integral(x, no_reentrant_curve(curve) - curve), x, normalize)
|
| 114 |
+
|
| 115 |
+
|
| 116 |
+
# the paper's name for compute_area_if_no_reentrant_minus_auc
|
| 117 |
+
compute_area_above_curve = compute_area_if_no_reentrant_minus_auc
|
| 118 |
+
|
| 119 |
+
|
| 120 |
+
def compute_concentration(
|
| 121 |
+
df: pd.DataFrame, window_size: int, threshold_prob: float = 0.8
|
| 122 |
+
) -> float:
|
| 123 |
+
"""c_sat: the first intensity at which the smoothed curve reaches threshold_prob.
|
| 124 |
+
|
| 125 |
+
inf when the sequence never condenses over the scanned range.
|
| 126 |
+
"""
|
| 127 |
+
intensity_levels = df["protein_intensity_level"].to_numpy(dtype=float)
|
| 128 |
+
predicted_classes = df["predicted_class"].to_numpy(dtype=float)
|
| 129 |
+
|
| 130 |
+
order = np.argsort(intensity_levels)
|
| 131 |
+
intensity_levels = intensity_levels[order]
|
| 132 |
+
curve = moving_average(predicted_classes[order], window_size)
|
| 133 |
+
|
| 134 |
+
hits = np.flatnonzero(curve >= threshold_prob)
|
| 135 |
+
return float(intensity_levels[hits[0]]) if hits.size else float("inf")
|
pipeline.py
ADDED
|
@@ -0,0 +1,312 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""CELL-FM CondenSeq: sequence -> condensate titration curve.
|
| 2 |
+
|
| 3 |
+
The demo runs the same three stages as the offline analysis, in memory:
|
| 4 |
+
|
| 5 |
+
1. generate CELL-FM CS samples one image per protein-intensity level, over a
|
| 6 |
+
log-spaced concentration ladder (evaluate_seq2img_dict.py)
|
| 7 |
+
2. classify a ViT labels each image condensed / diffuse (evaluate_single_img.py)
|
| 8 |
+
3. summarise a centred moving average over the binary calls gives the condensate
|
| 9 |
+
probability curve (ma_plot.py), and AUC / AAC integrate it
|
| 10 |
+
(analysis/ana_all_mutation_log_scale.py)
|
| 11 |
+
|
| 12 |
+
Weights come from a HF model repo by default; set CELLFM_LOCAL_WEIGHTS to a
|
| 13 |
+
directory holding the three .bin files to run against a local copy instead.
|
| 14 |
+
"""
|
| 15 |
+
|
| 16 |
+
import os
|
| 17 |
+
from dataclasses import dataclass, field
|
| 18 |
+
from functools import lru_cache
|
| 19 |
+
|
| 20 |
+
import numpy as np
|
| 21 |
+
import pandas as pd
|
| 22 |
+
import torch
|
| 23 |
+
|
| 24 |
+
from cell_fm.criterions.cell_fm.unidiffuser import UniDiffCriterions
|
| 25 |
+
from cell_fm.models.cell_fm.cell_fm_cs_config import CELLFMConfig
|
| 26 |
+
from cell_fm.models.cell_fm.cell_fm_cs_model import CELLFMCSModel
|
| 27 |
+
from cell_fm.models.vit_cls_condenseq_img.config import ViTConfig
|
| 28 |
+
from cell_fm.models.vit_cls_condenseq_img.model import ViTModel
|
| 29 |
+
|
| 30 |
+
import metrics
|
| 31 |
+
|
| 32 |
+
HERE = os.path.dirname(os.path.abspath(__file__))
|
| 33 |
+
|
| 34 |
+
MODEL_REPO = os.environ.get("CELLFM_MODEL_REPO", "BoHuangLab/CELL-FM")
|
| 35 |
+
LOCAL_WEIGHTS = os.environ.get("CELLFM_LOCAL_WEIGHTS", "")
|
| 36 |
+
|
| 37 |
+
# The weight repo holds one subfolder per model family, so paths are prefixed.
|
| 38 |
+
# CELLFM_LOCAL_WEIGHTS points at a flat directory, hence the basename() below.
|
| 39 |
+
GENERATOR_WEIGHTS = "condenseq/cellfm_seq2img.bin"
|
| 40 |
+
VAE_WEIGHTS = "condenseq/vae.bin"
|
| 41 |
+
CLASSIFIER_WEIGHTS = "condenseq/vit_cls.bin"
|
| 42 |
+
|
| 43 |
+
# The concentration ladder. The offline runs scanned 4096 levels; the demo scans
|
| 44 |
+
# fewer for latency, and the moving-average window scales with it (see WINDOW_DIVISOR).
|
| 45 |
+
MIN_INTENSITY = 20.0
|
| 46 |
+
MAX_INTENSITY = 2048.0
|
| 47 |
+
WINDOW_DIVISOR = 8 # window = num_images // 8, matching ana_all_mutation_log_scale.py
|
| 48 |
+
|
| 49 |
+
VALID_AA = set("ACDEFGHIKLMNPQRSTVWY")
|
| 50 |
+
|
| 51 |
+
# Generation starts from a random latent, so the curve moves a little between runs.
|
| 52 |
+
# The offline scripts fixed this at 6; keeping the same default makes the demo's
|
| 53 |
+
# numbers repeatable and comparable to them.
|
| 54 |
+
DEFAULT_SEED = 6
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
def resolve_weights(filename: str) -> str:
|
| 58 |
+
"""Local directory if CELLFM_LOCAL_WEIGHTS is set, else pull from the model repo."""
|
| 59 |
+
if LOCAL_WEIGHTS:
|
| 60 |
+
path = os.path.join(LOCAL_WEIGHTS, os.path.basename(filename))
|
| 61 |
+
if not os.path.exists(path):
|
| 62 |
+
raise FileNotFoundError(f"{path} not found (CELLFM_LOCAL_WEIGHTS={LOCAL_WEIGHTS})")
|
| 63 |
+
return path
|
| 64 |
+
|
| 65 |
+
from huggingface_hub import hf_hub_download
|
| 66 |
+
|
| 67 |
+
return hf_hub_download(repo_id=MODEL_REPO, filename=filename)
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
def generator_config() -> CELLFMConfig:
|
| 71 |
+
"""Hyperparameters from scripts/cell_fm_cs/evaluate_seq2img.sh."""
|
| 72 |
+
return CELLFMConfig(
|
| 73 |
+
img_resize=160,
|
| 74 |
+
img_crop_size=160,
|
| 75 |
+
img_type="GFP",
|
| 76 |
+
cell_image="nucl",
|
| 77 |
+
seq_zero_mask_ratio=0.0,
|
| 78 |
+
path_type="Linear",
|
| 79 |
+
prediction="velocity",
|
| 80 |
+
# VAE
|
| 81 |
+
num_down_blocks=3,
|
| 82 |
+
latent_channels=4,
|
| 83 |
+
vae_block_out_channels="128,256,512",
|
| 84 |
+
# CELL-FM
|
| 85 |
+
img_mask_ratio=0,
|
| 86 |
+
cond_out_channels="32,64",
|
| 87 |
+
sample_size=40,
|
| 88 |
+
esm_embedding="esmc_600m",
|
| 89 |
+
encoder_hidden_size=1152,
|
| 90 |
+
max_protein_sequence_len=2048,
|
| 91 |
+
encoder_num_hidden_layers=4,
|
| 92 |
+
num_heads=8,
|
| 93 |
+
dim_head=64,
|
| 94 |
+
dropout=0,
|
| 95 |
+
final_dropout=0,
|
| 96 |
+
encoder_patch_size=4,
|
| 97 |
+
# image generator
|
| 98 |
+
img_generator_num_layers=4,
|
| 99 |
+
img_generator_patch_size=2,
|
| 100 |
+
attention_head_dim=64,
|
| 101 |
+
num_attention_heads=8,
|
| 102 |
+
# image decoder
|
| 103 |
+
img_decoder_num_hidden_layers=2,
|
| 104 |
+
img_decoder_hidden_size=512,
|
| 105 |
+
img_decoder_num_heads=8,
|
| 106 |
+
img_decoder_dim_head=64,
|
| 107 |
+
cell_image_ratio=0.5,
|
| 108 |
+
# checkpoints
|
| 109 |
+
vae_loadcheck_path=resolve_weights(VAE_WEIGHTS),
|
| 110 |
+
loadcheck_path=resolve_weights(GENERATOR_WEIGHTS),
|
| 111 |
+
infer=True,
|
| 112 |
+
)
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
def classifier_config() -> ViTConfig:
|
| 116 |
+
"""Hyperparameters from scripts/vit_cls_condenseq_img/pretrain.sh."""
|
| 117 |
+
return ViTConfig(
|
| 118 |
+
image_size=160,
|
| 119 |
+
in_channels=2,
|
| 120 |
+
patch_size=4,
|
| 121 |
+
num_layers=8,
|
| 122 |
+
num_heads=8,
|
| 123 |
+
hidden_dim=512,
|
| 124 |
+
mlp_dim=2048,
|
| 125 |
+
num_classes=2,
|
| 126 |
+
loadcheck_path=resolve_weights(CLASSIFIER_WEIGHTS),
|
| 127 |
+
infer=True,
|
| 128 |
+
)
|
| 129 |
+
|
| 130 |
+
|
| 131 |
+
def get_device() -> torch.device:
|
| 132 |
+
return torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
| 133 |
+
|
| 134 |
+
|
| 135 |
+
@lru_cache(maxsize=1)
|
| 136 |
+
def load_models():
|
| 137 |
+
"""Build both models once and keep them resident."""
|
| 138 |
+
device = get_device()
|
| 139 |
+
|
| 140 |
+
generator = CELLFMCSModel(config=generator_config(), loss_fn=UniDiffCriterions)
|
| 141 |
+
generator.to(device).eval()
|
| 142 |
+
|
| 143 |
+
classifier = ViTModel(config=classifier_config())
|
| 144 |
+
classifier.to(device).eval()
|
| 145 |
+
|
| 146 |
+
from esm.tokenization.sequence_tokenizer import EsmSequenceTokenizer
|
| 147 |
+
|
| 148 |
+
return generator, classifier, EsmSequenceTokenizer()
|
| 149 |
+
|
| 150 |
+
|
| 151 |
+
@lru_cache(maxsize=1)
|
| 152 |
+
def reference_nucleus() -> torch.Tensor:
|
| 153 |
+
"""The DAPI channel every generated image is conditioned on, as (1, 1, 160, 160) in [-1, 1].
|
| 154 |
+
|
| 155 |
+
Taken from CondenSeq protein index 12626, image 0 -- the same one the offline
|
| 156 |
+
seq2img runs used, so demo curves are comparable to the published ones.
|
| 157 |
+
"""
|
| 158 |
+
arr = np.load(os.path.join(HERE, "assets", "reference_nucleus.npy"))
|
| 159 |
+
return torch.from_numpy(arr).float().unsqueeze(0)
|
| 160 |
+
|
| 161 |
+
|
| 162 |
+
def intensity_ladder(num_images: int) -> torch.Tensor:
|
| 163 |
+
"""Log-spaced protein intensity levels, the x axis of the titration curve."""
|
| 164 |
+
return torch.logspace(
|
| 165 |
+
start=float(np.log10(MIN_INTENSITY)),
|
| 166 |
+
end=float(np.log10(MAX_INTENSITY)),
|
| 167 |
+
steps=num_images,
|
| 168 |
+
)
|
| 169 |
+
|
| 170 |
+
|
| 171 |
+
def clean_sequence(sequence: str) -> str:
|
| 172 |
+
"""Strip FASTA headers, whitespace and case; reject anything not a standard residue."""
|
| 173 |
+
lines = [ln for ln in str(sequence).splitlines() if not ln.startswith(">")]
|
| 174 |
+
seq = "".join(lines).strip().upper()
|
| 175 |
+
seq = "".join(seq.split())
|
| 176 |
+
|
| 177 |
+
if not seq:
|
| 178 |
+
raise ValueError("Enter a protein sequence.")
|
| 179 |
+
bad = sorted(set(seq) - VALID_AA)
|
| 180 |
+
if bad:
|
| 181 |
+
raise ValueError(
|
| 182 |
+
f"Sequence contains non-standard residues: {', '.join(bad)}. "
|
| 183 |
+
"Use the 20 standard amino acids."
|
| 184 |
+
)
|
| 185 |
+
return seq
|
| 186 |
+
|
| 187 |
+
|
| 188 |
+
@dataclass
|
| 189 |
+
class Result:
|
| 190 |
+
sequence: str
|
| 191 |
+
intensities: np.ndarray # (N,) protein intensity levels
|
| 192 |
+
predictions: np.ndarray # (N,) 0 = diffuse, 1 = condensed
|
| 193 |
+
curve: np.ndarray # (N,) moving-averaged condensate probability
|
| 194 |
+
window: int
|
| 195 |
+
auc: float # area under the curve
|
| 196 |
+
aac: float # area above the curve (reentrant dissolution)
|
| 197 |
+
no_reentrant_auc: float
|
| 198 |
+
c_sat: float
|
| 199 |
+
images: np.ndarray = field(repr=False, default=None) # (K, 160, 160) sample panel
|
| 200 |
+
image_intensities: np.ndarray = field(repr=False, default=None)
|
| 201 |
+
|
| 202 |
+
|
| 203 |
+
@torch.no_grad()
|
| 204 |
+
def generate_images(sequence, num_images, num_steps, batch_size, seed=DEFAULT_SEED, progress=None):
|
| 205 |
+
"""Stage 1: one generated image per intensity level, returned as uint16 (N, 2, 160, 160).
|
| 206 |
+
|
| 207 |
+
uint16 is not incidental -- the offline pipeline wrote a TIFF between generation
|
| 208 |
+
and classification, so the classifier saw quantised pixels. Keeping the same
|
| 209 |
+
round-trip here keeps demo numbers comparable to the published ones.
|
| 210 |
+
"""
|
| 211 |
+
generator, _, vocab = load_models()
|
| 212 |
+
device = get_device()
|
| 213 |
+
|
| 214 |
+
from esm.utils import encoding
|
| 215 |
+
|
| 216 |
+
if seed is not None:
|
| 217 |
+
torch.manual_seed(int(seed))
|
| 218 |
+
if torch.cuda.is_available():
|
| 219 |
+
torch.cuda.manual_seed_all(int(seed))
|
| 220 |
+
|
| 221 |
+
tokens = encoding.tokenize_sequence(sequence, vocab, True).unsqueeze(0).to(device)
|
| 222 |
+
nucleus = reference_nucleus().to(device)
|
| 223 |
+
levels = intensity_ladder(num_images).to(device)
|
| 224 |
+
|
| 225 |
+
out = []
|
| 226 |
+
for start in range(0, num_images, batch_size):
|
| 227 |
+
end = min(start + batch_size, num_images)
|
| 228 |
+
n = end - start
|
| 229 |
+
|
| 230 |
+
if progress is not None:
|
| 231 |
+
progress(start / num_images, desc=f"Generating images {start}-{end} of {num_images}")
|
| 232 |
+
|
| 233 |
+
sample = generator.sequence_to_image(
|
| 234 |
+
tokens.repeat(n, 1),
|
| 235 |
+
nucleus.repeat(n, 1, 1, 1),
|
| 236 |
+
levels[start:end].unsqueeze(1),
|
| 237 |
+
num_steps=num_steps,
|
| 238 |
+
)
|
| 239 |
+
|
| 240 |
+
# channel 0 = nucleus (conditioning), channel 1 = generated protein
|
| 241 |
+
combined = torch.cat([nucleus.repeat(n, 1, 1, 1), sample], dim=1)
|
| 242 |
+
combined = ((combined + 1) / 2.0).clamp(0.0, 1.0)
|
| 243 |
+
out.append((combined * 65535).cpu().numpy().astype(np.uint16))
|
| 244 |
+
|
| 245 |
+
return np.concatenate(out, axis=0)
|
| 246 |
+
|
| 247 |
+
|
| 248 |
+
@torch.no_grad()
|
| 249 |
+
def classify_images(images, batch_size, progress=None):
|
| 250 |
+
"""Stage 2: condensed (1) vs diffuse (0) for each generated image."""
|
| 251 |
+
_, classifier, _ = load_models()
|
| 252 |
+
device = get_device()
|
| 253 |
+
|
| 254 |
+
preds = []
|
| 255 |
+
for start in range(0, images.shape[0], batch_size):
|
| 256 |
+
end = min(start + batch_size, images.shape[0])
|
| 257 |
+
|
| 258 |
+
if progress is not None:
|
| 259 |
+
progress(start / images.shape[0], desc=f"Classifying {start}-{end} of {images.shape[0]}")
|
| 260 |
+
|
| 261 |
+
batch = torch.from_numpy(images[start:end].astype(np.float32)).to(device)
|
| 262 |
+
batch = batch / 65535.0 * 2.0 - 1.0 # back to [-1, 1]
|
| 263 |
+
logits = classifier.predict(batch)
|
| 264 |
+
preds.append(torch.argmax(logits, dim=1).cpu().numpy())
|
| 265 |
+
|
| 266 |
+
return np.concatenate(preds).astype(int)
|
| 267 |
+
|
| 268 |
+
|
| 269 |
+
def summarise(intensities, predictions, num_images):
|
| 270 |
+
"""Stage 3: moving average, then AUC / AAC / c_sat over the smoothed curve."""
|
| 271 |
+
df = pd.DataFrame(
|
| 272 |
+
{"protein_intensity_level": intensities, "predicted_class": predictions}
|
| 273 |
+
)
|
| 274 |
+
window = max(1, num_images // WINDOW_DIVISOR)
|
| 275 |
+
kw = dict(window_size=window, use_log_scale=True)
|
| 276 |
+
|
| 277 |
+
return df, window, dict(
|
| 278 |
+
curve=metrics.moving_average(predictions, window),
|
| 279 |
+
auc=metrics.compute_area_under_curve(df, **kw),
|
| 280 |
+
aac=metrics.compute_area_above_curve(df, **kw),
|
| 281 |
+
no_reentrant_auc=metrics.compute_area_if_no_reentrant(df, **kw),
|
| 282 |
+
c_sat=metrics.compute_concentration(df, window_size=window),
|
| 283 |
+
)
|
| 284 |
+
|
| 285 |
+
|
| 286 |
+
def sample_panel(images, intensities, k=6):
|
| 287 |
+
"""Pick k images spread evenly along the ladder, for the strip shown under the curve."""
|
| 288 |
+
idx = np.linspace(0, len(intensities) - 1, k).astype(int)
|
| 289 |
+
return images[idx, 1].astype(np.float32) / 65535.0, intensities[idx]
|
| 290 |
+
|
| 291 |
+
|
| 292 |
+
def run(sequence, num_images=512, num_steps=100, batch_size=64, seed=DEFAULT_SEED,
|
| 293 |
+
progress=None) -> Result:
|
| 294 |
+
"""The whole pipeline: sequence in, titration curve and its areas out."""
|
| 295 |
+
sequence = clean_sequence(sequence)
|
| 296 |
+
|
| 297 |
+
images = generate_images(sequence, num_images, num_steps, batch_size, seed, progress)
|
| 298 |
+
predictions = classify_images(images, batch_size, progress)
|
| 299 |
+
|
| 300 |
+
intensities = intensity_ladder(num_images).numpy()
|
| 301 |
+
_, window, stats = summarise(intensities, predictions, num_images)
|
| 302 |
+
panel, panel_x = sample_panel(images, intensities)
|
| 303 |
+
|
| 304 |
+
return Result(
|
| 305 |
+
sequence=sequence,
|
| 306 |
+
intensities=intensities,
|
| 307 |
+
predictions=predictions,
|
| 308 |
+
window=window,
|
| 309 |
+
images=panel,
|
| 310 |
+
image_intensities=panel_x,
|
| 311 |
+
**stats,
|
| 312 |
+
)
|
requirements.txt
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Pinned to the versions the CELL-FM checkpoints were trained and evaluated with.
|
| 2 |
+
torch==2.4.1
|
| 3 |
+
torchvision==0.19.1
|
| 4 |
+
diffusers==0.31.0
|
| 5 |
+
transformers==4.44.2
|
| 6 |
+
esm==3.1.4
|
| 7 |
+
numpy==1.26.3
|
| 8 |
+
pandas==2.2.2
|
| 9 |
+
matplotlib==3.9.2
|
| 10 |
+
tifffile==2024.9.20
|
| 11 |
+
safetensors==0.4.5
|
| 12 |
+
einops==0.8.0
|
| 13 |
+
huggingface_hub>=0.24.6
|
| 14 |
+
gradio>=4.44.0
|
| 15 |
+
|
| 16 |
+
# flash-attn is deliberately NOT installed. ESM-C falls back to its pure-torch
|
| 17 |
+
# rotary when the package is absent (esm/models/esmc.py: is_flash_attn_available),
|
| 18 |
+
# and the two paths were verified bit-identical -- same AUC, same AAC, same 256/256
|
| 19 |
+
# per-image calls, same runtime -- so the Space skips a fragile CUDA build for free.
|
run_local.sh
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/bin/bash
|
| 2 |
+
# Run the demo on a cluster GPU and open a public tunnel to it.
|
| 3 |
+
#
|
| 4 |
+
# Works today, with no HF plan: gradio's --share flag serves the app from this
|
| 5 |
+
# machine over a temporary public URL (72 h, printed on startup). Useful for
|
| 6 |
+
# showing the demo to the lab while the Space is blocked.
|
| 7 |
+
#
|
| 8 |
+
# sbatch run_local.sh # or run the python line directly on a GPU node
|
| 9 |
+
#
|
| 10 |
+
#SBATCH --job-name=cellfm_demo
|
| 11 |
+
#SBATCH --time=08:00:00
|
| 12 |
+
#SBATCH --nodes=1
|
| 13 |
+
#SBATCH --ntasks=1
|
| 14 |
+
#SBATCH --partition=interactive
|
| 15 |
+
#SBATCH --cpus-per-task=8
|
| 16 |
+
#SBATCH --gpus=1
|
| 17 |
+
#SBATCH --output=cellfm_demo_%j.out
|
| 18 |
+
|
| 19 |
+
module load anaconda
|
| 20 |
+
conda activate esm3
|
| 21 |
+
export PYTHONUNBUFFERED=1
|
| 22 |
+
|
| 23 |
+
# Pull weights from the Hub (needs HF_TOKEN while HuangLab/CELL-FM is private),
|
| 24 |
+
# or point at a flat local directory holding the three .bin files:
|
| 25 |
+
# export CELLFM_LOCAL_WEIGHTS=/path/to/weights
|
| 26 |
+
|
| 27 |
+
cd "$(dirname "$0")"
|
| 28 |
+
GRADIO_SHARE=1 python app.py
|
upload_weights.py
ADDED
|
@@ -0,0 +1,106 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Push the three CELL-FM CondenSeq checkpoints to a HF model repo.
|
| 2 |
+
|
| 3 |
+
NOTE: never place credentials inside this directory -- it is uploaded verbatim to
|
| 4 |
+
the Space. Pass a token via the HF_TOKEN environment variable instead.
|
| 5 |
+
|
| 6 |
+
The Space pulls its weights from that repo at startup rather than carrying ~2.1 GB
|
| 7 |
+
of LFS itself. Run this once from the cluster:
|
| 8 |
+
|
| 9 |
+
python upload_weights.py --repo BoHuangLab/CELL-FM
|
| 10 |
+
|
| 11 |
+
Add --private to keep the weights unlisted, and --dry-run to see what would be
|
| 12 |
+
uploaded without touching the Hub.
|
| 13 |
+
"""
|
| 14 |
+
|
| 15 |
+
import argparse
|
| 16 |
+
import os
|
| 17 |
+
|
| 18 |
+
DEFAULT_SOURCES = {
|
| 19 |
+
# target name in the repo -> checkpoint on the cluster
|
| 20 |
+
"condenseq/cellfm_seq2img.bin": (
|
| 21 |
+
"/hpc/projects/group.huang/dihan.zheng/CELL-FM/"
|
| 22 |
+
"pretrain_condenseq/cellfm_seq2img/checkpoint-50000/pytorch_model.bin"
|
| 23 |
+
),
|
| 24 |
+
"condenseq/vae.bin": (
|
| 25 |
+
"/hpc/projects/group.huang/dihan.zheng/CELL-FM/"
|
| 26 |
+
"pretrain_condenseq/vae/checkpoint-50000/pytorch_model.bin"
|
| 27 |
+
),
|
| 28 |
+
"condenseq/vit_cls.bin": (
|
| 29 |
+
"/hpc/projects/group.huang/dihan.zheng/CELL-Diff2-Dev/"
|
| 30 |
+
"pretrain_condenseq_celldiff2_split/PT_CondenSeq_img_ViT_cls_R1/"
|
| 31 |
+
"checkpoint-10000/pytorch_model.bin"
|
| 32 |
+
),
|
| 33 |
+
}
|
| 34 |
+
|
| 35 |
+
CARD = """---
|
| 36 |
+
library_name: cell-fm
|
| 37 |
+
tags:
|
| 38 |
+
- biology
|
| 39 |
+
- microscopy
|
| 40 |
+
- protein
|
| 41 |
+
- condensate
|
| 42 |
+
- flow-matching
|
| 43 |
+
---
|
| 44 |
+
|
| 45 |
+
# CELL-FM weights
|
| 46 |
+
|
| 47 |
+
Checkpoints behind the [CELL-FM CondenSeq demo]({space_url}).
|
| 48 |
+
|
| 49 |
+
| File | Model | Source checkpoint |
|
| 50 |
+
|---|---|---|
|
| 51 |
+
| `condenseq/cellfm_seq2img.bin` | CELL-FM CS sequence-to-image generator (includes the ESM-C 600M encoder) | `pretrain_condenseq/cellfm_seq2img/checkpoint-50000` |
|
| 52 |
+
| `condenseq/vae.bin` | Image VAE, 160x160, 3 down blocks, 4 latent channels | `pretrain_condenseq/vae/checkpoint-50000` |
|
| 53 |
+
| `condenseq/vit_cls.bin` | ViT condensed/diffuse classifier, 2-channel 160x160 input | `PT_CondenSeq_img_ViT_cls_R1/checkpoint-10000` |
|
| 54 |
+
|
| 55 |
+
Hyperparameters are set in `pipeline.py` in the Space and mirror
|
| 56 |
+
`scripts/cell_fm_cs/evaluate_seq2img.sh` and
|
| 57 |
+
`scripts/vit_cls_condenseq_img/pretrain.sh` in the CELL-FM repository.
|
| 58 |
+
"""
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
def main() -> None:
|
| 62 |
+
ap = argparse.ArgumentParser()
|
| 63 |
+
ap.add_argument("--repo", default="BoHuangLab/CELL-FM")
|
| 64 |
+
ap.add_argument("--space-url", default="https://huggingface.co/spaces/BoHuangLab/CELL-FM")
|
| 65 |
+
ap.add_argument("--private", action="store_true")
|
| 66 |
+
ap.add_argument("--dry-run", action="store_true")
|
| 67 |
+
args = ap.parse_args()
|
| 68 |
+
|
| 69 |
+
missing = [p for p in DEFAULT_SOURCES.values() if not os.path.exists(p)]
|
| 70 |
+
if missing:
|
| 71 |
+
raise SystemExit("missing checkpoint(s):\n " + "\n ".join(missing))
|
| 72 |
+
|
| 73 |
+
total = sum(os.path.getsize(p) for p in DEFAULT_SOURCES.values())
|
| 74 |
+
for name, path in DEFAULT_SOURCES.items():
|
| 75 |
+
print(f" {name:22s} {os.path.getsize(path)/1e9:5.2f} GB <- {path}")
|
| 76 |
+
print(f" {'total':22s} {total/1e9:5.2f} GB -> {args.repo}")
|
| 77 |
+
|
| 78 |
+
if args.dry_run:
|
| 79 |
+
print("\ndry run, nothing uploaded")
|
| 80 |
+
return
|
| 81 |
+
|
| 82 |
+
from huggingface_hub import HfApi
|
| 83 |
+
|
| 84 |
+
api = HfApi()
|
| 85 |
+
api.create_repo(args.repo, repo_type="model", private=args.private, exist_ok=True)
|
| 86 |
+
|
| 87 |
+
for name, path in DEFAULT_SOURCES.items():
|
| 88 |
+
print(f"uploading {name} ...")
|
| 89 |
+
api.upload_file(
|
| 90 |
+
path_or_fileobj=path,
|
| 91 |
+
path_in_repo=name,
|
| 92 |
+
repo_id=args.repo,
|
| 93 |
+
repo_type="model",
|
| 94 |
+
)
|
| 95 |
+
|
| 96 |
+
api.upload_file(
|
| 97 |
+
path_or_fileobj=CARD.format(space_url=args.space_url).encode(),
|
| 98 |
+
path_in_repo="README.md",
|
| 99 |
+
repo_id=args.repo,
|
| 100 |
+
repo_type="model",
|
| 101 |
+
)
|
| 102 |
+
print(f"\ndone: https://huggingface.co/{args.repo}")
|
| 103 |
+
|
| 104 |
+
|
| 105 |
+
if __name__ == "__main__":
|
| 106 |
+
main()
|