beiken1201 commited on
Commit
faf1f6d
·
verified ·
1 Parent(s): 07bb78a

Make project paths portable and standardize English code comments

Browse files

Resolve bundled resource defaults from script locations, preserve caller-relative path overrides, and require missing external pretraining or legacy inputs explicitly. Translate comments and docstrings to English, update command examples and documentation, and add a path helper for legacy reference code. Validation: 97 Python 3 files and 65 notebook code cells compiled; 12 existing utility tests passed; parser, example-command, resource-path and local-link checks passed. Two historical DeepCDR entrypoints retain Python 2 syntax. No training or model inference was run. This commit updates 90 source/documentation files and adds one helper; pretrained weights, paired PKL, raw datasets and other files are unchanged.

This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .gitignore +42 -42
  2. BioMod/README.md +409 -399
  3. BioMod/cell_annotation/cmd_finetune.md +21 -15
  4. BioMod/cell_annotation/module_data/load_module.py +35 -32
  5. BioMod/cell_annotation/performer_module_finetune.py +0 -0
  6. BioMod/command.txt +31 -31
  7. BioMod/drug_combination/README.md +145 -143
  8. BioMod/drug_combination/train_drug_combination_performer_module.py +906 -900
  9. BioMod/drug_response/cmd.txt +10 -10
  10. BioMod/drug_response/train_drug_performer_module.py +1075 -1065
  11. BioMod/gene_perturbation/README.md +276 -267
  12. BioMod/gene_perturbation/gears_reduce_local.py +8 -2
  13. BioMod/gene_perturbation/inference_local.py +1038 -1034
  14. BioMod/gene_perturbation/model_module.py +761 -760
  15. BioMod/gene_perturbation/pertdata_local.py +523 -523
  16. BioMod/gene_perturbation/train_module.py +333 -325
  17. BioMod/gene_perturbation/utils.py +496 -489
  18. BioMod/interpretability/README.md +361 -413
  19. BioMod/interpretability/attention/README.md +111 -108
  20. BioMod/interpretability/attention/README_gene_attention.md +112 -114
  21. BioMod/interpretability/attention/module_crosstalk.py +606 -605
  22. BioMod/interpretability/attention/module_gene_attention.py +1093 -1090
  23. BioMod/interpretability/module_geometry.py +655 -654
  24. BioMod/interpretability/module_importance/README.md +319 -311
  25. BioMod/interpretability/module_importance/RESULTS.md +226 -221
  26. BioMod/interpretability/module_text_alignment/DESIGN.md +235 -225
  27. BioMod/interpretability/module_text_alignment/fetch_reactome_descriptions.py +97 -97
  28. BioMod/interpretability/module_text_alignment/module_text_alignment.py +665 -665
  29. BioMod/interpretability/module_text_alignment/module_text_alignment_extended.py +537 -537
  30. BioMod/interpretability/module_text_alignment/module_text_alignment_publication.py +0 -0
  31. BioMod/models_performer_module.py +0 -0
  32. BioMod/module_data_all/create_uncovered_modules.py +151 -146
  33. BioMod/module_data_all/merge_modules.py +96 -96
  34. BioMod/performer_module.py +18 -12
  35. BioMod/zero_shot_batch/README.md +159 -193
  36. BioMod/zero_shot_batch/performer_module_zero_shot.py +694 -689
  37. BioMod/zero_shot_batch/sequencing_depth/cmd.txt +41 -40
  38. BioMod/zero_shot_batch/sequencing_depth/run_zero_shot_low_depth.py +196 -196
  39. BioMod/zero_shot_batch/visualize_results.py +293 -293
  40. README.md +15 -10
  41. data/downstream/final_data/drug_combination_synergy/README.md +50 -48
  42. data/downstream/final_data/drug_combination_synergy/data_summary/README.md +77 -78
  43. data/downstream/final_data/drug_combination_synergy/data_summary/test_data/README.md +38 -46
  44. data/downstream/final_data/drug_response_pytorch/drug_combination/README.md +40 -9
  45. data/downstream/final_data/drug_response_pytorch/drug_combination/_reference_paths.py +64 -0
  46. data/downstream/final_data/drug_response_pytorch/drug_combination/cluster.py +26 -25
  47. data/downstream/final_data/drug_response_pytorch/drug_combination/creat_data_DC.py +124 -123
  48. data/downstream/final_data/drug_response_pytorch/drug_combination/data_summary/README.md +77 -78
  49. data/downstream/final_data/drug_response_pytorch/drug_combination/data_summary/test_data/README.md +38 -46
  50. data/downstream/final_data/drug_response_pytorch/drug_combination/heatmap.py +114 -113
.gitignore CHANGED
@@ -1,42 +1,42 @@
1
- # ===== 大文件:不进 GitHub,单独托管(HF / OSS)=====
2
- # weight/ 4.1GB 训练权重 checkpoint
3
- # data/ 17GB 下游公开 benchmark 数据集
4
- weight/
5
- data/
6
- !data/
7
- !data/downstream/
8
- !data/downstream/final_data/
9
- !data/downstream/final_data/norman/
10
- !data/downstream/final_data/norman/data_pyg/
11
- !data/downstream/final_data/norman/data_pyg/cell_graphs.pkl
12
- !data/downstream/final_data/norman/splits/
13
- !data/downstream/final_data/norman/splits/*.pkl
14
-
15
- # ===== 二进制 / 打包 / 缓存 =====
16
- *.zip
17
- !data/downstream/final_data/drug_response_pytorch/drug_response_pytorch/data/GDSC/drug_graph_feat.zip
18
- !data/downstream/final_data/single_drug_response/optional_reference/GDSC/drug_graph_feat.zip
19
- *.tar
20
- *.tar.gz
21
- *.pth
22
- *.pkl
23
- !data/downstream/final_data/norman/data_pyg/cell_graphs.pkl
24
- !data/downstream/final_data/norman/splits/*.pkl
25
- *.ckpt
26
- __pycache__/
27
- *.pyc
28
- .ipynb_checkpoints/
29
-
30
- # ===== 本地环境与密钥 =====
31
- .env
32
- *.local
33
-
34
- # Dated release checkpoints are intentionally versioned on Hugging Face.
35
- !weight/
36
- weight/*
37
- !weight/0629model/
38
- !weight/0629model/*.pkl
39
- !weight/0629model/*.pth
40
- !weight/0810model/
41
- !weight/0810model/*.pkl
42
- !weight/0810model/*.pth
 
1
+ # Large assets are hosted on Hugging Face or separate object storage.
2
+ # weight/ contains pretrained checkpoints and paired configurations.
3
+ # data/ contains downstream benchmark datasets and reference code.
4
+ weight/
5
+ data/
6
+ !data/
7
+ !data/downstream/
8
+ !data/downstream/final_data/
9
+ !data/downstream/final_data/norman/
10
+ !data/downstream/final_data/norman/data_pyg/
11
+ !data/downstream/final_data/norman/data_pyg/cell_graphs.pkl
12
+ !data/downstream/final_data/norman/splits/
13
+ !data/downstream/final_data/norman/splits/*.pkl
14
+
15
+ # Binary files, archives, and caches.
16
+ *.zip
17
+ !data/downstream/final_data/drug_response_pytorch/drug_response_pytorch/data/GDSC/drug_graph_feat.zip
18
+ !data/downstream/final_data/single_drug_response/optional_reference/GDSC/drug_graph_feat.zip
19
+ *.tar
20
+ *.tar.gz
21
+ *.pth
22
+ *.pkl
23
+ !data/downstream/final_data/norman/data_pyg/cell_graphs.pkl
24
+ !data/downstream/final_data/norman/splits/*.pkl
25
+ *.ckpt
26
+ __pycache__/
27
+ *.pyc
28
+ .ipynb_checkpoints/
29
+
30
+ # Local environment files and credentials.
31
+ .env
32
+ *.local
33
+
34
+ # Dated release checkpoints are intentionally versioned on Hugging Face.
35
+ !weight/
36
+ weight/*
37
+ !weight/0629model/
38
+ !weight/0629model/*.pkl
39
+ !weight/0629model/*.pth
40
+ !weight/0810model/
41
+ !weight/0810model/*.pkl
42
+ !weight/0810model/*.pth
BioMod/README.md CHANGED
@@ -1,399 +1,409 @@
1
- # BioMod / PerformerModule
2
-
3
- Source code and tutorial notebooks for BioMod single-cell pretraining and
4
- downstream experiments using PerformerModule representations.
5
-
6
- This source directory contains the pretraining entry point and shared training
7
- utilities, alongside downstream task runners, tutorial notebooks, environment
8
- files, and plotting helpers.
9
-
10
- ## Highlights
11
-
12
- | Area | What is included |
13
- | --- | --- |
14
- | Representation | Module-based single-cell encoder utilities and module metadata |
15
- | Annotation | Supervised cell-type annotation workflows |
16
- | Integration | Zero-shot batch integration and UMAP/scIB-style reporting |
17
- | Perturbation | Norman gene-perturbation tutorial and smoke workflow |
18
- | Drug response | Single-drug response prediction with GDSC/CCLE data and validation-based model selection |
19
- | Drug combinations | Drug-combination synergy prediction with validation-based model selection |
20
- | Figures | Notebook-level PDF/PNG/SVG plotting cells for paper and GitHub display |
21
-
22
- ## Repository Layout
23
-
24
- This is the `BioMod/` source directory inside the project root. See the
25
- [full project layout](../README.md#repository-layout) for sibling
26
- `model_dependencies/` and `weight/` directories; `data/` and `.cache/` are omitted
27
- from these repository trees.
28
-
29
- ```text
30
- BioMod/ # Task code: training, evaluation, interpretability and tutorials
31
- ├── cell_annotation/ # Cell-type annotation and model fine-tuning
32
- │ ├── module_data/ # Hallmark and Reactome modules with loading utilities
33
- │ │ ├── load_module.py
34
- │ │ ├── module1_hallmarks.json
35
- │ │ └── module2_reactome_2022_filtered.json
36
- │ ├── cmd_finetune.md
37
- │ └── performer_module_finetune.py
38
- ├── drug_combination/ # Drug-combination synergy prediction
39
- │ ├── README.md
40
- │ └── train_drug_combination_performer_module.py
41
- ├── drug_response/ # Single-drug response prediction and training
42
- │ ├── cmd.txt
43
- │ └── train_drug_performer_module.py
44
- ├── gene_perturbation/ # Perturbation response prediction: processing, training and inference
45
- │ ├── data_utils_local.py
46
- │ ├── gears_reduce_local.py
47
- │ ├── inference_local.py
48
- │ ├── model_module.py
49
- │ ├── pertdata_local.py
50
- │ ├── README.md
51
- │ ├── scfm_utils_local.py
52
- │ ├── train_module.py
53
- │ └── utils.py
54
- ├── interpretability/ # Module representations and biological interpretability
55
- │ ├── attention/ # Module crosstalk and within-module gene attention
56
- │ │ ├── __init__.py
57
- │ │ ├── module_crosstalk.py
58
- │ │ ├── module_gene_attention.py
59
- │ │ ├── README.md
60
- │ │ └── README_gene_attention.md
61
- │ ├── module_importance/ # Module importance, ablation studies and marker-gene validation
62
- │ │ ├── __init__.py
63
- │ │ ├── cross_class_similarity.py
64
- │ │ ├── family_ablation.py
65
- │ │ ├── family_importance_analysis.py
66
- │ │ ├── marker_gene_verification.py
67
- │ │ ├── masking_ablation.py
68
- │ │ ├── module_importance.py
69
- │ │ ├── module_importance_publication.py
70
- │ │ ├── pub_utils.py
71
- │ │ ├── random_baseline.py
72
- │ │ ├── README.md
73
- │ │ ├── RESULTS.md
74
- │ │ ├── tcell_reactome_analysis.py
75
- │ │ └── TODO.md
76
- │ ├── module_text_alignment/ # Semantic alignment of module representations and pathway text
77
- │ │ ├── __init__.py
78
- │ │ ├── DESIGN.md
79
- │ │ ├── fetch_reactome_descriptions.py
80
- │ │ ├── MANUSCRIPT_SECTION.md
81
- │ │ ├── module_text_alignment.py
82
- │ │ ├── module_text_alignment_extended.py
83
- │ │ ├── module_text_alignment_publication.py
84
- │ │ └── reactome_descriptions.json
85
- │ ├── __init__.py
86
- │ ├── module_geometry.py
87
- │ ├── README.md
88
- │ └── utils.py
89
- ├── module_data_all/ # Module definitions, merging and uncovered-gene tools
90
- │ ├── create_uncovered_modules.py
91
- │ ├── merge_modules.py
92
- │ └── merged_modules.json
93
- ├── tests/ # CPU tests for shared training utilities
94
- │ └── test_training_utils.py
95
- ├── toturials/ # Jupyter tutorials for five downstream tasks
96
- │ ├── cell_annotation.ipynb
97
- │ ├── drug_combination.ipynb
98
- │ ├── drug_response_prediction.ipynb
99
- │ ├── gene_perturbation.ipynb
100
- │ └── zero_shot_batch_integration.ipynb
101
- ├── zero_shot_batch/ # Zero-shot batch integration, evaluation and visualization
102
- │ ├── sequencing_depth/ # Low-depth data generation and zero-shot evaluation
103
- │ │ ├── cmd.txt
104
- │ │ ├── make_low_depth_h5ad.py
105
- │ │ └── run_zero_shot_low_depth.py
106
- │ ├── performer_module_zero_shot.py
107
- │ ├── README.md
108
- │ └── visualize_results.py
109
- ├── .env.example
110
- ├── .gitattributes
111
- ├── .gitignore
112
- ├── command.txt
113
- ├── environment.yml
114
- ├── models_performer_module.py
115
- ├── module_dataset.py
116
- ├── performer_module.py
117
- ├── README.md
118
- ├── requirements.txt
119
- └── training_utils.py
120
- ```
121
-
122
- The notebook directory is intentionally named `toturials` to preserve existing
123
- paths used by the project.
124
-
125
- ## Quick Start
126
-
127
- Create a fresh environment:
128
-
129
- ```bash
130
- cd /path/to/BioMod/BioMod
131
- conda env create -f environment.yml
132
- conda activate performer_module
133
- ```
134
-
135
- Or install Python packages into an existing CUDA/PyTorch environment:
136
-
137
- ```bash
138
- python -m pip install -r requirements.txt
139
- ```
140
-
141
- For the original server runs, the tutorials assume the `sclong` environment:
142
-
143
- ```bash
144
- conda activate sclong
145
- jupyter lab /path/to/BioMod/BioMod/toturials
146
- ```
147
-
148
- ### Training utility checks
149
-
150
- The pretraining entry point imports `training_utils.py` explicitly. This local
151
- module supplies random seeding, a sequential distributed validation sampler,
152
- cosine learning-rate scheduling with warmup and restarts, scalar metric
153
- reduction, and resumable checkpoint saving. It does not depend on a separate
154
- package named `utils` or on a downstream task's utility module.
155
-
156
- From this source directory, run the CPU utility tests and check entry-point
157
- imports after installing the dependencies:
158
-
159
- ```bash
160
- python -B -m unittest discover -s tests -p test_training_utils.py -v
161
- python -B performer_module.py --help
162
- ```
163
-
164
- The utility tests use synthetic tensors and temporary checkpoints; they do not
165
- load the supplied pretrained weights. Passing these checks does not validate a
166
- full training run. The pretraining entry point still requires CUDA and an NCCL
167
- process group, with training data and paths configured for the target machine.
168
-
169
- ## Shared Assets
170
-
171
- The BioMod model weights and matching configuration are stored in
172
- `weight/0810model/`. Use the supplied `.pth` and `.pkl` together, and update
173
- server-specific paths for the local environment.
174
-
175
- Tutorials expect a pretrained BioMod/PerformerModule checkpoint and shared
176
- gene assets. Set `BIOMODULEFM_ASSET_ROOT` to the directory containing
177
- `weight/0810model/` and `model_dependencies/` (the outer `BioMod/` project directory).
178
-
179
- ```bash
180
- export BIOMODULEFM_ASSET_ROOT=/path/to/BioMod
181
- ```
182
-
183
- Current local asset files:
184
-
185
- ```text
186
- $BIOMODULEFM_ASSET_ROOT/weight/0810model/ # BioMod model weights and matching configuration
187
- ├── mae_tost_70M_75mask_ec2.pkl
188
- └── mae_tost_70M_75mask_ec2_epoch0.pth
189
-
190
- $BIOMODULEFM_ASSET_ROOT/model_dependencies/ # Shared gene mappings, gene lists, embeddings and model components
191
- ├── module_data_all/ # Shared module definitions and construction tools
192
- │ ├── create_uncovered_modules.py
193
- │ ├── merge_modules.py
194
- │ └── merged_modules.json
195
- ├── performer_pytorch_cont/ # Performer attention and model components
196
- │ ├── performer_pytorch_cont/ # Nested package containing copies of the model component source files
197
- │ │ ├── __init__.py
198
- │ │ ├── ding_models.py
199
- │ │ ├── performer_pytorch.py
200
- │ │ ├── performer_pytorch_old.py
201
- │ │ └── reversible.py
202
- │ ├── __init__.py
203
- │ ├── ding_models.py
204
- │ ├── performer_pytorch.py
205
- │ ├── performer_pytorch_old.py
206
- │ └── reversible.py
207
- ├── gene2ens.csv
208
- ├── selected_gene2vec_27k.npy
209
- └── selected_genes_27k.txt
210
- ```
211
-
212
- Project-local module definitions are kept in:
213
-
214
- ```text
215
- BioMod/module_data_all/ # Module definitions, merging and uncovered-gene tools
216
- ├── create_uncovered_modules.py
217
- ├── merge_modules.py
218
- └── merged_modules.json
219
- ```
220
-
221
- ## BioMod model weights
222
-
223
- Use `mae_tost_70M_75mask_ec2_epoch0.pth` with
224
- `mae_tost_70M_75mask_ec2.pkl` from the shared-assets directory above. The supplied
225
- configuration specifies a 2-layer encoder, a 27,874-gene axis, and 660 module
226
- definitions. Keep this configuration paired with the weights so that task
227
- runners can restore the corresponding architecture settings.
228
-
229
- ## Tutorials
230
-
231
- | Task | Notebook | Main entry point | Default output style |
232
- | --- | --- | --- | --- |
233
- | Cell-type annotation | `toturials/cell_annotation.ipynb` | task script under `cell_annotation/` | metrics, classification report, confusion matrix |
234
- | Zero-shot batch integration | `toturials/zero_shot_batch_integration.ipynb` | `zero_shot_batch/` runner | UMAP PDF/PNG, avgBatch, avgBio |
235
- | Gene perturbation | `toturials/gene_perturbation.ipynb` | `gene_perturbation/` runner | GEARS-style metrics and smoke outputs |
236
- | Drug response prediction | `toturials/drug_response_prediction.ipynb` | `drug_response/train_drug_performer_module.py` | scatter, residuals, ROC, per-drug and per-cancer tables |
237
- | Drug-combination synergy | `toturials/drug_combination.ipynb` | `drug_combination/train_drug_combination_performer_module.py` | label balance, loss, AUROC, PR-AUC, confusion matrix when predictions exist |
238
-
239
- Each notebook uses absolute paths through configurable roots and keeps full
240
- training disabled by default when data are not present. Flip the notebook
241
- `RUN_TRAINING` flag only after checking the path table at the top of the
242
- notebook.
243
-
244
- ## Data Layout
245
-
246
- The Hugging Face repository includes datasets under the project-root `data/`
247
- directory. Individual tasks may require additional inputs or a different data
248
- layout. Check each task's required files, then configure its data paths for the
249
- local environment; the presence of `data/` does not imply that every tutorial
250
- has all of its inputs.
251
-
252
- ### Single-Drug Response
253
-
254
- Expected clean data root:
255
-
256
- ```text
257
- single_drug_response/data/
258
- |-- CCLE/
259
- | |-- Cell_lines_annotations_20181226.txt
260
- | |-- GDSC_IC50.csv
261
- | `-- genomic_expression_561celllines_697genes_demap_features.csv
262
- `-- GDSC/
263
- |-- 1.Drug_listMon Jun 24 09_00_55 2019.csv
264
- `-- drug_graph_feat/
265
- ```
266
-
267
- ### Drug-Combination Synergy
268
-
269
- Expected clean data root:
270
-
271
- ```text
272
- drug_combination/data_summary/
273
- |-- new_labels_0_10.csv
274
- |-- new_cell_features_954.csv
275
- |-- smiles_0_10.csv
276
- `-- test_data/
277
- |-- independent_input.csv
278
- `-- independent_cell_features_954.csv
279
- ```
280
-
281
- The drug-combination tutorial also supports old DeepDDS-style prediction CSV
282
- files for ROC, PR, and confusion-matrix visualization.
283
-
284
- ## Example Commands
285
-
286
- Drug response prediction with the BioMod model weights:
287
-
288
- ```bash
289
- python drug_response/train_drug_performer_module.py \
290
- --gpu_id 0 \
291
- --data_path /path/to/single_drug_response/data \
292
- --checkpoint_path $BIOMODULEFM_ASSET_ROOT/weight/0810model/mae_tost_70M_75mask_ec2_epoch0.pth \
293
- --hyper_params_path $BIOMODULEFM_ASSET_ROOT/weight/0810model/mae_tost_70M_75mask_ec2.pkl \
294
- --module_data_dir ./module_data_all \
295
- --gene_list_path $BIOMODULEFM_ASSET_ROOT/model_dependencies/selected_genes_27k.txt \
296
- --gene_mapping_path $BIOMODULEFM_ASSET_ROOT/model_dependencies/gene2ens.csv \
297
- --gene2vec_path $BIOMODULEFM_ASSET_ROOT/model_dependencies/selected_gene2vec_27k.npy \
298
- --symbol2ens_path $BIOMODULEFM_ASSET_ROOT/model_dependencies/gene2ens.csv \
299
- --save_dir ./results/drug_response \
300
- --epochs 300 \
301
- --batch_size 32 \
302
- --pool_method max
303
- ```
304
-
305
- Drug-combination synergy prediction:
306
-
307
- ```bash
308
- python drug_combination/train_drug_combination_performer_module.py \
309
- --checkpoint_path $BIOMODULEFM_ASSET_ROOT/weight/0810model/mae_tost_70M_75mask_ec2_epoch0.pth \
310
- --hyper_params_path $BIOMODULEFM_ASSET_ROOT/weight/0810model/mae_tost_70M_75mask_ec2.pkl \
311
- --module_data_dir ./module_data_all \
312
- --gene_list_path $BIOMODULEFM_ASSET_ROOT/model_dependencies/selected_genes_27k.txt \
313
- --gene_mapping_path $BIOMODULEFM_ASSET_ROOT/model_dependencies/gene2ens.csv \
314
- --gene2vec_path $BIOMODULEFM_ASSET_ROOT/model_dependencies/selected_gene2vec_27k.npy \
315
- --symbol2ens_path $BIOMODULEFM_ASSET_ROOT/model_dependencies/gene2ens.csv \
316
- --data_dir ./drug_combination/data_summary \
317
- --output_dir ./results/drug_combination \
318
- --epochs 500 \
319
- --batch_size 256 \
320
- --lr 1e-4 \
321
- --n_folds 1 \
322
- --pool_method max
323
- ```
324
-
325
- ## Display Outputs
326
-
327
- The notebooks regenerate figure files in task output directories. Typical files
328
- include:
329
-
330
- | Task | Generated display files |
331
- | --- | --- |
332
- | Zero-shot integration | A4 UMAP PDF/PNG panels, combined cell-type and batch views |
333
- | Drug response | `drug_response_overall_scatter.pdf`, `drug_response_median_threshold_roc.svg`, per-drug and per-cancer CSV summaries |
334
- | Drug combination | `drug_combination_metric_curves.pdf`, `drug_combination_roc_curves.pdf`, `drug_combination_label_balance.pdf`, optional PR and confusion-matrix panels |
335
- | Cell annotation | confusion matrices, classification reports, UMAP/prediction diagnostics |
336
- | Gene perturbation | smoke-run logs, perturbation metrics, downstream result tables |
337
-
338
- For GitHub display, use PNG files. For manuscripts, use the PDF/SVG exports.
339
- The plotting cells use embedded, editable PDF fonts where supported.
340
-
341
- ## Evaluation Results
342
-
343
- This README does not provide verified benchmark scores for the bundled BioMod
344
- model weights. Run the relevant task with a recorded data split and configuration
345
- before reporting performance. Notebook plotting cells and historical outputs
346
- alone do not establish performance for the supplied weights.
347
-
348
- ## Reproducibility Conventions
349
-
350
- - Record the exact command line, checkpoint, hyper-parameter pkl, data version,
351
- random seed, and output directory for every final run.
352
- - Keep raw data, cached embeddings, model checkpoints, metric tables, and paper
353
- figures in separate directories.
354
- - Prefer shared storage for large `.h5ad`, `.npy`, and checkpoint outputs.
355
- - Do not commit private data, raw model weights, API keys, or personal absolute
356
- paths.
357
-
358
- ## Troubleshooting
359
-
360
- ### Missing data
361
-
362
- Open the relevant notebook and run the path-check cell. It prints all required
363
- files and shows which entries are missing.
364
-
365
- ### Missing model assets
366
-
367
- Check `BIOMODULEFM_ASSET_ROOT` and verify that the BioMod model weights, matching
368
- `.pkl` configuration, gene2vec file, gene list, gene mapping, and
369
- `performer_pytorch_cont/` directory exist at the paths listed under Shared Assets.
370
-
371
- ### No space left on device
372
-
373
- Large `.h5ad`, `.npy`, checkpoint, and figure outputs can fill local disks.
374
- Move result directories to shared storage and remove partial failed outputs only
375
- after confirming they are not needed.
376
-
377
- ### Matplotlib font warnings
378
-
379
- The notebooks request Arial first and fall back to DejaVu Sans when Arial is not
380
- installed. PDF font embedding is enabled through `pdf.fonttype = 42`.
381
-
382
- ## GitHub Release Checklist
383
-
384
- - Keep `README.md`, `requirements.txt`, `environment.yml`, `.gitignore`, and
385
- `.gitattributes` at the repository root.
386
- - Confirm tutorial notebooks open without personal paths.
387
- - Keep large data and checkpoints outside Git, then document download or
388
- placement instructions.
389
- - Add final figure examples under a small `docs/` or `assets/` directory if the
390
- repository needs rendered images on the GitHub landing page.
391
- - Add license, citation, and data-availability text before public release.
392
-
393
- ## Changelog
394
-
395
- | Date | Change |
396
- | --- | --- |
397
- | 2026-06-15 | Added paper-style plotting cells for single-drug and drug-combination tutorials. |
398
- | 2026-06-15 | Removed `stage2_alignment/` from the active repository scope. |
399
- | 2026-06-15 | Added root environment files and GitHub-ready repository documentation. |
 
 
 
 
 
 
 
 
 
 
 
1
+ # BioMod / PerformerModule
2
+
3
+ Source code and tutorial notebooks for BioMod single-cell pretraining and
4
+ downstream experiments using PerformerModule representations.
5
+
6
+ This source directory contains the pretraining entry point and shared training
7
+ utilities, alongside downstream task runners, tutorial notebooks, environment
8
+ files, and plotting helpers.
9
+
10
+ ## Highlights
11
+
12
+ | Area | What is included |
13
+ | --- | --- |
14
+ | Representation | Module-based single-cell encoder utilities and module metadata |
15
+ | Annotation | Supervised cell-type annotation workflows |
16
+ | Integration | Zero-shot batch integration and UMAP/scIB-style reporting |
17
+ | Perturbation | Norman gene-perturbation training and evaluation tutorial |
18
+ | Drug response | Single-drug response prediction with GDSC/CCLE data and validation-based model selection |
19
+ | Drug combinations | Drug-combination synergy prediction with validation-based model selection |
20
+ | Figures | Notebook-level PDF/PNG/SVG plotting cells for paper and GitHub display |
21
+
22
+ ## Repository Layout
23
+
24
+ This is the `BioMod/` source directory inside the project root. See the
25
+ [full project layout](../README.md#repository-layout) for sibling
26
+ `model_dependencies/` and `weight/` directories; `data/` and `.cache/` are omitted
27
+ from these repository trees.
28
+
29
+ ```text
30
+ BioMod/ # Task code: training, evaluation, interpretability and tutorials
31
+ ├── cell_annotation/ # Cell-type annotation and model fine-tuning
32
+ │ ├── module_data/ # Hallmark and Reactome modules with loading utilities
33
+ │ │ ├── load_module.py
34
+ │ │ ├── module1_hallmarks.json
35
+ │ │ └── module2_reactome_2022_filtered.json
36
+ │ ├── cmd_finetune.md
37
+ │ └── performer_module_finetune.py
38
+ ├── drug_combination/ # Drug-combination synergy prediction
39
+ │ ├── README.md
40
+ │ └── train_drug_combination_performer_module.py
41
+ ├── drug_response/ # Single-drug response prediction and training
42
+ │ ├── cmd.txt
43
+ │ └── train_drug_performer_module.py
44
+ ├── gene_perturbation/ # Perturbation response prediction: processing, training and inference
45
+ │ ├── data_utils_local.py
46
+ │ ├── gears_reduce_local.py
47
+ │ ├── inference_local.py
48
+ │ ├── model_module.py
49
+ │ ├── pertdata_local.py
50
+ │ ├── README.md
51
+ │ ├── scfm_utils_local.py
52
+ │ ├── train_module.py
53
+ │ └── utils.py
54
+ ├── interpretability/ # Module representations and biological interpretability
55
+ │ ├── attention/ # Module crosstalk and within-module gene attention
56
+ │ │ ├── __init__.py
57
+ │ │ ├── module_crosstalk.py
58
+ │ │ ├── module_gene_attention.py
59
+ │ │ ├── README.md
60
+ │ │ └── README_gene_attention.md
61
+ │ ├── module_importance/ # Module importance, ablation studies and marker-gene validation
62
+ │ │ ├── __init__.py
63
+ │ │ ├── cross_class_similarity.py
64
+ │ │ ├── family_ablation.py
65
+ │ │ ├── family_importance_analysis.py
66
+ │ │ ├── marker_gene_verification.py
67
+ │ │ ├── masking_ablation.py
68
+ │ │ ├── module_importance.py
69
+ │ │ ├── module_importance_publication.py
70
+ │ │ ├── pub_utils.py
71
+ │ │ ├── random_baseline.py
72
+ │ │ ├── README.md
73
+ │ │ ├── RESULTS.md
74
+ │ │ ├── tcell_reactome_analysis.py
75
+ │ │ └── TODO.md
76
+ │ ├── module_text_alignment/ # Semantic alignment of module representations and pathway text
77
+ │ │ ├── __init__.py
78
+ │ │ ├── DESIGN.md
79
+ │ │ ├── fetch_reactome_descriptions.py
80
+ │ │ ├── MANUSCRIPT_SECTION.md
81
+ │ │ ├── module_text_alignment.py
82
+ │ │ ├── module_text_alignment_extended.py
83
+ │ │ ├── module_text_alignment_publication.py
84
+ │ │ └── reactome_descriptions.json
85
+ │ ├── __init__.py
86
+ │ ├── module_geometry.py
87
+ │ ├── README.md
88
+ │ └── utils.py
89
+ ├── module_data_all/ # Module definitions, merging and uncovered-gene tools
90
+ │ ├── create_uncovered_modules.py
91
+ │ ├── merge_modules.py
92
+ │ └── merged_modules.json
93
+ ├── tests/ # CPU tests for shared training utilities
94
+ │ └── test_training_utils.py
95
+ ├── toturials/ # Jupyter tutorials for five downstream tasks
96
+ │ ├── cell_annotation.ipynb
97
+ │ ├── drug_combination.ipynb
98
+ │ ├── drug_response_prediction.ipynb
99
+ │ ├── gene_perturbation.ipynb
100
+ │ └── zero_shot_batch_integration.ipynb
101
+ ├── zero_shot_batch/ # Zero-shot batch integration, evaluation and visualization
102
+ │ ├── sequencing_depth/ # Low-depth data generation and zero-shot evaluation
103
+ │ │ ├── cmd.txt
104
+ │ │ ├── make_low_depth_h5ad.py
105
+ │ │ └── run_zero_shot_low_depth.py
106
+ │ ├── performer_module_zero_shot.py
107
+ │ ├── README.md
108
+ │ └── visualize_results.py
109
+ ├── .env.example
110
+ ├── .gitattributes
111
+ ├── .gitignore
112
+ ├── command.txt
113
+ ├── environment.yml
114
+ ├── models_performer_module.py
115
+ ├── module_dataset.py
116
+ ├── performer_module.py
117
+ ├── README.md
118
+ ├── requirements.txt
119
+ └── training_utils.py
120
+ ```
121
+
122
+ The notebook directory is intentionally named `toturials` to preserve existing
123
+ paths used by the project.
124
+
125
+ ## Quick Start
126
+
127
+ From the outer repository root, enter the source directory and create a fresh
128
+ environment:
129
+
130
+ ```bash
131
+ cd BioMod
132
+ conda env create -f environment.yml
133
+ conda activate performer_module
134
+ ```
135
+
136
+ Or install Python packages into an existing CUDA/PyTorch environment:
137
+
138
+ ```bash
139
+ python -m pip install -r requirements.txt
140
+ ```
141
+
142
+ From this source directory, activate the supplied environment and start Jupyter
143
+ with the tutorial directory as its working directory:
144
+
145
+ ```bash
146
+ conda activate performer_module
147
+ cd toturials
148
+ jupyter lab
149
+ ```
150
+
151
+ ### Training utility checks
152
+
153
+ The pretraining entry point imports `training_utils.py` explicitly. This local
154
+ module supplies random seeding, a sequential distributed validation sampler,
155
+ cosine learning-rate scheduling with warmup and restarts, scalar metric
156
+ reduction, and resumable checkpoint saving. It does not depend on a separate
157
+ package named `utils` or on a downstream task's utility module.
158
+
159
+ From this source directory, run the CPU utility tests and check entry-point
160
+ imports after installing the dependencies:
161
+
162
+ ```bash
163
+ python -B -m unittest discover -s tests -p test_training_utils.py -v
164
+ python -B performer_module.py --help
165
+ ```
166
+
167
+ The utility tests use synthetic tensors and temporary checkpoints; they do not
168
+ load the supplied pretrained weights. Passing these checks does not validate a
169
+ full training run. The pretraining entry point requires CUDA and an NCCL
170
+ process group. Supply `--data_path` explicitly: the pretraining corpus is not
171
+ bundled. Configure module inputs with `--module_data_dir` or `--module_data_path`.
172
+ Generated checkpoints default to `tutorial_runs/pretraining/` under the outer
173
+ repository root, with periodic latest checkpoints under its `latest/` subdirectory.
174
+
175
+ ## Shared Assets
176
+
177
+ The BioMod model weights and matching configuration are stored in
178
+ the outer repository's `weight/0810model/`. Use the supplied `.pth` and `.pkl`
179
+ together.
180
+
181
+ Tutorials expect a pretrained BioMod/PerformerModule checkpoint and shared
182
+ gene assets. For the shell examples below, set `BIOMOD_ROOT` to the directory containing
183
+ `weight/0810model/` and `model_dependencies/` (the outer `BioMod/` project directory).
184
+
185
+ ```bash
186
+ # Run this from the inner BioMod/ source directory.
187
+ export BIOMOD_ROOT="$(cd .. && pwd)"
188
+ ```
189
+
190
+ Current local asset files:
191
+
192
+ ```text
193
+ $BIOMOD_ROOT/weight/0810model/ # BioMod model weights and matching configuration
194
+ ├── mae_tost_70M_75mask_ec2.pkl
195
+ └── mae_tost_70M_75mask_ec2_epoch0.pth
196
+
197
+ $BIOMOD_ROOT/model_dependencies/ # Shared gene mappings, gene lists, embeddings and model components
198
+ ├── module_data_all/ # Shared module definitions and construction tools
199
+ │ ├── create_uncovered_modules.py
200
+ │ ├── merge_modules.py
201
+ │ └── merged_modules.json
202
+ ├── performer_pytorch_cont/ # Performer attention and model components
203
+ │ ├── performer_pytorch_cont/ # Nested package containing copies of the model component source files
204
+ │ │ ├── __init__.py
205
+ │ │ ├── ding_models.py
206
+ │ │ ├── performer_pytorch.py
207
+ │ │ ├── performer_pytorch_old.py
208
+ │ │ └── reversible.py
209
+ │ ├── __init__.py
210
+ │ ├── ding_models.py
211
+ │ ├── performer_pytorch.py
212
+ │ ├── performer_pytorch_old.py
213
+ │ └── reversible.py
214
+ ├── gene2ens.csv
215
+ ├── selected_gene2vec_27k.npy
216
+ └── selected_genes_27k.txt
217
+ ```
218
+
219
+ Project-local module definitions are kept in:
220
+
221
+ ```text
222
+ BioMod/module_data_all/ # Module definitions, merging and uncovered-gene tools
223
+ ├── create_uncovered_modules.py
224
+ ├── merge_modules.py
225
+ └── merged_modules.json
226
+ ```
227
+
228
+ ## BioMod model weights
229
+
230
+ Use `mae_tost_70M_75mask_ec2_epoch0.pth` with
231
+ `mae_tost_70M_75mask_ec2.pkl` from the shared-assets directory above. The supplied
232
+ configuration specifies a 2-layer encoder, a 27,874-gene axis, and 660 module
233
+ definitions. Keep this configuration paired with the weights so that task
234
+ runners can restore the corresponding architecture settings.
235
+
236
+ ## Tutorials
237
+
238
+ | Task | Notebook | Main entry point | Default output style |
239
+ | --- | --- | --- | --- |
240
+ | Cell-type annotation | `toturials/cell_annotation.ipynb` | task script under `cell_annotation/` | metrics, classification report, confusion matrix |
241
+ | Zero-shot batch integration | `toturials/zero_shot_batch_integration.ipynb` | `zero_shot_batch/` runner | UMAP PDF/PNG, avgBatch, avgBio |
242
+ | Gene perturbation | `toturials/gene_perturbation.ipynb` | `gene_perturbation/` runner | selected model, configuration, parameters, and training log |
243
+ | Drug response prediction | `toturials/drug_response_prediction.ipynb` | `drug_response/train_drug_performer_module.py` | scatter, residuals, ROC, per-drug and per-cancer tables |
244
+ | Drug-combination synergy | `toturials/drug_combination.ipynb` | `drug_combination/train_drug_combination_performer_module.py` | label balance, loss, validation metrics, and ROC curves |
245
+
246
+ Each notebook derives paths from `PROJECT_ROOT = Path('../..').resolve()` when
247
+ Jupyter runs from `BioMod/toturials/`. From the repository root, change this to
248
+ `Path('.').resolve()`. Training is disabled by default with `RUN_TRAINING = False`;
249
+ zero-shot inference uses `RUN_EVALUATION = False`. Review the configuration and
250
+ path checks before enabling execution. The shell variable `BIOMOD_ROOT` is for
251
+ the command examples only and does not override notebook configuration cells.
252
+
253
+ ## Data Layout
254
+
255
+ The Hugging Face repository includes datasets under the project-root `data/`
256
+ directory. Individual tasks may require additional inputs or a different data
257
+ layout. Check each task's required files, then configure its data paths for the
258
+ local environment; the presence of `data/` does not imply that every tutorial
259
+ has all of its inputs.
260
+
261
+ ### Single-Drug Response
262
+
263
+ Expected clean data root:
264
+
265
+ ```text
266
+ data/downstream/final_data/single_drug_response/data/
267
+ |-- CCLE/
268
+ | |-- Cell_lines_annotations_20181226.txt
269
+ | |-- GDSC_IC50.csv
270
+ | `-- genomic_expression_561celllines_697genes_demap_features.csv
271
+ `-- GDSC/
272
+ |-- 1.Drug_listMon Jun 24 09_00_55 2019.csv
273
+ `-- drug_graph_feat/
274
+ ```
275
+
276
+ ### Drug-Combination Synergy
277
+
278
+ Expected clean data root:
279
+
280
+ ```text
281
+ data/downstream/final_data/drug_combination_synergy/data_summary/
282
+ |-- new_labels_0_10.csv
283
+ |-- new_cell_features_954.csv
284
+ |-- smiles_0_10.csv
285
+ `-- test_data/
286
+ |-- independent_input.csv
287
+ `-- independent_cell_features_954.csv
288
+ ```
289
+
290
+ The drug-combination tutorial reads the metric tables and ROC artifacts produced
291
+ by its training script. The script does not export per-example prediction CSVs.
292
+
293
+ ## Example Commands
294
+
295
+ Run these commands from the inner `BioMod/` source directory after setting
296
+ `BIOMOD_ROOT` as shown above. Drug response prediction with the BioMod model weights:
297
+
298
+ ```bash
299
+ python drug_response/train_drug_performer_module.py \
300
+ --gpu_id 0 \
301
+ --data_path "$BIOMOD_ROOT/data/downstream/final_data/single_drug_response/data" \
302
+ --checkpoint_path "$BIOMOD_ROOT/weight/0810model/mae_tost_70M_75mask_ec2_epoch0.pth" \
303
+ --hyper_params_path "$BIOMOD_ROOT/weight/0810model/mae_tost_70M_75mask_ec2.pkl" \
304
+ --module_data_dir "$BIOMOD_ROOT/model_dependencies/module_data_all" \
305
+ --gene_list_path "$BIOMOD_ROOT/model_dependencies/selected_genes_27k.txt" \
306
+ --gene_mapping_path "$BIOMOD_ROOT/model_dependencies/gene2ens.csv" \
307
+ --gene2vec_path "$BIOMOD_ROOT/model_dependencies/selected_gene2vec_27k.npy" \
308
+ --symbol2ens_path "$BIOMOD_ROOT/model_dependencies/gene2ens.csv" \
309
+ --save_dir "$BIOMOD_ROOT/tutorial_runs/drug_response_prediction" \
310
+ --epochs 300 \
311
+ --batch_size 32 \
312
+ --pool_method max
313
+ ```
314
+
315
+ Drug-combination synergy prediction:
316
+
317
+ ```bash
318
+ python drug_combination/train_drug_combination_performer_module.py \
319
+ --checkpoint_path "$BIOMOD_ROOT/weight/0810model/mae_tost_70M_75mask_ec2_epoch0.pth" \
320
+ --hyper_params_path "$BIOMOD_ROOT/weight/0810model/mae_tost_70M_75mask_ec2.pkl" \
321
+ --module_data_dir "$BIOMOD_ROOT/model_dependencies/module_data_all" \
322
+ --gene_list_path "$BIOMOD_ROOT/model_dependencies/selected_genes_27k.txt" \
323
+ --gene_mapping_path "$BIOMOD_ROOT/model_dependencies/gene2ens.csv" \
324
+ --gene2vec_path "$BIOMOD_ROOT/model_dependencies/selected_gene2vec_27k.npy" \
325
+ --symbol2ens_path "$BIOMOD_ROOT/model_dependencies/gene2ens.csv" \
326
+ --data_dir "$BIOMOD_ROOT/data/downstream/final_data/drug_combination_synergy/data_summary" \
327
+ --output_dir "$BIOMOD_ROOT/tutorial_runs/drug_combination/results" \
328
+ --epochs 500 \
329
+ --batch_size 256 \
330
+ --lr 1e-4 \
331
+ --n_folds 1 \
332
+ --pool_method max
333
+ ```
334
+
335
+ ## Display Outputs
336
+
337
+ The notebooks regenerate figure files in task output directories. Typical files
338
+ include:
339
+
340
+ | Task | Generated display files |
341
+ | --- | --- |
342
+ | Zero-shot integration | A4 UMAP PDF/PNG panels, combined cell-type and batch views |
343
+ | Drug response | `drug_response_overall_scatter.pdf`, `drug_response_median_threshold_roc.svg`, per-drug and per-cancer CSV summaries |
344
+ | Drug combination | `drug_combination_metric_curves.pdf`, `drug_combination_roc_curves.pdf`, `drug_combination_label_balance.pdf` |
345
+ | Cell annotation | confusion matrices and classification reports |
346
+ | Gene perturbation | training logs containing perturbation evaluation metrics |
347
+
348
+ For GitHub display, use PNG files. For manuscripts, use the PDF/SVG exports.
349
+ The plotting cells use embedded, editable PDF fonts where supported.
350
+
351
+ ## Evaluation Results
352
+
353
+ This README does not provide verified benchmark scores for the bundled BioMod
354
+ model weights. Run the relevant task with a recorded data split and configuration
355
+ before reporting performance. Notebook plotting cells and historical outputs
356
+ alone do not establish performance for the supplied weights.
357
+
358
+ ## Reproducibility Conventions
359
+
360
+ - Record the exact command line, checkpoint, hyper-parameter pkl, data version,
361
+ random seed, and output directory for every final run.
362
+ - Keep raw data, cached embeddings, model checkpoints, metric tables, and paper
363
+ figures in separate directories.
364
+ - Prefer shared storage for large `.h5ad`, `.npy`, and checkpoint outputs.
365
+ - Do not commit private data, raw model weights, API keys, or personal absolute
366
+ paths.
367
+
368
+ ## Troubleshooting
369
+
370
+ ### Missing data
371
+
372
+ Open the relevant notebook and run the path-check cell. It prints all required
373
+ files and shows which entries are missing.
374
+
375
+ ### Missing model assets
376
+
377
+ Check `BIOMOD_ROOT` for shell commands or `PROJECT_ROOT` in a notebook, and verify that the BioMod model weights, matching
378
+ `.pkl` configuration, gene2vec file, gene list, gene mapping, and
379
+ `performer_pytorch_cont/` directory exist at the paths listed under Shared Assets.
380
+
381
+ ### No space left on device
382
+
383
+ Large `.h5ad`, `.npy`, checkpoint, and figure outputs can fill local disks.
384
+ Move result directories to shared storage and remove partial failed outputs only
385
+ after confirming they are not needed.
386
+
387
+ ### Matplotlib font warnings
388
+
389
+ The notebooks request Arial first and fall back to DejaVu Sans when Arial is not
390
+ installed. PDF font embedding is enabled through `pdf.fonttype = 42`.
391
+
392
+ ## GitHub Release Checklist
393
+
394
+ - Keep `README.md`, `requirements.txt`, `environment.yml`, `.gitignore`, and
395
+ `.gitattributes` at the repository root.
396
+ - Confirm tutorial notebooks open without personal paths.
397
+ - Keep large data and checkpoints outside Git, then document download or
398
+ placement instructions.
399
+ - Add final figure examples under a small `docs/` or `assets/` directory if the
400
+ repository needs rendered images on the GitHub landing page.
401
+ - Add license, citation, and data-availability text before public release.
402
+
403
+ ## Changelog
404
+
405
+ | Date | Change |
406
+ | --- | --- |
407
+ | 2026-06-15 | Added paper-style plotting cells for single-drug and drug-combination tutorials. |
408
+ | 2026-06-15 | Removed `stage2_alignment/` from the active repository scope. |
409
+ | 2026-06-15 | Added root environment files and GitHub-ready repository documentation. |
BioMod/cell_annotation/cmd_finetune.md CHANGED
@@ -1,15 +1,21 @@
1
- torchrun --nproc_per_node=4 performer_module_finetune.py \
2
- --config_pkl ../ckpts_performer_module_multiply_all_module/performer_module_multiply_all_module.pkl \
3
- --performer_checkpoint ../ckpts_performer_module_multiply_all_module/performer_module_multiply_all_module_epoch6.pth \
4
- --data_path ../../cell_annotation/Zheng68K.h5ad \
5
- --label_column celltype \
6
- --head_type attn_pool \
7
- --finetune_mode last_n \
8
- --unfreeze_last_n 1 \
9
- --cls_decoder_layers 3 \
10
- --lr_head 1e-4 \
11
- --lr_backbone 1e-5 \
12
- --batch_size 256 \
13
- --epochs 300 \
14
- --module_data_dir ../module_data_all \
15
- --output_dir ../../../../share/extracted_feature/ckpts_performer_module_fintune/performer_module_multiply_all_module
 
 
 
 
 
 
 
1
+ # Cell annotation command
2
+
3
+ Run this command from the repository root in a CUDA/NCCL environment. Resource defaults resolve from the script location; explicit relative paths resolve from the working directory.
4
+
5
+ ```bash
6
+ torchrun --nproc_per_node=4 BioMod/cell_annotation/performer_module_finetune.py \
7
+ --config_pkl weight/0810model/mae_tost_70M_75mask_ec2.pkl \
8
+ --performer_checkpoint weight/0810model/mae_tost_70M_75mask_ec2_epoch0.pth \
9
+ --data_path data/downstream/final_data/Zheng68K.h5ad \
10
+ --label_column celltype \
11
+ --head_type attn_pool \
12
+ --finetune_mode last_n \
13
+ --unfreeze_last_n 1 \
14
+ --cls_decoder_layers 3 \
15
+ --lr_head 1e-4 \
16
+ --lr_backbone 1e-5 \
17
+ --batch_size 256 \
18
+ --epochs 300 \
19
+ --module_data_dir model_dependencies/module_data_all \
20
+ --output_dir tutorial_runs/cell_annotation_zheng68k
21
+ ```
BioMod/cell_annotation/module_data/load_module.py CHANGED
@@ -1,33 +1,36 @@
1
- def load_json_files(file1_path, file2_path):
2
- """
3
-
4
- Args:
5
- file1_path: hallmarks file path
6
- file2_path: reactome file path
7
-
8
- Returns:
9
- tuple: (hallmark_dict, filtered_reactome_dict)
10
- """
11
- # Read HALLMARKS
12
- with open(file1_path, 'r', encoding='utf-8') as f:
13
- hallmark_dict = json.load(f)
14
-
15
- # Read reactome data
16
- with open(file2_path, 'r', encoding='utf-8') as f:
17
- reactome_full = json.load(f)
18
-
19
- # Filter reactome data
20
- filtered_reactome = {
21
- k: v for k, v in reactome_full.items()
22
- if 50 <= len(v) <= 500
23
- }
24
-
25
- return hallmark_dict, filtered_reactome
26
-
27
-
28
- # hallmark_data, reactome_data = load_json_files(
29
- # 'module1_hallmarks.json',
30
- # 'module2_reactome.json'
31
- # )
32
- # len(hallmark_data.keys())
 
 
 
33
  # len(reactome_data.keys())
 
1
+ import json
2
+
3
+
4
+ def load_json_files(file1_path, file2_path):
5
+ """
6
+
7
+ Args:
8
+ file1_path: hallmarks file path
9
+ file2_path: reactome file path
10
+
11
+ Returns:
12
+ tuple: (hallmark_dict, filtered_reactome_dict)
13
+ """
14
+ # Read HALLMARKS
15
+ with open(file1_path, 'r', encoding='utf-8') as f:
16
+ hallmark_dict = json.load(f)
17
+
18
+ # Read reactome data
19
+ with open(file2_path, 'r', encoding='utf-8') as f:
20
+ reactome_full = json.load(f)
21
+
22
+ # Filter reactome data
23
+ filtered_reactome = {
24
+ k: v for k, v in reactome_full.items()
25
+ if 50 <= len(v) <= 500
26
+ }
27
+
28
+ return hallmark_dict, filtered_reactome
29
+
30
+
31
+ # hallmark_data, reactome_data = load_json_files(
32
+ # 'module1_hallmarks.json',
33
+ # 'module2_reactome_2022_filtered.json'
34
+ # )
35
+ # len(hallmark_data.keys())
36
  # len(reactome_data.keys())
BioMod/cell_annotation/performer_module_finetune.py CHANGED
The diff for this file is too large to render. See raw diff
 
BioMod/command.txt CHANGED
@@ -1,31 +1,31 @@
1
- source activate base
2
- conda activate sclong
3
- cd ../data/250010166/scLong/performer_module
4
- torchrun --nproc_per_node $SENSECORE_ACCELERATE_DEVICE_COUNT --nnodes $SENSECORE_PYTORCH_NNODES --node_rank $SENSECORE_PYTORCH_NODE_RANK --master_addr $MASTER_ADDR --master_port $MASTER_PORT \
5
- performer_module.py \
6
- --gene_list_path ../selected_genes_27k.txt \
7
- --cell_num 4000000 \
8
- --epoch 15 \
9
- --batch_size 60 \
10
- --valid_batch_size 120 \
11
- --learning_rate 1e-4 \
12
- --grad_acc 5 \
13
- --log_every 100 \
14
- --mask_ratio 0.75 \
15
- --data_path ../../../share/cell_files \
16
- --module_data_dir module_data_all/ \
17
- --gene_mapping_path ../gene2ens.csv \
18
- --gene2vec_data_path ../selected_gene2vec_27k.npy \
19
- --ckpt_dir ./performer_module_4M_all_module_hvg_600_singleton/ \
20
- --model_name performer_module_4M_all_module_hvg_600_singleton \
21
- --fusion_method multiply \
22
- -use_wandb \
23
- --api_key "$WANDB_API_KEY" \
24
- --loss_mode gene_mean \
25
- --decoder_type simple \
26
- --num_workers 12 \
27
- --prefetch_factor 4 \
28
- --extra_gene_rank_path hvg_rankings/hvg_seurat_v3_top4096_first4000000cells_gene27874.npz \
29
- --extra_gene_num 600 \
30
- --extra_gene_module_strategy singleton \
31
- --dec_embed_dim 1024
 
1
+ # Run from the outer repository root in a Linux CUDA/NCCL environment.
2
+ # Set BIOMOD_PRETRAIN_DATA to your preprocessed per-cell NPZ directory.
3
+ # The pretraining corpus is not included in this repository.
4
+ conda activate performer_module
5
+ : "${BIOMOD_PRETRAIN_DATA:?Set BIOMOD_PRETRAIN_DATA to the pretraining NPZ directory}"
6
+ torchrun --standalone --nproc_per_node="${NPROC_PER_NODE:-1}" \
7
+ BioMod/performer_module.py \
8
+ --gene_list_path model_dependencies/selected_genes_27k.txt \
9
+ --cell_num 4000000 \
10
+ --epoch 15 \
11
+ --batch_size 60 \
12
+ --valid_batch_size 120 \
13
+ --learning_rate 1e-4 \
14
+ --grad_acc 5 \
15
+ --log_every 100 \
16
+ --mask_ratio 0.75 \
17
+ --data_path "$BIOMOD_PRETRAIN_DATA" \
18
+ --module_data_dir model_dependencies/module_data_all \
19
+ --gene_mapping_path model_dependencies/gene2ens.csv \
20
+ --gene2vec_data_path model_dependencies/selected_gene2vec_27k.npy \
21
+ --ckpt_dir tutorial_runs/pretraining \
22
+ --model_name performer_module_pretraining \
23
+ --fusion_method multiply \
24
+ --loss_mode gene_mean \
25
+ --decoder_type simple \
26
+ --num_workers 12 \
27
+ --prefetch_factor 4 \
28
+ --dec_embed_dim 1024
29
+
30
+ # Adjust cell count and optimization settings to your own dataset and experiment.
31
+ # Add -use_wandb --api_key "$WANDB_API_KEY" to enable optional experiment logging.
BioMod/drug_combination/README.md CHANGED
@@ -1,143 +1,145 @@
1
- # Drug Combination Synergy Prediction with BioMod
2
-
3
- This task predicts drug-combination synergy using BioMod cell embeddings from
4
- the PerformerModule encoder and GCN-based drug molecular graph encoders. Use the
5
- [BioMod model weights and matching configuration](../../README.md#biomod-model-weights).
6
-
7
- It is adapted from [scLong/drug_combination](../../../drug_combination)
8
- (DeepDDs), replacing the scfm/geneformer cell-line representation with
9
- PerformerModule.
10
-
11
- ## Model Architecture
12
-
13
- | Component | Description |
14
- | --- | --- |
15
- | Drug branch | Three PyG GCNConv layers (78 -> 78 -> 156 -> 312), `global_max_pool`, and two fully connected layers to 128 dimensions |
16
- | Cell branch | Pretrained PerformerModule encoder, module mean pooling, and an MLP to 128 dimensions |
17
- | Fusion layer | `concat(drug1, drug2, cell) = 384 -> MLP -> 2-class prediction` |
18
-
19
- The drug1 and drug2 branches share weights, matching scLong.
20
-
21
- ## Data
22
-
23
- Place the data under `data_summary/`:
24
-
25
- ```text
26
- data_summary/
27
- |-- new_labels_0_10.csv # Training labels (drug1, drug2, cell, label)
28
- |-- new_cell_features_954.csv # Training cell features (30 cell lines x 954 genes)
29
- |-- smiles_0_10.csv # Drug SMILES
30
- `-- test_data/
31
- |-- independent_input.csv # Validation labels (668 samples)
32
- `-- independent_cell_features_954.csv # Validation cell features (1020 cell lines x 954 genes)
33
- ```
34
-
35
- The directory name `test_data/` is retained for compatibility. These files are
36
- used as the validation set for checkpoint selection.
37
-
38
- ## Dependencies
39
-
40
- - PyTorch
41
- - PyTorch Geometric (`torch_geometric`)
42
- - RDKit
43
- - networkx
44
- - scikit-learn
45
- - pandas, numpy
46
-
47
- ## Usage
48
-
49
- ```bash
50
- pip install RDKit -i https://pypi.tuna.tsinghua.edu.cn/simple
51
- ```
52
-
53
- ### Basic Run (Domain Generalization Mode, Default)
54
-
55
- ```bash
56
- python train_drug_combination_performer_module.py \
57
- --checkpoint_path /path/to/performer_module_checkpoint.pt \
58
- --hyper_params_path /path/to/hyper_params.pkl \
59
- --module_data_dir ../module_data_all \
60
- --data_dir ./data_summary
61
- ```
62
-
63
- ### Custom Paths
64
-
65
- ```bash
66
- python train_drug_combination_performer_module.py \
67
- --checkpoint_path ../mae_tost_48M_cross_attn_only_pure_memory/mae_tost_48M_cross_attn_only_pure_memory_epoch12.pth \
68
- --hyper_params_path ../mae_tost_48M_cross_attn_only_pure_memory/mae_tost_48M_cross_attn_only_pure_memory.pkl \
69
- --module_data_dir ../module_data_all \
70
- --gene_list_path ../../selected_genes_27k.txt\
71
- --gene_mapping_path ../gene_mapping.pkl \
72
- --gene2vec_path ../../selected_gene2vec_27k.npy \
73
- --symbol2ens_path ../../gene2ens.csv \
74
- --data_dir ./data_summary \
75
- --output_dir ./results \
76
- --epochs 500 \
77
- --batch_size 256 \
78
- --lr 1e-4 \
79
- --n_folds 1 \
80
- --pool_method max
81
- ```
82
-
83
- Reference note: `1e-4 500 0.6824301861702127`.
84
-
85
- ### Five-Fold Cross-Validation (Disable Domain Generalization)
86
-
87
- ```bash
88
- python train_drug_combination_performer_module.py \
89
- --checkpoint_path /path/to/checkpoint.pt \
90
- --hyper_params_path /path/to/hyper_params.pkl \
91
- --module_data_dir ../module_data_all \
92
- --no_domain_gen
93
- ```
94
-
95
- ### Force Reprocessing After Changing the Model or Data
96
-
97
- ```bash
98
- python train_drug_combination_performer_module.py \
99
- --checkpoint_path /path/to/checkpoint.pt \
100
- --hyper_params_path /path/to/hyper_params.pkl \
101
- --module_data_dir ../module_data_all \
102
- --force_preprocess
103
- ```
104
-
105
- ## Main Arguments
106
-
107
- | Argument | Default | Description |
108
- | --- | --- | --- |
109
- | `--checkpoint_path` | Required | Path to the BioMod model weights |
110
- | `--hyper_params_path` | Required | Path to the PerformerModule hyper-parameter pkl file |
111
- | `--module_data_dir` | None | Directory containing module data files |
112
- | `--data_dir` | `./data_summary` | Data directory |
113
- | `--validation_label_file` | `test_data/independent_input.csv` | Validation labels for checkpoint selection; legacy alias: `--test_label_file` |
114
- | `--validation_cell_file` | `test_data/independent_cell_features_954.csv` | Validation cell features; legacy alias: `--test_cell_file` |
115
- | `--batch_size` | 256 | Training batch size |
116
- | `--lr` | 1e-4 | Learning rate |
117
- | `--epochs` | 100 | Number of training epochs |
118
- | `--n_folds` | 5 | Number of folds |
119
- | `--domain_gen` | True | Domain generalization mode: train on the full training set and select checkpoints using a separate validation set |
120
- | `--eval_interval` | 5 | Evaluate every N epochs |
121
- | `--pool_method` | mean | PerformerModule pooling method (`mean`, `max`, or `concat`) |
122
- | `--cache_emb` | True | Cache cell embeddings to a `.npy` file |
123
- | `--force_preprocess` | False | Force regeneration of the PyG dataset |
124
-
125
- ## Outputs
126
-
127
- Results are saved under `--output_dir` (default: `./results/`):
128
-
129
- - `best_model_fold{i}.pt` - model weights selected by the highest validation AUC for each fold
130
- - `fpr_tpr_fold{i}.pkl` - validation FPR/TPR curve data for drawing ROC curves
131
- - `cell_emb_cache_{pool_method}.npy` - cached PerformerModule cell embeddings
132
-
133
- ## Evaluation Metrics
134
-
135
- Every `--eval_interval` epochs, the script evaluates AUC, PR_AUC, ACC, BACC,
136
- PREC, TPR, and KAPPA on the validation set, matching scLong's metric definitions.
137
- The checkpoint with the highest validation AUC is saved for each fold. In
138
- `--no_domain_gen` mode, each held-out fold serves as the validation set.
139
-
140
- This naming update preserves the data, splits, checkpoint-selection behavior,
141
- output filenames, metric column names, and legacy dataset cache IDs. It does not
142
- create or evaluate an independent final test set; the reported values are
143
- validation results used during model selection, not final test estimates.
 
 
 
1
+ # Drug Combination Synergy Prediction with BioMod
2
+
3
+ This task predicts drug-combination synergy using BioMod cell embeddings from
4
+ the PerformerModule encoder and GCN-based drug molecular graph encoders. Use the
5
+ [BioMod model weights and matching configuration](../../README.md#biomod-model-weights).
6
+
7
+ It is adapted from scLong's drug-combination implementation
8
+ (DeepDDs), replacing the scfm/geneformer cell-line representation with
9
+ PerformerModule.
10
+
11
+ ## Model Architecture
12
+
13
+ | Component | Description |
14
+ | --- | --- |
15
+ | Drug branch | Three PyG GCNConv layers (78 -> 78 -> 156 -> 312), `global_max_pool`, and two fully connected layers to 128 dimensions |
16
+ | Cell branch | Pretrained PerformerModule encoder, module mean pooling, and an MLP to 128 dimensions |
17
+ | Fusion layer | `concat(drug1, drug2, cell) = 384 -> MLP -> 2-class prediction` |
18
+
19
+ The drug1 and drug2 branches share weights, matching scLong.
20
+
21
+ ## Data
22
+
23
+ The bundled data are under `data/downstream/final_data/drug_combination_synergy/data_summary/`:
24
+
25
+ ```text
26
+ data_summary/
27
+ |-- new_labels_0_10.csv # Training labels (drug1, drug2, cell, label)
28
+ |-- new_cell_features_954.csv # Training cell features (30 cell lines x 954 genes)
29
+ |-- smiles_0_10.csv # Drug SMILES
30
+ `-- test_data/
31
+ |-- independent_input.csv # Validation labels (668 samples)
32
+ `-- independent_cell_features_954.csv # Validation cell features (1020 cell lines x 954 genes)
33
+ ```
34
+
35
+ The directory name `test_data/` is retained for compatibility. These files are
36
+ used as the validation set for checkpoint selection.
37
+
38
+ ## Dependencies
39
+
40
+ - PyTorch
41
+ - PyTorch Geometric (`torch_geometric`)
42
+ - RDKit
43
+ - networkx
44
+ - scikit-learn
45
+ - pandas, numpy
46
+
47
+ ## Usage
48
+
49
+ Run all commands below from the repository root. Built-in resource defaults resolve from the script location. Explicit relative paths resolve from the working directory.
50
+
51
+ ```bash
52
+ pip install RDKit -i https://pypi.tuna.tsinghua.edu.cn/simple
53
+ ```
54
+
55
+ ### Basic Run (Domain Generalization Mode, Default)
56
+
57
+ ```bash
58
+ python BioMod/drug_combination/train_drug_combination_performer_module.py \
59
+ --checkpoint_path weight/0810model/mae_tost_70M_75mask_ec2_epoch0.pth \
60
+ --hyper_params_path weight/0810model/mae_tost_70M_75mask_ec2.pkl \
61
+ --module_data_dir model_dependencies/module_data_all \
62
+ --data_dir data/downstream/final_data/drug_combination_synergy/data_summary
63
+ ```
64
+
65
+ ### Custom Paths
66
+
67
+ ```bash
68
+ python BioMod/drug_combination/train_drug_combination_performer_module.py \
69
+ --checkpoint_path weight/0810model/mae_tost_70M_75mask_ec2_epoch0.pth \
70
+ --hyper_params_path weight/0810model/mae_tost_70M_75mask_ec2.pkl \
71
+ --module_data_dir model_dependencies/module_data_all \
72
+ --gene_list_path model_dependencies/selected_genes_27k.txt\
73
+ --gene_mapping_path model_dependencies/gene2ens.csv \
74
+ --gene2vec_path model_dependencies/selected_gene2vec_27k.npy \
75
+ --symbol2ens_path model_dependencies/gene2ens.csv \
76
+ --data_dir data/downstream/final_data/drug_combination_synergy/data_summary \
77
+ --output_dir tutorial_runs/drug_combination \
78
+ --epochs 500 \
79
+ --batch_size 256 \
80
+ --lr 1e-4 \
81
+ --n_folds 1 \
82
+ --pool_method max
83
+ ```
84
+
85
+ Reference note: `1e-4 500 0.6824301861702127`.
86
+
87
+ ### Five-Fold Cross-Validation (Disable Domain Generalization)
88
+
89
+ ```bash
90
+ python BioMod/drug_combination/train_drug_combination_performer_module.py \
91
+ --checkpoint_path weight/0810model/mae_tost_70M_75mask_ec2_epoch0.pth \
92
+ --hyper_params_path weight/0810model/mae_tost_70M_75mask_ec2.pkl \
93
+ --module_data_dir model_dependencies/module_data_all \
94
+ --no_domain_gen
95
+ ```
96
+
97
+ ### Force Reprocessing After Changing the Model or Data
98
+
99
+ ```bash
100
+ python BioMod/drug_combination/train_drug_combination_performer_module.py \
101
+ --checkpoint_path weight/0810model/mae_tost_70M_75mask_ec2_epoch0.pth \
102
+ --hyper_params_path weight/0810model/mae_tost_70M_75mask_ec2.pkl \
103
+ --module_data_dir model_dependencies/module_data_all \
104
+ --force_preprocess
105
+ ```
106
+
107
+ ## Main Arguments
108
+
109
+ | Argument | Default | Description |
110
+ | --- | --- | --- |
111
+ | `--checkpoint_path` | Required | Path to the BioMod model weights |
112
+ | `--hyper_params_path` | Required | Path to the PerformerModule hyper-parameter pkl file |
113
+ | `--module_data_dir` | None | Directory containing module data files |
114
+ | `--data_dir` | `data/downstream/final_data/drug_combination_synergy/data_summary` | Data directory |
115
+ | `--validation_label_file` | `test_data/independent_input.csv` | Validation labels for checkpoint selection; legacy alias: `--test_label_file` |
116
+ | `--validation_cell_file` | `test_data/independent_cell_features_954.csv` | Validation cell features; legacy alias: `--test_cell_file` |
117
+ | `--batch_size` | 256 | Training batch size |
118
+ | `--lr` | 1e-4 | Learning rate |
119
+ | `--epochs` | 100 | Number of training epochs |
120
+ | `--n_folds` | 5 | Number of folds |
121
+ | `--domain_gen` | True | Domain generalization mode: train on the full training set and select checkpoints using a separate validation set |
122
+ | `--eval_interval` | 5 | Evaluate every N epochs |
123
+ | `--pool_method` | mean | PerformerModule pooling method (`mean`, `max`, or `concat`) |
124
+ | `--cache_emb` | True | Cache cell embeddings to a `.npy` file |
125
+ | `--force_preprocess` | False | Force regeneration of the PyG dataset |
126
+
127
+ ## Outputs
128
+
129
+ Results are saved under `--output_dir` (default: `BioMod/drug_combination/results/` relative to the repository root):
130
+
131
+ - `best_model_fold{i}.pt` - model weights selected by the highest validation AUC for each fold
132
+ - `fpr_tpr_fold{i}.pkl` - validation FPR/TPR curve data for drawing ROC curves
133
+ - `cell_emb_cache_{pool_method}.npy` - cached PerformerModule cell embeddings
134
+
135
+ ## Evaluation Metrics
136
+
137
+ Every `--eval_interval` epochs, the script evaluates AUC, PR_AUC, ACC, BACC,
138
+ PREC, TPR, and KAPPA on the validation set, matching scLong's metric definitions.
139
+ The checkpoint with the highest validation AUC is saved for each fold. In
140
+ `--no_domain_gen` mode, each held-out fold serves as the validation set.
141
+
142
+ This naming update preserves the data, splits, checkpoint-selection behavior,
143
+ output filenames, metric column names, and legacy dataset cache IDs. It does not
144
+ create or evaluate an independent final test set; the reported values are
145
+ validation results used during model selection, not final test estimates.
BioMod/drug_combination/train_drug_combination_performer_module.py CHANGED
@@ -1,900 +1,906 @@
1
- #!/usr/bin/env python
2
- # -*- coding: utf-8 -*-
3
- """
4
- Drug Combination Synergy Prediction with PerformerModule cell embeddings.
5
-
6
- Aligned with scLong/drug_combination (DeepDDs) architecture:
7
- - Drug branch: PyG GCNConv + global_max_pool (shared weights for drug1 & drug2)
8
- - Cell branch: PerformerModule pretrained encoder -> MLP
9
- - Fusion: concat(drug1, drug2, cell) -> MLP -> 2-class classification
10
-
11
- Usage:
12
- python train_drug_combination_performer_module.py \
13
- --checkpoint_path <ckpt> --hyper_params_path <pkl> \
14
- --module_data_dir ../module_data_all \
15
- --data_dir ./data_summary
16
- """
17
-
18
- import argparse
19
- import os
20
- import pickle
21
- import random
22
- import sys
23
- from pathlib import Path
24
- from typing import Dict, List, Optional, Tuple
25
-
26
- import matplotlib
27
- matplotlib.use('Agg')
28
- import matplotlib.pyplot as plt
29
- import networkx as nx
30
- import numpy as np
31
- import pandas as pd
32
- import torch
33
- import torch.nn as nn
34
- import torch.nn.functional as F
35
- from rdkit import Chem
36
- from sklearn import metrics
37
- from tqdm import tqdm
38
- from sklearn.metrics import (
39
- accuracy_score,
40
- balanced_accuracy_score,
41
- cohen_kappa_score,
42
- confusion_matrix,
43
- precision_score,
44
- recall_score,
45
- roc_auc_score,
46
- )
47
- from torch.optim.lr_scheduler import StepLR
48
-
49
- from torch_geometric import data as DATA
50
- from torch_geometric.data import InMemoryDataset
51
- from torch_geometric.loader import DataLoader
52
- from torch_geometric.nn import GCNConv, global_max_pool as gmp
53
-
54
- # ===================== Path setup =====================
55
- SCRIPT_DIR = Path(__file__).resolve().parent
56
- PERFORMER_MODULE_ROOT = SCRIPT_DIR.parent
57
- SCLONG_ROOT = PERFORMER_MODULE_ROOT.parent
58
-
59
- if str(PERFORMER_MODULE_ROOT) not in sys.path:
60
- sys.path.insert(0, str(PERFORMER_MODULE_ROOT))
61
- if str(SCLONG_ROOT) not in sys.path:
62
- sys.path.insert(0, str(SCLONG_ROOT))
63
-
64
- import importlib.util
65
-
66
- # Import PerformerModule model
67
- _models_path = os.path.join(PERFORMER_MODULE_ROOT, 'models_performer_module.py')
68
- _spec = importlib.util.spec_from_file_location("models_performer_module", _models_path)
69
- _models_mod = importlib.util.module_from_spec(_spec)
70
- _spec.loader.exec_module(_models_mod)
71
- PerformerEncoderOnly_Module = _models_mod.PerformerEncoderOnly_Module
72
- sanitize_performer_module_config = _models_mod.sanitize_performer_module_config
73
-
74
- # Import module_dataset
75
- _dataset_path = os.path.join(PERFORMER_MODULE_ROOT, 'module_dataset.py')
76
- _spec2 = importlib.util.spec_from_file_location("module_dataset", _dataset_path)
77
- _module_mod = importlib.util.module_from_spec(_spec2)
78
- _spec2.loader.exec_module(_module_mod)
79
- load_module_data = _module_mod.load_module_data
80
-
81
-
82
- # ===================== Utilities =====================
83
-
84
- def print_sys(*args, **kwargs):
85
- print(*args, **kwargs, flush=True)
86
-
87
-
88
- def reindex_tensor_universal(tensor, index_positions, dim, filler="0", device="cpu"):
89
- index_tensor = torch.tensor(index_positions, device=device)
90
- tensor_shape = list(tensor.shape)
91
- tensor_shape[dim] = len(index_positions)
92
- index_shape = [1] * len(tensor_shape)
93
- index_shape[dim] = len(index_positions)
94
- index_tensor = index_tensor.view(*index_shape).expand(*tensor_shape)
95
- if filler == "0":
96
- padder_shape = list(tensor_shape)
97
- padder_shape[dim] = 1
98
- padder = torch.zeros(padder_shape, dtype=tensor.dtype, device=device)
99
- elif filler == "mean":
100
- padder = tensor.mean(dim=dim, keepdim=True)
101
- else:
102
- raise ValueError("filler should be '0' or 'mean'")
103
- expanded_tensor = torch.cat((tensor, padder), dim=dim)
104
- reindexed = torch.gather(expanded_tensor, dim, index_tensor)
105
- return reindexed
106
-
107
-
108
- # ===================== PerformerModule Loading =====================
109
-
110
- def load_performer_module_model(
111
- checkpoint_path, hyper_params_path, module_data_dir, module_data_path,
112
- gene_list_path, gene_mapping_path, gene2vec_path, device,
113
- shuffle_module_genes=False, shuffle_seed=2021,
114
- ):
115
- print_sys(f"Loading PerformerModule model from: {checkpoint_path}")
116
-
117
- with open(hyper_params_path, 'rb') as f:
118
- model_params = pickle.load(f)
119
- model_params.pop('module_embed_chunk_padded_gene_budget', None)
120
- model_params = sanitize_performer_module_config(
121
- model_params, source=hyper_params_path, log_fn=print_sys,
122
- )
123
-
124
- if gene2vec_path:
125
- model_params['gene2vec_file'] = gene2vec_path
126
- model_params['use_gene2vec'] = model_params.get('use_gene2vec', True)
127
-
128
- if 'module_sets' not in model_params or model_params.get('module_sets') is None:
129
- if module_data_dir or module_data_path:
130
- module_sets = load_module_data(
131
- module_data_path=module_data_path,
132
- module_data_dir=module_data_dir,
133
- gene_list_path=gene_list_path,
134
- gene_mapping_path=gene_mapping_path,
135
- verbose=True,
136
- )
137
- model_params['module_sets'] = module_sets
138
- else:
139
- raise ValueError(
140
- "Model needs module_sets but neither --module_data_dir "
141
- "nor --module_data_path provided"
142
- )
143
-
144
- if shuffle_module_genes:
145
- print_sys(f"Shuffling genes across modules with seed={shuffle_seed}")
146
- all_gene_indices = []
147
- module_sizes = []
148
- for module in model_params['module_sets']:
149
- all_gene_indices.extend(module)
150
- module_sizes.append(len(module))
151
- rng = random.Random(shuffle_seed)
152
- rng.shuffle(all_gene_indices)
153
- shuffled_module_sets = []
154
- start_idx = 0
155
- for size in module_sizes:
156
- shuffled_module_sets.append(all_gene_indices[start_idx:start_idx + size])
157
- start_idx += size
158
- model_params['module_sets'] = shuffled_module_sets
159
- print_sys(f"Shuffled {len(all_gene_indices)} genes across {len(shuffled_module_sets)} modules")
160
-
161
- model = PerformerEncoderOnly_Module(**model_params).to(device)
162
-
163
- checkpoint = torch.load(checkpoint_path, map_location=device, weights_only=False)
164
- state_dict = checkpoint.get('model_state_dict', checkpoint)
165
- filtered = {
166
- k: v for k, v in state_dict.items()
167
- if 'go_conv' not in k and 'G_go' not in k.lower()
168
- }
169
- model.load_state_dict(filtered, strict=True)
170
- print_sys("Loaded pretrained PerformerModule weights.")
171
-
172
- model.eval()
173
- for p in model.parameters():
174
- p.requires_grad = False
175
-
176
- return model, len(model_params['module_sets']), model_params.get('base_dim', 1024)
177
-
178
-
179
- def build_performer_module_gene_mapping(symbol2ens_path, gene_list_path, expr_genes):
180
- gene2ens = pd.read_csv(symbol2ens_path, index_col=0)
181
- symbol2ens = gene2ens["Ensembl gene ID"].to_dict()
182
-
183
- input_genes = [symbol2ens.get(sym, sym) for sym in expr_genes]
184
- input_gene_num = len(input_genes)
185
- input_mapping = {gid: i for i, gid in enumerate(input_genes)}
186
-
187
- with open(gene_list_path, "r") as f:
188
- performer_genes = [line.rstrip("\n") for line in f.readlines()]
189
- performer_genes_pad = performer_genes + ["PAD"]
190
-
191
- performer_index_positions = [
192
- input_mapping.get(gid, input_gene_num) for gid in performer_genes_pad
193
- ]
194
- intersect = np.intersect1d(input_genes, performer_genes)
195
- print_sys(f"Target and PerformerModule genes intersect: {len(intersect)}")
196
- return input_mapping, performer_index_positions
197
-
198
-
199
- def build_cell_embeddings(
200
- cell_features_df, symbol2ens_path, gene_list_path, gene_mapping_path,
201
- gene2vec_path, checkpoint_path, hyper_params_path, module_data_dir,
202
- module_data_path, device, pool_method="mean",
203
- shuffle_module_genes=False, shuffle_seed=2021,
204
- ):
205
- model, num_modules, encoder_embed_dim = load_performer_module_model(
206
- checkpoint_path, hyper_params_path, module_data_dir, module_data_path,
207
- gene_list_path, gene_mapping_path, gene2vec_path, device,
208
- shuffle_module_genes=shuffle_module_genes, shuffle_seed=shuffle_seed,
209
- )
210
-
211
- gene_columns = list(cell_features_df.columns)
212
- _, performer_index_positions = build_performer_module_gene_mapping(
213
- symbol2ens_path, gene_list_path, gene_columns,
214
- )
215
-
216
- cell_emb_dict: Dict[str, np.ndarray] = {}
217
- first_emb_dim = -1
218
-
219
- with torch.no_grad():
220
- for cell_line_id, row in cell_features_df.iterrows():
221
- x = torch.tensor(row.values[None, :], dtype=torch.float32, device=device)
222
- x_performer = reindex_tensor_universal(
223
- x, performer_index_positions, dim=1, filler="0", device=device,
224
- )
225
- out = model(x_performer, mask_ratio=0.0, return_encodings=True)
226
-
227
- if 'module_encodings' in out:
228
- module_enc = out['module_encodings']
229
- if pool_method == "mean":
230
- cell_emb = module_enc.mean(dim=1)
231
- elif pool_method == "max":
232
- cell_emb = module_enc.max(dim=1)[0]
233
- elif pool_method == "concat":
234
- cell_emb = module_enc.reshape(1, -1)
235
- else:
236
- cell_emb = module_enc.mean(dim=1)
237
- else:
238
- raise ValueError(
239
- f"Expected 'module_encodings' in output, got: {list(out.keys())}"
240
- )
241
-
242
- cell_emb_np = cell_emb.squeeze(0).cpu().numpy()
243
- cell_emb_dict[cell_line_id] = cell_emb_np
244
- if first_emb_dim < 0:
245
- first_emb_dim = cell_emb_np.shape[0]
246
-
247
- print_sys(
248
- f"Built PerformerModule embeddings for {len(cell_emb_dict)} cell lines, "
249
- f"dim={first_emb_dim}, pool_method={pool_method}"
250
- )
251
- return cell_emb_dict, first_emb_dim
252
-
253
-
254
- # ===================== Drug Graph (scLong-aligned: edge_index format) =====================
255
-
256
- def one_of_k_encoding(x, allowable_set):
257
- if x not in allowable_set:
258
- raise Exception(f"input {x} not in allowable set {allowable_set}")
259
- return list(map(lambda s: x == s, allowable_set))
260
-
261
-
262
- def one_of_k_encoding_unk(x, allowable_set):
263
- if x not in allowable_set:
264
- x = allowable_set[-1]
265
- return list(map(lambda s: x == s, allowable_set))
266
-
267
-
268
- def atom_features(atom):
269
- return np.array(
270
- one_of_k_encoding_unk(atom.GetSymbol(), [
271
- 'C', 'N', 'O', 'S', 'F', 'Si', 'P', 'Cl', 'Br', 'Mg', 'Na', 'Ca',
272
- 'Fe', 'As', 'Al', 'I', 'B', 'V', 'K', 'Tl', 'Yb', 'Sb', 'Sn', 'Ag',
273
- 'Pd', 'Co', 'Se', 'Ti', 'Zn', 'H', 'Li', 'Ge', 'Cu', 'Au', 'Ni', 'Cd',
274
- 'In', 'Mn', 'Zr', 'Cr', 'Pt', 'Hg', 'Pb', 'Unknown',
275
- ]) +
276
- one_of_k_encoding(atom.GetDegree(), [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) +
277
- one_of_k_encoding_unk(atom.GetTotalNumHs(), [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) +
278
- one_of_k_encoding_unk(atom.GetImplicitValence(), [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) +
279
- [atom.GetIsAromatic()]
280
- )
281
-
282
-
283
- ATOM_FEAT_DIM = 78 # 44 + 11 + 11 + 11 + 1
284
-
285
-
286
- def smile_to_graph(smile):
287
- """Convert SMILES to (c_size, features, edge_index) — identical to scLong's creat_data_DC.py."""
288
- mol = Chem.MolFromSmiles(smile)
289
- if mol is None:
290
- return None
291
-
292
- c_size = mol.GetNumAtoms()
293
-
294
- features = []
295
- for atom in mol.GetAtoms():
296
- feature = atom_features(atom)
297
- features.append(feature / (sum(feature) + 1e-8))
298
-
299
- edges = []
300
- for bond in mol.GetBonds():
301
- edges.append([bond.GetBeginAtomIdx(), bond.GetEndAtomIdx()])
302
- g = nx.Graph(edges).to_directed()
303
- edge_index = []
304
- for e1, e2 in g.edges:
305
- edge_index.append([e1, e2])
306
-
307
- return c_size, features, edge_index
308
-
309
-
310
- # ===================== TestbedDataset (PyG InMemoryDataset, scLong-aligned) =====================
311
-
312
- class TestbedDataset(InMemoryDataset):
313
- """
314
- PyG InMemoryDataset for drug combination prediction.
315
- Same pattern as scLong/drug_combination/utils_test.py::TestbedDataset,
316
- but uses PerformerModule cell embeddings instead of scfm/geneformer.
317
- """
318
-
319
- def __init__(
320
- self, root='data_summary', dataset='drug1',
321
- xd=None, xt=None, y=None,
322
- cell_emb_dict=None, smile_graph=None,
323
- transform=None, pre_transform=None,
324
- ):
325
- self.dataset = dataset
326
- self._xd = xd
327
- self._xt = xt
328
- self._y = y
329
- self._cell_emb_dict = cell_emb_dict
330
- self._smile_graph = smile_graph
331
- super(TestbedDataset, self).__init__(root, transform, pre_transform)
332
-
333
- if os.path.isfile(self.processed_paths[0]):
334
- print_sys(f'Loading preprocessed data: {self.processed_paths[0]}')
335
- self.data, self.slices = torch.load(self.processed_paths[0], weights_only=False)
336
- else:
337
- print_sys(f'Preprocessing data: {self.processed_paths[0]}')
338
- self.process(xd, xt, y, cell_emb_dict, smile_graph)
339
- self.data, self.slices = torch.load(self.processed_paths[0], weights_only=False)
340
-
341
- @property
342
- def raw_file_names(self):
343
- return []
344
-
345
- @property
346
- def processed_file_names(self):
347
- return [self.dataset + '.pt']
348
-
349
- def download(self):
350
- pass
351
-
352
- def _download(self):
353
- pass
354
-
355
- def _process(self):
356
- if not os.path.exists(self.processed_dir):
357
- os.makedirs(self.processed_dir)
358
-
359
- def process(self, xd, xt, y, cell_emb_dict, smile_graph):
360
- assert len(xd) == len(xt) == len(y), "xd, xt, y must have the same length"
361
- data_list = []
362
- skipped = 0
363
-
364
- for i in range(len(xd)):
365
- smiles = xd[i]
366
- cell_name = xt[i]
367
- label = y[i]
368
-
369
- if smiles not in smile_graph:
370
- skipped += 1
371
- continue
372
- if cell_name not in cell_emb_dict:
373
- skipped += 1
374
- continue
375
-
376
- c_size, features, edge_index = smile_graph[smiles]
377
-
378
- GCNData = DATA.Data(
379
- x=torch.Tensor(features),
380
- edge_index=torch.LongTensor(edge_index).transpose(1, 0),
381
- y=torch.Tensor([label]),
382
- )
383
- cell_emb = cell_emb_dict[cell_name]
384
- GCNData.cell = torch.FloatTensor([cell_emb])
385
- GCNData.__setitem__('c_size', torch.LongTensor([c_size]))
386
- data_list.append(GCNData)
387
-
388
- if skipped > 0:
389
- print_sys(f'Skipped {skipped} samples (missing graph or cell embedding)')
390
-
391
- if self.pre_filter is not None:
392
- data_list = [d for d in data_list if self.pre_filter(d)]
393
- if self.pre_transform is not None:
394
- data_list = [self.pre_transform(d) for d in data_list]
395
-
396
- print_sys(f'Processed {len(data_list)} samples. Saving to {self.processed_paths[0]}')
397
- data, slices = self.collate(data_list)
398
- torch.save((data, slices), self.processed_paths[0])
399
-
400
-
401
- # ===================== Model: GCNNet (PyG, scLong-aligned) =====================
402
-
403
- class GCNNet(torch.nn.Module):
404
- """
405
- GCN-based drug combination model aligned with scLong/drug_combination/models/gcn.py.
406
-
407
- Drug branch: 3-layer GCNConv (78->78->156->312) + global_max_pool + 2 FC layers
408
- Drug1 and drug2 share weights (drug2 uses drug1_conv* layers, same as scLong).
409
- Cell branch: PerformerModule embedding -> MLP -> output_dim
410
- Fusion: concat(drug1, drug2, cell) -> fc1(256) -> fc2(128) -> out(2)
411
- """
412
-
413
- def __init__(
414
- self, n_output=2, num_features_xd=78, cell_emb_dim=1024,
415
- output_dim=128, dropout=0.2,
416
- ):
417
- super(GCNNet, self).__init__()
418
- self.relu = nn.ReLU()
419
- self.dropout = nn.Dropout(dropout)
420
- self.n_output = n_output
421
-
422
- # Drug graph branch (shared weights for drug1 & drug2)
423
- self.drug1_conv1 = GCNConv(num_features_xd, num_features_xd)
424
- self.drug1_conv2 = GCNConv(num_features_xd, num_features_xd * 2)
425
- self.drug1_conv3 = GCNConv(num_features_xd * 2, num_features_xd * 4)
426
- self.drug1_fc_g1 = nn.Linear(num_features_xd * 4, num_features_xd * 2)
427
- self.drug1_fc_g2 = nn.Linear(num_features_xd * 2, output_dim)
428
-
429
- # Cell branch (PerformerModule embedding -> MLP)
430
- self.reduction = nn.Sequential(
431
- nn.Linear(cell_emb_dim, 512),
432
- nn.ReLU(),
433
- nn.Linear(512, 256),
434
- nn.ReLU(),
435
- nn.Linear(256, output_dim),
436
- )
437
-
438
- # Fusion layers
439
- self.fc1 = nn.Linear(3 * output_dim, 256)
440
- self.fc2 = nn.Linear(256, 128)
441
- self.out = nn.Linear(128, self.n_output)
442
-
443
- def forward(self, data1, data2):
444
- x1, edge_index1, batch1, cell = (
445
- data1.x, data1.edge_index, data1.batch, data1.cell
446
- )
447
- x2, edge_index2, batch2 = data2.x, data2.edge_index, data2.batch
448
-
449
- # Drug1 branch
450
- x1 = self.relu(self.drug1_conv1(x1, edge_index1))
451
- x1 = self.relu(self.drug1_conv2(x1, edge_index1))
452
- x1 = self.relu(self.drug1_conv3(x1, edge_index1))
453
- x1 = gmp(x1, batch1)
454
- x1 = self.relu(self.drug1_fc_g1(x1))
455
- x1 = self.dropout(x1)
456
- x1 = self.drug1_fc_g2(x1)
457
- x1 = self.dropout(x1)
458
-
459
- # Drug2 branch (shared weights — uses drug1_conv* layers, same as scLong)
460
- x2 = self.relu(self.drug1_conv1(x2, edge_index2))
461
- x2 = self.relu(self.drug1_conv2(x2, edge_index2))
462
- x2 = self.relu(self.drug1_conv3(x2, edge_index2))
463
- x2 = gmp(x2, batch2)
464
- x2 = self.relu(self.drug1_fc_g1(x2))
465
- x2 = self.dropout(x2)
466
- x2 = self.drug1_fc_g2(x2)
467
- x2 = self.dropout(x2)
468
-
469
- # Cell branch
470
- cell_vector = F.normalize(cell, 2, 1)
471
- cell_vector = self.reduction(cell_vector)
472
-
473
- # Fusion
474
- xc = torch.cat((x1, x2, cell_vector), 1)
475
- xc = self.relu(self.fc1(xc))
476
- xc = self.dropout(xc)
477
- xc = self.relu(self.fc2(xc))
478
- xc = self.dropout(xc)
479
- out = self.out(xc)
480
- return out
481
-
482
-
483
- # ===================== Training & Evaluation =====================
484
-
485
- def train_epoch(model, device, drug1_loader, drug2_loader, optimizer, epoch):
486
- """Train one epoch. Returns average loss."""
487
- model.train()
488
- total_loss = 0.0
489
- n_batches = 0
490
- pbar = tqdm(
491
- zip(drug1_loader, drug2_loader),
492
- total=len(drug1_loader),
493
- desc=f'Epoch {epoch}',
494
- leave=True,
495
- )
496
- for data1, data2 in pbar:
497
- data1 = data1.to(device)
498
- data2 = data2.to(device)
499
- y = data1.y.view(-1, 1).long().to(device).squeeze(1)
500
- optimizer.zero_grad()
501
- output = model(data1, data2)
502
- loss = F.cross_entropy(output, y)
503
- loss.backward()
504
- optimizer.step()
505
- total_loss += loss.item()
506
- n_batches += 1
507
- pbar.set_postfix(loss=f'{loss.item():.4f}')
508
- avg_loss = total_loss / max(n_batches, 1)
509
- return avg_loss
510
-
511
-
512
- def predict(model, device, drug1_loader, drug2_loader):
513
- model.eval()
514
- total_preds = torch.Tensor()
515
- total_labels = torch.Tensor()
516
- total_prelabels = torch.Tensor()
517
- print_sys(f'Make prediction for {len(drug1_loader.dataset)} samples...')
518
-
519
- with torch.no_grad():
520
- for data in zip(drug1_loader, drug2_loader):
521
- data1 = data[0].to(device)
522
- data2 = data[1].to(device)
523
- output = model(data1, data2)
524
- ys = F.softmax(output, 1).to('cpu').data.numpy()
525
- predicted_labels = list(map(lambda x: np.argmax(x), ys))
526
- predicted_scores = list(map(lambda x: x[1], ys))
527
- total_preds = torch.cat((total_preds, torch.Tensor(predicted_scores)), 0)
528
- total_prelabels = torch.cat(
529
- (total_prelabels, torch.Tensor(predicted_labels)), 0
530
- )
531
- total_labels = torch.cat((total_labels, data1.y.view(-1, 1).cpu()), 0)
532
-
533
- return (
534
- total_labels.numpy().flatten(),
535
- total_preds.numpy().flatten(),
536
- total_prelabels.numpy().flatten(),
537
- )
538
-
539
-
540
- def evaluate_metrics(T, S, Y):
541
- """Compute all metrics matching scLong's evaluation."""
542
- try:
543
- AUC = roc_auc_score(T, S)
544
- except Exception:
545
- AUC = 0.0
546
- precision_arr, recall_arr, _ = metrics.precision_recall_curve(T, S)
547
- PR_AUC = metrics.auc(recall_arr, precision_arr)
548
- BACC = balanced_accuracy_score(T, Y)
549
- tn, fp, fn, tp = confusion_matrix(T, Y).ravel()
550
- TPR = tp / (tp + fn)
551
- PREC = precision_score(T, Y)
552
- ACC = accuracy_score(T, Y)
553
- KAPPA = cohen_kappa_score(T, Y)
554
- RECALL = recall_score(T, Y)
555
- return {
556
- 'AUC': AUC, 'PR_AUC': PR_AUC, 'ACC': ACC, 'BACC': BACC,
557
- 'PREC': PREC, 'TPR': TPR, 'KAPPA': KAPPA, 'RECALL': RECALL,
558
- }
559
-
560
-
561
- # ===================== Cell Feature Loading =====================
562
-
563
- def load_cell_features(cell_file):
564
- """Load cell features CSV. First row is header (gene_id), second is transcript_id (skipped)."""
565
- df = pd.read_csv(cell_file)
566
- df = df.iloc[1:].copy() # skip transcript_id row
567
- df = df.set_index(df.columns[0])
568
- df = df.astype(float)
569
- # Clean cell names: "X22RV1_PROSTATE" -> "22RV1"
570
- df.index = [
571
- idx.lstrip('X').split('_')[0] if '_' in idx else idx
572
- for idx in df.index
573
- ]
574
- return df
575
-
576
-
577
- # ===================== Main =====================
578
-
579
- def main():
580
- parser = argparse.ArgumentParser(
581
- description='Drug Combination Synergy Prediction with PerformerModule'
582
- )
583
-
584
- # PerformerModule model args
585
- parser.add_argument('--checkpoint_path', type=str, required=True,
586
- help='Path to PerformerModule pretrained checkpoint')
587
- parser.add_argument('--hyper_params_path', type=str, required=True,
588
- help='Path to PerformerModule hyper parameters pkl')
589
- parser.add_argument('--module_data_dir', type=str, default=None,
590
- help='Directory containing module data files')
591
- parser.add_argument('--module_data_path', type=str, default=None,
592
- help='Path to single module data file')
593
- parser.add_argument('--gene_list_path', type=str,
594
- default=os.path.join(PERFORMER_MODULE_ROOT, 'gene_list_27k.txt'))
595
- parser.add_argument('--gene_mapping_path', type=str,
596
- default=os.path.join(PERFORMER_MODULE_ROOT, 'gene_mapping.pkl'))
597
- parser.add_argument('--gene2vec_path', type=str,
598
- default=os.path.join(PERFORMER_MODULE_ROOT, 'gene2vec_27k.npy'))
599
- parser.add_argument('--symbol2ens_path', type=str,
600
- default=os.path.join(SCLONG_ROOT, 'gene2ensemble_ID.csv'))
601
- parser.add_argument('--pool_method', type=str, default='mean',
602
- choices=['mean', 'max', 'concat'])
603
- parser.add_argument('--shuffle_module_genes', action='store_true',
604
- help='Disrupt the gene allocation within the module (used for comparative experiments to verify the effectiveness of module division)')
605
- parser.add_argument('--shuffle_seed', type=int, default=2021,
606
- help='The random seed used when scrambling the module gene (needs to be consistent with the value used during pre-training)')
607
-
608
- # Data args
609
- parser.add_argument('--data_dir', type=str,
610
- default=os.path.join(SCRIPT_DIR, 'data_summary'))
611
- parser.add_argument('--label_file', type=str, default='new_labels_0_10.csv')
612
- parser.add_argument('--cell_file', type=str, default='new_cell_features_954.csv')
613
- parser.add_argument('--validation_label_file', '--test_label_file',
614
- dest='validation_label_file', type=str,
615
- default='test_data/independent_input.csv',
616
- help='Validation labels used for checkpoint selection; --test_label_file is a legacy alias')
617
- parser.add_argument('--validation_cell_file', '--test_cell_file',
618
- dest='validation_cell_file', type=str,
619
- default='test_data/independent_cell_features_954.csv',
620
- help='Validation cell features; --test_cell_file is a legacy alias')
621
-
622
- # Training args
623
- parser.add_argument('--batch_size', type=int, default=256)
624
- parser.add_argument('--lr', type=float, default=1e-4)
625
- parser.add_argument('--epochs', type=int, default=100)
626
- parser.add_argument('--n_folds', type=int, default=5)
627
- parser.add_argument('--domain_gen', action='store_true', default=True,
628
- help='Domain generalization: train full set, validate on a separate set for checkpoint selection')
629
- parser.add_argument('--no_domain_gen', dest='domain_gen', action='store_false')
630
- parser.add_argument('--eval_interval', type=int, default=5)
631
- parser.add_argument('--output_dir', type=str,
632
- default=os.path.join(SCRIPT_DIR, 'results'))
633
- parser.add_argument('--seed', type=int, default=42)
634
- parser.add_argument('--cache_emb', action='store_true', default=True,
635
- help='Cache PerformerModule cell embeddings to .npy')
636
- parser.add_argument('--no_cache_emb', dest='cache_emb', action='store_false')
637
- parser.add_argument('--force_preprocess', action='store_true', default=False,
638
- help='Force re-creation of PyG .pt datasets')
639
-
640
- args = parser.parse_args()
641
-
642
- # Device
643
- device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
644
- print_sys(f'Using device: {device}')
645
-
646
- # Seed
647
- random.seed(args.seed)
648
- np.random.seed(args.seed)
649
- torch.manual_seed(args.seed)
650
-
651
- os.makedirs(args.output_dir, exist_ok=True)
652
-
653
- # -------- Step 1: Build PerformerModule cell embeddings --------
654
- train_cell_file = os.path.join(args.data_dir, args.cell_file)
655
- train_cell_df = load_cell_features(train_cell_file)
656
- print_sys(f'Loaded train cell features: {train_cell_df.shape}')
657
-
658
- validation_cell_file = os.path.join(args.data_dir, args.validation_cell_file)
659
- validation_cell_df = load_cell_features(validation_cell_file)
660
- print_sys(f'Loaded validation cell features: {validation_cell_df.shape}')
661
-
662
- # Combine all cell lines and remove duplicates
663
- all_cell_df = pd.concat([train_cell_df, validation_cell_df])
664
- all_cell_df = all_cell_df[~all_cell_df.index.duplicated(keep='first')]
665
- print_sys(f'Total unique cell lines: {len(all_cell_df)}')
666
-
667
- # Cache embeddings
668
- cache_path = os.path.join(args.output_dir, f'cell_emb_cache_{args.pool_method}.npy')
669
- cache_dim_path = os.path.join(
670
- args.output_dir, f'cell_emb_dim_{args.pool_method}.txt'
671
- )
672
-
673
- if args.cache_emb and os.path.exists(cache_path) and not args.force_preprocess:
674
- print_sys(f'Loading cached cell embeddings from {cache_path}')
675
- cell_emb_dict = np.load(cache_path, allow_pickle=True).item()
676
- with open(cache_dim_path) as f:
677
- cell_emb_dim = int(f.read().strip())
678
- else:
679
- cell_emb_dict, cell_emb_dim = build_cell_embeddings(
680
- all_cell_df, args.symbol2ens_path, args.gene_list_path,
681
- args.gene_mapping_path, args.gene2vec_path, args.checkpoint_path,
682
- args.hyper_params_path, args.module_data_dir, args.module_data_path,
683
- device, args.pool_method,
684
- shuffle_module_genes=args.shuffle_module_genes,
685
- shuffle_seed=args.shuffle_seed,
686
- )
687
- if args.cache_emb:
688
- np.save(cache_path, cell_emb_dict)
689
- with open(cache_dim_path, 'w') as f:
690
- f.write(str(cell_emb_dim))
691
- print_sys(f'Cached cell embeddings to {cache_path}')
692
-
693
- # -------- Step 2: Build SMILES graphs --------
694
- train_label_path = os.path.join(args.data_dir, args.label_file)
695
- train_labels = pd.read_csv(train_label_path)
696
- print_sys(f'Train labels: {len(train_labels)} samples')
697
-
698
- validation_label_path = os.path.join(args.data_dir, args.validation_label_file)
699
- validation_labels = pd.read_csv(validation_label_path)
700
- print_sys(f'Validation labels: {len(validation_labels)} samples')
701
-
702
- all_smiles = set(
703
- train_labels['drug1'].tolist() + train_labels['drug2'].tolist()
704
- + validation_labels['drug1'].tolist() + validation_labels['drug2'].tolist()
705
- )
706
- smile_graph = {}
707
- failed = []
708
- for smi in all_smiles:
709
- result = smile_to_graph(smi)
710
- if result is not None:
711
- smile_graph[smi] = result
712
- else:
713
- failed.append(smi)
714
- if failed:
715
- print_sys(f'Warning: {len(failed)} SMILES failed to parse')
716
- print_sys(f'Built graphs for {len(smile_graph)} unique SMILES')
717
-
718
- # -------- Step 3: Create PyG datasets --------
719
- # Delete cached .pt files if force_preprocess
720
- if args.force_preprocess:
721
- processed_dir = os.path.join(args.data_dir, 'processed')
722
- if os.path.exists(processed_dir):
723
- import shutil
724
- shutil.rmtree(processed_dir)
725
- print_sys(f'Removed cached processed data: {processed_dir}')
726
-
727
- # Train datasets (drug1 and drug2 separate, scLong pattern)
728
- drug1_train = train_labels['drug1'].tolist()
729
- drug2_train = train_labels['drug2'].tolist()
730
- cells_train = train_labels['cell'].tolist()
731
- labels_train = train_labels['label'].tolist()
732
-
733
- train_d1_dataset = TestbedDataset(
734
- root=args.data_dir, dataset='train_drug1_performer_module',
735
- xd=drug1_train, xt=cells_train, y=labels_train,
736
- cell_emb_dict=cell_emb_dict, smile_graph=smile_graph,
737
- )
738
- train_d2_dataset = TestbedDataset(
739
- root=args.data_dir, dataset='train_drug2_performer_module',
740
- xd=drug2_train, xt=cells_train, y=labels_train,
741
- cell_emb_dict=cell_emb_dict, smile_graph=smile_graph,
742
- )
743
-
744
- # Validation datasets (separate set used for checkpoint selection)
745
- # Preserve legacy test_* cache IDs to reuse existing preprocessed datasets.
746
- drug1_validation = validation_labels['drug1'].tolist()
747
- drug2_validation = validation_labels['drug2'].tolist()
748
- cells_validation = validation_labels['cell'].tolist()
749
- labels_validation = validation_labels['label'].tolist()
750
-
751
- validation_d1_dataset = TestbedDataset(
752
- root=args.data_dir, dataset='test_drug1_performer_module',
753
- xd=drug1_validation, xt=cells_validation, y=labels_validation,
754
- cell_emb_dict=cell_emb_dict, smile_graph=smile_graph,
755
- )
756
- validation_d2_dataset = TestbedDataset(
757
- root=args.data_dir, dataset='test_drug2_performer_module',
758
- xd=drug2_validation, xt=cells_validation, y=labels_validation,
759
- cell_emb_dict=cell_emb_dict, smile_graph=smile_graph,
760
- )
761
-
762
- # -------- Step 4: Training (5-fold, scLong-aligned) --------
763
- lenth = len(train_d1_dataset)
764
- pot = lenth // args.n_folds
765
- random_num = random.sample(range(lenth), lenth)
766
- all_fold_aucs = []
767
-
768
- for fold_i in range(args.n_folds):
769
- print_sys(f'\n{"=" * 60}')
770
- print_sys(f'Fold {fold_i + 1}/{args.n_folds}')
771
- print_sys(f'{"=" * 60}')
772
-
773
- if args.domain_gen:
774
- # Domain generalization: train on full training set, select on validation set
775
- d1_train_data = train_d1_dataset
776
- d1_validation_data = validation_d1_dataset
777
- d2_train_data = train_d2_dataset
778
- d2_validation_data = validation_d2_dataset
779
- else:
780
- # Standard k-fold cross-validation
781
- validation_num = random_num[pot * fold_i : pot * (fold_i + 1)]
782
- train_num = (
783
- random_num[: pot * fold_i] + random_num[pot * (fold_i + 1) :]
784
- )
785
- d1_train_data = train_d1_dataset[train_num]
786
- d1_validation_data = train_d1_dataset[validation_num]
787
- d2_train_data = train_d2_dataset[train_num]
788
- d2_validation_data = train_d2_dataset[validation_num]
789
-
790
- d1_loader_train = DataLoader(
791
- d1_train_data, batch_size=args.batch_size, shuffle=None, pin_memory=True,
792
- )
793
- d1_loader_validation = DataLoader(
794
- d1_validation_data, batch_size=args.batch_size, shuffle=None, pin_memory=True,
795
- )
796
- d2_loader_train = DataLoader(
797
- d2_train_data, batch_size=args.batch_size, shuffle=None, pin_memory=True,
798
- )
799
- d2_loader_validation = DataLoader(
800
- d2_validation_data, batch_size=args.batch_size, shuffle=None, pin_memory=True,
801
- )
802
-
803
- model = GCNNet(cell_emb_dim=cell_emb_dim).to(device)
804
-
805
- if fold_i == 0:
806
- for name, param in model.named_parameters():
807
- print_sys(f" Parameter: {name}, Shape: {param.size()}")
808
- total_params = sum(p.numel() for p in model.parameters())
809
- print_sys(f'Total model parameters: {total_params}')
810
-
811
- optimizer = torch.optim.Adam(model.parameters(), lr=args.lr)
812
- scheduler = StepLR(optimizer, step_size=1000, gamma=0.9)
813
-
814
- best_auc = 0
815
- loss_history = []
816
- metric_history = []
817
-
818
- for epoch in range(1, args.epochs + 1):
819
- avg_loss = train_epoch(
820
- model, device, d1_loader_train, d2_loader_train,
821
- optimizer, epoch,
822
- )
823
- scheduler.step()
824
- loss_history.append(avg_loss)
825
-
826
- if epoch % args.eval_interval == 0:
827
- T, S, Y = predict(model, device, d1_loader_validation, d2_loader_validation)
828
- m = evaluate_metrics(T, S, Y)
829
- metric_history.append({'epoch': epoch, **m})
830
-
831
- if m['AUC'] > best_auc:
832
- best_auc = m['AUC']
833
- best_fpr, best_tpr = metrics.roc_curve(T, S)[:2]
834
- save_path = os.path.join(
835
- args.output_dir, f'best_model_fold{fold_i}.pt'
836
- )
837
- torch.save(model.state_dict(), save_path)
838
- print_sys(f'** New best validation AUROC: {best_auc:.4f} (saved)')
839
-
840
- print_sys(
841
- f'Epoch {epoch} | Validation | '
842
- f'AUROC: {m["AUC"]:.4f} PR_AUC: {m["PR_AUC"]:.4f} '
843
- f'ACC: {m["ACC"]:.4f} BACC: {m["BACC"]:.4f} '
844
- f'PREC: {m["PREC"]:.4f} TPR: {m["TPR"]:.4f} '
845
- f'KAPPA: {m["KAPPA"]:.4f}'
846
- )
847
-
848
- # Save loss curve
849
- loss_csv_path = os.path.join(args.output_dir, f'loss_fold{fold_i}.csv')
850
- loss_df = pd.DataFrame({'epoch': range(1, len(loss_history) + 1), 'avg_loss': loss_history})
851
- loss_df.to_csv(loss_csv_path, index=False)
852
-
853
- # Plot loss curve
854
- fig, ax = plt.subplots(figsize=(8, 5))
855
- ax.plot(loss_df['epoch'], loss_df['avg_loss'], linewidth=1.5)
856
- ax.set_xlabel('Epoch')
857
- ax.set_ylabel('Average Loss')
858
- ax.set_title(f'Training Loss - Fold {fold_i + 1}')
859
- ax.grid(True, alpha=0.3)
860
- fig.tight_layout()
861
- fig.savefig(os.path.join(args.output_dir, f'loss_curve_fold{fold_i}.png'), dpi=150)
862
- plt.close(fig)
863
-
864
- # Save validation metrics history (retain the existing CSV schema)
865
- if metric_history:
866
- metric_df = pd.DataFrame(metric_history)
867
- metric_df.to_csv(os.path.join(args.output_dir, f'metrics_fold{fold_i}.csv'), index=False)
868
-
869
- # Plot validation ROC curve (using the selected model's predictions)
870
- fig, ax = plt.subplots(figsize=(6, 6))
871
- ax.plot(best_fpr, best_tpr, linewidth=2, label=f'AUROC = {best_auc:.4f}')
872
- ax.plot([0, 1], [0, 1], 'k--', linewidth=1, alpha=0.5)
873
- ax.set_xlim([0, 1])
874
- ax.set_ylim([0, 1.02])
875
- ax.set_xlabel('False Positive Rate')
876
- ax.set_ylabel('True Positive Rate')
877
- ax.set_title(f'Validation ROC Curve - Fold {fold_i + 1}')
878
- ax.legend(loc='lower right')
879
- ax.grid(True, alpha=0.3)
880
- fig.tight_layout()
881
- fig.savefig(os.path.join(args.output_dir, f'roc_curve_fold{fold_i}.png'), dpi=150)
882
- plt.close(fig)
883
-
884
- # Save fpr/tpr data
885
- fpr_tpr_path = os.path.join(args.output_dir, f'fpr_tpr_fold{fold_i}.pkl')
886
- with open(fpr_tpr_path, 'wb') as fp:
887
- pickle.dump((best_fpr, best_tpr), fp)
888
-
889
- print_sys(f'Fold {fold_i + 1} best validation AUROC: {best_auc:.4f}')
890
- all_fold_aucs.append(best_auc)
891
-
892
- # Summary
893
- print_sys(f'\n{"=" * 60}')
894
- print_sys(f'All folds validation AUROC: {all_fold_aucs}')
895
- print_sys(f'Mean validation AUROC: {np.mean(all_fold_aucs):.4f} +/- {np.std(all_fold_aucs):.4f}')
896
- print_sys('Training complete.')
897
-
898
-
899
- if __name__ == '__main__':
900
- main()
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python
2
+ # -*- coding: utf-8 -*-
3
+ """
4
+ Drug Combination Synergy Prediction with PerformerModule cell embeddings.
5
+
6
+ Aligned with scLong/drug_combination (DeepDDs) architecture:
7
+ - Drug branch: PyG GCNConv + global_max_pool (shared weights for drug1 & drug2)
8
+ - Cell branch: PerformerModule pretrained encoder -> MLP
9
+ - Fusion: concat(drug1, drug2, cell) -> MLP -> 2-class classification
10
+
11
+ Usage:
12
+ # Run from the repository root.
13
+ python BioMod/drug_combination/train_drug_combination_performer_module.py \
14
+ --checkpoint_path <ckpt> --hyper_params_path <pkl> \
15
+ --module_data_dir model_dependencies/module_data_all \
16
+ --data_dir data/downstream/final_data/drug_combination_synergy/data_summary
17
+ """
18
+
19
+ import argparse
20
+ import os
21
+ import pickle
22
+ import random
23
+ import sys
24
+ from pathlib import Path
25
+ from typing import Dict, List, Optional, Tuple
26
+
27
+ import matplotlib
28
+ matplotlib.use('Agg')
29
+ import matplotlib.pyplot as plt
30
+ import networkx as nx
31
+ import numpy as np
32
+ import pandas as pd
33
+ import torch
34
+ import torch.nn as nn
35
+ import torch.nn.functional as F
36
+ from rdkit import Chem
37
+ from sklearn import metrics
38
+ from tqdm import tqdm
39
+ from sklearn.metrics import (
40
+ accuracy_score,
41
+ balanced_accuracy_score,
42
+ cohen_kappa_score,
43
+ confusion_matrix,
44
+ precision_score,
45
+ recall_score,
46
+ roc_auc_score,
47
+ )
48
+ from torch.optim.lr_scheduler import StepLR
49
+
50
+ from torch_geometric import data as DATA
51
+ from torch_geometric.data import InMemoryDataset
52
+ from torch_geometric.loader import DataLoader
53
+ from torch_geometric.nn import GCNConv, global_max_pool as gmp
54
+
55
+ # ===================== Path setup =====================
56
+ SCRIPT_DIR = Path(__file__).resolve().parent
57
+ PERFORMER_MODULE_ROOT = SCRIPT_DIR.parent
58
+ SCLONG_ROOT = PERFORMER_MODULE_ROOT.parent
59
+
60
+ REPO_ROOT = Path(__file__).resolve().parents[2]
61
+ ASSET_DIR = REPO_ROOT / "model_dependencies"
62
+ DOWNSTREAM_DATA_DIR = REPO_ROOT / "data" / "downstream" / "final_data"
63
+ WEIGHTS_DIR = REPO_ROOT / "weight" / "0810model"
64
+
65
+ if str(PERFORMER_MODULE_ROOT) not in sys.path:
66
+ sys.path.insert(0, str(PERFORMER_MODULE_ROOT))
67
+ if str(SCLONG_ROOT) not in sys.path:
68
+ sys.path.insert(0, str(SCLONG_ROOT))
69
+
70
+ import importlib.util
71
+
72
+ # Import PerformerModule model
73
+ _models_path = os.path.join(PERFORMER_MODULE_ROOT, 'models_performer_module.py')
74
+ _spec = importlib.util.spec_from_file_location("models_performer_module", _models_path)
75
+ _models_mod = importlib.util.module_from_spec(_spec)
76
+ _spec.loader.exec_module(_models_mod)
77
+ PerformerEncoderOnly_Module = _models_mod.PerformerEncoderOnly_Module
78
+ sanitize_performer_module_config = _models_mod.sanitize_performer_module_config
79
+
80
+ # Import module_dataset
81
+ _dataset_path = os.path.join(PERFORMER_MODULE_ROOT, 'module_dataset.py')
82
+ _spec2 = importlib.util.spec_from_file_location("module_dataset", _dataset_path)
83
+ _module_mod = importlib.util.module_from_spec(_spec2)
84
+ _spec2.loader.exec_module(_module_mod)
85
+ load_module_data = _module_mod.load_module_data
86
+
87
+
88
+ # ===================== Utilities =====================
89
+
90
+ def print_sys(*args, **kwargs):
91
+ print(*args, **kwargs, flush=True)
92
+
93
+
94
+ def reindex_tensor_universal(tensor, index_positions, dim, filler="0", device="cpu"):
95
+ index_tensor = torch.tensor(index_positions, device=device)
96
+ tensor_shape = list(tensor.shape)
97
+ tensor_shape[dim] = len(index_positions)
98
+ index_shape = [1] * len(tensor_shape)
99
+ index_shape[dim] = len(index_positions)
100
+ index_tensor = index_tensor.view(*index_shape).expand(*tensor_shape)
101
+ if filler == "0":
102
+ padder_shape = list(tensor_shape)
103
+ padder_shape[dim] = 1
104
+ padder = torch.zeros(padder_shape, dtype=tensor.dtype, device=device)
105
+ elif filler == "mean":
106
+ padder = tensor.mean(dim=dim, keepdim=True)
107
+ else:
108
+ raise ValueError("filler should be '0' or 'mean'")
109
+ expanded_tensor = torch.cat((tensor, padder), dim=dim)
110
+ reindexed = torch.gather(expanded_tensor, dim, index_tensor)
111
+ return reindexed
112
+
113
+
114
+ # ===================== PerformerModule Loading =====================
115
+
116
+ def load_performer_module_model(
117
+ checkpoint_path, hyper_params_path, module_data_dir, module_data_path,
118
+ gene_list_path, gene_mapping_path, gene2vec_path, device,
119
+ shuffle_module_genes=False, shuffle_seed=2021,
120
+ ):
121
+ print_sys(f"Loading PerformerModule model from: {checkpoint_path}")
122
+
123
+ with open(hyper_params_path, 'rb') as f:
124
+ model_params = pickle.load(f)
125
+ model_params.pop('module_embed_chunk_padded_gene_budget', None)
126
+ model_params = sanitize_performer_module_config(
127
+ model_params, source=hyper_params_path, log_fn=print_sys,
128
+ )
129
+
130
+ if gene2vec_path:
131
+ model_params['gene2vec_file'] = gene2vec_path
132
+ model_params['use_gene2vec'] = model_params.get('use_gene2vec', True)
133
+
134
+ if 'module_sets' not in model_params or model_params.get('module_sets') is None:
135
+ if module_data_dir or module_data_path:
136
+ module_sets = load_module_data(
137
+ module_data_path=module_data_path,
138
+ module_data_dir=module_data_dir,
139
+ gene_list_path=gene_list_path,
140
+ gene_mapping_path=gene_mapping_path,
141
+ verbose=True,
142
+ )
143
+ model_params['module_sets'] = module_sets
144
+ else:
145
+ raise ValueError(
146
+ "Model needs module_sets but neither --module_data_dir "
147
+ "nor --module_data_path provided"
148
+ )
149
+
150
+ if shuffle_module_genes:
151
+ print_sys(f"Shuffling genes across modules with seed={shuffle_seed}")
152
+ all_gene_indices = []
153
+ module_sizes = []
154
+ for module in model_params['module_sets']:
155
+ all_gene_indices.extend(module)
156
+ module_sizes.append(len(module))
157
+ rng = random.Random(shuffle_seed)
158
+ rng.shuffle(all_gene_indices)
159
+ shuffled_module_sets = []
160
+ start_idx = 0
161
+ for size in module_sizes:
162
+ shuffled_module_sets.append(all_gene_indices[start_idx:start_idx + size])
163
+ start_idx += size
164
+ model_params['module_sets'] = shuffled_module_sets
165
+ print_sys(f"Shuffled {len(all_gene_indices)} genes across {len(shuffled_module_sets)} modules")
166
+
167
+ model = PerformerEncoderOnly_Module(**model_params).to(device)
168
+
169
+ checkpoint = torch.load(checkpoint_path, map_location=device, weights_only=False)
170
+ state_dict = checkpoint.get('model_state_dict', checkpoint)
171
+ filtered = {
172
+ k: v for k, v in state_dict.items()
173
+ if 'go_conv' not in k and 'G_go' not in k.lower()
174
+ }
175
+ model.load_state_dict(filtered, strict=True)
176
+ print_sys("Loaded pretrained PerformerModule weights.")
177
+
178
+ model.eval()
179
+ for p in model.parameters():
180
+ p.requires_grad = False
181
+
182
+ return model, len(model_params['module_sets']), model_params.get('base_dim', 1024)
183
+
184
+
185
+ def build_performer_module_gene_mapping(symbol2ens_path, gene_list_path, expr_genes):
186
+ gene2ens = pd.read_csv(symbol2ens_path, index_col=0)
187
+ symbol2ens = gene2ens["Ensembl gene ID"].to_dict()
188
+
189
+ input_genes = [symbol2ens.get(sym, sym) for sym in expr_genes]
190
+ input_gene_num = len(input_genes)
191
+ input_mapping = {gid: i for i, gid in enumerate(input_genes)}
192
+
193
+ with open(gene_list_path, "r") as f:
194
+ performer_genes = [line.rstrip("\n") for line in f.readlines()]
195
+ performer_genes_pad = performer_genes + ["PAD"]
196
+
197
+ performer_index_positions = [
198
+ input_mapping.get(gid, input_gene_num) for gid in performer_genes_pad
199
+ ]
200
+ intersect = np.intersect1d(input_genes, performer_genes)
201
+ print_sys(f"Target and PerformerModule genes intersect: {len(intersect)}")
202
+ return input_mapping, performer_index_positions
203
+
204
+
205
+ def build_cell_embeddings(
206
+ cell_features_df, symbol2ens_path, gene_list_path, gene_mapping_path,
207
+ gene2vec_path, checkpoint_path, hyper_params_path, module_data_dir,
208
+ module_data_path, device, pool_method="mean",
209
+ shuffle_module_genes=False, shuffle_seed=2021,
210
+ ):
211
+ model, num_modules, encoder_embed_dim = load_performer_module_model(
212
+ checkpoint_path, hyper_params_path, module_data_dir, module_data_path,
213
+ gene_list_path, gene_mapping_path, gene2vec_path, device,
214
+ shuffle_module_genes=shuffle_module_genes, shuffle_seed=shuffle_seed,
215
+ )
216
+
217
+ gene_columns = list(cell_features_df.columns)
218
+ _, performer_index_positions = build_performer_module_gene_mapping(
219
+ symbol2ens_path, gene_list_path, gene_columns,
220
+ )
221
+
222
+ cell_emb_dict: Dict[str, np.ndarray] = {}
223
+ first_emb_dim = -1
224
+
225
+ with torch.no_grad():
226
+ for cell_line_id, row in cell_features_df.iterrows():
227
+ x = torch.tensor(row.values[None, :], dtype=torch.float32, device=device)
228
+ x_performer = reindex_tensor_universal(
229
+ x, performer_index_positions, dim=1, filler="0", device=device,
230
+ )
231
+ out = model(x_performer, mask_ratio=0.0, return_encodings=True)
232
+
233
+ if 'module_encodings' in out:
234
+ module_enc = out['module_encodings']
235
+ if pool_method == "mean":
236
+ cell_emb = module_enc.mean(dim=1)
237
+ elif pool_method == "max":
238
+ cell_emb = module_enc.max(dim=1)[0]
239
+ elif pool_method == "concat":
240
+ cell_emb = module_enc.reshape(1, -1)
241
+ else:
242
+ cell_emb = module_enc.mean(dim=1)
243
+ else:
244
+ raise ValueError(
245
+ f"Expected 'module_encodings' in output, got: {list(out.keys())}"
246
+ )
247
+
248
+ cell_emb_np = cell_emb.squeeze(0).cpu().numpy()
249
+ cell_emb_dict[cell_line_id] = cell_emb_np
250
+ if first_emb_dim < 0:
251
+ first_emb_dim = cell_emb_np.shape[0]
252
+
253
+ print_sys(
254
+ f"Built PerformerModule embeddings for {len(cell_emb_dict)} cell lines, "
255
+ f"dim={first_emb_dim}, pool_method={pool_method}"
256
+ )
257
+ return cell_emb_dict, first_emb_dim
258
+
259
+
260
+ # ===================== Drug Graph (scLong-aligned: edge_index format) =====================
261
+
262
+ def one_of_k_encoding(x, allowable_set):
263
+ if x not in allowable_set:
264
+ raise Exception(f"input {x} not in allowable set {allowable_set}")
265
+ return list(map(lambda s: x == s, allowable_set))
266
+
267
+
268
+ def one_of_k_encoding_unk(x, allowable_set):
269
+ if x not in allowable_set:
270
+ x = allowable_set[-1]
271
+ return list(map(lambda s: x == s, allowable_set))
272
+
273
+
274
+ def atom_features(atom):
275
+ return np.array(
276
+ one_of_k_encoding_unk(atom.GetSymbol(), [
277
+ 'C', 'N', 'O', 'S', 'F', 'Si', 'P', 'Cl', 'Br', 'Mg', 'Na', 'Ca',
278
+ 'Fe', 'As', 'Al', 'I', 'B', 'V', 'K', 'Tl', 'Yb', 'Sb', 'Sn', 'Ag',
279
+ 'Pd', 'Co', 'Se', 'Ti', 'Zn', 'H', 'Li', 'Ge', 'Cu', 'Au', 'Ni', 'Cd',
280
+ 'In', 'Mn', 'Zr', 'Cr', 'Pt', 'Hg', 'Pb', 'Unknown',
281
+ ]) +
282
+ one_of_k_encoding(atom.GetDegree(), [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) +
283
+ one_of_k_encoding_unk(atom.GetTotalNumHs(), [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) +
284
+ one_of_k_encoding_unk(atom.GetImplicitValence(), [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) +
285
+ [atom.GetIsAromatic()]
286
+ )
287
+
288
+
289
+ ATOM_FEAT_DIM = 78 # 44 + 11 + 11 + 11 + 1
290
+
291
+
292
+ def smile_to_graph(smile):
293
+ """Convert SMILES to (c_size, features, edge_index) — identical to scLong's creat_data_DC.py."""
294
+ mol = Chem.MolFromSmiles(smile)
295
+ if mol is None:
296
+ return None
297
+
298
+ c_size = mol.GetNumAtoms()
299
+
300
+ features = []
301
+ for atom in mol.GetAtoms():
302
+ feature = atom_features(atom)
303
+ features.append(feature / (sum(feature) + 1e-8))
304
+
305
+ edges = []
306
+ for bond in mol.GetBonds():
307
+ edges.append([bond.GetBeginAtomIdx(), bond.GetEndAtomIdx()])
308
+ g = nx.Graph(edges).to_directed()
309
+ edge_index = []
310
+ for e1, e2 in g.edges:
311
+ edge_index.append([e1, e2])
312
+
313
+ return c_size, features, edge_index
314
+
315
+
316
+ # ===================== TestbedDataset (PyG InMemoryDataset, scLong-aligned) =====================
317
+
318
+ class TestbedDataset(InMemoryDataset):
319
+ """
320
+ PyG InMemoryDataset for drug combination prediction.
321
+ Same pattern as scLong/drug_combination/utils_test.py::TestbedDataset,
322
+ but uses PerformerModule cell embeddings instead of scfm/geneformer.
323
+ """
324
+
325
+ def __init__(
326
+ self, root=str(DOWNSTREAM_DATA_DIR / 'drug_combination_synergy' / 'data_summary'), dataset='drug1',
327
+ xd=None, xt=None, y=None,
328
+ cell_emb_dict=None, smile_graph=None,
329
+ transform=None, pre_transform=None,
330
+ ):
331
+ self.dataset = dataset
332
+ self._xd = xd
333
+ self._xt = xt
334
+ self._y = y
335
+ self._cell_emb_dict = cell_emb_dict
336
+ self._smile_graph = smile_graph
337
+ super(TestbedDataset, self).__init__(root, transform, pre_transform)
338
+
339
+ if os.path.isfile(self.processed_paths[0]):
340
+ print_sys(f'Loading preprocessed data: {self.processed_paths[0]}')
341
+ self.data, self.slices = torch.load(self.processed_paths[0], weights_only=False)
342
+ else:
343
+ print_sys(f'Preprocessing data: {self.processed_paths[0]}')
344
+ self.process(xd, xt, y, cell_emb_dict, smile_graph)
345
+ self.data, self.slices = torch.load(self.processed_paths[0], weights_only=False)
346
+
347
+ @property
348
+ def raw_file_names(self):
349
+ return []
350
+
351
+ @property
352
+ def processed_file_names(self):
353
+ return [self.dataset + '.pt']
354
+
355
+ def download(self):
356
+ pass
357
+
358
+ def _download(self):
359
+ pass
360
+
361
+ def _process(self):
362
+ if not os.path.exists(self.processed_dir):
363
+ os.makedirs(self.processed_dir)
364
+
365
+ def process(self, xd, xt, y, cell_emb_dict, smile_graph):
366
+ assert len(xd) == len(xt) == len(y), "xd, xt, y must have the same length"
367
+ data_list = []
368
+ skipped = 0
369
+
370
+ for i in range(len(xd)):
371
+ smiles = xd[i]
372
+ cell_name = xt[i]
373
+ label = y[i]
374
+
375
+ if smiles not in smile_graph:
376
+ skipped += 1
377
+ continue
378
+ if cell_name not in cell_emb_dict:
379
+ skipped += 1
380
+ continue
381
+
382
+ c_size, features, edge_index = smile_graph[smiles]
383
+
384
+ GCNData = DATA.Data(
385
+ x=torch.Tensor(features),
386
+ edge_index=torch.LongTensor(edge_index).transpose(1, 0),
387
+ y=torch.Tensor([label]),
388
+ )
389
+ cell_emb = cell_emb_dict[cell_name]
390
+ GCNData.cell = torch.FloatTensor([cell_emb])
391
+ GCNData.__setitem__('c_size', torch.LongTensor([c_size]))
392
+ data_list.append(GCNData)
393
+
394
+ if skipped > 0:
395
+ print_sys(f'Skipped {skipped} samples (missing graph or cell embedding)')
396
+
397
+ if self.pre_filter is not None:
398
+ data_list = [d for d in data_list if self.pre_filter(d)]
399
+ if self.pre_transform is not None:
400
+ data_list = [self.pre_transform(d) for d in data_list]
401
+
402
+ print_sys(f'Processed {len(data_list)} samples. Saving to {self.processed_paths[0]}')
403
+ data, slices = self.collate(data_list)
404
+ torch.save((data, slices), self.processed_paths[0])
405
+
406
+
407
+ # ===================== Model: GCNNet (PyG, scLong-aligned) =====================
408
+
409
+ class GCNNet(torch.nn.Module):
410
+ """
411
+ GCN-based drug combination model aligned with scLong/drug_combination/models/gcn.py.
412
+
413
+ Drug branch: 3-layer GCNConv (78->78->156->312) + global_max_pool + 2 FC layers
414
+ Drug1 and drug2 share weights (drug2 uses drug1_conv* layers, same as scLong).
415
+ Cell branch: PerformerModule embedding -> MLP -> output_dim
416
+ Fusion: concat(drug1, drug2, cell) -> fc1(256) -> fc2(128) -> out(2)
417
+ """
418
+
419
+ def __init__(
420
+ self, n_output=2, num_features_xd=78, cell_emb_dim=1024,
421
+ output_dim=128, dropout=0.2,
422
+ ):
423
+ super(GCNNet, self).__init__()
424
+ self.relu = nn.ReLU()
425
+ self.dropout = nn.Dropout(dropout)
426
+ self.n_output = n_output
427
+
428
+ # Drug graph branch (shared weights for drug1 & drug2)
429
+ self.drug1_conv1 = GCNConv(num_features_xd, num_features_xd)
430
+ self.drug1_conv2 = GCNConv(num_features_xd, num_features_xd * 2)
431
+ self.drug1_conv3 = GCNConv(num_features_xd * 2, num_features_xd * 4)
432
+ self.drug1_fc_g1 = nn.Linear(num_features_xd * 4, num_features_xd * 2)
433
+ self.drug1_fc_g2 = nn.Linear(num_features_xd * 2, output_dim)
434
+
435
+ # Cell branch (PerformerModule embedding -> MLP)
436
+ self.reduction = nn.Sequential(
437
+ nn.Linear(cell_emb_dim, 512),
438
+ nn.ReLU(),
439
+ nn.Linear(512, 256),
440
+ nn.ReLU(),
441
+ nn.Linear(256, output_dim),
442
+ )
443
+
444
+ # Fusion layers
445
+ self.fc1 = nn.Linear(3 * output_dim, 256)
446
+ self.fc2 = nn.Linear(256, 128)
447
+ self.out = nn.Linear(128, self.n_output)
448
+
449
+ def forward(self, data1, data2):
450
+ x1, edge_index1, batch1, cell = (
451
+ data1.x, data1.edge_index, data1.batch, data1.cell
452
+ )
453
+ x2, edge_index2, batch2 = data2.x, data2.edge_index, data2.batch
454
+
455
+ # Drug1 branch
456
+ x1 = self.relu(self.drug1_conv1(x1, edge_index1))
457
+ x1 = self.relu(self.drug1_conv2(x1, edge_index1))
458
+ x1 = self.relu(self.drug1_conv3(x1, edge_index1))
459
+ x1 = gmp(x1, batch1)
460
+ x1 = self.relu(self.drug1_fc_g1(x1))
461
+ x1 = self.dropout(x1)
462
+ x1 = self.drug1_fc_g2(x1)
463
+ x1 = self.dropout(x1)
464
+
465
+ # Drug2 branch (shared weights — uses drug1_conv* layers, same as scLong)
466
+ x2 = self.relu(self.drug1_conv1(x2, edge_index2))
467
+ x2 = self.relu(self.drug1_conv2(x2, edge_index2))
468
+ x2 = self.relu(self.drug1_conv3(x2, edge_index2))
469
+ x2 = gmp(x2, batch2)
470
+ x2 = self.relu(self.drug1_fc_g1(x2))
471
+ x2 = self.dropout(x2)
472
+ x2 = self.drug1_fc_g2(x2)
473
+ x2 = self.dropout(x2)
474
+
475
+ # Cell branch
476
+ cell_vector = F.normalize(cell, 2, 1)
477
+ cell_vector = self.reduction(cell_vector)
478
+
479
+ # Fusion
480
+ xc = torch.cat((x1, x2, cell_vector), 1)
481
+ xc = self.relu(self.fc1(xc))
482
+ xc = self.dropout(xc)
483
+ xc = self.relu(self.fc2(xc))
484
+ xc = self.dropout(xc)
485
+ out = self.out(xc)
486
+ return out
487
+
488
+
489
+ # ===================== Training & Evaluation =====================
490
+
491
+ def train_epoch(model, device, drug1_loader, drug2_loader, optimizer, epoch):
492
+ """Train one epoch. Returns average loss."""
493
+ model.train()
494
+ total_loss = 0.0
495
+ n_batches = 0
496
+ pbar = tqdm(
497
+ zip(drug1_loader, drug2_loader),
498
+ total=len(drug1_loader),
499
+ desc=f'Epoch {epoch}',
500
+ leave=True,
501
+ )
502
+ for data1, data2 in pbar:
503
+ data1 = data1.to(device)
504
+ data2 = data2.to(device)
505
+ y = data1.y.view(-1, 1).long().to(device).squeeze(1)
506
+ optimizer.zero_grad()
507
+ output = model(data1, data2)
508
+ loss = F.cross_entropy(output, y)
509
+ loss.backward()
510
+ optimizer.step()
511
+ total_loss += loss.item()
512
+ n_batches += 1
513
+ pbar.set_postfix(loss=f'{loss.item():.4f}')
514
+ avg_loss = total_loss / max(n_batches, 1)
515
+ return avg_loss
516
+
517
+
518
+ def predict(model, device, drug1_loader, drug2_loader):
519
+ model.eval()
520
+ total_preds = torch.Tensor()
521
+ total_labels = torch.Tensor()
522
+ total_prelabels = torch.Tensor()
523
+ print_sys(f'Make prediction for {len(drug1_loader.dataset)} samples...')
524
+
525
+ with torch.no_grad():
526
+ for data in zip(drug1_loader, drug2_loader):
527
+ data1 = data[0].to(device)
528
+ data2 = data[1].to(device)
529
+ output = model(data1, data2)
530
+ ys = F.softmax(output, 1).to('cpu').data.numpy()
531
+ predicted_labels = list(map(lambda x: np.argmax(x), ys))
532
+ predicted_scores = list(map(lambda x: x[1], ys))
533
+ total_preds = torch.cat((total_preds, torch.Tensor(predicted_scores)), 0)
534
+ total_prelabels = torch.cat(
535
+ (total_prelabels, torch.Tensor(predicted_labels)), 0
536
+ )
537
+ total_labels = torch.cat((total_labels, data1.y.view(-1, 1).cpu()), 0)
538
+
539
+ return (
540
+ total_labels.numpy().flatten(),
541
+ total_preds.numpy().flatten(),
542
+ total_prelabels.numpy().flatten(),
543
+ )
544
+
545
+
546
+ def evaluate_metrics(T, S, Y):
547
+ """Compute all metrics matching scLong's evaluation."""
548
+ try:
549
+ AUC = roc_auc_score(T, S)
550
+ except Exception:
551
+ AUC = 0.0
552
+ precision_arr, recall_arr, _ = metrics.precision_recall_curve(T, S)
553
+ PR_AUC = metrics.auc(recall_arr, precision_arr)
554
+ BACC = balanced_accuracy_score(T, Y)
555
+ tn, fp, fn, tp = confusion_matrix(T, Y).ravel()
556
+ TPR = tp / (tp + fn)
557
+ PREC = precision_score(T, Y)
558
+ ACC = accuracy_score(T, Y)
559
+ KAPPA = cohen_kappa_score(T, Y)
560
+ RECALL = recall_score(T, Y)
561
+ return {
562
+ 'AUC': AUC, 'PR_AUC': PR_AUC, 'ACC': ACC, 'BACC': BACC,
563
+ 'PREC': PREC, 'TPR': TPR, 'KAPPA': KAPPA, 'RECALL': RECALL,
564
+ }
565
+
566
+
567
+ # ===================== Cell Feature Loading =====================
568
+
569
+ def load_cell_features(cell_file):
570
+ """Load cell features CSV. First row is header (gene_id), second is transcript_id (skipped)."""
571
+ df = pd.read_csv(cell_file)
572
+ df = df.iloc[1:].copy() # skip transcript_id row
573
+ df = df.set_index(df.columns[0])
574
+ df = df.astype(float)
575
+ # Clean cell names: "X22RV1_PROSTATE" -> "22RV1"
576
+ df.index = [
577
+ idx.lstrip('X').split('_')[0] if '_' in idx else idx
578
+ for idx in df.index
579
+ ]
580
+ return df
581
+
582
+
583
+ # ===================== Main =====================
584
+
585
+ def main():
586
+ parser = argparse.ArgumentParser(
587
+ description='Drug Combination Synergy Prediction with PerformerModule'
588
+ )
589
+
590
+ # PerformerModule model args
591
+ parser.add_argument('--checkpoint_path', type=str, required=True,
592
+ help='Path to PerformerModule pretrained checkpoint')
593
+ parser.add_argument('--hyper_params_path', type=str, required=True,
594
+ help='Path to PerformerModule hyper parameters pkl')
595
+ parser.add_argument('--module_data_dir', type=str, default=None,
596
+ help='Directory containing module data files')
597
+ parser.add_argument('--module_data_path', type=str, default=None,
598
+ help='Path to single module data file')
599
+ parser.add_argument('--gene_list_path', type=str,
600
+ default=str(ASSET_DIR / 'selected_genes_27k.txt'))
601
+ parser.add_argument('--gene_mapping_path', type=str,
602
+ default=str(ASSET_DIR / 'gene2ens.csv'))
603
+ parser.add_argument('--gene2vec_path', type=str,
604
+ default=str(ASSET_DIR / 'selected_gene2vec_27k.npy'))
605
+ parser.add_argument('--symbol2ens_path', type=str,
606
+ default=str(ASSET_DIR / 'gene2ens.csv'))
607
+ parser.add_argument('--pool_method', type=str, default='mean',
608
+ choices=['mean', 'max', 'concat'])
609
+ parser.add_argument('--shuffle_module_genes', action='store_true',
610
+ help='Disrupt the gene allocation within the module (used for comparative experiments to verify the effectiveness of module division)')
611
+ parser.add_argument('--shuffle_seed', type=int, default=2021,
612
+ help='The random seed used when scrambling the module gene (needs to be consistent with the value used during pre-training)')
613
+
614
+ # Data args
615
+ parser.add_argument('--data_dir', type=str,
616
+ default=str(DOWNSTREAM_DATA_DIR / 'drug_combination_synergy' / 'data_summary'))
617
+ parser.add_argument('--label_file', type=str, default='new_labels_0_10.csv')
618
+ parser.add_argument('--cell_file', type=str, default='new_cell_features_954.csv')
619
+ parser.add_argument('--validation_label_file', '--test_label_file',
620
+ dest='validation_label_file', type=str,
621
+ default='test_data/independent_input.csv',
622
+ help='Validation labels used for checkpoint selection; --test_label_file is a legacy alias')
623
+ parser.add_argument('--validation_cell_file', '--test_cell_file',
624
+ dest='validation_cell_file', type=str,
625
+ default='test_data/independent_cell_features_954.csv',
626
+ help='Validation cell features; --test_cell_file is a legacy alias')
627
+
628
+ # Training args
629
+ parser.add_argument('--batch_size', type=int, default=256)
630
+ parser.add_argument('--lr', type=float, default=1e-4)
631
+ parser.add_argument('--epochs', type=int, default=100)
632
+ parser.add_argument('--n_folds', type=int, default=5)
633
+ parser.add_argument('--domain_gen', action='store_true', default=True,
634
+ help='Domain generalization: train full set, validate on a separate set for checkpoint selection')
635
+ parser.add_argument('--no_domain_gen', dest='domain_gen', action='store_false')
636
+ parser.add_argument('--eval_interval', type=int, default=5)
637
+ parser.add_argument('--output_dir', type=str,
638
+ default=os.path.join(SCRIPT_DIR, 'results'))
639
+ parser.add_argument('--seed', type=int, default=42)
640
+ parser.add_argument('--cache_emb', action='store_true', default=True,
641
+ help='Cache PerformerModule cell embeddings to .npy')
642
+ parser.add_argument('--no_cache_emb', dest='cache_emb', action='store_false')
643
+ parser.add_argument('--force_preprocess', action='store_true', default=False,
644
+ help='Force re-creation of PyG .pt datasets')
645
+
646
+ args = parser.parse_args()
647
+
648
+ # Device
649
+ device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
650
+ print_sys(f'Using device: {device}')
651
+
652
+ # Seed
653
+ random.seed(args.seed)
654
+ np.random.seed(args.seed)
655
+ torch.manual_seed(args.seed)
656
+
657
+ os.makedirs(args.output_dir, exist_ok=True)
658
+
659
+ # -------- Step 1: Build PerformerModule cell embeddings --------
660
+ train_cell_file = os.path.join(args.data_dir, args.cell_file)
661
+ train_cell_df = load_cell_features(train_cell_file)
662
+ print_sys(f'Loaded train cell features: {train_cell_df.shape}')
663
+
664
+ validation_cell_file = os.path.join(args.data_dir, args.validation_cell_file)
665
+ validation_cell_df = load_cell_features(validation_cell_file)
666
+ print_sys(f'Loaded validation cell features: {validation_cell_df.shape}')
667
+
668
+ # Combine all cell lines and remove duplicates
669
+ all_cell_df = pd.concat([train_cell_df, validation_cell_df])
670
+ all_cell_df = all_cell_df[~all_cell_df.index.duplicated(keep='first')]
671
+ print_sys(f'Total unique cell lines: {len(all_cell_df)}')
672
+
673
+ # Cache embeddings
674
+ cache_path = os.path.join(args.output_dir, f'cell_emb_cache_{args.pool_method}.npy')
675
+ cache_dim_path = os.path.join(
676
+ args.output_dir, f'cell_emb_dim_{args.pool_method}.txt'
677
+ )
678
+
679
+ if args.cache_emb and os.path.exists(cache_path) and not args.force_preprocess:
680
+ print_sys(f'Loading cached cell embeddings from {cache_path}')
681
+ cell_emb_dict = np.load(cache_path, allow_pickle=True).item()
682
+ with open(cache_dim_path) as f:
683
+ cell_emb_dim = int(f.read().strip())
684
+ else:
685
+ cell_emb_dict, cell_emb_dim = build_cell_embeddings(
686
+ all_cell_df, args.symbol2ens_path, args.gene_list_path,
687
+ args.gene_mapping_path, args.gene2vec_path, args.checkpoint_path,
688
+ args.hyper_params_path, args.module_data_dir, args.module_data_path,
689
+ device, args.pool_method,
690
+ shuffle_module_genes=args.shuffle_module_genes,
691
+ shuffle_seed=args.shuffle_seed,
692
+ )
693
+ if args.cache_emb:
694
+ np.save(cache_path, cell_emb_dict)
695
+ with open(cache_dim_path, 'w') as f:
696
+ f.write(str(cell_emb_dim))
697
+ print_sys(f'Cached cell embeddings to {cache_path}')
698
+
699
+ # -------- Step 2: Build SMILES graphs --------
700
+ train_label_path = os.path.join(args.data_dir, args.label_file)
701
+ train_labels = pd.read_csv(train_label_path)
702
+ print_sys(f'Train labels: {len(train_labels)} samples')
703
+
704
+ validation_label_path = os.path.join(args.data_dir, args.validation_label_file)
705
+ validation_labels = pd.read_csv(validation_label_path)
706
+ print_sys(f'Validation labels: {len(validation_labels)} samples')
707
+
708
+ all_smiles = set(
709
+ train_labels['drug1'].tolist() + train_labels['drug2'].tolist()
710
+ + validation_labels['drug1'].tolist() + validation_labels['drug2'].tolist()
711
+ )
712
+ smile_graph = {}
713
+ failed = []
714
+ for smi in all_smiles:
715
+ result = smile_to_graph(smi)
716
+ if result is not None:
717
+ smile_graph[smi] = result
718
+ else:
719
+ failed.append(smi)
720
+ if failed:
721
+ print_sys(f'Warning: {len(failed)} SMILES failed to parse')
722
+ print_sys(f'Built graphs for {len(smile_graph)} unique SMILES')
723
+
724
+ # -------- Step 3: Create PyG datasets --------
725
+ # Delete cached .pt files if force_preprocess
726
+ if args.force_preprocess:
727
+ processed_dir = os.path.join(args.data_dir, 'processed')
728
+ if os.path.exists(processed_dir):
729
+ import shutil
730
+ shutil.rmtree(processed_dir)
731
+ print_sys(f'Removed cached processed data: {processed_dir}')
732
+
733
+ # Train datasets (drug1 and drug2 separate, scLong pattern)
734
+ drug1_train = train_labels['drug1'].tolist()
735
+ drug2_train = train_labels['drug2'].tolist()
736
+ cells_train = train_labels['cell'].tolist()
737
+ labels_train = train_labels['label'].tolist()
738
+
739
+ train_d1_dataset = TestbedDataset(
740
+ root=args.data_dir, dataset='train_drug1_performer_module',
741
+ xd=drug1_train, xt=cells_train, y=labels_train,
742
+ cell_emb_dict=cell_emb_dict, smile_graph=smile_graph,
743
+ )
744
+ train_d2_dataset = TestbedDataset(
745
+ root=args.data_dir, dataset='train_drug2_performer_module',
746
+ xd=drug2_train, xt=cells_train, y=labels_train,
747
+ cell_emb_dict=cell_emb_dict, smile_graph=smile_graph,
748
+ )
749
+
750
+ # Validation datasets (separate set used for checkpoint selection)
751
+ # Preserve legacy test_* cache IDs to reuse existing preprocessed datasets.
752
+ drug1_validation = validation_labels['drug1'].tolist()
753
+ drug2_validation = validation_labels['drug2'].tolist()
754
+ cells_validation = validation_labels['cell'].tolist()
755
+ labels_validation = validation_labels['label'].tolist()
756
+
757
+ validation_d1_dataset = TestbedDataset(
758
+ root=args.data_dir, dataset='test_drug1_performer_module',
759
+ xd=drug1_validation, xt=cells_validation, y=labels_validation,
760
+ cell_emb_dict=cell_emb_dict, smile_graph=smile_graph,
761
+ )
762
+ validation_d2_dataset = TestbedDataset(
763
+ root=args.data_dir, dataset='test_drug2_performer_module',
764
+ xd=drug2_validation, xt=cells_validation, y=labels_validation,
765
+ cell_emb_dict=cell_emb_dict, smile_graph=smile_graph,
766
+ )
767
+
768
+ # -------- Step 4: Training (5-fold, scLong-aligned) --------
769
+ lenth = len(train_d1_dataset)
770
+ pot = lenth // args.n_folds
771
+ random_num = random.sample(range(lenth), lenth)
772
+ all_fold_aucs = []
773
+
774
+ for fold_i in range(args.n_folds):
775
+ print_sys(f'\n{"=" * 60}')
776
+ print_sys(f'Fold {fold_i + 1}/{args.n_folds}')
777
+ print_sys(f'{"=" * 60}')
778
+
779
+ if args.domain_gen:
780
+ # Domain generalization: train on full training set, select on validation set
781
+ d1_train_data = train_d1_dataset
782
+ d1_validation_data = validation_d1_dataset
783
+ d2_train_data = train_d2_dataset
784
+ d2_validation_data = validation_d2_dataset
785
+ else:
786
+ # Standard k-fold cross-validation
787
+ validation_num = random_num[pot * fold_i : pot * (fold_i + 1)]
788
+ train_num = (
789
+ random_num[: pot * fold_i] + random_num[pot * (fold_i + 1) :]
790
+ )
791
+ d1_train_data = train_d1_dataset[train_num]
792
+ d1_validation_data = train_d1_dataset[validation_num]
793
+ d2_train_data = train_d2_dataset[train_num]
794
+ d2_validation_data = train_d2_dataset[validation_num]
795
+
796
+ d1_loader_train = DataLoader(
797
+ d1_train_data, batch_size=args.batch_size, shuffle=None, pin_memory=True,
798
+ )
799
+ d1_loader_validation = DataLoader(
800
+ d1_validation_data, batch_size=args.batch_size, shuffle=None, pin_memory=True,
801
+ )
802
+ d2_loader_train = DataLoader(
803
+ d2_train_data, batch_size=args.batch_size, shuffle=None, pin_memory=True,
804
+ )
805
+ d2_loader_validation = DataLoader(
806
+ d2_validation_data, batch_size=args.batch_size, shuffle=None, pin_memory=True,
807
+ )
808
+
809
+ model = GCNNet(cell_emb_dim=cell_emb_dim).to(device)
810
+
811
+ if fold_i == 0:
812
+ for name, param in model.named_parameters():
813
+ print_sys(f" Parameter: {name}, Shape: {param.size()}")
814
+ total_params = sum(p.numel() for p in model.parameters())
815
+ print_sys(f'Total model parameters: {total_params}')
816
+
817
+ optimizer = torch.optim.Adam(model.parameters(), lr=args.lr)
818
+ scheduler = StepLR(optimizer, step_size=1000, gamma=0.9)
819
+
820
+ best_auc = 0
821
+ loss_history = []
822
+ metric_history = []
823
+
824
+ for epoch in range(1, args.epochs + 1):
825
+ avg_loss = train_epoch(
826
+ model, device, d1_loader_train, d2_loader_train,
827
+ optimizer, epoch,
828
+ )
829
+ scheduler.step()
830
+ loss_history.append(avg_loss)
831
+
832
+ if epoch % args.eval_interval == 0:
833
+ T, S, Y = predict(model, device, d1_loader_validation, d2_loader_validation)
834
+ m = evaluate_metrics(T, S, Y)
835
+ metric_history.append({'epoch': epoch, **m})
836
+
837
+ if m['AUC'] > best_auc:
838
+ best_auc = m['AUC']
839
+ best_fpr, best_tpr = metrics.roc_curve(T, S)[:2]
840
+ save_path = os.path.join(
841
+ args.output_dir, f'best_model_fold{fold_i}.pt'
842
+ )
843
+ torch.save(model.state_dict(), save_path)
844
+ print_sys(f'** New best validation AUROC: {best_auc:.4f} (saved)')
845
+
846
+ print_sys(
847
+ f'Epoch {epoch} | Validation | '
848
+ f'AUROC: {m["AUC"]:.4f} PR_AUC: {m["PR_AUC"]:.4f} '
849
+ f'ACC: {m["ACC"]:.4f} BACC: {m["BACC"]:.4f} '
850
+ f'PREC: {m["PREC"]:.4f} TPR: {m["TPR"]:.4f} '
851
+ f'KAPPA: {m["KAPPA"]:.4f}'
852
+ )
853
+
854
+ # Save loss curve
855
+ loss_csv_path = os.path.join(args.output_dir, f'loss_fold{fold_i}.csv')
856
+ loss_df = pd.DataFrame({'epoch': range(1, len(loss_history) + 1), 'avg_loss': loss_history})
857
+ loss_df.to_csv(loss_csv_path, index=False)
858
+
859
+ # Plot loss curve
860
+ fig, ax = plt.subplots(figsize=(8, 5))
861
+ ax.plot(loss_df['epoch'], loss_df['avg_loss'], linewidth=1.5)
862
+ ax.set_xlabel('Epoch')
863
+ ax.set_ylabel('Average Loss')
864
+ ax.set_title(f'Training Loss - Fold {fold_i + 1}')
865
+ ax.grid(True, alpha=0.3)
866
+ fig.tight_layout()
867
+ fig.savefig(os.path.join(args.output_dir, f'loss_curve_fold{fold_i}.png'), dpi=150)
868
+ plt.close(fig)
869
+
870
+ # Save validation metrics history (retain the existing CSV schema)
871
+ if metric_history:
872
+ metric_df = pd.DataFrame(metric_history)
873
+ metric_df.to_csv(os.path.join(args.output_dir, f'metrics_fold{fold_i}.csv'), index=False)
874
+
875
+ # Plot validation ROC curve (using the selected model's predictions)
876
+ fig, ax = plt.subplots(figsize=(6, 6))
877
+ ax.plot(best_fpr, best_tpr, linewidth=2, label=f'AUROC = {best_auc:.4f}')
878
+ ax.plot([0, 1], [0, 1], 'k--', linewidth=1, alpha=0.5)
879
+ ax.set_xlim([0, 1])
880
+ ax.set_ylim([0, 1.02])
881
+ ax.set_xlabel('False Positive Rate')
882
+ ax.set_ylabel('True Positive Rate')
883
+ ax.set_title(f'Validation ROC Curve - Fold {fold_i + 1}')
884
+ ax.legend(loc='lower right')
885
+ ax.grid(True, alpha=0.3)
886
+ fig.tight_layout()
887
+ fig.savefig(os.path.join(args.output_dir, f'roc_curve_fold{fold_i}.png'), dpi=150)
888
+ plt.close(fig)
889
+
890
+ # Save fpr/tpr data
891
+ fpr_tpr_path = os.path.join(args.output_dir, f'fpr_tpr_fold{fold_i}.pkl')
892
+ with open(fpr_tpr_path, 'wb') as fp:
893
+ pickle.dump((best_fpr, best_tpr), fp)
894
+
895
+ print_sys(f'Fold {fold_i + 1} best validation AUROC: {best_auc:.4f}')
896
+ all_fold_aucs.append(best_auc)
897
+
898
+ # Summary
899
+ print_sys(f'\n{"=" * 60}')
900
+ print_sys(f'All folds validation AUROC: {all_fold_aucs}')
901
+ print_sys(f'Mean validation AUROC: {np.mean(all_fold_aucs):.4f} +/- {np.std(all_fold_aucs):.4f}')
902
+ print_sys('Training complete.')
903
+
904
+
905
+ if __name__ == '__main__':
906
+ main()
BioMod/drug_response/cmd.txt CHANGED
@@ -1,10 +1,10 @@
1
- python train_drug_performer_module.py \
2
- --gpu_id 0 \
3
- --data_path ../../drug_response_pytorch/data \
4
- --checkpoint_path ../performer_module_multiply_all_module_48Mcell/performer_module_multiply_all_module_48Mcell_epoch2.pth \
5
- --hyper_params_path ../performer_module_multiply_all_module_48Mcell/performer_module_multiply_all_module_48Mcell.pkl \
6
- --epochs 300 \
7
- --module_data_dir ../module_data_all \
8
- --save_dir ./performer_module_multiply_all_module_48Mcell \
9
- --fusion_method multiply \
10
- --pool_method max
 
1
+ # Run from the repository root.
2
+ python BioMod/drug_response/train_drug_performer_module.py \
3
+ --gpu_id 0 \
4
+ --data_path data/downstream/final_data/single_drug_response/data \
5
+ --checkpoint_path weight/0810model/mae_tost_70M_75mask_ec2_epoch0.pth \
6
+ --hyper_params_path weight/0810model/mae_tost_70M_75mask_ec2.pkl \
7
+ --epochs 300 \
8
+ --module_data_dir model_dependencies/module_data_all \
9
+ --save_dir tutorial_runs/drug_response \
10
+ --pool_method max
BioMod/drug_response/train_drug_performer_module.py CHANGED
@@ -1,1065 +1,1075 @@
1
- #!/usr/bin/env python
2
- # -*- coding: utf-8 -*-
3
- """
4
- Drug response prediction: using drug molecular structure + PerformerModule cell embedding
5
-
6
- Architecture:
7
- - Drug branch: 3-layer GCN + residual connection + double pooling
8
- - Cell branch: extract module-level features from pre-trained PerformerModule
9
- - Fusion layer: attention mechanism + deep MLP
10
- - Optimization strategy: AdamW + cosine annealing + early stopping
11
-
12
- How to use:
13
- python train_drug_performer_module.py --gpu_id 0 --data_path ../drug_response_pytorch/data --epochs 300
14
- """
15
-
16
- import argparse
17
- import os
18
- import random
19
- import sys
20
- import pickle
21
- from typing import Dict, List, Tuple, Union, Optional
22
-
23
- import hickle as hkl
24
- import numpy as np
25
- import pandas as pd
26
- import scipy.sparse as sp
27
- import torch
28
- import torch.nn as nn
29
- from scipy.stats import pearsonr
30
- from torch.utils.data import DataLoader, Dataset
31
-
32
- # Import PerformerModule model
33
- project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
34
- if project_root not in sys.path:
35
- sys.path.insert(0, project_root)
36
- sys.path.insert(0, os.path.dirname(project_root)) # Add scLong root directory
37
-
38
- # Use direct path import to avoid conflicts with files with the same name
39
- import importlib.util
40
- performer_module_dir = project_root
41
-
42
- # import models_performer_module
43
- models_path = os.path.join(performer_module_dir, 'models_performer_module.py')
44
- if not os.path.exists(models_path):
45
- raise FileNotFoundError(f"Model file not found: {models_path}")
46
- spec = importlib.util.spec_from_file_location("models_performer_module", models_path)
47
- models_performer_module = importlib.util.module_from_spec(spec)
48
- spec.loader.exec_module(models_performer_module)
49
- PerformerEncoderOnly_Module = models_performer_module.PerformerEncoderOnly_Module
50
- sanitize_performer_module_config = models_performer_module.sanitize_performer_module_config
51
-
52
- # Import module_dataset
53
- dataset_path = os.path.join(performer_module_dir, 'module_dataset.py')
54
- if not os.path.exists(dataset_path):
55
- raise FileNotFoundError(f"Dataset file not found: {dataset_path}")
56
- spec = importlib.util.spec_from_file_location("module_dataset", dataset_path)
57
- module_dataset_module = importlib.util.module_from_spec(spec)
58
- spec.loader.exec_module(module_dataset_module)
59
- load_module_data = module_dataset_module.load_module_data
60
-
61
-
62
- # ===================== Utility functions =====================
63
-
64
- def print_sys(*args, **kwargs):
65
- """Print and refresh output"""
66
- print(*args, **kwargs, flush=True)
67
-
68
-
69
- def reindex_tensor_universal(
70
- tensor: torch.Tensor,
71
- index_positions: List[int],
72
- dim: int,
73
- filler: str = "0",
74
- device: Union[torch.device, str] = "cpu",
75
- ) -> torch.Tensor:
76
- """Universal reordering operation"""
77
- index_tensor = torch.tensor(index_positions, device=device)
78
- tensor_shape = list(tensor.shape)
79
- tensor_shape[dim] = len(index_positions)
80
- index_shape = [1] * len(tensor_shape)
81
- index_shape[dim] = len(index_positions)
82
- index_tensor = index_tensor.view(*index_shape).expand(*tensor_shape)
83
-
84
- if filler == "0":
85
- padder_shape = list(tensor_shape)
86
- padder_shape[dim] = 1
87
- padder = torch.zeros(padder_shape, dtype=tensor.dtype, device=device)
88
- elif filler == "mean":
89
- padder = tensor.mean(dim=dim, keepdim=True)
90
- else:
91
- raise ValueError("filler should be '0' or 'mean'")
92
-
93
- expanded_tensor = torch.cat((tensor, padder), dim=dim)
94
- reindexed = torch.gather(expanded_tensor, dim, index_tensor)
95
- return reindexed
96
-
97
-
98
- # ===================== PerformerModule related =====================
99
-
100
- def load_performer_module_model(
101
- checkpoint_path: str,
102
- hyper_params_path: str,
103
- module_data_dir: Optional[str],
104
- module_data_path: Optional[str],
105
- gene_list_path: str,
106
- gene_mapping_path: str,
107
- gene2vec_path: str,
108
- device: torch.device,
109
- shuffle_module_genes: bool = False,
110
- seed: int = 0,
111
- ) -> PerformerEncoderOnly_Module:
112
- """Load a pretrained PerformerModule model"""
113
- print_sys(f"Loading PerformerModule model from: {checkpoint_path}")
114
-
115
- # Load model hyperparameters
116
- print_sys(f"Loading hyper parameters from: {hyper_params_path}")
117
- with open(hyper_params_path, 'rb') as f:
118
- model_params = pickle.load(f)
119
- model_params.pop('module_embed_chunk_padded_gene_budget', None)
120
- model_params = sanitize_performer_module_config(
121
- model_params,
122
- source=hyper_params_path,
123
- log_fn=print_sys,
124
- )
125
-
126
- # Update gene2vec path
127
- if gene2vec_path:
128
- model_params['gene2vec_file'] = gene2vec_path
129
-
130
- # Read fusion_method from pkl to ensure consistency with pre-training
131
- fusion_method = model_params.get('fusion_method', 'multiply')
132
- print_sys(f"Using fusion_method from pkl: {fusion_method}")
133
-
134
- # use_gene2vec strictly follows the pre-training pkl to avoid conflicts caused by downstream coverage.
135
- pkl_use_gene2vec = model_params.get('use_gene2vec', True)
136
- model_params['use_gene2vec'] = pkl_use_gene2vec
137
- print_sys(f"use_gene2vec={pkl_use_gene2vec} (from PKL)")
138
-
139
- # Load module data (if not present in hyperparameters)
140
- if 'module_sets' not in model_params or model_params.get('module_sets') is None:
141
- if module_data_dir or module_data_path:
142
- print_sys("Loading module data...")
143
- module_sets = load_module_data(
144
- module_data_path=module_data_path,
145
- module_data_dir=module_data_dir,
146
- gene_list_path=gene_list_path,
147
- gene_mapping_path=gene_mapping_path,
148
- verbose=True
149
- )
150
- model_params['module_sets'] = module_sets
151
- else:
152
- raise ValueError("Model requires module_sets, but --module_data_dir or --module_data_path was not provided")
153
- else:
154
- module_sets = model_params['module_sets']
155
- print_sys(f"Using module_sets from hyper parameters ({len(module_sets)} modules)")
156
-
157
- # If shuffle_module_genes is enabled, disrupt the gene distribution within the module.
158
- if shuffle_module_genes:
159
- print_sys(
160
- f"Applying downstream-only extra module reshuffle (seed={seed}). "
161
- f"Pretraining-time module_sets from PKL will be reshuffled again."
162
- )
163
- # Collect all gene indices
164
- all_gene_indices = []
165
- module_sizes = []
166
- for module in module_sets:
167
- all_gene_indices.extend(module)
168
- module_sizes.append(len(module))
169
-
170
- # Scramble the gene index using a fixed random seed
171
- rng = random.Random(seed)
172
- rng.shuffle(all_gene_indices)
173
-
174
- # Redistribute genes according to original module size
175
- shuffled_module_sets = []
176
- start_idx = 0
177
- for size in module_sizes:
178
- shuffled_module_sets.append(all_gene_indices[start_idx:start_idx + size])
179
- start_idx += size
180
- module_sets = shuffled_module_sets
181
- model_params['module_sets'] = module_sets
182
- print_sys(f"Shuffled {len(all_gene_indices)} genes across {len(module_sets)} modules")
183
-
184
- print_sys(f"PerformerModule config: base_dim={model_params.get('base_dim', 'N/A')}, "
185
- f"num_modules={len(module_sets)}")
186
-
187
- # Build model
188
- model = PerformerEncoderOnly_Module(**model_params).to(device)
189
-
190
- # Load weights
191
- print_sys(f"Loading checkpoint from: {checkpoint_path}")
192
- checkpoint = torch.load(checkpoint_path, map_location=device)
193
-
194
- # Handle parameters in checkpoint
195
- if 'model_state_dict' in checkpoint:
196
- state_dict = checkpoint['model_state_dict']
197
- else:
198
- state_dict = checkpoint
199
-
200
- # Filter out possible GO related parameters
201
- filtered_state_dict = {}
202
- for k, v in state_dict.items():
203
- if 'go_conv' not in k and 'G_go' not in k.lower():
204
- filtered_state_dict[k] = v
205
-
206
- # Use strict=True to ensure the weights match exactly (except for filtered GO parameters)
207
- model.load_state_dict(filtered_state_dict, strict=True)
208
-
209
- print_sys("Loaded pretrained PerformerModule weights.")
210
-
211
- # freeze model
212
- model.eval()
213
- for p in model.parameters():
214
- p.requires_grad = False
215
-
216
- return model, len(module_sets), model_params.get('base_dim', 1024)
217
-
218
-
219
- def build_performer_module_gene_mapping(
220
- symbol2ens_path: str,
221
- gene_list_path: str,
222
- expr_genes: List[str],
223
- ) -> Tuple[Dict[str, int], List[int]]:
224
- """Build a gene map"""
225
- gene2ens = pd.read_csv(symbol2ens_path, index_col=0)
226
- symbol2ens = gene2ens["Ensembl gene ID"].to_dict()
227
-
228
- input_genes = [symbol2ens.get(sym, sym) for sym in expr_genes]
229
- input_gene_num = len(input_genes)
230
- input_mapping = {gid: i for i, gid in enumerate(input_genes)}
231
-
232
- with open(gene_list_path, "r") as f:
233
- performer_genes = [line.rstrip("\n") for line in f.readlines()]
234
- performer_genes_pad = performer_genes + ["PAD"]
235
-
236
- performer_index_positions = [
237
- input_mapping.get(gid, input_gene_num) for gid in performer_genes_pad
238
- ]
239
- intersect = np.intersect1d(input_genes, performer_genes)
240
- print_sys(f"Target and PerformerModule genes intersect: {len(intersect)}")
241
- return input_mapping, performer_index_positions
242
-
243
-
244
- def build_cell_embeddings_with_performer_module(
245
- gexpr_feature: pd.DataFrame,
246
- symbol2ens_path: str,
247
- gene_list_path: str,
248
- gene_mapping_path: str,
249
- gene2vec_path: str,
250
- checkpoint_path: str,
251
- hyper_params_path: str,
252
- module_data_dir: Optional[str],
253
- module_data_path: Optional[str],
254
- device: torch.device,
255
- pool_method: str = "mean",
256
- shuffle_module_genes: bool = False,
257
- seed: int = 0,
258
- ) -> Tuple[Dict[str, np.ndarray], int]:
259
- """
260
- Use PerformerModule to construct embedding for each cell line
261
-
262
- Args:
263
- pool_method: pooling method, 'mean', 'max', 'cls', or 'concat'
264
- - 'mean': average all modules (default)
265
- - 'max': take the maximum value for all modules
266
- - 'cls': only use CLS token (if available)
267
- - 'concat': concatenate all modules (the dimension will be very large)
268
- use_gene2vec: automatically restored by pre-trained PKL, no longer covered downstream
269
- """
270
- model, num_modules, encoder_embed_dim = load_performer_module_model(
271
- checkpoint_path=checkpoint_path,
272
- hyper_params_path=hyper_params_path,
273
- module_data_dir=module_data_dir,
274
- module_data_path=module_data_path,
275
- gene_list_path=gene_list_path,
276
- gene_mapping_path=gene_mapping_path,
277
- gene2vec_path=gene2vec_path,
278
- device=device,
279
- shuffle_module_genes=shuffle_module_genes,
280
- seed=seed,
281
- )
282
-
283
- _, performer_index_positions = build_performer_module_gene_mapping(
284
- symbol2ens_path=symbol2ens_path,
285
- gene_list_path=gene_list_path,
286
- expr_genes=list(gexpr_feature.columns),
287
- )
288
-
289
- cell_emb_dict: Dict[str, np.ndarray] = {}
290
- first_emb_dim: int = -1
291
-
292
- with torch.no_grad():
293
- for cell_line_id, row in gexpr_feature.iterrows():
294
- x = torch.tensor(row.values[None, :], dtype=torch.float32, device=device)
295
- x_performer = reindex_tensor_universal(
296
- x, performer_index_positions, dim=1, filler="0", device=device
297
- )
298
-
299
- # Extract features through encoder (without masking, mask_ratio=0.0)
300
- out = model(x_performer, mask_ratio=0.0, return_encodings=True)
301
-
302
- # Extract module encodings
303
- if 'module_encodings' in out:
304
- module_encodings = out['module_encodings'] # (1, num_modules, encoder_embed_dim)
305
-
306
- # Processed according to pooling method
307
- if pool_method == "cls":
308
- if model.use_cls:
309
- # Use CLS token (in latent, but module_encodings does not contain CLS)
310
- # Need to get the complete latent from forward_encoder
311
- enc_out = model.forward_encoder(x_performer, mask_ratio=0.0)
312
- latent = enc_out[0]
313
- if model.use_cls:
314
- cell_emb = latent[:, 0, :] # (1, encoder_embed_dim)
315
- else:
316
- print_sys("Warning: pool_method='cls' but model has no CLS token, using 'mean' instead")
317
- cell_emb = module_encodings.mean(dim=1)
318
- else:
319
- print_sys("Warning: pool_method='cls' but model has no CLS token, using 'mean' instead")
320
- cell_emb = module_encodings.mean(dim=1)
321
- elif pool_method == "mean":
322
- cell_emb = module_encodings.mean(dim=1) # (1, encoder_embed_dim)
323
- elif pool_method == "max":
324
- cell_emb = module_encodings.max(dim=1)[0] # (1, encoder_embed_dim)
325
- elif pool_method == "concat":
326
- cell_emb = module_encodings.reshape(1, -1) # (1, num_modules * encoder_embed_dim)
327
- else:
328
- raise ValueError(f"Unknown pool_method: {pool_method}")
329
- else:
330
- raise ValueError(f"Expected 'module_encodings' in output, got: {list(out.keys())}")
331
-
332
- cell_emb_np = cell_emb.squeeze(0).cpu().numpy()
333
- cell_emb_dict[cell_line_id] = cell_emb_np
334
- if first_emb_dim < 0:
335
- first_emb_dim = cell_emb_np.shape[0]
336
-
337
- print_sys(
338
- f"Built PerformerModule embeddings for {len(cell_emb_dict)} cell lines, "
339
- f"dim={first_emb_dim}, pool_method={pool_method}"
340
- )
341
- return cell_emb_dict, first_emb_dim
342
-
343
-
344
- # ===================== Data processing =====================
345
-
346
- TCGA_LABEL_SET = [
347
- "ALL", "BLCA", "BRCA", "CESC", "DLBC", "LIHC", "LUAD", "ESCA", "GBM",
348
- "HNSC", "KIRC", "LAML", "LCML", "LGG", "LUSC", "MESO", "MM", "NB",
349
- "OV", "PAAD", "SCLC", "SKCM", "STAD", "THCA", "COAD/READ",
350
- ]
351
-
352
-
353
- def normalize_adj(adj: np.ndarray) -> np.ndarray:
354
- """Normalized adjacency matrix"""
355
- adj = adj + np.eye(adj.shape[0])
356
- d = sp.diags(np.power(np.array(adj.sum(1)), -0.5).flatten(), 0).toarray()
357
- a_norm = adj.dot(d).transpose().dot(d)
358
- return a_norm
359
-
360
-
361
- def calculate_graph_feat(
362
- feat_mat: np.ndarray,
363
- adj_list: List[List[int]],
364
- max_atoms: int,
365
- ) -> Tuple[np.ndarray, np.ndarray]:
366
- """Compute drug graph features"""
367
- assert feat_mat.shape[0] == len(adj_list)
368
- feat = np.zeros((max_atoms, feat_mat.shape[-1]), dtype=np.float32)
369
- adj_mat = np.zeros((max_atoms, max_atoms), dtype=np.float32)
370
-
371
- feat[: feat_mat.shape[0], :] = feat_mat
372
- for i, nodes in enumerate(adj_list):
373
- for each in nodes:
374
- adj_mat[i, int(each)] = 1
375
-
376
- assert np.allclose(adj_mat, adj_mat.T)
377
- adj_ = adj_mat[: len(adj_list), : len(adj_list)]
378
- norm_adj_ = normalize_adj(adj_)
379
- adj_mat[: len(adj_list), : len(adj_list)] = norm_adj_
380
- return feat, adj_mat
381
-
382
-
383
- def metadata_generate(
384
- drug_info_file: str,
385
- cell_line_info_file: str,
386
- drug_feature_dir: str,
387
- gene_expression_file: str,
388
- ) -> Tuple[dict, pd.DataFrame, list]:
389
- """Generate metadata"""
390
- # drug_id --> pubchem_id
391
- reader = pd.read_csv(drug_info_file)
392
- drugid2pubchemid = {# Build drug_id → pubchem_id mapping
393
- str(row.iloc[0]): str(row.iloc[5])# Mapping from column 0 of drug information CSV to column 5 (pubchem_id)
394
- for _, row in reader.iterrows()
395
- if str(row.iloc[5]).isdigit()
396
- }
397
-
398
- # cellline --> cancer type # Read from the TSV file, the first column is cellline_id, the last column is the cancer type
399
- cellline2cancertype: Dict[str, str] = {}
400
- with open(cell_line_info_file, "r") as f:
401
- for line in f.readlines()[1:]:
402
- cellline_id = line.split("\t")[1]
403
- tcga_label = line.strip().split("\t")[-1]
404
- cellline2cancertype[cellline_id] = tcga_label
405
-
406
- gexpr_feature = pd.read_csv(# Gene expression matrix for cell lines
407
- gene_expression_file, sep=",", header=0, index_col=0
408
- )
409
-
410
- # load drug graph features
411
- drug_feature: Dict[str, list] = {}
412
- drug_pubchem_id_set = []
413
- for fname in os.listdir(drug_feature_dir):
414
- if not fname.endswith(".hkl"):
415
- continue
416
- pubchem_id = fname.split(".")[0]
417
- feat_mat, adj_list, degree_list = hkl.load(
418
- os.path.join(drug_feature_dir, fname)
419
- )
420
- drug_feature[pubchem_id] = [feat_mat, adj_list, degree_list]# [Feature matrix, adjacency list, degree list]
421
- drug_pubchem_id_set.append(pubchem_id)
422
-
423
- # IC50
424
- # Read IC50 response data from GDSC database
425
- # Filter out drug-cell line pairs with corresponding pubchem_id and gene expression data
426
- cancer_response_exp_file = os.path.join(
427
- os.path.dirname(gene_expression_file), "GDSC_IC50.csv"
428
- )
429
- experiment_data = pd.read_csv(
430
- cancer_response_exp_file, sep=",", header=0, index_col=0
431
- )
432
- drug_match_list = [
433
- item
434
- for item in experiment_data.index
435
- if item.split(":")[1] in drugid2pubchemid.keys()
436
- ]
437
- experiment_data_filtered = experiment_data.loc[drug_match_list]
438
-
439
- data_idx: list = []# Each entry contains: cell line ID, drug pubchem ID, IC50 value, cancer type.
440
- for each_drug in experiment_data_filtered.index:
441
- for each_cellline in experiment_data_filtered.columns:
442
- pubchem_id = drugid2pubchemid[each_drug.split(":")[-1]]
443
- if (
444
- str(pubchem_id) in drug_pubchem_id_set
445
- and each_cellline in gexpr_feature.index
446
- ):
447
- if (
448
- not np.isnan(
449
- experiment_data_filtered.loc[each_drug, each_cellline]
450
- )
451
- and each_cellline in cellline2cancertype.keys()
452
- ):
453
- ln_ic50 = float(
454
- experiment_data_filtered.loc[each_drug, each_cellline]
455
- )
456
- data_idx.append(
457
- (
458
- each_cellline,
459
- pubchem_id,
460
- ln_ic50,
461
- cellline2cancertype[each_cellline],
462
- )
463
- )
464
-
465
- nb_celllines = len(set([item[0] for item in data_idx]))
466
- nb_drugs = len(set([item[1] for item in data_idx]))
467
- print_sys(
468
- f"{len(data_idx)} instances across {nb_celllines} cell lines and {nb_drugs} drugs."
469
- )
470
- return drug_feature, gexpr_feature, data_idx # Drug molecular graph features, cell line gene expression features, effective (cell line, drug, IC50, cancer type) four-tuple list
471
-
472
-
473
- def data_split(data_idx: list, ratio: float = 0.95):
474
- """Return training and validation splits stratified by cancer type."""
475
- train_idx, validation_idx = [], []
476
- for each_type in TCGA_LABEL_SET:
477
- data_subtype_idx = [item for item in data_idx if item[-1] == each_type]
478
- if len(data_subtype_idx) == 0:
479
- continue
480
- k = max(1, int(ratio * len(data_subtype_idx)))
481
- train_list = random.sample(data_subtype_idx, k)
482
- validation_list = [item for item in data_subtype_idx if item not in train_list]
483
- train_idx += train_list
484
- validation_idx += validation_list
485
- return train_idx, validation_idx
486
-
487
-
488
- def feature_extract(
489
- data_idx: list,
490
- drug_feature: dict,
491
- max_atoms: int,
492
- ) -> Tuple[list, np.ndarray, list, list]:
493
- """Extract drug map features"""
494
- nb_instance = len(data_idx)
495
-
496
- drug_data = [[] for _ in range(nb_instance)]
497
- target = np.zeros(nb_instance, dtype=np.float32)
498
- cancer_type_list = []
499
-
500
- for idx in range(nb_instance):
501
- cell_line_id, pubchem_id, ln_ic50, cancer_type = data_idx[idx]
502
-
503
- # Drug map features
504
- feat_mat, adj_list, _ = drug_feature[str(pubchem_id)]
505
- feat, adj = calculate_graph_feat(feat_mat, adj_list, max_atoms)
506
- drug_data[idx] = [feat, adj]
507
-
508
- # target value
509
- target[idx] = ln_ic50
510
-
511
- cancer_type_list.append([cancer_type, cell_line_id, pubchem_id])
512
-
513
- return drug_data, target, cancer_type_list, data_idx
514
-
515
-
516
- # ===================== Dataset =====================
517
-
518
- class DrugPerformerModuleDataset(Dataset):
519
- """Drug + PerformerModule Dataset"""
520
-
521
- def __init__(
522
- self,
523
- drug_data: list,
524
- targets: np.ndarray,
525
- index_list: list,
526
- cell_emb_dict: Dict[str, np.ndarray],
527
- ):
528
- super().__init__()
529
- self.drug_data = drug_data
530
- self.targets = targets.astype(np.float32)
531
- self.index_list = index_list
532
- self.cell_emb_dict = cell_emb_dict
533
-
534
- def __len__(self) -> int:
535
- return int(self.targets.shape[0])
536
-
537
- def __getitem__(self, idx: int):
538
- # Drug map features
539
- feat, adj = self.drug_data[idx]
540
- drug_feat = torch.from_numpy(feat).float()
541
- drug_adj = torch.from_numpy(adj).float()
542
-
543
- # PerformerModule cell embedding
544
- cell_line_id = self.index_list[idx][0]
545
- cell_emb_np = self.cell_emb_dict[cell_line_id]
546
- cell_emb = torch.from_numpy(cell_emb_np).float()
547
-
548
- # target value
549
- target = torch.tensor(self.targets[idx], dtype=torch.float32)
550
-
551
- return {
552
- "drug_feat": drug_feat,
553
- "drug_adj": drug_adj,
554
- "cell_emb": cell_emb,
555
- "target": target,
556
- }
557
-
558
-
559
- # ===================== Model Definition =====================
560
-
561
- class GraphConv(nn.Module):
562
- """graph convolution layer"""
563
-
564
- def __init__(self, in_features: int, out_features: int, bias: bool = True):
565
- super().__init__()
566
- self.in_features = in_features
567
- self.out_features = out_features
568
- self.weight = nn.Parameter(torch.Tensor(in_features, out_features))
569
- if bias:
570
- self.bias = nn.Parameter(torch.Tensor(out_features))
571
- else:
572
- self.bias = None
573
- self.reset_parameters()
574
-
575
- def reset_parameters(self):
576
- nn.init.xavier_uniform_(self.weight)
577
- if self.bias is not None:
578
- nn.init.zeros_(self.bias)
579
-
580
- def forward(self, x: torch.Tensor, adj: torch.Tensor) -> torch.Tensor:
581
- xw = torch.matmul(x, self.weight)
582
- if self.bias is not None:
583
- xw = xw + self.bias
584
- out = torch.bmm(adj.transpose(1, 2), xw)
585
- return out
586
-
587
-
588
- class DrugPerformerModuleModel(nn.Module):
589
- """
590
- Drugs + PerformerModule Predictive Model
591
-
592
- Architecture:
593
- - Drug branch: 3-layer GraphConv + residual connection + double pooling
594
- - Cell branch: 2-layer MLP (processing PerformerModule module embeddings)
595
- - Fusion layer: 4-layer MLP + attention mechanism
596
- """
597
-
598
- def __init__(
599
- self,
600
- drug_feat_dim: int,
601
- cell_emb_dim: int,
602
- hidden_dim: int = 256,
603
- dropout: float = 0.1,
604
- ):
605
- super().__init__()
606
- self.relu = nn.ReLU()
607
- self.dropout = nn.Dropout(dropout)
608
-
609
- # Drug branch: enhanced 3-layer GraphConv + residual connection
610
- self.graph_conv1 = GraphConv(drug_feat_dim, 256)
611
- self.graph_conv2 = GraphConv(256, 256)
612
- self.graph_conv3 = GraphConv(256, 128)
613
- self.drug_bn1 = nn.BatchNorm1d(256)
614
- self.drug_bn2 = nn.BatchNorm1d(256)
615
- self.drug_bn3 = nn.BatchNorm1d(128)
616
-
617
- # Cell branch: 2-layer MLP (processing PerformerModule embeddings)
618
- self.cell_fc1 = nn.Linear(cell_emb_dim, 256)
619
- self.cell_bn1 = nn.BatchNorm1d(256)
620
- self.cell_fc2 = nn.Linear(256, 128)
621
- self.cell_bn2 = nn.BatchNorm1d(128)
622
-
623
- # Fusion dimension: 128 (drug) + 128 (cell) = 256
624
- fusion_dim = 256
625
-
626
- # attention mechanism
627
- self.attention = nn.Sequential(
628
- nn.Linear(fusion_dim, fusion_dim // 2),
629
- nn.ReLU(),
630
- nn.Linear(fusion_dim // 2, fusion_dim),
631
- nn.Sigmoid()
632
- )
633
-
634
- # Fusion layer: 4-layer deep MLP
635
- self.fusion_fc1 = nn.Linear(fusion_dim, hidden_dim * 2)
636
- self.fusion_bn1 = nn.BatchNorm1d(hidden_dim * 2)
637
- self.fusion_fc2 = nn.Linear(hidden_dim * 2, hidden_dim)
638
- self.fusion_bn2 = nn.BatchNorm1d(hidden_dim)
639
- self.fusion_fc3 = nn.Linear(hidden_dim, hidden_dim // 2)
640
- self.fusion_bn3 = nn.BatchNorm1d(hidden_dim // 2)
641
- self.output_fc = nn.Linear(hidden_dim // 2, 1)
642
-
643
- print_sys(f"Model initialized: drug_feat_dim={drug_feat_dim}, "
644
- f"cell_emb_dim={cell_emb_dim}, hidden_dim={hidden_dim}")
645
-
646
- def forward(
647
- self,
648
- drug_feat: torch.Tensor,
649
- drug_adj: torch.Tensor,
650
- cell_emb: torch.Tensor,
651
- ) -> torch.Tensor:
652
- # Drug branch (enhanced version)
653
- # first floor
654
- x_drug = self.graph_conv1(drug_feat, drug_adj) # (B, N, 256)
655
- x_drug = x_drug.transpose(1, 2) # (B, 256, N) for BatchNorm
656
- x_drug = self.drug_bn1(x_drug)
657
- x_drug = x_drug.transpose(1, 2) # (B, N, 256)
658
- x_drug = self.relu(x_drug)
659
- x_drug = self.dropout(x_drug)
660
-
661
- # Second layer + residual connection
662
- x_drug_res = x_drug
663
- x_drug = self.graph_conv2(x_drug, drug_adj) # (B, N, 256)
664
- x_drug = x_drug.transpose(1, 2)
665
- x_drug = self.drug_bn2(x_drug)
666
- x_drug = x_drug.transpose(1, 2)
667
- x_drug = self.relu(x_drug + x_drug_res) # residual connection
668
- x_drug = self.dropout(x_drug)
669
-
670
- # third floor
671
- x_drug = self.graph_conv3(x_drug, drug_adj) # (B, N, 128)
672
- x_drug = x_drug.transpose(1, 2)
673
- x_drug = self.drug_bn3(x_drug)
674
- x_drug = x_drug.transpose(1, 2)
675
- x_drug = self.relu(x_drug)
676
-
677
- # Aggregation: max pooling + mean pooling
678
- x_drug_max = x_drug.max(dim=1)[0] # (B, 128)
679
- x_drug_mean = x_drug.mean(dim=1) # (B, 128)
680
- x_drug = x_drug_max + x_drug_mean # Fusion
681
-
682
- # Cell branching (handling PerformerModule embeddings)
683
- x_cell = self.cell_fc1(cell_emb) # (B, 256)
684
- x_cell = self.cell_bn1(x_cell)
685
- x_cell = self.relu(x_cell)
686
- x_cell = self.dropout(x_cell)
687
-
688
- x_cell = self.cell_fc2(x_cell) # (B, 128)
689
- x_cell = self.cell_bn2(x_cell)
690
- x_cell = self.relu(x_cell)
691
-
692
- # Feature fusion + attention
693
- h = torch.cat([x_drug, x_cell], dim=1) # (B, 256)
694
-
695
- # Apply attention mechanism
696
- att_weights = self.attention(h)
697
- h = h * att_weights # weighted features
698
-
699
- # Deep Fusion MLP
700
- h = self.fusion_fc1(h) # (B, hidden_dim * 2)
701
- h = self.fusion_bn1(h)
702
- h = self.relu(h)
703
- h = self.dropout(h)
704
-
705
- h = self.fusion_fc2(h) # (B, hidden_dim)
706
- h = self.fusion_bn2(h)
707
- h = self.relu(h)
708
- h = self.dropout(h)
709
-
710
- h = self.fusion_fc3(h) # (B, hidden_dim // 2)
711
- h = self.fusion_bn3(h)
712
- h = self.relu(h)
713
- h = self.dropout(h)
714
-
715
- # output layer
716
- out = self.output_fc(h) # (B, 1)
717
- return out
718
-
719
-
720
- # ===================== Training and Evaluation =====================
721
-
722
- def train_epoch(model, loader, optimizer, criterion, device):
723
- """train for one epoch"""
724
- model.train()
725
- total_loss = 0.0
726
- total = 0
727
- for batch in loader:
728
- drug_feat = batch["drug_feat"].to(device)
729
- drug_adj = batch["drug_adj"].to(device)
730
- cell_emb = batch["cell_emb"].to(device)
731
- target = batch["target"].to(device)
732
-
733
- optimizer.zero_grad(set_to_none=True)
734
- pred = model(drug_feat, drug_adj, cell_emb).squeeze(-1)
735
-
736
- loss = criterion(pred, target)
737
- loss.backward()
738
-
739
- # gradient crop
740
- torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)
741
-
742
- optimizer.step()
743
-
744
- total_loss += float(loss.item()) * target.size(0)
745
- total += target.size(0)
746
- return total_loss / total if total > 0 else 0.0
747
-
748
-
749
- def eval_epoch(model, loader, device):
750
- """Evaluate an epoch"""
751
- model.eval()
752
- preds = []
753
- trues = []
754
- with torch.no_grad():
755
- for batch in loader:
756
- drug_feat = batch["drug_feat"].to(device)
757
- drug_adj = batch["drug_adj"].to(device)
758
- cell_emb = batch["cell_emb"].to(device)
759
- target = batch["target"].to(device)
760
-
761
- pred = model(drug_feat, drug_adj, cell_emb).squeeze(-1)
762
-
763
- preds.append(pred.cpu().numpy())
764
- trues.append(target.cpu().numpy())
765
-
766
- preds = np.concatenate(preds, axis=0)
767
- trues = np.concatenate(trues, axis=0)
768
- pcc = pearsonr(preds, trues)[0]
769
- return preds, trues, pcc
770
-
771
-
772
- # ===================== Main =====================
773
-
774
- def main():
775
- parser = argparse.ArgumentParser(
776
- description="Drug response prediction using Drug + PerformerModule"
777
- )
778
- parser.add_argument("--gpu_id", type=str, default="0")
779
- parser.add_argument("--data_path", type=str, default="../drug_response_pytorch/data")
780
- parser.add_argument("--epochs", type=int, default=300)
781
- parser.add_argument("--batch_size", type=int, default=32)
782
- parser.add_argument("--max_atoms", type=int, default=100)
783
- parser.add_argument("--hidden_dim", type=int, default=256)
784
- parser.add_argument("--learning_rate", type=float, default=1e-3)
785
- parser.add_argument("--dropout", type=float, default=0.1)
786
- parser.add_argument("--patience", type=int, default=30)
787
-
788
- # PerformerModule related paths
789
- parser.add_argument(
790
- "--checkpoint_path",
791
- type=str,
792
- required=True,
793
- help="Path to pre-trained PerformerModule checkpoint"
794
- )
795
- parser.add_argument(
796
- "--hyper_params_path",
797
- type=str,
798
- required=True,
799
- help="Path to PerformerModule model hyper parameters pkl file"
800
- )
801
- parser.add_argument(
802
- "--module_data_dir",
803
- type=str,
804
- default=None,
805
- help="Directory containing module JSON files"
806
- )
807
- parser.add_argument(
808
- "--module_data_path",
809
- type=str,
810
- default=None,
811
- help="Path to module JSON file (single file)"
812
- )
813
- parser.add_argument(
814
- "--gene_list_path",
815
- type=str,
816
- default="../../selected_genes_27k.txt",
817
- help="Path to gene list file"
818
- )
819
- parser.add_argument(
820
- "--gene_mapping_path",
821
- type=str,
822
- default="../../gene2ens.csv",
823
- help="Path to gene symbol to Ensembl ID mapping"
824
- )
825
- parser.add_argument(
826
- "--gene2vec_path",
827
- type=str,
828
- default="../../selected_gene2vec_27k.npy",
829
- help="Path to gene2vec file"
830
- )
831
- parser.add_argument(
832
- "--symbol2ens_path",
833
- type=str,
834
- default="../../gene2ens.csv",
835
- help="Path to symbol to Ensembl mapping (for data processing)"
836
- )
837
- parser.add_argument(
838
- "--pool_method",
839
- type=str,
840
- default="mean",
841
- choices=["mean", "max", "cls", "concat"],
842
- help="Pooling method for PerformerModule module embeddings"
843
- )
844
- parser.add_argument(
845
- "--shuffle_module_genes",
846
- action="store_true",
847
- help="Optional downstream-only extra reshuffle on top of module_sets restored from PKL. "
848
- "Do not pass this when reproducing the pretrained backbone as-is."
849
- )
850
- parser.add_argument(
851
- "--shuffle_seed",
852
- type=int,
853
- default=2021,
854
- help="Random seed for the downstream extra reshuffle when --shuffle_module_genes is enabled."
855
- )
856
- # Output path
857
- parser.add_argument(
858
- "--save_dir",
859
- type=str,
860
- default="./models_drug_performer_module",
861
- help="Directory to save the model"
862
- )
863
-
864
- args = parser.parse_args()
865
-
866
- os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu_id
867
- device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
868
-
869
- print_sys("=" * 70)
870
- print_sys("Drug Response Prediction: Drug + PerformerModule")
871
- print_sys("=" * 70)
872
- print_sys(f"Device: {device}")
873
- print_sys(f"Epochs: {args.epochs}")
874
- print_sys(f"Batch size: {args.batch_size}")
875
- print_sys(f"Learning rate: {args.learning_rate}")
876
- print_sys(f"Hidden dim: {args.hidden_dim}")
877
- print_sys(f"Dropout: {args.dropout}")
878
- print_sys(f"Pool method: {args.pool_method}")
879
- print_sys(f"Early stopping patience: {args.patience}")
880
- print_sys("=" * 70)
881
-
882
- # Create save directory
883
- os.makedirs(args.save_dir, exist_ok=True)
884
-
885
- dpath = args.data_path
886
- drug_info_file = f"{dpath}/GDSC/1.Drug_listMon Jun 24 09_00_55 2019.csv"
887
- cell_line_info_file = f"{dpath}/CCLE/Cell_lines_annotations_20181226.txt"
888
- drug_feature_dir = f"{dpath}/GDSC/drug_graph_feat"
889
- gene_expression_file = (
890
- f"{dpath}/CCLE/genomic_expression_561celllines_697genes_demap_features.csv"
891
- )
892
-
893
- # fixed random seed
894
- random.seed(0)
895
- np.random.seed(0)
896
- torch.manual_seed(0)
897
- if torch.cuda.is_available():
898
- torch.cuda.manual_seed_all(0)
899
-
900
- # 1) Read metadata
901
- print_sys("\n[Step 1/6] Loading metadata...")
902
- drug_feature, gexpr_feature, data_idx = metadata_generate(
903
- drug_info_file,
904
- cell_line_info_file,
905
- drug_feature_dir,
906
- gene_expression_file,
907
- )
908
-
909
- # 2) Build PerformerModule embeddings
910
- print_sys("\n[Step 2/6] Building PerformerModule embeddings...")
911
- cell_emb_dict, cell_emb_dim = build_cell_embeddings_with_performer_module( # cell_emb_dict │ Dict[str, np.ndarray] │ {cell_line_id: embedding} mapping dictionary
912
- gexpr_feature=gexpr_feature,
913
- symbol2ens_path=args.symbol2ens_path,
914
- gene_list_path=args.gene_list_path,
915
- gene_mapping_path=args.gene_mapping_path,
916
- gene2vec_path=args.gene2vec_path,
917
- checkpoint_path=args.checkpoint_path,
918
- hyper_params_path=args.hyper_params_path,
919
- module_data_dir=args.module_data_dir,
920
- module_data_path=args.module_data_path,
921
- device=device,
922
- pool_method=args.pool_method,
923
- shuffle_module_genes=args.shuffle_module_genes,
924
- seed=args.shuffle_seed,
925
- )
926
-
927
- # 3) Split training and validation data; validation is used for model selection
928
- print_sys("\n[Step 3/6] Splitting dataset...")
929
- train_idx, validation_idx = data_split(data_idx)
930
- print_sys(f"Train samples: {len(train_idx)}, Validation samples: {len(validation_idx)}")
931
-
932
- # 4) Extract features
933
- print_sys("\n[Step 4/6] Extracting features...")
934
- X_drug_train, Y_train, train_cancer_info, train_index_list = feature_extract(
935
- train_idx, drug_feature, args.max_atoms
936
- )
937
- X_drug_validation, Y_validation, validation_cancer_info, validation_index_list = feature_extract(
938
- validation_idx, drug_feature, args.max_atoms
939
- )
940
-
941
- # 5) Build a data set
942
- print_sys("\n[Step 5/6] Creating datasets...")
943
- train_ds = DrugPerformerModuleDataset(
944
- X_drug_train, Y_train, train_index_list, cell_emb_dict
945
- )
946
- validation_ds = DrugPerformerModuleDataset(
947
- X_drug_validation, Y_validation, validation_index_list, cell_emb_dict
948
- )
949
-
950
- drug_dim = X_drug_train[0][0].shape[-1]
951
-
952
- num_workers = min(4, os.cpu_count() or 1)
953
- train_loader = DataLoader(
954
- train_ds,
955
- batch_size=args.batch_size,
956
- shuffle=True,
957
- num_workers=num_workers,
958
- pin_memory=torch.cuda.is_available(),
959
- )
960
- validation_loader = DataLoader(
961
- validation_ds,
962
- batch_size=args.batch_size,
963
- shuffle=False,
964
- num_workers=num_workers,
965
- pin_memory=torch.cuda.is_available(),
966
- )
967
-
968
- # 6) Build the model
969
- print_sys("\n[Step 6/6] Building model...")
970
- model = DrugPerformerModuleModel(
971
- drug_feat_dim=drug_dim,
972
- cell_emb_dim=cell_emb_dim,
973
- hidden_dim=args.hidden_dim,
974
- dropout=args.dropout,
975
- ).to(device)
976
-
977
- # Statistical parameters
978
- total_params = sum(p.numel() for p in model.parameters())
979
- trainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad)
980
- print_sys(f"Total parameters: {total_params:,}")
981
- print_sys(f"Trainable parameters: {trainable_params:,}")
982
-
983
- # Optimizers and schedulers
984
- optimizer = torch.optim.AdamW(
985
- model.parameters(), lr=args.learning_rate, weight_decay=1e-5
986
- )
987
- scheduler = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(
988
- optimizer, T_0=20, T_mult=2, eta_min=1e-6
989
- )
990
- criterion = torch.nn.HuberLoss(delta=1.0)
991
-
992
- # train
993
- print_sys("\n" + "=" * 70)
994
- print_sys("Starting training...")
995
- print_sys("=" * 70)
996
-
997
- best_pcc = -1.0
998
- best_pred = None
999
- best_true = None
1000
- best_epoch = 0
1001
- patience_counter = 0
1002
-
1003
- for epoch in range(1, args.epochs + 1):
1004
- train_loss = train_epoch(model, train_loader, optimizer, criterion, device)
1005
- preds, trues, pcc = eval_epoch(model, validation_loader, device)
1006
-
1007
- scheduler.step()
1008
- current_lr = optimizer.param_groups[0]["lr"]
1009
-
1010
- print_sys(
1011
- f"Epoch {epoch:03d}/{args.epochs} | "
1012
- f"train_loss={train_loss:.4f} | "
1013
- f"validation_pcc={pcc:.4f} | "
1014
- f"lr={current_lr:.6f}"
1015
- )
1016
-
1017
- if pcc > best_pcc:
1018
- best_pcc = pcc
1019
- best_pred = preds
1020
- best_true = trues
1021
- best_epoch = epoch
1022
- patience_counter = 0
1023
-
1024
- # Save the best model selected by validation PCC
1025
- model_save_path = os.path.join(args.save_dir, "best_model.pth")
1026
- torch.save(
1027
- {
1028
- "epoch": epoch,
1029
- "model_state_dict": model.state_dict(),
1030
- "optimizer_state_dict": optimizer.state_dict(),
1031
- "scheduler_state_dict": scheduler.state_dict(),
1032
- "best_pcc": best_pcc,
1033
- "train_loss": train_loss,
1034
- "args": vars(args),
1035
- },
1036
- model_save_path,
1037
- )
1038
- print_sys(f" → Best model saved! Validation PCC improved to {best_pcc:.4f}")
1039
- else:
1040
- patience_counter += 1
1041
- if patience_counter >= args.patience:
1042
- print_sys(
1043
- f"\n[Early Stopping] No improvement for {args.patience} epochs."
1044
- )
1045
- break
1046
-
1047
- # Save final result
1048
- print_sys("\n" + "=" * 70)
1049
- print_sys("Training completed!")
1050
- print_sys("=" * 70)
1051
- print_sys(f"Best epoch: {best_epoch}")
1052
- print_sys(f"Best validation PCC: {best_pcc:.4f}")
1053
-
1054
- if best_pred is not None:
1055
- result_path = os.path.join(args.save_dir, "predictions.npz")
1056
- np.savez(result_path, pred=best_pred, validation=best_true)
1057
- print_sys(f"Validation predictions saved to: {result_path}")
1058
-
1059
- print_sys(f"Best model saved to: {os.path.join(args.save_dir, 'best_model.pth')}")
1060
- print_sys("=" * 70)
1061
-
1062
-
1063
- if __name__ == "__main__":
1064
- main()
1065
-
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python
2
+ # -*- coding: utf-8 -*-
3
+ """
4
+ Drug response prediction: using drug molecular structure + PerformerModule cell embedding
5
+
6
+ Architecture:
7
+ - Drug branch: 3-layer GCN + residual connection + double pooling
8
+ - Cell branch: extract module-level features from pre-trained PerformerModule
9
+ - Fusion layer: attention mechanism + deep MLP
10
+ - Optimization strategy: AdamW + cosine annealing + early stopping
11
+
12
+ How to use:
13
+ # Run from the repository root; see drug_response/cmd.txt for a full example.
14
+ python BioMod/drug_response/train_drug_performer_module.py \
15
+ --gpu_id 0 --epochs 300 \
16
+ --checkpoint_path weight/0810model/mae_tost_70M_75mask_ec2_epoch0.pth \
17
+ --hyper_params_path weight/0810model/mae_tost_70M_75mask_ec2.pkl
18
+ """
19
+
20
+ import argparse
21
+ import os
22
+ import random
23
+ import sys
24
+ import pickle
25
+ from pathlib import Path
26
+
27
+ REPO_ROOT = Path(__file__).resolve().parents[2]
28
+ ASSET_DIR = REPO_ROOT / "model_dependencies"
29
+ DOWNSTREAM_DATA_DIR = REPO_ROOT / "data" / "downstream" / "final_data"
30
+ WEIGHTS_DIR = REPO_ROOT / "weight" / "0810model"
31
+ from typing import Dict, List, Tuple, Union, Optional
32
+
33
+ import hickle as hkl
34
+ import numpy as np
35
+ import pandas as pd
36
+ import scipy.sparse as sp
37
+ import torch
38
+ import torch.nn as nn
39
+ from scipy.stats import pearsonr
40
+ from torch.utils.data import DataLoader, Dataset
41
+
42
+ # Import PerformerModule model
43
+ project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
44
+ if project_root not in sys.path:
45
+ sys.path.insert(0, project_root)
46
+ sys.path.insert(0, os.path.dirname(project_root)) # Add the repository root directory
47
+
48
+ # Use direct path import to avoid conflicts with files with the same name
49
+ import importlib.util
50
+ performer_module_dir = project_root
51
+
52
+ # import models_performer_module
53
+ models_path = os.path.join(performer_module_dir, 'models_performer_module.py')
54
+ if not os.path.exists(models_path):
55
+ raise FileNotFoundError(f"Model file not found: {models_path}")
56
+ spec = importlib.util.spec_from_file_location("models_performer_module", models_path)
57
+ models_performer_module = importlib.util.module_from_spec(spec)
58
+ spec.loader.exec_module(models_performer_module)
59
+ PerformerEncoderOnly_Module = models_performer_module.PerformerEncoderOnly_Module
60
+ sanitize_performer_module_config = models_performer_module.sanitize_performer_module_config
61
+
62
+ # Import module_dataset
63
+ dataset_path = os.path.join(performer_module_dir, 'module_dataset.py')
64
+ if not os.path.exists(dataset_path):
65
+ raise FileNotFoundError(f"Dataset file not found: {dataset_path}")
66
+ spec = importlib.util.spec_from_file_location("module_dataset", dataset_path)
67
+ module_dataset_module = importlib.util.module_from_spec(spec)
68
+ spec.loader.exec_module(module_dataset_module)
69
+ load_module_data = module_dataset_module.load_module_data
70
+
71
+
72
+ # ===================== Utility functions =====================
73
+
74
+ def print_sys(*args, **kwargs):
75
+ """Print and refresh output"""
76
+ print(*args, **kwargs, flush=True)
77
+
78
+
79
+ def reindex_tensor_universal(
80
+ tensor: torch.Tensor,
81
+ index_positions: List[int],
82
+ dim: int,
83
+ filler: str = "0",
84
+ device: Union[torch.device, str] = "cpu",
85
+ ) -> torch.Tensor:
86
+ """Universal reordering operation"""
87
+ index_tensor = torch.tensor(index_positions, device=device)
88
+ tensor_shape = list(tensor.shape)
89
+ tensor_shape[dim] = len(index_positions)
90
+ index_shape = [1] * len(tensor_shape)
91
+ index_shape[dim] = len(index_positions)
92
+ index_tensor = index_tensor.view(*index_shape).expand(*tensor_shape)
93
+
94
+ if filler == "0":
95
+ padder_shape = list(tensor_shape)
96
+ padder_shape[dim] = 1
97
+ padder = torch.zeros(padder_shape, dtype=tensor.dtype, device=device)
98
+ elif filler == "mean":
99
+ padder = tensor.mean(dim=dim, keepdim=True)
100
+ else:
101
+ raise ValueError("filler should be '0' or 'mean'")
102
+
103
+ expanded_tensor = torch.cat((tensor, padder), dim=dim)
104
+ reindexed = torch.gather(expanded_tensor, dim, index_tensor)
105
+ return reindexed
106
+
107
+
108
+ # ===================== PerformerModule related =====================
109
+
110
+ def load_performer_module_model(
111
+ checkpoint_path: str,
112
+ hyper_params_path: str,
113
+ module_data_dir: Optional[str],
114
+ module_data_path: Optional[str],
115
+ gene_list_path: str,
116
+ gene_mapping_path: str,
117
+ gene2vec_path: str,
118
+ device: torch.device,
119
+ shuffle_module_genes: bool = False,
120
+ seed: int = 0,
121
+ ) -> PerformerEncoderOnly_Module:
122
+ """Load a pretrained PerformerModule model"""
123
+ print_sys(f"Loading PerformerModule model from: {checkpoint_path}")
124
+
125
+ # Load model hyperparameters
126
+ print_sys(f"Loading hyper parameters from: {hyper_params_path}")
127
+ with open(hyper_params_path, 'rb') as f:
128
+ model_params = pickle.load(f)
129
+ model_params.pop('module_embed_chunk_padded_gene_budget', None)
130
+ model_params = sanitize_performer_module_config(
131
+ model_params,
132
+ source=hyper_params_path,
133
+ log_fn=print_sys,
134
+ )
135
+
136
+ # Update gene2vec path
137
+ if gene2vec_path:
138
+ model_params['gene2vec_file'] = gene2vec_path
139
+
140
+ # Read fusion_method from pkl to ensure consistency with pre-training
141
+ fusion_method = model_params.get('fusion_method', 'multiply')
142
+ print_sys(f"Using fusion_method from pkl: {fusion_method}")
143
+
144
+ # use_gene2vec strictly follows the pre-training pkl to avoid conflicts caused by downstream coverage.
145
+ pkl_use_gene2vec = model_params.get('use_gene2vec', True)
146
+ model_params['use_gene2vec'] = pkl_use_gene2vec
147
+ print_sys(f"use_gene2vec={pkl_use_gene2vec} (from PKL)")
148
+
149
+ # Load module data (if not present in hyperparameters)
150
+ if 'module_sets' not in model_params or model_params.get('module_sets') is None:
151
+ if module_data_dir or module_data_path:
152
+ print_sys("Loading module data...")
153
+ module_sets = load_module_data(
154
+ module_data_path=module_data_path,
155
+ module_data_dir=module_data_dir,
156
+ gene_list_path=gene_list_path,
157
+ gene_mapping_path=gene_mapping_path,
158
+ verbose=True
159
+ )
160
+ model_params['module_sets'] = module_sets
161
+ else:
162
+ raise ValueError("Model requires module_sets, but --module_data_dir or --module_data_path was not provided")
163
+ else:
164
+ module_sets = model_params['module_sets']
165
+ print_sys(f"Using module_sets from hyper parameters ({len(module_sets)} modules)")
166
+
167
+ # If shuffle_module_genes is enabled, disrupt the gene distribution within the module.
168
+ if shuffle_module_genes:
169
+ print_sys(
170
+ f"Applying downstream-only extra module reshuffle (seed={seed}). "
171
+ f"Pretraining-time module_sets from PKL will be reshuffled again."
172
+ )
173
+ # Collect all gene indices
174
+ all_gene_indices = []
175
+ module_sizes = []
176
+ for module in module_sets:
177
+ all_gene_indices.extend(module)
178
+ module_sizes.append(len(module))
179
+
180
+ # Scramble the gene index using a fixed random seed
181
+ rng = random.Random(seed)
182
+ rng.shuffle(all_gene_indices)
183
+
184
+ # Redistribute genes according to original module size
185
+ shuffled_module_sets = []
186
+ start_idx = 0
187
+ for size in module_sizes:
188
+ shuffled_module_sets.append(all_gene_indices[start_idx:start_idx + size])
189
+ start_idx += size
190
+ module_sets = shuffled_module_sets
191
+ model_params['module_sets'] = module_sets
192
+ print_sys(f"Shuffled {len(all_gene_indices)} genes across {len(module_sets)} modules")
193
+
194
+ print_sys(f"PerformerModule config: base_dim={model_params.get('base_dim', 'N/A')}, "
195
+ f"num_modules={len(module_sets)}")
196
+
197
+ # Build model
198
+ model = PerformerEncoderOnly_Module(**model_params).to(device)
199
+
200
+ # Load weights
201
+ print_sys(f"Loading checkpoint from: {checkpoint_path}")
202
+ checkpoint = torch.load(checkpoint_path, map_location=device)
203
+
204
+ # Handle parameters in checkpoint
205
+ if 'model_state_dict' in checkpoint:
206
+ state_dict = checkpoint['model_state_dict']
207
+ else:
208
+ state_dict = checkpoint
209
+
210
+ # Filter out possible GO related parameters
211
+ filtered_state_dict = {}
212
+ for k, v in state_dict.items():
213
+ if 'go_conv' not in k and 'G_go' not in k.lower():
214
+ filtered_state_dict[k] = v
215
+
216
+ # Use strict=True to ensure the weights match exactly (except for filtered GO parameters)
217
+ model.load_state_dict(filtered_state_dict, strict=True)
218
+
219
+ print_sys("Loaded pretrained PerformerModule weights.")
220
+
221
+ # freeze model
222
+ model.eval()
223
+ for p in model.parameters():
224
+ p.requires_grad = False
225
+
226
+ return model, len(module_sets), model_params.get('base_dim', 1024)
227
+
228
+
229
+ def build_performer_module_gene_mapping(
230
+ symbol2ens_path: str,
231
+ gene_list_path: str,
232
+ expr_genes: List[str],
233
+ ) -> Tuple[Dict[str, int], List[int]]:
234
+ """Build a gene map"""
235
+ gene2ens = pd.read_csv(symbol2ens_path, index_col=0)
236
+ symbol2ens = gene2ens["Ensembl gene ID"].to_dict()
237
+
238
+ input_genes = [symbol2ens.get(sym, sym) for sym in expr_genes]
239
+ input_gene_num = len(input_genes)
240
+ input_mapping = {gid: i for i, gid in enumerate(input_genes)}
241
+
242
+ with open(gene_list_path, "r") as f:
243
+ performer_genes = [line.rstrip("\n") for line in f.readlines()]
244
+ performer_genes_pad = performer_genes + ["PAD"]
245
+
246
+ performer_index_positions = [
247
+ input_mapping.get(gid, input_gene_num) for gid in performer_genes_pad
248
+ ]
249
+ intersect = np.intersect1d(input_genes, performer_genes)
250
+ print_sys(f"Target and PerformerModule genes intersect: {len(intersect)}")
251
+ return input_mapping, performer_index_positions
252
+
253
+
254
+ def build_cell_embeddings_with_performer_module(
255
+ gexpr_feature: pd.DataFrame,
256
+ symbol2ens_path: str,
257
+ gene_list_path: str,
258
+ gene_mapping_path: str,
259
+ gene2vec_path: str,
260
+ checkpoint_path: str,
261
+ hyper_params_path: str,
262
+ module_data_dir: Optional[str],
263
+ module_data_path: Optional[str],
264
+ device: torch.device,
265
+ pool_method: str = "mean",
266
+ shuffle_module_genes: bool = False,
267
+ seed: int = 0,
268
+ ) -> Tuple[Dict[str, np.ndarray], int]:
269
+ """
270
+ Use PerformerModule to construct embedding for each cell line
271
+
272
+ Args:
273
+ pool_method: pooling method, 'mean', 'max', 'cls', or 'concat'
274
+ - 'mean': average all modules (default)
275
+ - 'max': take the maximum value for all modules
276
+ - 'cls': only use CLS token (if available)
277
+ - 'concat': concatenate all modules (the dimension will be very large)
278
+ use_gene2vec: automatically restored by pre-trained PKL, no longer covered downstream
279
+ """
280
+ model, num_modules, encoder_embed_dim = load_performer_module_model(
281
+ checkpoint_path=checkpoint_path,
282
+ hyper_params_path=hyper_params_path,
283
+ module_data_dir=module_data_dir,
284
+ module_data_path=module_data_path,
285
+ gene_list_path=gene_list_path,
286
+ gene_mapping_path=gene_mapping_path,
287
+ gene2vec_path=gene2vec_path,
288
+ device=device,
289
+ shuffle_module_genes=shuffle_module_genes,
290
+ seed=seed,
291
+ )
292
+
293
+ _, performer_index_positions = build_performer_module_gene_mapping(
294
+ symbol2ens_path=symbol2ens_path,
295
+ gene_list_path=gene_list_path,
296
+ expr_genes=list(gexpr_feature.columns),
297
+ )
298
+
299
+ cell_emb_dict: Dict[str, np.ndarray] = {}
300
+ first_emb_dim: int = -1
301
+
302
+ with torch.no_grad():
303
+ for cell_line_id, row in gexpr_feature.iterrows():
304
+ x = torch.tensor(row.values[None, :], dtype=torch.float32, device=device)
305
+ x_performer = reindex_tensor_universal(
306
+ x, performer_index_positions, dim=1, filler="0", device=device
307
+ )
308
+
309
+ # Extract features through encoder (without masking, mask_ratio=0.0)
310
+ out = model(x_performer, mask_ratio=0.0, return_encodings=True)
311
+
312
+ # Extract module encodings
313
+ if 'module_encodings' in out:
314
+ module_encodings = out['module_encodings'] # (1, num_modules, encoder_embed_dim)
315
+
316
+ # Processed according to pooling method
317
+ if pool_method == "cls":
318
+ if model.use_cls:
319
+ # Use CLS token (in latent, but module_encodings does not contain CLS)
320
+ # Need to get the complete latent from forward_encoder
321
+ enc_out = model.forward_encoder(x_performer, mask_ratio=0.0)
322
+ latent = enc_out[0]
323
+ if model.use_cls:
324
+ cell_emb = latent[:, 0, :] # (1, encoder_embed_dim)
325
+ else:
326
+ print_sys("Warning: pool_method='cls' but model has no CLS token, using 'mean' instead")
327
+ cell_emb = module_encodings.mean(dim=1)
328
+ else:
329
+ print_sys("Warning: pool_method='cls' but model has no CLS token, using 'mean' instead")
330
+ cell_emb = module_encodings.mean(dim=1)
331
+ elif pool_method == "mean":
332
+ cell_emb = module_encodings.mean(dim=1) # (1, encoder_embed_dim)
333
+ elif pool_method == "max":
334
+ cell_emb = module_encodings.max(dim=1)[0] # (1, encoder_embed_dim)
335
+ elif pool_method == "concat":
336
+ cell_emb = module_encodings.reshape(1, -1) # (1, num_modules * encoder_embed_dim)
337
+ else:
338
+ raise ValueError(f"Unknown pool_method: {pool_method}")
339
+ else:
340
+ raise ValueError(f"Expected 'module_encodings' in output, got: {list(out.keys())}")
341
+
342
+ cell_emb_np = cell_emb.squeeze(0).cpu().numpy()
343
+ cell_emb_dict[cell_line_id] = cell_emb_np
344
+ if first_emb_dim < 0:
345
+ first_emb_dim = cell_emb_np.shape[0]
346
+
347
+ print_sys(
348
+ f"Built PerformerModule embeddings for {len(cell_emb_dict)} cell lines, "
349
+ f"dim={first_emb_dim}, pool_method={pool_method}"
350
+ )
351
+ return cell_emb_dict, first_emb_dim
352
+
353
+
354
+ # ===================== Data processing =====================
355
+
356
+ TCGA_LABEL_SET = [
357
+ "ALL", "BLCA", "BRCA", "CESC", "DLBC", "LIHC", "LUAD", "ESCA", "GBM",
358
+ "HNSC", "KIRC", "LAML", "LCML", "LGG", "LUSC", "MESO", "MM", "NB",
359
+ "OV", "PAAD", "SCLC", "SKCM", "STAD", "THCA", "COAD/READ",
360
+ ]
361
+
362
+
363
+ def normalize_adj(adj: np.ndarray) -> np.ndarray:
364
+ """Normalized adjacency matrix"""
365
+ adj = adj + np.eye(adj.shape[0])
366
+ d = sp.diags(np.power(np.array(adj.sum(1)), -0.5).flatten(), 0).toarray()
367
+ a_norm = adj.dot(d).transpose().dot(d)
368
+ return a_norm
369
+
370
+
371
+ def calculate_graph_feat(
372
+ feat_mat: np.ndarray,
373
+ adj_list: List[List[int]],
374
+ max_atoms: int,
375
+ ) -> Tuple[np.ndarray, np.ndarray]:
376
+ """Compute drug graph features"""
377
+ assert feat_mat.shape[0] == len(adj_list)
378
+ feat = np.zeros((max_atoms, feat_mat.shape[-1]), dtype=np.float32)
379
+ adj_mat = np.zeros((max_atoms, max_atoms), dtype=np.float32)
380
+
381
+ feat[: feat_mat.shape[0], :] = feat_mat
382
+ for i, nodes in enumerate(adj_list):
383
+ for each in nodes:
384
+ adj_mat[i, int(each)] = 1
385
+
386
+ assert np.allclose(adj_mat, adj_mat.T)
387
+ adj_ = adj_mat[: len(adj_list), : len(adj_list)]
388
+ norm_adj_ = normalize_adj(adj_)
389
+ adj_mat[: len(adj_list), : len(adj_list)] = norm_adj_
390
+ return feat, adj_mat
391
+
392
+
393
+ def metadata_generate(
394
+ drug_info_file: str,
395
+ cell_line_info_file: str,
396
+ drug_feature_dir: str,
397
+ gene_expression_file: str,
398
+ ) -> Tuple[dict, pd.DataFrame, list]:
399
+ """Generate metadata"""
400
+ # drug_id --> pubchem_id
401
+ reader = pd.read_csv(drug_info_file)
402
+ drugid2pubchemid = {# Build drug_id → pubchem_id mapping
403
+ str(row.iloc[0]): str(row.iloc[5])# Mapping from column 0 of drug information CSV to column 5 (pubchem_id)
404
+ for _, row in reader.iterrows()
405
+ if str(row.iloc[5]).isdigit()
406
+ }
407
+
408
+ # cellline --> cancer type # Read from the TSV file, the first column is cellline_id, the last column is the cancer type
409
+ cellline2cancertype: Dict[str, str] = {}
410
+ with open(cell_line_info_file, "r") as f:
411
+ for line in f.readlines()[1:]:
412
+ cellline_id = line.split("\t")[1]
413
+ tcga_label = line.strip().split("\t")[-1]
414
+ cellline2cancertype[cellline_id] = tcga_label
415
+
416
+ gexpr_feature = pd.read_csv(# Gene expression matrix for cell lines
417
+ gene_expression_file, sep=",", header=0, index_col=0
418
+ )
419
+
420
+ # load drug graph features
421
+ drug_feature: Dict[str, list] = {}
422
+ drug_pubchem_id_set = []
423
+ for fname in os.listdir(drug_feature_dir):
424
+ if not fname.endswith(".hkl"):
425
+ continue
426
+ pubchem_id = fname.split(".")[0]
427
+ feat_mat, adj_list, degree_list = hkl.load(
428
+ os.path.join(drug_feature_dir, fname)
429
+ )
430
+ drug_feature[pubchem_id] = [feat_mat, adj_list, degree_list]# [Feature matrix, adjacency list, degree list]
431
+ drug_pubchem_id_set.append(pubchem_id)
432
+
433
+ # IC50
434
+ # Read IC50 response data from GDSC database
435
+ # Filter out drug-cell line pairs with corresponding pubchem_id and gene expression data
436
+ cancer_response_exp_file = os.path.join(
437
+ os.path.dirname(gene_expression_file), "GDSC_IC50.csv"
438
+ )
439
+ experiment_data = pd.read_csv(
440
+ cancer_response_exp_file, sep=",", header=0, index_col=0
441
+ )
442
+ drug_match_list = [
443
+ item
444
+ for item in experiment_data.index
445
+ if item.split(":")[1] in drugid2pubchemid.keys()
446
+ ]
447
+ experiment_data_filtered = experiment_data.loc[drug_match_list]
448
+
449
+ data_idx: list = []# Each entry contains: cell line ID, drug pubchem ID, IC50 value, cancer type.
450
+ for each_drug in experiment_data_filtered.index:
451
+ for each_cellline in experiment_data_filtered.columns:
452
+ pubchem_id = drugid2pubchemid[each_drug.split(":")[-1]]
453
+ if (
454
+ str(pubchem_id) in drug_pubchem_id_set
455
+ and each_cellline in gexpr_feature.index
456
+ ):
457
+ if (
458
+ not np.isnan(
459
+ experiment_data_filtered.loc[each_drug, each_cellline]
460
+ )
461
+ and each_cellline in cellline2cancertype.keys()
462
+ ):
463
+ ln_ic50 = float(
464
+ experiment_data_filtered.loc[each_drug, each_cellline]
465
+ )
466
+ data_idx.append(
467
+ (
468
+ each_cellline,
469
+ pubchem_id,
470
+ ln_ic50,
471
+ cellline2cancertype[each_cellline],
472
+ )
473
+ )
474
+
475
+ nb_celllines = len(set([item[0] for item in data_idx]))
476
+ nb_drugs = len(set([item[1] for item in data_idx]))
477
+ print_sys(
478
+ f"{len(data_idx)} instances across {nb_celllines} cell lines and {nb_drugs} drugs."
479
+ )
480
+ return drug_feature, gexpr_feature, data_idx # Drug molecular graph features, cell line gene expression features, effective (cell line, drug, IC50, cancer type) four-tuple list
481
+
482
+
483
+ def data_split(data_idx: list, ratio: float = 0.95):
484
+ """Return training and validation splits stratified by cancer type."""
485
+ train_idx, validation_idx = [], []
486
+ for each_type in TCGA_LABEL_SET:
487
+ data_subtype_idx = [item for item in data_idx if item[-1] == each_type]
488
+ if len(data_subtype_idx) == 0:
489
+ continue
490
+ k = max(1, int(ratio * len(data_subtype_idx)))
491
+ train_list = random.sample(data_subtype_idx, k)
492
+ validation_list = [item for item in data_subtype_idx if item not in train_list]
493
+ train_idx += train_list
494
+ validation_idx += validation_list
495
+ return train_idx, validation_idx
496
+
497
+
498
+ def feature_extract(
499
+ data_idx: list,
500
+ drug_feature: dict,
501
+ max_atoms: int,
502
+ ) -> Tuple[list, np.ndarray, list, list]:
503
+ """Extract drug map features"""
504
+ nb_instance = len(data_idx)
505
+
506
+ drug_data = [[] for _ in range(nb_instance)]
507
+ target = np.zeros(nb_instance, dtype=np.float32)
508
+ cancer_type_list = []
509
+
510
+ for idx in range(nb_instance):
511
+ cell_line_id, pubchem_id, ln_ic50, cancer_type = data_idx[idx]
512
+
513
+ # Drug map features
514
+ feat_mat, adj_list, _ = drug_feature[str(pubchem_id)]
515
+ feat, adj = calculate_graph_feat(feat_mat, adj_list, max_atoms)
516
+ drug_data[idx] = [feat, adj]
517
+
518
+ # target value
519
+ target[idx] = ln_ic50
520
+
521
+ cancer_type_list.append([cancer_type, cell_line_id, pubchem_id])
522
+
523
+ return drug_data, target, cancer_type_list, data_idx
524
+
525
+
526
+ # ===================== Dataset =====================
527
+
528
+ class DrugPerformerModuleDataset(Dataset):
529
+ """Drug + PerformerModule Dataset"""
530
+
531
+ def __init__(
532
+ self,
533
+ drug_data: list,
534
+ targets: np.ndarray,
535
+ index_list: list,
536
+ cell_emb_dict: Dict[str, np.ndarray],
537
+ ):
538
+ super().__init__()
539
+ self.drug_data = drug_data
540
+ self.targets = targets.astype(np.float32)
541
+ self.index_list = index_list
542
+ self.cell_emb_dict = cell_emb_dict
543
+
544
+ def __len__(self) -> int:
545
+ return int(self.targets.shape[0])
546
+
547
+ def __getitem__(self, idx: int):
548
+ # Drug map features
549
+ feat, adj = self.drug_data[idx]
550
+ drug_feat = torch.from_numpy(feat).float()
551
+ drug_adj = torch.from_numpy(adj).float()
552
+
553
+ # PerformerModule cell embedding
554
+ cell_line_id = self.index_list[idx][0]
555
+ cell_emb_np = self.cell_emb_dict[cell_line_id]
556
+ cell_emb = torch.from_numpy(cell_emb_np).float()
557
+
558
+ # target value
559
+ target = torch.tensor(self.targets[idx], dtype=torch.float32)
560
+
561
+ return {
562
+ "drug_feat": drug_feat,
563
+ "drug_adj": drug_adj,
564
+ "cell_emb": cell_emb,
565
+ "target": target,
566
+ }
567
+
568
+
569
+ # ===================== Model Definition =====================
570
+
571
+ class GraphConv(nn.Module):
572
+ """graph convolution layer"""
573
+
574
+ def __init__(self, in_features: int, out_features: int, bias: bool = True):
575
+ super().__init__()
576
+ self.in_features = in_features
577
+ self.out_features = out_features
578
+ self.weight = nn.Parameter(torch.Tensor(in_features, out_features))
579
+ if bias:
580
+ self.bias = nn.Parameter(torch.Tensor(out_features))
581
+ else:
582
+ self.bias = None
583
+ self.reset_parameters()
584
+
585
+ def reset_parameters(self):
586
+ nn.init.xavier_uniform_(self.weight)
587
+ if self.bias is not None:
588
+ nn.init.zeros_(self.bias)
589
+
590
+ def forward(self, x: torch.Tensor, adj: torch.Tensor) -> torch.Tensor:
591
+ xw = torch.matmul(x, self.weight)
592
+ if self.bias is not None:
593
+ xw = xw + self.bias
594
+ out = torch.bmm(adj.transpose(1, 2), xw)
595
+ return out
596
+
597
+
598
+ class DrugPerformerModuleModel(nn.Module):
599
+ """
600
+ Drugs + PerformerModule Predictive Model
601
+
602
+ Architecture:
603
+ - Drug branch: 3-layer GraphConv + residual connection + double pooling
604
+ - Cell branch: 2-layer MLP (processing PerformerModule module embeddings)
605
+ - Fusion layer: 4-layer MLP + attention mechanism
606
+ """
607
+
608
+ def __init__(
609
+ self,
610
+ drug_feat_dim: int,
611
+ cell_emb_dim: int,
612
+ hidden_dim: int = 256,
613
+ dropout: float = 0.1,
614
+ ):
615
+ super().__init__()
616
+ self.relu = nn.ReLU()
617
+ self.dropout = nn.Dropout(dropout)
618
+
619
+ # Drug branch: enhanced 3-layer GraphConv + residual connection
620
+ self.graph_conv1 = GraphConv(drug_feat_dim, 256)
621
+ self.graph_conv2 = GraphConv(256, 256)
622
+ self.graph_conv3 = GraphConv(256, 128)
623
+ self.drug_bn1 = nn.BatchNorm1d(256)
624
+ self.drug_bn2 = nn.BatchNorm1d(256)
625
+ self.drug_bn3 = nn.BatchNorm1d(128)
626
+
627
+ # Cell branch: 2-layer MLP (processing PerformerModule embeddings)
628
+ self.cell_fc1 = nn.Linear(cell_emb_dim, 256)
629
+ self.cell_bn1 = nn.BatchNorm1d(256)
630
+ self.cell_fc2 = nn.Linear(256, 128)
631
+ self.cell_bn2 = nn.BatchNorm1d(128)
632
+
633
+ # Fusion dimension: 128 (drug) + 128 (cell) = 256
634
+ fusion_dim = 256
635
+
636
+ # attention mechanism
637
+ self.attention = nn.Sequential(
638
+ nn.Linear(fusion_dim, fusion_dim // 2),
639
+ nn.ReLU(),
640
+ nn.Linear(fusion_dim // 2, fusion_dim),
641
+ nn.Sigmoid()
642
+ )
643
+
644
+ # Fusion layer: 4-layer deep MLP
645
+ self.fusion_fc1 = nn.Linear(fusion_dim, hidden_dim * 2)
646
+ self.fusion_bn1 = nn.BatchNorm1d(hidden_dim * 2)
647
+ self.fusion_fc2 = nn.Linear(hidden_dim * 2, hidden_dim)
648
+ self.fusion_bn2 = nn.BatchNorm1d(hidden_dim)
649
+ self.fusion_fc3 = nn.Linear(hidden_dim, hidden_dim // 2)
650
+ self.fusion_bn3 = nn.BatchNorm1d(hidden_dim // 2)
651
+ self.output_fc = nn.Linear(hidden_dim // 2, 1)
652
+
653
+ print_sys(f"Model initialized: drug_feat_dim={drug_feat_dim}, "
654
+ f"cell_emb_dim={cell_emb_dim}, hidden_dim={hidden_dim}")
655
+
656
+ def forward(
657
+ self,
658
+ drug_feat: torch.Tensor,
659
+ drug_adj: torch.Tensor,
660
+ cell_emb: torch.Tensor,
661
+ ) -> torch.Tensor:
662
+ # Drug branch (enhanced version)
663
+ # first floor
664
+ x_drug = self.graph_conv1(drug_feat, drug_adj) # (B, N, 256)
665
+ x_drug = x_drug.transpose(1, 2) # (B, 256, N) for BatchNorm
666
+ x_drug = self.drug_bn1(x_drug)
667
+ x_drug = x_drug.transpose(1, 2) # (B, N, 256)
668
+ x_drug = self.relu(x_drug)
669
+ x_drug = self.dropout(x_drug)
670
+
671
+ # Second layer + residual connection
672
+ x_drug_res = x_drug
673
+ x_drug = self.graph_conv2(x_drug, drug_adj) # (B, N, 256)
674
+ x_drug = x_drug.transpose(1, 2)
675
+ x_drug = self.drug_bn2(x_drug)
676
+ x_drug = x_drug.transpose(1, 2)
677
+ x_drug = self.relu(x_drug + x_drug_res) # residual connection
678
+ x_drug = self.dropout(x_drug)
679
+
680
+ # third floor
681
+ x_drug = self.graph_conv3(x_drug, drug_adj) # (B, N, 128)
682
+ x_drug = x_drug.transpose(1, 2)
683
+ x_drug = self.drug_bn3(x_drug)
684
+ x_drug = x_drug.transpose(1, 2)
685
+ x_drug = self.relu(x_drug)
686
+
687
+ # Aggregation: max pooling + mean pooling
688
+ x_drug_max = x_drug.max(dim=1)[0] # (B, 128)
689
+ x_drug_mean = x_drug.mean(dim=1) # (B, 128)
690
+ x_drug = x_drug_max + x_drug_mean # Fusion
691
+
692
+ # Cell branching (handling PerformerModule embeddings)
693
+ x_cell = self.cell_fc1(cell_emb) # (B, 256)
694
+ x_cell = self.cell_bn1(x_cell)
695
+ x_cell = self.relu(x_cell)
696
+ x_cell = self.dropout(x_cell)
697
+
698
+ x_cell = self.cell_fc2(x_cell) # (B, 128)
699
+ x_cell = self.cell_bn2(x_cell)
700
+ x_cell = self.relu(x_cell)
701
+
702
+ # Feature fusion + attention
703
+ h = torch.cat([x_drug, x_cell], dim=1) # (B, 256)
704
+
705
+ # Apply attention mechanism
706
+ att_weights = self.attention(h)
707
+ h = h * att_weights # weighted features
708
+
709
+ # Deep Fusion MLP
710
+ h = self.fusion_fc1(h) # (B, hidden_dim * 2)
711
+ h = self.fusion_bn1(h)
712
+ h = self.relu(h)
713
+ h = self.dropout(h)
714
+
715
+ h = self.fusion_fc2(h) # (B, hidden_dim)
716
+ h = self.fusion_bn2(h)
717
+ h = self.relu(h)
718
+ h = self.dropout(h)
719
+
720
+ h = self.fusion_fc3(h) # (B, hidden_dim // 2)
721
+ h = self.fusion_bn3(h)
722
+ h = self.relu(h)
723
+ h = self.dropout(h)
724
+
725
+ # output layer
726
+ out = self.output_fc(h) # (B, 1)
727
+ return out
728
+
729
+
730
+ # ===================== Training and Evaluation =====================
731
+
732
+ def train_epoch(model, loader, optimizer, criterion, device):
733
+ """train for one epoch"""
734
+ model.train()
735
+ total_loss = 0.0
736
+ total = 0
737
+ for batch in loader:
738
+ drug_feat = batch["drug_feat"].to(device)
739
+ drug_adj = batch["drug_adj"].to(device)
740
+ cell_emb = batch["cell_emb"].to(device)
741
+ target = batch["target"].to(device)
742
+
743
+ optimizer.zero_grad(set_to_none=True)
744
+ pred = model(drug_feat, drug_adj, cell_emb).squeeze(-1)
745
+
746
+ loss = criterion(pred, target)
747
+ loss.backward()
748
+
749
+ # gradient crop
750
+ torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)
751
+
752
+ optimizer.step()
753
+
754
+ total_loss += float(loss.item()) * target.size(0)
755
+ total += target.size(0)
756
+ return total_loss / total if total > 0 else 0.0
757
+
758
+
759
+ def eval_epoch(model, loader, device):
760
+ """Evaluate an epoch"""
761
+ model.eval()
762
+ preds = []
763
+ trues = []
764
+ with torch.no_grad():
765
+ for batch in loader:
766
+ drug_feat = batch["drug_feat"].to(device)
767
+ drug_adj = batch["drug_adj"].to(device)
768
+ cell_emb = batch["cell_emb"].to(device)
769
+ target = batch["target"].to(device)
770
+
771
+ pred = model(drug_feat, drug_adj, cell_emb).squeeze(-1)
772
+
773
+ preds.append(pred.cpu().numpy())
774
+ trues.append(target.cpu().numpy())
775
+
776
+ preds = np.concatenate(preds, axis=0)
777
+ trues = np.concatenate(trues, axis=0)
778
+ pcc = pearsonr(preds, trues)[0]
779
+ return preds, trues, pcc
780
+
781
+
782
+ # ===================== Main =====================
783
+
784
+ def main():
785
+ parser = argparse.ArgumentParser(
786
+ description="Drug response prediction using Drug + PerformerModule"
787
+ )
788
+ parser.add_argument("--gpu_id", type=str, default="0")
789
+ parser.add_argument("--data_path", type=str, default=str(DOWNSTREAM_DATA_DIR / "single_drug_response" / "data"))
790
+ parser.add_argument("--epochs", type=int, default=300)
791
+ parser.add_argument("--batch_size", type=int, default=32)
792
+ parser.add_argument("--max_atoms", type=int, default=100)
793
+ parser.add_argument("--hidden_dim", type=int, default=256)
794
+ parser.add_argument("--learning_rate", type=float, default=1e-3)
795
+ parser.add_argument("--dropout", type=float, default=0.1)
796
+ parser.add_argument("--patience", type=int, default=30)
797
+
798
+ # PerformerModule related paths
799
+ parser.add_argument(
800
+ "--checkpoint_path",
801
+ type=str,
802
+ required=True,
803
+ help="Path to pre-trained PerformerModule checkpoint"
804
+ )
805
+ parser.add_argument(
806
+ "--hyper_params_path",
807
+ type=str,
808
+ required=True,
809
+ help="Path to PerformerModule model hyper parameters pkl file"
810
+ )
811
+ parser.add_argument(
812
+ "--module_data_dir",
813
+ type=str,
814
+ default=None,
815
+ help="Directory containing module JSON files"
816
+ )
817
+ parser.add_argument(
818
+ "--module_data_path",
819
+ type=str,
820
+ default=None,
821
+ help="Path to module JSON file (single file)"
822
+ )
823
+ parser.add_argument(
824
+ "--gene_list_path",
825
+ type=str,
826
+ default=str(ASSET_DIR / "selected_genes_27k.txt"),
827
+ help="Path to gene list file"
828
+ )
829
+ parser.add_argument(
830
+ "--gene_mapping_path",
831
+ type=str,
832
+ default=str(ASSET_DIR / "gene2ens.csv"),
833
+ help="Path to gene symbol to Ensembl ID mapping"
834
+ )
835
+ parser.add_argument(
836
+ "--gene2vec_path",
837
+ type=str,
838
+ default=str(ASSET_DIR / "selected_gene2vec_27k.npy"),
839
+ help="Path to gene2vec file"
840
+ )
841
+ parser.add_argument(
842
+ "--symbol2ens_path",
843
+ type=str,
844
+ default=str(ASSET_DIR / "gene2ens.csv"),
845
+ help="Path to symbol to Ensembl mapping (for data processing)"
846
+ )
847
+ parser.add_argument(
848
+ "--pool_method",
849
+ type=str,
850
+ default="mean",
851
+ choices=["mean", "max", "cls", "concat"],
852
+ help="Pooling method for PerformerModule module embeddings"
853
+ )
854
+ parser.add_argument(
855
+ "--shuffle_module_genes",
856
+ action="store_true",
857
+ help="Optional downstream-only extra reshuffle on top of module_sets restored from PKL. "
858
+ "Do not pass this when reproducing the pretrained backbone as-is."
859
+ )
860
+ parser.add_argument(
861
+ "--shuffle_seed",
862
+ type=int,
863
+ default=2021,
864
+ help="Random seed for the downstream extra reshuffle when --shuffle_module_genes is enabled."
865
+ )
866
+ # Output path
867
+ parser.add_argument(
868
+ "--save_dir",
869
+ type=str,
870
+ default="./models_drug_performer_module",
871
+ help="Directory to save the model"
872
+ )
873
+
874
+ args = parser.parse_args()
875
+
876
+ os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu_id
877
+ device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
878
+
879
+ print_sys("=" * 70)
880
+ print_sys("Drug Response Prediction: Drug + PerformerModule")
881
+ print_sys("=" * 70)
882
+ print_sys(f"Device: {device}")
883
+ print_sys(f"Epochs: {args.epochs}")
884
+ print_sys(f"Batch size: {args.batch_size}")
885
+ print_sys(f"Learning rate: {args.learning_rate}")
886
+ print_sys(f"Hidden dim: {args.hidden_dim}")
887
+ print_sys(f"Dropout: {args.dropout}")
888
+ print_sys(f"Pool method: {args.pool_method}")
889
+ print_sys(f"Early stopping patience: {args.patience}")
890
+ print_sys("=" * 70)
891
+
892
+ # Create save directory
893
+ os.makedirs(args.save_dir, exist_ok=True)
894
+
895
+ dpath = args.data_path
896
+ drug_info_file = f"{dpath}/GDSC/1.Drug_listMon Jun 24 09_00_55 2019.csv"
897
+ cell_line_info_file = f"{dpath}/CCLE/Cell_lines_annotations_20181226.txt"
898
+ drug_feature_dir = f"{dpath}/GDSC/drug_graph_feat"
899
+ gene_expression_file = (
900
+ f"{dpath}/CCLE/genomic_expression_561celllines_697genes_demap_features.csv"
901
+ )
902
+
903
+ # fixed random seed
904
+ random.seed(0)
905
+ np.random.seed(0)
906
+ torch.manual_seed(0)
907
+ if torch.cuda.is_available():
908
+ torch.cuda.manual_seed_all(0)
909
+
910
+ # 1) Read metadata
911
+ print_sys("\n[Step 1/6] Loading metadata...")
912
+ drug_feature, gexpr_feature, data_idx = metadata_generate(
913
+ drug_info_file,
914
+ cell_line_info_file,
915
+ drug_feature_dir,
916
+ gene_expression_file,
917
+ )
918
+
919
+ # 2) Build PerformerModule embeddings
920
+ print_sys("\n[Step 2/6] Building PerformerModule embeddings...")
921
+ cell_emb_dict, cell_emb_dim = build_cell_embeddings_with_performer_module( # cell_emb_dict │ Dict[str, np.ndarray] │ {cell_line_id: embedding} mapping dictionary
922
+ gexpr_feature=gexpr_feature,
923
+ symbol2ens_path=args.symbol2ens_path,
924
+ gene_list_path=args.gene_list_path,
925
+ gene_mapping_path=args.gene_mapping_path,
926
+ gene2vec_path=args.gene2vec_path,
927
+ checkpoint_path=args.checkpoint_path,
928
+ hyper_params_path=args.hyper_params_path,
929
+ module_data_dir=args.module_data_dir,
930
+ module_data_path=args.module_data_path,
931
+ device=device,
932
+ pool_method=args.pool_method,
933
+ shuffle_module_genes=args.shuffle_module_genes,
934
+ seed=args.shuffle_seed,
935
+ )
936
+
937
+ # 3) Split training and validation data; validation is used for model selection
938
+ print_sys("\n[Step 3/6] Splitting dataset...")
939
+ train_idx, validation_idx = data_split(data_idx)
940
+ print_sys(f"Train samples: {len(train_idx)}, Validation samples: {len(validation_idx)}")
941
+
942
+ # 4) Extract features
943
+ print_sys("\n[Step 4/6] Extracting features...")
944
+ X_drug_train, Y_train, train_cancer_info, train_index_list = feature_extract(
945
+ train_idx, drug_feature, args.max_atoms
946
+ )
947
+ X_drug_validation, Y_validation, validation_cancer_info, validation_index_list = feature_extract(
948
+ validation_idx, drug_feature, args.max_atoms
949
+ )
950
+
951
+ # 5) Build a data set
952
+ print_sys("\n[Step 5/6] Creating datasets...")
953
+ train_ds = DrugPerformerModuleDataset(
954
+ X_drug_train, Y_train, train_index_list, cell_emb_dict
955
+ )
956
+ validation_ds = DrugPerformerModuleDataset(
957
+ X_drug_validation, Y_validation, validation_index_list, cell_emb_dict
958
+ )
959
+
960
+ drug_dim = X_drug_train[0][0].shape[-1]
961
+
962
+ num_workers = min(4, os.cpu_count() or 1)
963
+ train_loader = DataLoader(
964
+ train_ds,
965
+ batch_size=args.batch_size,
966
+ shuffle=True,
967
+ num_workers=num_workers,
968
+ pin_memory=torch.cuda.is_available(),
969
+ )
970
+ validation_loader = DataLoader(
971
+ validation_ds,
972
+ batch_size=args.batch_size,
973
+ shuffle=False,
974
+ num_workers=num_workers,
975
+ pin_memory=torch.cuda.is_available(),
976
+ )
977
+
978
+ # 6) Build the model
979
+ print_sys("\n[Step 6/6] Building model...")
980
+ model = DrugPerformerModuleModel(
981
+ drug_feat_dim=drug_dim,
982
+ cell_emb_dim=cell_emb_dim,
983
+ hidden_dim=args.hidden_dim,
984
+ dropout=args.dropout,
985
+ ).to(device)
986
+
987
+ # Statistical parameters
988
+ total_params = sum(p.numel() for p in model.parameters())
989
+ trainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad)
990
+ print_sys(f"Total parameters: {total_params:,}")
991
+ print_sys(f"Trainable parameters: {trainable_params:,}")
992
+
993
+ # Optimizers and schedulers
994
+ optimizer = torch.optim.AdamW(
995
+ model.parameters(), lr=args.learning_rate, weight_decay=1e-5
996
+ )
997
+ scheduler = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(
998
+ optimizer, T_0=20, T_mult=2, eta_min=1e-6
999
+ )
1000
+ criterion = torch.nn.HuberLoss(delta=1.0)
1001
+
1002
+ # train
1003
+ print_sys("\n" + "=" * 70)
1004
+ print_sys("Starting training...")
1005
+ print_sys("=" * 70)
1006
+
1007
+ best_pcc = -1.0
1008
+ best_pred = None
1009
+ best_true = None
1010
+ best_epoch = 0
1011
+ patience_counter = 0
1012
+
1013
+ for epoch in range(1, args.epochs + 1):
1014
+ train_loss = train_epoch(model, train_loader, optimizer, criterion, device)
1015
+ preds, trues, pcc = eval_epoch(model, validation_loader, device)
1016
+
1017
+ scheduler.step()
1018
+ current_lr = optimizer.param_groups[0]["lr"]
1019
+
1020
+ print_sys(
1021
+ f"Epoch {epoch:03d}/{args.epochs} | "
1022
+ f"train_loss={train_loss:.4f} | "
1023
+ f"validation_pcc={pcc:.4f} | "
1024
+ f"lr={current_lr:.6f}"
1025
+ )
1026
+
1027
+ if pcc > best_pcc:
1028
+ best_pcc = pcc
1029
+ best_pred = preds
1030
+ best_true = trues
1031
+ best_epoch = epoch
1032
+ patience_counter = 0
1033
+
1034
+ # Save the best model selected by validation PCC
1035
+ model_save_path = os.path.join(args.save_dir, "best_model.pth")
1036
+ torch.save(
1037
+ {
1038
+ "epoch": epoch,
1039
+ "model_state_dict": model.state_dict(),
1040
+ "optimizer_state_dict": optimizer.state_dict(),
1041
+ "scheduler_state_dict": scheduler.state_dict(),
1042
+ "best_pcc": best_pcc,
1043
+ "train_loss": train_loss,
1044
+ "args": vars(args),
1045
+ },
1046
+ model_save_path,
1047
+ )
1048
+ print_sys(f" → Best model saved! Validation PCC improved to {best_pcc:.4f}")
1049
+ else:
1050
+ patience_counter += 1
1051
+ if patience_counter >= args.patience:
1052
+ print_sys(
1053
+ f"\n[Early Stopping] No improvement for {args.patience} epochs."
1054
+ )
1055
+ break
1056
+
1057
+ # Save final result
1058
+ print_sys("\n" + "=" * 70)
1059
+ print_sys("Training completed!")
1060
+ print_sys("=" * 70)
1061
+ print_sys(f"Best epoch: {best_epoch}")
1062
+ print_sys(f"Best validation PCC: {best_pcc:.4f}")
1063
+
1064
+ if best_pred is not None:
1065
+ result_path = os.path.join(args.save_dir, "predictions.npz")
1066
+ np.savez(result_path, pred=best_pred, validation=best_true)
1067
+ print_sys(f"Validation predictions saved to: {result_path}")
1068
+
1069
+ print_sys(f"Best model saved to: {os.path.join(args.save_dir, 'best_model.pth')}")
1070
+ print_sys("=" * 70)
1071
+
1072
+
1073
+ if __name__ == "__main__":
1074
+ main()
1075
+
BioMod/gene_perturbation/README.md CHANGED
@@ -1,267 +1,276 @@
1
- # Gene Perturbation Prediction with BioMod
2
-
3
- This document describes the BioMod gene perturbation workflow (GEARS), using
4
- the `PerformerEncoderOnly_Module` encoder. The
5
- [BioMod model weights and matching configuration](../../README.md#biomod-model-weights)
6
- are shared with the other downstream tasks.
7
-
8
- ## Overview
9
-
10
- This adaptation replaces the original GEARS gene perturbation model based on
11
- `DualEncoderSCFM` with `PerformerEncoderOnly_Module`. The main adaptation points
12
- are:
13
-
14
- 1. **Model architecture adaptation**: replace the original dual-encoder model with a module-based encoder-only model.
15
- 2. **Data format adaptation**: handle module-based input data.
16
- 3. **Embedding extraction adaptation**: recover gene-level embeddings from module encodings.
17
-
18
- ## Main Adaptations
19
-
20
- ### 1. `SCFM_Module` (`model_module.py`)
21
-
22
- `SCFM_Module` wraps `PerformerEncoderOnly_Module` and provides these functions:
23
-
24
- - **Input handling**: reindex input gene expression values to the BioMod gene order.
25
- - **Module encoding**: obtain module-level encodings from `PerformerEncoderOnly_Module`.
26
- - **Gene embedding recovery**: recover gene-level embeddings from module encodings.
27
- - For each gene, find all modules that contain it.
28
- - Average the encodings of those modules to obtain the gene embedding.
29
- - If a gene is not included in any module, use the global average as a fallback.
30
- - **Output reindexing**: reindex gene embeddings back to the input gene order.
31
-
32
- **Key adaptation**:
33
-
34
- ```python
35
- # Recover gene-level embeddings from module encodings.
36
- for gene_idx in range(len(self.scfm_index_positions) - 1):
37
- if gene_idx in self.gene_to_modules:
38
- module_indices = self.gene_to_modules[gene_idx]
39
- module_embs = module_encodings[:, module_indices, :] # B, num_modules_for_gene, D
40
- gene_emb = module_embs.mean(dim=1) # B, D
41
- else:
42
- gene_emb = module_encodings.mean(dim=1) # B, D (global average)
43
- ```
44
-
45
- ### 2. `GEARS_Model_Module` (`model_module.py`)
46
-
47
- `GEARS_Model_Module` inherits the original GEARS architecture. Main changes:
48
-
49
- - **Use `SCFM_Module`** instead of the original `SCFM_different_genes`.
50
- - **Module data loading** supports loading module data from hyper-parameters or file paths.
51
- - **Other components remain unchanged**, including the GNN layers and decoder.
52
-
53
- ### 3. Training Script (`train_module.py`)
54
-
55
- The training script is based on `GEARS/train_dual_ddp_reduce_dec.py`. Main changes:
56
-
57
- - **Model class**: use `GEARS_Model_Module` instead of `GEARS_Model_Acc`.
58
- - **New arguments**:
59
- - `--module_data_path`: path to one module JSON file.
60
- - `--module_data_dir`: directory containing module JSON files.
61
- - `--gene_mapping_path`: CSV file mapping gene symbols to Ensembl IDs.
62
-
63
- ## Usage
64
-
65
- ### 1. Prepare Data
66
-
67
- Make sure the following files exist:
68
-
69
- - Perturbation data, such as Norman, Adamson, or Dixit.
70
- - BioMod model weights and their matching configuration file.
71
- - Module data in JSON format.
72
- - Gene mapping file, such as `gene2ens.csv`.
73
-
74
- ### 2. Run Training
75
-
76
- ```bash
77
- # DDP training on multiple GPUs.
78
- python -m torch.distributed.launch --nproc_per_node=4 \
79
- train_module.py \
80
- --data_dir /path/to/data \
81
- --data_name norman \
82
- --scfm_hyper_params_path /path/to/hyper_params.pkl \
83
- --scfm_ckpt_path /path/to/checkpoint.pth \
84
- --module_data_dir /path/to/module_data \
85
- --gene_mapping_path /path/to/gene2ens.csv \
86
- --scfm_genes_list_path /path/to/selected_genes_27k.txt \
87
- --scfm_gene2vec_file /path/to/selected_gene2vec_27k.npy \
88
- --result_dir ./results_module \
89
- --run_name norman_module_experiment \
90
- --epochs 20 \
91
- --batch_size 32 \
92
- --lr 1e-3
93
- ```
94
-
95
- ### 3. Arguments
96
-
97
- **Required arguments**:
98
-
99
- - `--scfm_hyper_params_path`: path to the BioMod model configuration file.
100
- - `--scfm_ckpt_path`: path to the BioMod model weights.
101
- - `--module_data_path` or `--module_data_dir`: module data path; provide one of them.
102
-
103
- **Recommended arguments**:
104
-
105
- - `--gene_mapping_path`: gene mapping file, especially when module data use gene symbols.
106
- - `--scfm_genes_list_path`: BioMod gene list file.
107
- - `--scfm_gene2vec_file`: path to the gene2vec file.
108
-
109
- **Optional arguments**:
110
-
111
- - `--use_module_level`: use module-level features. This is experimental and is not fully supported at the moment.
112
- - Default: False, meaning gene-level features are used and recommended.
113
- - See `METHODS_COMPARISON.md` for details.
114
-
115
- ## Comparison with Original sclong
116
-
117
- ### Main Differences
118
-
119
- 1. **Model architecture**:
120
- - **Original sclong**: uses `DualEncoderSCFM` and directly processes gene sequences.
121
- - **Module version**: uses `PerformerEncoderOnly_Module` and processes module-based data.
122
-
123
- 2. **Data representation**:
124
- - **Original sclong**: gene-level embeddings come directly from encoder outputs.
125
- - **Module version**: gene-level embeddings are recovered from module encodings by mean pooling.
126
-
127
- 3. **Input format**:
128
- - **Original sclong**: directly inputs gene expression vectors.
129
- - **Module version**: keeps the same external input format but internally processes data with module embeddings.
130
-
131
- ### Performance Comparison
132
-
133
- To compare model performance:
134
-
135
- 1. **Run original sclong**:
136
-
137
- ```bash
138
- cd GEARS
139
- python train_dual_ddp_reduce_dec.py [arguments...]
140
- ```
141
-
142
- 2. **Run the module version**:
143
-
144
- ```bash
145
- cd performer_module/gene_perturbation
146
- python train_module.py [arguments...]
147
- ```
148
-
149
- 3. **Compare metrics**:
150
-
151
- - MSE (Mean Squared Error)
152
- - Pearson correlation
153
- - MSE and Pearson correlation for the top 20 DE genes
154
- - Direction accuracy and related metrics
155
-
156
- Results are saved under each `result_dir`. Compare these metrics to evaluate the
157
- module version.
158
-
159
- ## Technical Details
160
-
161
- ### Module-to-Gene Mapping
162
-
163
- In `SCFM_Module`, `gene_to_modules` records which modules contain each gene:
164
-
165
- ```python
166
- self.gene_to_modules = {} # {gene_idx: [module_idx1, module_idx2, ...]}
167
- for module_idx, module_genes in enumerate(module_sets):
168
- for gene_idx in module_genes:
169
- if gene_idx < len(scfm_genes):
170
- if gene_idx not in self.gene_to_modules:
171
- self.gene_to_modules[gene_idx] = []
172
- self.gene_to_modules[gene_idx].append(module_idx)
173
- ```
174
-
175
- ### Gene Embedding Recovery Strategy
176
-
177
- For each gene:
178
-
179
- 1. If the gene appears in some modules, average the encodings of those modules.
180
- 2. If the gene does not appear in any module, use the global average of all modules as a fallback.
181
-
182
- This strategy:
183
-
184
- - Makes full use of module information.
185
- - Provides a reasonable embedding for genes absent from all modules.
186
- - Keeps embedding dimensionality similar to the original model.
187
-
188
- ## File Structure
189
-
190
- ```text
191
- BioMod/gene_perturbation/ # Perturbation response prediction: processing, training and inference
192
- ├── data_utils_local.py
193
- ├── gears_reduce_local.py
194
- ├── inference_local.py
195
- ├── model_module.py
196
- ├── pertdata_local.py
197
- ├── README.md
198
- ├── scfm_utils_local.py
199
- ├── train_module.py
200
- └── utils.py
201
- ```
202
-
203
- ## Notes
204
-
205
- 1. **Module data format**: make sure the gene indices or names in the module data match the BioMod gene list.
206
- 2. **Gene mapping**: provide `gene_mapping_path` if the module data use gene symbols.
207
- 3. **Memory usage**: the module version may require more memory because it maintains the module-to-gene mapping.
208
- 4. **Training stability**: use the same learning rate and training strategy as original sclong when possible.
209
-
210
- ## Troubleshooting
211
-
212
- ### Issue 1: Module Data Not Found
213
-
214
- - Check whether `--module_data_path` or `--module_data_dir` is correct.
215
- - Make sure the module data file exists and has the expected format.
216
-
217
- ### Issue 2: Gene Mapping Failed
218
-
219
- - Check whether `--gene_mapping_path` is correct.
220
- - Make sure the gene mapping file is a CSV containing `Approved symbol` and `Ensembl gene ID` columns.
221
-
222
- ### Issue 3: Model Loading Failed
223
-
224
- - Check whether the checkpoint path is correct.
225
- - Make sure `hyper_params` contains the required module information.
226
-
227
- ## Gene-Based vs Module-Based Methods
228
-
229
- This implementation supports two feature extraction modes controlled by
230
- `--use_module_level`.
231
-
232
- ### Method 1: Gene-Based (Default, Recommended)
233
-
234
- ```bash
235
- # Default mode; do not pass --use_module_level or set it to False.
236
- python train_module.py [other arguments...]
237
- ```
238
-
239
- **Characteristics**:
240
-
241
- - Recovers gene-level embeddings from module encodings.
242
- - Fully compatible with the original GEARS architecture.
243
- - Downstream tasks operate at gene level, including the GNN and decoder.
244
- - Outputs gene-level predictions.
245
-
246
- ### Method 2: Module-Based (Experimental, Not Fully Implemented)
247
-
248
- ```bash
249
- # Enable module-level mode.
250
- python train_module.py --use_module_level [other arguments...]
251
- ```
252
-
253
- **Characteristics**:
254
-
255
- - Directly uses module-level encodings.
256
- - Preserves the full module-level information.
257
- - **Note**: module-level downstream tasks are not fully implemented in the current version.
258
- - Requires redesigning the downstream architecture, including the GNN and decoder.
259
-
260
- See `METHODS_COMPARISON.md` for the detailed comparison and rationale.
261
-
262
- ## References
263
-
264
- - Original GEARS code: `GEARS/gears_reduce/`
265
- - PerformerEncoderOnly_Module model: `performer_module/models_performer_module.py`
266
- - Module data processing: `performer_module/module_dataset.py`
267
- - Method comparison: `METHODS_COMPARISON.md`
 
 
 
 
 
 
 
 
 
 
1
+ # Gene Perturbation Prediction with BioMod
2
+
3
+ This document describes the BioMod gene perturbation workflow (GEARS), using
4
+ the `PerformerEncoderOnly_Module` encoder. The
5
+ [BioMod model weights and matching configuration](../../README.md#biomod-model-weights)
6
+ are shared with the other downstream tasks.
7
+
8
+ ## Overview
9
+
10
+ This adaptation replaces the original GEARS gene perturbation model based on
11
+ `DualEncoderSCFM` with `PerformerEncoderOnly_Module`. The main adaptation points
12
+ are:
13
+
14
+ 1. **Model architecture adaptation**: replace the original dual-encoder model with a module-based encoder-only model.
15
+ 2. **Data format adaptation**: handle module-based input data.
16
+ 3. **Embedding extraction adaptation**: recover gene-level embeddings from module encodings.
17
+
18
+ ## Main Adaptations
19
+
20
+ ### 1. `SCFM_Module` (`model_module.py`)
21
+
22
+ `SCFM_Module` wraps `PerformerEncoderOnly_Module` and provides these functions:
23
+
24
+ - **Input handling**: reindex input gene expression values to the BioMod gene order.
25
+ - **Module encoding**: obtain module-level encodings from `PerformerEncoderOnly_Module`.
26
+ - **Gene embedding recovery**: recover gene-level embeddings from module encodings.
27
+ - For each gene, find all modules that contain it.
28
+ - Average the encodings of those modules to obtain the gene embedding.
29
+ - If a gene is not included in any module, use the global average as a fallback.
30
+ - **Output reindexing**: reindex gene embeddings back to the input gene order.
31
+
32
+ **Key adaptation**:
33
+
34
+ ```python
35
+ # Recover gene-level embeddings from module encodings.
36
+ for gene_idx in range(len(self.scfm_index_positions) - 1):
37
+ if gene_idx in self.gene_to_modules:
38
+ module_indices = self.gene_to_modules[gene_idx]
39
+ module_embs = module_encodings[:, module_indices, :] # B, num_modules_for_gene, D
40
+ gene_emb = module_embs.mean(dim=1) # B, D
41
+ else:
42
+ gene_emb = module_encodings.mean(dim=1) # B, D (global average)
43
+ ```
44
+
45
+ ### 2. `GEARS_Model_Module` (`model_module.py`)
46
+
47
+ `GEARS_Model_Module` inherits the original GEARS architecture. Main changes:
48
+
49
+ - **Use `SCFM_Module`** instead of the original `SCFM_different_genes`.
50
+ - **Module data loading** supports loading module data from hyper-parameters or file paths.
51
+ - **Other components remain unchanged**, including the GNN layers and decoder.
52
+
53
+ ### 3. Training Script (`train_module.py`)
54
+
55
+ The training script is based on `GEARS/train_dual_ddp_reduce_dec.py`. Main changes:
56
+
57
+ - **Model class**: use `GEARS_Model_Module` instead of `GEARS_Model_Acc`.
58
+ - **New arguments**:
59
+ - `--module_data_path`: path to one module JSON file.
60
+ - `--module_data_dir`: directory containing module JSON files.
61
+ - `--gene_mapping_path`: CSV file mapping gene symbols to Ensembl IDs.
62
+
63
+ ## Usage
64
+
65
+ ### 1. Prepare Data
66
+
67
+ Make sure the following files exist:
68
+
69
+ - Perturbation data, such as Norman, Adamson, or Dixit.
70
+ - BioMod model weights and their matching configuration file.
71
+ - Module data in JSON format.
72
+ - Gene mapping file, such as `gene2ens.csv`.
73
+
74
+ ### 2. Run Training
75
+
76
+ Run from the repository root in a Linux CUDA/NCCL environment. Built-in asset defaults are derived from the script location; explicit relative paths resolve from the working directory.
77
+
78
+ ```bash
79
+ # DDP training on multiple GPUs.
80
+ python -m torch.distributed.run --standalone --nproc_per_node=4 \
81
+ BioMod/gene_perturbation/train_module.py \
82
+ --data_dir data/downstream/final_data \
83
+ --data_name norman \
84
+ --scfm_hyper_params_path weight/0810model/mae_tost_70M_75mask_ec2.pkl \
85
+ --scfm_ckpt_path weight/0810model/mae_tost_70M_75mask_ec2_epoch0.pth \
86
+ --module_data_dir model_dependencies/module_data_all \
87
+ --gene_mapping_path model_dependencies/gene2ens.csv \
88
+ --scfm_genes_list_path model_dependencies/selected_genes_27k.txt \
89
+ --scfm_gene2vec_file model_dependencies/selected_gene2vec_27k.npy \
90
+ --result_dir tutorial_runs/gene_perturbation \
91
+ --run_name norman_module_experiment \
92
+ --epochs 20 \
93
+ --batch_size 32 \
94
+ --lr 1e-3
95
+ ```
96
+
97
+ ### 3. Arguments
98
+
99
+ **Model arguments** (defaults point to the bundled BioMod assets):
100
+
101
+ - `--scfm_hyper_params_path`: path to the BioMod model configuration file.
102
+ - `--scfm_ckpt_path`: path to the BioMod model weights.
103
+ - `--module_data_path` or `--module_data_dir`: required only when the model configuration does not already contain module definitions.
104
+
105
+ **Recommended arguments**:
106
+
107
+ - `--gene_mapping_path`: gene mapping file, especially when module data use gene symbols.
108
+ - `--scfm_genes_list_path`: BioMod gene list file.
109
+ - `--scfm_gene2vec_file`: path to the gene2vec file.
110
+
111
+ **Optional arguments**:
112
+
113
+ - `--use_module_level`: use module-level features. This is experimental and is not fully supported at the moment.
114
+ - Default: False, meaning gene-level features are used and recommended.
115
+
116
+ ## Comparison with Original sclong
117
+
118
+ ### Main Differences
119
+
120
+ 1. **Model architecture**:
121
+ - **Original sclong**: uses `DualEncoderSCFM` and directly processes gene sequences.
122
+ - **Module version**: uses `PerformerEncoderOnly_Module` and processes module-based data.
123
+
124
+ 2. **Data representation**:
125
+ - **Original sclong**: gene-level embeddings come directly from encoder outputs.
126
+ - **Module version**: gene-level embeddings are recovered from module encodings by mean pooling.
127
+
128
+ 3. **Input format**:
129
+ - **Original sclong**: directly inputs gene expression vectors.
130
+ - **Module version**: keeps the same external input format but internally processes data with module embeddings.
131
+
132
+ ### Performance Comparison
133
+
134
+ To compare model performance:
135
+
136
+ 1. **Run original sclong**:
137
+
138
+ ```bash
139
+ # Requires a separate checkout of the original scLong/GEARS code.
140
+ python /path/to/upstream/GEARS/train_dual_ddp_reduce_dec.py [arguments...]
141
+ ```
142
+
143
+ 2. **Run the module version**:
144
+
145
+ ```bash
146
+ # Run from the BioMod repository root.
147
+ python -m torch.distributed.run --standalone --nproc_per_node=1 BioMod/gene_perturbation/train_module.py [arguments...]
148
+ ```
149
+
150
+ 3. **Compare metrics**:
151
+
152
+ - MSE (Mean Squared Error)
153
+ - Pearson correlation
154
+ - MSE and Pearson correlation for the top 20 DE genes
155
+ - Direction accuracy and related metrics
156
+
157
+ Results are saved under each `result_dir`. Compare these metrics to evaluate the
158
+ module version.
159
+
160
+ ## Technical Details
161
+
162
+ ### Module-to-Gene Mapping
163
+
164
+ In `SCFM_Module`, `gene_to_modules` records which modules contain each gene:
165
+
166
+ ```python
167
+ self.gene_to_modules = {} # {gene_idx: [module_idx1, module_idx2, ...]}
168
+ for module_idx, module_genes in enumerate(module_sets):
169
+ for gene_idx in module_genes:
170
+ if gene_idx < len(scfm_genes):
171
+ if gene_idx not in self.gene_to_modules:
172
+ self.gene_to_modules[gene_idx] = []
173
+ self.gene_to_modules[gene_idx].append(module_idx)
174
+ ```
175
+
176
+ ### Gene Embedding Recovery Strategy
177
+
178
+ For each gene:
179
+
180
+ 1. If the gene appears in some modules, average the encodings of those modules.
181
+ 2. If the gene does not appear in any module, use the global average of all modules as a fallback.
182
+
183
+ This strategy:
184
+
185
+ - Makes full use of module information.
186
+ - Provides a reasonable embedding for genes absent from all modules.
187
+ - Keeps embedding dimensionality similar to the original model.
188
+
189
+ ## File Structure
190
+
191
+ ```text
192
+ BioMod/gene_perturbation/ # Perturbation response prediction: processing, training and inference
193
+ ├── data_utils_local.py
194
+ ├── gears_reduce_local.py
195
+ ├── inference_local.py
196
+ ├── model_module.py
197
+ ├── pertdata_local.py
198
+ ├── README.md
199
+ ├── scfm_utils_local.py
200
+ ├── train_module.py
201
+ └── utils.py
202
+ ```
203
+
204
+ ## Notes
205
+
206
+ 1. **Module data format**: make sure the gene indices or names in the module data match the BioMod gene list.
207
+ 2. **Gene mapping**: provide `gene_mapping_path` if the module data use gene symbols.
208
+ 3. **Memory usage**: the module version may require more memory because it maintains the module-to-gene mapping.
209
+ 4. **Training stability**: use the same learning rate and training strategy as original sclong when possible.
210
+
211
+ ## Troubleshooting
212
+
213
+ ### Issue 1: Module Data Not Found
214
+
215
+ - Check whether `--module_data_path` or `--module_data_dir` is correct.
216
+ - Make sure the module data file exists and has the expected format.
217
+
218
+ ### Issue 2: Gene Mapping Failed
219
+
220
+ - Check whether `--gene_mapping_path` is correct.
221
+ - Make sure the gene mapping file is a CSV containing `Approved symbol` and `Ensembl gene ID` columns.
222
+
223
+ ### Issue 3: Model Loading Failed
224
+
225
+ - Check whether the checkpoint path is correct.
226
+ - Make sure `hyper_params` contains the required module information.
227
+
228
+ ## Gene-Based vs Module-Based Methods
229
+
230
+ This implementation supports two feature extraction modes controlled by
231
+ `--use_module_level`.
232
+
233
+ ### Method 1: Gene-Based (Default, Recommended)
234
+
235
+ ```bash
236
+ # Default mode; do not pass --use_module_level or set it to False.
237
+ python -m torch.distributed.run --standalone --nproc_per_node=1 BioMod/gene_perturbation/train_module.py [other arguments...]
238
+ ```
239
+
240
+ **Characteristics**:
241
+
242
+ - Recovers gene-level embeddings from module encodings.
243
+ - Fully compatible with the original GEARS architecture.
244
+ - Downstream tasks operate at gene level, including the GNN and decoder.
245
+ - Outputs gene-level predictions.
246
+
247
+ ### Method 2: Module-Based (Experimental, Not Fully Implemented)
248
+
249
+ ```bash
250
+ # Enable module-level mode.
251
+ python -m torch.distributed.run --standalone --nproc_per_node=1 BioMod/gene_perturbation/train_module.py --use_module_level [other arguments...]
252
+ ```
253
+
254
+ **Characteristics**:
255
+
256
+ - Directly uses module-level encodings.
257
+ - Preserves the full module-level information.
258
+ - **Note**: module-level downstream tasks are not fully implemented in the current version.
259
+ - Requires redesigning the downstream architecture, including the GNN and decoder.
260
+
261
+
262
+ ## References
263
+
264
+ - Original GEARS code is an external reference and is not included in this source directory.
265
+ - PerformerEncoderOnly_Module model: `BioMod/models_performer_module.py`
266
+ - Module data processing: `BioMod/module_dataset.py`
267
+
268
+ ## Optional external annotation resources
269
+
270
+ The bundled Norman data include `go.csv`. If that network must be regenerated,
271
+ provide `gene2go_path` to `get_go_auto` or set `BIOMOD_GENE2GO_PATH` to the
272
+ appropriate `gene2go_all.pkl` file. This annotation file is not bundled.
273
+
274
+ The optional `get_high_umi_idx` helper requires `high_umi_path` or
275
+ `BIOMOD_HIGH_UMI_PATH` pointing to `genes_with_hi_mean.npy`; that resource is also
276
+ not bundled. Neither helper silently searches an old workstation path.
BioMod/gene_perturbation/gears_reduce_local.py CHANGED
@@ -459,7 +459,7 @@ def predict(self, pert_list):
459
  else:
460
  return results_pred
461
 
462
- def GI_predict(self, combo, GI_genes_file='./genes_with_hi_mean.npy'):
463
  """
464
  Predict the genetic interaction effect of gene pairs
465
 
@@ -472,8 +472,14 @@ def GI_predict(self, combo, GI_genes_file='./genes_with_hi_mean.npy'):
472
 
473
  [parameter]
474
  - combo: gene pair, such as ('MAPK1', 'KRAS')
475
- - GI_genes_file: Gene list file used for GI analysis
476
  """
 
 
 
 
 
 
477
  try:
478
  # If there is already a prediction, use it directly
479
  pred = {}
 
459
  else:
460
  return results_pred
461
 
462
+ def GI_predict(self, combo, GI_genes_file=None):
463
  """
464
  Predict the genetic interaction effect of gene pairs
465
 
 
472
 
473
  [parameter]
474
  - combo: gene pair, such as ('MAPK1', 'KRAS')
475
+ - GI_genes_file: Caller-supplied gene list used for GI analysis; alternatively set BIOMOD_HIGH_UMI_PATH.
476
  """
477
+ GI_genes_file = GI_genes_file or os.environ.get('BIOMOD_HIGH_UMI_PATH')
478
+ if not GI_genes_file or not os.path.isfile(GI_genes_file):
479
+ raise FileNotFoundError(
480
+ "Provide GI_genes_file or set BIOMOD_HIGH_UMI_PATH to the "
481
+ "genes_with_hi_mean.npy resource; this file is not bundled."
482
+ )
483
  try:
484
  # If there is already a prediction, use it directly
485
  pred = {}
BioMod/gene_perturbation/inference_local.py CHANGED
@@ -1,1034 +1,1038 @@
1
- """
2
- ================================================================================
3
- Model evaluation module (inference_local.py)
4
- ================================================================================
5
-
6
- [Module function]
7
- Provides evaluation and analysis functions for genetic perturbation models, including:
8
-
9
- 1. evaluate(): main evaluation function, calculates MSE, Pearson correlation coefficient and other indicators
10
- 2. compute_metrics(): Calculate the complete dictionary of evaluation metrics
11
- 3. deeper_analysis(): In-depth analysis of prediction results
12
- 4. non_dropout_analysis(): analyze on non-dropout genes
13
-
14
- [Description of key indicators]
15
- - MSE: mean square error of all genes
16
- - MSE_DE: MSE of Top 20 differentially expressed genes (paying more attention to biological significance)
17
- - Pearson: Pearson correlation coefficient of all genes
18
- - Pearson_DE: Pearson correlation coefficient of Top 20 DE genes
19
-
20
- [Evaluation Notes]
21
- - real_num_genes: only evaluate on real genes, excluding extended/completed genes
22
- - de_idx: The differentially expressed gene index of each sample may be different
23
- ================================================================================
24
- """
25
-
26
- import torch
27
- import numpy as np
28
- import scanpy as sc
29
-
30
- from sklearn.metrics import r2_score
31
- from scipy.stats import pearsonr, spearmanr
32
- from sklearn.metrics import mean_squared_error as mse
33
- from sklearn.metrics import mean_absolute_error as mae
34
-
35
- def evaluate(loader, model, uncertainty, device, real_num_genes = None):
36
- """
37
- main evaluation function
38
-
39
- [Function]
40
- Runs the model on the given data loader, collecting predicted and true values.
41
-
42
- [Key steps]
43
- 1. Traverse all batches of the data loader
44
- 2. Run model inference (without calculating gradients)
45
- 3. Collect predicted values, true values, and DE gene predicted values
46
-
47
- [Parameter description]
48
- - real_num_genes: only evaluate the first N real genes to avoid expanding gene interference indicators
49
- """
50
-
51
- model.eval()
52
- model.to(device)
53
- pert_cat = []
54
- pred = []
55
- truth = []
56
- pred_de = []
57
- truth_de = []
58
- results = {}
59
- logvar = []
60
-
61
- for itr, batch in enumerate(loader):
62
-
63
- batch.to(device)
64
- pert_cat.extend(batch.pert)
65
-
66
- with torch.no_grad():
67
- if uncertainty:
68
- p, unc = model(batch)
69
- logvar.extend(unc.cpu())
70
- else:
71
- p = model(batch)
72
- # Only evaluate in the real gene dimension to avoid the interference of completed/expanded genes on indicators.
73
- p = p[:, :real_num_genes]
74
- t = batch.y[:, :real_num_genes]
75
- pred.extend(p.cpu())
76
- truth.extend(t.cpu())
77
-
78
- # The DE indicator is only counted on the de_idx corresponding to the sample and is one of the core calibers of downstream reporting.
79
- for itr, de_idx in enumerate(batch.de_idx):
80
- pred_de.append(p[itr, de_idx])
81
- truth_de.append(t[itr, de_idx])
82
-
83
- # all genes
84
- results['pert_cat'] = np.array(pert_cat)
85
- pred = torch.stack(pred)
86
- truth = torch.stack(truth)
87
- results['pred']= pred.detach().cpu().numpy()
88
- results['truth']= truth.detach().cpu().numpy()
89
-
90
- pred_de = torch.stack(pred_de)
91
- truth_de = torch.stack(truth_de)
92
- results['pred_de']= pred_de.detach().cpu().numpy()
93
- results['truth_de']= truth_de.detach().cpu().numpy()
94
-
95
- if uncertainty:
96
- results['logvar'] = torch.stack(logvar).detach().cpu().numpy()
97
-
98
- return results
99
-
100
- #This evaluation is just as test. IT HAS INFORMATION LEAKAGE!
101
- def evaluate_with_nonzero_mask(loader, model, uncertainty, device):
102
- """
103
- Run model in inference mode using a given data loader
104
- """
105
-
106
- model.eval()
107
- model.to(device)
108
- pert_cat = []
109
- pred = []
110
- truth = []
111
- pred_de = []
112
- truth_de = []
113
- results = {}
114
- logvar = []
115
-
116
- for itr, batch in enumerate(loader):
117
-
118
- batch.to(device)
119
- pert_cat.extend(batch.pert)
120
-
121
- with torch.no_grad():
122
- if uncertainty:
123
- p, unc = model(batch)
124
- logvar.extend(unc.cpu())
125
- else:
126
- p = model(batch)
127
- t = batch.y
128
- p = p * (t != 0)
129
- pred.extend(p.cpu())
130
- truth.extend(t.cpu())
131
-
132
- # Differentially expressed genes
133
- for itr, de_idx in enumerate(batch.de_idx):
134
- pred_de.append(p[itr, de_idx])
135
- truth_de.append(t[itr, de_idx])
136
-
137
- # all genes
138
- results['pert_cat'] = np.array(pert_cat)
139
- pred = torch.stack(pred)
140
- truth = torch.stack(truth)
141
- results['pred']= pred.detach().cpu().numpy()
142
- results['truth']= truth.detach().cpu().numpy()
143
-
144
- pred_de = torch.stack(pred_de)
145
- truth_de = torch.stack(truth_de)
146
- results['pred_de']= pred_de.detach().cpu().numpy()
147
- results['truth_de']= truth_de.detach().cpu().numpy()
148
-
149
- if uncertainty:
150
- results['logvar'] = torch.stack(logvar).detach().cpu().numpy()
151
-
152
- return results
153
-
154
-
155
- def non_zero_evaluate(loader, model, uncertainty, device,
156
- delta_list, penalty = {
157
- "false_neg": 1,
158
- "false_pos": 1 }):
159
- """
160
- Run model in inference mode using a given data loader
161
- """
162
-
163
- model.eval()
164
- model.to(device)
165
- pert_cat = []
166
- pred = []
167
- truth = []
168
- pred_de = []
169
- truth_de = []
170
- results = {}
171
- logvar = []
172
-
173
- for itr, batch in enumerate(loader):
174
-
175
- batch.to(device)
176
- pert_cat.extend(batch.pert)
177
-
178
- with torch.no_grad():
179
- if uncertainty:
180
- p, unc = model(batch)
181
- logvar.extend(unc.cpu())
182
- else:
183
- p = model(batch)
184
- t = batch.y
185
- pred.extend(p.cpu())
186
- truth.extend(t.cpu())
187
-
188
- # Differentially expressed genes
189
- for itr, de_idx in enumerate(batch.de_idx):
190
- pred_de.append(p[itr, de_idx])
191
- truth_de.append(t[itr, de_idx])
192
-
193
- # all genes
194
- results['pert_cat'] = np.array(pert_cat)
195
- pred = torch.stack(pred)
196
- truth = torch.stack(truth)
197
- results['pred']= pred.detach().cpu().numpy()
198
- results['truth']= truth.detach().cpu().numpy()
199
-
200
- pred_de = torch.stack(pred_de)
201
- truth_de = torch.stack(truth_de)
202
- results['pred_de']= pred_de.detach().cpu().numpy()
203
- results['truth_de']= truth_de.detach().cpu().numpy()
204
-
205
- if uncertainty:
206
- results['logvar'] = torch.stack(logvar).detach().cpu().numpy()
207
-
208
- return results
209
-
210
-
211
- def compute_metrics(results, gene_idx=None):
212
- """
213
- Given results from a model run and the ground truth, compute metrics
214
-
215
- """
216
- metrics = {}
217
- metrics_pert = {}
218
-
219
- metric2fct = {
220
- 'mse': mse,
221
- 'pearson': pearsonr
222
- }
223
-
224
- for m in metric2fct.keys():
225
- metrics[m] = []
226
- metrics[m + '_de'] = []
227
-
228
- for pert in np.unique(results['pert_cat']):
229
-
230
- metrics_pert[pert] = {}
231
- p_idx = np.where(results['pert_cat'] == pert)[0]
232
-
233
- for m, fct in metric2fct.items():
234
- # First average the cells of the same perturbation, and then calculate the indicator of the perturbation.
235
- if m == 'pearson':
236
- val = fct(results['pred'][p_idx].mean(0), results['truth'][p_idx].mean(0))[0]
237
- if np.isnan(val):
238
- val = 0
239
- else:
240
- val = fct(results['pred'][p_idx].mean(0), results['truth'][p_idx].mean(0))
241
-
242
- metrics_pert[pert][m] = val
243
- metrics[m].append(metrics_pert[pert][m])
244
-
245
-
246
- if pert != 'ctrl':
247
-
248
- for m, fct in metric2fct.items():
249
- if m == 'pearson':
250
- val = fct(results['pred_de'][p_idx].mean(0), results['truth_de'][p_idx].mean(0))[0]
251
- if np.isnan(val):
252
- val = 0
253
- else:
254
- val = fct(results['pred_de'][p_idx].mean(0), results['truth_de'][p_idx].mean(0))
255
-
256
- metrics_pert[pert][m + '_de'] = val
257
- metrics[m + '_de'].append(metrics_pert[pert][m + '_de'])
258
-
259
- else:
260
- for m, fct in metric2fct.items():
261
- metrics_pert[pert][m + '_de'] = 0
262
-
263
- for m in metric2fct.keys():
264
- # The result is a perturbation-level macro average (each perturbation is equally weighted).
265
- metrics[m] = np.mean(metrics[m])
266
- metrics[m + '_de'] = np.mean(metrics[m + '_de'])
267
-
268
- return metrics, metrics_pert
269
-
270
-
271
- def compute_synergy_loss(results, mean_control, high_umi_idx, subtype = 'POTENTIATION'):
272
- pred_res = get_test_set_results_seen2(results, subtype)
273
- all_perts = np.unique(results['pert_cat'])
274
- linear_params = get_linear_params(pred_res, high_umi_idx,
275
- mean_control,all_perts)
276
- synergy_loss = np.sum([np.abs(linear_params[k]['pred']['mag']
277
- - linear_params[k]['truth']['mag']) for k in
278
- linear_params])
279
-
280
- mag = np.sum([np.abs(linear_params[k]['pred']['mag']) for k in
281
- linear_params])
282
- return synergy_loss, mag
283
-
284
-
285
- def non_zero_analysis(adata, test_res):
286
- metric2fct = {
287
- 'pearson': pearsonr,
288
- 'mse': mse
289
- }
290
-
291
- pert_metric = {}
292
-
293
- ## in silico modeling and upperbounding
294
- pert2pert_full_id = dict(adata.obs[['condition', 'condition_name']].values)
295
- geneid2name = dict(zip(adata.var.index.values, adata.var['gene_name']))
296
- geneid2idx = dict(zip(adata.var.index.values, range(len(adata.var.index.values))))
297
-
298
- # calculate mean expression for each condition
299
- unique_conditions = adata.obs.condition.unique()
300
- conditions2index = {}
301
- for i in unique_conditions:
302
- conditions2index[i] = np.where(adata.obs.condition == i)[0]
303
-
304
- condition2mean_expression = {}
305
- for i, j in conditions2index.items():
306
- condition2mean_expression[i] = np.mean(adata.X[j], axis = 0)
307
- pert_list = np.array(list(condition2mean_expression.keys()))
308
- mean_expression = np.array(list(condition2mean_expression.values())).reshape(len(adata.obs.condition.unique()), adata.X.toarray().shape[1])
309
- ctrl = mean_expression[np.where(pert_list == 'ctrl')[0]]
310
-
311
- gene_list = adata.var['gene_name'].values
312
-
313
- for pert in np.unique(test_res['pert_cat']):
314
- pert_metric[pert] = {}
315
-
316
- pert_idx = np.where(test_res['pert_cat'] == pert)[0]
317
- de_idx = [geneid2idx[i] for i in adata.uns['top_non_zero_de_20'][pert2pert_full_id[pert]]]
318
-
319
- direc_change = np.abs(np.sign(test_res['pred'][pert_idx].mean(0)[de_idx] - ctrl[0][de_idx]) - np.sign(test_res['truth'][pert_idx].mean(0)[de_idx] - ctrl[0][de_idx]))
320
- frac_correct_direction = len(np.where(direc_change == 0)[0])/len(de_idx)
321
- pert_metric[pert]['frac_correct_direction_top20_non_zero'] = frac_correct_direction
322
-
323
- frac_direction_opposite = len(np.where(direc_change == 2)[0])/len(de_idx)
324
- pert_metric[pert]['frac_opposite_direction_top20_non_zero'] = frac_direction_opposite
325
-
326
- frac_direction_opposite = len(np.where(direc_change == 1)[0])/len(de_idx)
327
- pert_metric[pert]['frac_0/1_direction_top20_non_zero'] = frac_direction_opposite
328
-
329
- mean = np.mean(test_res['truth'][pert_idx][:, de_idx], axis = 0)
330
- std = np.std(test_res['truth'][pert_idx][:, de_idx], axis = 0)
331
- min_ = np.min(test_res['truth'][pert_idx][:, de_idx], axis = 0)
332
- max_ = np.max(test_res['truth'][pert_idx][:, de_idx], axis = 0)
333
- q25 = np.quantile(test_res['truth'][pert_idx][:, de_idx], 0.25, axis = 0)
334
- q75 = np.quantile(test_res['truth'][pert_idx][:, de_idx], 0.75, axis = 0)
335
- q55 = np.quantile(test_res['truth'][pert_idx][:, de_idx], 0.55, axis = 0)
336
- q45 = np.quantile(test_res['truth'][pert_idx][:, de_idx], 0.45, axis = 0)
337
- q40 = np.quantile(test_res['truth'][pert_idx][:, de_idx], 0.4, axis = 0)
338
- q60 = np.quantile(test_res['truth'][pert_idx][:, de_idx], 0.6, axis = 0)
339
-
340
- zero_des = np.intersect1d(np.where(min_ == 0)[0], np.where(max_ == 0)[0])
341
- nonzero_des = np.setdiff1d(list(range(20)), zero_des)
342
-
343
- if len(nonzero_des) == 0:
344
- pass
345
- # pert that all de genes are 0...
346
- else:
347
- pred_mean = np.mean(test_res['pred'][pert_idx][:, de_idx], axis = 0).reshape(-1,)
348
- true_mean = np.mean(test_res['truth'][pert_idx][:, de_idx], axis = 0).reshape(-1,)
349
-
350
- in_range = (pred_mean[nonzero_des] >= min_[nonzero_des]) & (pred_mean[nonzero_des] <= max_[nonzero_des])
351
- frac_in_range = sum(in_range)/len(nonzero_des)
352
- pert_metric[pert]['frac_in_range_non_zero'] = frac_in_range
353
-
354
- in_range_5 = (pred_mean[nonzero_des] >= q45[nonzero_des]) & (pred_mean[nonzero_des] <= q55[nonzero_des])
355
- frac_in_range_45_55 = sum(in_range_5)/len(nonzero_des)
356
- pert_metric[pert]['frac_in_range_45_55_non_zero'] = frac_in_range_45_55
357
-
358
- in_range_10 = (pred_mean[nonzero_des] >= q40[nonzero_des]) & (pred_mean[nonzero_des] <= q60[nonzero_des])
359
- frac_in_range_40_60 = sum(in_range_10)/len(nonzero_des)
360
- pert_metric[pert]['frac_in_range_40_60_non_zero'] = frac_in_range_40_60
361
-
362
- in_range_25 = (pred_mean[nonzero_des] >= q25[nonzero_des]) & (pred_mean[nonzero_des] <= q75[nonzero_des])
363
- frac_in_range_25_75 = sum(in_range_25)/len(nonzero_des)
364
- pert_metric[pert]['frac_in_range_25_75_non_zero'] = frac_in_range_25_75
365
-
366
- zero_idx = np.where(std > 0)[0]
367
- sigma = (np.abs(pred_mean[zero_idx] - mean[zero_idx]))/(std[zero_idx])
368
- pert_metric[pert]['mean_sigma_non_zero'] = np.mean(sigma)
369
- pert_metric[pert]['std_sigma_non_zero'] = np.std(sigma)
370
- pert_metric[pert]['frac_sigma_below_1_non_zero'] = 1 - len(np.where(sigma > 1)[0])/len(zero_idx)
371
- pert_metric[pert]['frac_sigma_below_2_non_zero'] = 1 - len(np.where(sigma > 2)[0])/len(zero_idx)
372
-
373
- p_idx = np.where(test_res['pert_cat'] == pert)[0]
374
- for m, fct in metric2fct.items():
375
- if m != 'mse':
376
- val = fct(test_res['pred'][p_idx].mean(0)[de_idx] - ctrl[0][de_idx], test_res['truth'][p_idx].mean(0)[de_idx]-ctrl[0][de_idx])[0]
377
- if np.isnan(val):
378
- val = 0
379
- pert_metric[pert][m + '_delta_top20_de_non_zero'] = val
380
-
381
-
382
- val = fct(test_res['pred'][p_idx].mean(0)[de_idx], test_res['truth'][p_idx].mean(0)[de_idx])[0]
383
- if np.isnan(val):
384
- val = 0
385
- pert_metric[pert][m + '_top20_de_non_zero'] = val
386
- else:
387
- val = fct(test_res['pred'][p_idx].mean(0)[de_idx] - ctrl[0][de_idx], test_res['truth'][p_idx].mean(0)[de_idx]-ctrl[0][de_idx])
388
- pert_metric[pert][m + '_top20_de_non_zero'] = val
389
-
390
- return pert_metric
391
-
392
- def non_dropout_analysis(adata, test_res):
393
- metric2fct = {
394
- 'pearson': pearsonr,
395
- 'mse': mse
396
- }
397
-
398
- pert_metric = {}
399
-
400
- ## in silico modeling and upperbounding
401
- pert2pert_full_id = dict(adata.obs[['condition', 'condition_name']].values)
402
- geneid2name = dict(zip(adata.var.index.values, adata.var['gene_name']))
403
- geneid2idx = dict(zip(adata.var.index.values, range(len(adata.var.index.values))))
404
-
405
- # calculate mean expression for each condition
406
- unique_conditions = adata.obs.condition.unique()
407
- conditions2index = {}
408
- for i in unique_conditions:
409
- conditions2index[i] = np.where(adata.obs.condition == i)[0]
410
-
411
- condition2mean_expression = {}
412
- for i, j in conditions2index.items():
413
- condition2mean_expression[i] = np.mean(adata.X[j], axis = 0)
414
- pert_list = np.array(list(condition2mean_expression.keys()))
415
- mean_expression = np.array(list(condition2mean_expression.values())).reshape(len(adata.obs.condition.unique()), adata.X.toarray().shape[1])
416
- ctrl = mean_expression[np.where(pert_list == 'ctrl')[0]]
417
-
418
- gene_list = adata.var['gene_name'].values
419
-
420
- for pert in np.unique(test_res['pert_cat']):
421
- pert_metric[pert] = {}
422
-
423
- pert_idx = np.where(test_res['pert_cat'] == pert)[0]
424
- de_idx = [geneid2idx[i] for i in adata.uns['top_non_dropout_de_20'][pert2pert_full_id[pert]]]
425
- non_zero_idx = adata.uns['non_zeros_gene_idx'][pert2pert_full_id[pert]]
426
- non_dropout_gene_idx = adata.uns['non_dropout_gene_idx'][pert2pert_full_id[pert]]
427
-
428
- direc_change = np.abs(np.sign(test_res['pred'][pert_idx].mean(0)[de_idx] - ctrl[0][de_idx]) - np.sign(test_res['truth'][pert_idx].mean(0)[de_idx] - ctrl[0][de_idx]))
429
- frac_correct_direction = len(np.where(direc_change == 0)[0])/len(de_idx)
430
- pert_metric[pert]['frac_correct_direction_top20_non_dropout'] = frac_correct_direction
431
-
432
- frac_direction_opposite = len(np.where(direc_change == 2)[0])/len(de_idx)
433
- pert_metric[pert]['frac_opposite_direction_top20_non_dropout'] = frac_direction_opposite
434
-
435
- frac_direction_opposite = len(np.where(direc_change == 1)[0])/len(de_idx)
436
- pert_metric[pert]['frac_0/1_direction_top20_non_dropout'] = frac_direction_opposite
437
-
438
- direc_change = np.abs(np.sign(test_res['pred'][pert_idx].mean(0)[non_zero_idx] - ctrl[0][non_zero_idx]) - np.sign(test_res['truth'][pert_idx].mean(0)[non_zero_idx] - ctrl[0][non_zero_idx]))
439
- frac_correct_direction = len(np.where(direc_change == 0)[0])/len(non_zero_idx)
440
- pert_metric[pert]['frac_correct_direction_non_zero'] = frac_correct_direction
441
-
442
- frac_direction_opposite = len(np.where(direc_change == 2)[0])/len(non_zero_idx)
443
- pert_metric[pert]['frac_opposite_direction_non_zero'] = frac_direction_opposite
444
-
445
- frac_direction_opposite = len(np.where(direc_change == 1)[0])/len(non_zero_idx)
446
- pert_metric[pert]['frac_0/1_direction_non_zero'] = frac_direction_opposite
447
-
448
- direc_change = np.abs(np.sign(test_res['pred'][pert_idx].mean(0)[non_dropout_gene_idx] - ctrl[0][non_dropout_gene_idx]) - np.sign(test_res['truth'][pert_idx].mean(0)[non_dropout_gene_idx] - ctrl[0][non_dropout_gene_idx]))
449
- frac_correct_direction = len(np.where(direc_change == 0)[0])/len(non_dropout_gene_idx)
450
- pert_metric[pert]['frac_correct_direction_non_dropout'] = frac_correct_direction
451
-
452
- frac_direction_opposite = len(np.where(direc_change == 2)[0])/len(non_dropout_gene_idx)
453
- pert_metric[pert]['frac_opposite_direction_non_dropout'] = frac_direction_opposite
454
-
455
- frac_direction_opposite = len(np.where(direc_change == 1)[0])/len(non_dropout_gene_idx)
456
- pert_metric[pert]['frac_0/1_direction_non_dropout'] = frac_direction_opposite
457
-
458
- mean = np.mean(test_res['truth'][pert_idx][:, de_idx], axis = 0)
459
- std = np.std(test_res['truth'][pert_idx][:, de_idx], axis = 0)
460
- min_ = np.min(test_res['truth'][pert_idx][:, de_idx], axis = 0)
461
- max_ = np.max(test_res['truth'][pert_idx][:, de_idx], axis = 0)
462
- q25 = np.quantile(test_res['truth'][pert_idx][:, de_idx], 0.25, axis = 0)
463
- q75 = np.quantile(test_res['truth'][pert_idx][:, de_idx], 0.75, axis = 0)
464
- q55 = np.quantile(test_res['truth'][pert_idx][:, de_idx], 0.55, axis = 0)
465
- q45 = np.quantile(test_res['truth'][pert_idx][:, de_idx], 0.45, axis = 0)
466
- q40 = np.quantile(test_res['truth'][pert_idx][:, de_idx], 0.4, axis = 0)
467
- q60 = np.quantile(test_res['truth'][pert_idx][:, de_idx], 0.6, axis = 0)
468
-
469
- zero_des = np.intersect1d(np.where(min_ == 0)[0], np.where(max_ == 0)[0])
470
- nonzero_des = np.setdiff1d(list(range(20)), zero_des)
471
-
472
- if len(nonzero_des) == 0:
473
- pass
474
- # pert that all de genes are 0...
475
- else:
476
- pred_mean = np.mean(test_res['pred'][pert_idx][:, de_idx], axis = 0).reshape(-1,)
477
- true_mean = np.mean(test_res['truth'][pert_idx][:, de_idx], axis = 0).reshape(-1,)
478
-
479
- in_range = (pred_mean[nonzero_des] >= min_[nonzero_des]) & (pred_mean[nonzero_des] <= max_[nonzero_des])
480
- frac_in_range = sum(in_range)/len(nonzero_des)
481
- pert_metric[pert]['frac_in_range_non_dropout'] = frac_in_range
482
-
483
- in_range_5 = (pred_mean[nonzero_des] >= q45[nonzero_des]) & (pred_mean[nonzero_des] <= q55[nonzero_des])
484
- frac_in_range_45_55 = sum(in_range_5)/len(nonzero_des)
485
- pert_metric[pert]['frac_in_range_45_55_non_dropout'] = frac_in_range_45_55
486
-
487
- in_range_10 = (pred_mean[nonzero_des] >= q40[nonzero_des]) & (pred_mean[nonzero_des] <= q60[nonzero_des])
488
- frac_in_range_40_60 = sum(in_range_10)/len(nonzero_des)
489
- pert_metric[pert]['frac_in_range_40_60_non_dropout'] = frac_in_range_40_60
490
-
491
- in_range_25 = (pred_mean[nonzero_des] >= q25[nonzero_des]) & (pred_mean[nonzero_des] <= q75[nonzero_des])
492
- frac_in_range_25_75 = sum(in_range_25)/len(nonzero_des)
493
- pert_metric[pert]['frac_in_range_25_75_non_dropout'] = frac_in_range_25_75
494
-
495
- zero_idx = np.where(std > 0)[0]
496
- sigma = (np.abs(pred_mean[zero_idx] - mean[zero_idx]))/(std[zero_idx])
497
- pert_metric[pert]['mean_sigma_non_dropout'] = np.mean(sigma)
498
- pert_metric[pert]['std_sigma_non_dropout'] = np.std(sigma)
499
- pert_metric[pert]['frac_sigma_below_1_non_dropout'] = 1 - len(np.where(sigma > 1)[0])/len(zero_idx)
500
- pert_metric[pert]['frac_sigma_below_2_non_dropout'] = 1 - len(np.where(sigma > 2)[0])/len(zero_idx)
501
-
502
- p_idx = np.where(test_res['pert_cat'] == pert)[0]
503
- for m, fct in metric2fct.items():
504
- if m != 'mse':
505
- val = fct(test_res['pred'][p_idx].mean(0)[de_idx] - ctrl[0][de_idx], test_res['truth'][p_idx].mean(0)[de_idx]-ctrl[0][de_idx])[0]
506
- if np.isnan(val):
507
- val = 0
508
- pert_metric[pert][m + '_delta_top20_de_non_dropout'] = val
509
-
510
-
511
- val = fct(test_res['pred'][p_idx].mean(0)[de_idx], test_res['truth'][p_idx].mean(0)[de_idx])[0]
512
- if np.isnan(val):
513
- val = 0
514
- pert_metric[pert][m + '_top20_de_non_dropout'] = val
515
- else:
516
- val = fct(test_res['pred'][p_idx].mean(0)[de_idx] - ctrl[0][de_idx], test_res['truth'][p_idx].mean(0)[de_idx]-ctrl[0][de_idx])
517
- pert_metric[pert][m + '_top20_de_non_dropout'] = val
518
-
519
- return pert_metric
520
-
521
- def deeper_analysis(adata, test_res, de_column_prefix = 'rank_genes_groups_cov', most_variable_genes = None):
522
-
523
- metric2fct = {
524
- 'pearson': pearsonr,
525
- 'mse': mse
526
- }
527
-
528
- pert_metric = {}
529
-
530
- ## in silico modeling and upperbounding
531
- pert2pert_full_id = dict(adata.obs[['condition', 'condition_name']].values)
532
- geneid2name = dict(zip(adata.var.index.values, adata.var['gene_name']))
533
- geneid2idx = dict(zip(adata.var.index.values, range(len(adata.var.index.values))))
534
-
535
- # calculate mean expression for each condition
536
- unique_conditions = adata.obs.condition.unique()
537
- conditions2index = {}
538
- for i in unique_conditions:
539
- conditions2index[i] = np.where(adata.obs.condition == i)[0]
540
-
541
- condition2mean_expression = {}
542
- for i, j in conditions2index.items():
543
- condition2mean_expression[i] = np.mean(adata.X[j], axis = 0)
544
- pert_list = np.array(list(condition2mean_expression.keys()))
545
- mean_expression = np.array(list(condition2mean_expression.values())).reshape(len(adata.obs.condition.unique()), adata.X.toarray().shape[1])
546
- ctrl = mean_expression[np.where(pert_list == 'ctrl')[0]]
547
-
548
- if most_variable_genes is None:
549
- most_variable_genes = np.argsort(np.std(mean_expression, axis = 0))[-200:]
550
-
551
- gene_list = adata.var['gene_name'].values
552
-
553
- for pert in np.unique(test_res['pert_cat']):
554
- pert_metric[pert] = {}
555
- de_idx = [geneid2idx[i] for i in adata.uns['rank_genes_groups_cov_all'][pert2pert_full_id[pert]][:20]]
556
- de_idx_200 = [geneid2idx[i] for i in adata.uns['rank_genes_groups_cov_all'][pert2pert_full_id[pert]][:200]]
557
- de_idx_100 = [geneid2idx[i] for i in adata.uns['rank_genes_groups_cov_all'][pert2pert_full_id[pert]][:100]]
558
- de_idx_50 = [geneid2idx[i] for i in adata.uns['rank_genes_groups_cov_all'][pert2pert_full_id[pert]][:50]]
559
-
560
- pert_idx = np.where(test_res['pert_cat'] == pert)[0]
561
- pred_mean = np.mean(test_res['pred_de'][pert_idx], axis = 0).reshape(-1,)
562
- true_mean = np.mean(test_res['truth_de'][pert_idx], axis = 0).reshape(-1,)
563
-
564
- direc_change = np.abs(np.sign(test_res['pred'][pert_idx].mean(0) - ctrl[0]) - np.sign(test_res['truth'][pert_idx].mean(0) - ctrl[0]))
565
- frac_correct_direction = len(np.where(direc_change == 0)[0])/len(geneid2name)
566
- pert_metric[pert]['frac_correct_direction_all'] = frac_correct_direction
567
-
568
- de_idx_map = {20: de_idx,
569
- 50: de_idx_50,
570
- 100: de_idx_100,
571
- 200: de_idx_200
572
- }
573
-
574
- for val in [20, 50, 100, 200]:
575
-
576
- direc_change = np.abs(np.sign(test_res['pred'][pert_idx].mean(0)[de_idx_map[val]] - ctrl[0][de_idx_map[val]]) - np.sign(test_res['truth'][pert_idx].mean(0)[de_idx_map[val]] - ctrl[0][de_idx_map[val]]))
577
- frac_correct_direction = len(np.where(direc_change == 0)[0])/val
578
- pert_metric[pert]['frac_correct_direction_' + str(val)] = frac_correct_direction
579
-
580
- mean = np.mean(test_res['truth_de'][pert_idx], axis = 0)
581
- std = np.std(test_res['truth_de'][pert_idx], axis = 0)
582
- min_ = np.min(test_res['truth_de'][pert_idx], axis = 0)
583
- max_ = np.max(test_res['truth_de'][pert_idx], axis = 0)
584
- q25 = np.quantile(test_res['truth_de'][pert_idx], 0.25, axis = 0)
585
- q75 = np.quantile(test_res['truth_de'][pert_idx], 0.75, axis = 0)
586
- q55 = np.quantile(test_res['truth_de'][pert_idx], 0.55, axis = 0)
587
- q45 = np.quantile(test_res['truth_de'][pert_idx], 0.45, axis = 0)
588
- q40 = np.quantile(test_res['truth_de'][pert_idx], 0.4, axis = 0)
589
- q60 = np.quantile(test_res['truth_de'][pert_idx], 0.6, axis = 0)
590
-
591
- zero_des = np.intersect1d(np.where(min_ == 0)[0], np.where(max_ == 0)[0])
592
- nonzero_des = np.setdiff1d(list(range(20)), zero_des)
593
- if len(nonzero_des) == 0:
594
- pass
595
- # pert that all de genes are 0...
596
- else:
597
-
598
- direc_change = np.abs(np.sign(pred_mean[nonzero_des] - ctrl[0][de_idx][nonzero_des]) - np.sign(true_mean[nonzero_des] - ctrl[0][de_idx][nonzero_des]))
599
- frac_correct_direction = len(np.where(direc_change == 0)[0])/len(nonzero_des)
600
- pert_metric[pert]['frac_correct_direction_20_nonzero'] = frac_correct_direction
601
-
602
- in_range = (pred_mean[nonzero_des] >= min_[nonzero_des]) & (pred_mean[nonzero_des] <= max_[nonzero_des])
603
- frac_in_range = sum(in_range)/len(nonzero_des)
604
- pert_metric[pert]['frac_in_range'] = frac_in_range
605
-
606
- in_range_5 = (pred_mean[nonzero_des] >= q45[nonzero_des]) & (pred_mean[nonzero_des] <= q55[nonzero_des])
607
- frac_in_range_45_55 = sum(in_range_5)/len(nonzero_des)
608
- pert_metric[pert]['frac_in_range_45_55'] = frac_in_range_45_55
609
-
610
- in_range_10 = (pred_mean[nonzero_des] >= q40[nonzero_des]) & (pred_mean[nonzero_des] <= q60[nonzero_des])
611
- frac_in_range_40_60 = sum(in_range_10)/len(nonzero_des)
612
- pert_metric[pert]['frac_in_range_40_60'] = frac_in_range_40_60
613
-
614
- in_range_25 = (pred_mean[nonzero_des] >= q25[nonzero_des]) & (pred_mean[nonzero_des] <= q75[nonzero_des])
615
- frac_in_range_25_75 = sum(in_range_25)/len(nonzero_des)
616
- pert_metric[pert]['frac_in_range_25_75'] = frac_in_range_25_75
617
-
618
- zero_idx = np.where(std > 0)[0]
619
- sigma = (np.abs(pred_mean[zero_idx] - mean[zero_idx]))/(std[zero_idx])
620
- pert_metric[pert]['mean_sigma'] = np.mean(sigma)
621
- pert_metric[pert]['std_sigma'] = np.std(sigma)
622
- pert_metric[pert]['frac_sigma_below_1'] = 1 - len(np.where(sigma > 1)[0])/len(zero_idx)
623
- pert_metric[pert]['frac_sigma_below_2'] = 1 - len(np.where(sigma > 2)[0])/len(zero_idx)
624
-
625
- ## correlation on delta
626
- p_idx = np.where(test_res['pert_cat'] == pert)[0]
627
-
628
- for m, fct in metric2fct.items():
629
- if m != 'mse':
630
- val = fct(test_res['pred'][p_idx].mean(0)- ctrl[0], test_res['truth'][p_idx].mean(0)-ctrl[0])[0]
631
- if np.isnan(val):
632
- val = 0
633
-
634
- pert_metric[pert][m + '_delta'] = val
635
-
636
- val = fct(test_res['pred'][p_idx].mean(0)[de_idx] - ctrl[0][de_idx], test_res['truth'][p_idx].mean(0)[de_idx]-ctrl[0][de_idx])[0]
637
- if np.isnan(val):
638
- val = 0
639
-
640
- pert_metric[pert][m + '_delta_de'] = val
641
-
642
- ## up fold changes > 10,
643
- pert_mean = np.mean(test_res['truth'][p_idx], axis = 0).reshape(-1,)
644
-
645
- fold_change = pert_mean/ctrl
646
- fold_change[np.isnan(fold_change)] = 0
647
- fold_change[np.isinf(fold_change)] = 0
648
- ## this is to remove the ones that are super low and the fold change becomes unmeaningful
649
- fold_change[0][np.where(pert_mean < 0.5)[0]] = 0
650
-
651
- o = np.where(fold_change[0] > 0)[0]
652
-
653
- pred_fc = test_res['pred'][p_idx].mean(0)[o]
654
- true_fc = test_res['truth'][p_idx].mean(0)[o]
655
- ctrl_fc = ctrl[0][o]
656
-
657
- if len(o) > 0:
658
- pert_metric[pert]['fold_change_gap_all'] = np.mean(np.abs(pred_fc/ctrl_fc - true_fc/ctrl_fc))
659
-
660
-
661
- o = np.intersect1d(np.where(fold_change[0] <0.333)[0], np.where(fold_change[0] > 0)[0])
662
-
663
- pred_fc = test_res['pred'][p_idx].mean(0)[o]
664
- true_fc = test_res['truth'][p_idx].mean(0)[o]
665
- ctrl_fc = ctrl[0][o]
666
-
667
- if len(o) > 0:
668
- pert_metric[pert]['fold_change_gap_downreg_0.33'] = np.mean(np.abs(pred_fc/ctrl_fc - true_fc/ctrl_fc))
669
-
670
-
671
- o = np.intersect1d(np.where(fold_change[0] <0.1)[0], np.where(fold_change[0] > 0)[0])
672
-
673
- pred_fc = test_res['pred'][p_idx].mean(0)[o]
674
- true_fc = test_res['truth'][p_idx].mean(0)[o]
675
- ctrl_fc = ctrl[0][o]
676
-
677
- if len(o) > 0:
678
- pert_metric[pert]['fold_change_gap_downreg_0.1'] = np.mean(np.abs(pred_fc/ctrl_fc - true_fc/ctrl_fc))
679
-
680
- o = np.where(fold_change[0] > 3)[0]
681
-
682
- pred_fc = test_res['pred'][p_idx].mean(0)[o]
683
- true_fc = test_res['truth'][p_idx].mean(0)[o]
684
- ctrl_fc = ctrl[0][o]
685
-
686
- if len(o) > 0:
687
- pert_metric[pert]['fold_change_gap_upreg_3'] = np.mean(np.abs(pred_fc/ctrl_fc - true_fc/ctrl_fc))
688
-
689
- o = np.where(fold_change[0] > 10)[0]
690
-
691
- pred_fc = test_res['pred'][p_idx].mean(0)[o]
692
- true_fc = test_res['truth'][p_idx].mean(0)[o]
693
- ctrl_fc = ctrl[0][o]
694
-
695
- if len(o) > 0:
696
- pert_metric[pert]['fold_change_gap_upreg_10'] = np.mean(np.abs(pred_fc/ctrl_fc - true_fc/ctrl_fc))
697
-
698
- ## most variable genes
699
- for m, fct in metric2fct.items():
700
- if m != 'mse':
701
- val = fct(test_res['pred'][p_idx].mean(0)[most_variable_genes] - ctrl[0][most_variable_genes], test_res['truth'][p_idx].mean(0)[most_variable_genes]-ctrl[0][most_variable_genes])[0]
702
- if np.isnan(val):
703
- val = 0
704
- pert_metric[pert][m + '_delta_top200_hvg'] = val
705
-
706
-
707
- val = fct(test_res['pred'][p_idx].mean(0)[most_variable_genes], test_res['truth'][p_idx].mean(0)[most_variable_genes])[0]
708
- if np.isnan(val):
709
- val = 0
710
- pert_metric[pert][m + '_top200_hvg'] = val
711
- else:
712
- val = fct(test_res['pred'][p_idx].mean(0)[most_variable_genes], test_res['truth'][p_idx].mean(0)[most_variable_genes])
713
- pert_metric[pert][m + '_top200_hvg'] = val
714
-
715
-
716
- ## top 20/50/100/200 DEs
717
- for m, fct in metric2fct.items():
718
- if m != 'mse':
719
- val = fct(test_res['pred'][p_idx].mean(0)[de_idx] - ctrl[0][de_idx], test_res['truth'][p_idx].mean(0)[de_idx]-ctrl[0][de_idx])[0]
720
- if np.isnan(val):
721
- val = 0
722
- pert_metric[pert][m + '_delta_top20_de'] = val
723
-
724
-
725
- val = fct(test_res['pred'][p_idx].mean(0)[de_idx], test_res['truth'][p_idx].mean(0)[de_idx])[0]
726
- if np.isnan(val):
727
- val = 0
728
- pert_metric[pert][m + '_top20_de'] = val
729
- else:
730
- val = fct(test_res['pred'][p_idx].mean(0)[de_idx] - ctrl[0][de_idx], test_res['truth'][p_idx].mean(0)[de_idx]-ctrl[0][de_idx])
731
- pert_metric[pert][m + '_top20_de'] = val
732
-
733
-
734
- for m, fct in metric2fct.items():
735
- if m != 'mse':
736
- val = fct(test_res['pred'][p_idx].mean(0)[de_idx_200] - ctrl[0][de_idx_200], test_res['truth'][p_idx].mean(0)[de_idx_200]-ctrl[0][de_idx_200])[0]
737
- if np.isnan(val):
738
- val = 0
739
- pert_metric[pert][m + '_delta_top200_de'] = val
740
-
741
-
742
- val = fct(test_res['pred'][p_idx].mean(0)[de_idx_200], test_res['truth'][p_idx].mean(0)[de_idx_200])[0]
743
- if np.isnan(val):
744
- val = 0
745
- pert_metric[pert][m + '_top200_de'] = val
746
- else:
747
- val = fct(test_res['pred'][p_idx].mean(0)[de_idx_200] - ctrl[0][de_idx_200], test_res['truth'][p_idx].mean(0)[de_idx_200]-ctrl[0][de_idx_200])
748
- pert_metric[pert][m + '_top200_de'] = val
749
-
750
- for m, fct in metric2fct.items():
751
- if m != 'mse':
752
-
753
- val = fct(test_res['pred'][p_idx].mean(0)[de_idx_100] - ctrl[0][de_idx_100], test_res['truth'][p_idx].mean(0)[de_idx_100]-ctrl[0][de_idx_100])[0]
754
- if np.isnan(val):
755
- val = 0
756
- pert_metric[pert][m + '_delta_top100_de'] = val
757
-
758
-
759
- val = fct(test_res['pred'][p_idx].mean(0)[de_idx_100], test_res['truth'][p_idx].mean(0)[de_idx_100])[0]
760
- if np.isnan(val):
761
- val = 0
762
- pert_metric[pert][m + '_top100_de'] = val
763
- else:
764
- val = fct(test_res['pred'][p_idx].mean(0)[de_idx_100] - ctrl[0][de_idx_100], test_res['truth'][p_idx].mean(0)[de_idx_100]-ctrl[0][de_idx_100])
765
- pert_metric[pert][m + '_top100_de'] = val
766
-
767
- for m, fct in metric2fct.items():
768
- if m != 'mse':
769
-
770
- val = fct(test_res['pred'][p_idx].mean(0)[de_idx_50] - ctrl[0][de_idx_50], test_res['truth'][p_idx].mean(0)[de_idx_50]-ctrl[0][de_idx_50])[0]
771
- if np.isnan(val):
772
- val = 0
773
- pert_metric[pert][m + '_delta_top50_de'] = val
774
-
775
-
776
- val = fct(test_res['pred'][p_idx].mean(0)[de_idx_50], test_res['truth'][p_idx].mean(0)[de_idx_50])[0]
777
- if np.isnan(val):
778
- val = 0
779
- pert_metric[pert][m + '_top50_de'] = val
780
- else:
781
- val = fct(test_res['pred'][p_idx].mean(0)[de_idx_50] - ctrl[0][de_idx_50], test_res['truth'][p_idx].mean(0)[de_idx_50]-ctrl[0][de_idx_50])
782
- pert_metric[pert][m + '_top50_de'] = val
783
-
784
-
785
-
786
- return pert_metric
787
-
788
-
789
- def GI_subgroup(pert_metric):
790
- GI_type2Score = {}
791
- test_pert_list = list(pert_metric.keys())
792
- for GI_type, gi_list in GIs.items():
793
- intersect = np.intersect1d(gi_list, test_pert_list)
794
- if len(intersect) != 0:
795
- GI_type2Score[GI_type] = {}
796
-
797
- for m in list(list(pert_metric.values())[0].keys()):
798
- GI_type2Score[GI_type][m] = np.mean([pert_metric[i][m] for i in intersect if m in pert_metric[i]])
799
-
800
- return GI_type2Score
801
-
802
- def node_specific_batch_out(models, batch):
803
- # Returns output for all node specific models as a matrix of dimension batch_size x nodes
804
- outs = []
805
- for idx in range(len(models)):
806
- outs.append(models[idx](batch).detach().cpu().numpy()[:,idx])
807
- return np.vstack(outs).T
808
-
809
- # Run prediction over all batches
810
- def batch_predict(loader, loaded_models, args):
811
- # Prediction for node specific GNNs
812
- preds = []
813
- print("Loader size: ", len(loader))
814
- for itr, batch in enumerate(loader):
815
- print(itr)
816
- batch = batch.to(args['device'])
817
- preds.append(node_specific_batch_out(loaded_models, batch))
818
-
819
- preds = np.vstack(preds)
820
- return preds
821
-
822
- def get_high_umi_idx(gene_list):
823
- # Genes used for linear model fitting
824
- try:
825
- high_umi = np.load('../genes_with_hi_mean.npy', allow_pickle=True)
826
- except:
827
- high_umi = np.load('./genes_with_hi_mean.npy', allow_pickle=True)
828
- high_umi_idx = np.where([g in high_umi for g in gene_list])[0]
829
- return high_umi_idx
830
-
831
- def get_mean_ctrl(adata):
832
- return adata[adata.obs['condition'] == 'ctrl'].to_df().mean().reset_index(
833
- drop=True)
834
-
835
- def get_single_name(g, all_perts):
836
- name = g+'+ctrl'
837
- if name in all_perts:
838
- return name
839
- else:
840
- return 'ctrl+'+g
841
-
842
-
843
- def get_test_set_results_seen2(res, sel_GI_type):
844
- # Get relevant test set results
845
- test_pert_cats = [p for p in np.unique(res['pert_cat']) if
846
- p in GIs[sel_GI_type] or 'ctrl' in p]
847
- pred_idx = np.where([t in test_pert_cats for t in res['pert_cat']])
848
- out = {}
849
- for key in res:
850
- out[key] = res[key][pred_idx]
851
- return out
852
-
853
- ## Synergy loss calculation functions
854
- def get_all_vectors(all_res, mean_control, double,
855
- single1, single2, high_umi_idx):
856
- # Pred
857
- pred_df = pd.DataFrame(all_res['pred'])
858
- pred_df['condition'] = all_res['pert_cat']
859
- subset_df = pred_df[pred_df['condition'] == double].iloc[:, :-1]
860
- delta_double_pred = subset_df.mean(0) - mean_control
861
- single_df_1_pred = pred_df[pred_df['condition'] == single1].iloc[:, :-1]
862
- single_df_2_pred = pred_df[pred_df['condition'] == single2].iloc[:, :-1]
863
-
864
- # True
865
- truth_df = pd.DataFrame(all_res['truth'])
866
- truth_df['condition'] = all_res['pert_cat']
867
- subset_df = truth_df[truth_df['condition'] == double].iloc[:, :-1]
868
- delta_double_truth = subset_df.mean(0) - mean_control
869
- single_df_1_truth = truth_df[truth_df['condition'] == single1].iloc[:, :-1]
870
- single_df_2_truth = truth_df[truth_df['condition'] == single2].iloc[:, :-1]
871
-
872
- delta_single_truth_1 = single_df_1_truth.mean(0) - mean_control
873
- delta_single_truth_2 = single_df_2_truth.mean(0) - mean_control
874
- delta_single_pred_1 = single_df_1_pred.mean(0) - mean_control
875
- delta_single_pred_2 = single_df_2_pred.mean(0) - mean_control
876
-
877
- return {'single_pred_1': delta_single_pred_1.values[high_umi_idx],
878
- 'single_pred_2': delta_single_pred_2.values[high_umi_idx],
879
- 'double_pred': delta_double_pred.values[high_umi_idx],
880
- 'single_truth_1': delta_single_truth_1.values[high_umi_idx],
881
- 'single_truth_2': delta_single_truth_2.values[high_umi_idx],
882
- 'double_truth': delta_double_truth.values[high_umi_idx]}
883
-
884
-
885
- def get_coeffs_synergy(singles_expr, double_expr):
886
- results = {}
887
- results['ts'] = TheilSenRegressor(fit_intercept=False,
888
- max_subpopulation=1e5,
889
- max_iter=1000,
890
- random_state=1000)
891
- X = singles_expr
892
- y = double_expr
893
- try:
894
- results['ts'].fit(X, y.ravel())
895
- except:
896
- print(X)
897
- print(y)
898
- results['c1'] = results['ts'].coef_[0]
899
- results['c2'] = results['ts'].coef_[1]
900
- results['mag'] = np.sqrt((results['c1'] ** 2 + results['c2'] ** 2))
901
- return results
902
-
903
-
904
- def Fit(all_vectors, type_='pertnet'):
905
- if type_ == 'pertnet':
906
- singles_expr = np.array(
907
- [all_vectors['single_pred_1'], all_vectors['single_pred_2']]).T
908
- first_expr = np.array([all_vectors['single_pred_1']]).T
909
- second_expr = np.array([all_vectors['single_pred_2']]).T
910
- double_expr = np.array(all_vectors['double_pred']).T
911
-
912
- elif type_ == 'truth':
913
- singles_expr = np.array(
914
- [all_vectors['single_truth_1'], all_vectors['single_truth_2']]).T
915
- first_expr = np.array([all_vectors['single_truth_1']]).T
916
- second_expr = np.array([all_vectors['single_truth_2']]).T
917
- double_expr = np.array(all_vectors['double_truth']).T
918
-
919
- return get_coeffs_synergy(singles_expr, double_expr)
920
-
921
-
922
- def get_linear_params(pred_res, high_umi_idx, mean_control, all_perts):
923
- results = {}
924
- for c in set(pred_res['pert_cat']):
925
- if 'ctrl' in c:
926
- continue
927
- double = c
928
- single1 = get_single_name(double.split('+')[0], all_perts)
929
- single2 = get_single_name(double.split('+')[1], all_perts)
930
- all_vectors = get_all_vectors(pred_res, mean_control, double,
931
- single1, single2, high_umi_idx)
932
-
933
- pertnet_results = Fit(all_vectors, type_='pertnet')
934
- truth_results = Fit(all_vectors, type_='truth')
935
-
936
- results[c] = {
937
- 'truth': truth_results,
938
- 'pred': pertnet_results}
939
-
940
- return results
941
-
942
-
943
- # Read in model for each gene
944
-
945
- GIs = {
946
- 'NEOMORPHIC': ['CBL+TGFBR2',
947
- 'KLF1+TGFBR2',
948
- 'MAP2K6+SPI1',
949
- 'SAMD1+TGFBR2',
950
- 'TGFBR2+C19orf26',
951
- 'TGFBR2+ETS2',
952
- 'CBL+UBASH3A',
953
- 'CEBPE+KLF1',
954
- 'DUSP9+MAPK1',
955
- 'FOSB+PTPN12',
956
- 'PLK4+STIL',
957
- 'PTPN12+OSR2',
958
- 'ZC3HAV1+CEBPE'],
959
- 'ADDITIVE': ['BPGM+SAMD1',
960
- 'CEBPB+MAPK1',
961
- 'CEBPB+OSR2',
962
- 'DUSP9+PRTG',
963
- 'FOSB+OSR2',
964
- 'IRF1+SET',
965
- 'MAP2K3+ELMSAN1',
966
- 'MAP2K6+ELMSAN1',
967
- 'POU3F2+FOXL2',
968
- 'RHOXF2BB+SET',
969
- 'SAMD1+PTPN12',
970
- 'SAMD1+UBASH3B',
971
- 'SAMD1+ZBTB1',
972
- 'SGK1+TBX2',
973
- 'TBX3+TBX2',
974
- 'ZBTB10+SNAI1'],
975
- 'EPISTASIS': ['AHR+KLF1',
976
- 'MAPK1+TGFBR2',
977
- 'TGFBR2+IGDCC3',
978
- 'TGFBR2+PRTG',
979
- 'UBASH3B+OSR2',
980
- 'DUSP9+ETS2',
981
- 'KLF1+CEBPA',
982
- 'MAP2K6+IKZF3',
983
- 'ZC3HAV1+CEBPA'],
984
- 'REDUNDANT': ['CDKN1C+CDKN1A',
985
- 'MAP2K3+MAP2K6',
986
- 'CEBPB+CEBPA',
987
- 'CEBPE+CEBPA',
988
- 'CEBPE+SPI1',
989
- 'ETS2+MAPK1',
990
- 'FOSB+CEBPE',
991
- 'FOXA3+FOXA1'],
992
- 'POTENTIATION': ['CNN1+UBASH3A',
993
- 'ETS2+MAP7D1',
994
- 'FEV+CBFA2T3',
995
- 'FEV+ISL2',
996
- 'FEV+MAP7D1',
997
- 'PTPN12+UBASH3A'],
998
- 'SYNERGY_SIMILAR_PHENO':['CBL+CNN1',
999
- 'CBL+PTPN12',
1000
- 'CBL+PTPN9',
1001
- 'CBL+UBASH3B',
1002
- 'FOXA3+FOXL2',
1003
- 'FOXA3+HOXB9',
1004
- 'FOXL2+HOXB9',
1005
- 'UBASH3B+CNN1',
1006
- 'UBASH3B+PTPN12',
1007
- 'UBASH3B+PTPN9',
1008
- 'UBASH3B+ZBTB25'],
1009
- 'SYNERGY_DISSIMILAR_PHENO': ['AHR+FEV',
1010
- 'DUSP9+SNAI1',
1011
- 'FOXA1+FOXF1',
1012
- 'FOXA1+FOXL2',
1013
- 'FOXA1+HOXB9',
1014
- 'FOXF1+FOXL2',
1015
- 'FOXF1+HOXB9',
1016
- 'FOXL2+MEIS1',
1017
- 'IGDCC3+ZBTB25',
1018
- 'POU3F2+CBFA2T3',
1019
- 'PTPN12+ZBTB25',
1020
- 'SNAI1+DLX2',
1021
- 'SNAI1+UBASH3B'],
1022
- 'SUPPRESSOR': ['CEBPB+PTPN12',
1023
- 'CEBPE+CNN1',
1024
- 'CEBPE+PTPN12',
1025
- 'CNN1+MAPK1',
1026
- 'ETS2+CNN1',
1027
- 'ETS2+IGDCC3',
1028
- 'ETS2+PRTG',
1029
- 'FOSB+UBASH3B',
1030
- 'IGDCC3+MAPK1',
1031
- 'LYL1+CEBPB',
1032
- 'MAPK1+PRTG',
1033
- 'PTPN12+SNAI1']
1034
- }
 
 
 
 
 
1
+ """
2
+ ================================================================================
3
+ Model evaluation module (inference_local.py)
4
+ ================================================================================
5
+
6
+ [Module function]
7
+ Provides evaluation and analysis functions for genetic perturbation models, including:
8
+
9
+ 1. evaluate(): main evaluation function, calculates MSE, Pearson correlation coefficient and other indicators
10
+ 2. compute_metrics(): Calculate the complete dictionary of evaluation metrics
11
+ 3. deeper_analysis(): In-depth analysis of prediction results
12
+ 4. non_dropout_analysis(): analyze on non-dropout genes
13
+
14
+ [Description of key indicators]
15
+ - MSE: mean square error of all genes
16
+ - MSE_DE: MSE of Top 20 differentially expressed genes (paying more attention to biological significance)
17
+ - Pearson: Pearson correlation coefficient of all genes
18
+ - Pearson_DE: Pearson correlation coefficient of Top 20 DE genes
19
+
20
+ [Evaluation Notes]
21
+ - real_num_genes: only evaluate on real genes, excluding extended/completed genes
22
+ - de_idx: The differentially expressed gene index of each sample may be different
23
+ ================================================================================
24
+ """
25
+
26
+ import os
27
+ import torch
28
+ import numpy as np
29
+ import scanpy as sc
30
+
31
+ from sklearn.metrics import r2_score
32
+ from scipy.stats import pearsonr, spearmanr
33
+ from sklearn.metrics import mean_squared_error as mse
34
+ from sklearn.metrics import mean_absolute_error as mae
35
+
36
+ def evaluate(loader, model, uncertainty, device, real_num_genes = None):
37
+ """
38
+ main evaluation function
39
+
40
+ [Function]
41
+ Runs the model on the given data loader, collecting predicted and true values.
42
+
43
+ [Key steps]
44
+ 1. Traverse all batches of the data loader
45
+ 2. Run model inference (without calculating gradients)
46
+ 3. Collect predicted values, true values, and DE gene predicted values
47
+
48
+ [Parameter description]
49
+ - real_num_genes: only evaluate the first N real genes to avoid expanding gene interference indicators
50
+ """
51
+
52
+ model.eval()
53
+ model.to(device)
54
+ pert_cat = []
55
+ pred = []
56
+ truth = []
57
+ pred_de = []
58
+ truth_de = []
59
+ results = {}
60
+ logvar = []
61
+
62
+ for itr, batch in enumerate(loader):
63
+
64
+ batch.to(device)
65
+ pert_cat.extend(batch.pert)
66
+
67
+ with torch.no_grad():
68
+ if uncertainty:
69
+ p, unc = model(batch)
70
+ logvar.extend(unc.cpu())
71
+ else:
72
+ p = model(batch)
73
+ # Only evaluate in the real gene dimension to avoid the interference of completed/expanded genes on indicators.
74
+ p = p[:, :real_num_genes]
75
+ t = batch.y[:, :real_num_genes]
76
+ pred.extend(p.cpu())
77
+ truth.extend(t.cpu())
78
+
79
+ # The DE indicator is only counted on the de_idx corresponding to the sample and is one of the core calibers of downstream reporting.
80
+ for itr, de_idx in enumerate(batch.de_idx):
81
+ pred_de.append(p[itr, de_idx])
82
+ truth_de.append(t[itr, de_idx])
83
+
84
+ # all genes
85
+ results['pert_cat'] = np.array(pert_cat)
86
+ pred = torch.stack(pred)
87
+ truth = torch.stack(truth)
88
+ results['pred']= pred.detach().cpu().numpy()
89
+ results['truth']= truth.detach().cpu().numpy()
90
+
91
+ pred_de = torch.stack(pred_de)
92
+ truth_de = torch.stack(truth_de)
93
+ results['pred_de']= pred_de.detach().cpu().numpy()
94
+ results['truth_de']= truth_de.detach().cpu().numpy()
95
+
96
+ if uncertainty:
97
+ results['logvar'] = torch.stack(logvar).detach().cpu().numpy()
98
+
99
+ return results
100
+
101
+ #This evaluation is just as test. IT HAS INFORMATION LEAKAGE!
102
+ def evaluate_with_nonzero_mask(loader, model, uncertainty, device):
103
+ """
104
+ Run model in inference mode using a given data loader
105
+ """
106
+
107
+ model.eval()
108
+ model.to(device)
109
+ pert_cat = []
110
+ pred = []
111
+ truth = []
112
+ pred_de = []
113
+ truth_de = []
114
+ results = {}
115
+ logvar = []
116
+
117
+ for itr, batch in enumerate(loader):
118
+
119
+ batch.to(device)
120
+ pert_cat.extend(batch.pert)
121
+
122
+ with torch.no_grad():
123
+ if uncertainty:
124
+ p, unc = model(batch)
125
+ logvar.extend(unc.cpu())
126
+ else:
127
+ p = model(batch)
128
+ t = batch.y
129
+ p = p * (t != 0)
130
+ pred.extend(p.cpu())
131
+ truth.extend(t.cpu())
132
+
133
+ # Differentially expressed genes
134
+ for itr, de_idx in enumerate(batch.de_idx):
135
+ pred_de.append(p[itr, de_idx])
136
+ truth_de.append(t[itr, de_idx])
137
+
138
+ # all genes
139
+ results['pert_cat'] = np.array(pert_cat)
140
+ pred = torch.stack(pred)
141
+ truth = torch.stack(truth)
142
+ results['pred']= pred.detach().cpu().numpy()
143
+ results['truth']= truth.detach().cpu().numpy()
144
+
145
+ pred_de = torch.stack(pred_de)
146
+ truth_de = torch.stack(truth_de)
147
+ results['pred_de']= pred_de.detach().cpu().numpy()
148
+ results['truth_de']= truth_de.detach().cpu().numpy()
149
+
150
+ if uncertainty:
151
+ results['logvar'] = torch.stack(logvar).detach().cpu().numpy()
152
+
153
+ return results
154
+
155
+
156
+ def non_zero_evaluate(loader, model, uncertainty, device,
157
+ delta_list, penalty = {
158
+ "false_neg": 1,
159
+ "false_pos": 1 }):
160
+ """
161
+ Run model in inference mode using a given data loader
162
+ """
163
+
164
+ model.eval()
165
+ model.to(device)
166
+ pert_cat = []
167
+ pred = []
168
+ truth = []
169
+ pred_de = []
170
+ truth_de = []
171
+ results = {}
172
+ logvar = []
173
+
174
+ for itr, batch in enumerate(loader):
175
+
176
+ batch.to(device)
177
+ pert_cat.extend(batch.pert)
178
+
179
+ with torch.no_grad():
180
+ if uncertainty:
181
+ p, unc = model(batch)
182
+ logvar.extend(unc.cpu())
183
+ else:
184
+ p = model(batch)
185
+ t = batch.y
186
+ pred.extend(p.cpu())
187
+ truth.extend(t.cpu())
188
+
189
+ # Differentially expressed genes
190
+ for itr, de_idx in enumerate(batch.de_idx):
191
+ pred_de.append(p[itr, de_idx])
192
+ truth_de.append(t[itr, de_idx])
193
+
194
+ # all genes
195
+ results['pert_cat'] = np.array(pert_cat)
196
+ pred = torch.stack(pred)
197
+ truth = torch.stack(truth)
198
+ results['pred']= pred.detach().cpu().numpy()
199
+ results['truth']= truth.detach().cpu().numpy()
200
+
201
+ pred_de = torch.stack(pred_de)
202
+ truth_de = torch.stack(truth_de)
203
+ results['pred_de']= pred_de.detach().cpu().numpy()
204
+ results['truth_de']= truth_de.detach().cpu().numpy()
205
+
206
+ if uncertainty:
207
+ results['logvar'] = torch.stack(logvar).detach().cpu().numpy()
208
+
209
+ return results
210
+
211
+
212
+ def compute_metrics(results, gene_idx=None):
213
+ """
214
+ Given results from a model run and the ground truth, compute metrics
215
+
216
+ """
217
+ metrics = {}
218
+ metrics_pert = {}
219
+
220
+ metric2fct = {
221
+ 'mse': mse,
222
+ 'pearson': pearsonr
223
+ }
224
+
225
+ for m in metric2fct.keys():
226
+ metrics[m] = []
227
+ metrics[m + '_de'] = []
228
+
229
+ for pert in np.unique(results['pert_cat']):
230
+
231
+ metrics_pert[pert] = {}
232
+ p_idx = np.where(results['pert_cat'] == pert)[0]
233
+
234
+ for m, fct in metric2fct.items():
235
+ # First average the cells of the same perturbation, and then calculate the indicator of the perturbation.
236
+ if m == 'pearson':
237
+ val = fct(results['pred'][p_idx].mean(0), results['truth'][p_idx].mean(0))[0]
238
+ if np.isnan(val):
239
+ val = 0
240
+ else:
241
+ val = fct(results['pred'][p_idx].mean(0), results['truth'][p_idx].mean(0))
242
+
243
+ metrics_pert[pert][m] = val
244
+ metrics[m].append(metrics_pert[pert][m])
245
+
246
+
247
+ if pert != 'ctrl':
248
+
249
+ for m, fct in metric2fct.items():
250
+ if m == 'pearson':
251
+ val = fct(results['pred_de'][p_idx].mean(0), results['truth_de'][p_idx].mean(0))[0]
252
+ if np.isnan(val):
253
+ val = 0
254
+ else:
255
+ val = fct(results['pred_de'][p_idx].mean(0), results['truth_de'][p_idx].mean(0))
256
+
257
+ metrics_pert[pert][m + '_de'] = val
258
+ metrics[m + '_de'].append(metrics_pert[pert][m + '_de'])
259
+
260
+ else:
261
+ for m, fct in metric2fct.items():
262
+ metrics_pert[pert][m + '_de'] = 0
263
+
264
+ for m in metric2fct.keys():
265
+ # The result is a perturbation-level macro average (each perturbation is equally weighted).
266
+ metrics[m] = np.mean(metrics[m])
267
+ metrics[m + '_de'] = np.mean(metrics[m + '_de'])
268
+
269
+ return metrics, metrics_pert
270
+
271
+
272
+ def compute_synergy_loss(results, mean_control, high_umi_idx, subtype = 'POTENTIATION'):
273
+ pred_res = get_test_set_results_seen2(results, subtype)
274
+ all_perts = np.unique(results['pert_cat'])
275
+ linear_params = get_linear_params(pred_res, high_umi_idx,
276
+ mean_control,all_perts)
277
+ synergy_loss = np.sum([np.abs(linear_params[k]['pred']['mag']
278
+ - linear_params[k]['truth']['mag']) for k in
279
+ linear_params])
280
+
281
+ mag = np.sum([np.abs(linear_params[k]['pred']['mag']) for k in
282
+ linear_params])
283
+ return synergy_loss, mag
284
+
285
+
286
+ def non_zero_analysis(adata, test_res):
287
+ metric2fct = {
288
+ 'pearson': pearsonr,
289
+ 'mse': mse
290
+ }
291
+
292
+ pert_metric = {}
293
+
294
+ ## in silico modeling and upperbounding
295
+ pert2pert_full_id = dict(adata.obs[['condition', 'condition_name']].values)
296
+ geneid2name = dict(zip(adata.var.index.values, adata.var['gene_name']))
297
+ geneid2idx = dict(zip(adata.var.index.values, range(len(adata.var.index.values))))
298
+
299
+ # calculate mean expression for each condition
300
+ unique_conditions = adata.obs.condition.unique()
301
+ conditions2index = {}
302
+ for i in unique_conditions:
303
+ conditions2index[i] = np.where(adata.obs.condition == i)[0]
304
+
305
+ condition2mean_expression = {}
306
+ for i, j in conditions2index.items():
307
+ condition2mean_expression[i] = np.mean(adata.X[j], axis = 0)
308
+ pert_list = np.array(list(condition2mean_expression.keys()))
309
+ mean_expression = np.array(list(condition2mean_expression.values())).reshape(len(adata.obs.condition.unique()), adata.X.toarray().shape[1])
310
+ ctrl = mean_expression[np.where(pert_list == 'ctrl')[0]]
311
+
312
+ gene_list = adata.var['gene_name'].values
313
+
314
+ for pert in np.unique(test_res['pert_cat']):
315
+ pert_metric[pert] = {}
316
+
317
+ pert_idx = np.where(test_res['pert_cat'] == pert)[0]
318
+ de_idx = [geneid2idx[i] for i in adata.uns['top_non_zero_de_20'][pert2pert_full_id[pert]]]
319
+
320
+ direc_change = np.abs(np.sign(test_res['pred'][pert_idx].mean(0)[de_idx] - ctrl[0][de_idx]) - np.sign(test_res['truth'][pert_idx].mean(0)[de_idx] - ctrl[0][de_idx]))
321
+ frac_correct_direction = len(np.where(direc_change == 0)[0])/len(de_idx)
322
+ pert_metric[pert]['frac_correct_direction_top20_non_zero'] = frac_correct_direction
323
+
324
+ frac_direction_opposite = len(np.where(direc_change == 2)[0])/len(de_idx)
325
+ pert_metric[pert]['frac_opposite_direction_top20_non_zero'] = frac_direction_opposite
326
+
327
+ frac_direction_opposite = len(np.where(direc_change == 1)[0])/len(de_idx)
328
+ pert_metric[pert]['frac_0/1_direction_top20_non_zero'] = frac_direction_opposite
329
+
330
+ mean = np.mean(test_res['truth'][pert_idx][:, de_idx], axis = 0)
331
+ std = np.std(test_res['truth'][pert_idx][:, de_idx], axis = 0)
332
+ min_ = np.min(test_res['truth'][pert_idx][:, de_idx], axis = 0)
333
+ max_ = np.max(test_res['truth'][pert_idx][:, de_idx], axis = 0)
334
+ q25 = np.quantile(test_res['truth'][pert_idx][:, de_idx], 0.25, axis = 0)
335
+ q75 = np.quantile(test_res['truth'][pert_idx][:, de_idx], 0.75, axis = 0)
336
+ q55 = np.quantile(test_res['truth'][pert_idx][:, de_idx], 0.55, axis = 0)
337
+ q45 = np.quantile(test_res['truth'][pert_idx][:, de_idx], 0.45, axis = 0)
338
+ q40 = np.quantile(test_res['truth'][pert_idx][:, de_idx], 0.4, axis = 0)
339
+ q60 = np.quantile(test_res['truth'][pert_idx][:, de_idx], 0.6, axis = 0)
340
+
341
+ zero_des = np.intersect1d(np.where(min_ == 0)[0], np.where(max_ == 0)[0])
342
+ nonzero_des = np.setdiff1d(list(range(20)), zero_des)
343
+
344
+ if len(nonzero_des) == 0:
345
+ pass
346
+ # pert that all de genes are 0...
347
+ else:
348
+ pred_mean = np.mean(test_res['pred'][pert_idx][:, de_idx], axis = 0).reshape(-1,)
349
+ true_mean = np.mean(test_res['truth'][pert_idx][:, de_idx], axis = 0).reshape(-1,)
350
+
351
+ in_range = (pred_mean[nonzero_des] >= min_[nonzero_des]) & (pred_mean[nonzero_des] <= max_[nonzero_des])
352
+ frac_in_range = sum(in_range)/len(nonzero_des)
353
+ pert_metric[pert]['frac_in_range_non_zero'] = frac_in_range
354
+
355
+ in_range_5 = (pred_mean[nonzero_des] >= q45[nonzero_des]) & (pred_mean[nonzero_des] <= q55[nonzero_des])
356
+ frac_in_range_45_55 = sum(in_range_5)/len(nonzero_des)
357
+ pert_metric[pert]['frac_in_range_45_55_non_zero'] = frac_in_range_45_55
358
+
359
+ in_range_10 = (pred_mean[nonzero_des] >= q40[nonzero_des]) & (pred_mean[nonzero_des] <= q60[nonzero_des])
360
+ frac_in_range_40_60 = sum(in_range_10)/len(nonzero_des)
361
+ pert_metric[pert]['frac_in_range_40_60_non_zero'] = frac_in_range_40_60
362
+
363
+ in_range_25 = (pred_mean[nonzero_des] >= q25[nonzero_des]) & (pred_mean[nonzero_des] <= q75[nonzero_des])
364
+ frac_in_range_25_75 = sum(in_range_25)/len(nonzero_des)
365
+ pert_metric[pert]['frac_in_range_25_75_non_zero'] = frac_in_range_25_75
366
+
367
+ zero_idx = np.where(std > 0)[0]
368
+ sigma = (np.abs(pred_mean[zero_idx] - mean[zero_idx]))/(std[zero_idx])
369
+ pert_metric[pert]['mean_sigma_non_zero'] = np.mean(sigma)
370
+ pert_metric[pert]['std_sigma_non_zero'] = np.std(sigma)
371
+ pert_metric[pert]['frac_sigma_below_1_non_zero'] = 1 - len(np.where(sigma > 1)[0])/len(zero_idx)
372
+ pert_metric[pert]['frac_sigma_below_2_non_zero'] = 1 - len(np.where(sigma > 2)[0])/len(zero_idx)
373
+
374
+ p_idx = np.where(test_res['pert_cat'] == pert)[0]
375
+ for m, fct in metric2fct.items():
376
+ if m != 'mse':
377
+ val = fct(test_res['pred'][p_idx].mean(0)[de_idx] - ctrl[0][de_idx], test_res['truth'][p_idx].mean(0)[de_idx]-ctrl[0][de_idx])[0]
378
+ if np.isnan(val):
379
+ val = 0
380
+ pert_metric[pert][m + '_delta_top20_de_non_zero'] = val
381
+
382
+
383
+ val = fct(test_res['pred'][p_idx].mean(0)[de_idx], test_res['truth'][p_idx].mean(0)[de_idx])[0]
384
+ if np.isnan(val):
385
+ val = 0
386
+ pert_metric[pert][m + '_top20_de_non_zero'] = val
387
+ else:
388
+ val = fct(test_res['pred'][p_idx].mean(0)[de_idx] - ctrl[0][de_idx], test_res['truth'][p_idx].mean(0)[de_idx]-ctrl[0][de_idx])
389
+ pert_metric[pert][m + '_top20_de_non_zero'] = val
390
+
391
+ return pert_metric
392
+
393
+ def non_dropout_analysis(adata, test_res):
394
+ metric2fct = {
395
+ 'pearson': pearsonr,
396
+ 'mse': mse
397
+ }
398
+
399
+ pert_metric = {}
400
+
401
+ ## in silico modeling and upperbounding
402
+ pert2pert_full_id = dict(adata.obs[['condition', 'condition_name']].values)
403
+ geneid2name = dict(zip(adata.var.index.values, adata.var['gene_name']))
404
+ geneid2idx = dict(zip(adata.var.index.values, range(len(adata.var.index.values))))
405
+
406
+ # calculate mean expression for each condition
407
+ unique_conditions = adata.obs.condition.unique()
408
+ conditions2index = {}
409
+ for i in unique_conditions:
410
+ conditions2index[i] = np.where(adata.obs.condition == i)[0]
411
+
412
+ condition2mean_expression = {}
413
+ for i, j in conditions2index.items():
414
+ condition2mean_expression[i] = np.mean(adata.X[j], axis = 0)
415
+ pert_list = np.array(list(condition2mean_expression.keys()))
416
+ mean_expression = np.array(list(condition2mean_expression.values())).reshape(len(adata.obs.condition.unique()), adata.X.toarray().shape[1])
417
+ ctrl = mean_expression[np.where(pert_list == 'ctrl')[0]]
418
+
419
+ gene_list = adata.var['gene_name'].values
420
+
421
+ for pert in np.unique(test_res['pert_cat']):
422
+ pert_metric[pert] = {}
423
+
424
+ pert_idx = np.where(test_res['pert_cat'] == pert)[0]
425
+ de_idx = [geneid2idx[i] for i in adata.uns['top_non_dropout_de_20'][pert2pert_full_id[pert]]]
426
+ non_zero_idx = adata.uns['non_zeros_gene_idx'][pert2pert_full_id[pert]]
427
+ non_dropout_gene_idx = adata.uns['non_dropout_gene_idx'][pert2pert_full_id[pert]]
428
+
429
+ direc_change = np.abs(np.sign(test_res['pred'][pert_idx].mean(0)[de_idx] - ctrl[0][de_idx]) - np.sign(test_res['truth'][pert_idx].mean(0)[de_idx] - ctrl[0][de_idx]))
430
+ frac_correct_direction = len(np.where(direc_change == 0)[0])/len(de_idx)
431
+ pert_metric[pert]['frac_correct_direction_top20_non_dropout'] = frac_correct_direction
432
+
433
+ frac_direction_opposite = len(np.where(direc_change == 2)[0])/len(de_idx)
434
+ pert_metric[pert]['frac_opposite_direction_top20_non_dropout'] = frac_direction_opposite
435
+
436
+ frac_direction_opposite = len(np.where(direc_change == 1)[0])/len(de_idx)
437
+ pert_metric[pert]['frac_0/1_direction_top20_non_dropout'] = frac_direction_opposite
438
+
439
+ direc_change = np.abs(np.sign(test_res['pred'][pert_idx].mean(0)[non_zero_idx] - ctrl[0][non_zero_idx]) - np.sign(test_res['truth'][pert_idx].mean(0)[non_zero_idx] - ctrl[0][non_zero_idx]))
440
+ frac_correct_direction = len(np.where(direc_change == 0)[0])/len(non_zero_idx)
441
+ pert_metric[pert]['frac_correct_direction_non_zero'] = frac_correct_direction
442
+
443
+ frac_direction_opposite = len(np.where(direc_change == 2)[0])/len(non_zero_idx)
444
+ pert_metric[pert]['frac_opposite_direction_non_zero'] = frac_direction_opposite
445
+
446
+ frac_direction_opposite = len(np.where(direc_change == 1)[0])/len(non_zero_idx)
447
+ pert_metric[pert]['frac_0/1_direction_non_zero'] = frac_direction_opposite
448
+
449
+ direc_change = np.abs(np.sign(test_res['pred'][pert_idx].mean(0)[non_dropout_gene_idx] - ctrl[0][non_dropout_gene_idx]) - np.sign(test_res['truth'][pert_idx].mean(0)[non_dropout_gene_idx] - ctrl[0][non_dropout_gene_idx]))
450
+ frac_correct_direction = len(np.where(direc_change == 0)[0])/len(non_dropout_gene_idx)
451
+ pert_metric[pert]['frac_correct_direction_non_dropout'] = frac_correct_direction
452
+
453
+ frac_direction_opposite = len(np.where(direc_change == 2)[0])/len(non_dropout_gene_idx)
454
+ pert_metric[pert]['frac_opposite_direction_non_dropout'] = frac_direction_opposite
455
+
456
+ frac_direction_opposite = len(np.where(direc_change == 1)[0])/len(non_dropout_gene_idx)
457
+ pert_metric[pert]['frac_0/1_direction_non_dropout'] = frac_direction_opposite
458
+
459
+ mean = np.mean(test_res['truth'][pert_idx][:, de_idx], axis = 0)
460
+ std = np.std(test_res['truth'][pert_idx][:, de_idx], axis = 0)
461
+ min_ = np.min(test_res['truth'][pert_idx][:, de_idx], axis = 0)
462
+ max_ = np.max(test_res['truth'][pert_idx][:, de_idx], axis = 0)
463
+ q25 = np.quantile(test_res['truth'][pert_idx][:, de_idx], 0.25, axis = 0)
464
+ q75 = np.quantile(test_res['truth'][pert_idx][:, de_idx], 0.75, axis = 0)
465
+ q55 = np.quantile(test_res['truth'][pert_idx][:, de_idx], 0.55, axis = 0)
466
+ q45 = np.quantile(test_res['truth'][pert_idx][:, de_idx], 0.45, axis = 0)
467
+ q40 = np.quantile(test_res['truth'][pert_idx][:, de_idx], 0.4, axis = 0)
468
+ q60 = np.quantile(test_res['truth'][pert_idx][:, de_idx], 0.6, axis = 0)
469
+
470
+ zero_des = np.intersect1d(np.where(min_ == 0)[0], np.where(max_ == 0)[0])
471
+ nonzero_des = np.setdiff1d(list(range(20)), zero_des)
472
+
473
+ if len(nonzero_des) == 0:
474
+ pass
475
+ # pert that all de genes are 0...
476
+ else:
477
+ pred_mean = np.mean(test_res['pred'][pert_idx][:, de_idx], axis = 0).reshape(-1,)
478
+ true_mean = np.mean(test_res['truth'][pert_idx][:, de_idx], axis = 0).reshape(-1,)
479
+
480
+ in_range = (pred_mean[nonzero_des] >= min_[nonzero_des]) & (pred_mean[nonzero_des] <= max_[nonzero_des])
481
+ frac_in_range = sum(in_range)/len(nonzero_des)
482
+ pert_metric[pert]['frac_in_range_non_dropout'] = frac_in_range
483
+
484
+ in_range_5 = (pred_mean[nonzero_des] >= q45[nonzero_des]) & (pred_mean[nonzero_des] <= q55[nonzero_des])
485
+ frac_in_range_45_55 = sum(in_range_5)/len(nonzero_des)
486
+ pert_metric[pert]['frac_in_range_45_55_non_dropout'] = frac_in_range_45_55
487
+
488
+ in_range_10 = (pred_mean[nonzero_des] >= q40[nonzero_des]) & (pred_mean[nonzero_des] <= q60[nonzero_des])
489
+ frac_in_range_40_60 = sum(in_range_10)/len(nonzero_des)
490
+ pert_metric[pert]['frac_in_range_40_60_non_dropout'] = frac_in_range_40_60
491
+
492
+ in_range_25 = (pred_mean[nonzero_des] >= q25[nonzero_des]) & (pred_mean[nonzero_des] <= q75[nonzero_des])
493
+ frac_in_range_25_75 = sum(in_range_25)/len(nonzero_des)
494
+ pert_metric[pert]['frac_in_range_25_75_non_dropout'] = frac_in_range_25_75
495
+
496
+ zero_idx = np.where(std > 0)[0]
497
+ sigma = (np.abs(pred_mean[zero_idx] - mean[zero_idx]))/(std[zero_idx])
498
+ pert_metric[pert]['mean_sigma_non_dropout'] = np.mean(sigma)
499
+ pert_metric[pert]['std_sigma_non_dropout'] = np.std(sigma)
500
+ pert_metric[pert]['frac_sigma_below_1_non_dropout'] = 1 - len(np.where(sigma > 1)[0])/len(zero_idx)
501
+ pert_metric[pert]['frac_sigma_below_2_non_dropout'] = 1 - len(np.where(sigma > 2)[0])/len(zero_idx)
502
+
503
+ p_idx = np.where(test_res['pert_cat'] == pert)[0]
504
+ for m, fct in metric2fct.items():
505
+ if m != 'mse':
506
+ val = fct(test_res['pred'][p_idx].mean(0)[de_idx] - ctrl[0][de_idx], test_res['truth'][p_idx].mean(0)[de_idx]-ctrl[0][de_idx])[0]
507
+ if np.isnan(val):
508
+ val = 0
509
+ pert_metric[pert][m + '_delta_top20_de_non_dropout'] = val
510
+
511
+
512
+ val = fct(test_res['pred'][p_idx].mean(0)[de_idx], test_res['truth'][p_idx].mean(0)[de_idx])[0]
513
+ if np.isnan(val):
514
+ val = 0
515
+ pert_metric[pert][m + '_top20_de_non_dropout'] = val
516
+ else:
517
+ val = fct(test_res['pred'][p_idx].mean(0)[de_idx] - ctrl[0][de_idx], test_res['truth'][p_idx].mean(0)[de_idx]-ctrl[0][de_idx])
518
+ pert_metric[pert][m + '_top20_de_non_dropout'] = val
519
+
520
+ return pert_metric
521
+
522
+ def deeper_analysis(adata, test_res, de_column_prefix = 'rank_genes_groups_cov', most_variable_genes = None):
523
+
524
+ metric2fct = {
525
+ 'pearson': pearsonr,
526
+ 'mse': mse
527
+ }
528
+
529
+ pert_metric = {}
530
+
531
+ ## in silico modeling and upperbounding
532
+ pert2pert_full_id = dict(adata.obs[['condition', 'condition_name']].values)
533
+ geneid2name = dict(zip(adata.var.index.values, adata.var['gene_name']))
534
+ geneid2idx = dict(zip(adata.var.index.values, range(len(adata.var.index.values))))
535
+
536
+ # calculate mean expression for each condition
537
+ unique_conditions = adata.obs.condition.unique()
538
+ conditions2index = {}
539
+ for i in unique_conditions:
540
+ conditions2index[i] = np.where(adata.obs.condition == i)[0]
541
+
542
+ condition2mean_expression = {}
543
+ for i, j in conditions2index.items():
544
+ condition2mean_expression[i] = np.mean(adata.X[j], axis = 0)
545
+ pert_list = np.array(list(condition2mean_expression.keys()))
546
+ mean_expression = np.array(list(condition2mean_expression.values())).reshape(len(adata.obs.condition.unique()), adata.X.toarray().shape[1])
547
+ ctrl = mean_expression[np.where(pert_list == 'ctrl')[0]]
548
+
549
+ if most_variable_genes is None:
550
+ most_variable_genes = np.argsort(np.std(mean_expression, axis = 0))[-200:]
551
+
552
+ gene_list = adata.var['gene_name'].values
553
+
554
+ for pert in np.unique(test_res['pert_cat']):
555
+ pert_metric[pert] = {}
556
+ de_idx = [geneid2idx[i] for i in adata.uns['rank_genes_groups_cov_all'][pert2pert_full_id[pert]][:20]]
557
+ de_idx_200 = [geneid2idx[i] for i in adata.uns['rank_genes_groups_cov_all'][pert2pert_full_id[pert]][:200]]
558
+ de_idx_100 = [geneid2idx[i] for i in adata.uns['rank_genes_groups_cov_all'][pert2pert_full_id[pert]][:100]]
559
+ de_idx_50 = [geneid2idx[i] for i in adata.uns['rank_genes_groups_cov_all'][pert2pert_full_id[pert]][:50]]
560
+
561
+ pert_idx = np.where(test_res['pert_cat'] == pert)[0]
562
+ pred_mean = np.mean(test_res['pred_de'][pert_idx], axis = 0).reshape(-1,)
563
+ true_mean = np.mean(test_res['truth_de'][pert_idx], axis = 0).reshape(-1,)
564
+
565
+ direc_change = np.abs(np.sign(test_res['pred'][pert_idx].mean(0) - ctrl[0]) - np.sign(test_res['truth'][pert_idx].mean(0) - ctrl[0]))
566
+ frac_correct_direction = len(np.where(direc_change == 0)[0])/len(geneid2name)
567
+ pert_metric[pert]['frac_correct_direction_all'] = frac_correct_direction
568
+
569
+ de_idx_map = {20: de_idx,
570
+ 50: de_idx_50,
571
+ 100: de_idx_100,
572
+ 200: de_idx_200
573
+ }
574
+
575
+ for val in [20, 50, 100, 200]:
576
+
577
+ direc_change = np.abs(np.sign(test_res['pred'][pert_idx].mean(0)[de_idx_map[val]] - ctrl[0][de_idx_map[val]]) - np.sign(test_res['truth'][pert_idx].mean(0)[de_idx_map[val]] - ctrl[0][de_idx_map[val]]))
578
+ frac_correct_direction = len(np.where(direc_change == 0)[0])/val
579
+ pert_metric[pert]['frac_correct_direction_' + str(val)] = frac_correct_direction
580
+
581
+ mean = np.mean(test_res['truth_de'][pert_idx], axis = 0)
582
+ std = np.std(test_res['truth_de'][pert_idx], axis = 0)
583
+ min_ = np.min(test_res['truth_de'][pert_idx], axis = 0)
584
+ max_ = np.max(test_res['truth_de'][pert_idx], axis = 0)
585
+ q25 = np.quantile(test_res['truth_de'][pert_idx], 0.25, axis = 0)
586
+ q75 = np.quantile(test_res['truth_de'][pert_idx], 0.75, axis = 0)
587
+ q55 = np.quantile(test_res['truth_de'][pert_idx], 0.55, axis = 0)
588
+ q45 = np.quantile(test_res['truth_de'][pert_idx], 0.45, axis = 0)
589
+ q40 = np.quantile(test_res['truth_de'][pert_idx], 0.4, axis = 0)
590
+ q60 = np.quantile(test_res['truth_de'][pert_idx], 0.6, axis = 0)
591
+
592
+ zero_des = np.intersect1d(np.where(min_ == 0)[0], np.where(max_ == 0)[0])
593
+ nonzero_des = np.setdiff1d(list(range(20)), zero_des)
594
+ if len(nonzero_des) == 0:
595
+ pass
596
+ # pert that all de genes are 0...
597
+ else:
598
+
599
+ direc_change = np.abs(np.sign(pred_mean[nonzero_des] - ctrl[0][de_idx][nonzero_des]) - np.sign(true_mean[nonzero_des] - ctrl[0][de_idx][nonzero_des]))
600
+ frac_correct_direction = len(np.where(direc_change == 0)[0])/len(nonzero_des)
601
+ pert_metric[pert]['frac_correct_direction_20_nonzero'] = frac_correct_direction
602
+
603
+ in_range = (pred_mean[nonzero_des] >= min_[nonzero_des]) & (pred_mean[nonzero_des] <= max_[nonzero_des])
604
+ frac_in_range = sum(in_range)/len(nonzero_des)
605
+ pert_metric[pert]['frac_in_range'] = frac_in_range
606
+
607
+ in_range_5 = (pred_mean[nonzero_des] >= q45[nonzero_des]) & (pred_mean[nonzero_des] <= q55[nonzero_des])
608
+ frac_in_range_45_55 = sum(in_range_5)/len(nonzero_des)
609
+ pert_metric[pert]['frac_in_range_45_55'] = frac_in_range_45_55
610
+
611
+ in_range_10 = (pred_mean[nonzero_des] >= q40[nonzero_des]) & (pred_mean[nonzero_des] <= q60[nonzero_des])
612
+ frac_in_range_40_60 = sum(in_range_10)/len(nonzero_des)
613
+ pert_metric[pert]['frac_in_range_40_60'] = frac_in_range_40_60
614
+
615
+ in_range_25 = (pred_mean[nonzero_des] >= q25[nonzero_des]) & (pred_mean[nonzero_des] <= q75[nonzero_des])
616
+ frac_in_range_25_75 = sum(in_range_25)/len(nonzero_des)
617
+ pert_metric[pert]['frac_in_range_25_75'] = frac_in_range_25_75
618
+
619
+ zero_idx = np.where(std > 0)[0]
620
+ sigma = (np.abs(pred_mean[zero_idx] - mean[zero_idx]))/(std[zero_idx])
621
+ pert_metric[pert]['mean_sigma'] = np.mean(sigma)
622
+ pert_metric[pert]['std_sigma'] = np.std(sigma)
623
+ pert_metric[pert]['frac_sigma_below_1'] = 1 - len(np.where(sigma > 1)[0])/len(zero_idx)
624
+ pert_metric[pert]['frac_sigma_below_2'] = 1 - len(np.where(sigma > 2)[0])/len(zero_idx)
625
+
626
+ ## correlation on delta
627
+ p_idx = np.where(test_res['pert_cat'] == pert)[0]
628
+
629
+ for m, fct in metric2fct.items():
630
+ if m != 'mse':
631
+ val = fct(test_res['pred'][p_idx].mean(0)- ctrl[0], test_res['truth'][p_idx].mean(0)-ctrl[0])[0]
632
+ if np.isnan(val):
633
+ val = 0
634
+
635
+ pert_metric[pert][m + '_delta'] = val
636
+
637
+ val = fct(test_res['pred'][p_idx].mean(0)[de_idx] - ctrl[0][de_idx], test_res['truth'][p_idx].mean(0)[de_idx]-ctrl[0][de_idx])[0]
638
+ if np.isnan(val):
639
+ val = 0
640
+
641
+ pert_metric[pert][m + '_delta_de'] = val
642
+
643
+ ## up fold changes > 10,
644
+ pert_mean = np.mean(test_res['truth'][p_idx], axis = 0).reshape(-1,)
645
+
646
+ fold_change = pert_mean/ctrl
647
+ fold_change[np.isnan(fold_change)] = 0
648
+ fold_change[np.isinf(fold_change)] = 0
649
+ ## this is to remove the ones that are super low and the fold change becomes unmeaningful
650
+ fold_change[0][np.where(pert_mean < 0.5)[0]] = 0
651
+
652
+ o = np.where(fold_change[0] > 0)[0]
653
+
654
+ pred_fc = test_res['pred'][p_idx].mean(0)[o]
655
+ true_fc = test_res['truth'][p_idx].mean(0)[o]
656
+ ctrl_fc = ctrl[0][o]
657
+
658
+ if len(o) > 0:
659
+ pert_metric[pert]['fold_change_gap_all'] = np.mean(np.abs(pred_fc/ctrl_fc - true_fc/ctrl_fc))
660
+
661
+
662
+ o = np.intersect1d(np.where(fold_change[0] <0.333)[0], np.where(fold_change[0] > 0)[0])
663
+
664
+ pred_fc = test_res['pred'][p_idx].mean(0)[o]
665
+ true_fc = test_res['truth'][p_idx].mean(0)[o]
666
+ ctrl_fc = ctrl[0][o]
667
+
668
+ if len(o) > 0:
669
+ pert_metric[pert]['fold_change_gap_downreg_0.33'] = np.mean(np.abs(pred_fc/ctrl_fc - true_fc/ctrl_fc))
670
+
671
+
672
+ o = np.intersect1d(np.where(fold_change[0] <0.1)[0], np.where(fold_change[0] > 0)[0])
673
+
674
+ pred_fc = test_res['pred'][p_idx].mean(0)[o]
675
+ true_fc = test_res['truth'][p_idx].mean(0)[o]
676
+ ctrl_fc = ctrl[0][o]
677
+
678
+ if len(o) > 0:
679
+ pert_metric[pert]['fold_change_gap_downreg_0.1'] = np.mean(np.abs(pred_fc/ctrl_fc - true_fc/ctrl_fc))
680
+
681
+ o = np.where(fold_change[0] > 3)[0]
682
+
683
+ pred_fc = test_res['pred'][p_idx].mean(0)[o]
684
+ true_fc = test_res['truth'][p_idx].mean(0)[o]
685
+ ctrl_fc = ctrl[0][o]
686
+
687
+ if len(o) > 0:
688
+ pert_metric[pert]['fold_change_gap_upreg_3'] = np.mean(np.abs(pred_fc/ctrl_fc - true_fc/ctrl_fc))
689
+
690
+ o = np.where(fold_change[0] > 10)[0]
691
+
692
+ pred_fc = test_res['pred'][p_idx].mean(0)[o]
693
+ true_fc = test_res['truth'][p_idx].mean(0)[o]
694
+ ctrl_fc = ctrl[0][o]
695
+
696
+ if len(o) > 0:
697
+ pert_metric[pert]['fold_change_gap_upreg_10'] = np.mean(np.abs(pred_fc/ctrl_fc - true_fc/ctrl_fc))
698
+
699
+ ## most variable genes
700
+ for m, fct in metric2fct.items():
701
+ if m != 'mse':
702
+ val = fct(test_res['pred'][p_idx].mean(0)[most_variable_genes] - ctrl[0][most_variable_genes], test_res['truth'][p_idx].mean(0)[most_variable_genes]-ctrl[0][most_variable_genes])[0]
703
+ if np.isnan(val):
704
+ val = 0
705
+ pert_metric[pert][m + '_delta_top200_hvg'] = val
706
+
707
+
708
+ val = fct(test_res['pred'][p_idx].mean(0)[most_variable_genes], test_res['truth'][p_idx].mean(0)[most_variable_genes])[0]
709
+ if np.isnan(val):
710
+ val = 0
711
+ pert_metric[pert][m + '_top200_hvg'] = val
712
+ else:
713
+ val = fct(test_res['pred'][p_idx].mean(0)[most_variable_genes], test_res['truth'][p_idx].mean(0)[most_variable_genes])
714
+ pert_metric[pert][m + '_top200_hvg'] = val
715
+
716
+
717
+ ## top 20/50/100/200 DEs
718
+ for m, fct in metric2fct.items():
719
+ if m != 'mse':
720
+ val = fct(test_res['pred'][p_idx].mean(0)[de_idx] - ctrl[0][de_idx], test_res['truth'][p_idx].mean(0)[de_idx]-ctrl[0][de_idx])[0]
721
+ if np.isnan(val):
722
+ val = 0
723
+ pert_metric[pert][m + '_delta_top20_de'] = val
724
+
725
+
726
+ val = fct(test_res['pred'][p_idx].mean(0)[de_idx], test_res['truth'][p_idx].mean(0)[de_idx])[0]
727
+ if np.isnan(val):
728
+ val = 0
729
+ pert_metric[pert][m + '_top20_de'] = val
730
+ else:
731
+ val = fct(test_res['pred'][p_idx].mean(0)[de_idx] - ctrl[0][de_idx], test_res['truth'][p_idx].mean(0)[de_idx]-ctrl[0][de_idx])
732
+ pert_metric[pert][m + '_top20_de'] = val
733
+
734
+
735
+ for m, fct in metric2fct.items():
736
+ if m != 'mse':
737
+ val = fct(test_res['pred'][p_idx].mean(0)[de_idx_200] - ctrl[0][de_idx_200], test_res['truth'][p_idx].mean(0)[de_idx_200]-ctrl[0][de_idx_200])[0]
738
+ if np.isnan(val):
739
+ val = 0
740
+ pert_metric[pert][m + '_delta_top200_de'] = val
741
+
742
+
743
+ val = fct(test_res['pred'][p_idx].mean(0)[de_idx_200], test_res['truth'][p_idx].mean(0)[de_idx_200])[0]
744
+ if np.isnan(val):
745
+ val = 0
746
+ pert_metric[pert][m + '_top200_de'] = val
747
+ else:
748
+ val = fct(test_res['pred'][p_idx].mean(0)[de_idx_200] - ctrl[0][de_idx_200], test_res['truth'][p_idx].mean(0)[de_idx_200]-ctrl[0][de_idx_200])
749
+ pert_metric[pert][m + '_top200_de'] = val
750
+
751
+ for m, fct in metric2fct.items():
752
+ if m != 'mse':
753
+
754
+ val = fct(test_res['pred'][p_idx].mean(0)[de_idx_100] - ctrl[0][de_idx_100], test_res['truth'][p_idx].mean(0)[de_idx_100]-ctrl[0][de_idx_100])[0]
755
+ if np.isnan(val):
756
+ val = 0
757
+ pert_metric[pert][m + '_delta_top100_de'] = val
758
+
759
+
760
+ val = fct(test_res['pred'][p_idx].mean(0)[de_idx_100], test_res['truth'][p_idx].mean(0)[de_idx_100])[0]
761
+ if np.isnan(val):
762
+ val = 0
763
+ pert_metric[pert][m + '_top100_de'] = val
764
+ else:
765
+ val = fct(test_res['pred'][p_idx].mean(0)[de_idx_100] - ctrl[0][de_idx_100], test_res['truth'][p_idx].mean(0)[de_idx_100]-ctrl[0][de_idx_100])
766
+ pert_metric[pert][m + '_top100_de'] = val
767
+
768
+ for m, fct in metric2fct.items():
769
+ if m != 'mse':
770
+
771
+ val = fct(test_res['pred'][p_idx].mean(0)[de_idx_50] - ctrl[0][de_idx_50], test_res['truth'][p_idx].mean(0)[de_idx_50]-ctrl[0][de_idx_50])[0]
772
+ if np.isnan(val):
773
+ val = 0
774
+ pert_metric[pert][m + '_delta_top50_de'] = val
775
+
776
+
777
+ val = fct(test_res['pred'][p_idx].mean(0)[de_idx_50], test_res['truth'][p_idx].mean(0)[de_idx_50])[0]
778
+ if np.isnan(val):
779
+ val = 0
780
+ pert_metric[pert][m + '_top50_de'] = val
781
+ else:
782
+ val = fct(test_res['pred'][p_idx].mean(0)[de_idx_50] - ctrl[0][de_idx_50], test_res['truth'][p_idx].mean(0)[de_idx_50]-ctrl[0][de_idx_50])
783
+ pert_metric[pert][m + '_top50_de'] = val
784
+
785
+
786
+
787
+ return pert_metric
788
+
789
+
790
+ def GI_subgroup(pert_metric):
791
+ GI_type2Score = {}
792
+ test_pert_list = list(pert_metric.keys())
793
+ for GI_type, gi_list in GIs.items():
794
+ intersect = np.intersect1d(gi_list, test_pert_list)
795
+ if len(intersect) != 0:
796
+ GI_type2Score[GI_type] = {}
797
+
798
+ for m in list(list(pert_metric.values())[0].keys()):
799
+ GI_type2Score[GI_type][m] = np.mean([pert_metric[i][m] for i in intersect if m in pert_metric[i]])
800
+
801
+ return GI_type2Score
802
+
803
+ def node_specific_batch_out(models, batch):
804
+ # Returns output for all node specific models as a matrix of dimension batch_size x nodes
805
+ outs = []
806
+ for idx in range(len(models)):
807
+ outs.append(models[idx](batch).detach().cpu().numpy()[:,idx])
808
+ return np.vstack(outs).T
809
+
810
+ # Run prediction over all batches
811
+ def batch_predict(loader, loaded_models, args):
812
+ # Prediction for node specific GNNs
813
+ preds = []
814
+ print("Loader size: ", len(loader))
815
+ for itr, batch in enumerate(loader):
816
+ print(itr)
817
+ batch = batch.to(args['device'])
818
+ preds.append(node_specific_batch_out(loaded_models, batch))
819
+
820
+ preds = np.vstack(preds)
821
+ return preds
822
+
823
+ def get_high_umi_idx(gene_list, high_umi_path=None):
824
+ """Return indices for a caller-supplied high-expression gene list."""
825
+ high_umi_path = high_umi_path or os.environ.get('BIOMOD_HIGH_UMI_PATH')
826
+ if not high_umi_path or not os.path.isfile(high_umi_path):
827
+ raise FileNotFoundError(
828
+ "Provide high_umi_path or set BIOMOD_HIGH_UMI_PATH to the "
829
+ "genes_with_hi_mean.npy resource; this file is not bundled."
830
+ )
831
+ high_umi = np.load(high_umi_path, allow_pickle=True)
832
+ high_umi_idx = np.where([g in high_umi for g in gene_list])[0]
833
+ return high_umi_idx
834
+
835
+ def get_mean_ctrl(adata):
836
+ return adata[adata.obs['condition'] == 'ctrl'].to_df().mean().reset_index(
837
+ drop=True)
838
+
839
+ def get_single_name(g, all_perts):
840
+ name = g+'+ctrl'
841
+ if name in all_perts:
842
+ return name
843
+ else:
844
+ return 'ctrl+'+g
845
+
846
+
847
+ def get_test_set_results_seen2(res, sel_GI_type):
848
+ # Get relevant test set results
849
+ test_pert_cats = [p for p in np.unique(res['pert_cat']) if
850
+ p in GIs[sel_GI_type] or 'ctrl' in p]
851
+ pred_idx = np.where([t in test_pert_cats for t in res['pert_cat']])
852
+ out = {}
853
+ for key in res:
854
+ out[key] = res[key][pred_idx]
855
+ return out
856
+
857
+ ## Synergy loss calculation functions
858
+ def get_all_vectors(all_res, mean_control, double,
859
+ single1, single2, high_umi_idx):
860
+ # Pred
861
+ pred_df = pd.DataFrame(all_res['pred'])
862
+ pred_df['condition'] = all_res['pert_cat']
863
+ subset_df = pred_df[pred_df['condition'] == double].iloc[:, :-1]
864
+ delta_double_pred = subset_df.mean(0) - mean_control
865
+ single_df_1_pred = pred_df[pred_df['condition'] == single1].iloc[:, :-1]
866
+ single_df_2_pred = pred_df[pred_df['condition'] == single2].iloc[:, :-1]
867
+
868
+ # True
869
+ truth_df = pd.DataFrame(all_res['truth'])
870
+ truth_df['condition'] = all_res['pert_cat']
871
+ subset_df = truth_df[truth_df['condition'] == double].iloc[:, :-1]
872
+ delta_double_truth = subset_df.mean(0) - mean_control
873
+ single_df_1_truth = truth_df[truth_df['condition'] == single1].iloc[:, :-1]
874
+ single_df_2_truth = truth_df[truth_df['condition'] == single2].iloc[:, :-1]
875
+
876
+ delta_single_truth_1 = single_df_1_truth.mean(0) - mean_control
877
+ delta_single_truth_2 = single_df_2_truth.mean(0) - mean_control
878
+ delta_single_pred_1 = single_df_1_pred.mean(0) - mean_control
879
+ delta_single_pred_2 = single_df_2_pred.mean(0) - mean_control
880
+
881
+ return {'single_pred_1': delta_single_pred_1.values[high_umi_idx],
882
+ 'single_pred_2': delta_single_pred_2.values[high_umi_idx],
883
+ 'double_pred': delta_double_pred.values[high_umi_idx],
884
+ 'single_truth_1': delta_single_truth_1.values[high_umi_idx],
885
+ 'single_truth_2': delta_single_truth_2.values[high_umi_idx],
886
+ 'double_truth': delta_double_truth.values[high_umi_idx]}
887
+
888
+
889
+ def get_coeffs_synergy(singles_expr, double_expr):
890
+ results = {}
891
+ results['ts'] = TheilSenRegressor(fit_intercept=False,
892
+ max_subpopulation=1e5,
893
+ max_iter=1000,
894
+ random_state=1000)
895
+ X = singles_expr
896
+ y = double_expr
897
+ try:
898
+ results['ts'].fit(X, y.ravel())
899
+ except:
900
+ print(X)
901
+ print(y)
902
+ results['c1'] = results['ts'].coef_[0]
903
+ results['c2'] = results['ts'].coef_[1]
904
+ results['mag'] = np.sqrt((results['c1'] ** 2 + results['c2'] ** 2))
905
+ return results
906
+
907
+
908
+ def Fit(all_vectors, type_='pertnet'):
909
+ if type_ == 'pertnet':
910
+ singles_expr = np.array(
911
+ [all_vectors['single_pred_1'], all_vectors['single_pred_2']]).T
912
+ first_expr = np.array([all_vectors['single_pred_1']]).T
913
+ second_expr = np.array([all_vectors['single_pred_2']]).T
914
+ double_expr = np.array(all_vectors['double_pred']).T
915
+
916
+ elif type_ == 'truth':
917
+ singles_expr = np.array(
918
+ [all_vectors['single_truth_1'], all_vectors['single_truth_2']]).T
919
+ first_expr = np.array([all_vectors['single_truth_1']]).T
920
+ second_expr = np.array([all_vectors['single_truth_2']]).T
921
+ double_expr = np.array(all_vectors['double_truth']).T
922
+
923
+ return get_coeffs_synergy(singles_expr, double_expr)
924
+
925
+
926
+ def get_linear_params(pred_res, high_umi_idx, mean_control, all_perts):
927
+ results = {}
928
+ for c in set(pred_res['pert_cat']):
929
+ if 'ctrl' in c:
930
+ continue
931
+ double = c
932
+ single1 = get_single_name(double.split('+')[0], all_perts)
933
+ single2 = get_single_name(double.split('+')[1], all_perts)
934
+ all_vectors = get_all_vectors(pred_res, mean_control, double,
935
+ single1, single2, high_umi_idx)
936
+
937
+ pertnet_results = Fit(all_vectors, type_='pertnet')
938
+ truth_results = Fit(all_vectors, type_='truth')
939
+
940
+ results[c] = {
941
+ 'truth': truth_results,
942
+ 'pred': pertnet_results}
943
+
944
+ return results
945
+
946
+
947
+ # Read in model for each gene
948
+
949
+ GIs = {
950
+ 'NEOMORPHIC': ['CBL+TGFBR2',
951
+ 'KLF1+TGFBR2',
952
+ 'MAP2K6+SPI1',
953
+ 'SAMD1+TGFBR2',
954
+ 'TGFBR2+C19orf26',
955
+ 'TGFBR2+ETS2',
956
+ 'CBL+UBASH3A',
957
+ 'CEBPE+KLF1',
958
+ 'DUSP9+MAPK1',
959
+ 'FOSB+PTPN12',
960
+ 'PLK4+STIL',
961
+ 'PTPN12+OSR2',
962
+ 'ZC3HAV1+CEBPE'],
963
+ 'ADDITIVE': ['BPGM+SAMD1',
964
+ 'CEBPB+MAPK1',
965
+ 'CEBPB+OSR2',
966
+ 'DUSP9+PRTG',
967
+ 'FOSB+OSR2',
968
+ 'IRF1+SET',
969
+ 'MAP2K3+ELMSAN1',
970
+ 'MAP2K6+ELMSAN1',
971
+ 'POU3F2+FOXL2',
972
+ 'RHOXF2BB+SET',
973
+ 'SAMD1+PTPN12',
974
+ 'SAMD1+UBASH3B',
975
+ 'SAMD1+ZBTB1',
976
+ 'SGK1+TBX2',
977
+ 'TBX3+TBX2',
978
+ 'ZBTB10+SNAI1'],
979
+ 'EPISTASIS': ['AHR+KLF1',
980
+ 'MAPK1+TGFBR2',
981
+ 'TGFBR2+IGDCC3',
982
+ 'TGFBR2+PRTG',
983
+ 'UBASH3B+OSR2',
984
+ 'DUSP9+ETS2',
985
+ 'KLF1+CEBPA',
986
+ 'MAP2K6+IKZF3',
987
+ 'ZC3HAV1+CEBPA'],
988
+ 'REDUNDANT': ['CDKN1C+CDKN1A',
989
+ 'MAP2K3+MAP2K6',
990
+ 'CEBPB+CEBPA',
991
+ 'CEBPE+CEBPA',
992
+ 'CEBPE+SPI1',
993
+ 'ETS2+MAPK1',
994
+ 'FOSB+CEBPE',
995
+ 'FOXA3+FOXA1'],
996
+ 'POTENTIATION': ['CNN1+UBASH3A',
997
+ 'ETS2+MAP7D1',
998
+ 'FEV+CBFA2T3',
999
+ 'FEV+ISL2',
1000
+ 'FEV+MAP7D1',
1001
+ 'PTPN12+UBASH3A'],
1002
+ 'SYNERGY_SIMILAR_PHENO':['CBL+CNN1',
1003
+ 'CBL+PTPN12',
1004
+ 'CBL+PTPN9',
1005
+ 'CBL+UBASH3B',
1006
+ 'FOXA3+FOXL2',
1007
+ 'FOXA3+HOXB9',
1008
+ 'FOXL2+HOXB9',
1009
+ 'UBASH3B+CNN1',
1010
+ 'UBASH3B+PTPN12',
1011
+ 'UBASH3B+PTPN9',
1012
+ 'UBASH3B+ZBTB25'],
1013
+ 'SYNERGY_DISSIMILAR_PHENO': ['AHR+FEV',
1014
+ 'DUSP9+SNAI1',
1015
+ 'FOXA1+FOXF1',
1016
+ 'FOXA1+FOXL2',
1017
+ 'FOXA1+HOXB9',
1018
+ 'FOXF1+FOXL2',
1019
+ 'FOXF1+HOXB9',
1020
+ 'FOXL2+MEIS1',
1021
+ 'IGDCC3+ZBTB25',
1022
+ 'POU3F2+CBFA2T3',
1023
+ 'PTPN12+ZBTB25',
1024
+ 'SNAI1+DLX2',
1025
+ 'SNAI1+UBASH3B'],
1026
+ 'SUPPRESSOR': ['CEBPB+PTPN12',
1027
+ 'CEBPE+CNN1',
1028
+ 'CEBPE+PTPN12',
1029
+ 'CNN1+MAPK1',
1030
+ 'ETS2+CNN1',
1031
+ 'ETS2+IGDCC3',
1032
+ 'ETS2+PRTG',
1033
+ 'FOSB+UBASH3B',
1034
+ 'IGDCC3+MAPK1',
1035
+ 'LYL1+CEBPB',
1036
+ 'MAPK1+PRTG',
1037
+ 'PTPN12+SNAI1']
1038
+ }
BioMod/gene_perturbation/model_module.py CHANGED
@@ -1,760 +1,761 @@
1
- """
2
- ================================================================================
3
- Genetic perturbation task model definition (model_module.py)
4
- ================================================================================
5
-
6
- [Model Architecture Overview]
7
- This document implements a deep learning model for genetic perturbation prediction. The core components include:
8
-
9
- 1. ModuleEmbedding: Convert gene-level expression values into module-level tokens
10
- - Supports multiple integration methods (multiply/add/gated_residual)
11
- - Use attention pooling to aggregate genetic information within the module
12
-
13
- 2. PerformerEncoderOnly_Module: Encoder-only Transformer model
14
- - Use Performer (Linear Complexity Attention) to process large numbers of module tokens
15
- - Support Masked Autoencoder (MAE) pre-training paradigm
16
-
17
- 3. SCFM_Module: wrapper for pre-trained models
18
- - Process the mapping of input genes to pre-trained model genes
19
- - Recover gene-level embeddings from module encoding
20
-
21
- 4. GEARS_Model_Module: Genetic perturbation prediction model
22
- - Integrate pre-trained encoder + GNN + decoder
23
- - Predict expression changes after gene perturbation
24
-
25
- [Key technical points]
26
- - Module-based gene representation: Group thousands of genes into hundreds of modules to reduce computational complexity
27
- - Gene embedding recovery: Recover gene-level features from module encoding through average pooling
28
- - GO graph neural network: using gene ontology knowledge to enhance perturbation representation
29
-
30
- ================================================================================
31
- """
32
-
33
- import torch
34
- import torch.nn as nn
35
- import torch.nn.functional as F
36
- from torch.nn import Sequential, Linear, ReLU
37
- import numpy as np
38
- import pickle
39
- from torch_geometric.nn import SGConv
40
- from typing import List, Tuple, Optional
41
- import sys
42
- import os
43
- from copy import deepcopy
44
- import scipy
45
-
46
- # ============================================================
47
- # Import Performer related components (does not depend on gears_reduce)
48
- # Performer is a linear complexity Transformer variant suitable for processing long sequences
49
- # ============================================================
50
- try:
51
- from BioMod.models_performer_module import (
52
- ModuleEmbedding,
53
- PerformerEncoderOnly_Module,
54
- sanitize_performer_module_config,
55
- )
56
- except ModuleNotFoundError:
57
- project_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
58
- if project_root not in sys.path:
59
- sys.path.insert(0, project_root)
60
- from BioMod.models_performer_module import (
61
- ModuleEmbedding,
62
- PerformerEncoderOnly_Module,
63
- sanitize_performer_module_config,
64
- )
65
-
66
- # ============================================================
67
- # Original GEARS related classes
68
- # ============================================================
69
-
70
- class MLP(torch.nn.Module):
71
- """MLP network"""
72
- def __init__(self, sizes, batch_norm=True, last_layer_act="linear"):
73
- super(MLP, self).__init__()
74
- layers = []
75
- for s in range(len(sizes) - 1):
76
- layers = layers + [
77
- torch.nn.Linear(sizes[s], sizes[s + 1]),
78
- torch.nn.BatchNorm1d(sizes[s + 1])
79
- if batch_norm and s < len(sizes) - 1 else None,
80
- torch.nn.ReLU()
81
- ]
82
-
83
- layers = [l for l in layers if l is not None][:-1]
84
- self.activation = last_layer_act
85
- self.network = torch.nn.Sequential(*layers)
86
- self.relu = torch.nn.ReLU()
87
-
88
- def forward(self, x):
89
- return self.network(x)
90
-
91
-
92
- def reindex_tensor_universal(tensor, index_positions, dim, filler='0', device='cpu'):
93
- """
94
- Reindex tensor along specified dimensions
95
- """
96
- index_tensor = torch.tensor(index_positions, device=device)
97
- tensor_shape = list(tensor.shape)
98
- tensor_shape[dim] = len(index_positions)
99
- index_shape = [1] * len(tensor_shape)
100
- index_shape[dim] = len(index_positions)
101
- index_tensor = index_tensor.view(*index_shape).expand(*tensor_shape)
102
-
103
- if filler == '0':
104
- padder_shape = deepcopy(tensor_shape)
105
- padder_shape[dim] = 1
106
- padder = torch.zeros(padder_shape, dtype=tensor.dtype, device=device)
107
- elif filler == 'mean':
108
- padder = tensor.mean(dim=dim, keepdim=True)
109
- else:
110
- raise ValueError("filler should be 0 or mean!")
111
- expanded_tensor = torch.cat((tensor, padder), dim=dim)
112
- reindexed = torch.gather(expanded_tensor, dim, index_tensor)
113
- return reindexed
114
-
115
-
116
- class SCFM_Module(torch.nn.Module):
117
- """
118
- ================================================================================
119
- SCFM_Module: wrapper for pre-trained single cell base models
120
-
121
- [Core functions]
122
- Adapt the PerformerEncoderOnly_Module model to the genetic perturbation task to address the following key issues:
123
-
124
- 1. Gene mapping: The gene order of the input data may be different from that of the pre-trained model, and re-indexing is required.
125
- 2. Module-to-gene recovery: Recovering gene-level embeddings from module encodings
126
- 3. Dimension adjustment: adjust the encoder output to the dimensions required by downstream tasks
127
-
128
- [Key Design Decisions]
129
- - use_module_level=False (recommended): restore gene level embedding from module encoding
130
- Advantages: Compatible with existing GEARS architecture, no need to modify downstream tasks
131
- - use_module_level=True (experimental): use module level encoding directly
132
- Downstream architecture needs to be redesigned
133
-
134
- [Gene Embedding Recovery Strategy]
135
- For each gene:
136
- 1. Find all modules containing the gene
137
- 2. Perform average pooling on the encoding of these modules
138
- 3. If the gene is not in any module, use the global average as fallback
139
-
140
- [Performance Optimization]
141
- - Precompute scatter index to avoid Python loops
142
- - Vectorized aggregation using scatter_add
143
- ================================================================================
144
- """
145
- def __init__(self, input_genes, scfm_genes,
146
- scfm_hyper_params, scfm_ckpt,
147
- module_sets=None,
148
- requires_grad=False,
149
- device='cpu',
150
- output_dim=200,
151
- use_module_level=False):
152
- super(SCFM_Module, self).__init__()
153
- self.device = device
154
- self.use_module_level = use_module_level
155
-
156
- # ============================================================
157
- # [Gene Mapping] Establish index mapping between input genes and pre-trained model genes
158
- # This is a critical step in dealing with inconsistencies in gene order across different datasets
159
- # ============================================================
160
- input_mapping = {idx: i for i, idx in enumerate(input_genes)}
161
- self.input_gene_num = len(input_genes)
162
-
163
- # ============================================================
164
- # [Key] fusion_method processing
165
- # Read from pkl to ensure consistency with pre-training
166
- # ============================================================
167
- fusion_method = scfm_hyper_params.get('fusion_method', 'multiply')
168
- print(f"[SCFM_Module] Using fusion_method from hyper_params: {fusion_method}")
169
-
170
- # Convert new indices to a list of index positions from the original tensor
171
- scfm_genes_pad = scfm_genes + ['PAD']
172
- self.scfm_seq_len = len(scfm_genes_pad)
173
- scfm_mapping = {idx: i for i, idx in enumerate(scfm_genes_pad)}
174
-
175
- # Create bidirectional index mapping
176
- self.scfm_index_positions = [input_mapping.get(idx, self.input_gene_num) for idx in scfm_genes_pad]
177
- self.input_index_positions = [scfm_mapping.get(idx, self.scfm_seq_len) for idx in input_genes]
178
-
179
- # Load module data (if not provided in hyper_params)
180
- if module_sets is None:
181
- if 'module_sets' in scfm_hyper_params and scfm_hyper_params['module_sets'] is not None:
182
- module_sets = scfm_hyper_params['module_sets']
183
- else:
184
- raise ValueError("module_sets must be provided (via parameters or hyper_params)")
185
-
186
- self.module_sets = module_sets
187
- self.num_modules = len(module_sets)
188
- self.scfm_seq_len = len(scfm_genes)
189
-
190
- # ============================================================
191
- # [Key] Create a gene-to-module mapping
192
- # Each gene may appear in multiple modules for subsequent average pooling.
193
- # This is the core data structure of module representation.
194
- # ============================================================
195
- self.gene_to_modules = {} # {gene_idx: [module_idx1, module_idx2, ...]}
196
- for module_idx, module_genes in enumerate(module_sets):
197
- for gene_idx in module_genes:
198
- if gene_idx < len(scfm_genes): # Only consider genes in scfm_genes
199
- if gene_idx not in self.gene_to_modules:
200
- self.gene_to_modules[gene_idx] = []
201
- self.gene_to_modules[gene_idx].append(module_idx)
202
-
203
- # ============================================================
204
- # [Performance Optimization] Pre-built sparse index for efficient module->gene aggregation
205
- # Use scatter_add to replace Python loop, greatly improving forward efficiency
206
- # This is the standard optimization method in PyTorch for handling irregular aggregations
207
- # ============================================================
208
- import torch.sparse
209
- max_modules_per_gene = max(len(v) for v in self.gene_to_modules.values()) if self.gene_to_modules else 1
210
-
211
- # Construct a sparse index matrix: shape (num_genes, max_modules_per_gene)
212
- gene_to_module_matrix = []
213
- for gene_idx in range(self.scfm_seq_len):
214
- if gene_idx in self.gene_to_modules:
215
- modules = self.gene_to_modules[gene_idx]
216
- padded = modules + [-1] * (max_modules_per_gene - len(modules))
217
- gene_to_module_matrix.append(padded)
218
- else:
219
- gene_to_module_matrix.append([-1] * max_modules_per_gene)
220
-
221
- # Registered as a buffer (will not be updated by the optimizer, but will be moved to the correct device with the model)
222
- self.register_buffer('gene_to_module_matrix',
223
- torch.tensor(gene_to_module_matrix, dtype=torch.long))
224
- gene_module_counts = torch.tensor([
225
- len(self.gene_to_modules.get(i, [])) for i in range(self.scfm_seq_len)
226
- ], dtype=torch.long)
227
- self.register_buffer('gene_module_counts', gene_module_counts)
228
-
229
- # ============================================================
230
- # [Performance Optimization] Precompute scatter_add index
231
- # Module->gene aggregation for fully vectorized, avoiding Python loops
232
- # ============================================================
233
- module_indices = []
234
- gene_indices = []
235
- for module_idx, module_genes in enumerate(module_sets):
236
- for gene_idx in module_genes:
237
- if gene_idx < self.scfm_seq_len:
238
- module_indices.append(module_idx)
239
- gene_indices.append(gene_idx)
240
-
241
- self.register_buffer('scatter_module_idx', torch.tensor(module_indices, dtype=torch.long))
242
- self.register_buffer('scatter_gene_idx', torch.tensor(gene_indices, dtype=torch.long))
243
-
244
- # Create PerformerEncoderOnly_Module model
245
- scfm_hyper_params['module_sets'] = module_sets
246
- scfm_hyper_params['num_genes'] = len(scfm_genes)
247
- scfm_hyper_params['max_seq_len'] = len(module_sets)
248
-
249
- self.scfm = PerformerEncoderOnly_Module(**scfm_hyper_params)
250
-
251
- # ============================================================
252
- # [Key] Load pre-trained weights
253
- # Filter out incompatible parameters (GO related, module_heads, etc.)
254
- # strict=False allows partial parameter mismatch (used for fine-tuning mode)
255
- # ============================================================
256
- if scfm_ckpt != 'None':
257
- ckpt = torch.load(scfm_ckpt, map_location=device)
258
- model_state_dict = ckpt.get('model_state_dict', ckpt)
259
- current_state_dict = self.scfm.state_dict()
260
- filtered_state_dict = {}
261
- for k, v in model_state_dict.items():
262
- # Filter out GO related parameters (may not be available in different data sets)
263
- if 'go_conv' in k or 'G_go' in k.lower():
264
- continue
265
- # Filter out module_heads (not needed for fine-tuning)
266
- if 'module_heads' in k:
267
- continue
268
- # Filter out parameters that do not exist or have mismatched shapes in the current model
269
- if k not in current_state_dict:
270
- continue
271
- if current_state_dict[k].shape != v.shape:
272
- print(
273
- f"[SCFM_Module] skip shape-mismatched key: {k}, "
274
- f"ckpt={tuple(v.shape)} current={tuple(current_state_dict[k].shape)}"
275
- )
276
- continue
277
- filtered_state_dict[k] = v
278
- # Use strict=False because module_heads is intentionally removed
279
- missing, unexpected = self.scfm.load_state_dict(filtered_state_dict, strict=False)
280
- for param in self.scfm.parameters():
281
- param.requires_grad = requires_grad
282
- else:
283
- for param in self.scfm.parameters():
284
- param.requires_grad = True
285
-
286
- # Output dimension adjustment (if needed)
287
- self.output_dim = output_dim
288
- if self.output_dim != 200:
289
- try:
290
- self.feedforward = nn.Linear(self.scfm.base_dim, self.output_dim)
291
- except AttributeError:
292
- self.feedforward = nn.Linear(scfm_hyper_params.get('base_dim', 200), self.output_dim)
293
-
294
- def forward(self, x):
295
- """
296
- Forward propagation: from gene expression values to gene-level embeddings
297
-
298
- [Processing Process]
299
- 1. Re-index the input gene expression into the gene order of the pre-trained model
300
- 2. Get the module encoding through PerformerEncoderOnly_Module
301
- 3. Restore gene-level embedding from module encoding (or return module encoding directly)
302
- 4. Reindex gene embeddings back to input gene order
303
-
304
- [Input and output]
305
- Input: x (B, input_gene_num) - gene expression value
306
- Output:
307
- - use_module_level=False: output_reindex (B, input_gene_num, D) - gene level embedding
308
- - use_module_level=True: module_encodings (B, num_modules, D) - module level encoding
309
- """
310
- x = x.view((-1, self.input_gene_num)) # B, M
311
-
312
- # ============================================================
313
- # [Step 1] Re-index: Map the input gene expression to the gene order of the pre-trained model
314
- # Genes that are not present are filled with 0 (PAD position)
315
- # ============================================================
316
- x_scfm = reindex_tensor_universal(x, self.scfm_index_positions, dim=1, filler='0', device=self.device) # B, N
317
-
318
- # ============================================================
319
- # [Step 2] Obtain the module code through the pre-trained model
320
- # mask_ratio=0 means not to mask any modules (fine-tuning mode)
321
- # Use mask_ratio>0 for MAE pre-training during pre-training.
322
- # ============================================================
323
- out = self.scfm(x_scfm, mask_ratio=0.0, return_encodings=True)
324
-
325
- if 'encodings' in out:
326
- module_encodings = out['encodings'] # Local legacy interface
327
- elif 'module_encodings' in out:
328
- module_encodings = out['module_encodings'] # Reuse pre-trained shared implementation interfaces
329
- else:
330
- raise ValueError(f"Expected one of ['encodings', 'module_encodings'] in output, got: {list(out.keys())}")
331
-
332
- # ============================================================
333
- # [Option] Directly return to module level encoding
334
- # For experimental module-level downstream tasks
335
- # ============================================================
336
- if self.use_module_level:
337
- if self.output_dim != 200:
338
- B, num_modules, D = module_encodings.shape
339
- module_encodings_flat = module_encodings.reshape(B * num_modules, D)
340
- module_encodings_flat = self.feedforward(module_encodings_flat)
341
- module_encodings = module_encodings_flat.reshape(B, num_modules, self.output_dim)
342
- return module_encodings # (B, num_modules, D)
343
-
344
- # ============================================================
345
- # [Step 3] Restore gene-level embedding from module encoding
346
- # Core idea: The embedding of each gene = the average encoding of all modules containing the gene
347
- # Vectorized aggregation using scatter_add to avoid Python loops
348
- # ============================================================
349
- B, num_modules, D = module_encodings.shape
350
- G = self.scfm_seq_len
351
-
352
- # Calculate global average (for genes without any modules)
353
- global_mean = module_encodings.mean(dim=1) # (B, D)
354
-
355
- # Extract the corresponding module embeddings: (B, num_pairs, D)
356
- module_embs = module_encodings[:, self.scatter_module_idx, :] # (B, num_pairs, D)
357
-
358
- # Aggregate to gene dimension using scatter_add
359
- scfm_gene_embeddings = torch.zeros(B, G, D, device=module_encodings.device)
360
- gene_idx_expanded = self.scatter_gene_idx.unsqueeze(0).unsqueeze(-1).expand(B, -1, D)
361
- scfm_gene_embeddings.scatter_add_(1, gene_idx_expanded, module_embs)
362
-
363
- # ============================================================
364
- # [Key] Divide by the number of modules for each gene to get the average
365
- # This is the core operation from module to gene aggregation
366
- # clamp(min=1) avoids division by zero errors
367
- # ============================================================
368
- gene_counts = self.gene_module_counts.float().view(1, G, 1).clamp(min=1)
369
- scfm_gene_embeddings = scfm_gene_embeddings / gene_counts
370
-
371
- # For genes without any modules, use global average as fallback
372
- no_module_mask = (self.gene_module_counts == 0).view(1, G, 1).float()
373
- scfm_gene_embeddings = scfm_gene_embeddings * (1 - no_module_mask) + global_mean.unsqueeze(1) * no_module_mask
374
-
375
- # Add embedding of PAD positions (for handling non-existent genes)
376
- pad_emb = torch.zeros(B, 1, D, device=module_encodings.device)
377
- output_encodings = torch.cat([scfm_gene_embeddings, pad_emb], dim=1) # (B, G+1, D)
378
-
379
- # ============================================================
380
- # [Step 4] Reindex back to the input gene sequence
381
- # Use mean as fallback to fill in non-existent genes
382
- # ============================================================
383
- output_reindex = reindex_tensor_universal(
384
- output_encodings,
385
- self.input_index_positions,
386
- dim=1,
387
- filler='mean',
388
- device=self.device
389
- ) # B, M, D
390
-
391
- if self.output_dim != 200:
392
- output_reindex = self.feedforward(output_reindex)
393
-
394
- return output_reindex
395
-
396
-
397
- class GEARS_Model_Module(torch.nn.Module):
398
- """
399
- ================================================================================
400
- GEARS_Model_Module: Genetic perturbation prediction model
401
-
402
- [Core functions]
403
- Predicting transcriptomic changes in cells following genetic perturbation. Enter a perturbation combination (such as knockout gene A),
404
- Output the expression value prediction of each gene after perturbation.
405
-
406
- [Model Architecture]
407
- 1. Pretrained encoder (SCFM_Module): Encode gene expression into embedding vectors
408
- 2. Perturbation embedding layer: generate perturbation features for perturbed genes
409
- 3. GO graph neural network (GNN): using gene ontology knowledge to enhance perturbation representation
410
- 4. Decoder: predict gene expression values after perturbation
411
-
412
- [Key Design Decisions]
413
- - Use GO graph neural network: gene ontology contains gene functional relationships, which can guide model understanding
414
- How perturbation effects spread between genes
415
- - Gene specific decoder: Each gene has independent decoding parameters to capture gene specificity
416
- - Cross-gene MLP: Modeling interdependencies between genes
417
-
418
- [Input and output]
419
- Input: data (PyG Data object)
420
- - x: (B*N, 2) feature matrix, [:, 0] is gene expression, [:, 1] is perturbation mark
421
- - y: true expression value (used for training)
422
- Output: out (B, N) - predicted gene expression value after perturbation
423
- ================================================================================
424
- """
425
- def __init__(self, args):
426
- super(GEARS_Model_Module, self).__init__()
427
- self.args = args
428
-
429
- # Forecast record related settings
430
- if args.get('record_pred', False):
431
- self.pred_dir = f"./result_process/preds/{self.args['exp_name']}"
432
- if not os.path.exists(self.pred_dir):
433
- os.makedirs(self.pred_dir)
434
- self.pred_batch_idx = 0
435
-
436
- # ============================================================
437
- # [Basic parameters]
438
- # hidden_size: embedding dimension, affecting model capacity
439
- # num_genes: number of genes, determines the output dimension
440
- # ============================================================
441
- self.num_genes = args['num_genes']
442
- hidden_size = args['hidden_size']
443
- self.uncertainty = args['uncertainty']
444
- self.num_layers = args['num_go_gnn_layers'] # GO GNN layers
445
- self.indv_out_hidden_size = args['decoder_hidden_size']
446
- self.num_layers_gene_pos = args['num_gene_gnn_layers']
447
- self.pert_emb_lambda = 0.2
448
-
449
- self.input_genes_list = args['input_genes_ens_ids']
450
- self.scfm_genes_list = args['scfm_genes_ens_ids']
451
-
452
- # ============================================================
453
- # [Load pre-trained model configuration]
454
- # hyper_params contains model architecture parameters, which must be consistent with pre-training
455
- # ============================================================
456
- try:
457
- with open(args["scfm_hyper_params_path"], 'rb') as handle:
458
- self.scfm_hyper_params = pickle.load(handle)
459
- except ValueError:
460
- import pickle5
461
- with open(args["scfm_hyper_params_path"], 'rb') as handle:
462
- self.scfm_hyper_params = pickle5.load(handle)
463
- self.scfm_hyper_params.pop('module_embed_chunk_padded_gene_budget', None)
464
- self.scfm_hyper_params = sanitize_performer_module_config(
465
- self.scfm_hyper_params,
466
- source=args["scfm_hyper_params_path"],
467
- log_fn=print,
468
- )
469
-
470
- self.scfm_hyper_params['gene2vec_file'] = args['scfm_gene2vec_file']
471
- self.scfm_ckpt = args["scfm_ckpt"]
472
-
473
- # ============================================================
474
- # [Module data loading]
475
- # module_sets defines how genes are grouped into modules
476
- # Can be loaded from hyper_params or external files
477
- # ============================================================
478
- module_sets = None
479
- if 'module_sets' in self.scfm_hyper_params and self.scfm_hyper_params['module_sets'] is not None:
480
- module_sets = self.scfm_hyper_params['module_sets']
481
- elif args.get('module_data_path') or args.get('module_data_dir'):
482
- from BioMod.module_dataset import load_module_data
483
- module_sets = load_module_data(
484
- module_data_path=args.get('module_data_path'),
485
- module_data_dir=args.get('module_data_dir'),
486
- gene_list_path=args.get('scfm_genes_list_path'),
487
- gene_mapping_path=args.get('gene_mapping_path', '../gene2ens.csv'),
488
- verbose=True
489
- )
490
- else:
491
- raise ValueError("module_sets must be provided (via hyper_params or module_data_path/module_data_dir)")
492
-
493
- # ============================================================
494
- # [Ablation Experiment]shuffle_module_genes
495
- # Disrupt the gene assignment within the module and verify the importance of the module structure (gene grouping)
496
- # If performance decreases after scrambling, it means that the module structure contains useful biological information.
497
- # ============================================================
498
- if args.get('shuffle_module_genes', False):
499
- shuffle_seed = args.get('shuffle_seed', 2021)
500
- print(
501
- f"[GEARS_Model_Module] Applying downstream-only extra module reshuffle "
502
- f"(seed={shuffle_seed}). Pretraining-time module_sets from PKL will be reshuffled again."
503
- )
504
- all_gene_indices = []
505
- module_sizes = []
506
- for module in module_sets:
507
- all_gene_indices.extend(module)
508
- module_sizes.append(len(module))
509
-
510
- import random
511
- rng = random.Random(shuffle_seed)
512
- rng.shuffle(all_gene_indices)
513
-
514
- shuffled_module_sets = []
515
- start_idx = 0
516
- for size in module_sizes:
517
- shuffled_module_sets.append(all_gene_indices[start_idx:start_idx + size])
518
- start_idx += size
519
- module_sets = shuffled_module_sets
520
- print(f"[GEARS_Model_Module] Shuffled {len(all_gene_indices)} genes across {len(module_sets)} modules")
521
-
522
- # ============================================================
523
- # [Perturbation Embedding Layer]
524
- # pert_w: Convert perturbation markers (0/1) to embedding vectors
525
- # gene_emb: Basic embedding of genes (used when pre-trained models are not available)
526
- # pert_emb: dedicated embedding of perturbed genes
527
- # ============================================================
528
- self.pert_w = nn.Linear(1, hidden_size)
529
-
530
- # gene/global perturbation embedding dictionary lookup
531
- self.gene_emb = nn.Embedding(self.num_genes, hidden_size, max_norm=True)
532
- self.pert_emb = nn.Embedding(self.num_genes, hidden_size, max_norm=True)
533
-
534
- # transformation layer
535
- self.emb_trans = nn.ReLU()
536
- self.pert_base_trans = nn.ReLU()
537
- self.transform = nn.ReLU()
538
- self.pert_fuse = MLP([hidden_size, hidden_size, hidden_size], last_layer_act='ReLU')
539
-
540
- # ============================================================
541
- # [GO graph neural network]
542
- # G_sim: edge index of GO similarity graph
543
- # G_sim_weight: edge weight
544
- # Using SGConv (Simplified Graph Convolution) for message passing
545
- # GO diagram helps the model understand gene functional relationships, and perturbation effects propagate along the diagram.
546
- # ============================================================
547
- self.G_sim = args['G_go'].to(args['device'])
548
- self.G_sim_weight = args['G_go_weight'].to(args['device'])
549
-
550
- self.sim_layers = torch.nn.ModuleList()
551
- for i in range(1, self.num_layers + 1):
552
- self.sim_layers.append(SGConv(hidden_size, hidden_size, 1))
553
-
554
- # ============================================================
555
- # [Decoder]
556
- # recovery_w: shared MLP, projects embeddings into decoding space
557
- # indv_w1/b1: Gene specific parameters, each gene has independent decoding weight
558
- # cross_gene_state: cross-gene MLP, modeling inter-gene dependencies
559
- # indv_w2/b2: final decoding layer
560
- # ============================================================
561
- self.recovery_w = MLP([hidden_size, hidden_size*2, hidden_size], last_layer_act='linear')
562
-
563
- # gene specific decoder
564
- self.indv_w1 = nn.Parameter(torch.rand(self.num_genes, hidden_size, 1))
565
- self.indv_b1 = nn.Parameter(torch.rand(self.num_genes, 1))
566
- self.act = nn.ReLU()
567
- nn.init.xavier_normal_(self.indv_w1)
568
- nn.init.xavier_normal_(self.indv_b1)
569
-
570
- # Cross gene MLP
571
- self.cross_gene_state = MLP([self.num_genes, hidden_size, hidden_size])
572
- self.indv_w2 = nn.Parameter(torch.rand(1, self.num_genes, hidden_size+1))
573
- self.indv_b2 = nn.Parameter(torch.rand(1, self.num_genes))
574
- nn.init.xavier_normal_(self.indv_w2)
575
- nn.init.xavier_normal_(self.indv_b2)
576
-
577
- # BatchNorm layers
578
- self.bn_emb = nn.BatchNorm1d(hidden_size)
579
- self.bn_pert_base = nn.BatchNorm1d(hidden_size)
580
- self.bn_pert_base_trans = nn.BatchNorm1d(hidden_size)
581
-
582
- # uncertainty mode (optional)
583
- if self.uncertainty:
584
- self.uncertainty_w = MLP([hidden_size, hidden_size*2, hidden_size, 1], last_layer_act='linear')
585
-
586
- # ============================================================
587
- # [Core] Initialize pre-trained encoder (SCFM_Module)
588
- # requires_grad: whether to fine-tune pre-training parameters
589
- # - finetune_method='frozen': freeze pre-training parameters
590
- # - Others: Full parameter fine-tuning
591
- # ============================================================
592
- self.use_module_level = args.get('use_module_level', False)
593
- self.singlecell_model = SCFM_Module(
594
- self.input_genes_list,
595
- self.scfm_genes_list,
596
- self.scfm_hyper_params,
597
- self.scfm_ckpt,
598
- module_sets=module_sets,
599
- requires_grad=args.get('finetune_method') != 'frozen',
600
- device=args['device'],
601
- output_dim=hidden_size,
602
- use_module_level=self.use_module_level
603
- )
604
- self.pretrained = True
605
-
606
- # ============================================================
607
- # [Experimental feature] module level features
608
- # In the current implementation, module-level downstream tasks are not fully implemented.
609
- # GNN and decoder need to be redesigned to handle module dimensions
610
- # ============================================================
611
- if self.use_module_level:
612
- self.num_modules = len(module_sets)
613
- import warnings
614
- warnings.warn(
615
- f"Using module-level features. This requires compatible downstream architecture. "
616
- f"Number of modules: {self.num_modules}, Number of genes: {self.num_genes}. "
617
- f"Currently module-level downstream task is not fully implemented.",
618
- UserWarning
619
- )
620
-
621
- def forward(self, data):
622
- """
623
- Forward propagation: predicting expression values after gene perturbation
624
-
625
- [enter]
626
- data: PyG Data object
627
- - x: (B*N, 2) characteristic matrix
628
- [:, 0]: gene expression value (control group)
629
- [:, 1]: perturbation flag (1 indicates that the gene is perturbed)
630
- - y: real expression value (used during training)
631
- - pert: disturbance type identifier
632
-
633
- [Output]
634
- out: (B, N) predicted gene expression value after perturbation
635
-
636
- [Processing Process]
637
- 1. Obtain base gene embeddings (from pretrained encoder or random embeddings)
638
- 2. Generate perturbation embedding (enhanced by GO-GNN)
639
- 3. Merge base embedding and perturbation embedding
640
- 4. Predict the expression value through the decoder
641
- """
642
- if str(type(data)) == "<class 'torch.Tensor'>":
643
- x = data.reshape((-1, 2))
644
- else:
645
- x, batch = data.x, data.batch
646
- x_pert = x[:, 1].reshape((-1, self.num_genes)) # Disturbance mark
647
- num_graphs = len(data.batch.unique())
648
-
649
- # ============================================================
650
- # [Step 1] Obtain basic gene embedding
651
- # If using a pretrained model, get embeddings from gene expression encoding
652
- # Otherwise use randomly initialized gene embeddings
653
- # ============================================================
654
- if self.pretrained:
655
- exp = x[:, 0].reshape((-1, self.num_genes)) # B, N - gene expression value
656
- emb = self.singlecell_model(exp) # B, N, D - by pre-trained encoder
657
- emb = emb.reshape((num_graphs * self.num_genes, -1))
658
- else:
659
- emb = self.gene_emb(torch.LongTensor(list(range(self.num_genes))).repeat(num_graphs, ).to(self.args['device']))
660
- emb = self.bn_emb(emb)
661
- base_emb = self.emb_trans(emb)
662
-
663
- base_emb = base_emb.reshape(num_graphs, self.num_genes, -1)
664
-
665
- pert = x_pert.reshape(-1, 1) # B * N, 1
666
-
667
- # ============================================================
668
- # [Step 2] Generate perturbation embedding (GO-GNN enhancement)
669
- # Perturbation embeddings are propagated through GO similarity graphs to capture gene functional relationships
670
- # This enables the model to exploit a priori knowledge of gene function
671
- # ============================================================
672
- pert_embeddings = self.pert_emb(torch.LongTensor(list(range(self.num_genes))).to(self.args['device'])) # N, D
673
-
674
- # Propagating perturbation embeddings via GO-GNN
675
- for idx, layer in enumerate(self.sim_layers):
676
- pert_embeddings = layer(pert_embeddings, self.G_sim, self.G_sim_weight)
677
- if idx < self.num_layers - 1:
678
- pert_embeddings = pert_embeddings.relu()
679
-
680
- # After GNN: N, D
681
- pert_embeddings = torch.stack(list((pert_embeddings,)) * num_graphs) # B, N, D
682
- pert_embeddings = pert_embeddings * x_pert.reshape((num_graphs, self.num_genes, 1)) # Only perturbed genes are retained
683
- pert_embeddings = pert_embeddings.sum(1) # B, D - aggregate all perturbed genes
684
- pert_mask = (x_pert.sum(1) != 0).reshape((num_graphs, 1)) # B, 1
685
- pert_embeddings = self.pert_fuse(pert_embeddings) * pert_mask # B, D
686
- pert_embeddings = pert_embeddings.reshape((num_graphs, 1, pert_embeddings.shape[1])) # B, 1, D
687
-
688
- # ============================================================
689
- # [Step 3] Merge base embedding and perturbation embedding
690
- # Perturbation information is added to the embedding of each gene
691
- # ============================================================
692
- base_emb = base_emb + pert_embeddings
693
-
694
- base_emb = base_emb.reshape(num_graphs * self.num_genes, -1)
695
-
696
- # Add perturbed position embedding
697
- pert_emb = self.pert_w(pert)
698
- combined = pert_emb + base_emb
699
- combined = self.bn_pert_base_trans(combined)
700
- base_emb = self.pert_base_trans(combined)
701
- base_emb = self.bn_pert_base(base_emb)
702
-
703
- # ============================================================
704
- # [Step 4] Decoder: predict the expression value after perturbation
705
- # Using gene-specific decoders + cross-gene MLP
706
- # ============================================================
707
- base_emb = self.transform(base_emb)
708
- out = self.recovery_w(base_emb) # B * N, D
709
- out = out.reshape(num_graphs, self.num_genes, -1) # B, N, D
710
- out = out.unsqueeze(-1) * self.indv_w1 # B, N, D, 1 - Gene specific transformation
711
- w = torch.sum(out, axis=2) # B, N, 1
712
- out = w + self.indv_b1 # B, N, 1
713
-
714
- # Cross gene MLP: Modeling inter-gene dependencies
715
- cross_gene_embed = self.cross_gene_state(out.reshape(num_graphs, self.num_genes, -1).squeeze(2)) # B, D
716
- cross_gene_embed = cross_gene_embed.repeat(1, self.num_genes) # B, N*D
717
-
718
- cross_gene_embed = cross_gene_embed.reshape([num_graphs, self.num_genes, -1]) # B, N, D
719
- cross_gene_out = torch.cat([out, cross_gene_embed], 2) # B, N, D+1
720
-
721
- cross_gene_out = cross_gene_out * self.indv_w2 # B, N, D+1
722
- cross_gene_out = torch.sum(cross_gene_out, axis=2) # B, N
723
- out = cross_gene_out + self.indv_b2 # B, N
724
-
725
- # ============================================================
726
- # [Key] Residual connection: predicted value = change + original value
727
- # This is the core design of GEARS. The model predicts expression changes rather than absolute values.
728
- # ============================================================
729
- out = out.reshape(num_graphs * self.num_genes, -1) + x[:, 0].reshape(-1, 1) # B * N, 1
730
-
731
- # ============================================================
732
- # [Prediction record] for subsequent analysis
733
- # ============================================================
734
- if self.args.get('start_record', False):
735
- y_flat = data.y.reshape(-1, 1) # B * N, 1
736
- pred = torch.cat((pert, out, y_flat), dim=1) # B * N, 3
737
- pred = pred.cpu().to_sparse()
738
-
739
- values = pred.values().numpy()
740
- indices = pred.indices().numpy()
741
- sparse_matrix = scipy.sparse.coo_matrix((values, indices), shape=pred.shape)
742
-
743
- scipy.sparse.save_npz(f'{self.pred_dir}/b{self.pred_batch_idx}_{num_graphs}.npz', sparse_matrix)
744
- self.pred_batch_idx += 1
745
-
746
- # ============================================================
747
- # [Key] reshape output is (B, N) format
748
- # In this way, the loss function can correctly index the predicted value and the true value.
749
- # ============================================================
750
- out = torch.split(torch.flatten(out), self.num_genes)
751
- out = torch.stack(out) # (B, N)
752
-
753
- # uncertainty mode (optional)
754
- if self.uncertainty:
755
- logvar = self.uncertainty_w(base_emb)
756
- logvar = torch.split(torch.flatten(logvar), self.num_genes)
757
- return out, torch.stack(logvar)
758
-
759
- return out # (B, N)
760
-
 
 
1
+ """
2
+ ================================================================================
3
+ Genetic perturbation task model definition (model_module.py)
4
+ ================================================================================
5
+
6
+ [Model Architecture Overview]
7
+ This document implements a deep learning model for genetic perturbation prediction. The core components include:
8
+
9
+ 1. ModuleEmbedding: Convert gene-level expression values into module-level tokens
10
+ - Supports multiple integration methods (multiply/add/gated_residual)
11
+ - Use attention pooling to aggregate genetic information within the module
12
+
13
+ 2. PerformerEncoderOnly_Module: Encoder-only Transformer model
14
+ - Use Performer (Linear Complexity Attention) to process large numbers of module tokens
15
+ - Support Masked Autoencoder (MAE) pre-training paradigm
16
+
17
+ 3. SCFM_Module: wrapper for pre-trained models
18
+ - Process the mapping of input genes to pre-trained model genes
19
+ - Recover gene-level embeddings from module encoding
20
+
21
+ 4. GEARS_Model_Module: Genetic perturbation prediction model
22
+ - Integrate pre-trained encoder + GNN + decoder
23
+ - Predict expression changes after gene perturbation
24
+
25
+ [Key technical points]
26
+ - Module-based gene representation: Group thousands of genes into hundreds of modules to reduce computational complexity
27
+ - Gene embedding recovery: Recover gene-level features from module encoding through average pooling
28
+ - GO graph neural network: using gene ontology knowledge to enhance perturbation representation
29
+
30
+ ================================================================================
31
+ """
32
+
33
+ import torch
34
+ import torch.nn as nn
35
+ import torch.nn.functional as F
36
+ from torch.nn import Sequential, Linear, ReLU
37
+ import numpy as np
38
+ import pickle
39
+ from torch_geometric.nn import SGConv
40
+ from typing import List, Tuple, Optional
41
+ import sys
42
+ from pathlib import Path
43
+ import os
44
+ from copy import deepcopy
45
+ import scipy
46
+
47
+ # ============================================================
48
+ # Import Performer related components (does not depend on gears_reduce)
49
+ # Performer is a linear complexity Transformer variant suitable for processing long sequences
50
+ # ============================================================
51
+ try:
52
+ from BioMod.models_performer_module import (
53
+ ModuleEmbedding,
54
+ PerformerEncoderOnly_Module,
55
+ sanitize_performer_module_config,
56
+ )
57
+ except ModuleNotFoundError:
58
+ project_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
59
+ if project_root not in sys.path:
60
+ sys.path.insert(0, project_root)
61
+ from BioMod.models_performer_module import (
62
+ ModuleEmbedding,
63
+ PerformerEncoderOnly_Module,
64
+ sanitize_performer_module_config,
65
+ )
66
+
67
+ # ============================================================
68
+ # Original GEARS related classes
69
+ # ============================================================
70
+
71
+ class MLP(torch.nn.Module):
72
+ """MLP network"""
73
+ def __init__(self, sizes, batch_norm=True, last_layer_act="linear"):
74
+ super(MLP, self).__init__()
75
+ layers = []
76
+ for s in range(len(sizes) - 1):
77
+ layers = layers + [
78
+ torch.nn.Linear(sizes[s], sizes[s + 1]),
79
+ torch.nn.BatchNorm1d(sizes[s + 1])
80
+ if batch_norm and s < len(sizes) - 1 else None,
81
+ torch.nn.ReLU()
82
+ ]
83
+
84
+ layers = [l for l in layers if l is not None][:-1]
85
+ self.activation = last_layer_act
86
+ self.network = torch.nn.Sequential(*layers)
87
+ self.relu = torch.nn.ReLU()
88
+
89
+ def forward(self, x):
90
+ return self.network(x)
91
+
92
+
93
+ def reindex_tensor_universal(tensor, index_positions, dim, filler='0', device='cpu'):
94
+ """
95
+ Reindex tensor along specified dimensions
96
+ """
97
+ index_tensor = torch.tensor(index_positions, device=device)
98
+ tensor_shape = list(tensor.shape)
99
+ tensor_shape[dim] = len(index_positions)
100
+ index_shape = [1] * len(tensor_shape)
101
+ index_shape[dim] = len(index_positions)
102
+ index_tensor = index_tensor.view(*index_shape).expand(*tensor_shape)
103
+
104
+ if filler == '0':
105
+ padder_shape = deepcopy(tensor_shape)
106
+ padder_shape[dim] = 1
107
+ padder = torch.zeros(padder_shape, dtype=tensor.dtype, device=device)
108
+ elif filler == 'mean':
109
+ padder = tensor.mean(dim=dim, keepdim=True)
110
+ else:
111
+ raise ValueError("filler should be 0 or mean!")
112
+ expanded_tensor = torch.cat((tensor, padder), dim=dim)
113
+ reindexed = torch.gather(expanded_tensor, dim, index_tensor)
114
+ return reindexed
115
+
116
+
117
+ class SCFM_Module(torch.nn.Module):
118
+ """
119
+ ================================================================================
120
+ SCFM_Module: wrapper for pre-trained single cell base models
121
+
122
+ [Core functions]
123
+ Adapt the PerformerEncoderOnly_Module model to the genetic perturbation task to address the following key issues:
124
+
125
+ 1. Gene mapping: The gene order of the input data may be different from that of the pre-trained model, and re-indexing is required.
126
+ 2. Module-to-gene recovery: Recovering gene-level embeddings from module encodings
127
+ 3. Dimension adjustment: adjust the encoder output to the dimensions required by downstream tasks
128
+
129
+ [Key Design Decisions]
130
+ - use_module_level=False (recommended): restore gene level embedding from module encoding
131
+ Advantages: Compatible with existing GEARS architecture, no need to modify downstream tasks
132
+ - use_module_level=True (experimental): use module level encoding directly
133
+ Downstream architecture needs to be redesigned
134
+
135
+ [Gene Embedding Recovery Strategy]
136
+ For each gene:
137
+ 1. Find all modules containing the gene
138
+ 2. Perform average pooling on the encoding of these modules
139
+ 3. If the gene is not in any module, use the global average as fallback
140
+
141
+ [Performance Optimization]
142
+ - Precompute scatter index to avoid Python loops
143
+ - Vectorized aggregation using scatter_add
144
+ ================================================================================
145
+ """
146
+ def __init__(self, input_genes, scfm_genes,
147
+ scfm_hyper_params, scfm_ckpt,
148
+ module_sets=None,
149
+ requires_grad=False,
150
+ device='cpu',
151
+ output_dim=200,
152
+ use_module_level=False):
153
+ super(SCFM_Module, self).__init__()
154
+ self.device = device
155
+ self.use_module_level = use_module_level
156
+
157
+ # ============================================================
158
+ # [Gene Mapping] Establish index mapping between input genes and pre-trained model genes
159
+ # This is a critical step in dealing with inconsistencies in gene order across different datasets
160
+ # ============================================================
161
+ input_mapping = {idx: i for i, idx in enumerate(input_genes)}
162
+ self.input_gene_num = len(input_genes)
163
+
164
+ # ============================================================
165
+ # [Key] fusion_method processing
166
+ # Read from pkl to ensure consistency with pre-training
167
+ # ============================================================
168
+ fusion_method = scfm_hyper_params.get('fusion_method', 'multiply')
169
+ print(f"[SCFM_Module] Using fusion_method from hyper_params: {fusion_method}")
170
+
171
+ # Convert new indices to a list of index positions from the original tensor
172
+ scfm_genes_pad = scfm_genes + ['PAD']
173
+ self.scfm_seq_len = len(scfm_genes_pad)
174
+ scfm_mapping = {idx: i for i, idx in enumerate(scfm_genes_pad)}
175
+
176
+ # Create bidirectional index mapping
177
+ self.scfm_index_positions = [input_mapping.get(idx, self.input_gene_num) for idx in scfm_genes_pad]
178
+ self.input_index_positions = [scfm_mapping.get(idx, self.scfm_seq_len) for idx in input_genes]
179
+
180
+ # Load module data (if not provided in hyper_params)
181
+ if module_sets is None:
182
+ if 'module_sets' in scfm_hyper_params and scfm_hyper_params['module_sets'] is not None:
183
+ module_sets = scfm_hyper_params['module_sets']
184
+ else:
185
+ raise ValueError("module_sets must be provided (via parameters or hyper_params)")
186
+
187
+ self.module_sets = module_sets
188
+ self.num_modules = len(module_sets)
189
+ self.scfm_seq_len = len(scfm_genes)
190
+
191
+ # ============================================================
192
+ # [Key] Create a gene-to-module mapping
193
+ # Each gene may appear in multiple modules for subsequent average pooling.
194
+ # This is the core data structure of module representation.
195
+ # ============================================================
196
+ self.gene_to_modules = {} # {gene_idx: [module_idx1, module_idx2, ...]}
197
+ for module_idx, module_genes in enumerate(module_sets):
198
+ for gene_idx in module_genes:
199
+ if gene_idx < len(scfm_genes): # Only consider genes in scfm_genes
200
+ if gene_idx not in self.gene_to_modules:
201
+ self.gene_to_modules[gene_idx] = []
202
+ self.gene_to_modules[gene_idx].append(module_idx)
203
+
204
+ # ============================================================
205
+ # [Performance Optimization] Pre-built sparse index for efficient module->gene aggregation
206
+ # Use scatter_add to replace Python loop, greatly improving forward efficiency
207
+ # This is the standard optimization method in PyTorch for handling irregular aggregations
208
+ # ============================================================
209
+ import torch.sparse
210
+ max_modules_per_gene = max(len(v) for v in self.gene_to_modules.values()) if self.gene_to_modules else 1
211
+
212
+ # Construct a sparse index matrix: shape (num_genes, max_modules_per_gene)
213
+ gene_to_module_matrix = []
214
+ for gene_idx in range(self.scfm_seq_len):
215
+ if gene_idx in self.gene_to_modules:
216
+ modules = self.gene_to_modules[gene_idx]
217
+ padded = modules + [-1] * (max_modules_per_gene - len(modules))
218
+ gene_to_module_matrix.append(padded)
219
+ else:
220
+ gene_to_module_matrix.append([-1] * max_modules_per_gene)
221
+
222
+ # Registered as a buffer (will not be updated by the optimizer, but will be moved to the correct device with the model)
223
+ self.register_buffer('gene_to_module_matrix',
224
+ torch.tensor(gene_to_module_matrix, dtype=torch.long))
225
+ gene_module_counts = torch.tensor([
226
+ len(self.gene_to_modules.get(i, [])) for i in range(self.scfm_seq_len)
227
+ ], dtype=torch.long)
228
+ self.register_buffer('gene_module_counts', gene_module_counts)
229
+
230
+ # ============================================================
231
+ # [Performance Optimization] Precompute scatter_add index
232
+ # Module->gene aggregation for fully vectorized, avoiding Python loops
233
+ # ============================================================
234
+ module_indices = []
235
+ gene_indices = []
236
+ for module_idx, module_genes in enumerate(module_sets):
237
+ for gene_idx in module_genes:
238
+ if gene_idx < self.scfm_seq_len:
239
+ module_indices.append(module_idx)
240
+ gene_indices.append(gene_idx)
241
+
242
+ self.register_buffer('scatter_module_idx', torch.tensor(module_indices, dtype=torch.long))
243
+ self.register_buffer('scatter_gene_idx', torch.tensor(gene_indices, dtype=torch.long))
244
+
245
+ # Create PerformerEncoderOnly_Module model
246
+ scfm_hyper_params['module_sets'] = module_sets
247
+ scfm_hyper_params['num_genes'] = len(scfm_genes)
248
+ scfm_hyper_params['max_seq_len'] = len(module_sets)
249
+
250
+ self.scfm = PerformerEncoderOnly_Module(**scfm_hyper_params)
251
+
252
+ # ============================================================
253
+ # [Key] Load pre-trained weights
254
+ # Filter out incompatible parameters (GO related, module_heads, etc.)
255
+ # strict=False allows partial parameter mismatch (used for fine-tuning mode)
256
+ # ============================================================
257
+ if scfm_ckpt != 'None':
258
+ ckpt = torch.load(scfm_ckpt, map_location=device)
259
+ model_state_dict = ckpt.get('model_state_dict', ckpt)
260
+ current_state_dict = self.scfm.state_dict()
261
+ filtered_state_dict = {}
262
+ for k, v in model_state_dict.items():
263
+ # Filter out GO related parameters (may not be available in different data sets)
264
+ if 'go_conv' in k or 'G_go' in k.lower():
265
+ continue
266
+ # Filter out module_heads (not needed for fine-tuning)
267
+ if 'module_heads' in k:
268
+ continue
269
+ # Filter out parameters that do not exist or have mismatched shapes in the current model
270
+ if k not in current_state_dict:
271
+ continue
272
+ if current_state_dict[k].shape != v.shape:
273
+ print(
274
+ f"[SCFM_Module] skip shape-mismatched key: {k}, "
275
+ f"ckpt={tuple(v.shape)} current={tuple(current_state_dict[k].shape)}"
276
+ )
277
+ continue
278
+ filtered_state_dict[k] = v
279
+ # Use strict=False because module_heads is intentionally removed
280
+ missing, unexpected = self.scfm.load_state_dict(filtered_state_dict, strict=False)
281
+ for param in self.scfm.parameters():
282
+ param.requires_grad = requires_grad
283
+ else:
284
+ for param in self.scfm.parameters():
285
+ param.requires_grad = True
286
+
287
+ # Output dimension adjustment (if needed)
288
+ self.output_dim = output_dim
289
+ if self.output_dim != 200:
290
+ try:
291
+ self.feedforward = nn.Linear(self.scfm.base_dim, self.output_dim)
292
+ except AttributeError:
293
+ self.feedforward = nn.Linear(scfm_hyper_params.get('base_dim', 200), self.output_dim)
294
+
295
+ def forward(self, x):
296
+ """
297
+ Forward propagation: from gene expression values to gene-level embeddings
298
+
299
+ [Processing Process]
300
+ 1. Re-index the input gene expression into the gene order of the pre-trained model
301
+ 2. Get the module encoding through PerformerEncoderOnly_Module
302
+ 3. Restore gene-level embedding from module encoding (or return module encoding directly)
303
+ 4. Reindex gene embeddings back to input gene order
304
+
305
+ [Input and output]
306
+ Input: x (B, input_gene_num) - gene expression value
307
+ Output:
308
+ - use_module_level=False: output_reindex (B, input_gene_num, D) - gene level embedding
309
+ - use_module_level=True: module_encodings (B, num_modules, D) - module level encoding
310
+ """
311
+ x = x.view((-1, self.input_gene_num)) # B, M
312
+
313
+ # ============================================================
314
+ # [Step 1] Re-index: Map the input gene expression to the gene order of the pre-trained model
315
+ # Genes that are not present are filled with 0 (PAD position)
316
+ # ============================================================
317
+ x_scfm = reindex_tensor_universal(x, self.scfm_index_positions, dim=1, filler='0', device=self.device) # B, N
318
+
319
+ # ============================================================
320
+ # [Step 2] Obtain the module code through the pre-trained model
321
+ # mask_ratio=0 means not to mask any modules (fine-tuning mode)
322
+ # Use mask_ratio>0 for MAE pre-training during pre-training.
323
+ # ============================================================
324
+ out = self.scfm(x_scfm, mask_ratio=0.0, return_encodings=True)
325
+
326
+ if 'encodings' in out:
327
+ module_encodings = out['encodings'] # Local legacy interface
328
+ elif 'module_encodings' in out:
329
+ module_encodings = out['module_encodings'] # Reuse pre-trained shared implementation interfaces
330
+ else:
331
+ raise ValueError(f"Expected one of ['encodings', 'module_encodings'] in output, got: {list(out.keys())}")
332
+
333
+ # ============================================================
334
+ # [Option] Directly return to module level encoding
335
+ # For experimental module-level downstream tasks
336
+ # ============================================================
337
+ if self.use_module_level:
338
+ if self.output_dim != 200:
339
+ B, num_modules, D = module_encodings.shape
340
+ module_encodings_flat = module_encodings.reshape(B * num_modules, D)
341
+ module_encodings_flat = self.feedforward(module_encodings_flat)
342
+ module_encodings = module_encodings_flat.reshape(B, num_modules, self.output_dim)
343
+ return module_encodings # (B, num_modules, D)
344
+
345
+ # ============================================================
346
+ # [Step 3] Restore gene-level embedding from module encoding
347
+ # Core idea: The embedding of each gene = the average encoding of all modules containing the gene
348
+ # Vectorized aggregation using scatter_add to avoid Python loops
349
+ # ============================================================
350
+ B, num_modules, D = module_encodings.shape
351
+ G = self.scfm_seq_len
352
+
353
+ # Calculate global average (for genes without any modules)
354
+ global_mean = module_encodings.mean(dim=1) # (B, D)
355
+
356
+ # Extract the corresponding module embeddings: (B, num_pairs, D)
357
+ module_embs = module_encodings[:, self.scatter_module_idx, :] # (B, num_pairs, D)
358
+
359
+ # Aggregate to gene dimension using scatter_add
360
+ scfm_gene_embeddings = torch.zeros(B, G, D, device=module_encodings.device)
361
+ gene_idx_expanded = self.scatter_gene_idx.unsqueeze(0).unsqueeze(-1).expand(B, -1, D)
362
+ scfm_gene_embeddings.scatter_add_(1, gene_idx_expanded, module_embs)
363
+
364
+ # ============================================================
365
+ # [Key] Divide by the number of modules for each gene to get the average
366
+ # This is the core operation from module to gene aggregation
367
+ # clamp(min=1) avoids division by zero errors
368
+ # ============================================================
369
+ gene_counts = self.gene_module_counts.float().view(1, G, 1).clamp(min=1)
370
+ scfm_gene_embeddings = scfm_gene_embeddings / gene_counts
371
+
372
+ # For genes without any modules, use global average as fallback
373
+ no_module_mask = (self.gene_module_counts == 0).view(1, G, 1).float()
374
+ scfm_gene_embeddings = scfm_gene_embeddings * (1 - no_module_mask) + global_mean.unsqueeze(1) * no_module_mask
375
+
376
+ # Add embedding of PAD positions (for handling non-existent genes)
377
+ pad_emb = torch.zeros(B, 1, D, device=module_encodings.device)
378
+ output_encodings = torch.cat([scfm_gene_embeddings, pad_emb], dim=1) # (B, G+1, D)
379
+
380
+ # ============================================================
381
+ # [Step 4] Reindex back to the input gene sequence
382
+ # Use mean as fallback to fill in non-existent genes
383
+ # ============================================================
384
+ output_reindex = reindex_tensor_universal(
385
+ output_encodings,
386
+ self.input_index_positions,
387
+ dim=1,
388
+ filler='mean',
389
+ device=self.device
390
+ ) # B, M, D
391
+
392
+ if self.output_dim != 200:
393
+ output_reindex = self.feedforward(output_reindex)
394
+
395
+ return output_reindex
396
+
397
+
398
+ class GEARS_Model_Module(torch.nn.Module):
399
+ """
400
+ ================================================================================
401
+ GEARS_Model_Module: Genetic perturbation prediction model
402
+
403
+ [Core functions]
404
+ Predicting transcriptomic changes in cells following genetic perturbation. Enter a perturbation combination (such as knockout gene A),
405
+ Output the expression value prediction of each gene after perturbation.
406
+
407
+ [Model Architecture]
408
+ 1. Pretrained encoder (SCFM_Module): Encode gene expression into embedding vectors
409
+ 2. Perturbation embedding layer: generate perturbation features for perturbed genes
410
+ 3. GO graph neural network (GNN): using gene ontology knowledge to enhance perturbation representation
411
+ 4. Decoder: predict gene expression values after perturbation
412
+
413
+ [Key Design Decisions]
414
+ - Use GO graph neural network: gene ontology contains gene functional relationships, which can guide model understanding
415
+ How perturbation effects spread between genes
416
+ - Gene specific decoder: Each gene has independent decoding parameters to capture gene specificity
417
+ - Cross-gene MLP: Modeling interdependencies between genes
418
+
419
+ [Input and output]
420
+ Input: data (PyG Data object)
421
+ - x: (B*N, 2) feature matrix, [:, 0] is gene expression, [:, 1] is perturbation mark
422
+ - y: true expression value (used for training)
423
+ Output: out (B, N) - predicted gene expression value after perturbation
424
+ ================================================================================
425
+ """
426
+ def __init__(self, args):
427
+ super(GEARS_Model_Module, self).__init__()
428
+ self.args = args
429
+
430
+ # Forecast record related settings
431
+ if args.get('record_pred', False):
432
+ self.pred_dir = f"./result_process/preds/{self.args['exp_name']}"
433
+ if not os.path.exists(self.pred_dir):
434
+ os.makedirs(self.pred_dir)
435
+ self.pred_batch_idx = 0
436
+
437
+ # ============================================================
438
+ # [Basic parameters]
439
+ # hidden_size: embedding dimension, affecting model capacity
440
+ # num_genes: number of genes, determines the output dimension
441
+ # ============================================================
442
+ self.num_genes = args['num_genes']
443
+ hidden_size = args['hidden_size']
444
+ self.uncertainty = args['uncertainty']
445
+ self.num_layers = args['num_go_gnn_layers'] # GO GNN layers
446
+ self.indv_out_hidden_size = args['decoder_hidden_size']
447
+ self.num_layers_gene_pos = args['num_gene_gnn_layers']
448
+ self.pert_emb_lambda = 0.2
449
+
450
+ self.input_genes_list = args['input_genes_ens_ids']
451
+ self.scfm_genes_list = args['scfm_genes_ens_ids']
452
+
453
+ # ============================================================
454
+ # [Load pre-trained model configuration]
455
+ # hyper_params contains model architecture parameters, which must be consistent with pre-training
456
+ # ============================================================
457
+ try:
458
+ with open(args["scfm_hyper_params_path"], 'rb') as handle:
459
+ self.scfm_hyper_params = pickle.load(handle)
460
+ except ValueError:
461
+ import pickle5
462
+ with open(args["scfm_hyper_params_path"], 'rb') as handle:
463
+ self.scfm_hyper_params = pickle5.load(handle)
464
+ self.scfm_hyper_params.pop('module_embed_chunk_padded_gene_budget', None)
465
+ self.scfm_hyper_params = sanitize_performer_module_config(
466
+ self.scfm_hyper_params,
467
+ source=args["scfm_hyper_params_path"],
468
+ log_fn=print,
469
+ )
470
+
471
+ self.scfm_hyper_params['gene2vec_file'] = args['scfm_gene2vec_file']
472
+ self.scfm_ckpt = args["scfm_ckpt"]
473
+
474
+ # ============================================================
475
+ # [Module data loading]
476
+ # module_sets defines how genes are grouped into modules
477
+ # Can be loaded from hyper_params or external files
478
+ # ============================================================
479
+ module_sets = None
480
+ if 'module_sets' in self.scfm_hyper_params and self.scfm_hyper_params['module_sets'] is not None:
481
+ module_sets = self.scfm_hyper_params['module_sets']
482
+ elif args.get('module_data_path') or args.get('module_data_dir'):
483
+ from BioMod.module_dataset import load_module_data
484
+ module_sets = load_module_data(
485
+ module_data_path=args.get('module_data_path'),
486
+ module_data_dir=args.get('module_data_dir'),
487
+ gene_list_path=args.get('scfm_genes_list_path'),
488
+ gene_mapping_path=args.get('gene_mapping_path', str(Path(__file__).resolve().parents[2] / 'model_dependencies' / 'gene2ens.csv')),
489
+ verbose=True
490
+ )
491
+ else:
492
+ raise ValueError("module_sets must be provided (via hyper_params or module_data_path/module_data_dir)")
493
+
494
+ # ============================================================
495
+ # [Ablation Experiment]shuffle_module_genes
496
+ # Disrupt the gene assignment within the module and verify the importance of the module structure (gene grouping)
497
+ # If performance decreases after scrambling, it means that the module structure contains useful biological information.
498
+ # ============================================================
499
+ if args.get('shuffle_module_genes', False):
500
+ shuffle_seed = args.get('shuffle_seed', 2021)
501
+ print(
502
+ f"[GEARS_Model_Module] Applying downstream-only extra module reshuffle "
503
+ f"(seed={shuffle_seed}). Pretraining-time module_sets from PKL will be reshuffled again."
504
+ )
505
+ all_gene_indices = []
506
+ module_sizes = []
507
+ for module in module_sets:
508
+ all_gene_indices.extend(module)
509
+ module_sizes.append(len(module))
510
+
511
+ import random
512
+ rng = random.Random(shuffle_seed)
513
+ rng.shuffle(all_gene_indices)
514
+
515
+ shuffled_module_sets = []
516
+ start_idx = 0
517
+ for size in module_sizes:
518
+ shuffled_module_sets.append(all_gene_indices[start_idx:start_idx + size])
519
+ start_idx += size
520
+ module_sets = shuffled_module_sets
521
+ print(f"[GEARS_Model_Module] Shuffled {len(all_gene_indices)} genes across {len(module_sets)} modules")
522
+
523
+ # ============================================================
524
+ # [Perturbation Embedding Layer]
525
+ # pert_w: Convert perturbation markers (0/1) to embedding vectors
526
+ # gene_emb: Basic embedding of genes (used when pre-trained models are not available)
527
+ # pert_emb: dedicated embedding of perturbed genes
528
+ # ============================================================
529
+ self.pert_w = nn.Linear(1, hidden_size)
530
+
531
+ # gene/global perturbation embedding dictionary lookup
532
+ self.gene_emb = nn.Embedding(self.num_genes, hidden_size, max_norm=True)
533
+ self.pert_emb = nn.Embedding(self.num_genes, hidden_size, max_norm=True)
534
+
535
+ # transformation layer
536
+ self.emb_trans = nn.ReLU()
537
+ self.pert_base_trans = nn.ReLU()
538
+ self.transform = nn.ReLU()
539
+ self.pert_fuse = MLP([hidden_size, hidden_size, hidden_size], last_layer_act='ReLU')
540
+
541
+ # ============================================================
542
+ # [GO graph neural network]
543
+ # G_sim: edge index of GO similarity graph
544
+ # G_sim_weight: edge weight
545
+ # Using SGConv (Simplified Graph Convolution) for message passing
546
+ # GO diagram helps the model understand gene functional relationships, and perturbation effects propagate along the diagram.
547
+ # ============================================================
548
+ self.G_sim = args['G_go'].to(args['device'])
549
+ self.G_sim_weight = args['G_go_weight'].to(args['device'])
550
+
551
+ self.sim_layers = torch.nn.ModuleList()
552
+ for i in range(1, self.num_layers + 1):
553
+ self.sim_layers.append(SGConv(hidden_size, hidden_size, 1))
554
+
555
+ # ============================================================
556
+ # [Decoder]
557
+ # recovery_w: shared MLP, projects embeddings into decoding space
558
+ # indv_w1/b1: Gene specific parameters, each gene has independent decoding weight
559
+ # cross_gene_state: cross-gene MLP, modeling inter-gene dependencies
560
+ # indv_w2/b2: final decoding layer
561
+ # ============================================================
562
+ self.recovery_w = MLP([hidden_size, hidden_size*2, hidden_size], last_layer_act='linear')
563
+
564
+ # gene specific decoder
565
+ self.indv_w1 = nn.Parameter(torch.rand(self.num_genes, hidden_size, 1))
566
+ self.indv_b1 = nn.Parameter(torch.rand(self.num_genes, 1))
567
+ self.act = nn.ReLU()
568
+ nn.init.xavier_normal_(self.indv_w1)
569
+ nn.init.xavier_normal_(self.indv_b1)
570
+
571
+ # Cross gene MLP
572
+ self.cross_gene_state = MLP([self.num_genes, hidden_size, hidden_size])
573
+ self.indv_w2 = nn.Parameter(torch.rand(1, self.num_genes, hidden_size+1))
574
+ self.indv_b2 = nn.Parameter(torch.rand(1, self.num_genes))
575
+ nn.init.xavier_normal_(self.indv_w2)
576
+ nn.init.xavier_normal_(self.indv_b2)
577
+
578
+ # BatchNorm layers
579
+ self.bn_emb = nn.BatchNorm1d(hidden_size)
580
+ self.bn_pert_base = nn.BatchNorm1d(hidden_size)
581
+ self.bn_pert_base_trans = nn.BatchNorm1d(hidden_size)
582
+
583
+ # uncertainty mode (optional)
584
+ if self.uncertainty:
585
+ self.uncertainty_w = MLP([hidden_size, hidden_size*2, hidden_size, 1], last_layer_act='linear')
586
+
587
+ # ============================================================
588
+ # [Core] Initialize pre-trained encoder (SCFM_Module)
589
+ # requires_grad: whether to fine-tune pre-training parameters
590
+ # - finetune_method='frozen': freeze pre-training parameters
591
+ # - Others: Full parameter fine-tuning
592
+ # ============================================================
593
+ self.use_module_level = args.get('use_module_level', False)
594
+ self.singlecell_model = SCFM_Module(
595
+ self.input_genes_list,
596
+ self.scfm_genes_list,
597
+ self.scfm_hyper_params,
598
+ self.scfm_ckpt,
599
+ module_sets=module_sets,
600
+ requires_grad=args.get('finetune_method') != 'frozen',
601
+ device=args['device'],
602
+ output_dim=hidden_size,
603
+ use_module_level=self.use_module_level
604
+ )
605
+ self.pretrained = True
606
+
607
+ # ============================================================
608
+ # [Experimental feature] module level features
609
+ # In the current implementation, module-level downstream tasks are not fully implemented.
610
+ # GNN and decoder need to be redesigned to handle module dimensions
611
+ # ============================================================
612
+ if self.use_module_level:
613
+ self.num_modules = len(module_sets)
614
+ import warnings
615
+ warnings.warn(
616
+ f"Using module-level features. This requires compatible downstream architecture. "
617
+ f"Number of modules: {self.num_modules}, Number of genes: {self.num_genes}. "
618
+ f"Currently module-level downstream task is not fully implemented.",
619
+ UserWarning
620
+ )
621
+
622
+ def forward(self, data):
623
+ """
624
+ Forward propagation: predicting expression values after gene perturbation
625
+
626
+ [enter]
627
+ data: PyG Data object
628
+ - x: (B*N, 2) characteristic matrix
629
+ [:, 0]: gene expression value (control group)
630
+ [:, 1]: perturbation flag (1 indicates that the gene is perturbed)
631
+ - y: real expression value (used during training)
632
+ - pert: disturbance type identifier
633
+
634
+ [Output]
635
+ out: (B, N) predicted gene expression value after perturbation
636
+
637
+ [Processing Process]
638
+ 1. Obtain base gene embeddings (from pretrained encoder or random embeddings)
639
+ 2. Generate perturbation embedding (enhanced by GO-GNN)
640
+ 3. Merge base embedding and perturbation embedding
641
+ 4. Predict the expression value through the decoder
642
+ """
643
+ if str(type(data)) == "<class 'torch.Tensor'>":
644
+ x = data.reshape((-1, 2))
645
+ else:
646
+ x, batch = data.x, data.batch
647
+ x_pert = x[:, 1].reshape((-1, self.num_genes)) # Disturbance mark
648
+ num_graphs = len(data.batch.unique())
649
+
650
+ # ============================================================
651
+ # [Step 1] Obtain basic gene embedding
652
+ # If using a pretrained model, get embeddings from gene expression encoding
653
+ # Otherwise use randomly initialized gene embeddings
654
+ # ============================================================
655
+ if self.pretrained:
656
+ exp = x[:, 0].reshape((-1, self.num_genes)) # B, N - gene expression value
657
+ emb = self.singlecell_model(exp) # B, N, D - by pre-trained encoder
658
+ emb = emb.reshape((num_graphs * self.num_genes, -1))
659
+ else:
660
+ emb = self.gene_emb(torch.LongTensor(list(range(self.num_genes))).repeat(num_graphs, ).to(self.args['device']))
661
+ emb = self.bn_emb(emb)
662
+ base_emb = self.emb_trans(emb)
663
+
664
+ base_emb = base_emb.reshape(num_graphs, self.num_genes, -1)
665
+
666
+ pert = x_pert.reshape(-1, 1) # B * N, 1
667
+
668
+ # ============================================================
669
+ # [Step 2] Generate perturbation embedding (GO-GNN enhancement)
670
+ # Perturbation embeddings are propagated through GO similarity graphs to capture gene functional relationships
671
+ # This enables the model to exploit a priori knowledge of gene function
672
+ # ============================================================
673
+ pert_embeddings = self.pert_emb(torch.LongTensor(list(range(self.num_genes))).to(self.args['device'])) # N, D
674
+
675
+ # Propagating perturbation embeddings via GO-GNN
676
+ for idx, layer in enumerate(self.sim_layers):
677
+ pert_embeddings = layer(pert_embeddings, self.G_sim, self.G_sim_weight)
678
+ if idx < self.num_layers - 1:
679
+ pert_embeddings = pert_embeddings.relu()
680
+
681
+ # After GNN: N, D
682
+ pert_embeddings = torch.stack(list((pert_embeddings,)) * num_graphs) # B, N, D
683
+ pert_embeddings = pert_embeddings * x_pert.reshape((num_graphs, self.num_genes, 1)) # Only perturbed genes are retained
684
+ pert_embeddings = pert_embeddings.sum(1) # B, D - aggregate all perturbed genes
685
+ pert_mask = (x_pert.sum(1) != 0).reshape((num_graphs, 1)) # B, 1
686
+ pert_embeddings = self.pert_fuse(pert_embeddings) * pert_mask # B, D
687
+ pert_embeddings = pert_embeddings.reshape((num_graphs, 1, pert_embeddings.shape[1])) # B, 1, D
688
+
689
+ # ============================================================
690
+ # [Step 3] Merge base embedding and perturbation embedding
691
+ # Perturbation information is added to the embedding of each gene
692
+ # ============================================================
693
+ base_emb = base_emb + pert_embeddings
694
+
695
+ base_emb = base_emb.reshape(num_graphs * self.num_genes, -1)
696
+
697
+ # Add perturbed position embedding
698
+ pert_emb = self.pert_w(pert)
699
+ combined = pert_emb + base_emb
700
+ combined = self.bn_pert_base_trans(combined)
701
+ base_emb = self.pert_base_trans(combined)
702
+ base_emb = self.bn_pert_base(base_emb)
703
+
704
+ # ============================================================
705
+ # [Step 4] Decoder: predict the expression value after perturbation
706
+ # Using gene-specific decoders + cross-gene MLP
707
+ # ============================================================
708
+ base_emb = self.transform(base_emb)
709
+ out = self.recovery_w(base_emb) # B * N, D
710
+ out = out.reshape(num_graphs, self.num_genes, -1) # B, N, D
711
+ out = out.unsqueeze(-1) * self.indv_w1 # B, N, D, 1 - Gene specific transformation
712
+ w = torch.sum(out, axis=2) # B, N, 1
713
+ out = w + self.indv_b1 # B, N, 1
714
+
715
+ # Cross gene MLP: Modeling inter-gene dependencies
716
+ cross_gene_embed = self.cross_gene_state(out.reshape(num_graphs, self.num_genes, -1).squeeze(2)) # B, D
717
+ cross_gene_embed = cross_gene_embed.repeat(1, self.num_genes) # B, N*D
718
+
719
+ cross_gene_embed = cross_gene_embed.reshape([num_graphs, self.num_genes, -1]) # B, N, D
720
+ cross_gene_out = torch.cat([out, cross_gene_embed], 2) # B, N, D+1
721
+
722
+ cross_gene_out = cross_gene_out * self.indv_w2 # B, N, D+1
723
+ cross_gene_out = torch.sum(cross_gene_out, axis=2) # B, N
724
+ out = cross_gene_out + self.indv_b2 # B, N
725
+
726
+ # ============================================================
727
+ # [Key] Residual connection: predicted value = change + original value
728
+ # This is the core design of GEARS. The model predicts expression changes rather than absolute values.
729
+ # ============================================================
730
+ out = out.reshape(num_graphs * self.num_genes, -1) + x[:, 0].reshape(-1, 1) # B * N, 1
731
+
732
+ # ============================================================
733
+ # [Prediction record] for subsequent analysis
734
+ # ============================================================
735
+ if self.args.get('start_record', False):
736
+ y_flat = data.y.reshape(-1, 1) # B * N, 1
737
+ pred = torch.cat((pert, out, y_flat), dim=1) # B * N, 3
738
+ pred = pred.cpu().to_sparse()
739
+
740
+ values = pred.values().numpy()
741
+ indices = pred.indices().numpy()
742
+ sparse_matrix = scipy.sparse.coo_matrix((values, indices), shape=pred.shape)
743
+
744
+ scipy.sparse.save_npz(f'{self.pred_dir}/b{self.pred_batch_idx}_{num_graphs}.npz', sparse_matrix)
745
+ self.pred_batch_idx += 1
746
+
747
+ # ============================================================
748
+ # [Key] reshape output is (B, N) format
749
+ # In this way, the loss function can correctly index the predicted value and the true value.
750
+ # ============================================================
751
+ out = torch.split(torch.flatten(out), self.num_genes)
752
+ out = torch.stack(out) # (B, N)
753
+
754
+ # uncertainty mode (optional)
755
+ if self.uncertainty:
756
+ logvar = self.uncertainty_w(base_emb)
757
+ logvar = torch.split(torch.flatten(logvar), self.num_genes)
758
+ return out, torch.stack(logvar)
759
+
760
+ return out # (B, N)
761
+
BioMod/gene_perturbation/pertdata_local.py CHANGED
@@ -1,524 +1,524 @@
1
- """
2
- ================================================================================
3
- Perturbation data processing module (pertdata_local.py)
4
- ================================================================================
5
-
6
- [Module function]
7
- Load and preprocess the genetic perturbation dataset, converting the raw data into a model-usable PyG graph format.
8
-
9
- [Supported data sets]
10
- - norman: Norman et al. 2019, Large-scale CRISPR perturbation data
11
- - adamson: Adamson et al. 2016, single gene perturbation data
12
- - dixit: Dixit et al. 2016, single gene perturbation data
13
- - replogle_k562_essential/replogle_rpe1_essential: Replogle et al. 2022
14
-
15
- [Data format]
16
- Each cell is represented as a graph data object:
17
- - x: (num_genes, 2) feature matrix, [:, 0] is the expression value, [:, 1] is the perturbation mark
18
- - y: target expression value
19
- - de_idx: differentially expressed gene index
20
- - pert: disturbance type identifier
21
-
22
- [Key steps]
23
- 1. Load the original h5ad file
24
- 2. Calculate differentially expressed genes
25
- 3. Divide training/validation/test sets
26
- 4. Create PyG graph data objects
27
- ================================================================================
28
- """
29
-
30
- from torch_geometric.data import Data
31
- import torch
32
- import numpy as np
33
- import pickle
34
- from torch_geometric.data import DataLoader
35
- import os
36
- import scanpy as sc
37
- import networkx as nx
38
- from tqdm import tqdm
39
- import scipy.sparse as sp
40
-
41
- import warnings
42
- warnings.filterwarnings("ignore")
43
- sc.settings.verbosity = 0
44
-
45
- # ============================================================
46
- # Use local modules and avoid external gears_reduce imports
47
- # ============================================================
48
- from utils import print_sys, zip_data_download_wrapper, filter_pert_cond_in_go
49
- from data_utils_local import get_DE_genes, get_dropout_non_zero_genes, DataSplitter
50
-
51
- class PertData:
52
-
53
- def __init__(self, data_path):
54
- """
55
- Initialize the perturbation data processing class
56
-
57
- Args:
58
- data_path: data storage directory path
59
- """
60
- self.data_path = data_path
61
- self.gene_names = None
62
- if not os.path.exists(self.data_path):
63
- os.mkdir(self.data_path)
64
-
65
- def load(self, data_name = None,
66
- data_path = None, exclude_pert = set()):
67
- """
68
- Load perturbation dataset
69
-
70
- [Data source]
71
- - Download public data sets such as norman/adamson/dixit from Harvard Dataverse
72
- - You can also provide the local h5ad file path
73
-
74
- [Processing Process]
75
- 1. Download or load data
76
- 2. Filter disturbances that are not in the GO graph
77
- 3. Create PyG graph data objects
78
- """
79
- if data_name in ['norman', 'adamson', 'dixit',
80
- 'replogle_k562_essential',
81
- 'replogle_rpe1_essential']:
82
- # Download public datasets from Harvard Dataverse
83
- if data_name == 'norman':
84
- url = 'https://dataverse.harvard.edu/api/access/datafile/6154020'
85
- elif data_name == 'adamson':
86
- url = 'https://dataverse.harvard.edu/api/access/datafile/6154417'
87
- elif data_name == 'dixit':
88
- url = 'https://dataverse.harvard.edu/api/access/datafile/6154416'
89
- elif data_name == 'replogle_k562_essential':
90
- url = 'https://dataverse.harvard.edu/api/access/datafile/7458695'
91
- elif data_name == 'replogle_rpe1_essential':
92
- url = 'https://dataverse.harvard.edu/api/access/datafile/7458694'
93
- data_path = os.path.join(self.data_path, data_name)
94
- zip_data_download_wrapper(url, data_path, self.data_path)
95
- self.dataset_name = data_path.split('/')[-1]
96
- self.dataset_path = data_path
97
- adata_path = os.path.join(data_path, 'perturb_processed.h5ad')
98
- self.adata = sc.read_h5ad(adata_path)
99
-
100
- elif os.path.exists(data_path):
101
- # Load local data
102
- adata_path = os.path.join(data_path, 'perturb_processed.h5ad')
103
- self.adata = sc.read_h5ad(adata_path)
104
- self.dataset_name = data_path.split('/')[-1]
105
- self.dataset_path = data_path
106
- else:
107
- raise ValueError("data is either Norman/Adamson/Dixit or a path to an h5ad file")
108
-
109
- # Filter perturbed genes that are not in the GO graph
110
- print_sys('These perturbations are not in the GO graph and their '
111
- 'perturbation can thus not be predicted')
112
- print_sys(exclude_pert)
113
- if len(exclude_pert) > 0:
114
- filter_go = self.adata.obs[self.adata.obs.condition.apply(
115
- lambda x: filter_pert_cond_in_go(x, exclude_pert))]
116
- self.adata = self.adata[filter_go.index.values, :]
117
-
118
- # ============================================================
119
- # [Key] Create or load PyG graph data objects
120
- # This is the core step of data preprocessing, converting each cell into graph format
121
- # ============================================================
122
- pyg_path = os.path.join(data_path, 'data_pyg')
123
- if not os.path.exists(pyg_path):
124
- os.mkdir(pyg_path)
125
- dataset_fname = os.path.join(pyg_path, 'cell_graphs.pkl')
126
-
127
- if os.path.isfile(dataset_fname):
128
- # Already processed data can be loaded directly.
129
- print_sys("Local copy of pyg dataset is detected. Loading...")
130
- self.dataset_processed = pickle.load(open(dataset_fname, "rb"))
131
- print_sys("Done!")
132
- else:
133
- # Data needs to be reprocessed
134
- self.ctrl_adata = self.adata[self.adata.obs['condition'] == 'ctrl']
135
- self.gene_names = list(self.adata.var.gene_name)
136
- conditions = list(self.adata.obs.condition.unique())
137
- pert_genes = []
138
- for condition in conditions:
139
- if condition == 'ctrl':
140
- continue
141
- # Analyze perturbation combinations (format: gene1+gene2)
142
- cond1 = condition.split('+')[0]
143
- cond2 = condition.split('+')[1]
144
- if cond1 != 'ctrl':
145
- if cond1 not in pert_genes:
146
- pert_genes.append(cond1)
147
- if cond1 not in self.gene_names:
148
- self.gene_names.append(cond1)
149
- if cond2 != 'ctrl':
150
- if cond2 not in pert_genes:
151
- pert_genes.append(cond2)
152
- if cond2 not in self.gene_names:
153
- self.gene_names.append(cond2)
154
- self.gene_names = np.array(self.gene_names)
155
- self.real_gene_names = np.array(list(self.adata.var.gene_name))
156
- print_sys(f"len(self.gene_names): {len(self.gene_names)}")
157
- print_sys(f"len(self.real_gene_names): {len(self.real_gene_names)}")
158
- print_sys("Creating pyg object for each cell in the data...")
159
- self.dataset_processed = self.create_dataset_file()
160
- print_sys("Saving new dataset pyg object at " + dataset_fname)
161
- pickle.dump(self.dataset_processed, open(dataset_fname, "wb"))
162
- print_sys("Done!")
163
-
164
- def new_data_process(self, dataset_name,
165
- adata = None):
166
-
167
- if 'condition' not in adata.obs.columns.values:
168
- raise ValueError("Please specify condition")
169
- if 'gene_name' not in adata.var.columns.values:
170
- raise ValueError("Please specify gene name")
171
- if 'cell_type' not in adata.obs.columns.values:
172
- raise ValueError("Please specify cell type")
173
-
174
- dataset_name = dataset_name.lower()
175
- self.dataset_name = dataset_name
176
- save_data_folder = os.path.join(self.data_path, dataset_name)
177
-
178
- if not os.path.exists(save_data_folder):
179
- os.mkdir(save_data_folder)
180
- self.dataset_path = save_data_folder
181
- self.adata = get_DE_genes(adata)
182
- self.adata = get_dropout_non_zero_genes(self.adata)
183
- self.adata.write_h5ad(os.path.join(save_data_folder, 'perturb_processed.h5ad'))
184
-
185
- self.ctrl_adata = self.adata[self.adata.obs['condition'] == 'ctrl']
186
-
187
- self.gene_names = list(self.adata.var.gene_name)
188
- conditions = list(self.adata.obs.condition.unique())
189
- pert_genes = []
190
- for condition in conditions:
191
- if condition == 'ctrl':
192
- continue
193
- cond1 = condition.split('+')[0]
194
- cond2 = condition.split('+')[1]
195
- if cond1 != 'ctrl':
196
- if cond1 not in pert_genes:
197
- pert_genes.append(cond1)
198
- if cond1 not in self.gene_names:
199
- self.gene_names.append(cond1)
200
- if cond2 != 'ctrl':
201
- if cond2 not in pert_genes:
202
- pert_genes.append(cond2)
203
- if cond2 not in self.gene_names:
204
- self.gene_names.append(cond2)
205
- self.gene_names = np.array(self.gene_names)
206
- self.real_gene_names = np.array(list(self.adata.var.gene_name))
207
-
208
-
209
- pyg_path = os.path.join(save_data_folder, 'data_pyg')
210
- if not os.path.exists(pyg_path):
211
- os.mkdir(pyg_path)
212
- dataset_fname = os.path.join(pyg_path, 'cell_graphs.pkl')
213
- print_sys("Creating pyg object for each cell in the data...")
214
- self.dataset_processed = self.create_dataset_file()
215
- print_sys("Saving new dataset pyg object at " + dataset_fname)
216
- pickle.dump(self.dataset_processed, open(dataset_fname, "wb"))
217
- print_sys("Done!")
218
-
219
- def prepare_split(self, split = 'simulation',
220
- seed = 1,
221
- train_gene_set_size = 0.75,
222
- combo_seen2_train_frac = 0.75,
223
- combo_single_split_test_set_fraction = 0.1,
224
- test_perts = None,
225
- only_test_set_perts = False,
226
- test_pert_genes = None):
227
- """
228
- Prepare data partitioning
229
-
230
- [Description of partition strategy]
231
- - 'simulation': simulate the division of real application scenarios
232
- The test set contains: unseen_single (unseen single gene perturbation),
233
- combo_seen0/1/2 (combination perturbations with different familiarity levels)
234
- This is the most commonly used division method to evaluate the generalization ability of the model.
235
- - 'combo_seen0/1/2': divided according to the number of genes seen in the combination
236
- - 'single': only evaluate single gene perturbations
237
- - 'no_test': Do not divide the test set
238
-
239
- [Key parameters]
240
- - train_gene_set_size: training set gene proportion (0.75 means 75% of genes are used for training)
241
- - seed: random seed to ensure the experiment is reproducible
242
- """
243
- available_splits = ['simulation', 'simulation_single', 'combo_seen0', 'combo_seen1',
244
- 'combo_seen2', 'single', 'no_test', 'no_split']
245
- if split not in available_splits:
246
- raise ValueError('currently, we only support ' + ','.join(available_splits))
247
- self.split = split
248
- self.seed = seed
249
- self.subgroup = None
250
- self.train_gene_set_size = train_gene_set_size
251
-
252
- # Divide file path
253
- split_folder = os.path.join(self.dataset_path, 'splits')
254
- if not os.path.exists(split_folder):
255
- os.mkdir(split_folder)
256
- split_file = self.dataset_name + '_' + split + '_' + str(seed) + '_' + str(train_gene_set_size) + '.pkl'
257
- split_path = os.path.join(split_folder, split_file)
258
-
259
- if test_perts:
260
- split_path = split_path[:-4] + '_' + test_perts + '.pkl'
261
-
262
- if os.path.exists(split_path):
263
- # Already divided, load directly
264
- print_sys("Local copy of split is detected. Loading...")
265
- set2conditions = pickle.load(open(split_path, "rb"))
266
- if split == 'simulation':
267
- subgroup_path = split_path[:-4] + '_subgroup.pkl'
268
- subgroup = pickle.load(open(subgroup_path, "rb"))
269
- self.subgroup = subgroup
270
- else:
271
- # Create new partition
272
- print_sys("Creating new splits....")
273
- if test_perts:
274
- test_perts = test_perts.split('_')
275
-
276
- if split in ['simulation', 'simulation_single']:
277
- DS = DataSplitter(self.adata, split_type=split)
278
-
279
- adata, subgroup = DS.split_data(train_gene_set_size = train_gene_set_size,
280
- combo_seen2_train_frac = combo_seen2_train_frac,
281
- seed=seed,
282
- test_perts = test_perts,
283
- only_test_set_perts = only_test_set_perts
284
- )
285
- subgroup_path = split_path[:-4] + '_subgroup.pkl'
286
- pickle.dump(subgroup, open(subgroup_path, "wb"))
287
- self.subgroup = subgroup
288
-
289
- elif split[:5] == 'combo':
290
- split_type = 'combo'
291
- seen = int(split[-1])
292
-
293
- if test_pert_genes:
294
- test_pert_genes = test_pert_genes.split('_')
295
-
296
- DS = DataSplitter(self.adata, split_type=split_type, seen=int(seen))
297
- adata = DS.split_data(test_size=combo_single_split_test_set_fraction,
298
- test_perts=test_perts,
299
- test_pert_genes=test_pert_genes,
300
- seed=seed)
301
-
302
- elif split == 'single':
303
- DS = DataSplitter(self.adata, split_type=split)
304
- adata = DS.split_data(test_size=combo_single_split_test_set_fraction, seed=seed)
305
-
306
- elif split == 'no_test':
307
- DS = DataSplitter(self.adata, split_type=split)
308
- adata = DS.split_data(test_size=combo_single_split_test_set_fraction, seed=seed)
309
-
310
- elif split == 'no_split':
311
- adata = self.adata
312
- adata.obs['split'] = 'test'
313
-
314
- # Save partition results
315
- set2conditions = dict(adata.obs.groupby('split').agg({'condition': lambda x: x}).condition)
316
- set2conditions = {i: j.unique().tolist() for i,j in set2conditions.items()}
317
- pickle.dump(set2conditions, open(split_path, "wb"))
318
- print_sys("Saving new splits at " + split_path)
319
-
320
- self.set2conditions = set2conditions
321
-
322
- if split == 'simulation':
323
- print_sys('Simulation split test composition:')
324
- for i,j in subgroup['test_subgroup'].items():
325
- print_sys(i + ':' + str(len(j)))
326
- print_sys("Done!")
327
-
328
- def get_dataloader(self, batch_size, test_batch_size = None):
329
- """
330
- Create data loader
331
-
332
- [Data loader]
333
- Encapsulate PyG graph data objects as DataLoader, supporting batch processing and shuffling.
334
-
335
- [Key settings]
336
- - Training set: shuffle=True, drop_last=True (make sure each batch size is consistent)
337
- - Validation/test set: shuffle=False (maintain order for easy analysis)
338
- """
339
- if test_batch_size is None:
340
- test_batch_size = batch_size
341
- if self.gene_names is None:
342
- self.gene_names = list(self.adata.var.gene_name)
343
- conditions = list(self.adata.obs.condition.unique())
344
- pert_genes = []
345
- for condition in conditions:
346
- if condition == 'ctrl':
347
- continue
348
- cond1 = condition.split('+')[0]
349
- cond2 = condition.split('+')[1]
350
- if cond1 != 'ctrl':
351
- if cond1 not in pert_genes:
352
- pert_genes.append(cond1)
353
- if cond1 not in self.gene_names:
354
- self.gene_names.append(cond1)
355
- if cond2 != 'ctrl':
356
- if cond2 not in pert_genes:
357
- pert_genes.append(cond2)
358
- if cond2 not in self.gene_names:
359
- self.gene_names.append(cond2)
360
- self.gene_names = np.array(self.gene_names)
361
- self.real_gene_names = np.array(list(self.adata.var.gene_name))
362
-
363
- # Gene name to index mapping
364
- self.node_map = {x: it for it, x in enumerate(self.gene_names)}
365
- if (self.gene_names[: len(self.real_gene_names)] != self.real_gene_names).any():
366
- raise ValueError("self.real_gene_names must be in the start of self.gene_names")
367
-
368
- # Create cytogram data
369
- cell_graphs = {}
370
- if self.split == 'no_split':
371
- i = 'test'
372
- cell_graphs[i] = []
373
- for p in self.set2conditions[i]:
374
- if p != 'ctrl':
375
- cell_graphs[i].extend(self.dataset_processed[p])
376
-
377
- print_sys("Creating dataloaders....")
378
- test_loader = DataLoader(cell_graphs['test'],
379
- batch_size=batch_size, shuffle=False)
380
-
381
- print_sys("Dataloaders created...")
382
- return {'test_loader': test_loader}
383
- else:
384
- if self.split =='no_test':
385
- splits = ['train','val']
386
- else:
387
- splits = ['train','val','test']
388
- for i in splits:
389
- cell_graphs[i] = []
390
- for p in self.set2conditions[i]:
391
- cell_graphs[i].extend(self.dataset_processed[p])
392
-
393
- print_sys("Creating dataloaders....")
394
-
395
- # Set up data loader
396
- train_loader = DataLoader(cell_graphs['train'],
397
- batch_size=batch_size, shuffle=True, drop_last = True)
398
- val_loader = DataLoader(cell_graphs['val'],
399
- batch_size=batch_size, shuffle=True)
400
-
401
- if self.split !='no_test':
402
- test_loader = DataLoader(cell_graphs['test'],
403
- batch_size=batch_size, shuffle=False)
404
- self.dataloader = {'train_loader': train_loader,
405
- 'val_loader': val_loader,
406
- 'test_loader': test_loader}
407
-
408
- else:
409
- self.dataloader = {'train_loader': train_loader,
410
- 'val_loader': val_loader}
411
- print_sys("Done!")
412
- del self.dataset_processed # Clean memory
413
-
414
-
415
- def create_dataset_file(self):
416
- dl = {}
417
- for p in tqdm(self.adata.obs['condition'].unique()):
418
- cell_graph_dataset = self.create_cell_graph_dataset(self.adata, p, num_samples=1)
419
- dl[p] = cell_graph_dataset
420
- return dl
421
-
422
- def get_pert_idx(self, pert_category, adata_):
423
- pert_idx = [np.where(p == self.gene_names)[0][0]
424
- for p in pert_category.split('+')
425
- if p != 'ctrl']
426
- return pert_idx
427
-
428
- # Set up feature matrix and output
429
- def create_cell_graph(self, X, y, de_idx, pert, pert_idx=None):
430
- """
431
- Create a graph data object of a single cell
432
-
433
- [enter]
434
- - X: Control group gene expression value (expression before perturbation)
435
- - y: gene expression value after perturbation (target value)
436
- - de_idx: index of differentially expressed genes
437
- - pert: disturbance type identifier
438
- - pert_idx: index list of perturbed genes
439
-
440
- [Output]
441
- PyG Data object:
442
- - x: (num_genes, 2) feature matrix
443
- [:, 0]: Control group expression value
444
- [:, 1]: perturbation flag (1=the gene is perturbed)
445
- - y: target expression value
446
- - de_idx: differentially expressed gene index
447
- - pert: perturbation type
448
- """
449
- # Create perturbation feature vectors (mark which genes are perturbed)
450
- pert_feats = np.zeros(len(X[0]))
451
- if pert_idx is not None:
452
- for p in pert_idx:
453
- pert_feats[int(np.abs(p))] = 1
454
- pert_feats = np.expand_dims(pert_feats, 0)
455
- # Spliced expression values and perturbation markers
456
- feature_mat = torch.Tensor(np.concatenate([X, pert_feats])).T
457
-
458
- return Data(x=feature_mat, edge_index=None, edge_attr=None,
459
- y=torch.Tensor(y), de_idx=de_idx, pert=pert)
460
-
461
- def create_cell_graph_dataset(self, split_adata, pert_category,
462
- num_samples=1):
463
- """
464
- Create a graph dataset of perturbation categories
465
-
466
- [Processing logic]
467
- 1. For non-control group disturbances:
468
- - Get perturbed gene index
469
- - Expression before perturbation = randomly sample control cells
470
- - Target = expression of cells after perturbation
471
- 2. For the control group:
472
- - Expression before perturbation = target = expression in control cells
473
-
474
- [num_samples parameter]
475
- Each perturbed cell can be paired with multiple control group cells,
476
- Increase the diversity of training data
477
- """
478
- num_de_genes = 20 # Only focus on the top 20 differentially expressed genes for evaluation
479
- adata_ = split_adata[split_adata.obs['condition'] == pert_category]
480
- de_genes = adata_.uns['rank_genes_groups_cov_all']
481
- Xs = []
482
- ys = []
483
-
484
- # Handling non-control group disturbances
485
- if pert_category != 'ctrl':
486
- # Get the index of the perturbed gene
487
- pert_idx = self.get_pert_idx(pert_category, adata_)
488
-
489
- # Get differentially expressed gene index
490
- pert_de_category = adata_.obs['condition_name'][0]
491
- de_idx = np.where(adata_.var_names.isin(
492
- np.array(de_genes[pert_de_category][:num_de_genes])))[0]
493
-
494
- # Expand feature matrix to include perturbed genes
495
- padded_X = sp.lil_matrix((adata_.X.shape[0], len(self.gene_names)), dtype=adata_.X.dtype)
496
- padded_X[:, :len(self.real_gene_names)] = adata_.X
497
-
498
- for cell_z in padded_X:
499
- # Use control group samples as expression before perturbation
500
- ctrl_samples = self.ctrl_adata[np.random.randint(0,
501
- len(self.ctrl_adata), num_samples), :]
502
- padded_ctrl_X = sp.lil_matrix((ctrl_samples.X.shape[0], len(self.gene_names)), dtype=ctrl_samples.X.dtype)
503
- padded_ctrl_X[:, :len(self.real_gene_names)] = ctrl_samples.X
504
- for c in padded_ctrl_X:
505
- Xs.append(c) # control group expression
506
- ys.append(cell_z) # expression after perturbation
507
-
508
- # Process control group
509
- else:
510
- pert_idx = None
511
- de_idx = [-1] * num_de_genes
512
- padded_X = sp.lil_matrix((adata_.X.shape[0], len(self.gene_names)), dtype=adata_.X.dtype)
513
- padded_X[:, :len(self.real_gene_names)] = adata_.X
514
- for cell_z in padded_X:
515
- Xs.append(cell_z)
516
- ys.append(cell_z)
517
-
518
- # Create a cell diagram data object
519
- cell_graphs = []
520
- for X, y in zip(Xs, ys):
521
- cell_graphs.append(self.create_cell_graph(X.toarray(),
522
- y.toarray(), de_idx, pert_category, pert_idx))
523
-
524
  return cell_graphs
 
1
+ """
2
+ ================================================================================
3
+ Perturbation data processing module (pertdata_local.py)
4
+ ================================================================================
5
+
6
+ [Module function]
7
+ Load and preprocess the genetic perturbation dataset, converting the raw data into a model-usable PyG graph format.
8
+
9
+ [Supported data sets]
10
+ - norman: Norman et al. 2019, Large-scale CRISPR perturbation data
11
+ - adamson: Adamson et al. 2016, single gene perturbation data
12
+ - dixit: Dixit et al. 2016, single gene perturbation data
13
+ - replogle_k562_essential/replogle_rpe1_essential: Replogle et al. 2022
14
+
15
+ [Data format]
16
+ Each cell is represented as a graph data object:
17
+ - x: (num_genes, 2) feature matrix, [:, 0] is the expression value, [:, 1] is the perturbation mark
18
+ - y: target expression value
19
+ - de_idx: differentially expressed gene index
20
+ - pert: disturbance type identifier
21
+
22
+ [Key steps]
23
+ 1. Load the original h5ad file
24
+ 2. Calculate differentially expressed genes
25
+ 3. Divide training/validation/test sets
26
+ 4. Create PyG graph data objects
27
+ ================================================================================
28
+ """
29
+
30
+ from torch_geometric.data import Data
31
+ import torch
32
+ import numpy as np
33
+ import pickle
34
+ from torch_geometric.data import DataLoader
35
+ import os
36
+ import scanpy as sc
37
+ import networkx as nx
38
+ from tqdm import tqdm
39
+ import scipy.sparse as sp
40
+
41
+ import warnings
42
+ warnings.filterwarnings("ignore")
43
+ sc.settings.verbosity = 0
44
+
45
+ # ============================================================
46
+ # Use local modules and avoid external gears_reduce imports
47
+ # ============================================================
48
+ from utils import print_sys, zip_data_download_wrapper, filter_pert_cond_in_go
49
+ from data_utils_local import get_DE_genes, get_dropout_non_zero_genes, DataSplitter
50
+
51
+ class PertData:
52
+
53
+ def __init__(self, data_path):
54
+ """
55
+ Initialize the perturbation data processing class
56
+
57
+ Args:
58
+ data_path: data storage directory path
59
+ """
60
+ self.data_path = data_path
61
+ self.gene_names = None
62
+ if not os.path.exists(self.data_path):
63
+ os.mkdir(self.data_path)
64
+
65
+ def load(self, data_name = None,
66
+ data_path = None, exclude_pert = set()):
67
+ """
68
+ Load perturbation dataset
69
+
70
+ [Data source]
71
+ - Download public data sets such as norman/adamson/dixit from Harvard Dataverse
72
+ - You can also provide the local h5ad file path
73
+
74
+ [Processing Process]
75
+ 1. Download or load data
76
+ 2. Filter disturbances that are not in the GO graph
77
+ 3. Create PyG graph data objects
78
+ """
79
+ if data_name in ['norman', 'adamson', 'dixit',
80
+ 'replogle_k562_essential',
81
+ 'replogle_rpe1_essential']:
82
+ # Download public datasets from Harvard Dataverse
83
+ if data_name == 'norman':
84
+ url = 'https://dataverse.harvard.edu/api/access/datafile/6154020'
85
+ elif data_name == 'adamson':
86
+ url = 'https://dataverse.harvard.edu/api/access/datafile/6154417'
87
+ elif data_name == 'dixit':
88
+ url = 'https://dataverse.harvard.edu/api/access/datafile/6154416'
89
+ elif data_name == 'replogle_k562_essential':
90
+ url = 'https://dataverse.harvard.edu/api/access/datafile/7458695'
91
+ elif data_name == 'replogle_rpe1_essential':
92
+ url = 'https://dataverse.harvard.edu/api/access/datafile/7458694'
93
+ data_path = os.path.join(self.data_path, data_name)
94
+ zip_data_download_wrapper(url, data_path, self.data_path)
95
+ self.dataset_name = os.path.basename(os.path.normpath(data_path))
96
+ self.dataset_path = data_path
97
+ adata_path = os.path.join(data_path, 'perturb_processed.h5ad')
98
+ self.adata = sc.read_h5ad(adata_path)
99
+
100
+ elif os.path.exists(data_path):
101
+ # Load local data
102
+ adata_path = os.path.join(data_path, 'perturb_processed.h5ad')
103
+ self.adata = sc.read_h5ad(adata_path)
104
+ self.dataset_name = os.path.basename(os.path.normpath(data_path))
105
+ self.dataset_path = data_path
106
+ else:
107
+ raise ValueError("data is either Norman/Adamson/Dixit or a path to an h5ad file")
108
+
109
+ # Filter perturbed genes that are not in the GO graph
110
+ print_sys('These perturbations are not in the GO graph and their '
111
+ 'perturbation can thus not be predicted')
112
+ print_sys(exclude_pert)
113
+ if len(exclude_pert) > 0:
114
+ filter_go = self.adata.obs[self.adata.obs.condition.apply(
115
+ lambda x: filter_pert_cond_in_go(x, exclude_pert))]
116
+ self.adata = self.adata[filter_go.index.values, :]
117
+
118
+ # ============================================================
119
+ # [Key] Create or load PyG graph data objects
120
+ # This is the core step of data preprocessing, converting each cell into graph format
121
+ # ============================================================
122
+ pyg_path = os.path.join(data_path, 'data_pyg')
123
+ if not os.path.exists(pyg_path):
124
+ os.mkdir(pyg_path)
125
+ dataset_fname = os.path.join(pyg_path, 'cell_graphs.pkl')
126
+
127
+ if os.path.isfile(dataset_fname):
128
+ # Already processed data can be loaded directly.
129
+ print_sys("Local copy of pyg dataset is detected. Loading...")
130
+ self.dataset_processed = pickle.load(open(dataset_fname, "rb"))
131
+ print_sys("Done!")
132
+ else:
133
+ # Data needs to be reprocessed
134
+ self.ctrl_adata = self.adata[self.adata.obs['condition'] == 'ctrl']
135
+ self.gene_names = list(self.adata.var.gene_name)
136
+ conditions = list(self.adata.obs.condition.unique())
137
+ pert_genes = []
138
+ for condition in conditions:
139
+ if condition == 'ctrl':
140
+ continue
141
+ # Analyze perturbation combinations (format: gene1+gene2)
142
+ cond1 = condition.split('+')[0]
143
+ cond2 = condition.split('+')[1]
144
+ if cond1 != 'ctrl':
145
+ if cond1 not in pert_genes:
146
+ pert_genes.append(cond1)
147
+ if cond1 not in self.gene_names:
148
+ self.gene_names.append(cond1)
149
+ if cond2 != 'ctrl':
150
+ if cond2 not in pert_genes:
151
+ pert_genes.append(cond2)
152
+ if cond2 not in self.gene_names:
153
+ self.gene_names.append(cond2)
154
+ self.gene_names = np.array(self.gene_names)
155
+ self.real_gene_names = np.array(list(self.adata.var.gene_name))
156
+ print_sys(f"len(self.gene_names): {len(self.gene_names)}")
157
+ print_sys(f"len(self.real_gene_names): {len(self.real_gene_names)}")
158
+ print_sys("Creating pyg object for each cell in the data...")
159
+ self.dataset_processed = self.create_dataset_file()
160
+ print_sys("Saving new dataset pyg object at " + dataset_fname)
161
+ pickle.dump(self.dataset_processed, open(dataset_fname, "wb"))
162
+ print_sys("Done!")
163
+
164
+ def new_data_process(self, dataset_name,
165
+ adata = None):
166
+
167
+ if 'condition' not in adata.obs.columns.values:
168
+ raise ValueError("Please specify condition")
169
+ if 'gene_name' not in adata.var.columns.values:
170
+ raise ValueError("Please specify gene name")
171
+ if 'cell_type' not in adata.obs.columns.values:
172
+ raise ValueError("Please specify cell type")
173
+
174
+ dataset_name = dataset_name.lower()
175
+ self.dataset_name = dataset_name
176
+ save_data_folder = os.path.join(self.data_path, dataset_name)
177
+
178
+ if not os.path.exists(save_data_folder):
179
+ os.mkdir(save_data_folder)
180
+ self.dataset_path = save_data_folder
181
+ self.adata = get_DE_genes(adata)
182
+ self.adata = get_dropout_non_zero_genes(self.adata)
183
+ self.adata.write_h5ad(os.path.join(save_data_folder, 'perturb_processed.h5ad'))
184
+
185
+ self.ctrl_adata = self.adata[self.adata.obs['condition'] == 'ctrl']
186
+
187
+ self.gene_names = list(self.adata.var.gene_name)
188
+ conditions = list(self.adata.obs.condition.unique())
189
+ pert_genes = []
190
+ for condition in conditions:
191
+ if condition == 'ctrl':
192
+ continue
193
+ cond1 = condition.split('+')[0]
194
+ cond2 = condition.split('+')[1]
195
+ if cond1 != 'ctrl':
196
+ if cond1 not in pert_genes:
197
+ pert_genes.append(cond1)
198
+ if cond1 not in self.gene_names:
199
+ self.gene_names.append(cond1)
200
+ if cond2 != 'ctrl':
201
+ if cond2 not in pert_genes:
202
+ pert_genes.append(cond2)
203
+ if cond2 not in self.gene_names:
204
+ self.gene_names.append(cond2)
205
+ self.gene_names = np.array(self.gene_names)
206
+ self.real_gene_names = np.array(list(self.adata.var.gene_name))
207
+
208
+
209
+ pyg_path = os.path.join(save_data_folder, 'data_pyg')
210
+ if not os.path.exists(pyg_path):
211
+ os.mkdir(pyg_path)
212
+ dataset_fname = os.path.join(pyg_path, 'cell_graphs.pkl')
213
+ print_sys("Creating pyg object for each cell in the data...")
214
+ self.dataset_processed = self.create_dataset_file()
215
+ print_sys("Saving new dataset pyg object at " + dataset_fname)
216
+ pickle.dump(self.dataset_processed, open(dataset_fname, "wb"))
217
+ print_sys("Done!")
218
+
219
+ def prepare_split(self, split = 'simulation',
220
+ seed = 1,
221
+ train_gene_set_size = 0.75,
222
+ combo_seen2_train_frac = 0.75,
223
+ combo_single_split_test_set_fraction = 0.1,
224
+ test_perts = None,
225
+ only_test_set_perts = False,
226
+ test_pert_genes = None):
227
+ """
228
+ Prepare data partitioning
229
+
230
+ [Description of partition strategy]
231
+ - 'simulation': simulate the division of real application scenarios
232
+ The test set contains: unseen_single (unseen single gene perturbation),
233
+ combo_seen0/1/2 (combination perturbations with different familiarity levels)
234
+ This is the most commonly used division method to evaluate the generalization ability of the model.
235
+ - 'combo_seen0/1/2': divided according to the number of genes seen in the combination
236
+ - 'single': only evaluate single gene perturbations
237
+ - 'no_test': Do not divide the test set
238
+
239
+ [Key parameters]
240
+ - train_gene_set_size: training set gene proportion (0.75 means 75% of genes are used for training)
241
+ - seed: random seed to ensure the experiment is reproducible
242
+ """
243
+ available_splits = ['simulation', 'simulation_single', 'combo_seen0', 'combo_seen1',
244
+ 'combo_seen2', 'single', 'no_test', 'no_split']
245
+ if split not in available_splits:
246
+ raise ValueError('currently, we only support ' + ','.join(available_splits))
247
+ self.split = split
248
+ self.seed = seed
249
+ self.subgroup = None
250
+ self.train_gene_set_size = train_gene_set_size
251
+
252
+ # Divide file path
253
+ split_folder = os.path.join(self.dataset_path, 'splits')
254
+ if not os.path.exists(split_folder):
255
+ os.mkdir(split_folder)
256
+ split_file = self.dataset_name + '_' + split + '_' + str(seed) + '_' + str(train_gene_set_size) + '.pkl'
257
+ split_path = os.path.join(split_folder, split_file)
258
+
259
+ if test_perts:
260
+ split_path = split_path[:-4] + '_' + test_perts + '.pkl'
261
+
262
+ if os.path.exists(split_path):
263
+ # Already divided, load directly
264
+ print_sys("Local copy of split is detected. Loading...")
265
+ set2conditions = pickle.load(open(split_path, "rb"))
266
+ if split == 'simulation':
267
+ subgroup_path = split_path[:-4] + '_subgroup.pkl'
268
+ subgroup = pickle.load(open(subgroup_path, "rb"))
269
+ self.subgroup = subgroup
270
+ else:
271
+ # Create new partition
272
+ print_sys("Creating new splits....")
273
+ if test_perts:
274
+ test_perts = test_perts.split('_')
275
+
276
+ if split in ['simulation', 'simulation_single']:
277
+ DS = DataSplitter(self.adata, split_type=split)
278
+
279
+ adata, subgroup = DS.split_data(train_gene_set_size = train_gene_set_size,
280
+ combo_seen2_train_frac = combo_seen2_train_frac,
281
+ seed=seed,
282
+ test_perts = test_perts,
283
+ only_test_set_perts = only_test_set_perts
284
+ )
285
+ subgroup_path = split_path[:-4] + '_subgroup.pkl'
286
+ pickle.dump(subgroup, open(subgroup_path, "wb"))
287
+ self.subgroup = subgroup
288
+
289
+ elif split[:5] == 'combo':
290
+ split_type = 'combo'
291
+ seen = int(split[-1])
292
+
293
+ if test_pert_genes:
294
+ test_pert_genes = test_pert_genes.split('_')
295
+
296
+ DS = DataSplitter(self.adata, split_type=split_type, seen=int(seen))
297
+ adata = DS.split_data(test_size=combo_single_split_test_set_fraction,
298
+ test_perts=test_perts,
299
+ test_pert_genes=test_pert_genes,
300
+ seed=seed)
301
+
302
+ elif split == 'single':
303
+ DS = DataSplitter(self.adata, split_type=split)
304
+ adata = DS.split_data(test_size=combo_single_split_test_set_fraction, seed=seed)
305
+
306
+ elif split == 'no_test':
307
+ DS = DataSplitter(self.adata, split_type=split)
308
+ adata = DS.split_data(test_size=combo_single_split_test_set_fraction, seed=seed)
309
+
310
+ elif split == 'no_split':
311
+ adata = self.adata
312
+ adata.obs['split'] = 'test'
313
+
314
+ # Save partition results
315
+ set2conditions = dict(adata.obs.groupby('split').agg({'condition': lambda x: x}).condition)
316
+ set2conditions = {i: j.unique().tolist() for i,j in set2conditions.items()}
317
+ pickle.dump(set2conditions, open(split_path, "wb"))
318
+ print_sys("Saving new splits at " + split_path)
319
+
320
+ self.set2conditions = set2conditions
321
+
322
+ if split == 'simulation':
323
+ print_sys('Simulation split test composition:')
324
+ for i,j in subgroup['test_subgroup'].items():
325
+ print_sys(i + ':' + str(len(j)))
326
+ print_sys("Done!")
327
+
328
+ def get_dataloader(self, batch_size, test_batch_size = None):
329
+ """
330
+ Create data loader
331
+
332
+ [Data loader]
333
+ Encapsulate PyG graph data objects as DataLoader, supporting batch processing and shuffling.
334
+
335
+ [Key settings]
336
+ - Training set: shuffle=True, drop_last=True (make sure each batch size is consistent)
337
+ - Validation/test set: shuffle=False (maintain order for easy analysis)
338
+ """
339
+ if test_batch_size is None:
340
+ test_batch_size = batch_size
341
+ if self.gene_names is None:
342
+ self.gene_names = list(self.adata.var.gene_name)
343
+ conditions = list(self.adata.obs.condition.unique())
344
+ pert_genes = []
345
+ for condition in conditions:
346
+ if condition == 'ctrl':
347
+ continue
348
+ cond1 = condition.split('+')[0]
349
+ cond2 = condition.split('+')[1]
350
+ if cond1 != 'ctrl':
351
+ if cond1 not in pert_genes:
352
+ pert_genes.append(cond1)
353
+ if cond1 not in self.gene_names:
354
+ self.gene_names.append(cond1)
355
+ if cond2 != 'ctrl':
356
+ if cond2 not in pert_genes:
357
+ pert_genes.append(cond2)
358
+ if cond2 not in self.gene_names:
359
+ self.gene_names.append(cond2)
360
+ self.gene_names = np.array(self.gene_names)
361
+ self.real_gene_names = np.array(list(self.adata.var.gene_name))
362
+
363
+ # Gene name to index mapping
364
+ self.node_map = {x: it for it, x in enumerate(self.gene_names)}
365
+ if (self.gene_names[: len(self.real_gene_names)] != self.real_gene_names).any():
366
+ raise ValueError("self.real_gene_names must be in the start of self.gene_names")
367
+
368
+ # Create cytogram data
369
+ cell_graphs = {}
370
+ if self.split == 'no_split':
371
+ i = 'test'
372
+ cell_graphs[i] = []
373
+ for p in self.set2conditions[i]:
374
+ if p != 'ctrl':
375
+ cell_graphs[i].extend(self.dataset_processed[p])
376
+
377
+ print_sys("Creating dataloaders....")
378
+ test_loader = DataLoader(cell_graphs['test'],
379
+ batch_size=batch_size, shuffle=False)
380
+
381
+ print_sys("Dataloaders created...")
382
+ return {'test_loader': test_loader}
383
+ else:
384
+ if self.split =='no_test':
385
+ splits = ['train','val']
386
+ else:
387
+ splits = ['train','val','test']
388
+ for i in splits:
389
+ cell_graphs[i] = []
390
+ for p in self.set2conditions[i]:
391
+ cell_graphs[i].extend(self.dataset_processed[p])
392
+
393
+ print_sys("Creating dataloaders....")
394
+
395
+ # Set up data loader
396
+ train_loader = DataLoader(cell_graphs['train'],
397
+ batch_size=batch_size, shuffle=True, drop_last = True)
398
+ val_loader = DataLoader(cell_graphs['val'],
399
+ batch_size=batch_size, shuffle=True)
400
+
401
+ if self.split !='no_test':
402
+ test_loader = DataLoader(cell_graphs['test'],
403
+ batch_size=batch_size, shuffle=False)
404
+ self.dataloader = {'train_loader': train_loader,
405
+ 'val_loader': val_loader,
406
+ 'test_loader': test_loader}
407
+
408
+ else:
409
+ self.dataloader = {'train_loader': train_loader,
410
+ 'val_loader': val_loader}
411
+ print_sys("Done!")
412
+ del self.dataset_processed # Clean memory
413
+
414
+
415
+ def create_dataset_file(self):
416
+ dl = {}
417
+ for p in tqdm(self.adata.obs['condition'].unique()):
418
+ cell_graph_dataset = self.create_cell_graph_dataset(self.adata, p, num_samples=1)
419
+ dl[p] = cell_graph_dataset
420
+ return dl
421
+
422
+ def get_pert_idx(self, pert_category, adata_):
423
+ pert_idx = [np.where(p == self.gene_names)[0][0]
424
+ for p in pert_category.split('+')
425
+ if p != 'ctrl']
426
+ return pert_idx
427
+
428
+ # Set up feature matrix and output
429
+ def create_cell_graph(self, X, y, de_idx, pert, pert_idx=None):
430
+ """
431
+ Create a graph data object of a single cell
432
+
433
+ [enter]
434
+ - X: Control group gene expression value (expression before perturbation)
435
+ - y: gene expression value after perturbation (target value)
436
+ - de_idx: index of differentially expressed genes
437
+ - pert: disturbance type identifier
438
+ - pert_idx: index list of perturbed genes
439
+
440
+ [Output]
441
+ PyG Data object:
442
+ - x: (num_genes, 2) feature matrix
443
+ [:, 0]: Control group expression value
444
+ [:, 1]: perturbation flag (1=the gene is perturbed)
445
+ - y: target expression value
446
+ - de_idx: differentially expressed gene index
447
+ - pert: perturbation type
448
+ """
449
+ # Create perturbation feature vectors (mark which genes are perturbed)
450
+ pert_feats = np.zeros(len(X[0]))
451
+ if pert_idx is not None:
452
+ for p in pert_idx:
453
+ pert_feats[int(np.abs(p))] = 1
454
+ pert_feats = np.expand_dims(pert_feats, 0)
455
+ # Spliced expression values and perturbation markers
456
+ feature_mat = torch.Tensor(np.concatenate([X, pert_feats])).T
457
+
458
+ return Data(x=feature_mat, edge_index=None, edge_attr=None,
459
+ y=torch.Tensor(y), de_idx=de_idx, pert=pert)
460
+
461
+ def create_cell_graph_dataset(self, split_adata, pert_category,
462
+ num_samples=1):
463
+ """
464
+ Create a graph dataset of perturbation categories
465
+
466
+ [Processing logic]
467
+ 1. For non-control group disturbances:
468
+ - Get perturbed gene index
469
+ - Expression before perturbation = randomly sample control cells
470
+ - Target = expression of cells after perturbation
471
+ 2. For the control group:
472
+ - Expression before perturbation = target = expression in control cells
473
+
474
+ [num_samples parameter]
475
+ Each perturbed cell can be paired with multiple control group cells,
476
+ Increase the diversity of training data
477
+ """
478
+ num_de_genes = 20 # Only focus on the top 20 differentially expressed genes for evaluation
479
+ adata_ = split_adata[split_adata.obs['condition'] == pert_category]
480
+ de_genes = adata_.uns['rank_genes_groups_cov_all']
481
+ Xs = []
482
+ ys = []
483
+
484
+ # Handling non-control group disturbances
485
+ if pert_category != 'ctrl':
486
+ # Get the index of the perturbed gene
487
+ pert_idx = self.get_pert_idx(pert_category, adata_)
488
+
489
+ # Get differentially expressed gene index
490
+ pert_de_category = adata_.obs['condition_name'][0]
491
+ de_idx = np.where(adata_.var_names.isin(
492
+ np.array(de_genes[pert_de_category][:num_de_genes])))[0]
493
+
494
+ # Expand feature matrix to include perturbed genes
495
+ padded_X = sp.lil_matrix((adata_.X.shape[0], len(self.gene_names)), dtype=adata_.X.dtype)
496
+ padded_X[:, :len(self.real_gene_names)] = adata_.X
497
+
498
+ for cell_z in padded_X:
499
+ # Use control group samples as expression before perturbation
500
+ ctrl_samples = self.ctrl_adata[np.random.randint(0,
501
+ len(self.ctrl_adata), num_samples), :]
502
+ padded_ctrl_X = sp.lil_matrix((ctrl_samples.X.shape[0], len(self.gene_names)), dtype=ctrl_samples.X.dtype)
503
+ padded_ctrl_X[:, :len(self.real_gene_names)] = ctrl_samples.X
504
+ for c in padded_ctrl_X:
505
+ Xs.append(c) # control group expression
506
+ ys.append(cell_z) # expression after perturbation
507
+
508
+ # Process control group
509
+ else:
510
+ pert_idx = None
511
+ de_idx = [-1] * num_de_genes
512
+ padded_X = sp.lil_matrix((adata_.X.shape[0], len(self.gene_names)), dtype=adata_.X.dtype)
513
+ padded_X[:, :len(self.real_gene_names)] = adata_.X
514
+ for cell_z in padded_X:
515
+ Xs.append(cell_z)
516
+ ys.append(cell_z)
517
+
518
+ # Create a cell diagram data object
519
+ cell_graphs = []
520
+ for X, y in zip(Xs, ys):
521
+ cell_graphs.append(self.create_cell_graph(X.toarray(),
522
+ y.toarray(), de_idx, pert_category, pert_idx))
523
+
524
  return cell_graphs
BioMod/gene_perturbation/train_module.py CHANGED
@@ -1,325 +1,333 @@
1
- """
2
- ================================================================================
3
- Genetic perturbation task training script (train_module.py)
4
- ================================================================================
5
-
6
- [Task Overview]
7
- This script is used to train a model that predicts the effects of genetic perturbation. Gene perturbation refers to knockout or
8
- Overexpress a gene and observe its effect on the cell transcriptome. The goal of the model is: given a combination of genetic perturbations,
9
- Predict the gene expression profile of cells after perturbation.
10
-
11
- [Core process]
12
- 1. Load perturbation data sets (such as Norman, Adamson and other public data sets)
13
- 2. Initialize the pre-trained single-cell basic model (PerformerEncoderOnly_Module)
14
- 3. Fine-tune the model on perturbed data and learn to predict the effect of perturbation
15
- 4. Evaluate model performance (MSE, Pearson correlation coefficient and other indicators)
16
-
17
- [Key technical points]
18
- - Use DDP (DistributedDataParallel) for multi-GPU distributed training
19
- - Use module-based gene representation to group genes and then encode them
20
- - Supports multiple gene expression and gene2vec fusion methods
21
-
22
- [Adaptation instructions]
23
- This script is based on GEARS/train_dual_ddp_reduce_dec.py and adapted to the module-based model.
24
- Use a completely separate native module, avoiding any external gears_reduce imports.
25
- ================================================================================
26
- """
27
-
28
- import os
29
- # [Key] Disable Numba JIT compilation to avoid compilation conflicts in multi-process environments
30
- # Numba may cause problems during distributed training. Use pure Python to implement it after disabling it.
31
- os.environ['NUMBA_DISABLE_JIT'] = '1'
32
- import sys
33
- import time
34
- import argparse
35
- import pandas as pd
36
- import numpy as np
37
- import scanpy as sc
38
- from os.path import join as pjoin
39
-
40
- import torch
41
- from torch.utils.data.distributed import DistributedSampler
42
- from torch.nn.parallel import DistributedDataParallel as DDP
43
- import torch.distributed as dist
44
- import tempfile
45
-
46
- # ============================================================
47
- # [Key] Correctly set the order of sys.path
48
- # Local modules must take precedence and avoid importing into external utils.py
49
- # This is a common Python module import trap: if a module with the same name exists in the project root directory,
50
- # can be imported incorrectly, causing problems that are difficult to debug
51
- # ============================================================
52
- current_dir = os.path.dirname(os.path.abspath(__file__))
53
- project_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
54
-
55
- # First add the project root directory (for performer_pytorch_cont)
56
- if project_root not in sys.path:
57
- sys.path.append(project_root)
58
-
59
- # Finally add the current directory (make sure local modules take precedence)
60
- # Use append instead of insert(0), but since the current directory is empty there will be no conflict
61
- if current_dir not in sys.path:
62
- sys.path.insert(0, current_dir)
63
-
64
- # ============================================================
65
- # [Key] Create an independent Numba cache directory for each process
66
- # In distributed training, multiple processes sharing the same cache directory can cause file conflicts.
67
- # This is a common problem with Numba in a DDP environment. An independent cache must be allocated for each rank.
68
- # ============================================================
69
- if "RANK" in os.environ: # Distributed environment
70
- cache_dir = f"/tmp/numba_cache_{os.environ['RANK']}"
71
- else:
72
- cache_dir = tempfile.mkdtemp(prefix="numba_cache_")
73
- os.environ["NUMBA_CACHE_DIR"] = cache_dir
74
-
75
- # ============================================================
76
- # [Module Import] Use completely independent local modules
77
- # These local modules are to avoid conflicts with modules of the same name in other parts of the project
78
- # ============================================================
79
- from gears_reduce_local import GEARS
80
- from utils import print_sys
81
- from pertdata_local import PertData # Local version, does not rely on external gears_reduce
82
-
83
- # Import module model
84
- from model_module import GEARS_Model_Module
85
-
86
- def main(parser):
87
- """
88
- main training function
89
-
90
- [Training process]
91
- 1. Initialize the distributed training environment (DDP)
92
- 2. Loading and preprocessing perturbation data
93
- 3. Prepare data partitioning (training/validation/testing)
94
- 4. Initialize model and optimizer
95
- 5. Training loop + validation + testing
96
- """
97
- # Turn on anomaly detection to help debug gradient calculation problems
98
- torch.autograd.set_detect_anomaly(True)
99
-
100
- args = parser.parse_args()
101
-
102
- # ============================================================
103
- # [Distributed training initialization]
104
- # DDP uses NCCL backend for inter-GPU communication
105
- # local_rank: the GPU number of the current process on this machine
106
- # RANK: global process number (different machines have different ranks during multi-machine training)
107
- # ============================================================
108
- local_rank = args.local_rank
109
- rank = int(os.environ["RANK"])
110
- is_master = (rank == 0) # Rank 0 is the main process, responsible for printing logs and saving models
111
-
112
- dist.init_process_group(backend='nccl', init_method='env://')
113
- print('DDP init!')
114
- local_rank = int(os.environ['LOCAL_RANK'])
115
- device = f'cuda:{local_rank}'
116
- torch.cuda.set_device(device)
117
- world_size = torch.distributed.get_world_size() # Total number of processes
118
-
119
- print("local_rank: ", local_rank)
120
-
121
- valid_every = args.valid_every # Verify every epoch
122
-
123
- # ============================================================
124
- # [Data loading]
125
- # The PertData class is responsible for loading and processing perturbation data
126
- # Supported data sets: norman, adamson, dixit and other public data sets
127
- # ============================================================
128
- pert_data = PertData(args.data_dir)
129
- # load dataset in paper: norman, adamson, dixit.
130
- try:
131
- if args.data_name in ['norman', 'adamson', 'dixit']:
132
- pert_data.load(data_name=args.data_name)
133
- else:
134
- print('load data')
135
- pert_data.load(data_path=pjoin(args.data_dir, args.data_name))
136
- except:
137
- # If the data format is not standard, try loading from h5ad file
138
- adata = sc.read_h5ad(pjoin(args.data_dir, args.data_name+'.h5ad'))
139
- adata.uns['log1p'] = {}
140
- adata.uns['log1p']['base'] = None
141
- pert_data.new_data_process(dataset_name=args.data_name, adata=adata)
142
-
143
- # ============================================================
144
- # [Gene Mapping]
145
- # Check the overlap between the genes of the input data and the genes of the pre-trained model
146
- # An overlap ratio that is too low may result in poor model performance.
147
- # ============================================================
148
- input_genes_ens_ids = pert_data.adata.var.index.tolist()
149
- with open(args.scfm_genes_list_path, 'r') as f:
150
- scfm_genes_ens_ids = [line.rstrip('\n') for line in f.readlines()]
151
- if is_master:
152
- print("target and scfm genes intersect: ", len(np.intersect1d(input_genes_ens_ids, scfm_genes_ens_ids)))
153
-
154
- # ============================================================
155
- # [Data division]
156
- # The split parameter determines the data partitioning strategy:
157
- # - 'simulation': simulates the division of real application scenarios, and the test set contains unseen gene combinations
158
- # - 'combo_seen0/1/2': divided according to the number of genes seen in the combination
159
- # train_gene_set_size: training set gene proportion
160
- # ============================================================
161
- pert_data.prepare_split(split=args.split, seed=args.seed, train_gene_set_size=args.train_gene_set_size)
162
- # get dataloader with batch size
163
- pert_data.get_dataloader(batch_size=args.batch_size, test_batch_size=args.test_batch_size)
164
-
165
- # set up and train a model
166
- if is_master:
167
- print_sys("EXPERIMENT: " + args.run_name)
168
-
169
- # ============================================================
170
- # [Model initialization]
171
- # The GEARS class encapsulates the complete process of training, verification, and testing
172
- # ============================================================
173
- gears_model = GEARS(pert_data,
174
- local_rank=local_rank,
175
- is_master=is_master,
176
- world_size=world_size,
177
- weight_bias_track=args.wandb,
178
- train_bs=args.batch_size,
179
- test_bs=args.test_batch_size,
180
- device=device,
181
- exp_name=args.run_name)
182
-
183
- # ============================================================
184
- # [Model parameter configuration]
185
- # Use GEARS_Model_Module instead of GEARS_Model_Acc
186
- # Key parameter description:
187
- # - hidden_size: hidden layer dimension, affecting model capacity
188
- # - finetune_method: fine-tuning method ('frozen' freezes pre-training parameters, None means fine-tuning all parameters)
189
- # - accumulation_steps: the number of gradient accumulation steps, used to simulate larger batch size
190
- # ============================================================
191
- gears_model.model_initialize(
192
- hidden_size=args.hidden_size,
193
- model_type=args.model_type,
194
- bin_set=args.bin_set,
195
- load_path=args.singlecell_model_path,
196
- finetune_method=args.finetune_method,
197
- accumulation_steps=args.accumulation_steps,
198
- mode=args.mode,
199
- input_genes_ens_ids=input_genes_ens_ids,
200
- scfm_genes_ens_ids=scfm_genes_ens_ids,
201
- scfm_hyper_params_path=args.scfm_hyper_params_path,
202
- scfm_ckpt_path=args.scfm_ckpt_path,
203
- scfm_class=None, # Do not use scfm_class, use module model directly
204
- key_enc=None,
205
- scfm_gene2vec_file=args.scfm_gene2vec_file,
206
- record_pred=args.record_pred,
207
- model_class=GEARS_Model_Module, # Use module model class
208
- # ============================================================
209
- # [Module related parameters] The following parameters affect how the model processes module-based data
210
- # module_data_path/module_data_dir: module divides the data path
211
- # gene_mapping_path: mapping file from gene symbol to Ensembl ID
212
- # use_module_level: whether to use module level features directly (default False, reverts to gene level)
213
- # fusion_method: Read from pkl to ensure consistency with pre-training
214
- # shuffle_module_genes: [Ablation experiment] Disrupt the gene distribution within the module
215
- # ============================================================
216
- module_data_path=args.module_data_path if hasattr(args, 'module_data_path') else None,
217
- module_data_dir=args.module_data_dir if hasattr(args, 'module_data_dir') else None,
218
- gene_mapping_path=args.gene_mapping_path if hasattr(args, 'gene_mapping_path') else '../gene2ens.csv',
219
- scfm_genes_list_path=args.scfm_genes_list_path,
220
- use_module_level=args.use_module_level if hasattr(args, 'use_module_level') else False,
221
- shuffle_module_genes=args.shuffle_module_genes if hasattr(args, 'shuffle_module_genes') else False,
222
- shuffle_seed=args.shuffle_seed if hasattr(args, 'shuffle_seed') else 2021,
223
- )
224
-
225
- gears_model.train(
226
- epochs=args.epochs,
227
- result_dir=args.result_dir,
228
- lr=args.lr,
229
- valid_every=valid_every
230
- )
231
-
232
- # save params
233
- if is_master:
234
- param_pd = pd.DataFrame(vars(args), index=['params']).T
235
- param_pd.to_csv(f'{args.result_dir}/params.csv')
236
-
237
- if __name__ == '__main__':
238
- parser = argparse.ArgumentParser(description='GEARS with PerformerEncoderOnly_Module')
239
-
240
- # ============================================================
241
- # [Distributed training parameters]
242
- # local-rank: automatically set by torch.distributed.launch, no need to specify manually
243
- # ============================================================
244
- parser.add_argument("--local-rank", type=int, default=-1, help='Local process rank.')
245
-
246
- # ============================================================
247
- # [Data related parameters]
248
- # data_dir: data storage directory
249
- # data_name: Data set name (norman/adamson/dixit is a public data set)
250
- # split: data partitioning strategy, simulation is the most commonly used
251
- # train_gene_set_size: The proportion of genes in the training set, which affects the evaluation of model generalization ability
252
- # ============================================================
253
- parser.add_argument('--data_dir', type=str, default='../../GEARS/data')
254
- parser.add_argument('--data_name', type=str, default='norman')
255
- parser.add_argument('--split', type=str, default='simulation')
256
- parser.add_argument('--result_dir', type=str, default='./results')
257
- parser.add_argument('--seed', type=int, default=1)
258
-
259
- # ============================================================
260
- # [Training hyperparameters]
261
- # epochs: number of training rounds, usually 15-20 rounds is enough
262
- # batch_size: Batch size, adjusted according to GPU memory
263
- # lr: learning rate, usually a smaller learning rate (1e-3 or less) is used during fine-tuning
264
- # accumulation_steps: Number of gradient accumulation steps, used to simulate larger batch size
265
- # ============================================================
266
- parser.add_argument('--epochs', type=int, default=20)
267
- parser.add_argument('--batch_size', type=int, default=32)
268
- parser.add_argument('--valid_every', type=int, default=1)
269
- parser.add_argument('--test_batch_size', type=int, default=128)
270
- parser.add_argument('--train_gene_set_size', type=float, default=0.75)
271
- parser.add_argument('--hidden_size', type=int, default=64)
272
- parser.add_argument('--device', type=str, default='cuda')
273
- parser.add_argument('--model_type', type=str, default=None)
274
- parser.add_argument('--bin_set', type=str, default=None)
275
- parser.add_argument('--singlecell_model_path', type=str, default=None)
276
- parser.add_argument('--finetune_method', type=str, default=None)
277
- parser.add_argument('--mode', type=str, default='v1')
278
- parser.add_argument('--accumulation_steps', type=int, default=1)
279
- parser.add_argument('-wandb', action='store_true', help='Enable Weights & Biases logging')
280
- parser.add_argument('-record_pred', action='store_true', help='Record predictions for analysis')
281
-
282
- # ============================================================
283
- # [Related parameters of pre-training model]
284
- # scfm_genes_list_path: list of genes used by the pre-trained model
285
- # scfm_hyper_params_path: Hyperparameter configuration file of the pre-trained model
286
- # scfm_ckpt_path: checkpoint file of the pre-trained model
287
- # scfm_gene2vec_file: gene2vec pre-trained embedding file
288
- # ============================================================
289
- parser.add_argument('--scfm_genes_list_path', type=str,
290
- default="../../selected_genes_27k.txt")
291
- parser.add_argument('--scfm_hyper_params_path', type=str,
292
- default="../ckpts_performer_module_test/performer_module_test.pkl")
293
- parser.add_argument('--scfm_ckpt_path', type=str,
294
- default="../ckpts_performer_module_test/performer_module_test_epoch6.pth")
295
- parser.add_argument('--run_name', type=str,
296
- default="default_run_module")
297
- parser.add_argument('--scfm_gene2vec_file', type=str,
298
- default='../../selected_gene2vec_27k.npy')
299
-
300
- # ============================================================
301
- # [Module related parameters] These parameters affect how the model handles module-based gene representation.
302
- # module_data_path: single module JSON file path
303
- # module_data_dir: module JSON file directory
304
- # gene_mapping_path: mapping file from gene symbol to Ensembl ID
305
- # use_module_level: whether to use module level features directly (experimental feature)
306
- # fusion_method: Read from pkl to ensure consistency with pre-training
307
- # shuffle_module_genes: [Ablation experiment] Disrupt the genes in the module to verify the importance of the module structure
308
- # ============================================================
309
- parser.add_argument('--module_data_path', type=str, default=None,
310
- help='Path to module JSON file (single file)')
311
- parser.add_argument('--module_data_dir', type=str, default='../module_data',
312
- help='Path to directory containing module JSON files')
313
- parser.add_argument('--gene_mapping_path', type=str, default='../../gene2ens.csv',
314
- help='Path to gene symbol to Ensembl ID mapping CSV')
315
- parser.add_argument('--use_module_level', action='store_true',
316
- help='Use module-level features directly instead of recovering to gene level. '
317
- 'Note: This requires compatible downstream architecture (not fully implemented).')
318
- parser.add_argument('--shuffle_module_genes', action='store_true',
319
- help='[Ablation experiment] Additionally shuffle module_sets after restoring them from the pretrained PKL. '
320
- 'Do not pass this parameter when reproducing the experiment normally.')
321
- parser.add_argument('--shuffle_seed', type=int, default=2021,
322
- help='Only takes effect when --shuffle_module_genes is turned on and is used for additional downstream shuffling.')
323
- parser.add_argument('--lr', type=float, default=1e-3)
324
-
325
- main(parser)
 
 
 
 
 
 
 
 
 
1
+ """
2
+ ================================================================================
3
+ Genetic perturbation task training script (train_module.py)
4
+ ================================================================================
5
+
6
+ [Task Overview]
7
+ This script is used to train a model that predicts the effects of genetic perturbation. Gene perturbation refers to knockout or
8
+ Overexpress a gene and observe its effect on the cell transcriptome. The goal of the model is: given a combination of genetic perturbations,
9
+ Predict the gene expression profile of cells after perturbation.
10
+
11
+ [Core process]
12
+ 1. Load perturbation data sets (such as Norman, Adamson and other public data sets)
13
+ 2. Initialize the pre-trained single-cell basic model (PerformerEncoderOnly_Module)
14
+ 3. Fine-tune the model on perturbed data and learn to predict the effect of perturbation
15
+ 4. Evaluate model performance (MSE, Pearson correlation coefficient and other indicators)
16
+
17
+ [Key technical points]
18
+ - Use DDP (DistributedDataParallel) for multi-GPU distributed training
19
+ - Use module-based gene representation to group genes and then encode them
20
+ - Supports multiple gene expression and gene2vec fusion methods
21
+
22
+ [Adaptation instructions]
23
+ This script is based on GEARS/train_dual_ddp_reduce_dec.py and adapted to the module-based model.
24
+ Use a completely separate native module, avoiding any external gears_reduce imports.
25
+ ================================================================================
26
+ """
27
+
28
+ import os
29
+ # [Key] Disable Numba JIT compilation to avoid compilation conflicts in multi-process environments
30
+ # Numba may cause problems during distributed training. Use pure Python to implement it after disabling it.
31
+ os.environ['NUMBA_DISABLE_JIT'] = '1'
32
+ import sys
33
+ import time
34
+ import argparse
35
+ import pandas as pd
36
+ import numpy as np
37
+ import scanpy as sc
38
+ from os.path import join as pjoin
39
+
40
+ import torch
41
+ from torch.utils.data.distributed import DistributedSampler
42
+ from torch.nn.parallel import DistributedDataParallel as DDP
43
+ import torch.distributed as dist
44
+ import tempfile
45
+ from pathlib import Path
46
+
47
+ REPO_ROOT = Path(__file__).resolve().parents[2]
48
+ ASSET_DIR = REPO_ROOT / "model_dependencies"
49
+ DOWNSTREAM_DATA_DIR = REPO_ROOT / "data" / "downstream" / "final_data"
50
+ WEIGHTS_DIR = REPO_ROOT / "weight" / "0810model"
51
+
52
+ # ============================================================
53
+ # [Key] Correctly set the order of sys.path
54
+ # Local modules must take precedence and avoid importing into external utils.py
55
+ # This is a common Python module import trap: if a module with the same name exists in the project root directory,
56
+ # can be imported incorrectly, causing problems that are difficult to debug
57
+ # ============================================================
58
+ current_dir = os.path.dirname(os.path.abspath(__file__))
59
+ project_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
60
+
61
+ # First add the project root directory (for performer_pytorch_cont)
62
+ if project_root not in sys.path:
63
+ sys.path.append(project_root)
64
+
65
+ # Finally add the current directory (make sure local modules take precedence)
66
+ # Use append instead of insert(0), but since the current directory is empty there will be no conflict
67
+ if current_dir not in sys.path:
68
+ sys.path.insert(0, current_dir)
69
+
70
+ # ============================================================
71
+ # [Key] Create an independent Numba cache directory for each process
72
+ # In distributed training, multiple processes sharing the same cache directory can cause file conflicts.
73
+ # This is a common problem with Numba in a DDP environment. An independent cache must be allocated for each rank.
74
+ # ============================================================
75
+ if "RANK" in os.environ: # Distributed environment
76
+ cache_dir = tempfile.mkdtemp(prefix=f"biomod_numba_rank{os.environ['RANK']}_")
77
+ else:
78
+ cache_dir = tempfile.mkdtemp(prefix="numba_cache_")
79
+ os.environ["NUMBA_CACHE_DIR"] = cache_dir
80
+
81
+ # ============================================================
82
+ # [Module Import] Use completely independent local modules
83
+ # These local modules are to avoid conflicts with modules of the same name in other parts of the project
84
+ # ============================================================
85
+ from gears_reduce_local import GEARS
86
+ from utils import print_sys
87
+ from pertdata_local import PertData # Local version, does not rely on external gears_reduce
88
+
89
+ # Import module model
90
+ from model_module import GEARS_Model_Module
91
+
92
+ def main(parser):
93
+ """
94
+ main training function
95
+
96
+ [Training process]
97
+ 1. Initialize the distributed training environment (DDP)
98
+ 2. Loading and preprocessing perturbation data
99
+ 3. Prepare data partitioning (training/validation/testing)
100
+ 4. Initialize model and optimizer
101
+ 5. Training loop + validation + testing
102
+ """
103
+ # Turn on anomaly detection to help debug gradient calculation problems
104
+ torch.autograd.set_detect_anomaly(True)
105
+
106
+ args = parser.parse_args()
107
+ if args.module_data_path is None and args.module_data_dir is None:
108
+ args.module_data_dir = str(ASSET_DIR / 'module_data_all')
109
+
110
+ # ============================================================
111
+ # [Distributed training initialization]
112
+ # DDP uses NCCL backend for inter-GPU communication
113
+ # local_rank: the GPU number of the current process on this machine
114
+ # RANK: global process number (different machines have different ranks during multi-machine training)
115
+ # ============================================================
116
+ local_rank = args.local_rank
117
+ rank = int(os.environ["RANK"])
118
+ is_master = (rank == 0) # Rank 0 is the main process, responsible for printing logs and saving models
119
+
120
+ dist.init_process_group(backend='nccl', init_method='env://')
121
+ print('DDP init!')
122
+ local_rank = int(os.environ['LOCAL_RANK'])
123
+ device = f'cuda:{local_rank}'
124
+ torch.cuda.set_device(device)
125
+ world_size = torch.distributed.get_world_size() # Total number of processes
126
+
127
+ print("local_rank: ", local_rank)
128
+
129
+ valid_every = args.valid_every # Verify every epoch
130
+
131
+ # ============================================================
132
+ # [Data loading]
133
+ # The PertData class is responsible for loading and processing perturbation data
134
+ # Supported data sets: norman, adamson, dixit and other public data sets
135
+ # ============================================================
136
+ pert_data = PertData(args.data_dir)
137
+ # load dataset in paper: norman, adamson, dixit.
138
+ try:
139
+ if args.data_name in ['norman', 'adamson', 'dixit']:
140
+ pert_data.load(data_name=args.data_name)
141
+ else:
142
+ print('load data')
143
+ pert_data.load(data_path=pjoin(args.data_dir, args.data_name))
144
+ except:
145
+ # If the data format is not standard, try loading from h5ad file
146
+ adata = sc.read_h5ad(pjoin(args.data_dir, args.data_name+'.h5ad'))
147
+ adata.uns['log1p'] = {}
148
+ adata.uns['log1p']['base'] = None
149
+ pert_data.new_data_process(dataset_name=args.data_name, adata=adata)
150
+
151
+ # ============================================================
152
+ # [Gene Mapping]
153
+ # Check the overlap between the genes of the input data and the genes of the pre-trained model
154
+ # An overlap ratio that is too low may result in poor model performance.
155
+ # ============================================================
156
+ input_genes_ens_ids = pert_data.adata.var.index.tolist()
157
+ with open(args.scfm_genes_list_path, 'r') as f:
158
+ scfm_genes_ens_ids = [line.rstrip('\n') for line in f.readlines()]
159
+ if is_master:
160
+ print("target and scfm genes intersect: ", len(np.intersect1d(input_genes_ens_ids, scfm_genes_ens_ids)))
161
+
162
+ # ============================================================
163
+ # [Data division]
164
+ # The split parameter determines the data partitioning strategy:
165
+ # - 'simulation': simulates the division of real application scenarios, and the test set contains unseen gene combinations
166
+ # - 'combo_seen0/1/2': divided according to the number of genes seen in the combination
167
+ # train_gene_set_size: training set gene proportion
168
+ # ============================================================
169
+ pert_data.prepare_split(split=args.split, seed=args.seed, train_gene_set_size=args.train_gene_set_size)
170
+ # get dataloader with batch size
171
+ pert_data.get_dataloader(batch_size=args.batch_size, test_batch_size=args.test_batch_size)
172
+
173
+ # set up and train a model
174
+ if is_master:
175
+ print_sys("EXPERIMENT: " + args.run_name)
176
+
177
+ # ============================================================
178
+ # [Model initialization]
179
+ # The GEARS class encapsulates the complete process of training, verification, and testing
180
+ # ============================================================
181
+ gears_model = GEARS(pert_data,
182
+ local_rank=local_rank,
183
+ is_master=is_master,
184
+ world_size=world_size,
185
+ weight_bias_track=args.wandb,
186
+ train_bs=args.batch_size,
187
+ test_bs=args.test_batch_size,
188
+ device=device,
189
+ exp_name=args.run_name)
190
+
191
+ # ============================================================
192
+ # [Model parameter configuration]
193
+ # Use GEARS_Model_Module instead of GEARS_Model_Acc
194
+ # Key parameter description:
195
+ # - hidden_size: hidden layer dimension, affecting model capacity
196
+ # - finetune_method: fine-tuning method ('frozen' freezes pre-training parameters, None means fine-tuning all parameters)
197
+ # - accumulation_steps: the number of gradient accumulation steps, used to simulate larger batch size
198
+ # ============================================================
199
+ gears_model.model_initialize(
200
+ hidden_size=args.hidden_size,
201
+ model_type=args.model_type,
202
+ bin_set=args.bin_set,
203
+ load_path=args.singlecell_model_path,
204
+ finetune_method=args.finetune_method,
205
+ accumulation_steps=args.accumulation_steps,
206
+ mode=args.mode,
207
+ input_genes_ens_ids=input_genes_ens_ids,
208
+ scfm_genes_ens_ids=scfm_genes_ens_ids,
209
+ scfm_hyper_params_path=args.scfm_hyper_params_path,
210
+ scfm_ckpt_path=args.scfm_ckpt_path,
211
+ scfm_class=None, # Do not use scfm_class, use module model directly
212
+ key_enc=None,
213
+ scfm_gene2vec_file=args.scfm_gene2vec_file,
214
+ record_pred=args.record_pred,
215
+ model_class=GEARS_Model_Module, # Use module model class
216
+ # ============================================================
217
+ # [Module related parameters] The following parameters affect how the model processes module-based data
218
+ # module_data_path/module_data_dir: module divides the data path
219
+ # gene_mapping_path: mapping file from gene symbol to Ensembl ID
220
+ # use_module_level: whether to use module level features directly (default False, reverts to gene level)
221
+ # fusion_method: Read from pkl to ensure consistency with pre-training
222
+ # shuffle_module_genes: [Ablation experiment] Disrupt the gene distribution within the module
223
+ # ============================================================
224
+ module_data_path=args.module_data_path if hasattr(args, 'module_data_path') else None,
225
+ module_data_dir=args.module_data_dir if hasattr(args, 'module_data_dir') else None,
226
+ gene_mapping_path=args.gene_mapping_path if hasattr(args, 'gene_mapping_path') else str(ASSET_DIR / 'gene2ens.csv'),
227
+ scfm_genes_list_path=args.scfm_genes_list_path,
228
+ use_module_level=args.use_module_level if hasattr(args, 'use_module_level') else False,
229
+ shuffle_module_genes=args.shuffle_module_genes if hasattr(args, 'shuffle_module_genes') else False,
230
+ shuffle_seed=args.shuffle_seed if hasattr(args, 'shuffle_seed') else 2021,
231
+ )
232
+
233
+ gears_model.train(
234
+ epochs=args.epochs,
235
+ result_dir=args.result_dir,
236
+ lr=args.lr,
237
+ valid_every=valid_every
238
+ )
239
+
240
+ # save params
241
+ if is_master:
242
+ param_pd = pd.DataFrame(vars(args), index=['params']).T
243
+ param_pd.to_csv(f'{args.result_dir}/params.csv')
244
+
245
+ if __name__ == '__main__':
246
+ parser = argparse.ArgumentParser(description='GEARS with PerformerEncoderOnly_Module')
247
+
248
+ # ============================================================
249
+ # [Distributed training parameters]
250
+ # local-rank: automatically set by torch.distributed.launch, no need to specify manually
251
+ # ============================================================
252
+ parser.add_argument("--local-rank", type=int, default=-1, help='Local process rank.')
253
+
254
+ # ============================================================
255
+ # [Data related parameters]
256
+ # data_dir: data storage directory
257
+ # data_name: Data set name (norman/adamson/dixit is a public data set)
258
+ # split: data partitioning strategy, simulation is the most commonly used
259
+ # train_gene_set_size: The proportion of genes in the training set, which affects the evaluation of model generalization ability
260
+ # ============================================================
261
+ parser.add_argument('--data_dir', type=str, default=str(DOWNSTREAM_DATA_DIR))
262
+ parser.add_argument('--data_name', type=str, default='norman')
263
+ parser.add_argument('--split', type=str, default='simulation')
264
+ parser.add_argument('--result_dir', type=str, default='./results')
265
+ parser.add_argument('--seed', type=int, default=1)
266
+
267
+ # ============================================================
268
+ # [Training hyperparameters]
269
+ # epochs: number of training rounds, usually 15-20 rounds is enough
270
+ # batch_size: Batch size, adjusted according to GPU memory
271
+ # lr: learning rate, usually a smaller learning rate (1e-3 or less) is used during fine-tuning
272
+ # accumulation_steps: Number of gradient accumulation steps, used to simulate larger batch size
273
+ # ============================================================
274
+ parser.add_argument('--epochs', type=int, default=20)
275
+ parser.add_argument('--batch_size', type=int, default=32)
276
+ parser.add_argument('--valid_every', type=int, default=1)
277
+ parser.add_argument('--test_batch_size', type=int, default=128)
278
+ parser.add_argument('--train_gene_set_size', type=float, default=0.75)
279
+ parser.add_argument('--hidden_size', type=int, default=64)
280
+ parser.add_argument('--device', type=str, default='cuda')
281
+ parser.add_argument('--model_type', type=str, default=None)
282
+ parser.add_argument('--bin_set', type=str, default=None)
283
+ parser.add_argument('--singlecell_model_path', type=str, default=None)
284
+ parser.add_argument('--finetune_method', type=str, default=None)
285
+ parser.add_argument('--mode', type=str, default='v1')
286
+ parser.add_argument('--accumulation_steps', type=int, default=1)
287
+ parser.add_argument('-wandb', action='store_true', help='Enable Weights & Biases logging')
288
+ parser.add_argument('-record_pred', action='store_true', help='Record predictions for analysis')
289
+
290
+ # ============================================================
291
+ # [Related parameters of pre-training model]
292
+ # scfm_genes_list_path: list of genes used by the pre-trained model
293
+ # scfm_hyper_params_path: Hyperparameter configuration file of the pre-trained model
294
+ # scfm_ckpt_path: checkpoint file of the pre-trained model
295
+ # scfm_gene2vec_file: gene2vec pre-trained embedding file
296
+ # ============================================================
297
+ parser.add_argument('--scfm_genes_list_path', type=str,
298
+ default=str(ASSET_DIR / "selected_genes_27k.txt"))
299
+ parser.add_argument('--scfm_hyper_params_path', type=str,
300
+ default=str(WEIGHTS_DIR / "mae_tost_70M_75mask_ec2.pkl"))
301
+ parser.add_argument('--scfm_ckpt_path', type=str,
302
+ default=str(WEIGHTS_DIR / "mae_tost_70M_75mask_ec2_epoch0.pth"))
303
+ parser.add_argument('--run_name', type=str,
304
+ default="default_run_module")
305
+ parser.add_argument('--scfm_gene2vec_file', type=str,
306
+ default=str(ASSET_DIR / 'selected_gene2vec_27k.npy'))
307
+
308
+ # ============================================================
309
+ # [Module related parameters] These parameters affect how the model handles module-based gene representation.
310
+ # module_data_path: single module JSON file path
311
+ # module_data_dir: module JSON file directory
312
+ # gene_mapping_path: mapping file from gene symbol to Ensembl ID
313
+ # use_module_level: whether to use module level features directly (experimental feature)
314
+ # fusion_method: Read from pkl to ensure consistency with pre-training
315
+ # shuffle_module_genes: [Ablation experiment] Disrupt the genes in the module to verify the importance of the module structure
316
+ # ============================================================
317
+ parser.add_argument('--module_data_path', type=str, default=None,
318
+ help='Path to module JSON file (single file)')
319
+ parser.add_argument('--module_data_dir', type=str, default=None,
320
+ help='Module JSON directory; defaults to bundled module_data_all only when no module file or directory is supplied')
321
+ parser.add_argument('--gene_mapping_path', type=str, default=str(ASSET_DIR / 'gene2ens.csv'),
322
+ help='Path to gene symbol to Ensembl ID mapping CSV')
323
+ parser.add_argument('--use_module_level', action='store_true',
324
+ help='Use module-level features directly instead of recovering to gene level. '
325
+ 'Note: This requires compatible downstream architecture (not fully implemented).')
326
+ parser.add_argument('--shuffle_module_genes', action='store_true',
327
+ help='[Ablation experiment] Additionally shuffle module_sets after restoring them from the pretrained PKL. '
328
+ 'Do not pass this parameter when reproducing the experiment normally.')
329
+ parser.add_argument('--shuffle_seed', type=int, default=2021,
330
+ help='Only takes effect when --shuffle_module_genes is turned on and is used for additional downstream shuffling.')
331
+ parser.add_argument('--lr', type=float, default=1e-3)
332
+
333
+ main(parser)
BioMod/gene_perturbation/utils.py CHANGED
@@ -1,489 +1,496 @@
1
- """
2
- ================================================================================
3
- Utility function module (utils.py)
4
- ================================================================================
5
-
6
- [Module function]
7
- Provides various tool functions required for genetic perturbation tasks, including:
8
-
9
- 1. Perturbation analysis function: analyze the gene name of single gene/combination perturbation
10
- 2. Gene relationship graph construction: co-expression graph and GO similarity graph
11
- 3. Loss function: training loss and uncertainty loss
12
- 4. Genetic interaction (GI) analysis: Calculate genetic interaction coefficients
13
-
14
- [Key concepts]
15
- - Perturbation format: "gene1+gene2", such as "ctrl+MAPK1" or "MAPK1+KRAS"
16
- - Co-expression map: constructed based on gene expression correlations in the data
17
- - GO graph: Jaccard similarity construction based on Gene Ontology annotation
18
-
19
- [Key parameters that may affect the results]
20
- - coexpress_threshold: Threshold for co-expression graph construction
21
- - dict_filter: training loss is only calculated on non-zero genes
22
- - direction_lambda: direction consistency loss weight
23
- ================================================================================
24
- """
25
-
26
- import torch
27
- import numpy as np
28
- import pandas as pd
29
- from sklearn.linear_model import TheilSenRegressor
30
- import torch.nn as nn
31
-
32
- import networkx as nx
33
- from tqdm import tqdm
34
- import pickle
35
- import sys, os
36
- import requests
37
- from torch_geometric.data import Data
38
-
39
- from zipfile import ZipFile
40
-
41
- import statsmodels.api as sm
42
- from sklearn.linear_model import LinearRegression, TheilSenRegressor
43
- # ============================================================
44
- # Lazy loading of dcor to avoid numba compatibility issues in DDP multi-process environments
45
- # dcor will be imported when needed (in the get_coeffs function)
46
- # ============================================================
47
- from sklearn.metrics import r2_score
48
-
49
- def parse_single_pert(i):
50
- """
51
- Parsing single gene perturbation strings
52
-
53
- [Format]
54
- - "ctrl+GENE": The perturbed gene is GENE
55
- - "GENE+ctrl": The perturbed gene is GENE
56
-
57
- [return]
58
- Perturbed gene name
59
- """
60
- a = i.split('+')[0]
61
- b = i.split('+')[1]
62
- if a == 'ctrl':
63
- pert = b
64
- else:
65
- pert = a
66
- return pert
67
-
68
- def filter_pert_cond_in_go(condition, exclude_pert):
69
- """
70
- Filter perturbations not in the GO graph
71
-
72
- [use]
73
- Genes without annotations in the GO diagram cannot predict their perturbation effects.
74
- These disturbing conditions need to be eliminated during data loading.
75
- """
76
- if condition == 'ctrl':
77
- return True
78
- else:
79
- if condition in exclude_pert:
80
- return False
81
- else:
82
- return True
83
-
84
- def parse_combo_pert(i):
85
- """
86
- Parsing combined perturbation strings
87
-
88
- [Format]
89
- - "GENE1+GENE2": perturb two genes simultaneously
90
-
91
- [return]
92
- (gene1, gene2) tuple
93
- """
94
- return i.split('+')[0], i.split('+')[1]
95
-
96
- def combine_res(res_1, res_2):
97
- """Merge two evaluation result dictionaries"""
98
- res_out = {}
99
- for key in res_1:
100
- res_out[key] = np.concatenate([res_1[key], res_2[key]])
101
- return res_out
102
-
103
- def parse_any_pert(p):
104
- """
105
- Parse arbitrary perturbation strings (single genes or combinations)
106
-
107
- [return]
108
- Gene list:
109
- - Single gene perturbation: [gene]
110
- - Combinatorial perturbation: [gene1, gene2]
111
- """
112
- if ('ctrl' in p) and (p != 'ctrl'):
113
- return [parse_single_pert(p)]
114
- elif 'ctrl' not in p:
115
- out = parse_combo_pert(p)
116
- return [out[0], out[1]]
117
-
118
- def np_pearson_cor(x, y):
119
- """
120
- Calculate Pearson correlation coefficient matrix
121
-
122
- [use]
123
- Used to construct co-expressed gene relationship diagrams
124
- """
125
- xv = x - x.mean(axis=0)
126
- yv = y - y.mean(axis=0)
127
- xvss = (xv * xv).sum(axis=0)
128
- yvss = (yv * yv).sum(axis=0)
129
- result = np.matmul(xv.transpose(), yv) / np.sqrt(np.outer(xvss, yvss))
130
- return np.maximum(np.minimum(result, 1.0), -1.0)
131
-
132
-
133
- def dataverse_download(url, save_path):
134
- """dataverse download helper with progress bar
135
-
136
- Args:
137
- url (str): the url of the dataset
138
- path (str): the path to save the dataset
139
- """
140
-
141
- if os.path.exists(save_path):
142
- print_sys('Found local copy...')
143
- else:
144
- print_sys("Downloading...")
145
- response = requests.get(url, stream=True)
146
- total_size_in_bytes= int(response.headers.get('content-length', 0))
147
- block_size = 1024
148
- progress_bar = tqdm(total=total_size_in_bytes, unit='iB', unit_scale=True)
149
- with open(save_path, 'wb') as file:
150
- for data in response.iter_content(block_size):
151
- progress_bar.update(len(data))
152
- file.write(data)
153
- progress_bar.close()
154
-
155
-
156
- def zip_data_download_wrapper(url, save_path, data_path):
157
-
158
- if os.path.exists(save_path):
159
- print_sys('Found local copy...')
160
- else:
161
- dataverse_download(url, save_path + '.zip')
162
- print_sys('Extracting zip file...')
163
- with ZipFile((save_path + '.zip'), 'r') as zip:
164
- zip.extractall(path = data_path)
165
- print_sys("Done!")
166
-
167
- def get_go_auto(gene_list, data_path, data_name):
168
- go_path = os.path.join(data_path, data_name, 'go.csv')
169
-
170
- if os.path.exists(go_path):
171
- return pd.read_csv(go_path)
172
- else:
173
- with open(os.path.join('/home/ding.bai/pert_new/GEARS/data_all/gene2go_all.pkl'), 'rb') as f:
174
- gene2go = pickle.load(f)
175
-
176
- gene2go = {i: list(gene2go[i]) for i in gene_list if i in gene2go}
177
- edge_list = []
178
- for g1 in tqdm(gene2go.keys()):
179
- for g2 in gene2go.keys():
180
- edge_list.append((g1, g2, len(np.intersect1d(gene2go[g1], gene2go[g2]))/len(np.union1d(gene2go[g1], gene2go[g2]))))
181
-
182
- edge_list_filter = [i for i in edge_list if i[2] > 0]
183
- further_filter = [i for i in edge_list if i[2] > 0.1]
184
- df_edge_list = pd.DataFrame(further_filter).rename(columns = {0: 'gene1', 1: 'gene2', 2: 'score'})
185
-
186
- df_edge_list = df_edge_list.rename(columns = {'gene1': 'source', 'gene2': 'target', 'score': 'importance'})
187
-
188
- df_edge_list.to_csv(go_path, index = False)
189
- return df_edge_list
190
-
191
- def get_go(df_gene2go):
192
- df_gene2go['Entry name'] = df_gene2go['Entry name'].apply(lambda x: x.split('_')[0])
193
- df_gene2go = df_gene2go[df_gene2go['Gene ontology IDs'].notnull()]
194
- df_gene2go = df_gene2go.rename(columns = {[i for i in df_gene2go.columns.values if 'yourlist' in i][0]: 'gene_id'})
195
-
196
- geneid2go = dict(df_gene2go[['gene_id', 'Gene ontology IDs']].values)
197
-
198
- gene2go = {}
199
- for i,j in geneid2go.items():
200
- j = [k.strip() for k in j.split(';')]
201
- for k in i.split(','):
202
- gene2go[ensembl2genename[k]] = j
203
-
204
- from tqdm import tqdm
205
- edge_list = []
206
- for g1 in tqdm(gene2go.keys()):
207
- for g2 in gene2go.keys():
208
-
209
- edge_list.append((g1, g2, len(np.intersect1d(gene2go[g1], gene2go[g2]))/len(np.union1d(gene2go[g1], gene2go[g2]))))
210
-
211
- edge_list_filter = [i for i in edge_list if i[2] > 0]
212
- further_filter = [i for i in edge_list if i[2] > 0.1]
213
- df_edge_list = pd.DataFrame(further_filter).rename(columns = {0: 'gene1', 1: 'gene2', 2: 'score'})
214
-
215
- df_edge_list = df_edge_list.rename(columns = {'gene1': 'source', 'gene2': 'target', 'score': 'importance'})
216
-
217
- return df_edge_list
218
-
219
- class GeneSimNetwork():
220
- '''
221
- Represents a gene similarity network. Has the following attributes:
222
- - self.edge_list
223
- - self.gene_list
224
- - self.G
225
- - self.edge_index (Tensor, can be used to set G_go in model_initialize)
226
- - self.edge_weight (Tensor, can be used to set G_go_weight in model_initialize)
227
- '''
228
- def __init__(self, edge_list, gene_list, node_map):
229
- # Keeps edges if both source and target exist in node_map
230
- keep_edges = np.zeros(len(edge_list)).astype(bool)
231
- for i, (source, target, importance) in edge_list.iterrows():
232
- keep_edges[i] = source in node_map and target in node_map
233
- self.edge_list = edge_list[keep_edges]
234
-
235
- # Constructs NetworkX graph
236
- self.G = nx.from_pandas_edgelist(self.edge_list, source='source',
237
- target='target', edge_attr=['importance'],
238
- create_using=nx.DiGraph())
239
- self.gene_list = gene_list
240
-
241
- # Add any isolated nodes (represented in the tensor with self edges)
242
- for n in self.gene_list:
243
- added_nodes = []
244
- if n not in self.G.nodes():
245
- self.G.add_node(n)
246
- added_nodes.append(n)
247
- if len(added_nodes) > 0:
248
- print('The following nodes were not present in the edge list and have been added to the graph as self-edges:', added_nodes)
249
-
250
- # Convert graph edges to tensor
251
- edge_index_ = [(node_map[e[0]], node_map[e[1]]) for e in self.G.edges]
252
- self.edge_index = torch.tensor(edge_index_, dtype=torch.long).T
253
-
254
- # Convert edge weights to tensor
255
- edge_attr = nx.get_edge_attributes(self.G, 'importance')
256
- importance = np.array([edge_attr[e] for e in self.G.edges])
257
- self.edge_weight = torch.Tensor(importance)
258
-
259
-
260
- def get_similarity_network(network_type, adata, threshold, k, gene_list, data_path, data_name, split, seed, train_gene_set_size, set2conditions):
261
-
262
- if network_type == 'co-express':
263
- df_out = get_coexpression_network_from_train(adata, threshold, k, data_path, data_name, split, seed, train_gene_set_size, set2conditions)
264
- elif network_type == 'go':
265
- df_jaccard = get_go_auto(gene_list, data_path, data_name)
266
- df_out = df_jaccard.groupby('target').apply(lambda x: x.nlargest(k + 1,['importance'])).reset_index(drop = True)
267
-
268
- return df_out
269
-
270
- def get_coexpression_network_from_train(adata, threshold, k, data_path, data_name, split, seed, train_gene_set_size, set2conditions):
271
-
272
- fname = os.path.join(os.path.join(data_path, data_name), split + '_' + str(seed) + '_' + str(train_gene_set_size) + '_' + str(threshold) + '_' + str(k) + '_co_expression_network.csv')
273
-
274
-
275
- if os.path.exists(fname):
276
- return pd.read_csv(fname)
277
- else:
278
- gene_list = [f for f in adata.var.gene_name.values]
279
- idx2gene = dict(zip(range(len(gene_list)), gene_list))
280
- X = adata.X
281
- train_perts = set2conditions['train']
282
- X_tr = X[np.isin(adata.obs.condition, [i for i in train_perts if 'ctrl' in i])]
283
- gene_list = adata.var['gene_name'].values
284
-
285
- X_tr = X_tr.toarray()
286
- out = np_pearson_cor(X_tr, X_tr)
287
- out[np.isnan(out)] = 0
288
- out = np.abs(out)
289
-
290
- out_sort_idx = np.argsort(out)[:, -(k + 1):]
291
- out_sort_val = np.sort(out)[:, -(k + 1):]
292
-
293
- df_g = []
294
- for i in range(out_sort_idx.shape[0]):
295
- target = idx2gene[i]
296
- for j in range(out_sort_idx.shape[1]):
297
- df_g.append((idx2gene[out_sort_idx[i, j]], target, out_sort_val[i, j]))
298
-
299
- df_g = [i for i in df_g if i[2] > threshold]
300
-
301
- df_co_expression = pd.DataFrame(df_g).rename(columns = {0: 'source', 1: 'target', 2: 'importance'})
302
- df_co_expression.to_csv(fname, index = False)
303
- return df_co_expression
304
-
305
-
306
- def uncertainty_loss_fct(pred, logvar, y, perts, reg = 0.1, ctrl = None, direction_lambda = 1e-3, dict_filter = None):
307
- """
308
- Uncertain version loss:
309
- 1) The error term is weighted by logvar (heteroscedastic modeling)
310
- 2) Optional direction penalty (relative to the up/down direction of control)
311
- 3) `dict_filter` is only calculated on a non-zero gene subset, and the caliber is consistent with the evaluation
312
- """
313
- gamma = 2
314
- perts = np.array(perts)
315
- losses = torch.tensor(0.0, requires_grad=True).to(pred.device)
316
- for p in set(perts):
317
- if p!= 'ctrl':
318
- # Training only on non-zero genes of this perturbation can reduce the impact of dropout noise.
319
- retain_idx = dict_filter[p]
320
- pred_p = pred[np.where(perts==p)[0]][:, retain_idx]
321
- y_p = y[np.where(perts==p)[0]][:, retain_idx]
322
- logvar_p = logvar[np.where(perts==p)[0]][:, retain_idx]
323
- else:
324
- pred_p = pred[np.where(perts==p)[0]]
325
- y_p = y[np.where(perts==p)[0]]
326
- logvar_p = logvar[np.where(perts==p)[0]]
327
-
328
- # uncertainty based loss
329
-
330
- losses += torch.sum((pred_p - y_p)**(2 + gamma) + reg * torch.exp(-logvar_p) * (pred_p - y_p)**(2 + gamma))/pred_p.shape[0]/pred_p.shape[1]
331
-
332
- # direction loss
333
- if p!= 'ctrl':
334
- # Directional consistency term: The constraint prediction is consistent with the true direction of change relative to control.
335
- losses += torch.sum(direction_lambda * (torch.sign(y_p - ctrl[retain_idx]) - torch.sign(pred_p - ctrl[retain_idx]))**2)/pred_p.shape[0]/pred_p.shape[1]
336
- else:
337
- losses += torch.sum(direction_lambda * (torch.sign(y_p - ctrl) - torch.sign(pred_p - ctrl))**2)/pred_p.shape[0]/pred_p.shape[1]
338
-
339
- return losses/(len(set(perts)))
340
-
341
- def loss_fct(pred, y, perts, ctrl = None, direction_lambda = 1e-3, dict_filter = None):
342
- """
343
- Main training loss (without uncertainty):
344
- - Basic item: MSE
345
- - Optional direction item: Penalize upward/downward direction errors
346
- - `dict_filter` will change the training gene set and is an important switch that affects the results.
347
- """
348
- gamma = 2
349
- mse_p = torch.nn.MSELoss()
350
- perts = np.array(perts)
351
- losses = torch.tensor(0.0).to(pred.device)
352
-
353
- for p in set(perts):
354
- pert_idx = np.where(perts == p)[0]
355
-
356
-
357
- # during training, we remove the all zero genes into calculation of loss. this gives a cleaner direction loss. empirically, the performance stays the same.
358
-
359
- # during training, we remove the all zero genes into calculation of loss.
360
- # this gives a cleaner direction loss. empirically, the performance stays the same.
361
-
362
- if p!= 'ctrl':
363
- # Generally more stable when consistent with the evaluated non-zero / non-dropout settings.
364
- retain_idx = dict_filter[p]
365
- pred_p = pred[pert_idx][:, retain_idx]
366
- y_p = y[pert_idx][:, retain_idx]
367
- else:
368
- pred_p = pred[pert_idx]
369
- y_p = y[pert_idx]
370
-
371
-
372
- losses += torch.sum((pred_p - y_p)**(2 + gamma))/pred_p.shape[0]/pred_p.shape[1]
373
-
374
- ## direction loss
375
- if (p!= 'ctrl'):
376
- losses += torch.sum(direction_lambda * (torch.sign(y_p - ctrl[retain_idx]) - torch.sign(pred_p - ctrl[retain_idx]))**2)/pred_p.shape[0]/pred_p.shape[1]
377
- else:
378
- losses += torch.sum(direction_lambda * (torch.sign(y_p - ctrl) - torch.sign(pred_p - ctrl))**2)/pred_p.shape[0]/pred_p.shape[1]
379
-
380
- return losses/(len(set(perts)))
381
-
382
-
383
- def print_sys(s):
384
- """system print
385
-
386
- Args:
387
- s (str): the string to print
388
- """
389
- print(s, flush = True, file = sys.stderr)
390
-
391
- def create_cell_graph_for_prediction(X, pert_idx, pert_gene):
392
-
393
-
394
- # If perturbations will be represented as node features
395
- pert_feats = np.zeros(len(X))
396
- for p in pert_idx:
397
- pert_feats[int(np.abs(p))] = np.sign(p)
398
- feature_mat = torch.Tensor(np.vstack([X, pert_feats])).T
399
-
400
- return Data(x=feature_mat, pert=pert_gene)
401
-
402
-
403
- def create_cell_graph_dataset_for_prediction(pert_gene, ctrl_adata, gene_names, device, num_samples = 300):
404
- Xs = []
405
-
406
- # Get the indices (and signs) of applied perturbation
407
- pert_idx = [np.where(p == np.array(gene_names))[0][0] for p in pert_gene]
408
-
409
- Xs = ctrl_adata[np.random.randint(0, len(ctrl_adata), num_samples), :].X.toarray()
410
- # Create cell graphs
411
- cell_graphs = [create_cell_graph_for_prediction(X, pert_idx, pert_gene).to(device) for X in Xs]
412
- return cell_graphs
413
-
414
- ##
415
- ##GI related utils
416
- ##
417
-
418
- def get_coeffs(singles_expr, first_expr, second_expr, double_expr):
419
- # ============================================================
420
- # Lazy loading of dcor to avoid numba conflicts when importing into multiple processes
421
- # Import dcor only when the function is called
422
- # ============================================================
423
- try:
424
- from dcor import distance_correlation
425
- except Exception as e:
426
- # Use alternative if dcor import fails (numba compatibility issue)
427
- import warnings
428
- warnings.warn(f"dcor import failed: {e}. Using fallback distance correlation implementation.")
429
- # Use a simplified scipy/numpy based implementation as an alternative
430
- def distance_correlation(x, y):
431
- """Simplified version of distance correlation coefficient, does not rely on numba"""
432
- from scipy.spatial.distance import pdist, squareform
433
- x = np.asarray(x)
434
- y = np.asarray(y)
435
- a = squareform(pdist(x))
436
- b = squareform(pdist(y))
437
- A = a - a.mean(axis=0)[None, :] - a.mean(axis=1)[:, None] + a.mean()
438
- B = b - b.mean(axis=0)[None, :] - b.mean(axis=1)[:, None] + b.mean()
439
- dcov2 = (A * B).sum() / (A.size * A.size)
440
- dvar_x = (A * A).sum() / (A.size * A.size)
441
- dvar_y = (B * B).sum() / (B.size * B.size)
442
- if dvar_x == 0 or dvar_y == 0:
443
- return 0.0
444
- return np.sqrt(dcov2 / np.sqrt(dvar_x * dvar_y))
445
-
446
- results = {}
447
- results['ts'] = TheilSenRegressor(fit_intercept=False,
448
- max_subpopulation=1e5,
449
- max_iter=1000,
450
- random_state=1000)
451
- X = singles_expr
452
- y = double_expr
453
- results['ts'].fit(X, y.ravel())
454
- Zts = results['ts'].predict(X)
455
- results['c1'] = results['ts'].coef_[0]
456
- results['c2'] = results['ts'].coef_[1]
457
- results['mag'] = np.sqrt((results['c1']**2 + results['c2']**2))
458
-
459
- results['dcor'] = distance_correlation(singles_expr, double_expr)
460
- results['dcor_singles'] = distance_correlation(first_expr, second_expr)
461
- results['dcor_first'] = distance_correlation(first_expr, double_expr)
462
- results['dcor_second'] = distance_correlation(second_expr, double_expr)
463
- results['corr_fit'] = np.corrcoef(Zts.flatten(), double_expr.flatten())[0,1]
464
- results['dominance'] = np.abs(np.log10(results['c1']/results['c2']))
465
- results['eq_contr'] = np.min([results['dcor_first'], results['dcor_second']])/\
466
- np.max([results['dcor_first'], results['dcor_second']])
467
-
468
- return results
469
-
470
- def get_GI_params(preds, combo):
471
-
472
- singles_expr = np.array([preds[combo[0]], preds[combo[1]]]).T
473
- first_expr = np.array(preds[combo[0]]).T
474
- second_expr = np.array(preds[combo[1]]).T
475
- double_expr = np.array(preds[combo[0]+'_'+combo[1]]).T
476
-
477
- return get_coeffs(singles_expr, first_expr, second_expr, double_expr)
478
-
479
- def get_GI_genes_idx(adata, GI_gene_file):
480
- # Genes used for linear model fitting
481
- GI_genes = np.load(GI_gene_file, allow_pickle=True)
482
- GI_genes_idx = np.where([g in GI_genes for g in adata.var.gene_name.values])[0]
483
-
484
- return GI_genes_idx
485
-
486
- def get_mean_control(adata):
487
- cols = adata.var.gene_name.values.astype('str')
488
- mean_ctrl_exp = adata[adata.obs['condition'] == 'ctrl'].to_df().mean()
489
- return mean_ctrl_exp
 
 
 
 
 
 
 
 
1
+ """
2
+ ================================================================================
3
+ Utility function module (utils.py)
4
+ ================================================================================
5
+
6
+ [Module function]
7
+ Provides various tool functions required for genetic perturbation tasks, including:
8
+
9
+ 1. Perturbation analysis function: analyze the gene name of single gene/combination perturbation
10
+ 2. Gene relationship graph construction: co-expression graph and GO similarity graph
11
+ 3. Loss function: training loss and uncertainty loss
12
+ 4. Genetic interaction (GI) analysis: Calculate genetic interaction coefficients
13
+
14
+ [Key concepts]
15
+ - Perturbation format: "gene1+gene2", such as "ctrl+MAPK1" or "MAPK1+KRAS"
16
+ - Co-expression map: constructed based on gene expression correlations in the data
17
+ - GO graph: Jaccard similarity construction based on Gene Ontology annotation
18
+
19
+ [Key parameters that may affect the results]
20
+ - coexpress_threshold: Threshold for co-expression graph construction
21
+ - dict_filter: training loss is only calculated on non-zero genes
22
+ - direction_lambda: direction consistency loss weight
23
+ ================================================================================
24
+ """
25
+
26
+ import torch
27
+ import numpy as np
28
+ import pandas as pd
29
+ from sklearn.linear_model import TheilSenRegressor
30
+ import torch.nn as nn
31
+
32
+ import networkx as nx
33
+ from tqdm import tqdm
34
+ import pickle
35
+ import sys, os
36
+ import requests
37
+ from torch_geometric.data import Data
38
+
39
+ from zipfile import ZipFile
40
+
41
+ import statsmodels.api as sm
42
+ from sklearn.linear_model import LinearRegression, TheilSenRegressor
43
+ # ============================================================
44
+ # Lazy loading of dcor to avoid numba compatibility issues in DDP multi-process environments
45
+ # dcor will be imported when needed (in the get_coeffs function)
46
+ # ============================================================
47
+ from sklearn.metrics import r2_score
48
+
49
+ def parse_single_pert(i):
50
+ """
51
+ Parsing single gene perturbation strings
52
+
53
+ [Format]
54
+ - "ctrl+GENE": The perturbed gene is GENE
55
+ - "GENE+ctrl": The perturbed gene is GENE
56
+
57
+ [return]
58
+ Perturbed gene name
59
+ """
60
+ a = i.split('+')[0]
61
+ b = i.split('+')[1]
62
+ if a == 'ctrl':
63
+ pert = b
64
+ else:
65
+ pert = a
66
+ return pert
67
+
68
+ def filter_pert_cond_in_go(condition, exclude_pert):
69
+ """
70
+ Filter perturbations not in the GO graph
71
+
72
+ [use]
73
+ Genes without annotations in the GO diagram cannot predict their perturbation effects.
74
+ These disturbing conditions need to be eliminated during data loading.
75
+ """
76
+ if condition == 'ctrl':
77
+ return True
78
+ else:
79
+ if condition in exclude_pert:
80
+ return False
81
+ else:
82
+ return True
83
+
84
+ def parse_combo_pert(i):
85
+ """
86
+ Parsing combined perturbation strings
87
+
88
+ [Format]
89
+ - "GENE1+GENE2": perturb two genes simultaneously
90
+
91
+ [return]
92
+ (gene1, gene2) tuple
93
+ """
94
+ return i.split('+')[0], i.split('+')[1]
95
+
96
+ def combine_res(res_1, res_2):
97
+ """Merge two evaluation result dictionaries"""
98
+ res_out = {}
99
+ for key in res_1:
100
+ res_out[key] = np.concatenate([res_1[key], res_2[key]])
101
+ return res_out
102
+
103
+ def parse_any_pert(p):
104
+ """
105
+ Parse arbitrary perturbation strings (single genes or combinations)
106
+
107
+ [return]
108
+ Gene list:
109
+ - Single gene perturbation: [gene]
110
+ - Combinatorial perturbation: [gene1, gene2]
111
+ """
112
+ if ('ctrl' in p) and (p != 'ctrl'):
113
+ return [parse_single_pert(p)]
114
+ elif 'ctrl' not in p:
115
+ out = parse_combo_pert(p)
116
+ return [out[0], out[1]]
117
+
118
+ def np_pearson_cor(x, y):
119
+ """
120
+ Calculate Pearson correlation coefficient matrix
121
+
122
+ [use]
123
+ Used to construct co-expressed gene relationship diagrams
124
+ """
125
+ xv = x - x.mean(axis=0)
126
+ yv = y - y.mean(axis=0)
127
+ xvss = (xv * xv).sum(axis=0)
128
+ yvss = (yv * yv).sum(axis=0)
129
+ result = np.matmul(xv.transpose(), yv) / np.sqrt(np.outer(xvss, yvss))
130
+ return np.maximum(np.minimum(result, 1.0), -1.0)
131
+
132
+
133
+ def dataverse_download(url, save_path):
134
+ """dataverse download helper with progress bar
135
+
136
+ Args:
137
+ url (str): the url of the dataset
138
+ path (str): the path to save the dataset
139
+ """
140
+
141
+ if os.path.exists(save_path):
142
+ print_sys('Found local copy...')
143
+ else:
144
+ print_sys("Downloading...")
145
+ response = requests.get(url, stream=True)
146
+ total_size_in_bytes= int(response.headers.get('content-length', 0))
147
+ block_size = 1024
148
+ progress_bar = tqdm(total=total_size_in_bytes, unit='iB', unit_scale=True)
149
+ with open(save_path, 'wb') as file:
150
+ for data in response.iter_content(block_size):
151
+ progress_bar.update(len(data))
152
+ file.write(data)
153
+ progress_bar.close()
154
+
155
+
156
+ def zip_data_download_wrapper(url, save_path, data_path):
157
+
158
+ if os.path.exists(save_path):
159
+ print_sys('Found local copy...')
160
+ else:
161
+ dataverse_download(url, save_path + '.zip')
162
+ print_sys('Extracting zip file...')
163
+ with ZipFile((save_path + '.zip'), 'r') as zip:
164
+ zip.extractall(path = data_path)
165
+ print_sys("Done!")
166
+
167
+ def get_go_auto(gene_list, data_path, data_name, gene2go_path=None):
168
+ go_path = os.path.join(data_path, data_name, 'go.csv')
169
+
170
+ if os.path.exists(go_path):
171
+ return pd.read_csv(go_path)
172
+ else:
173
+ # This optional annotation resource is not bundled with the project.
174
+ gene2go_path = gene2go_path or os.environ.get('BIOMOD_GENE2GO_PATH')
175
+ if not gene2go_path or not os.path.isfile(gene2go_path):
176
+ raise FileNotFoundError(
177
+ f"GO network not found: {go_path}. Provide gene2go_path or set "
178
+ "BIOMOD_GENE2GO_PATH to a gene2go_all.pkl annotation file to regenerate it."
179
+ )
180
+ with open(gene2go_path, 'rb') as f:
181
+ gene2go = pickle.load(f)
182
+
183
+ gene2go = {i: list(gene2go[i]) for i in gene_list if i in gene2go}
184
+ edge_list = []
185
+ for g1 in tqdm(gene2go.keys()):
186
+ for g2 in gene2go.keys():
187
+ edge_list.append((g1, g2, len(np.intersect1d(gene2go[g1], gene2go[g2]))/len(np.union1d(gene2go[g1], gene2go[g2]))))
188
+
189
+ edge_list_filter = [i for i in edge_list if i[2] > 0]
190
+ further_filter = [i for i in edge_list if i[2] > 0.1]
191
+ df_edge_list = pd.DataFrame(further_filter).rename(columns = {0: 'gene1', 1: 'gene2', 2: 'score'})
192
+
193
+ df_edge_list = df_edge_list.rename(columns = {'gene1': 'source', 'gene2': 'target', 'score': 'importance'})
194
+
195
+ df_edge_list.to_csv(go_path, index = False)
196
+ return df_edge_list
197
+
198
+ def get_go(df_gene2go):
199
+ df_gene2go['Entry name'] = df_gene2go['Entry name'].apply(lambda x: x.split('_')[0])
200
+ df_gene2go = df_gene2go[df_gene2go['Gene ontology IDs'].notnull()]
201
+ df_gene2go = df_gene2go.rename(columns = {[i for i in df_gene2go.columns.values if 'yourlist' in i][0]: 'gene_id'})
202
+
203
+ geneid2go = dict(df_gene2go[['gene_id', 'Gene ontology IDs']].values)
204
+
205
+ gene2go = {}
206
+ for i,j in geneid2go.items():
207
+ j = [k.strip() for k in j.split(';')]
208
+ for k in i.split(','):
209
+ gene2go[ensembl2genename[k]] = j
210
+
211
+ from tqdm import tqdm
212
+ edge_list = []
213
+ for g1 in tqdm(gene2go.keys()):
214
+ for g2 in gene2go.keys():
215
+
216
+ edge_list.append((g1, g2, len(np.intersect1d(gene2go[g1], gene2go[g2]))/len(np.union1d(gene2go[g1], gene2go[g2]))))
217
+
218
+ edge_list_filter = [i for i in edge_list if i[2] > 0]
219
+ further_filter = [i for i in edge_list if i[2] > 0.1]
220
+ df_edge_list = pd.DataFrame(further_filter).rename(columns = {0: 'gene1', 1: 'gene2', 2: 'score'})
221
+
222
+ df_edge_list = df_edge_list.rename(columns = {'gene1': 'source', 'gene2': 'target', 'score': 'importance'})
223
+
224
+ return df_edge_list
225
+
226
+ class GeneSimNetwork():
227
+ '''
228
+ Represents a gene similarity network. Has the following attributes:
229
+ - self.edge_list
230
+ - self.gene_list
231
+ - self.G
232
+ - self.edge_index (Tensor, can be used to set G_go in model_initialize)
233
+ - self.edge_weight (Tensor, can be used to set G_go_weight in model_initialize)
234
+ '''
235
+ def __init__(self, edge_list, gene_list, node_map):
236
+ # Keeps edges if both source and target exist in node_map
237
+ keep_edges = np.zeros(len(edge_list)).astype(bool)
238
+ for i, (source, target, importance) in edge_list.iterrows():
239
+ keep_edges[i] = source in node_map and target in node_map
240
+ self.edge_list = edge_list[keep_edges]
241
+
242
+ # Constructs NetworkX graph
243
+ self.G = nx.from_pandas_edgelist(self.edge_list, source='source',
244
+ target='target', edge_attr=['importance'],
245
+ create_using=nx.DiGraph())
246
+ self.gene_list = gene_list
247
+
248
+ # Add any isolated nodes (represented in the tensor with self edges)
249
+ for n in self.gene_list:
250
+ added_nodes = []
251
+ if n not in self.G.nodes():
252
+ self.G.add_node(n)
253
+ added_nodes.append(n)
254
+ if len(added_nodes) > 0:
255
+ print('The following nodes were not present in the edge list and have been added to the graph as self-edges:', added_nodes)
256
+
257
+ # Convert graph edges to tensor
258
+ edge_index_ = [(node_map[e[0]], node_map[e[1]]) for e in self.G.edges]
259
+ self.edge_index = torch.tensor(edge_index_, dtype=torch.long).T
260
+
261
+ # Convert edge weights to tensor
262
+ edge_attr = nx.get_edge_attributes(self.G, 'importance')
263
+ importance = np.array([edge_attr[e] for e in self.G.edges])
264
+ self.edge_weight = torch.Tensor(importance)
265
+
266
+
267
+ def get_similarity_network(network_type, adata, threshold, k, gene_list, data_path, data_name, split, seed, train_gene_set_size, set2conditions):
268
+
269
+ if network_type == 'co-express':
270
+ df_out = get_coexpression_network_from_train(adata, threshold, k, data_path, data_name, split, seed, train_gene_set_size, set2conditions)
271
+ elif network_type == 'go':
272
+ df_jaccard = get_go_auto(gene_list, data_path, data_name)
273
+ df_out = df_jaccard.groupby('target').apply(lambda x: x.nlargest(k + 1,['importance'])).reset_index(drop = True)
274
+
275
+ return df_out
276
+
277
+ def get_coexpression_network_from_train(adata, threshold, k, data_path, data_name, split, seed, train_gene_set_size, set2conditions):
278
+
279
+ fname = os.path.join(os.path.join(data_path, data_name), split + '_' + str(seed) + '_' + str(train_gene_set_size) + '_' + str(threshold) + '_' + str(k) + '_co_expression_network.csv')
280
+
281
+
282
+ if os.path.exists(fname):
283
+ return pd.read_csv(fname)
284
+ else:
285
+ gene_list = [f for f in adata.var.gene_name.values]
286
+ idx2gene = dict(zip(range(len(gene_list)), gene_list))
287
+ X = adata.X
288
+ train_perts = set2conditions['train']
289
+ X_tr = X[np.isin(adata.obs.condition, [i for i in train_perts if 'ctrl' in i])]
290
+ gene_list = adata.var['gene_name'].values
291
+
292
+ X_tr = X_tr.toarray()
293
+ out = np_pearson_cor(X_tr, X_tr)
294
+ out[np.isnan(out)] = 0
295
+ out = np.abs(out)
296
+
297
+ out_sort_idx = np.argsort(out)[:, -(k + 1):]
298
+ out_sort_val = np.sort(out)[:, -(k + 1):]
299
+
300
+ df_g = []
301
+ for i in range(out_sort_idx.shape[0]):
302
+ target = idx2gene[i]
303
+ for j in range(out_sort_idx.shape[1]):
304
+ df_g.append((idx2gene[out_sort_idx[i, j]], target, out_sort_val[i, j]))
305
+
306
+ df_g = [i for i in df_g if i[2] > threshold]
307
+
308
+ df_co_expression = pd.DataFrame(df_g).rename(columns = {0: 'source', 1: 'target', 2: 'importance'})
309
+ df_co_expression.to_csv(fname, index = False)
310
+ return df_co_expression
311
+
312
+
313
+ def uncertainty_loss_fct(pred, logvar, y, perts, reg = 0.1, ctrl = None, direction_lambda = 1e-3, dict_filter = None):
314
+ """
315
+ Uncertain version loss:
316
+ 1) The error term is weighted by logvar (heteroscedastic modeling)
317
+ 2) Optional direction penalty (relative to the up/down direction of control)
318
+ 3) `dict_filter` is only calculated on a non-zero gene subset, and the caliber is consistent with the evaluation
319
+ """
320
+ gamma = 2
321
+ perts = np.array(perts)
322
+ losses = torch.tensor(0.0, requires_grad=True).to(pred.device)
323
+ for p in set(perts):
324
+ if p!= 'ctrl':
325
+ # Training only on non-zero genes of this perturbation can reduce the impact of dropout noise.
326
+ retain_idx = dict_filter[p]
327
+ pred_p = pred[np.where(perts==p)[0]][:, retain_idx]
328
+ y_p = y[np.where(perts==p)[0]][:, retain_idx]
329
+ logvar_p = logvar[np.where(perts==p)[0]][:, retain_idx]
330
+ else:
331
+ pred_p = pred[np.where(perts==p)[0]]
332
+ y_p = y[np.where(perts==p)[0]]
333
+ logvar_p = logvar[np.where(perts==p)[0]]
334
+
335
+ # uncertainty based loss
336
+
337
+ losses += torch.sum((pred_p - y_p)**(2 + gamma) + reg * torch.exp(-logvar_p) * (pred_p - y_p)**(2 + gamma))/pred_p.shape[0]/pred_p.shape[1]
338
+
339
+ # direction loss
340
+ if p!= 'ctrl':
341
+ # Directional consistency term: The constraint prediction is consistent with the true direction of change relative to control.
342
+ losses += torch.sum(direction_lambda * (torch.sign(y_p - ctrl[retain_idx]) - torch.sign(pred_p - ctrl[retain_idx]))**2)/pred_p.shape[0]/pred_p.shape[1]
343
+ else:
344
+ losses += torch.sum(direction_lambda * (torch.sign(y_p - ctrl) - torch.sign(pred_p - ctrl))**2)/pred_p.shape[0]/pred_p.shape[1]
345
+
346
+ return losses/(len(set(perts)))
347
+
348
+ def loss_fct(pred, y, perts, ctrl = None, direction_lambda = 1e-3, dict_filter = None):
349
+ """
350
+ Main training loss (without uncertainty):
351
+ - Basic item: MSE
352
+ - Optional direction item: Penalize upward/downward direction errors
353
+ - `dict_filter` will change the training gene set and is an important switch that affects the results.
354
+ """
355
+ gamma = 2
356
+ mse_p = torch.nn.MSELoss()
357
+ perts = np.array(perts)
358
+ losses = torch.tensor(0.0).to(pred.device)
359
+
360
+ for p in set(perts):
361
+ pert_idx = np.where(perts == p)[0]
362
+
363
+
364
+ # during training, we remove the all zero genes into calculation of loss. this gives a cleaner direction loss. empirically, the performance stays the same.
365
+
366
+ # during training, we remove the all zero genes into calculation of loss.
367
+ # this gives a cleaner direction loss. empirically, the performance stays the same.
368
+
369
+ if p!= 'ctrl':
370
+ # Generally more stable when consistent with the evaluated non-zero / non-dropout settings.
371
+ retain_idx = dict_filter[p]
372
+ pred_p = pred[pert_idx][:, retain_idx]
373
+ y_p = y[pert_idx][:, retain_idx]
374
+ else:
375
+ pred_p = pred[pert_idx]
376
+ y_p = y[pert_idx]
377
+
378
+
379
+ losses += torch.sum((pred_p - y_p)**(2 + gamma))/pred_p.shape[0]/pred_p.shape[1]
380
+
381
+ ## direction loss
382
+ if (p!= 'ctrl'):
383
+ losses += torch.sum(direction_lambda * (torch.sign(y_p - ctrl[retain_idx]) - torch.sign(pred_p - ctrl[retain_idx]))**2)/pred_p.shape[0]/pred_p.shape[1]
384
+ else:
385
+ losses += torch.sum(direction_lambda * (torch.sign(y_p - ctrl) - torch.sign(pred_p - ctrl))**2)/pred_p.shape[0]/pred_p.shape[1]
386
+
387
+ return losses/(len(set(perts)))
388
+
389
+
390
+ def print_sys(s):
391
+ """system print
392
+
393
+ Args:
394
+ s (str): the string to print
395
+ """
396
+ print(s, flush = True, file = sys.stderr)
397
+
398
+ def create_cell_graph_for_prediction(X, pert_idx, pert_gene):
399
+
400
+
401
+ # If perturbations will be represented as node features
402
+ pert_feats = np.zeros(len(X))
403
+ for p in pert_idx:
404
+ pert_feats[int(np.abs(p))] = np.sign(p)
405
+ feature_mat = torch.Tensor(np.vstack([X, pert_feats])).T
406
+
407
+ return Data(x=feature_mat, pert=pert_gene)
408
+
409
+
410
+ def create_cell_graph_dataset_for_prediction(pert_gene, ctrl_adata, gene_names, device, num_samples = 300):
411
+ Xs = []
412
+
413
+ # Get the indices (and signs) of applied perturbation
414
+ pert_idx = [np.where(p == np.array(gene_names))[0][0] for p in pert_gene]
415
+
416
+ Xs = ctrl_adata[np.random.randint(0, len(ctrl_adata), num_samples), :].X.toarray()
417
+ # Create cell graphs
418
+ cell_graphs = [create_cell_graph_for_prediction(X, pert_idx, pert_gene).to(device) for X in Xs]
419
+ return cell_graphs
420
+
421
+ ##
422
+ ##GI related utils
423
+ ##
424
+
425
+ def get_coeffs(singles_expr, first_expr, second_expr, double_expr):
426
+ # ============================================================
427
+ # Lazy loading of dcor to avoid numba conflicts when importing into multiple processes
428
+ # Import dcor only when the function is called
429
+ # ============================================================
430
+ try:
431
+ from dcor import distance_correlation
432
+ except Exception as e:
433
+ # Use alternative if dcor import fails (numba compatibility issue)
434
+ import warnings
435
+ warnings.warn(f"dcor import failed: {e}. Using fallback distance correlation implementation.")
436
+ # Use a simplified scipy/numpy based implementation as an alternative
437
+ def distance_correlation(x, y):
438
+ """Simplified version of distance correlation coefficient, does not rely on numba"""
439
+ from scipy.spatial.distance import pdist, squareform
440
+ x = np.asarray(x)
441
+ y = np.asarray(y)
442
+ a = squareform(pdist(x))
443
+ b = squareform(pdist(y))
444
+ A = a - a.mean(axis=0)[None, :] - a.mean(axis=1)[:, None] + a.mean()
445
+ B = b - b.mean(axis=0)[None, :] - b.mean(axis=1)[:, None] + b.mean()
446
+ dcov2 = (A * B).sum() / (A.size * A.size)
447
+ dvar_x = (A * A).sum() / (A.size * A.size)
448
+ dvar_y = (B * B).sum() / (B.size * B.size)
449
+ if dvar_x == 0 or dvar_y == 0:
450
+ return 0.0
451
+ return np.sqrt(dcov2 / np.sqrt(dvar_x * dvar_y))
452
+
453
+ results = {}
454
+ results['ts'] = TheilSenRegressor(fit_intercept=False,
455
+ max_subpopulation=1e5,
456
+ max_iter=1000,
457
+ random_state=1000)
458
+ X = singles_expr
459
+ y = double_expr
460
+ results['ts'].fit(X, y.ravel())
461
+ Zts = results['ts'].predict(X)
462
+ results['c1'] = results['ts'].coef_[0]
463
+ results['c2'] = results['ts'].coef_[1]
464
+ results['mag'] = np.sqrt((results['c1']**2 + results['c2']**2))
465
+
466
+ results['dcor'] = distance_correlation(singles_expr, double_expr)
467
+ results['dcor_singles'] = distance_correlation(first_expr, second_expr)
468
+ results['dcor_first'] = distance_correlation(first_expr, double_expr)
469
+ results['dcor_second'] = distance_correlation(second_expr, double_expr)
470
+ results['corr_fit'] = np.corrcoef(Zts.flatten(), double_expr.flatten())[0,1]
471
+ results['dominance'] = np.abs(np.log10(results['c1']/results['c2']))
472
+ results['eq_contr'] = np.min([results['dcor_first'], results['dcor_second']])/\
473
+ np.max([results['dcor_first'], results['dcor_second']])
474
+
475
+ return results
476
+
477
+ def get_GI_params(preds, combo):
478
+
479
+ singles_expr = np.array([preds[combo[0]], preds[combo[1]]]).T
480
+ first_expr = np.array(preds[combo[0]]).T
481
+ second_expr = np.array(preds[combo[1]]).T
482
+ double_expr = np.array(preds[combo[0]+'_'+combo[1]]).T
483
+
484
+ return get_coeffs(singles_expr, first_expr, second_expr, double_expr)
485
+
486
+ def get_GI_genes_idx(adata, GI_gene_file):
487
+ # Genes used for linear model fitting
488
+ GI_genes = np.load(GI_gene_file, allow_pickle=True)
489
+ GI_genes_idx = np.where([g in GI_genes for g in adata.var.gene_name.values])[0]
490
+
491
+ return GI_genes_idx
492
+
493
+ def get_mean_control(adata):
494
+ cols = adata.var.gene_name.values.astype('str')
495
+ mean_ctrl_exp = adata[adata.obs['condition'] == 'ctrl'].to_df().mean()
496
+ return mean_ctrl_exp
BioMod/interpretability/README.md CHANGED
@@ -1,413 +1,361 @@
1
- # Module Interpretability Experiments for BioMod
2
-
3
- This directory contains analyses of BioMod module representations, covering
4
- latent geometry, pathway-text alignment, module importance, and attention.
5
- These workflows examine structural, semantic, and functional properties of the
6
- learned modules. Use the
7
- [BioMod model weights and matching configuration](../../README.md#biomod-model-weights)
8
- when loading the pretrained backbone.
9
-
10
- ## Experiment Roadmap, Ordered by Priority
11
-
12
- | # | Script | Core question | Status |
13
- | --- | --- | --- | --- |
14
- | 1 | `module_geometry.py` | Does the module latent space self-organize by pathway family, | Completed |
15
- | 2 | `module_text_alignment/module_text_alignment.py` | Do module embeddings align with natural-language pathway descriptions, | Implemented |
16
- | 3 | `module_importance/module_importance.py` | Does task-driven module importance match known pathway-cell-type or pathway-disease associations, | Implemented |
17
- | 4 | `module_vs_gene.py` | Is BioMod better than GenePT/scGPT for pathway-level interpretability, | TODO |
18
- | 6 | `attention/module_crosstalk.py` | Do module interactions match Reactome/KEGG pathway relationships, | Implemented |
19
-
20
- Experiments 1 and 2 are the core differentiators because gene-level baselines
21
- cannot structurally reproduce them. Experiment 3 provides task-level functional
22
- validation. Experiment 4 provides direct comparison to baselines. Additional
23
- experiments, such as module crosstalk vs Reactome, module dynamics along
24
- developmental trajectories, and retraining-based module-granularity ablation, can
25
- be pursued after the core experiments are complete.
26
-
27
- ## Experiment 1: Intrinsic Geometry of Module Embeddings (`module_geometry.py`)
28
-
29
- **Hypothesis:** the BioMod encoder is trained with a masked-module
30
- reconstruction task, but it should still encode semantically related modules,
31
- such as modules from the same pathway family or MSigDB hallmark biological
32
- process, into nearby regions of latent space, even though family labels are
33
- never seen during training.
34
-
35
- **Workflow:**
36
-
37
- 1. Load the pretrained backbone and run inference with `mask_ratio=0.0` and `return_encodings=True` to obtain module encodings shaped `[B, num_modules, base_dim]`.
38
- 2. Average over the cell population to obtain `[num_modules, base_dim]`, using streaming accumulation rather than storing the full table in memory.
39
- 3. Parse family assignments from the keys in `merged_modules.json`: `Hallmarks` (50), `Reactome` (491), `Hotspot` (73), and `Processed` (46). Hallmark modules are also annotated with one of eight MSigDB biological-process categories: immune, metabolic, signaling, proliferation, development, dna_damage, pathway, and cellular_component.
40
- 4. Quantify family separation:
41
- - silhouette coefficient, using both cosine and Euclidean metrics.
42
- - kNN purity at `k=10`, using both cosine and Euclidean metrics.
43
- - ratio of mean inter-family to intra-family distance.
44
- 5. Compare to a permutation null distribution by shuffling family labels `K=200` times, recomputing metrics, and reporting the observed value, null mean, null standard deviation, and right-tail empirical p-value.
45
- 6. Visualize with UMAP and PCA in 2D, colored by family or hallmark biological process.
46
-
47
- **Differentiation from gene-level baselines:** gene-level foundation models embed
48
- single genes rather than pathway groups. To reproduce this experiment, they would
49
- first have to manually aggregate gene embeddings by pathway, which compresses
50
- gene-level signals and introduces a hand-designed step. BioMod learns this
51
- aggregation end to end during pretraining.
52
-
53
- ## File Structure
54
-
55
- ```text
56
- BioMod/interpretability/ # Module representations and biological interpretability
57
- ├── attention/ # Module crosstalk and within-module gene attention
58
- │ ├── __init__.py
59
- │ ├── module_crosstalk.py
60
- │ ├── module_gene_attention.py
61
- │ ├── README.md
62
- │ └── README_gene_attention.md
63
- ├── module_importance/ # Module importance, ablation studies and marker-gene validation
64
- │ ├── __init__.py
65
- │ ├── cross_class_similarity.py
66
- │ ├── family_ablation.py
67
- │ ├── family_importance_analysis.py
68
- │ ├── marker_gene_verification.py
69
- │ ├── masking_ablation.py
70
- │ ├── module_importance.py
71
- │ ├── module_importance_publication.py
72
- │ ├── pub_utils.py
73
- │ ├── random_baseline.py
74
- │ ├── README.md
75
- │ ├── RESULTS.md
76
- │ ├── tcell_reactome_analysis.py
77
- │ └── TODO.md
78
- ├── module_text_alignment/ # Semantic alignment of module representations and pathway text
79
- │ ├── __init__.py
80
- │ ├── DESIGN.md
81
- │ ├── fetch_reactome_descriptions.py
82
- │ ├── MANUSCRIPT_SECTION.md
83
- │ ├── module_text_alignment.py
84
- │ ├── module_text_alignment_extended.py
85
- │ ��── module_text_alignment_publication.py
86
- │ └── reactome_descriptions.json
87
- ├── __init__.py
88
- ├── module_geometry.py
89
- ├── README.md
90
- └── utils.py
91
- ```
92
-
93
- ## Command-Line Usage
94
-
95
- Run scripts from `BioMod/interpretability/` within the project root; relative paths below are
96
- based on that directory.
97
-
98
- ```bash
99
- # General h5ad input: cell annotation, zero-shot, or any annotated dataset.
100
- python module_geometry.py \
101
- --performer_checkpoint ../mae_tost_48M_cross_attn_only_pure_memory/mae_tost_48M_cross_attn_only_pure_memory_epoch12.pth \
102
- --config_pkl ../mae_tost_48M_cross_attn_only_pure_memory/mae_tost_48M_cross_attn_only_pure_memory.pkl \
103
- --module_data_dir ../module_data_all \
104
- --gene_list_path ../../selected_genes_27k.txt \
105
- --gene_mapping_path ../../gene2ens.csv \
106
- --gene2vec_data_path ../../selected_gene2vec_27k.npy \
107
- --gene2ens_file ../../gene2ens.csv \
108
- --data_path ../../zero-shot-batch/pancreas_scib.h5ad \
109
- --data_mode h5ad \
110
- --gene_name_column auto \
111
- --batch_size 640 \
112
- --output_dir ./results/geometry/pancreas_scib
113
- ```
114
-
115
- For pretraining data, where each cell is an `.npz` file already in the 27,874
116
- model-gene space:
117
-
118
- ```bash
119
- python module_geometry.py \
120
- --performer_checkpoint ../mae_tost_48M_cross_attn_only_pure_memory/mae_tost_48M_cross_attn_only_pure_memory_epoch12.pth \
121
- --config_pkl ../mae_tost_48M_cross_attn_only_pure_memory/mae_tost_48M_cross_attn_only_pure_memory.pkl \
122
- --module_data_dir ../module_data_all \
123
- --gene_list_path ../../selected_genes_27k.txt \
124
- --gene_mapping_path ../../gene2ens.csv \
125
- --gene2vec_data_path ../../selected_gene2vec_27k.npy \
126
- --data_path ../../../../share/cell_files \
127
- --data_mode pretrain_npz \
128
- --cell_num 48024242 \
129
- --max_cells 50000 \
130
- --batch_size 512 \
131
- --output_dir ./results/geometry/pretrain_subset
132
- ```
133
-
134
- ### Multi-GPU DDP Acceleration
135
-
136
- `torchrun` multi-GPU inference is supported. When `WORLD_SIZE > 1`, each GPU
137
- processes a subset of cells, embeddings are aggregated with `all_reduce`, and
138
- rank 0 computes metrics and figures. Single-GPU and CPU behavior is unchanged.
139
-
140
- ```bash
141
- torchrun --nproc_per_node=2 module_geometry.py \
142
- --performer_checkpoint ../mae_tost_48M_cross_attn_only_pure_memory/mae_tost_48M_cross_attn_only_pure_memory_epoch2.pth \
143
- --config_pkl ../mae_tost_48M_cross_attn_only_pure_memory/mae_tost_48M_cross_attn_only_pure_memory.pkl \
144
- --module_data_dir ../module_data_all \
145
- --gene_list_path ../../selected_genes_27k.txt \
146
- --gene_mapping_path ../../gene2ens.csv \
147
- --gene2vec_data_path ../../selected_gene2vec_27k.npy \
148
- --data_path ../../../share/cell_files \
149
- --data_mode pretrain_npz \
150
- --cell_num 49738024 \
151
- --max_cells 100000 \
152
- --batch_size 16 \
153
- --output_dir ./results/geometry/pretrain_ddp
154
- ```
155
-
156
- ### Common Arguments
157
-
158
- | Argument | Default | Description |
159
- | --- | --- | --- |
160
- | `--max_cells` | None | Number of cells to subsample; the mean is stable from a few thousand cells onward. |
161
- | `--cell_num` | None | Total cell count in `pretrain_npz` mode, used for sequential names `cell0.npz` to `cellN.npz`. |
162
- | `--knn_k` | 10 | `k` for kNN purity. |
163
- | `--null_permutations` | 200 | Number of label permutations for the empirical null distribution. |
164
- | `--include_other_in_metrics` | Off | By default, modules from the `Other` family are excluded. |
165
- | `--shuffle_module_genes` | Off | Randomly shuffle gene-to-module assignments as a negative control or ablation. Uses the same flag as pretraining and downstream finetuning. |
166
- | `--umap_n_neighbors` | 15 | UMAP graph-construction parameter. |
167
- | `--umap_min_dist` | 0.3 | UMAP layout parameter. |
168
-
169
- ## Output Files
170
-
171
- Files are written to `<output_dir>/`:
172
-
173
- | File | Content |
174
- | --- | --- |
175
- | `module_embeddings.npy` | Cell-averaged module embeddings with shape `[num_modules, base_dim]` |
176
- | `module_metadata.csv` | Metadata for each module: `module_name`, `family`, and `hallmark_process` |
177
- | `metrics.json` | Observed metrics, null mean/std, and right-tail p-values for family-level and hallmark-process-level analyses |
178
- | `pca_family.png` | 2D PCA colored by family |
179
- | `umap_family.png` | 2D UMAP colored by family; requires `umap-learn` |
180
- | `pca_hallmark_process.png` | PCA for Hallmark modules only, colored by MSigDB biological process |
181
- | `umap_hallmark_process.png` | UMAP for Hallmark modules only, colored by MSigDB biological process |
182
- | `run.log` | Full log with formatted core metrics |
183
-
184
- ## Recommended Negative Controls
185
-
186
- Two control experiments make Experiment 1 more rigorous:
187
-
188
- 1. **Random module assignment**: rerun with `--shuffle_module_genes`. This flag preserves module sizes but randomly assigns genes to modules, matching the same ablation flag used for pretraining and finetuning. If geometry is pathway-driven rather than a generic property of the encoder, family separation should drop close to the null distribution.
189
-
190
- 2. **Untrained backbone**: set `--performer_checkpoint` to the `*_init.pth` checkpoint generated during pretraining with `--save_init_ckpt_and_exit`. Family separation should be close to the null distribution. The gap between trained and untrained models quantifies the contribution of pretraining.
191
-
192
- Together, these controls support the claim that the family structure of the module
193
- latent space is produced by pathway-based module design plus pretraining, rather
194
- than by module design alone or by a generic encoder architecture.
195
-
196
- ### Control 1: Random Module Assignment (`shuffle_module_genes`)
197
-
198
- Add `--shuffle_module_genes` to the original command and write to a control output
199
- directory.
200
-
201
- For h5ad data, using `pancreas_scib` as an example:
202
-
203
- ```bash
204
- python module_geometry.py \
205
- --performer_checkpoint ../mae_tost_48M_cross_attn_only_pure_memory/mae_tost_48M_cross_attn_only_pure_memory_epoch12.pth \
206
- --config_pkl ../mae_tost_48M_cross_attn_only_pure_memory/mae_tost_48M_cross_attn_only_pure_memory.pkl \
207
- --module_data_dir ../module_data_all \
208
- --gene_list_path ../../selected_genes_27k.txt \
209
- --gene_mapping_path ../../gene2ens.csv \
210
- --gene2vec_data_path ../../selected_gene2vec_27k.npy \
211
- --gene2ens_file ../../gene2ens.csv \
212
- --data_path ../../zero-shot-batch/pancreas_scib.h5ad \
213
- --data_mode h5ad \
214
- --gene_name_column auto \
215
- --batch_size 640 \
216
- --shuffle_module_genes \
217
- --output_dir ./results/geometry/pancreas_scib_shuffle
218
- ```
219
-
220
- For pretraining npz data:
221
-
222
- ```bash
223
- python module_geometry.py \
224
- --performer_checkpoint ../mae_tost_48M_cross_attn_only_pure_memory/mae_tost_48M_cross_attn_only_pure_memory_epoch12.pth \
225
- --config_pkl ../mae_tost_48M_cross_attn_only_pure_memory/mae_tost_48M_cross_attn_only_pure_memory.pkl \
226
- --module_data_dir ../module_data_all \
227
- --gene_list_path ../../selected_genes_27k.txt \
228
- --gene_mapping_path ../../gene2ens.csv \
229
- --gene2vec_data_path ../../selected_gene2vec_27k.npy \
230
- --gene2ens_file ../../gene2ens.csv \
231
- --data_path ../../../../share/cell_files \
232
- --data_mode pretrain_npz \
233
- --cell_num 48024242 \
234
- --max_cells 50000 \
235
- --batch_size 640 \
236
- --shuffle_module_genes \
237
- --output_dir ./results/geometry/pretrain_subset_shuffle
238
- ```
239
-
240
- How `--shuffle_module_genes` works: inside `load_backbone()`, module sizes are
241
- kept unchanged, all genes are collected into one list, the list is randomly
242
- shuffled, and genes are reassigned to modules. The seed is controlled by
243
- `--shuffle_seed`, defaulting to 2021. Model weights are unchanged; only the
244
- gene-to-module mapping is shuffled.
245
-
246
- **Expected result:** kNN purity should drop to about 0.576, the null level, and
247
- the inter-/intra-family distance ratio should drop to about 1.0. `metrics.json`
248
- records `"shuffle_module_genes": true`.
249
-
250
- ### Control 2: Untrained Backbone with Random Initialization
251
-
252
- First generate a randomly initialized checkpoint, then run `module_geometry.py`
253
- with it.
254
-
255
- **Step 1: Generate `_init.pth`**
256
-
257
- Use the pretraining script with `--save_init_ckpt_and_exit`. The command
258
- initializes the model, saves a random-weight checkpoint and the matching config
259
- PKL, then exits immediately without entering training.
260
-
261
- ```bash
262
- cd /mnt/c/Users/250010166/Desktop/single_cell/scLong/performer_module
263
-
264
- torchrun --nproc_per_node=1 performer_module.py \
265
- --model_name mae_tost_48M_cross_attn_only_pure_memory \
266
- --ckpt_dir ./mae_tost_48M_cross_attn_only_pure_memory \
267
- --gene_list_path ../selected_genes_27k.txt \
268
- --gene2vec_data_path ../selected_gene2vec_27k.npy \
269
- --module_data_dir module_data_all \
270
- --gene_mapping_path ../gene2ens.csv \
271
- --data_path ../../../share/cell_files \
272
- --cell_num 48024242 \
273
- --enc_depth 32 \
274
- --enc_heads 16 \
275
- --base_dim 1024 \
276
- --pretrain_arch mae \
277
- --decoder_type cross_attn \
278
- --save_init_ckpt_and_exit
279
- ```
280
-
281
- The following files are generated under
282
- `mae_tost_48M_cross_attn_only_pure_memory/`:
283
-
284
- - `mae_tost_48M_cross_attn_only_pure_memory_init.pth`, randomly initialized weights.
285
- - `mae_tost_48M_cross_attn_only_pure_memory_init.pkl`, model config.
286
-
287
- **Step 2: Run `module_geometry.py` with `_init.pth`**
288
-
289
- For h5ad data:
290
-
291
- ```bash
292
- cd /mnt/c/Users/250010166/Desktop/single_cell/scLong/performer_module/interpretability
293
-
294
- python module_geometry.py \
295
- --performer_checkpoint ../mae_tost_48M_cross_attn_only_pure_memory_init/mae_tost_48M_cross_attn_only_pure_memory_init.pth \
296
- --config_pkl ../mae_tost_48M_cross_attn_only_pure_memory_init/mae_tost_48M_cross_attn_only_pure_memory_init.pkl \
297
- --module_data_dir ../module_data_all \
298
- --gene_list_path ../../selected_genes_27k.txt \
299
- --gene_mapping_path ../../gene2ens.csv \
300
- --gene2vec_data_path ../../selected_gene2vec_27k.npy \
301
- --gene2ens_file ../../gene2ens.csv \
302
- --data_path ../../zero-shot-batch/pancreas_scib.h5ad \
303
- --data_mode h5ad \
304
- --gene_name_column auto \
305
- --batch_size 640 \
306
- --output_dir ./results/geometry/pancreas_scib_init
307
- ```
308
-
309
- For pretraining npz data:
310
-
311
- ```bash
312
- python module_geometry.py \
313
- --performer_checkpoint ../mae_tost_48M_cross_attn_only_pure_memory_init/mae_tost_48M_cross_attn_only_pure_memory_init.pth \
314
- --config_pkl ../mae_tost_48M_cross_attn_only_pure_memory_init/mae_tost_48M_cross_attn_only_pure_memory_init.pkl \
315
- --module_data_dir ../module_data_all \
316
- --gene_list_path ../../selected_genes_27k.txt \
317
- --gene_mapping_path ../../gene2ens.csv \
318
- --gene2vec_data_path ../../selected_gene2vec_27k.npy \
319
- --gene2ens_file ../../gene2ens.csv \
320
- --data_path ../../../../share/cell_files \
321
- --data_mode pretrain_npz \
322
- --cell_num 48024242 \
323
- --max_cells 50000 \
324
- --batch_size 640 \
325
- --output_dir ./results/geometry/pretrain_subset_init
326
- ```
327
-
328
- **Expected result:** all family-level metrics should be close to the null
329
- distribution: kNN purity about 0.576, distance ratio about 1.0, and p-value
330
- above 0.05. This indicates that a randomly initialized encoder cannot separate
331
- pathway families.
332
-
333
- ### Result Comparison Matrix
334
-
335
- After running all controls, construct the following table for the paper:
336
-
337
- | Condition | kNN purity, cosine | Inter-/intra-family ratio, cosine | Conclusion |
338
- | --- | --- | --- | --- |
339
- | **Normal**, trained + pathway module | about 0.72, p<0.005 | 3.8-6.2, p<0.005 | Significant family structure |
340
- | **Shuffle**, trained + random module | about 0.576 expected | about 1.0 expected | Structure is driven by pathway design |
341
- | **Init**, untrained + pathway module | about 0.576 expected | about 1.0 expected | Structure is learned during pretraining |
342
-
343
- The three experiments jointly show that the family structure of the module latent
344
- space is generated by both pathway-based module design and pretraining; both are
345
- necessary.
346
-
347
- ## Experiment 2: Module Embedding to Pathway Text Alignment (`module_text_alignment/`)
348
-
349
- **Hypothesis:** the structure of the BioMod module embedding space significantly
350
- aligns with natural-language pathway semantics. Semantically similar pathways
351
- are also encoded nearby by the model. Gene-level models cannot reproduce this
352
- experiment because individual gene names do not carry phrase-level semantics.
353
-
354
- **Method:**
355
-
356
- 1. Parse 541 semantically named modules from `merged_modules.json`: 50 Hallmark and 491 Reactome.
357
- 2. Encode pathway names with S-PubMedBert-MS-MARCO, a sentence-level fine-tuned PubMedBERT model.
358
- 3. Load `module_embeddings.npy` from Experiment 1, containing cell-averaged module embeddings.
359
- 4. Measure alignment:
360
- - **RSA / Mantel test**, the main metric: Spearman correlation between paired distance matrices from the two spaces, with a permutation test.
361
- - **CKA**: global representation-structure alignment.
362
- - **LOO Ridge retrieval**: cross-modal retrieval recall@k.
363
-
364
- **Data requirement:** only `module_embeddings.npy` from Experiment 1 is needed;
365
- raw expression data are not required.
366
-
367
- ### Commands
368
-
369
- ```bash
370
- cd performer_module/interpretability/module_text_alignment
371
-
372
- # Download the sentence encoder.
373
- wget https://hf-mirror.com/hfd/hfd.sh && chmod a+x hfd.sh && export HF_ENDPOINT=https://hf-mirror.com
374
- ./hfd.sh pritamdeka/S-PubMedBert-MS-MARCO --local-dir ./S-PubMedBert-MS-MARCO
375
-
376
- # Main experiment: RSA + CKA + LOO retrieval.
377
- python module_text_alignment.py \
378
- --module_embedding_path ../results/geometry/pretrain_subset/module_embeddings.npy \
379
- --module_json ../../module_data_all/merged_modules.json \
380
- --sentence_model ./S-PubMedBert-MS-MARCO \
381
- --n_permutations 1000 \
382
- --output_dir ../results/text_alignment/pretrain_subset_v2
383
-
384
- # Extended analysis: broad Reactome categories and Hallmark deep dive.
385
- python module_text_alignment_extended.py \
386
- --module_embedding_path ../results/geometry/pretrain_subset/module_embeddings.npy \
387
- --module_json ../../module_data_all/merged_modules.json \
388
- --sentence_model ./S-PubMedBert-MS-MARCO \
389
- --descriptions_file ./reactome_descriptions.json \
390
- --n_permutations 1000 \
391
- --output_dir ../results/text_alignment/pretrain_subset_v2_extended
392
- ```
393
-
394
- ### Output Files
395
-
396
- | File | Content |
397
- | --- | --- |
398
- | `alignment_metrics.json` | RSA, CKA, retrieval recall@k, and p-values |
399
- | `rsa_scatter.png` | Paired-distance scatter plot, module space vs text space |
400
- | `rsa_null_distribution.png` | Mantel test: observed value vs null distribution |
401
- | `retrieval_ranks.png` | Histogram of retrieval ranks |
402
- | `retrieval_examples.csv` | Top-5 retrieval results for representative pathways |
403
- | `extended_metrics.json` | Extended-analysis results: broad-category RSA and Hallmark deep dive |
404
- | `reactome_category_heatmaps.png` | Distance-matrix comparison at the broad Reactome category level |
405
- | `hallmarks_deep_dive.png` | Hallmark per-module alignment scores and similarity matrix |
406
- | `hallmarks_per_module_alignment.csv` | Per-module alignment ranking for 50 Hallmarks |
407
-
408
- ## Dependencies
409
-
410
- `torch`, `numpy`, `pandas`, `scikit-learn`, `matplotlib`, `anndata`, and `scipy`.
411
- UMAP plots additionally require `umap-learn`; if it is not installed, the script
412
- warns and skips only UMAP plots. Experiment 2 also requires
413
- `sentence-transformers`, installable with `pip install sentence-transformers`.
 
1
+ # Module Interpretability Experiments for BioMod
2
+
3
+ This directory contains analyses of BioMod module representations, covering
4
+ latent geometry, pathway-text alignment, module importance, and attention.
5
+ These workflows examine structural, semantic, and functional properties of the
6
+ learned modules. Use the
7
+ [BioMod model weights and matching configuration](../../README.md#biomod-model-weights)
8
+ when loading the pretrained backbone.
9
+
10
+ ## Experiment Roadmap, Ordered by Priority
11
+
12
+ | # | Script | Core question | Status |
13
+ | --- | --- | --- | --- |
14
+ | 1 | `module_geometry.py` | Does the module latent space self-organize by pathway family, | Completed |
15
+ | 2 | `module_text_alignment/module_text_alignment.py` | Do module embeddings align with natural-language pathway descriptions, | Implemented |
16
+ | 3 | `module_importance/module_importance.py` | Does task-driven module importance match known pathway-cell-type or pathway-disease associations, | Implemented |
17
+ | 4 | `module_vs_gene.py` | Is BioMod better than GenePT/scGPT for pathway-level interpretability, | TODO |
18
+ | 6 | `attention/module_crosstalk.py` | Do module interactions match Reactome/KEGG pathway relationships, | Implemented |
19
+
20
+ Experiments 1 and 2 are the core differentiators because gene-level baselines
21
+ cannot structurally reproduce them. Experiment 3 provides task-level functional
22
+ validation. Experiment 4 provides direct comparison to baselines. Additional
23
+ experiments, such as module crosstalk vs Reactome, module dynamics along
24
+ developmental trajectories, and retraining-based module-granularity ablation, can
25
+ be pursued after the core experiments are complete.
26
+
27
+ ## Experiment 1: Intrinsic Geometry of Module Embeddings (`module_geometry.py`)
28
+
29
+ **Hypothesis:** the BioMod encoder is trained with a masked-module
30
+ reconstruction task, but it should still encode semantically related modules,
31
+ such as modules from the same pathway family or MSigDB hallmark biological
32
+ process, into nearby regions of latent space, even though family labels are
33
+ never seen during training.
34
+
35
+ **Workflow:**
36
+
37
+ 1. Load the pretrained backbone and run inference with `mask_ratio=0.0` and `return_encodings=True` to obtain module encodings shaped `[B, num_modules, base_dim]`.
38
+ 2. Average over the cell population to obtain `[num_modules, base_dim]`, using streaming accumulation rather than storing the full table in memory.
39
+ 3. Parse family assignments from the keys in `merged_modules.json`: `Hallmarks` (50), `Reactome` (491), `Hotspot` (73), and `Processed` (46). Hallmark modules are also annotated with one of eight MSigDB biological-process categories: immune, metabolic, signaling, proliferation, development, dna_damage, pathway, and cellular_component.
40
+ 4. Quantify family separation:
41
+ - silhouette coefficient, using both cosine and Euclidean metrics.
42
+ - kNN purity at `k=10`, using both cosine and Euclidean metrics.
43
+ - ratio of mean inter-family to intra-family distance.
44
+ 5. Compare to a permutation null distribution by shuffling family labels `K=200` times, recomputing metrics, and reporting the observed value, null mean, null standard deviation, and right-tail empirical p-value.
45
+ 6. Visualize with UMAP and PCA in 2D, colored by family or hallmark biological process.
46
+
47
+ **Differentiation from gene-level baselines:** gene-level foundation models embed
48
+ single genes rather than pathway groups. To reproduce this experiment, they would
49
+ first have to manually aggregate gene embeddings by pathway, which compresses
50
+ gene-level signals and introduces a hand-designed step. BioMod learns this
51
+ aggregation end to end during pretraining.
52
+
53
+ ## File Structure
54
+
55
+ ```text
56
+ BioMod/interpretability/ # Module representations and biological interpretability
57
+ ├── attention/ # Module crosstalk and within-module gene attention
58
+ │ ├── __init__.py
59
+ │ ├── module_crosstalk.py
60
+ │ ├── module_gene_attention.py
61
+ │ ├── README.md
62
+ │ └── README_gene_attention.md
63
+ ├── module_importance/ # Module importance, ablation studies and marker-gene validation
64
+ │ ├── __init__.py
65
+ │ ├── cross_class_similarity.py
66
+ │ ├── family_ablation.py
67
+ │ ├── family_importance_analysis.py
68
+ │ ├── marker_gene_verification.py
69
+ │ ├── masking_ablation.py
70
+ │ ├── module_importance.py
71
+ │ ├── module_importance_publication.py
72
+ │ ├── pub_utils.py
73
+ │ ├── random_baseline.py
74
+ │ ├── README.md
75
+ │ ├── RESULTS.md
76
+ │ ├── tcell_reactome_analysis.py
77
+ │ └── TODO.md
78
+ ├── module_text_alignment/ # Semantic alignment of module representations and pathway text
79
+ │ ├── __init__.py
80
+ │ ├── DESIGN.md
81
+ │ ├── fetch_reactome_descriptions.py
82
+ │ ├── MANUSCRIPT_SECTION.md
83
+ │ ├── module_text_alignment.py
84
+ │ ├── module_text_alignment_extended.py
85
+ │ ├── module_text_alignment_publication.py
86
+ │ └── reactome_descriptions.json
87
+ ├── __init__.py
88
+ ├── module_geometry.py
89
+ ├── README.md
90
+ └── utils.py
91
+ ```
92
+
93
+ ## Command-Line Usage
94
+
95
+ Run all commands below from the BioMod repository root. Bundled default assets
96
+ are located relative to the script files, so those defaults do not depend on the
97
+ working directory. Explicit relative CLI paths still resolve from your current
98
+ working directory.
99
+
100
+ The h5ad examples use the bundled pancreas dataset. Pretraining `.npz` cell files
101
+ are not included: replace `/absolute/path/to/pretraining_cell_npz` with your own
102
+ prepared directory. Generated embeddings and analysis results are also not
103
+ bundled; run the corresponding upstream analysis before using those paths.
104
+
105
+ ```bash
106
+ # General h5ad input: cell annotation, zero-shot, or any annotated dataset.
107
+ python BioMod/interpretability/module_geometry.py \
108
+ --performer_checkpoint weight/0810model/mae_tost_70M_75mask_ec2_epoch0.pth \
109
+ --config_pkl weight/0810model/mae_tost_70M_75mask_ec2.pkl \
110
+ --module_data_dir model_dependencies/module_data_all \
111
+ --gene_list_path model_dependencies/selected_genes_27k.txt \
112
+ --gene_mapping_path model_dependencies/gene2ens.csv \
113
+ --gene2vec_data_path model_dependencies/selected_gene2vec_27k.npy \
114
+ --gene2ens_file model_dependencies/gene2ens.csv \
115
+ --data_path data/downstream/final_data/human_pancreas_norm_complexBatch_with_gene_name.h5ad \
116
+ --data_mode h5ad \
117
+ --gene_name_column auto \
118
+ --batch_size 640 \
119
+ --output_dir outputs/interpretability/geometry/pancreas_scib
120
+ ```
121
+
122
+ For pretraining data, where each cell is an `.npz` file already in the 27,874
123
+ model-gene space:
124
+
125
+ ```bash
126
+ python BioMod/interpretability/module_geometry.py \
127
+ --performer_checkpoint weight/0810model/mae_tost_70M_75mask_ec2_epoch0.pth \
128
+ --config_pkl weight/0810model/mae_tost_70M_75mask_ec2.pkl \
129
+ --module_data_dir model_dependencies/module_data_all \
130
+ --gene_list_path model_dependencies/selected_genes_27k.txt \
131
+ --gene_mapping_path model_dependencies/gene2ens.csv \
132
+ --gene2vec_data_path model_dependencies/selected_gene2vec_27k.npy \
133
+ --data_path /absolute/path/to/pretraining_cell_npz \
134
+ --data_mode pretrain_npz \
135
+ --max_cells 50000 \
136
+ --batch_size 512 \
137
+ --output_dir outputs/interpretability/geometry/pretrain_subset
138
+ ```
139
+
140
+ ### Multi-GPU DDP Acceleration
141
+
142
+ `torchrun` multi-GPU inference is supported. When `WORLD_SIZE > 1`, each GPU
143
+ processes a subset of cells, embeddings are aggregated with `all_reduce`, and
144
+ rank 0 computes metrics and figures. Single-GPU and CPU behavior is unchanged.
145
+
146
+ ```bash
147
+ torchrun --nproc_per_node=2 BioMod/interpretability/module_geometry.py \
148
+ --performer_checkpoint weight/0810model/mae_tost_70M_75mask_ec2_epoch0.pth \
149
+ --config_pkl weight/0810model/mae_tost_70M_75mask_ec2.pkl \
150
+ --module_data_dir model_dependencies/module_data_all \
151
+ --gene_list_path model_dependencies/selected_genes_27k.txt \
152
+ --gene_mapping_path model_dependencies/gene2ens.csv \
153
+ --gene2vec_data_path model_dependencies/selected_gene2vec_27k.npy \
154
+ --data_path /absolute/path/to/pretraining_cell_npz \
155
+ --data_mode pretrain_npz \
156
+ --max_cells 100000 \
157
+ --batch_size 16 \
158
+ --output_dir outputs/interpretability/geometry/pretrain_ddp
159
+ ```
160
+
161
+ ### Common Arguments
162
+
163
+ | Argument | Default | Description |
164
+ | --- | --- | --- |
165
+ | `--max_cells` | None | Number of cells to subsample; the mean is stable from a few thousand cells onward. |
166
+ | `--cell_num` | None | Total cell count in `pretrain_npz` mode, used for sequential names `cell0.npz` to `cellN.npz`. |
167
+ | `--knn_k` | 10 | `k` for kNN purity. |
168
+ | `--null_permutations` | 200 | Number of label permutations for the empirical null distribution. |
169
+ | `--include_other_in_metrics` | Off | By default, modules from the `Other` family are excluded. |
170
+ | `--shuffle_module_genes` | Off | Randomly shuffle gene-to-module assignments as a negative control or ablation. Uses the same flag as pretraining and downstream finetuning. |
171
+ | `--umap_n_neighbors` | 15 | UMAP graph-construction parameter. |
172
+ | `--umap_min_dist` | 0.3 | UMAP layout parameter. |
173
+
174
+ ## Output Files
175
+
176
+ Files are written to `<output_dir>/`:
177
+
178
+ | File | Content |
179
+ | --- | --- |
180
+ | `module_embeddings.npy` | Cell-averaged module embeddings with shape `[num_modules, base_dim]` |
181
+ | `module_metadata.csv` | Metadata for each module: `module_name`, `family`, and `hallmark_process` |
182
+ | `metrics.json` | Observed metrics, null mean/std, and right-tail p-values for family-level and hallmark-process-level analyses |
183
+ | `pca_family.png` | 2D PCA colored by family |
184
+ | `umap_family.png` | 2D UMAP colored by family; requires `umap-learn` |
185
+ | `pca_hallmark_process.png` | PCA for Hallmark modules only, colored by MSigDB biological process |
186
+ | `umap_hallmark_process.png` | UMAP for Hallmark modules only, colored by MSigDB biological process |
187
+ | `run.log` | Full log with formatted core metrics |
188
+
189
+ ## Recommended Negative Controls
190
+
191
+ Two control experiments make Experiment 1 more rigorous:
192
+
193
+ 1. **Random module assignment**: rerun with `--shuffle_module_genes`. This flag preserves module sizes but randomly assigns genes to modules, matching the same ablation flag used for pretraining and finetuning. If geometry is pathway-driven rather than a generic property of the encoder, family separation should drop close to the null distribution.
194
+
195
+ 2. **Untrained backbone**: set `--performer_checkpoint` to the `*_init.pth` checkpoint generated during pretraining with `--save_init_ckpt_and_exit`. Family separation should be close to the null distribution. The gap between trained and untrained models quantifies the contribution of pretraining.
196
+
197
+ Together, these controls support the claim that the family structure of the module
198
+ latent space is produced by pathway-based module design plus pretraining, rather
199
+ than by module design alone or by a generic encoder architecture.
200
+
201
+ ### Control 1: Random Module Assignment (`shuffle_module_genes`)
202
+
203
+ Add `--shuffle_module_genes` to the original command and write to a control output
204
+ directory.
205
+
206
+ For h5ad data, using `pancreas_scib` as an example:
207
+
208
+ ```bash
209
+ python BioMod/interpretability/module_geometry.py \
210
+ --performer_checkpoint weight/0810model/mae_tost_70M_75mask_ec2_epoch0.pth \
211
+ --config_pkl weight/0810model/mae_tost_70M_75mask_ec2.pkl \
212
+ --module_data_dir model_dependencies/module_data_all \
213
+ --gene_list_path model_dependencies/selected_genes_27k.txt \
214
+ --gene_mapping_path model_dependencies/gene2ens.csv \
215
+ --gene2vec_data_path model_dependencies/selected_gene2vec_27k.npy \
216
+ --gene2ens_file model_dependencies/gene2ens.csv \
217
+ --data_path data/downstream/final_data/human_pancreas_norm_complexBatch_with_gene_name.h5ad \
218
+ --data_mode h5ad \
219
+ --gene_name_column auto \
220
+ --batch_size 640 \
221
+ --shuffle_module_genes \
222
+ --output_dir outputs/interpretability/geometry/pancreas_scib_shuffle
223
+ ```
224
+
225
+ For pretraining npz data:
226
+
227
+ ```bash
228
+ python BioMod/interpretability/module_geometry.py \
229
+ --performer_checkpoint weight/0810model/mae_tost_70M_75mask_ec2_epoch0.pth \
230
+ --config_pkl weight/0810model/mae_tost_70M_75mask_ec2.pkl \
231
+ --module_data_dir model_dependencies/module_data_all \
232
+ --gene_list_path model_dependencies/selected_genes_27k.txt \
233
+ --gene_mapping_path model_dependencies/gene2ens.csv \
234
+ --gene2vec_data_path model_dependencies/selected_gene2vec_27k.npy \
235
+ --gene2ens_file model_dependencies/gene2ens.csv \
236
+ --data_path /absolute/path/to/pretraining_cell_npz \
237
+ --data_mode pretrain_npz \
238
+ --max_cells 50000 \
239
+ --batch_size 640 \
240
+ --shuffle_module_genes \
241
+ --output_dir outputs/interpretability/geometry/pretrain_subset_shuffle
242
+ ```
243
+
244
+ How `--shuffle_module_genes` works: inside `load_backbone()`, module sizes are
245
+ kept unchanged, all genes are collected into one list, the list is randomly
246
+ shuffled, and genes are reassigned to modules. The seed is controlled by
247
+ `--shuffle_seed`, defaulting to 2021. Model weights are unchanged; only the
248
+ gene-to-module mapping is shuffled.
249
+
250
+ **Expected result:** kNN purity should drop to about 0.576, the null level, and
251
+ the inter-/intra-family distance ratio should drop to about 1.0. `metrics.json`
252
+ records `"shuffle_module_genes": true`.
253
+
254
+ ### Control 2: Untrained Backbone with Random Initialization
255
+
256
+ First generate a randomly initialized checkpoint, then run `module_geometry.py`
257
+ with it.
258
+
259
+ A matching randomly initialized checkpoint and configuration are not bundled.
260
+ Create them separately with the pretraining script's `--save_init_ckpt_and_exit`
261
+ option and architecture settings matching the BioMod checkpoint being compared.
262
+ Keep the generated control files in a separate output directory. Then pass their
263
+ paths explicitly; do not use the pretrained BioMod weights for this control.
264
+
265
+ ```bash
266
+ python BioMod/interpretability/module_geometry.py \
267
+ --performer_checkpoint /absolute/path/to/random_init.pth \
268
+ --config_pkl /absolute/path/to/random_init.pkl \
269
+ --data_path data/downstream/final_data/human_pancreas_norm_complexBatch_with_gene_name.h5ad \
270
+ --data_mode h5ad \
271
+ --gene_name_column auto \
272
+ --batch_size 8 \
273
+ --output_dir outputs/interpretability/geometry/pancreas_init
274
+ ```
275
+
276
+ **Expected result:** all family-level metrics should be close to the null
277
+ distribution: kNN purity about 0.576, distance ratio about 1.0, and p-value
278
+ above 0.05. This indicates that a randomly initialized encoder cannot separate
279
+ pathway families.
280
+
281
+ ### Result Comparison Matrix
282
+
283
+ After running all controls, construct the following table for the paper:
284
+
285
+ | Condition | kNN purity, cosine | Inter-/intra-family ratio, cosine | Conclusion |
286
+ | --- | --- | --- | --- |
287
+ | **Normal**, trained + pathway module | about 0.72, p<0.005 | 3.8-6.2, p<0.005 | Significant family structure |
288
+ | **Shuffle**, trained + random module | about 0.576 expected | about 1.0 expected | Structure is driven by pathway design |
289
+ | **Init**, untrained + pathway module | about 0.576 expected | about 1.0 expected | Structure is learned during pretraining |
290
+
291
+ The three experiments jointly show that the family structure of the module latent
292
+ space is generated by both pathway-based module design and pretraining; both are
293
+ necessary.
294
+
295
+ ## Experiment 2: Module Embedding to Pathway Text Alignment (`module_text_alignment/`)
296
+
297
+ **Hypothesis:** the structure of the BioMod module embedding space significantly
298
+ aligns with natural-language pathway semantics. Semantically similar pathways
299
+ are also encoded nearby by the model. Gene-level models cannot reproduce this
300
+ experiment because individual gene names do not carry phrase-level semantics.
301
+
302
+ **Method:**
303
+
304
+ 1. Parse 541 semantically named modules from `merged_modules.json`: 50 Hallmark and 491 Reactome.
305
+ 2. Encode pathway names with S-PubMedBert-MS-MARCO, a sentence-level fine-tuned PubMedBERT model.
306
+ 3. Load `module_embeddings.npy` from Experiment 1, containing cell-averaged module embeddings.
307
+ 4. Measure alignment:
308
+ - **RSA / Mantel test**, the main metric: Spearman correlation between paired distance matrices from the two spaces, with a permutation test.
309
+ - **CKA**: global representation-structure alignment.
310
+ - **LOO Ridge retrieval**: cross-modal retrieval recall@k.
311
+
312
+ **Data requirement:** only `module_embeddings.npy` from Experiment 1 is needed;
313
+ raw expression data are not required.
314
+
315
+ ### Commands
316
+
317
+ ```bash
318
+ # Run from the BioMod repository root.
319
+
320
+ # Download the sentence encoder.
321
+ # The sentence-transformers loader downloads this model on first use.
322
+ # Network access or an existing local model cache is required.
323
+
324
+ # Main experiment: RSA + CKA + LOO retrieval.
325
+ python BioMod/interpretability/module_text_alignment/module_text_alignment.py \
326
+ --module_embedding_path outputs/interpretability/geometry/pretrain_subset/module_embeddings.npy \
327
+ --module_json model_dependencies/module_data_all/merged_modules.json \
328
+ --sentence_model pritamdeka/S-PubMedBert-MS-MARCO \
329
+ --n_permutations 1000 \
330
+ --output_dir outputs/interpretability/text_alignment/pretrain_subset_v2
331
+
332
+ # Extended analysis: broad Reactome categories and Hallmark deep dive.
333
+ python BioMod/interpretability/module_text_alignment/module_text_alignment_extended.py \
334
+ --module_embedding_path outputs/interpretability/geometry/pretrain_subset/module_embeddings.npy \
335
+ --module_json model_dependencies/module_data_all/merged_modules.json \
336
+ --sentence_model pritamdeka/S-PubMedBert-MS-MARCO \
337
+ --descriptions_file BioMod/interpretability/module_text_alignment/reactome_descriptions.json \
338
+ --n_permutations 1000 \
339
+ --output_dir outputs/interpretability/text_alignment/pretrain_subset_v2_extended
340
+ ```
341
+
342
+ ### Output Files
343
+
344
+ | File | Content |
345
+ | --- | --- |
346
+ | `alignment_metrics.json` | RSA, CKA, retrieval recall@k, and p-values |
347
+ | `rsa_scatter.png` | Paired-distance scatter plot, module space vs text space |
348
+ | `rsa_null_distribution.png` | Mantel test: observed value vs null distribution |
349
+ | `retrieval_ranks.png` | Histogram of retrieval ranks |
350
+ | `retrieval_examples.csv` | Top-5 retrieval results for representative pathways |
351
+ | `extended_metrics.json` | Extended-analysis results: broad-category RSA and Hallmark deep dive |
352
+ | `reactome_category_heatmaps.png` | Distance-matrix comparison at the broad Reactome category level |
353
+ | `hallmarks_deep_dive.png` | Hallmark per-module alignment scores and similarity matrix |
354
+ | `hallmarks_per_module_alignment.csv` | Per-module alignment ranking for 50 Hallmarks |
355
+
356
+ ## Dependencies
357
+
358
+ `torch`, `numpy`, `pandas`, `scikit-learn`, `matplotlib`, `anndata`, and `scipy`.
359
+ UMAP plots additionally require `umap-learn`; if it is not installed, the script
360
+ warns and skips only UMAP plots. Experiment 2 also requires
361
+ `sentence-transformers`, installable with `pip install sentence-transformers`.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
BioMod/interpretability/attention/README.md CHANGED
@@ -1,108 +1,111 @@
1
- # Experiment 6: Module Crosstalk Graph vs Pathway Interaction Database
2
-
3
- This experiment tests whether module-module interactions learned by the BioMod
4
- encoder are consistent with known pathway-pathway relationships.
5
-
6
- ## Methods
7
-
8
- | Method | Principle | Compute cost |
9
- | --- | --- | --- |
10
- | A, cosine | Cosine similarity between cell-averaged module embeddings | Low |
11
- | C, jacobian | Cell-averaged norm of `d encoder_out[j] / d module_emb[i]` | High |
12
-
13
- Ground truth: Jaccard index and overlap coefficient of shared genes between
14
- modules.
15
-
16
- ## Evaluation Metrics
17
-
18
- - Spearman correlation between the model interaction matrix and database overlap matrix.
19
- - AUROC, using database edges as positives and model scores as predictions.
20
- - Precision@k, the fraction of top-k model interactions that exist in the database.
21
- - p-value from a permutation null distribution.
22
-
23
- ## Run
24
-
25
- ```bash
26
- cd /path/to/performer_module/interpretability/attention
27
-
28
- # Full run, methods A + C.
29
- python module_crosstalk.py \
30
- --performer_checkpoint ../../mae_tost_48M_cross_attn_only_pure_memory/mae_tost_48M_cross_attn_only_pure_memory_epoch12.pth \
31
- --config_pkl ../../mae_tost_48M_cross_attn_only_pure_memory/mae_tost_48M_cross_attn_only_pure_memory.pkl \
32
- --module_data_dir ../../module_data_all \
33
- --gene_list_path ../../../selected_genes_27k.txt \
34
- --gene_mapping_path ../../../gene2ens.csv \
35
- --gene2vec_data_path ../../../selected_gene2vec_27k.npy \
36
- --gene2ens_file ../../../gene2ens.csv \
37
- --data_path ../../../zero-shot-batch/pancreas_scib.h5ad \
38
- --data_mode h5ad \
39
- --batch_size 640 \
40
- --jacobian_cells 200 \
41
- --output_dir ../results/crosstalk/pancreas_max \
42
- --pool_mode max
43
-
44
- # Method A only, for quick validation.
45
- python module_crosstalk.py \
46
- --performer_checkpoint ../../mae_tost_48M_cross_attn_only_pure_memory/mae_tost_48M_cross_attn_only_pure_memory_epoch12.pth \
47
- --config_pkl ../../mae_tost_48M_cross_attn_only_pure_memory/mae_tost_48M_cross_attn_only_pure_memory.pkl \
48
- --module_data_dir ../../module_data_all \
49
- --gene_list_path ../../../selected_genes_27k.txt \
50
- --gene_mapping_path ../../../gene2ens.csv \
51
- --gene2vec_data_path ../../../selected_gene2vec_27k.npy \
52
- --gene2ens_file ../../../gene2ens.csv \
53
- --data_path ../../../zero-shot-batch/pancreas_scib.h5ad \
54
- --data_mode h5ad \
55
- --batch_size 64 \
56
- --skip_jacobian \
57
- --output_dir ../results/crosstalk/pancreas_cosine_only
58
-
59
- # Pretraining npz data.
60
- python module_crosstalk.py \
61
- --performer_checkpoint ../../mae_tost_48M_cross_attn_only_pure_memory/mae_tost_48M_cross_attn_only_pure_memory_epoch12.pth \
62
- --config_pkl ../../mae_tost_48M_cross_attn_only_pure_memory/mae_tost_48M_cross_attn_only_pure_memory.pkl \
63
- --module_data_dir ../../module_data_all \
64
- --gene_list_path ../../../selected_genes_27k.txt \
65
- --gene_mapping_path ../../../gene2ens.csv \
66
- --gene2vec_data_path ../../../selected_gene2vec_27k.npy \
67
- --gene2ens_file ../../../gene2ens.csv \
68
- --data_path ../../../../share/cell_files \
69
- --data_mode pretrain_npz \
70
- --cell_num 49738024 \
71
- --max_cells 5000 \
72
- --batch_size 16 \
73
- --jacobian_cells 100 \
74
- --output_dir ../results/crosstalk/pretrain
75
- ```
76
-
77
- ## Key Arguments
78
-
79
- | Argument | Default | Description |
80
- | --- | --- | --- |
81
- | `--jacobian_cells` | 200 | Number of cells sampled for Jacobian computation; more is stabler but slower |
82
- | `--skip_jacobian` | off | Run method A only |
83
- | `--max_cells` | 5000 | Maximum cell count used by method A |
84
- | `--null_permutations` | 200 | Number of permutations |
85
- | `--top_k_fractions` | 0.01,0.05,0.1 | Fractions used for Precision@k |
86
-
87
- ## Outputs
88
-
89
- ```text
90
- output_dir/
91
- |-- cosine_interaction.npy # (660, 660) method A interaction matrix
92
- |-- jacobian_interaction.npy # (660, 660) method C interaction matrix
93
- |-- mean_module_embeddings.npy # (660, 1024) cell-averaged module embeddings
94
- |-- gt_jaccard.npy # (660, 660) gene Jaccard overlap
95
- |-- gt_overlap_coef.npy # (660, 660) overlap coefficient
96
- |-- metrics.json # All evaluation metrics and p-values
97
- |-- heatmap_cosine.png # Method A heatmap, sorted by family
98
- |-- heatmap_jacobian.png # Method C heatmap
99
- |-- heatmap_gt_jaccard.png # Ground-truth heatmap
100
- |-- scatter_cosine_vs_jaccard.png
101
- `-- scatter_jacobian_vs_jaccard.png
102
- ```
103
-
104
- ## Notes
105
-
106
- - Jacobian computation requires 660 backward passes per cell. `--jacobian_cells 200` takes about 10-30 minutes on one A100.
107
- - Method A has almost no extra overhead, so use `--skip_jacobian` for quick validation first.
108
- - The current model uses TSSA attention rather than standard softmax attention, so attention weights cannot be read directly. Jacobian is used as a substitute.
 
 
 
 
1
+ # Experiment 6: Module Crosstalk Graph vs Pathway Interaction Database
2
+
3
+ This experiment tests whether module-module interactions learned by the BioMod
4
+ encoder are consistent with known pathway-pathway relationships.
5
+
6
+ ## Methods
7
+
8
+ | Method | Principle | Compute cost |
9
+ | --- | --- | --- |
10
+ | A, cosine | Cosine similarity between cell-averaged module embeddings | Low |
11
+ | C, jacobian | Cell-averaged norm of `d encoder_out[j] / d module_emb[i]` | High |
12
+
13
+ Ground truth: Jaccard index and overlap coefficient of shared genes between
14
+ modules.
15
+
16
+ ## Evaluation Metrics
17
+
18
+ - Spearman correlation between the model interaction matrix and database overlap matrix.
19
+ - AUROC, using database edges as positives and model scores as predictions.
20
+ - Precision@k, the fraction of top-k model interactions that exist in the database.
21
+ - p-value from a permutation null distribution.
22
+
23
+ ## Run
24
+
25
+ Run from the BioMod repository root. The h5ad examples use the bundled pancreas
26
+ dataset and BioMod model weights. The optional pretraining `.npz` dataset is not
27
+ included; replace `/absolute/path/to/pretraining_cell_npz` with your own directory.
28
+
29
+ ```bash
30
+ # Run from the BioMod repository root.
31
+
32
+ # Full run, methods A + C.
33
+ python BioMod/interpretability/attention/module_crosstalk.py \
34
+ --performer_checkpoint weight/0810model/mae_tost_70M_75mask_ec2_epoch0.pth \
35
+ --config_pkl weight/0810model/mae_tost_70M_75mask_ec2.pkl \
36
+ --module_data_dir model_dependencies/module_data_all \
37
+ --gene_list_path model_dependencies/selected_genes_27k.txt \
38
+ --gene_mapping_path model_dependencies/gene2ens.csv \
39
+ --gene2vec_data_path model_dependencies/selected_gene2vec_27k.npy \
40
+ --gene2ens_file model_dependencies/gene2ens.csv \
41
+ --data_path data/downstream/final_data/human_pancreas_norm_complexBatch_with_gene_name.h5ad \
42
+ --data_mode h5ad \
43
+ --batch_size 640 \
44
+ --jacobian_cells 200 \
45
+ --output_dir outputs/interpretability/crosstalk/pancreas_max \
46
+ --pool_mode max
47
+
48
+ # Method A only, for quick validation.
49
+ python BioMod/interpretability/attention/module_crosstalk.py \
50
+ --performer_checkpoint weight/0810model/mae_tost_70M_75mask_ec2_epoch0.pth \
51
+ --config_pkl weight/0810model/mae_tost_70M_75mask_ec2.pkl \
52
+ --module_data_dir model_dependencies/module_data_all \
53
+ --gene_list_path model_dependencies/selected_genes_27k.txt \
54
+ --gene_mapping_path model_dependencies/gene2ens.csv \
55
+ --gene2vec_data_path model_dependencies/selected_gene2vec_27k.npy \
56
+ --gene2ens_file model_dependencies/gene2ens.csv \
57
+ --data_path data/downstream/final_data/human_pancreas_norm_complexBatch_with_gene_name.h5ad \
58
+ --data_mode h5ad \
59
+ --batch_size 64 \
60
+ --skip_jacobian \
61
+ --output_dir outputs/interpretability/crosstalk/pancreas_cosine_only
62
+
63
+ # Pretraining npz data.
64
+ python BioMod/interpretability/attention/module_crosstalk.py \
65
+ --performer_checkpoint weight/0810model/mae_tost_70M_75mask_ec2_epoch0.pth \
66
+ --config_pkl weight/0810model/mae_tost_70M_75mask_ec2.pkl \
67
+ --module_data_dir model_dependencies/module_data_all \
68
+ --gene_list_path model_dependencies/selected_genes_27k.txt \
69
+ --gene_mapping_path model_dependencies/gene2ens.csv \
70
+ --gene2vec_data_path model_dependencies/selected_gene2vec_27k.npy \
71
+ --gene2ens_file model_dependencies/gene2ens.csv \
72
+ --data_path /absolute/path/to/pretraining_cell_npz \
73
+ --data_mode pretrain_npz \
74
+ --max_cells 5000 \
75
+ --batch_size 16 \
76
+ --jacobian_cells 100 \
77
+ --output_dir outputs/interpretability/crosstalk/pretrain
78
+ ```
79
+
80
+ ## Key Arguments
81
+
82
+ | Argument | Default | Description |
83
+ | --- | --- | --- |
84
+ | `--jacobian_cells` | 200 | Number of cells sampled for Jacobian computation; more is stabler but slower |
85
+ | `--skip_jacobian` | off | Run method A only |
86
+ | `--max_cells` | 5000 | Maximum cell count used by method A |
87
+ | `--null_permutations` | 200 | Number of permutations |
88
+ | `--top_k_fractions` | 0.01,0.05,0.1 | Fractions used for Precision@k |
89
+
90
+ ## Outputs
91
+
92
+ ```text
93
+ output_dir/
94
+ |-- cosine_interaction.npy # (660, 660) method A interaction matrix
95
+ |-- jacobian_interaction.npy # (660, 660) method C interaction matrix
96
+ |-- mean_module_embeddings.npy # (660, 1024) cell-averaged module embeddings
97
+ |-- gt_jaccard.npy # (660, 660) gene Jaccard overlap
98
+ |-- gt_overlap_coef.npy # (660, 660) overlap coefficient
99
+ |-- metrics.json # All evaluation metrics and p-values
100
+ |-- heatmap_cosine.png # Method A heatmap, sorted by family
101
+ |-- heatmap_jacobian.png # Method C heatmap
102
+ |-- heatmap_gt_jaccard.png # Ground-truth heatmap
103
+ |-- scatter_cosine_vs_jaccard.png
104
+ `-- scatter_jacobian_vs_jaccard.png
105
+ ```
106
+
107
+ ## Notes
108
+
109
+ - Jacobian computation requires 660 backward passes per cell. `--jacobian_cells 200` takes about 10-30 minutes on one A100.
110
+ - Method A has almost no extra overhead, so use `--skip_jacobian` for quick validation first.
111
+ - The current model uses TSSA attention rather than standard softmax attention, so attention weights cannot be read directly. Jacobian is used as a substitute.
BioMod/interpretability/attention/README_gene_attention.md CHANGED
@@ -1,114 +1,112 @@
1
- # Gene Attention Interpretability: ModuleEmbedding
2
-
3
- Analyze gene contributions within modules using the content-based attention
4
- pooling in ModuleEmbedding.
5
-
6
- ## Analyses
7
-
8
- 1. Global gene-importance ranking, averaged attention over all cells.
9
- 2. Cell-type specificity with Mann-Whitney U test and fold change.
10
- 3. Marker gene validation with precision@k and recall@k.
11
- 4. Pathway enrichment analysis with GO BP, requiring `gseapy`.
12
- 5. Comparison with DEG results using Spearman correlation.
13
-
14
- ## Dependencies
15
-
16
- ```bash
17
- pip install torch numpy pandas scipy matplotlib seaborn anndata
18
- # Optional
19
- pip install gseapy # Pathway enrichment analysis
20
- ```
21
-
22
- ## Run Commands
23
-
24
- ### Basic Usage on the Pancreas Dataset with Built-In Markers
25
-
26
- ```bash
27
- cd ../data/250010166/scLong/performer_module/interpretability/attention
28
-
29
- python module_gene_attention.py \
30
- --performer_checkpoint ../../mae_tost_48M_cross_attn_only_pure_memory/mae_tost_48M_cross_attn_only_pure_memory_epoch12.pth \
31
- --config_pkl ../../mae_tost_48M_cross_attn_only_pure_memory/mae_tost_48M_cross_attn_only_pure_memory.pkl \
32
- --module_data_dir ../../module_data_all \
33
- --gene_list_path ../../../selected_genes_27k.txt \
34
- --gene_mapping_path ../../../gene2ens.csv \
35
- --gene2vec_data_path ../../../selected_gene2vec_27k.npy \
36
- --gene2ens_file ../../../gene2ens.csv \
37
- --data_path ../../../zero-shot-batch/pancreas_scib.h5ad \
38
- --celltype_column celltype \
39
- --batch_size 640 \
40
- --top_k_genes 20 \
41
- --output_dir ../results/gene_attention/pancreas
42
- ```
43
-
44
- ### With DEG Comparison
45
-
46
- ```bash
47
- python module_gene_attention.py \
48
- --performer_checkpoint ../../mae_tost_48M_cross_attn_only_pure_memory/mae_tost_48M_cross_attn_only_pure_memory_epoch12.pth \
49
- --config_pkl ../../mae_tost_48M_cross_attn_only_pure_memory/mae_tost_48M_cross_attn_only_pure_memory.pkl \
50
- --module_data_dir ../../module_data_all \
51
- --gene_list_path ../../../selected_genes_27k.txt \
52
- --gene_mapping_path ../../../gene2ens.csv \
53
- --gene2vec_data_path ../../../selected_gene2vec_27k.npy \
54
- --gene2ens_file ../../../gene2ens.csv \
55
- --data_path ../../../zero-shot-batch/pancreas_scib.h5ad \
56
- --celltype_column celltype \
57
- --batch_size 64 \
58
- --deg_csv /path/to/deg_results.csv \
59
- --output_dir ../results/gene_attention/pancreas_with_deg
60
- ```
61
-
62
- DEG CSV columns must include `gene`, `cell_type`, `log2fc`, and `pval`.
63
-
64
- ### Custom Marker Database
65
-
66
- ```bash
67
- python module_gene_attention.py \
68
- ... \
69
- --marker_db /path/to/markers.csv \
70
- --output_dir ../results/gene_attention/custom_markers
71
- ```
72
-
73
- Marker CSV columns must include `cell_type` and `gene_symbol`.
74
-
75
- ### Limit Cell Count for Faster Testing
76
-
77
- ```bash
78
- python module_gene_attention.py \
79
- ... \
80
- --max_cells 5000 \
81
- --output_dir ../results/gene_attention/pancreas_5k
82
- ```
83
-
84
- ## Key Arguments
85
-
86
- | Argument | Default | Description |
87
- | --- | --- | --- |
88
- | `--batch_size` | 64 | Inference batch size |
89
- | `--top_k_genes` | 20 | Number of top genes shown for each cell type in visualizations |
90
- | `--max_cells` | None | Number of cells to subsample |
91
- | `--marker_db` | None | Custom marker CSV; built-in pancreas markers are used when omitted |
92
- | `--deg_csv` | None | DEG result CSV; DEG correlation analysis is skipped when omitted |
93
- | `--celltype_column` | celltype | Cell-type column name in h5ad |
94
- | `--shuffle_module_genes` | off | Randomly shuffle gene-to-module assignment as a negative control |
95
-
96
- ## Output Files
97
-
98
- ```text
99
- output_dir/
100
- |-- global_gene_importance.csv # Global gene importance across all modules and genes
101
- |-- global_top_genes.png # Top-N gene bar chart
102
- |-- celltype_<ct>_attention.csv # Gene attention for each cell type
103
- |-- celltype_attention_heatmap.png # Cell type x gene heatmap
104
- |-- gene_specificity.csv # Gene specificity with fold change and p-value
105
- |-- specificity_volcano.png # Volcano plot
106
- |-- marker_validation.json # Marker validation results
107
- |-- marker_recall.png # Recall@50/100 bar chart
108
- |-- deg_correlation.json # DEG correlation, if --deg_csv is provided
109
- |-- deg_correlation.png # Spearman rho bar chart
110
- |-- enrichment/ # Pathway enrichment, if gseapy is installed
111
- | |-- <ct>_enrichment.csv
112
- | `-- enrichment_summary.json
113
- `-- summary.json # Run summary
114
- ```
 
1
+ # Gene Attention Interpretability: ModuleEmbedding
2
+
3
+ Analyze gene contributions within modules using the content-based attention
4
+ pooling in ModuleEmbedding.
5
+
6
+ ## Analyses
7
+
8
+ 1. Global gene-importance ranking, averaged attention over all cells.
9
+ 2. Cell-type specificity with Mann-Whitney U test and fold change.
10
+ 3. Marker gene validation with precision@k and recall@k.
11
+ 4. Pathway enrichment analysis with GO BP, requiring `gseapy`.
12
+ 5. Comparison with DEG results using Spearman correlation.
13
+
14
+ ## Dependencies
15
+
16
+ ```bash
17
+ pip install torch numpy pandas scipy matplotlib seaborn anndata
18
+ # Optional
19
+ pip install gseapy # Pathway enrichment analysis
20
+ ```
21
+
22
+ ## Run Commands
23
+
24
+ Run from the BioMod repository root. The examples use bundled BioMod model
25
+ weights and pancreas data. Custom DEG and marker CSV files are optional external
26
+ inputs; replace their example paths with your own files before using those flags.
27
+ Set `CELL_TYPE_COLUMN` below to the actual cell-type field in your h5ad metadata;
28
+ the column name has not been assumed for the bundled pancreas file.
29
+
30
+ ### Basic Usage on the Pancreas Dataset with Built-In Markers
31
+
32
+ ```bash
33
+ # Run from the BioMod repository root.
34
+
35
+ python BioMod/interpretability/attention/module_gene_attention.py \
36
+ --performer_checkpoint weight/0810model/mae_tost_70M_75mask_ec2_epoch0.pth \
37
+ --config_pkl weight/0810model/mae_tost_70M_75mask_ec2.pkl \
38
+ --module_data_dir model_dependencies/module_data_all \
39
+ --gene_list_path model_dependencies/selected_genes_27k.txt \
40
+ --gene_mapping_path model_dependencies/gene2ens.csv \
41
+ --gene2vec_data_path model_dependencies/selected_gene2vec_27k.npy \
42
+ --gene2ens_file model_dependencies/gene2ens.csv \
43
+ --data_path data/downstream/final_data/human_pancreas_norm_complexBatch_with_gene_name.h5ad \
44
+ --celltype_column CELL_TYPE_COLUMN \
45
+ --batch_size 640 \
46
+ --top_k_genes 20 \
47
+ --output_dir outputs/interpretability/gene_attention/pancreas
48
+ ```
49
+
50
+ ### With DEG Comparison
51
+
52
+ ```bash
53
+ python BioMod/interpretability/attention/module_gene_attention.py \
54
+ --performer_checkpoint weight/0810model/mae_tost_70M_75mask_ec2_epoch0.pth \
55
+ --config_pkl weight/0810model/mae_tost_70M_75mask_ec2.pkl \
56
+ --module_data_dir model_dependencies/module_data_all \
57
+ --gene_list_path model_dependencies/selected_genes_27k.txt \
58
+ --gene_mapping_path model_dependencies/gene2ens.csv \
59
+ --gene2vec_data_path model_dependencies/selected_gene2vec_27k.npy \
60
+ --gene2ens_file model_dependencies/gene2ens.csv \
61
+ --data_path data/downstream/final_data/human_pancreas_norm_complexBatch_with_gene_name.h5ad \
62
+ --celltype_column CELL_TYPE_COLUMN \
63
+ --batch_size 64 \
64
+ --deg_csv /path/to/deg_results.csv \
65
+ --output_dir outputs/interpretability/gene_attention/pancreas_with_deg
66
+ ```
67
+
68
+ DEG CSV columns must include `gene`, `cell_type`, `log2fc`, and `pval`.
69
+
70
+ ### Custom Marker Database
71
+
72
+ Add `--marker_db /path/to/markers.csv` to the complete basic command above and
73
+ change `--output_dir` to `outputs/interpretability/gene_attention/custom_markers`.
74
+
75
+ Marker CSV columns must include `cell_type` and `gene_symbol`.
76
+
77
+ ### Limit Cell Count for Faster Testing
78
+
79
+ Add `--max_cells 5000` to the complete basic command above and change
80
+ `--output_dir` to `outputs/interpretability/gene_attention/pancreas_5k`.
81
+
82
+ ## Key Arguments
83
+
84
+ | Argument | Default | Description |
85
+ | --- | --- | --- |
86
+ | `--batch_size` | 64 | Inference batch size |
87
+ | `--top_k_genes` | 20 | Number of top genes shown for each cell type in visualizations |
88
+ | `--max_cells` | None | Number of cells to subsample |
89
+ | `--marker_db` | None | Custom marker CSV; built-in pancreas markers are used when omitted |
90
+ | `--deg_csv` | None | DEG result CSV; DEG correlation analysis is skipped when omitted |
91
+ | `--celltype_column` | celltype | Cell-type column name in h5ad |
92
+ | `--shuffle_module_genes` | off | Randomly shuffle gene-to-module assignment as a negative control |
93
+
94
+ ## Output Files
95
+
96
+ ```text
97
+ output_dir/
98
+ |-- global_gene_importance.csv # Global gene importance across all modules and genes
99
+ |-- global_top_genes.png # Top-N gene bar chart
100
+ |-- celltype_<ct>_attention.csv # Gene attention for each cell type
101
+ |-- celltype_attention_heatmap.png # Cell type x gene heatmap
102
+ |-- gene_specificity.csv # Gene specificity with fold change and p-value
103
+ |-- specificity_volcano.png # Volcano plot
104
+ |-- marker_validation.json # Marker validation results
105
+ |-- marker_recall.png # Recall@50/100 bar chart
106
+ |-- deg_correlation.json # DEG correlation, if --deg_csv is provided
107
+ |-- deg_correlation.png # Spearman rho bar chart
108
+ |-- enrichment/ # Pathway enrichment, if gseapy is installed
109
+ | |-- <ct>_enrichment.csv
110
+ | `-- enrichment_summary.json
111
+ `-- summary.json # Run summary
112
+ ```
 
 
BioMod/interpretability/attention/module_crosstalk.py CHANGED
@@ -1,605 +1,606 @@
1
- """Experiment 6 — Module crosstalk graph vs Reactome/KEGG pathway interactions.
2
-
3
- Two complementary methods to quantify module-to-module interaction strength:
4
-
5
- Method A (cosine similarity): Compute cosine similarity between cell-averaged
6
- module encoder outputs. Fast, no gradient needed.
7
-
8
- Method C (Jacobian): Compute ||d encoder_out[j] / d module_input[i]|| averaged
9
- over cells. Measures causal influence of module i on module j's representation.
10
- Requires gradient; more expensive but more principled.
11
-
12
- Both interaction matrices are compared against known pathway-pathway relationships
13
- from Reactome (shared genes / hierarchical overlap) and optionally KEGG.
14
-
15
- Metrics:
16
- - Spearman correlation between model interaction matrix and DB overlap matrix
17
- - Precision@k: among top-k model interactions, fraction present in DB
18
- - AUROC: treating DB edges as positives, model scores as predictions
19
- - Permutation null p-values for all metrics
20
-
21
- Usage:
22
- python module_crosstalk.py \
23
- --performer_checkpoint ../mae_tost_48M_cross_attn_only_pure_memory/mae_tost_48M_cross_attn_only_pure_memory_epoch12.pth \
24
- --config_pkl ../mae_tost_48M_cross_attn_only_pure_memory/mae_tost_48M_cross_attn_only_pure_memory.pkl \
25
- --module_data_dir ../module_data_all \
26
- --gene_list_path ../../selected_genes_27k.txt \
27
- --gene_mapping_path ../../gene2ens.csv \
28
- --gene2vec_data_path ../../selected_gene2vec_27k.npy \
29
- --data_path ../../zero-shot-batch/pancreas_scib.h5ad \
30
- --data_mode h5ad \
31
- --batch_size 16 \
32
- --jacobian_cells 200 \
33
- --output_dir ./results/crosstalk/pancreas
34
- """
35
- from __future__ import annotations
36
-
37
- import argparse
38
- import json
39
- import logging
40
- import sys
41
- from pathlib import Path
42
- from typing import Dict, List, Optional, Tuple
43
-
44
- import numpy as np
45
- import torch
46
- from torch.utils.data import DataLoader, Subset
47
-
48
- import matplotlib
49
- matplotlib.use("Agg")
50
- import matplotlib.pyplot as plt
51
-
52
- from scipy.stats import spearmanr
53
- from sklearn.metrics import roc_auc_score
54
-
55
- HERE = Path(__file__).resolve()
56
- PERFORMER_MODULE = HERE.parents[2]
57
- REPO_ROOT = HERE.parents[3]
58
- for p in (str(PERFORMER_MODULE), str(REPO_ROOT)):
59
- if p not in sys.path:
60
- sys.path.insert(0, p)
61
-
62
- from cell_annotation.performer_module_finetune import (
63
- DataPreprocessor,
64
- load_backbone,
65
- seed_all,
66
- )
67
-
68
- from interpretability.utils import (
69
- MatrixRowDataset,
70
- NpzCellDataset,
71
- build_module_metadata,
72
- ensure_dir,
73
- load_module_names,
74
- )
75
-
76
- logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
77
- logger = logging.getLogger(__name__)
78
-
79
- # ---------------------------------------------------------------------------
80
- # Argparse
81
- # ---------------------------------------------------------------------------
82
-
83
- def parse_args() -> argparse.Namespace:
84
- p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
85
-
86
- p.add_argument("--performer_checkpoint", type=str, required=True)
87
- p.add_argument("--config_pkl", type=str, required=True)
88
- p.add_argument("--module_data_dir", type=str, default="../module_data_all")
89
- p.add_argument("--module_json", type=str, default=None)
90
- p.add_argument("--gene_list_path", type=str, default="../../selected_genes_27k.txt")
91
- p.add_argument("--gene_mapping_path", type=str, default="../../gene2ens.csv")
92
- p.add_argument("--gene2vec_data_path", type=str, default="../../selected_gene2vec_27k.npy")
93
- p.add_argument("--gene2ens_file", type=str, default="../../gene2ens.csv")
94
- p.add_argument("--gene2ens_column", type=str, default="Ensembl gene ID")
95
- p.add_argument("--shuffle_module_genes", action="store_true")
96
- p.add_argument("--shuffle_seed", type=int, default=2021)
97
-
98
- p.add_argument("--data_path", type=str, required=True)
99
- p.add_argument("--data_mode", type=str, default="h5ad",
100
- choices=["h5ad", "pretrain_h5ad", "pretrain_npz"])
101
- p.add_argument("--cell_num", type=int, default=None)
102
- p.add_argument("--gene_name_column", type=str, default="auto")
103
- p.add_argument("--dataset_name", type=str, default="crosstalk")
104
- p.add_argument("--max_cells", type=int, default=5000,
105
- help="Max cells for method A (cosine). Method C uses --jacobian_cells.")
106
-
107
- p.add_argument("--batch_size", type=int, default=16)
108
- p.add_argument("--num_workers", type=int, default=0)
109
- p.add_argument("--device", type=str, default="cuda")
110
- p.add_argument("--seed", type=int, default=2021)
111
- p.add_argument("--pool_mode", type=str, default="mean",
112
- choices=["mean", "max"],
113
- help="Pooling over cells: 'mean' or 'max' (element-wise max).")
114
-
115
- p.add_argument("--jacobian_cells", type=int, default=200,
116
- help="Number of cells for Jacobian computation (expensive).")
117
- p.add_argument("--jacobian_batch", type=int, default=4,
118
- help="Cells per Jacobian forward-backward pass.")
119
- p.add_argument("--skip_jacobian", action="store_true",
120
- help="Only run method A (cosine), skip method C (Jacobian).")
121
-
122
- p.add_argument("--null_permutations", type=int, default=200)
123
- p.add_argument("--top_k_fractions", type=str, default="0.01,0.05,0.1",
124
- help="Comma-separated fractions for precision@k.")
125
-
126
- p.add_argument("--output_dir", type=str, required=True)
127
- return p.parse_args()
128
-
129
-
130
- # ---------------------------------------------------------------------------
131
- # Data loading (reuses patterns from module_geometry.py)
132
- # ---------------------------------------------------------------------------
133
-
134
- def build_dataset(args, preprocessor):
135
- """Build dataset and optional reindex_state."""
136
- if args.data_mode == "h5ad":
137
- import anndata as ad
138
- adata, reindex_state = preprocessor.preprocess_adata(
139
- adata_path=args.data_path,
140
- min_genes=0, min_cells=0,
141
- max_cells=args.max_cells,
142
- dataset_name=args.dataset_name,
143
- gene_name_column=args.gene_name_column,
144
- )
145
- dataset = MatrixRowDataset(adata.X)
146
- n_cells = adata.n_obs
147
- elif args.data_mode == "pretrain_h5ad":
148
- import anndata as ad
149
- adata = ad.read_h5ad(args.data_path)
150
- if args.max_cells and adata.n_obs > args.max_cells:
151
- rng = np.random.default_rng(args.seed)
152
- chosen = np.sort(rng.choice(adata.n_obs, size=args.max_cells, replace=False))
153
- adata = adata[chosen].copy()
154
- dataset = MatrixRowDataset(adata.X)
155
- n_cells = adata.n_obs
156
- reindex_state = None
157
- elif args.data_mode == "pretrain_npz":
158
- dataset = NpzCellDataset(
159
- data_dir=args.data_path,
160
- cell_num=args.cell_num,
161
- max_cells=args.max_cells,
162
- seed=args.seed,
163
- )
164
- n_cells = len(dataset)
165
- reindex_state = None
166
- else:
167
- raise ValueError(f"Unsupported data_mode: {args.data_mode}")
168
- return dataset, reindex_state, n_cells
169
-
170
-
171
- # ---------------------------------------------------------------------------
172
- # Method A: Cosine similarity of cell-averaged module embeddings
173
- # ---------------------------------------------------------------------------
174
-
175
- @torch.no_grad()
176
- def compute_cosine_interaction(backbone, loader, preprocessor, reindex_state,
177
- device, num_modules, base_dim, expected_genes,
178
- pool_mode="mean"):
179
- """Compute pooled module embeddings, return cosine similarity matrix."""
180
- backbone.eval()
181
- if pool_mode == "mean":
182
- agg_emb = torch.zeros(num_modules, base_dim, device=device, dtype=torch.float64)
183
- else:
184
- agg_emb = torch.full((num_modules, base_dim), -float("inf"), device=device, dtype=torch.float64)
185
- n_cells = 0
186
-
187
- for batch_idx, x in enumerate(loader):
188
- x = x.to(device)
189
- if reindex_state is not None:
190
- x = preprocessor.reindex_tensor_to_model_space(x, reindex_state=reindex_state)
191
- out = backbone(x, mask_ratio=0.0, return_encodings=True)
192
- enc = out["module_encodings"].to(torch.float64) # (B, num_modules, D)
193
- if pool_mode == "mean":
194
- agg_emb += enc.sum(dim=0)
195
- else:
196
- batch_max = enc.max(dim=0).values # (num_modules, D)
197
- agg_emb = torch.maximum(agg_emb, batch_max)
198
- n_cells += enc.shape[0]
199
- if (batch_idx + 1) % 50 == 0:
200
- logger.info(" [cosine/%s] batch %d, %d cells", pool_mode, batch_idx + 1, n_cells)
201
-
202
- if pool_mode == "mean":
203
- pooled_emb = (agg_emb / max(n_cells, 1)).float().cpu().numpy()
204
- else:
205
- pooled_emb = agg_emb.float().cpu().numpy()
206
-
207
- norms = np.linalg.norm(pooled_emb, axis=1, keepdims=True)
208
- norms = np.clip(norms, 1e-8, None)
209
- normed = pooled_emb / norms
210
- cos_sim = normed @ normed.T
211
- np.fill_diagonal(cos_sim, 0.0)
212
- logger.info("Method A (%s) done: %d cells, interaction matrix shape %s", pool_mode, n_cells, cos_sim.shape)
213
- return cos_sim, pooled_emb
214
-
215
-
216
- # ---------------------------------------------------------------------------
217
- # Method C: Jacobian-based module interaction
218
- # ---------------------------------------------------------------------------
219
-
220
- def compute_jacobian_interaction(backbone, loader, preprocessor, reindex_state,
221
- device, num_modules, base_dim, expected_genes,
222
- max_cells, batch_size):
223
- """Compute average ||d encoder_out[j] / d module_emb[i]|| interaction matrix."""
224
- backbone.eval()
225
- for param in backbone.parameters():
226
- param.requires_grad_(False)
227
-
228
- interaction = torch.zeros(num_modules, num_modules, device=device, dtype=torch.float64)
229
- n_cells = 0
230
-
231
- for x in loader:
232
- if n_cells >= max_cells:
233
- break
234
- x = x.to(device)
235
- if reindex_state is not None:
236
- x = preprocessor.reindex_tensor_to_model_space(x, reindex_state=reindex_state)
237
-
238
- actual_batch = min(x.shape[0], max_cells - n_cells)
239
- x = x[:actual_batch]
240
-
241
- for cell_idx in range(x.shape[0]):
242
- cell = x[cell_idx:cell_idx+1]
243
- J = _compute_single_cell_jacobian(backbone, cell, num_modules, base_dim, device)
244
- interaction += J
245
- n_cells += 1
246
- if n_cells % 20 == 0:
247
- logger.info(" [jacobian] %d / %d cells", n_cells, max_cells)
248
- if n_cells >= max_cells:
249
- break
250
-
251
- interaction /= max(n_cells, 1)
252
- result = interaction.cpu().numpy()
253
- np.fill_diagonal(result, 0.0)
254
- logger.info("Method C done: %d cells, interaction matrix shape %s", n_cells, result.shape)
255
- return result
256
-
257
-
258
- def _compute_single_cell_jacobian(backbone, cell, num_modules, base_dim, device):
259
- """For one cell, compute (num_modules, num_modules) Jacobian norm matrix.
260
-
261
- J[i, j] = ||d encoder_out[j] / d module_emb[i]||_F
262
-
263
- Strategy: run module_embed to get intermediate, detach, enable grad,
264
- then replicate the encoder forward path and backprop from each output module.
265
- """
266
- J = torch.zeros(num_modules, num_modules, device=device, dtype=torch.float32)
267
-
268
- with torch.no_grad():
269
- x_raw, _ = backbone.module_embed(cell) # (1, num_modules, D)
270
-
271
- x_modules = x_raw.detach().clone().requires_grad_(True)
272
-
273
- with torch.enable_grad():
274
- if backbone.mask_style == "replace":
275
- x = backbone._add_module_position_embedding(x_modules)
276
- else:
277
- x = backbone._add_module_position_embedding(x_modules)
278
-
279
- if backbone.use_cls and backbone.cls_token is not None:
280
- cls_token = backbone.cls_token + backbone.pos_emb[:, :1, :]
281
- cls_tokens = cls_token.expand(x.shape[0], -1, -1)
282
- x = torch.cat((cls_tokens, x), dim=1)
283
-
284
- x_emb = backbone.dropout(x)
285
- layer_pos_emb = backbone.layer_pos_emb(x_emb)
286
- enc_out = backbone.encoder(x_emb, pos_emb=layer_pos_emb)
287
- enc_out = backbone.encoder_norm(enc_out) # (1, L, D)
288
-
289
- if backbone.use_cls and backbone.cls_token is not None:
290
- enc_out = enc_out[:, 1:, :] # skip CLS
291
-
292
- for j in range(num_modules):
293
- if x_modules.grad is not None:
294
- x_modules.grad.zero_()
295
- scalar = enc_out[0, j].pow(2).sum()
296
- scalar.backward(retain_graph=(j < num_modules - 1))
297
- grad = x_modules.grad[0] # (num_modules, D)
298
- J[:, j] = grad.norm(dim=1)
299
-
300
- return J.to(torch.float64)
301
-
302
-
303
- # ---------------------------------------------------------------------------
304
- # Ground truth: pathway overlap from module gene sets
305
- # ---------------------------------------------------------------------------
306
-
307
- def compute_pathway_overlap(module_names: List[str], module_gene_dict: dict,
308
- metric: str = "jaccard") -> np.ndarray:
309
- """Compute pairwise pathway overlap matrix from shared genes.
310
-
311
- metric: 'jaccard' (|A∩B| / |A∪B|) or 'overlap_coef' (|A∩B| / min(|A|,|B|))
312
- """
313
- n = len(module_names)
314
- gene_sets = []
315
- for name in module_names:
316
- genes = module_gene_dict.get(name, [])
317
- gene_sets.append(set(genes))
318
-
319
- overlap = np.zeros((n, n), dtype=np.float32)
320
- for i in range(n):
321
- for j in range(i + 1, n):
322
- inter = len(gene_sets[i] & gene_sets[j])
323
- if inter == 0:
324
- continue
325
- if metric == "jaccard":
326
- union = len(gene_sets[i] | gene_sets[j])
327
- score = inter / union if union > 0 else 0.0
328
- else:
329
- min_size = min(len(gene_sets[i]), len(gene_sets[j]))
330
- score = inter / min_size if min_size > 0 else 0.0
331
- overlap[i, j] = score
332
- overlap[j, i] = score
333
- return overlap
334
-
335
-
336
- # ---------------------------------------------------------------------------
337
- # Evaluation metrics
338
- # ---------------------------------------------------------------------------
339
-
340
- def evaluate_interaction(model_matrix: np.ndarray, gt_matrix: np.ndarray,
341
- top_k_fracs: List[float], n_permutations: int = 200):
342
- """Compare model interaction matrix against ground truth overlap.
343
-
344
- Returns dict with spearman_r, auroc, precision@k, and permutation p-values.
345
- """
346
- n = model_matrix.shape[0]
347
- triu_idx = np.triu_indices(n, k=1)
348
- model_flat = model_matrix[triu_idx]
349
- gt_flat = gt_matrix[triu_idx]
350
-
351
- has_positive = gt_flat.max() > 0
352
- has_variation = model_flat.std() > 1e-10
353
-
354
- rho, _ = spearmanr(model_flat, gt_flat) if has_variation else (0.0, 1.0)
355
-
356
- gt_binary = (gt_flat > 0).astype(np.float32)
357
- n_pos = gt_binary.sum()
358
- if n_pos > 0 and n_pos < len(gt_binary) and has_variation:
359
- auroc = roc_auc_score(gt_binary, model_flat)
360
- else:
361
- auroc = 0.5
362
-
363
- precisions = {}
364
- n_pairs = len(model_flat)
365
- for frac in top_k_fracs:
366
- k = max(1, int(n_pairs * frac))
367
- top_k_idx = np.argsort(model_flat)[-k:]
368
- precisions[f"precision@{frac}"] = float(gt_binary[top_k_idx].mean())
369
-
370
- null_rhos = []
371
- null_aurocs = []
372
- rng = np.random.default_rng(42)
373
- for _ in range(n_permutations):
374
- perm = rng.permutation(n)
375
- perm_matrix = model_matrix[perm][:, perm]
376
- perm_flat = perm_matrix[triu_idx]
377
- r, _ = spearmanr(perm_flat, gt_flat) if has_variation else (0.0, 1.0)
378
- null_rhos.append(r)
379
- if n_pos > 0 and n_pos < len(gt_binary):
380
- null_aurocs.append(roc_auc_score(gt_binary, perm_flat))
381
- else:
382
- null_aurocs.append(0.5)
383
-
384
- p_rho = (np.sum(np.array(null_rhos) >= rho) + 1) / (n_permutations + 1)
385
- p_auroc = (np.sum(np.array(null_aurocs) >= auroc) + 1) / (n_permutations + 1)
386
-
387
- return {
388
- "spearman_r": float(rho),
389
- "spearman_p_perm": float(p_rho),
390
- "auroc": float(auroc),
391
- "auroc_p_perm": float(p_auroc),
392
- "null_rho_mean": float(np.mean(null_rhos)),
393
- "null_rho_std": float(np.std(null_rhos)),
394
- **precisions,
395
- }
396
-
397
-
398
- # ---------------------------------------------------------------------------
399
- # Visualization
400
- # ---------------------------------------------------------------------------
401
-
402
- def plot_interaction_matrix(matrix: np.ndarray, title: str, path: Path,
403
- module_families: Optional[List[str]] = None):
404
- """Plot heatmap of interaction matrix, optionally sorted by family."""
405
- if module_families is not None:
406
- order = np.argsort(module_families)
407
- matrix = matrix[order][:, order]
408
- sorted_families = [module_families[i] for i in order]
409
- else:
410
- sorted_families = None
411
-
412
- fig, ax = plt.subplots(figsize=(10, 9))
413
- vmax = np.percentile(np.abs(matrix), 99)
414
- im = ax.imshow(matrix, cmap="RdBu_r", vmin=-vmax, vmax=vmax, aspect="equal")
415
- ax.set_title(title, fontsize=12)
416
- ax.set_xlabel("Module j")
417
- ax.set_ylabel("Module i")
418
- plt.colorbar(im, ax=ax, fraction=0.046)
419
-
420
- if sorted_families is not None:
421
- unique_fams = []
422
- boundaries = [0]
423
- prev = sorted_families[0]
424
- for idx, f in enumerate(sorted_families):
425
- if f != prev:
426
- unique_fams.append(prev)
427
- boundaries.append(idx)
428
- prev = f
429
- unique_fams.append(prev)
430
- boundaries.append(len(sorted_families))
431
- for b in boundaries[1:-1]:
432
- ax.axhline(b - 0.5, color="black", linewidth=0.3, alpha=0.5)
433
- ax.axvline(b - 0.5, color="black", linewidth=0.3, alpha=0.5)
434
-
435
- plt.tight_layout()
436
- fig.savefig(path, dpi=150, bbox_inches="tight")
437
- plt.close(fig)
438
-
439
-
440
- def plot_scatter(model_flat, gt_flat, title, path):
441
- """Scatter plot of model interaction vs ground truth overlap."""
442
- fig, ax = plt.subplots(figsize=(6, 5))
443
- subsample = min(len(model_flat), 50000)
444
- if subsample < len(model_flat):
445
- rng = np.random.default_rng(0)
446
- idx = rng.choice(len(model_flat), size=subsample, replace=False)
447
- model_flat = model_flat[idx]
448
- gt_flat = gt_flat[idx]
449
- ax.scatter(gt_flat, model_flat, alpha=0.1, s=2, rasterized=True)
450
- ax.set_xlabel("Ground truth overlap (Jaccard)")
451
- ax.set_ylabel("Model interaction score")
452
- ax.set_title(title)
453
- plt.tight_layout()
454
- fig.savefig(path, dpi=150, bbox_inches="tight")
455
- plt.close(fig)
456
-
457
-
458
- # ---------------------------------------------------------------------------
459
- # Main
460
- # ---------------------------------------------------------------------------
461
-
462
- def main():
463
- args = parse_args()
464
- seed_all(args.seed)
465
- device = torch.device(args.device if torch.cuda.is_available() else "cpu")
466
- output_dir = Path(args.output_dir)
467
- ensure_dir(output_dir)
468
-
469
- # Resolve all paths to absolute (relative to CWD) so that load_backbone's
470
- # internal resolve_path (which resolves relative to cell_annotation/) is bypassed.
471
- for attr in ("performer_checkpoint", "config_pkl", "module_data_dir",
472
- "gene_list_path", "gene_mapping_path", "gene2vec_data_path",
473
- "gene2ens_file", "data_path"):
474
- val = getattr(args, attr, None)
475
- if val is not None and not Path(val).is_absolute():
476
- setattr(args, attr, str(Path(val).resolve()))
477
-
478
- logger.info("Loading backbone...")
479
- backbone, module_data, model_config = load_backbone(args, device)
480
- backbone.eval()
481
-
482
- num_modules = len(module_data)
483
- base_dim = int(model_config["base_dim"])
484
- expected_genes = int(model_config.get("num_genes", 27874))
485
- logger.info("Model: %d modules, dim=%d, genes=%d", num_modules, base_dim, expected_genes)
486
-
487
- module_json_path = args.module_json or str(Path(args.module_data_dir) / "merged_modules.json")
488
- module_names = load_module_names(module_json_path)
489
- with open(module_json_path, "r", encoding="utf-8") as f:
490
- module_gene_dict = json.load(f)
491
-
492
- preprocessor = None
493
- if args.data_mode == "h5ad":
494
- preprocessor = DataPreprocessor(
495
- gene2ens_file=args.gene2ens_file,
496
- gene2ens_column=args.gene2ens_column,
497
- model_genes_file=args.gene_list_path,
498
- )
499
- preprocessor.load_gene_mapping()
500
- preprocessor.load_model_genes()
501
-
502
- dataset, reindex_state, n_cells = build_dataset(args, preprocessor)
503
- logger.info("Dataset: %d cells", n_cells)
504
-
505
- # --- Method A: Cosine ---
506
- logger.info("=" * 60)
507
- logger.info("Method A: Cosine similarity of module embeddings")
508
- logger.info("=" * 60)
509
- loader_a = DataLoader(dataset, batch_size=args.batch_size, shuffle=False,
510
- num_workers=args.num_workers, pin_memory=True)
511
- cos_matrix, mean_emb = compute_cosine_interaction(
512
- backbone, loader_a, preprocessor, reindex_state,
513
- device, num_modules, base_dim, expected_genes,
514
- pool_mode=args.pool_mode,
515
- )
516
-
517
- # --- Method C: Jacobian ---
518
- jac_matrix = None
519
- if not args.skip_jacobian:
520
- logger.info("=" * 60)
521
- logger.info("Method C: Jacobian-based module interaction")
522
- logger.info("=" * 60)
523
- jac_n = min(args.jacobian_cells, n_cells)
524
- if jac_n < n_cells:
525
- rng = np.random.default_rng(args.seed)
526
- jac_indices = np.sort(rng.choice(n_cells, size=jac_n, replace=False))
527
- jac_dataset = Subset(dataset, jac_indices.tolist())
528
- else:
529
- jac_dataset = dataset
530
- loader_c = DataLoader(jac_dataset, batch_size=args.jacobian_batch, shuffle=False,
531
- num_workers=0, pin_memory=True)
532
- jac_matrix = compute_jacobian_interaction(
533
- backbone, loader_c, preprocessor, reindex_state,
534
- device, num_modules, base_dim, expected_genes,
535
- max_cells=jac_n, batch_size=args.jacobian_batch,
536
- )
537
-
538
- # --- Ground truth overlap ---
539
- logger.info("Computing pathway gene overlap (Jaccard)...")
540
- gt_jaccard = compute_pathway_overlap(module_names, module_gene_dict, metric="jaccard")
541
- gt_overlap = compute_pathway_overlap(module_names, module_gene_dict, metric="overlap_coef")
542
-
543
- # --- Evaluate ---
544
- top_k_fracs = [float(f) for f in args.top_k_fractions.split(",")]
545
- results = {}
546
-
547
- logger.info("Evaluating Method A vs Jaccard overlap...")
548
- results["cosine_vs_jaccard"] = evaluate_interaction(
549
- cos_matrix, gt_jaccard, top_k_fracs, args.null_permutations)
550
- results["cosine_vs_overlap_coef"] = evaluate_interaction(
551
- cos_matrix, gt_overlap, top_k_fracs, args.null_permutations)
552
-
553
- if jac_matrix is not None:
554
- logger.info("Evaluating Method C vs Jaccard overlap...")
555
- jac_sym = (jac_matrix + jac_matrix.T) / 2
556
- results["jacobian_vs_jaccard"] = evaluate_interaction(
557
- jac_sym, gt_jaccard, top_k_fracs, args.null_permutations)
558
- results["jacobian_vs_overlap_coef"] = evaluate_interaction(
559
- jac_sym, gt_overlap, top_k_fracs, args.null_permutations)
560
-
561
- # --- Save ---
562
- np.save(output_dir / "cosine_interaction.npy", cos_matrix)
563
- np.save(output_dir / "mean_module_embeddings.npy", mean_emb)
564
- np.save(output_dir / "gt_jaccard.npy", gt_jaccard)
565
- np.save(output_dir / "gt_overlap_coef.npy", gt_overlap)
566
- if jac_matrix is not None:
567
- np.save(output_dir / "jacobian_interaction.npy", jac_matrix)
568
-
569
- with open(output_dir / "metrics.json", "w", encoding="utf-8") as f:
570
- json.dump(results, f, indent=2, ensure_ascii=False)
571
-
572
- # --- Plots ---
573
- meta = build_module_metadata(module_names)
574
- families = [row["family"] for row in meta]
575
-
576
- plot_interaction_matrix(cos_matrix, "Method A: Cosine Similarity",
577
- output_dir / "heatmap_cosine.png", families)
578
- if jac_matrix is not None:
579
- plot_interaction_matrix(jac_sym, "Method C: Jacobian Interaction",
580
- output_dir / "heatmap_jacobian.png", families)
581
- plot_interaction_matrix(gt_jaccard, "Ground Truth: Jaccard Gene Overlap",
582
- output_dir / "heatmap_gt_jaccard.png", families)
583
-
584
- triu_idx = np.triu_indices(num_modules, k=1)
585
- plot_scatter(cos_matrix[triu_idx], gt_jaccard[triu_idx],
586
- "Cosine vs Jaccard", output_dir / "scatter_cosine_vs_jaccard.png")
587
- if jac_matrix is not None:
588
- plot_scatter(jac_sym[triu_idx], gt_jaccard[triu_idx],
589
- "Jacobian vs Jaccard", output_dir / "scatter_jacobian_vs_jaccard.png")
590
-
591
- # --- Print summary ---
592
- logger.info("=" * 60)
593
- logger.info("Results Summary")
594
- logger.info("=" * 60)
595
- for method_key, metrics in results.items():
596
- logger.info(" %s:", method_key)
597
- logger.info(" Spearman r = %.4f (p_perm = %.4f)", metrics["spearman_r"], metrics["spearman_p_perm"])
598
- logger.info(" AUROC = %.4f (p_perm = %.4f)", metrics["auroc"], metrics["auroc_p_perm"])
599
- for k in top_k_fracs:
600
- logger.info(" Precision@%.2f = %.4f", k, metrics[f"precision@{k}"])
601
- logger.info("Output saved to: %s", output_dir)
602
-
603
-
604
- if __name__ == "__main__":
605
- main()
 
 
1
+ """Experiment 6 — Module crosstalk graph vs Reactome/KEGG pathway interactions.
2
+
3
+ Two complementary methods to quantify module-to-module interaction strength:
4
+
5
+ Method A (cosine similarity): Compute cosine similarity between cell-averaged
6
+ module encoder outputs. Fast, no gradient needed.
7
+
8
+ Method C (Jacobian): Compute ||d encoder_out[j] / d module_input[i]|| averaged
9
+ over cells. Measures causal influence of module i on module j's representation.
10
+ Requires gradient; more expensive but more principled.
11
+
12
+ Both interaction matrices are compared against known pathway-pathway relationships
13
+ from Reactome (shared genes / hierarchical overlap) and optionally KEGG.
14
+
15
+ Metrics:
16
+ - Spearman correlation between model interaction matrix and DB overlap matrix
17
+ - Precision@k: among top-k model interactions, fraction present in DB
18
+ - AUROC: treating DB edges as positives, model scores as predictions
19
+ - Permutation null p-values for all metrics
20
+
21
+ Usage (run from the BioMod repository root):
22
+ python BioMod/interpretability/attention/module_crosstalk.py \
23
+ --performer_checkpoint weight/0810model/mae_tost_70M_75mask_ec2_epoch0.pth \
24
+ --config_pkl weight/0810model/mae_tost_70M_75mask_ec2.pkl \
25
+ --module_data_dir model_dependencies/module_data_all \
26
+ --gene_list_path model_dependencies/selected_genes_27k.txt \
27
+ --gene_mapping_path model_dependencies/gene2ens.csv \
28
+ --gene2vec_data_path model_dependencies/selected_gene2vec_27k.npy \
29
+ --data_path data/downstream/final_data/human_pancreas_norm_complexBatch_with_gene_name.h5ad \
30
+ --data_mode h5ad \
31
+ --batch_size 16 \
32
+ --jacobian_cells 200 \
33
+ --output_dir outputs/interpretability/crosstalk/pancreas
34
+ """
35
+ from __future__ import annotations
36
+
37
+ import argparse
38
+ import json
39
+ import logging
40
+ import sys
41
+ from pathlib import Path
42
+ from typing import Dict, List, Optional, Tuple
43
+
44
+ import numpy as np
45
+ import torch
46
+ from torch.utils.data import DataLoader, Subset
47
+
48
+ import matplotlib
49
+ matplotlib.use("Agg")
50
+ import matplotlib.pyplot as plt
51
+
52
+ from scipy.stats import spearmanr
53
+ from sklearn.metrics import roc_auc_score
54
+
55
+ HERE = Path(__file__).resolve()
56
+ PERFORMER_MODULE = HERE.parents[2]
57
+ REPO_ROOT = HERE.parents[3]
58
+ ASSET_ROOT = REPO_ROOT / "model_dependencies"
59
+ for p in (str(PERFORMER_MODULE), str(REPO_ROOT)):
60
+ if p not in sys.path:
61
+ sys.path.insert(0, p)
62
+
63
+ from cell_annotation.performer_module_finetune import (
64
+ DataPreprocessor,
65
+ load_backbone,
66
+ seed_all,
67
+ )
68
+
69
+ from interpretability.utils import (
70
+ MatrixRowDataset,
71
+ NpzCellDataset,
72
+ build_module_metadata,
73
+ ensure_dir,
74
+ load_module_names,
75
+ )
76
+
77
+ logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
78
+ logger = logging.getLogger(__name__)
79
+
80
+ # ---------------------------------------------------------------------------
81
+ # Argparse
82
+ # ---------------------------------------------------------------------------
83
+
84
+ def parse_args() -> argparse.Namespace:
85
+ p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
86
+
87
+ p.add_argument("--performer_checkpoint", type=str, required=True)
88
+ p.add_argument("--config_pkl", type=str, required=True)
89
+ p.add_argument("--module_data_dir", type=str, default=str(ASSET_ROOT / "module_data_all"))
90
+ p.add_argument("--module_json", type=str, default=None)
91
+ p.add_argument("--gene_list_path", type=str, default=str(ASSET_ROOT / "selected_genes_27k.txt"))
92
+ p.add_argument("--gene_mapping_path", type=str, default=str(ASSET_ROOT / "gene2ens.csv"))
93
+ p.add_argument("--gene2vec_data_path", type=str, default=str(ASSET_ROOT / "selected_gene2vec_27k.npy"))
94
+ p.add_argument("--gene2ens_file", type=str, default=str(ASSET_ROOT / "gene2ens.csv"))
95
+ p.add_argument("--gene2ens_column", type=str, default="Ensembl gene ID")
96
+ p.add_argument("--shuffle_module_genes", action="store_true")
97
+ p.add_argument("--shuffle_seed", type=int, default=2021)
98
+
99
+ p.add_argument("--data_path", type=str, required=True)
100
+ p.add_argument("--data_mode", type=str, default="h5ad",
101
+ choices=["h5ad", "pretrain_h5ad", "pretrain_npz"])
102
+ p.add_argument("--cell_num", type=int, default=None)
103
+ p.add_argument("--gene_name_column", type=str, default="auto")
104
+ p.add_argument("--dataset_name", type=str, default="crosstalk")
105
+ p.add_argument("--max_cells", type=int, default=5000,
106
+ help="Max cells for method A (cosine). Method C uses --jacobian_cells.")
107
+
108
+ p.add_argument("--batch_size", type=int, default=16)
109
+ p.add_argument("--num_workers", type=int, default=0)
110
+ p.add_argument("--device", type=str, default="cuda")
111
+ p.add_argument("--seed", type=int, default=2021)
112
+ p.add_argument("--pool_mode", type=str, default="mean",
113
+ choices=["mean", "max"],
114
+ help="Pooling over cells: 'mean' or 'max' (element-wise max).")
115
+
116
+ p.add_argument("--jacobian_cells", type=int, default=200,
117
+ help="Number of cells for Jacobian computation (expensive).")
118
+ p.add_argument("--jacobian_batch", type=int, default=4,
119
+ help="Cells per Jacobian forward-backward pass.")
120
+ p.add_argument("--skip_jacobian", action="store_true",
121
+ help="Only run method A (cosine), skip method C (Jacobian).")
122
+
123
+ p.add_argument("--null_permutations", type=int, default=200)
124
+ p.add_argument("--top_k_fractions", type=str, default="0.01,0.05,0.1",
125
+ help="Comma-separated fractions for precision@k.")
126
+
127
+ p.add_argument("--output_dir", type=str, required=True)
128
+ return p.parse_args()
129
+
130
+
131
+ # ---------------------------------------------------------------------------
132
+ # Data loading (reuses patterns from module_geometry.py)
133
+ # ---------------------------------------------------------------------------
134
+
135
+ def build_dataset(args, preprocessor):
136
+ """Build dataset and optional reindex_state."""
137
+ if args.data_mode == "h5ad":
138
+ import anndata as ad
139
+ adata, reindex_state = preprocessor.preprocess_adata(
140
+ adata_path=args.data_path,
141
+ min_genes=0, min_cells=0,
142
+ max_cells=args.max_cells,
143
+ dataset_name=args.dataset_name,
144
+ gene_name_column=args.gene_name_column,
145
+ )
146
+ dataset = MatrixRowDataset(adata.X)
147
+ n_cells = adata.n_obs
148
+ elif args.data_mode == "pretrain_h5ad":
149
+ import anndata as ad
150
+ adata = ad.read_h5ad(args.data_path)
151
+ if args.max_cells and adata.n_obs > args.max_cells:
152
+ rng = np.random.default_rng(args.seed)
153
+ chosen = np.sort(rng.choice(adata.n_obs, size=args.max_cells, replace=False))
154
+ adata = adata[chosen].copy()
155
+ dataset = MatrixRowDataset(adata.X)
156
+ n_cells = adata.n_obs
157
+ reindex_state = None
158
+ elif args.data_mode == "pretrain_npz":
159
+ dataset = NpzCellDataset(
160
+ data_dir=args.data_path,
161
+ cell_num=args.cell_num,
162
+ max_cells=args.max_cells,
163
+ seed=args.seed,
164
+ )
165
+ n_cells = len(dataset)
166
+ reindex_state = None
167
+ else:
168
+ raise ValueError(f"Unsupported data_mode: {args.data_mode}")
169
+ return dataset, reindex_state, n_cells
170
+
171
+
172
+ # ---------------------------------------------------------------------------
173
+ # Method A: Cosine similarity of cell-averaged module embeddings
174
+ # ---------------------------------------------------------------------------
175
+
176
+ @torch.no_grad()
177
+ def compute_cosine_interaction(backbone, loader, preprocessor, reindex_state,
178
+ device, num_modules, base_dim, expected_genes,
179
+ pool_mode="mean"):
180
+ """Compute pooled module embeddings, return cosine similarity matrix."""
181
+ backbone.eval()
182
+ if pool_mode == "mean":
183
+ agg_emb = torch.zeros(num_modules, base_dim, device=device, dtype=torch.float64)
184
+ else:
185
+ agg_emb = torch.full((num_modules, base_dim), -float("inf"), device=device, dtype=torch.float64)
186
+ n_cells = 0
187
+
188
+ for batch_idx, x in enumerate(loader):
189
+ x = x.to(device)
190
+ if reindex_state is not None:
191
+ x = preprocessor.reindex_tensor_to_model_space(x, reindex_state=reindex_state)
192
+ out = backbone(x, mask_ratio=0.0, return_encodings=True)
193
+ enc = out["module_encodings"].to(torch.float64) # (B, num_modules, D)
194
+ if pool_mode == "mean":
195
+ agg_emb += enc.sum(dim=0)
196
+ else:
197
+ batch_max = enc.max(dim=0).values # (num_modules, D)
198
+ agg_emb = torch.maximum(agg_emb, batch_max)
199
+ n_cells += enc.shape[0]
200
+ if (batch_idx + 1) % 50 == 0:
201
+ logger.info(" [cosine/%s] batch %d, %d cells", pool_mode, batch_idx + 1, n_cells)
202
+
203
+ if pool_mode == "mean":
204
+ pooled_emb = (agg_emb / max(n_cells, 1)).float().cpu().numpy()
205
+ else:
206
+ pooled_emb = agg_emb.float().cpu().numpy()
207
+
208
+ norms = np.linalg.norm(pooled_emb, axis=1, keepdims=True)
209
+ norms = np.clip(norms, 1e-8, None)
210
+ normed = pooled_emb / norms
211
+ cos_sim = normed @ normed.T
212
+ np.fill_diagonal(cos_sim, 0.0)
213
+ logger.info("Method A (%s) done: %d cells, interaction matrix shape %s", pool_mode, n_cells, cos_sim.shape)
214
+ return cos_sim, pooled_emb
215
+
216
+
217
+ # ---------------------------------------------------------------------------
218
+ # Method C: Jacobian-based module interaction
219
+ # ---------------------------------------------------------------------------
220
+
221
+ def compute_jacobian_interaction(backbone, loader, preprocessor, reindex_state,
222
+ device, num_modules, base_dim, expected_genes,
223
+ max_cells, batch_size):
224
+ """Compute average ||d encoder_out[j] / d module_emb[i]|| interaction matrix."""
225
+ backbone.eval()
226
+ for param in backbone.parameters():
227
+ param.requires_grad_(False)
228
+
229
+ interaction = torch.zeros(num_modules, num_modules, device=device, dtype=torch.float64)
230
+ n_cells = 0
231
+
232
+ for x in loader:
233
+ if n_cells >= max_cells:
234
+ break
235
+ x = x.to(device)
236
+ if reindex_state is not None:
237
+ x = preprocessor.reindex_tensor_to_model_space(x, reindex_state=reindex_state)
238
+
239
+ actual_batch = min(x.shape[0], max_cells - n_cells)
240
+ x = x[:actual_batch]
241
+
242
+ for cell_idx in range(x.shape[0]):
243
+ cell = x[cell_idx:cell_idx+1]
244
+ J = _compute_single_cell_jacobian(backbone, cell, num_modules, base_dim, device)
245
+ interaction += J
246
+ n_cells += 1
247
+ if n_cells % 20 == 0:
248
+ logger.info(" [jacobian] %d / %d cells", n_cells, max_cells)
249
+ if n_cells >= max_cells:
250
+ break
251
+
252
+ interaction /= max(n_cells, 1)
253
+ result = interaction.cpu().numpy()
254
+ np.fill_diagonal(result, 0.0)
255
+ logger.info("Method C done: %d cells, interaction matrix shape %s", n_cells, result.shape)
256
+ return result
257
+
258
+
259
+ def _compute_single_cell_jacobian(backbone, cell, num_modules, base_dim, device):
260
+ """For one cell, compute (num_modules, num_modules) Jacobian norm matrix.
261
+
262
+ J[i, j] = ||d encoder_out[j] / d module_emb[i]||_F
263
+
264
+ Strategy: run module_embed to get intermediate, detach, enable grad,
265
+ then replicate the encoder forward path and backprop from each output module.
266
+ """
267
+ J = torch.zeros(num_modules, num_modules, device=device, dtype=torch.float32)
268
+
269
+ with torch.no_grad():
270
+ x_raw, _ = backbone.module_embed(cell) # (1, num_modules, D)
271
+
272
+ x_modules = x_raw.detach().clone().requires_grad_(True)
273
+
274
+ with torch.enable_grad():
275
+ if backbone.mask_style == "replace":
276
+ x = backbone._add_module_position_embedding(x_modules)
277
+ else:
278
+ x = backbone._add_module_position_embedding(x_modules)
279
+
280
+ if backbone.use_cls and backbone.cls_token is not None:
281
+ cls_token = backbone.cls_token + backbone.pos_emb[:, :1, :]
282
+ cls_tokens = cls_token.expand(x.shape[0], -1, -1)
283
+ x = torch.cat((cls_tokens, x), dim=1)
284
+
285
+ x_emb = backbone.dropout(x)
286
+ layer_pos_emb = backbone.layer_pos_emb(x_emb)
287
+ enc_out = backbone.encoder(x_emb, pos_emb=layer_pos_emb)
288
+ enc_out = backbone.encoder_norm(enc_out) # (1, L, D)
289
+
290
+ if backbone.use_cls and backbone.cls_token is not None:
291
+ enc_out = enc_out[:, 1:, :] # skip CLS
292
+
293
+ for j in range(num_modules):
294
+ if x_modules.grad is not None:
295
+ x_modules.grad.zero_()
296
+ scalar = enc_out[0, j].pow(2).sum()
297
+ scalar.backward(retain_graph=(j < num_modules - 1))
298
+ grad = x_modules.grad[0] # (num_modules, D)
299
+ J[:, j] = grad.norm(dim=1)
300
+
301
+ return J.to(torch.float64)
302
+
303
+
304
+ # ---------------------------------------------------------------------------
305
+ # Ground truth: pathway overlap from module gene sets
306
+ # ---------------------------------------------------------------------------
307
+
308
+ def compute_pathway_overlap(module_names: List[str], module_gene_dict: dict,
309
+ metric: str = "jaccard") -> np.ndarray:
310
+ """Compute pairwise pathway overlap matrix from shared genes.
311
+
312
+ metric: 'jaccard' (|A∩B| / |A∪B|) or 'overlap_coef' (|A∩B| / min(|A|,|B|))
313
+ """
314
+ n = len(module_names)
315
+ gene_sets = []
316
+ for name in module_names:
317
+ genes = module_gene_dict.get(name, [])
318
+ gene_sets.append(set(genes))
319
+
320
+ overlap = np.zeros((n, n), dtype=np.float32)
321
+ for i in range(n):
322
+ for j in range(i + 1, n):
323
+ inter = len(gene_sets[i] & gene_sets[j])
324
+ if inter == 0:
325
+ continue
326
+ if metric == "jaccard":
327
+ union = len(gene_sets[i] | gene_sets[j])
328
+ score = inter / union if union > 0 else 0.0
329
+ else:
330
+ min_size = min(len(gene_sets[i]), len(gene_sets[j]))
331
+ score = inter / min_size if min_size > 0 else 0.0
332
+ overlap[i, j] = score
333
+ overlap[j, i] = score
334
+ return overlap
335
+
336
+
337
+ # ---------------------------------------------------------------------------
338
+ # Evaluation metrics
339
+ # ---------------------------------------------------------------------------
340
+
341
+ def evaluate_interaction(model_matrix: np.ndarray, gt_matrix: np.ndarray,
342
+ top_k_fracs: List[float], n_permutations: int = 200):
343
+ """Compare model interaction matrix against ground truth overlap.
344
+
345
+ Returns dict with spearman_r, auroc, precision@k, and permutation p-values.
346
+ """
347
+ n = model_matrix.shape[0]
348
+ triu_idx = np.triu_indices(n, k=1)
349
+ model_flat = model_matrix[triu_idx]
350
+ gt_flat = gt_matrix[triu_idx]
351
+
352
+ has_positive = gt_flat.max() > 0
353
+ has_variation = model_flat.std() > 1e-10
354
+
355
+ rho, _ = spearmanr(model_flat, gt_flat) if has_variation else (0.0, 1.0)
356
+
357
+ gt_binary = (gt_flat > 0).astype(np.float32)
358
+ n_pos = gt_binary.sum()
359
+ if n_pos > 0 and n_pos < len(gt_binary) and has_variation:
360
+ auroc = roc_auc_score(gt_binary, model_flat)
361
+ else:
362
+ auroc = 0.5
363
+
364
+ precisions = {}
365
+ n_pairs = len(model_flat)
366
+ for frac in top_k_fracs:
367
+ k = max(1, int(n_pairs * frac))
368
+ top_k_idx = np.argsort(model_flat)[-k:]
369
+ precisions[f"precision@{frac}"] = float(gt_binary[top_k_idx].mean())
370
+
371
+ null_rhos = []
372
+ null_aurocs = []
373
+ rng = np.random.default_rng(42)
374
+ for _ in range(n_permutations):
375
+ perm = rng.permutation(n)
376
+ perm_matrix = model_matrix[perm][:, perm]
377
+ perm_flat = perm_matrix[triu_idx]
378
+ r, _ = spearmanr(perm_flat, gt_flat) if has_variation else (0.0, 1.0)
379
+ null_rhos.append(r)
380
+ if n_pos > 0 and n_pos < len(gt_binary):
381
+ null_aurocs.append(roc_auc_score(gt_binary, perm_flat))
382
+ else:
383
+ null_aurocs.append(0.5)
384
+
385
+ p_rho = (np.sum(np.array(null_rhos) >= rho) + 1) / (n_permutations + 1)
386
+ p_auroc = (np.sum(np.array(null_aurocs) >= auroc) + 1) / (n_permutations + 1)
387
+
388
+ return {
389
+ "spearman_r": float(rho),
390
+ "spearman_p_perm": float(p_rho),
391
+ "auroc": float(auroc),
392
+ "auroc_p_perm": float(p_auroc),
393
+ "null_rho_mean": float(np.mean(null_rhos)),
394
+ "null_rho_std": float(np.std(null_rhos)),
395
+ **precisions,
396
+ }
397
+
398
+
399
+ # ---------------------------------------------------------------------------
400
+ # Visualization
401
+ # ---------------------------------------------------------------------------
402
+
403
+ def plot_interaction_matrix(matrix: np.ndarray, title: str, path: Path,
404
+ module_families: Optional[List[str]] = None):
405
+ """Plot heatmap of interaction matrix, optionally sorted by family."""
406
+ if module_families is not None:
407
+ order = np.argsort(module_families)
408
+ matrix = matrix[order][:, order]
409
+ sorted_families = [module_families[i] for i in order]
410
+ else:
411
+ sorted_families = None
412
+
413
+ fig, ax = plt.subplots(figsize=(10, 9))
414
+ vmax = np.percentile(np.abs(matrix), 99)
415
+ im = ax.imshow(matrix, cmap="RdBu_r", vmin=-vmax, vmax=vmax, aspect="equal")
416
+ ax.set_title(title, fontsize=12)
417
+ ax.set_xlabel("Module j")
418
+ ax.set_ylabel("Module i")
419
+ plt.colorbar(im, ax=ax, fraction=0.046)
420
+
421
+ if sorted_families is not None:
422
+ unique_fams = []
423
+ boundaries = [0]
424
+ prev = sorted_families[0]
425
+ for idx, f in enumerate(sorted_families):
426
+ if f != prev:
427
+ unique_fams.append(prev)
428
+ boundaries.append(idx)
429
+ prev = f
430
+ unique_fams.append(prev)
431
+ boundaries.append(len(sorted_families))
432
+ for b in boundaries[1:-1]:
433
+ ax.axhline(b - 0.5, color="black", linewidth=0.3, alpha=0.5)
434
+ ax.axvline(b - 0.5, color="black", linewidth=0.3, alpha=0.5)
435
+
436
+ plt.tight_layout()
437
+ fig.savefig(path, dpi=150, bbox_inches="tight")
438
+ plt.close(fig)
439
+
440
+
441
+ def plot_scatter(model_flat, gt_flat, title, path):
442
+ """Scatter plot of model interaction vs ground truth overlap."""
443
+ fig, ax = plt.subplots(figsize=(6, 5))
444
+ subsample = min(len(model_flat), 50000)
445
+ if subsample < len(model_flat):
446
+ rng = np.random.default_rng(0)
447
+ idx = rng.choice(len(model_flat), size=subsample, replace=False)
448
+ model_flat = model_flat[idx]
449
+ gt_flat = gt_flat[idx]
450
+ ax.scatter(gt_flat, model_flat, alpha=0.1, s=2, rasterized=True)
451
+ ax.set_xlabel("Ground truth overlap (Jaccard)")
452
+ ax.set_ylabel("Model interaction score")
453
+ ax.set_title(title)
454
+ plt.tight_layout()
455
+ fig.savefig(path, dpi=150, bbox_inches="tight")
456
+ plt.close(fig)
457
+
458
+
459
+ # ---------------------------------------------------------------------------
460
+ # Main
461
+ # ---------------------------------------------------------------------------
462
+
463
+ def main():
464
+ args = parse_args()
465
+ seed_all(args.seed)
466
+ device = torch.device(args.device if torch.cuda.is_available() else "cpu")
467
+ output_dir = Path(args.output_dir)
468
+ ensure_dir(output_dir)
469
+
470
+ # Resolve all paths to absolute (relative to CWD) so that load_backbone's
471
+ # internal resolve_path (which resolves relative to cell_annotation/) is bypassed.
472
+ for attr in ("performer_checkpoint", "config_pkl", "module_data_dir",
473
+ "gene_list_path", "gene_mapping_path", "gene2vec_data_path",
474
+ "gene2ens_file", "data_path"):
475
+ val = getattr(args, attr, None)
476
+ if val is not None and not Path(val).is_absolute():
477
+ setattr(args, attr, str(Path(val).resolve()))
478
+
479
+ logger.info("Loading backbone...")
480
+ backbone, module_data, model_config = load_backbone(args, device)
481
+ backbone.eval()
482
+
483
+ num_modules = len(module_data)
484
+ base_dim = int(model_config["base_dim"])
485
+ expected_genes = int(model_config.get("num_genes", 27874))
486
+ logger.info("Model: %d modules, dim=%d, genes=%d", num_modules, base_dim, expected_genes)
487
+
488
+ module_json_path = args.module_json or str(Path(args.module_data_dir) / "merged_modules.json")
489
+ module_names = load_module_names(module_json_path)
490
+ with open(module_json_path, "r", encoding="utf-8") as f:
491
+ module_gene_dict = json.load(f)
492
+
493
+ preprocessor = None
494
+ if args.data_mode == "h5ad":
495
+ preprocessor = DataPreprocessor(
496
+ gene2ens_file=args.gene2ens_file,
497
+ gene2ens_column=args.gene2ens_column,
498
+ model_genes_file=args.gene_list_path,
499
+ )
500
+ preprocessor.load_gene_mapping()
501
+ preprocessor.load_model_genes()
502
+
503
+ dataset, reindex_state, n_cells = build_dataset(args, preprocessor)
504
+ logger.info("Dataset: %d cells", n_cells)
505
+
506
+ # --- Method A: Cosine ---
507
+ logger.info("=" * 60)
508
+ logger.info("Method A: Cosine similarity of module embeddings")
509
+ logger.info("=" * 60)
510
+ loader_a = DataLoader(dataset, batch_size=args.batch_size, shuffle=False,
511
+ num_workers=args.num_workers, pin_memory=True)
512
+ cos_matrix, mean_emb = compute_cosine_interaction(
513
+ backbone, loader_a, preprocessor, reindex_state,
514
+ device, num_modules, base_dim, expected_genes,
515
+ pool_mode=args.pool_mode,
516
+ )
517
+
518
+ # --- Method C: Jacobian ---
519
+ jac_matrix = None
520
+ if not args.skip_jacobian:
521
+ logger.info("=" * 60)
522
+ logger.info("Method C: Jacobian-based module interaction")
523
+ logger.info("=" * 60)
524
+ jac_n = min(args.jacobian_cells, n_cells)
525
+ if jac_n < n_cells:
526
+ rng = np.random.default_rng(args.seed)
527
+ jac_indices = np.sort(rng.choice(n_cells, size=jac_n, replace=False))
528
+ jac_dataset = Subset(dataset, jac_indices.tolist())
529
+ else:
530
+ jac_dataset = dataset
531
+ loader_c = DataLoader(jac_dataset, batch_size=args.jacobian_batch, shuffle=False,
532
+ num_workers=0, pin_memory=True)
533
+ jac_matrix = compute_jacobian_interaction(
534
+ backbone, loader_c, preprocessor, reindex_state,
535
+ device, num_modules, base_dim, expected_genes,
536
+ max_cells=jac_n, batch_size=args.jacobian_batch,
537
+ )
538
+
539
+ # --- Ground truth overlap ---
540
+ logger.info("Computing pathway gene overlap (Jaccard)...")
541
+ gt_jaccard = compute_pathway_overlap(module_names, module_gene_dict, metric="jaccard")
542
+ gt_overlap = compute_pathway_overlap(module_names, module_gene_dict, metric="overlap_coef")
543
+
544
+ # --- Evaluate ---
545
+ top_k_fracs = [float(f) for f in args.top_k_fractions.split(",")]
546
+ results = {}
547
+
548
+ logger.info("Evaluating Method A vs Jaccard overlap...")
549
+ results["cosine_vs_jaccard"] = evaluate_interaction(
550
+ cos_matrix, gt_jaccard, top_k_fracs, args.null_permutations)
551
+ results["cosine_vs_overlap_coef"] = evaluate_interaction(
552
+ cos_matrix, gt_overlap, top_k_fracs, args.null_permutations)
553
+
554
+ if jac_matrix is not None:
555
+ logger.info("Evaluating Method C vs Jaccard overlap...")
556
+ jac_sym = (jac_matrix + jac_matrix.T) / 2
557
+ results["jacobian_vs_jaccard"] = evaluate_interaction(
558
+ jac_sym, gt_jaccard, top_k_fracs, args.null_permutations)
559
+ results["jacobian_vs_overlap_coef"] = evaluate_interaction(
560
+ jac_sym, gt_overlap, top_k_fracs, args.null_permutations)
561
+
562
+ # --- Save ---
563
+ np.save(output_dir / "cosine_interaction.npy", cos_matrix)
564
+ np.save(output_dir / "mean_module_embeddings.npy", mean_emb)
565
+ np.save(output_dir / "gt_jaccard.npy", gt_jaccard)
566
+ np.save(output_dir / "gt_overlap_coef.npy", gt_overlap)
567
+ if jac_matrix is not None:
568
+ np.save(output_dir / "jacobian_interaction.npy", jac_matrix)
569
+
570
+ with open(output_dir / "metrics.json", "w", encoding="utf-8") as f:
571
+ json.dump(results, f, indent=2, ensure_ascii=False)
572
+
573
+ # --- Plots ---
574
+ meta = build_module_metadata(module_names)
575
+ families = [row["family"] for row in meta]
576
+
577
+ plot_interaction_matrix(cos_matrix, "Method A: Cosine Similarity",
578
+ output_dir / "heatmap_cosine.png", families)
579
+ if jac_matrix is not None:
580
+ plot_interaction_matrix(jac_sym, "Method C: Jacobian Interaction",
581
+ output_dir / "heatmap_jacobian.png", families)
582
+ plot_interaction_matrix(gt_jaccard, "Ground Truth: Jaccard Gene Overlap",
583
+ output_dir / "heatmap_gt_jaccard.png", families)
584
+
585
+ triu_idx = np.triu_indices(num_modules, k=1)
586
+ plot_scatter(cos_matrix[triu_idx], gt_jaccard[triu_idx],
587
+ "Cosine vs Jaccard", output_dir / "scatter_cosine_vs_jaccard.png")
588
+ if jac_matrix is not None:
589
+ plot_scatter(jac_sym[triu_idx], gt_jaccard[triu_idx],
590
+ "Jacobian vs Jaccard", output_dir / "scatter_jacobian_vs_jaccard.png")
591
+
592
+ # --- Print summary ---
593
+ logger.info("=" * 60)
594
+ logger.info("Results Summary")
595
+ logger.info("=" * 60)
596
+ for method_key, metrics in results.items():
597
+ logger.info(" %s:", method_key)
598
+ logger.info(" Spearman r = %.4f (p_perm = %.4f)", metrics["spearman_r"], metrics["spearman_p_perm"])
599
+ logger.info(" AUROC = %.4f (p_perm = %.4f)", metrics["auroc"], metrics["auroc_p_perm"])
600
+ for k in top_k_fracs:
601
+ logger.info(" Precision@%.2f = %.4f", k, metrics[f"precision@{k}"])
602
+ logger.info("Output saved to: %s", output_dir)
603
+
604
+
605
+ if __name__ == "__main__":
606
+ main()
BioMod/interpretability/attention/module_gene_attention.py CHANGED
@@ -1,1090 +1,1093 @@
1
- """ModuleEmbedding Attention-based Gene Interpretability Analysis (Streaming).
2
-
3
- Interpretability analysis using single-head content-based attention pooling in ModuleEmbedding.
4
- Streaming computing mode: memory usage O(n_celltypes × n_genes).
5
-
6
- Analysis content:
7
- 1. Global gene importance ranking
8
- 2. Cell type specificity (z-score, Cohen's d, Welch's t)
9
- 3. Marker gene verification
10
- 4. Attention-Expression correlation
11
- 5. Random baseline comparison (KL divergence)
12
- 6. Attention concentration (Gini coefficient)
13
- 7. Pathway Enrichment Analysis
14
- 8. DEG comparison
15
-
16
- Usage:
17
- python module_gene_attention.py \
18
- --performer_checkpoint ../../mae_tost_48M_cross_attn_only_pure_memory/mae_tost_48M_cross_attn_only_pure_memory_epoch12.pth \
19
- --config_pkl ../../mae_tost_48M_cross_attn_only_pure_memory/mae_tost_48M_cross_attn_only_pure_memory.pkl \
20
- --module_data_dir ../../module_data_all \
21
- --gene_list_path ../../../selected_genes_27k.txt \
22
- --gene_mapping_path ../../../gene2ens.csv \
23
- --gene2vec_data_path ../../../selected_gene2vec_27k.npy \
24
- --gene2ens_file ../../../gene2ens.csv \
25
- --data_path ../../../zero-shot-batch/pancreas_scib.h5ad \
26
- --celltype_column celltype \
27
- --batch_size 64 \
28
- --output_dir ../results/gene_attention/pancreas
29
- """
30
- from __future__ import annotations
31
-
32
- import argparse
33
- import json
34
- import logging
35
- import sys
36
- from pathlib import Path
37
- from typing import Dict, List, Optional, Tuple
38
-
39
- import numpy as np
40
- import pandas as pd
41
- import torch
42
- from torch.utils.data import DataLoader
43
-
44
- import matplotlib
45
- matplotlib.use("Agg")
46
- import matplotlib.pyplot as plt
47
- import seaborn as sns
48
-
49
- from scipy.stats import spearmanr, t as t_dist
50
-
51
- HERE = Path(__file__).resolve()
52
- PERFORMER_MODULE = HERE.parents[2]
53
- REPO_ROOT = HERE.parents[3]
54
- for p in (str(PERFORMER_MODULE), str(REPO_ROOT)):
55
- if p not in sys.path:
56
- sys.path.insert(0, p)
57
-
58
- # Publication utilities
59
- MODULE_IMPORTANCE_DIR = PERFORMER_MODULE / "interpretability" / "module_importance"
60
- if str(MODULE_IMPORTANCE_DIR) not in sys.path:
61
- sys.path.insert(0, str(MODULE_IMPORTANCE_DIR))
62
-
63
- from pub_utils import setup_publication_style, savefig as pub_savefig
64
-
65
- from cell_annotation.performer_module_finetune import (
66
- DataPreprocessor,
67
- load_backbone,
68
- seed_all,
69
- )
70
- from interpretability.utils import (
71
- MatrixRowDataset,
72
- build_module_metadata,
73
- ensure_dir,
74
- load_module_names,
75
- )
76
-
77
- logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
78
- logger = logging.getLogger(__name__)
79
-
80
-
81
- # ---------------------------------------------------------------------------
82
- # Argparse
83
- # ---------------------------------------------------------------------------
84
-
85
- def parse_args() -> argparse.Namespace:
86
- p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
87
-
88
- p.add_argument("--performer_checkpoint", type=str, required=True)
89
- p.add_argument("--config_pkl", type=str, required=True)
90
- p.add_argument("--module_data_dir", type=str, default="../../module_data_all")
91
- p.add_argument("--module_json", type=str, default=None)
92
- p.add_argument("--gene_list_path", type=str, default="../../../selected_genes_27k.txt")
93
- p.add_argument("--gene_mapping_path", type=str, default="../../../gene2ens.csv")
94
- p.add_argument("--gene2vec_data_path", type=str, default="../../../selected_gene2vec_27k.npy")
95
- p.add_argument("--gene2ens_file", type=str, default="../../../gene2ens.csv")
96
- p.add_argument("--gene2ens_column", type=str, default="Ensembl gene ID")
97
- p.add_argument("--shuffle_module_genes", action="store_true")
98
- p.add_argument("--shuffle_seed", type=int, default=2021)
99
-
100
- p.add_argument("--data_path", type=str, required=True)
101
- p.add_argument("--celltype_column", type=str, default="celltype")
102
- p.add_argument("--gene_name_column", type=str, default="auto")
103
- p.add_argument("--dataset_name", type=str, default="gene_attention")
104
- p.add_argument("--max_cells", type=int, default=None)
105
-
106
- p.add_argument("--batch_size", type=int, default=64)
107
- p.add_argument("--num_workers", type=int, default=0)
108
- p.add_argument("--device", type=str, default="cuda")
109
- p.add_argument("--seed", type=int, default=2021)
110
-
111
- p.add_argument("--top_k_genes", type=int, default=20)
112
- p.add_argument("--marker_db", type=str, default=None)
113
- p.add_argument("--deg_csv", type=str, default=None)
114
- p.add_argument("--min_cells", type=int, default=30)
115
-
116
- p.add_argument("--output_dir", type=str, required=True)
117
- return p.parse_args()
118
-
119
-
120
- # ---------------------------------------------------------------------------
121
- # Built-in pancreas marker genes (from PanglaoDB + CellMarker)
122
- # ---------------------------------------------------------------------------
123
-
124
- PANCREAS_MARKERS: Dict[str, List[str]] = {
125
- "alpha": ["GCG", "ARX", "IRX2", "TTR", "LOXL4", "GC", "CRYBA2", "DPP4", "IRX1", "PCSK2"],
126
- "beta": ["INS", "IAPP", "MAFA", "NKX6-1", "PDX1", "HADH", "SLC30A8", "NPTX2", "DLK1", "PCSK1"],
127
- "delta": ["SST", "HHEX", "RBP4", "LEPR", "PCSK1", "SEC11C"],
128
- "gamma": ["PPY", "SERTM1", "CARTPT", "MEIS2", "ETV1", "ABCC9"],
129
- "ductal": ["KRT19", "KRT7", "SPP1", "CFTR", "MUC1", "AMBP", "ANXA4", "BICC1", "SOX9"],
130
- "acinar": ["PRSS1", "CELA3A", "CELA2A", "CPA1", "CPA2", "PNLIP", "CEL", "CLPS", "REG1A", "CTRB1"],
131
- "activated_stellate": ["COL1A1", "COL1A2", "COL3A1", "ACTA2", "TAGLN", "RGS5", "PDGFRB"],
132
- "endothelial": ["PECAM1", "VWF", "CDH5", "KDR", "ENG", "PLVAP", "ESAM"],
133
- "macrophage": ["CD68", "CD163", "CSF1R", "MARCO", "MSR1", "C1QA", "C1QB"],
134
- "epsilon": ["GHRL", "PCSK1", "ARX"],
135
- }
136
-
137
- PANCREAS_COLORS: Dict[str, str] = {
138
- "alpha": "#E41A1C", "beta": "#377EB8", "delta": "#4DAF4A",
139
- "gamma": "#984EA3", "ductal": "#FF7F00", "acinar": "#A65628",
140
- "activated_stellate": "#F781BF", "endothelial": "#66C2A5",
141
- "macrophage": "#FC8D62", "epsilon": "#8DA0CB", "t_cell": "#E78AC3",
142
- "schwann": "#B3B3B3", "mast": "#FDBF6F", "quiescent_stellate": "#CAB2D6",
143
- }
144
-
145
-
146
- # ---------------------------------------------------------------------------
147
- # Streaming attention extraction + statistics accumulation
148
- # ---------------------------------------------------------------------------
149
-
150
- @torch.no_grad()
151
- def extract_attention_streaming(
152
- backbone, loader, preprocessor, reindex_state, device,
153
- cell_labels: np.ndarray, num_modules: int, module_sizes: List[int],
154
- ) -> Tuple:
155
- """Extract attention and accumulate statistics in streaming fashion.
156
-
157
- Returns:
158
- global_mean, global_std, ct_mean, ct_std, ct_count, n_cells,
159
- expr_mean, attn_expr_prod_sum
160
- """
161
- backbone.eval()
162
- module_embed = backbone.module_embed
163
-
164
- global_sum = {i: np.zeros(module_sizes[i], dtype=np.float64) for i in range(num_modules)}
165
- global_sum_sq = {i: np.zeros(module_sizes[i], dtype=np.float64) for i in range(num_modules)}
166
- expr_sum = {i: np.zeros(module_sizes[i], dtype=np.float64) for i in range(num_modules)}
167
- attn_expr_prod_sum = {i: np.zeros(module_sizes[i], dtype=np.float64) for i in range(num_modules)}
168
-
169
- celltypes = np.unique(cell_labels)
170
- ct_sum = {ct: {i: np.zeros(module_sizes[i], dtype=np.float64) for i in range(num_modules)} for ct in celltypes}
171
- ct_sum_sq = {ct: {i: np.zeros(module_sizes[i], dtype=np.float64) for i in range(num_modules)} for ct in celltypes}
172
- ct_count = {ct: 0 for ct in celltypes}
173
-
174
- n_cells = 0
175
- cell_idx = 0
176
-
177
- for batch_idx, x in enumerate(loader):
178
- x = x.to(device)
179
- if reindex_state is not None:
180
- x = preprocessor.reindex_tensor_to_model_space(x, reindex_state=reindex_state)
181
-
182
- batch_size = x.shape[0]
183
- batch_labels = cell_labels[cell_idx:cell_idx + batch_size]
184
- cell_idx += batch_size
185
-
186
- for module_i, gene_idx in enumerate(module_embed.module_sets):
187
- gene_idx_dev = gene_idx.to(device)
188
- expr_slice = torch.index_select(x, dim=1, index=gene_idx_dev)
189
-
190
- gene_emb = module_embed.gene_embed(gene_idx_dev)
191
- gene_emb = gene_emb.unsqueeze(0).expand(batch_size, -1, -1)
192
- gene_tok = module_embed._fuse_expr_and_gene_emb(expr_slice, gene_emb)
193
- gene_tok = module_embed.proj(gene_tok)
194
-
195
- attn = module_embed.attn_score(gene_tok).squeeze(-1)
196
- attn = torch.softmax(attn, dim=1)
197
- attn_np = attn.cpu().numpy().astype(np.float64)
198
-
199
- # Expression for correlation
200
- expr_np = expr_slice.cpu().numpy().astype(np.float64)
201
- expr_sum[module_i] += expr_np.sum(axis=0)
202
- attn_expr_prod_sum[module_i] += (attn_np * expr_np).sum(axis=0)
203
-
204
- # Global accumulation
205
- global_sum[module_i] += attn_np.sum(axis=0)
206
- global_sum_sq[module_i] += (attn_np ** 2).sum(axis=0)
207
-
208
- # Per-celltype accumulation
209
- for ct in celltypes:
210
- mask = batch_labels == ct
211
- if mask.any():
212
- ct_attn = attn_np[mask]
213
- ct_sum[ct][module_i] += ct_attn.sum(axis=0)
214
- ct_sum_sq[ct][module_i] += (ct_attn ** 2).sum(axis=0)
215
-
216
- for ct in celltypes:
217
- ct_count[ct] += int((batch_labels == ct).sum())
218
-
219
- n_cells += batch_size
220
- if (batch_idx + 1) % 50 == 0:
221
- logger.info(" [attention] batch %d, %d cells processed", batch_idx + 1, n_cells)
222
-
223
- # Compute means and stds
224
- global_mean = {}
225
- global_std = {}
226
- for i in range(num_modules):
227
- global_mean[i] = (global_sum[i] / n_cells).astype(np.float32)
228
- variance = global_sum_sq[i] / n_cells - global_mean[i].astype(np.float64) ** 2
229
- global_std[i] = np.sqrt(np.maximum(variance, 0)).astype(np.float32)
230
-
231
- ct_mean = {}
232
- ct_std = {}
233
- for ct in celltypes:
234
- ct_mean[ct] = {}
235
- ct_std[ct] = {}
236
- for i in range(num_modules):
237
- if ct_count[ct] > 0:
238
- ct_mean[ct][i] = (ct_sum[ct][i] / ct_count[ct]).astype(np.float32)
239
- else:
240
- ct_mean[ct][i] = np.zeros(module_sizes[i], dtype=np.float32)
241
- if ct_count[ct] > 1:
242
- var = ct_sum_sq[ct][i] / ct_count[ct] - ct_mean[ct][i].astype(np.float64) ** 2
243
- ct_std[ct][i] = np.sqrt(np.maximum(var, 0)).astype(np.float32)
244
- else:
245
- ct_std[ct][i] = np.zeros(module_sizes[i], dtype=np.float32)
246
-
247
- expr_mean = {i: (expr_sum[i] / n_cells).astype(np.float32) for i in range(num_modules)}
248
-
249
- logger.info("Extracted attention for %d cells across %d modules (streaming)", n_cells, num_modules)
250
- return global_mean, global_std, ct_mean, ct_std, ct_count, n_cells, expr_mean, attn_expr_prod_sum
251
-
252
-
253
- # ---------------------------------------------------------------------------
254
- # Analysis functions
255
- # ---------------------------------------------------------------------------
256
-
257
- def build_global_importance_df(
258
- global_mean: Dict[int, np.ndarray],
259
- global_std: Dict[int, np.ndarray],
260
- module_gene_names: Dict[int, List[str]],
261
- num_modules: int,
262
- ) -> pd.DataFrame:
263
- """Build global gene importance DataFrame from streaming means."""
264
- rows = []
265
- for module_i in range(num_modules):
266
- mean_attn = global_mean[module_i]
267
- std_attn = global_std[module_i]
268
- genes = module_gene_names[module_i]
269
- rank_order = np.argsort(-mean_attn)
270
- for g_idx, gene in enumerate(genes):
271
- rows.append({
272
- "module_idx": module_i,
273
- "gene": gene,
274
- "mean_attention": float(mean_attn[g_idx]),
275
- "std_attention": float(std_attn[g_idx]),
276
- "rank_in_module": int(np.where(rank_order == g_idx)[0][0]) + 1,
277
- "module_size": len(genes),
278
- })
279
- df = pd.DataFrame(rows)
280
- df["normalized_rank"] = df["rank_in_module"] / df["module_size"]
281
- return df
282
-
283
-
284
- def build_celltype_dfs(
285
- ct_mean: Dict[str, Dict[int, np.ndarray]],
286
- ct_count: Dict[str, int],
287
- module_gene_names: Dict[int, List[str]],
288
- num_modules: int,
289
- ) -> Dict[str, pd.DataFrame]:
290
- """Build per-celltype DataFrames from streaming means."""
291
- ct_dfs = {}
292
- for ct, module_means in ct_mean.items():
293
- rows = []
294
- for module_i in range(num_modules):
295
- mean_attn = module_means[module_i]
296
- genes = module_gene_names[module_i]
297
- for g_idx, gene in enumerate(genes):
298
- rows.append({
299
- "module_idx": module_i,
300
- "gene": gene,
301
- "mean_attention": float(mean_attn[g_idx]),
302
- "n_cells": ct_count[ct],
303
- })
304
- ct_dfs[ct] = pd.DataFrame(rows)
305
- return ct_dfs
306
-
307
-
308
- def compute_specificity_from_means(
309
- global_mean: Dict[int, np.ndarray],
310
- ct_mean: Dict[str, Dict[int, np.ndarray]],
311
- module_gene_names: Dict[int, List[str]],
312
- num_modules: int,
313
- ) -> pd.DataFrame:
314
- """Fold-change specificity (legacy): max_ct(mean_attn_ct) / global_mean_attn."""
315
- celltypes = list(ct_mean.keys())
316
- rows = []
317
- for module_i in range(num_modules):
318
- genes = module_gene_names[module_i]
319
- g_mean = global_mean[module_i]
320
- for g_idx, gene in enumerate(genes):
321
- ct_scores = {ct: float(ct_mean[ct][module_i][g_idx]) for ct in celltypes}
322
- top_ct = max(ct_scores, key=ct_scores.get)
323
- top_score = ct_scores[top_ct]
324
- fold_change = top_score / max(float(g_mean[g_idx]), 1e-10)
325
- rows.append({
326
- "module_idx": module_i, "gene": gene, "top_celltype": top_ct,
327
- "top_ct_mean_attn": top_score, "global_mean_attn": float(g_mean[g_idx]),
328
- "fold_change": fold_change,
329
- })
330
- return pd.DataFrame(rows)
331
-
332
-
333
- def compute_specificity_zscore(
334
- global_mean: Dict[int, np.ndarray],
335
- global_std: Dict[int, np.ndarray],
336
- ct_mean: Dict[str, Dict[int, np.ndarray]],
337
- ct_count: Dict[str, int],
338
- module_gene_names: Dict[int, List[str]],
339
- num_modules: int,
340
- min_cells: int = 30,
341
- ) -> pd.DataFrame:
342
- """Z-score specificity: (ct_mean - global_mean) / global_std."""
343
- valid_cts = [ct for ct, n in ct_count.items() if n >= min_cells]
344
- rows = []
345
- for module_i in range(num_modules):
346
- genes = module_gene_names[module_i]
347
- g_mean = global_mean[module_i]
348
- g_std = global_std[module_i]
349
- for g_idx, gene in enumerate(genes):
350
- std_val = max(float(g_std[g_idx]), 1e-8)
351
- ct_zscores = {}
352
- for ct in valid_cts:
353
- z = (float(ct_mean[ct][module_i][g_idx]) - float(g_mean[g_idx])) / std_val
354
- ct_zscores[ct] = z
355
- top_ct = max(ct_zscores, key=ct_zscores.get)
356
- rows.append({
357
- "module_idx": module_i, "gene": gene, "top_celltype": top_ct,
358
- "z_score": ct_zscores[top_ct],
359
- "global_mean_attn": float(g_mean[g_idx]),
360
- "ct_mean_attn": float(ct_mean[top_ct][module_i][g_idx]),
361
- })
362
- return pd.DataFrame(rows)
363
-
364
-
365
- def compute_cohens_d(
366
- global_mean: Dict[int, np.ndarray],
367
- ct_mean: Dict[str, Dict[int, np.ndarray]],
368
- ct_std: Dict[str, Dict[int, np.ndarray]],
369
- ct_count: Dict[str, int],
370
- module_gene_names: Dict[int, List[str]],
371
- num_modules: int,
372
- n_cells: int,
373
- min_cells: int = 30,
374
- ) -> pd.DataFrame:
375
- """Cohen's d effect size for cell-type specificity."""
376
- valid_cts = [ct for ct, n in ct_count.items() if n >= min_cells]
377
- rows = []
378
- for module_i in range(num_modules):
379
- genes = module_gene_names[module_i]
380
- for g_idx, gene in enumerate(genes):
381
- best_d = -np.inf
382
- best_ct = valid_cts[0]
383
- for ct in valid_cts:
384
- n_ct = ct_count[ct]
385
- n_other = n_cells - n_ct
386
- ct_m = float(ct_mean[ct][module_i][g_idx])
387
- ct_s = float(ct_std[ct][module_i][g_idx])
388
- other_m = (float(global_mean[module_i][g_idx]) * n_cells - ct_m * n_ct) / max(n_other, 1)
389
- pooled = max(ct_s, 1e-10)
390
- d = (ct_m - other_m) / pooled
391
- if d > best_d:
392
- best_d = d
393
- best_ct = ct
394
- rows.append({"module_idx": module_i, "gene": gene, "top_celltype": best_ct, "cohens_d": best_d})
395
- return pd.DataFrame(rows)
396
-
397
-
398
- def compute_welch_ttest(
399
- global_mean: Dict[int, np.ndarray],
400
- global_std: Dict[int, np.ndarray],
401
- ct_mean: Dict[str, Dict[int, np.ndarray]],
402
- ct_std: Dict[str, Dict[int, np.ndarray]],
403
- ct_count: Dict[str, int],
404
- module_gene_names: Dict[int, List[str]],
405
- num_modules: int,
406
- n_cells: int,
407
- min_cells: int = 30,
408
- ) -> pd.DataFrame:
409
- """Welch's t-test from streaming statistics with BH FDR correction."""
410
- valid_cts = [ct for ct, n in ct_count.items() if n >= min_cells]
411
- rows = []
412
- for module_i in range(num_modules):
413
- genes = module_gene_names[module_i]
414
- for g_idx, gene in enumerate(genes):
415
- best_t = -np.inf
416
- best_ct = valid_cts[0]
417
- best_pval = 1.0
418
- for ct in valid_cts:
419
- n_ct = ct_count[ct]
420
- ct_m = float(ct_mean[ct][module_i][g_idx])
421
- ct_v = float(ct_std[ct][module_i][g_idx]) ** 2
422
- g_m = float(global_mean[module_i][g_idx])
423
- g_v = float(global_std[module_i][g_idx]) ** 2
424
- se = np.sqrt(ct_v / max(n_ct, 1) + g_v / max(n_cells, 1))
425
- if se < 1e-12:
426
- continue
427
- t_stat = (ct_m - g_m) / se
428
- num = (ct_v / n_ct + g_v / n_cells) ** 2
429
- denom = (ct_v / n_ct) ** 2 / max(n_ct - 1, 1) + (g_v / n_cells) ** 2 / max(n_cells - 1, 1)
430
- df = num / max(denom, 1e-12)
431
- pval = float(t_dist.sf(abs(t_stat), df) * 2)
432
- if t_stat > best_t:
433
- best_t = t_stat
434
- best_ct = ct
435
- best_pval = pval
436
- rows.append({"module_idx": module_i, "gene": gene, "top_celltype": best_ct,
437
- "t_stat": best_t, "pval": best_pval})
438
-
439
- df = pd.DataFrame(rows)
440
- # BH FDR correction
441
- pvals = df["pval"].values
442
- n = len(pvals)
443
- sorted_idx = np.argsort(pvals)
444
- sorted_pvals = pvals[sorted_idx]
445
- fdr = np.minimum(1.0, sorted_pvals * n / (np.arange(n) + 1))
446
- for i in range(n - 2, -1, -1):
447
- fdr[i] = min(fdr[i], fdr[i + 1])
448
- fdr_all = np.empty(n)
449
- fdr_all[sorted_idx] = fdr
450
- df["fdr"] = fdr_all
451
- return df
452
-
453
-
454
- def compute_attention_expression_corr(
455
- global_mean: Dict[int, np.ndarray],
456
- expr_mean: Dict[int, np.ndarray],
457
- module_gene_names: Dict[int, List[str]],
458
- num_modules: int,
459
- ) -> Dict[str, float]:
460
- """Correlation between mean attention and mean expression across genes."""
461
- all_attn = []
462
- all_expr = []
463
- for module_i in range(num_modules):
464
- all_attn.extend(global_mean[module_i].tolist())
465
- all_expr.extend(expr_mean[module_i].tolist())
466
- all_attn = np.array(all_attn)
467
- all_expr = np.array(all_expr)
468
- mask = (all_attn > 0) & (all_expr > 0)
469
- if mask.sum() > 10:
470
- pearson_r = float(np.corrcoef(all_attn[mask], all_expr[mask])[0, 1])
471
- sp_r, sp_p = spearmanr(all_attn[mask], all_expr[mask])
472
- else:
473
- pearson_r, sp_r, sp_p = 0.0, 0.0, 1.0
474
- return {"pearson_r": pearson_r, "spearman_r": float(sp_r),
475
- "spearman_pval": float(sp_p), "n_genes": int(mask.sum())}
476
-
477
-
478
- def compute_random_baseline(
479
- global_mean: Dict[int, np.ndarray],
480
- num_modules: int,
481
- ) -> Dict[str, float]:
482
- """Compare learned attention to uniform baseline."""
483
- kl_divs = []
484
- learned_entropies = []
485
- for module_i in range(num_modules):
486
- attn = global_mean[module_i].astype(np.float64)
487
- attn = attn / max(attn.sum(), 1e-10)
488
- g_size = len(attn)
489
- uniform = np.ones(g_size) / g_size
490
- mask = attn > 1e-12
491
- kl = float(np.sum(attn[mask] * np.log(attn[mask] / uniform[mask])))
492
- kl_divs.append(kl)
493
- h_learned = -float(np.sum(attn[mask] * np.log(attn[mask])))
494
- h_max = np.log(g_size)
495
- learned_entropies.append(h_learned / max(h_max, 1e-10))
496
- return {
497
- "mean_kl_divergence": float(np.mean(kl_divs)),
498
- "std_kl_divergence": float(np.std(kl_divs)),
499
- "mean_normalized_entropy": float(np.mean(learned_entropies)),
500
- "entropy_reduction": float(1.0 - np.mean(learned_entropies)),
501
- "n_modules": num_modules,
502
- }
503
-
504
-
505
- def compute_attention_concentration(
506
- global_mean: Dict[int, np.ndarray],
507
- ct_mean: Dict[str, Dict[int, np.ndarray]],
508
- ct_count: Dict[str, int],
509
- num_modules: int,
510
- min_cells: int = 30,
511
- ) -> pd.DataFrame:
512
- """Gini coefficient per module measuring attention selectivity."""
513
- def gini(arr):
514
- arr = np.sort(arr.flatten())
515
- n = len(arr)
516
- if n == 0 or arr.sum() == 0:
517
- return 0.0
518
- index = np.arange(1, n + 1)
519
- return float((2 * np.sum(index * arr) - (n + 1) * np.sum(arr)) / (n * np.sum(arr)))
520
-
521
- valid_cts = [ct for ct, n in ct_count.items() if n >= min_cells]
522
- rows = []
523
- for module_i in range(num_modules):
524
- g_attn = global_mean[module_i]
525
- g_gini = gini(g_attn)
526
- ct_ginis = [gini(ct_mean[ct][module_i]) for ct in valid_cts]
527
- rows.append({
528
- "module_idx": module_i, "global_gini": g_gini,
529
- "mean_ct_gini": float(np.mean(ct_ginis)) if ct_ginis else 0.0,
530
- "module_size": len(g_attn),
531
- })
532
- return pd.DataFrame(rows)
533
-
534
-
535
- def compute_marker_rank_within_celltype(
536
- ct_mean: Dict[str, Dict[int, np.ndarray]],
537
- ct_count: Dict[str, int],
538
- module_gene_names: Dict[int, List[str]],
539
- num_modules: int,
540
- marker_dict: Dict[str, List[str]],
541
- min_cells: int = 30,
542
- ) -> pd.DataFrame:
543
- """Rank marker genes within their expected cell type by attention."""
544
- rows = []
545
- for ct, markers in marker_dict.items():
546
- if ct not in ct_count or ct_count[ct] < min_cells:
547
- continue
548
- gene_attn = {}
549
- for module_i in range(num_modules):
550
- genes = module_gene_names[module_i]
551
- attn = ct_mean[ct][module_i]
552
- for g_idx, gene in enumerate(genes):
553
- if gene not in gene_attn or float(attn[g_idx]) > gene_attn[gene]:
554
- gene_attn[gene] = float(attn[g_idx])
555
- sorted_genes = sorted(gene_attn.items(), key=lambda x: -x[1])
556
- gene_rank = {g: rank + 1 for rank, (g, _) in enumerate(sorted_genes)}
557
- n_total = len(gene_rank)
558
- for marker in markers:
559
- if marker in gene_rank:
560
- rank = gene_rank[marker]
561
- rows.append({
562
- "celltype": ct, "marker_gene": marker, "rank": rank,
563
- "percentile": rank / n_total * 100, "n_total_genes": n_total,
564
- "attention": gene_attn[marker],
565
- })
566
- return pd.DataFrame(rows)
567
-
568
-
569
- # ---------------------------------------------------------------------------
570
- # Marker gene validation
571
- # ---------------------------------------------------------------------------
572
-
573
- def validate_markers(
574
- specificity_df: pd.DataFrame,
575
- marker_dict: Dict[str, List[str]],
576
- sort_col: str = "fold_change",
577
- ) -> Dict[str, object]:
578
- """Validate whether high-attention genes match known markers."""
579
- results = {}
580
- all_genes_in_model = set(specificity_df["gene"].unique())
581
- for ct, markers in marker_dict.items():
582
- markers_in_model = [g for g in markers if g in all_genes_in_model]
583
- if not markers_in_model:
584
- continue
585
- ct_specific = specificity_df[specificity_df["top_celltype"] == ct].copy()
586
- ct_specific = ct_specific.sort_values(sort_col, ascending=False).reset_index(drop=True)
587
- marker_set = set(markers_in_model)
588
- n_total = len(ct_specific)
589
- if n_total == 0:
590
- continue
591
- top_50_genes = set(ct_specific.head(50)["gene"])
592
- top_100_genes = set(ct_specific.head(100)["gene"])
593
- hits_50 = len(marker_set & top_50_genes)
594
- hits_100 = len(marker_set & top_100_genes)
595
- marker_rows = ct_specific[ct_specific["gene"].isin(marker_set)]
596
- if len(marker_rows) > 0:
597
- marker_ranks = (marker_rows.index + 1).tolist()
598
- mean_rank = np.mean(marker_ranks)
599
- median_rank = np.median(marker_ranks)
600
- else:
601
- mean_rank = median_rank = n_total
602
- results[ct] = {
603
- "n_markers_in_model": len(markers_in_model),
604
- "markers_in_model": markers_in_model,
605
- "n_genes_assigned_to_ct": n_total,
606
- "hits_in_top50": hits_50, "hits_in_top100": hits_100,
607
- "precision_at_50": hits_50 / min(50, n_total),
608
- "precision_at_100": hits_100 / min(100, n_total),
609
- "marker_mean_rank": float(mean_rank),
610
- "marker_median_rank": float(median_rank),
611
- "recall_at_50": hits_50 / len(markers_in_model) if markers_in_model else 0,
612
- "recall_at_100": hits_100 / len(markers_in_model) if markers_in_model else 0,
613
- }
614
- return results
615
-
616
-
617
- def validate_deg_correlation(
618
- ct_dfs: Dict[str, pd.DataFrame],
619
- deg_df: pd.DataFrame,
620
- ) -> Dict[str, Dict[str, float]]:
621
- """Compare attention ranking with DEG ranking per cell type."""
622
- results = {}
623
- for ct, attn_df in ct_dfs.items():
624
- ct_deg = deg_df[deg_df["cell_type"] == ct].copy()
625
- if len(ct_deg) == 0:
626
- continue
627
- merged = attn_df.merge(ct_deg[["gene", "log2fc"]], on="gene", how="inner")
628
- if len(merged) < 10:
629
- continue
630
- rho, pval = spearmanr(merged["mean_attention"], merged["log2fc"].abs())
631
- results[ct] = {"spearman_r": float(rho), "pval": float(pval), "n_genes": len(merged)}
632
- return results
633
-
634
-
635
- # ---------------------------------------------------------------------------
636
- # Publication-quality visualization
637
- # ---------------------------------------------------------------------------
638
-
639
- def plot_global_importance_pub(global_df, specificity_zscore_df, top_k, output_dir):
640
- """Fig1: Top-K genes with std error bars, colored by top cell type."""
641
- setup_publication_style()
642
- gene_stats = global_df.groupby("gene").agg(
643
- mean_attn=("mean_attention", "mean"), std_attn=("std_attention", "mean"),
644
- ).nlargest(top_k, "mean_attn")
645
- gene_ct = specificity_zscore_df.drop_duplicates("gene").set_index("gene")["top_celltype"]
646
- fig, ax = plt.subplots(figsize=(4.5, 5))
647
- genes = gene_stats.index.tolist()
648
- y_pos = np.arange(len(genes))
649
- colors = [PANCREAS_COLORS.get(gene_ct.get(g, ""), "#999999") for g in genes]
650
- ax.barh(y_pos, gene_stats["mean_attn"], xerr=gene_stats["std_attn"],
651
- color=colors, edgecolor="none", height=0.7, capsize=2, error_kw={"linewidth": 0.8})
652
- ax.set_yticks(y_pos)
653
- ax.set_yticklabels(genes)
654
- ax.set_xlabel("Mean attention weight")
655
- ax.set_title("Top genes by global attention")
656
- ax.invert_yaxis()
657
- pub_savefig(fig, output_dir / "fig1_global_importance")
658
- logger.info("Saved fig1_global_importance (PDF+PNG)")
659
-
660
-
661
- def plot_celltype_heatmap_pub(ct_dfs, ct_count, top_k, output_dir, min_cells=30):
662
- """Fig2: Clustered heatmap of gene attention (z-score normalized)."""
663
- setup_publication_style()
664
- valid_cts = {ct for ct, n in ct_count.items() if n >= min_cells}
665
- all_genes_attn = {}
666
- for ct, df in ct_dfs.items():
667
- if ct not in valid_cts:
668
- continue
669
- all_genes_attn[ct] = df.groupby("gene")["mean_attention"].mean()
670
- combined = pd.DataFrame(all_genes_attn)
671
- gene_var = combined.var(axis=1).nlargest(top_k).index
672
- heatmap_data = combined.loc[gene_var]
673
- row_mean = heatmap_data.mean(axis=1)
674
- row_std = heatmap_data.std(axis=1).replace(0, 1)
675
- heatmap_z = heatmap_data.sub(row_mean, axis=0).div(row_std, axis=0)
676
- g = sns.clustermap(heatmap_z, cmap="RdBu_r", center=0, figsize=(6, 7),
677
- xticklabels=True, yticklabels=True,
678
- dendrogram_ratio=(0.1, 0.15), cbar_kws={"label": "Z-score"},
679
- linewidths=0.3)
680
- g.ax_heatmap.set_xlabel("")
681
- g.ax_heatmap.set_ylabel("")
682
- g.fig.suptitle(f"Top {top_k} variable genes (z-scored attention)", y=1.02, fontsize=11)
683
- pub_savefig(g.fig, output_dir / "fig2_celltype_heatmap")
684
- logger.info("Saved fig2_celltype_heatmap (PDF+PNG)")
685
-
686
-
687
- def plot_specificity_volcano(welch_df, zscore_df, top_k, output_dir):
688
- """Fig3: Volcano plot (z-score vs -log10 FDR)."""
689
- setup_publication_style()
690
- merged = zscore_df[["gene", "top_celltype", "z_score"]].drop_duplicates("gene")
691
- welch_best = welch_df.loc[welch_df.groupby("gene")["t_stat"].idxmax()][["gene", "fdr"]]
692
- merged = merged.merge(welch_best, on="gene", how="left")
693
- merged["neg_log10_fdr"] = -np.log10(merged["fdr"].clip(lower=1e-300))
694
- fig, ax = plt.subplots(figsize=(5.5, 5))
695
- for ct in merged["top_celltype"].unique():
696
- subset = merged[merged["top_celltype"] == ct]
697
- color = PANCREAS_COLORS.get(ct, "#999999")
698
- ax.scatter(subset["z_score"], subset["neg_log10_fdr"],
699
- c=color, s=8, alpha=0.5, label=ct, rasterized=True)
700
- top_genes = merged.nlargest(top_k, "z_score")
701
- for _, row in top_genes.iterrows():
702
- ax.annotate(row["gene"], (row["z_score"], row["neg_log10_fdr"]),
703
- fontsize=6, ha="left", va="bottom")
704
- ax.axhline(-np.log10(0.05), color="gray", linestyle="--", linewidth=0.6)
705
- ax.axvline(2, color="gray", linestyle="--", linewidth=0.6)
706
- ax.set_xlabel("Z-score (cell-type specificity)")
707
- ax.set_ylabel(r"$-\log_{10}$(FDR)")
708
- ax.set_title("Gene specificity volcano")
709
- ax.legend(bbox_to_anchor=(1.02, 1), loc="upper left", fontsize=7, frameon=False, markerscale=2)
710
- pub_savefig(fig, output_dir / "fig3_specificity_volcano")
711
- logger.info("Saved fig3_specificity_volcano (PDF+PNG)")
712
-
713
-
714
- def plot_marker_dotplot(marker_rank_df, output_dir):
715
- """Fig4: Dot plot of marker gene rank percentile per cell type."""
716
- setup_publication_style()
717
- if marker_rank_df.empty:
718
- return
719
- fig, ax = plt.subplots(figsize=(6, 4.5))
720
- celltypes = marker_rank_df["celltype"].unique()
721
- ct_positions = {ct: i for i, ct in enumerate(celltypes)}
722
- for _, row in marker_rank_df.iterrows():
723
- y = ct_positions[row["celltype"]]
724
- size = max(10, 200 * (1 - row["percentile"] / 100))
725
- color = PANCREAS_COLORS.get(row["celltype"], "#999999")
726
- ax.scatter(row["percentile"], y, s=size, c=color, alpha=0.7,
727
- edgecolors="white", linewidth=0.3)
728
- if row["percentile"] < 5:
729
- ax.annotate(row["marker_gene"], (row["percentile"], y),
730
- fontsize=6, ha="left", va="center", xytext=(3, 0),
731
- textcoords="offset points")
732
- ax.set_yticks(range(len(celltypes)))
733
- ax.set_yticklabels(celltypes)
734
- ax.set_xlabel("Rank percentile (lower = higher attention)")
735
- ax.set_title("Marker gene rank within expected cell type")
736
- ax.axvline(5, color="gray", linestyle="--", linewidth=0.6, alpha=0.5, label="Top 5%")
737
- ax.legend(fontsize=7, frameon=False)
738
- ax.set_xlim(-2, 105)
739
- pub_savefig(fig, output_dir / "fig4_marker_dotplot")
740
- logger.info("Saved fig4_marker_dotplot (PDF+PNG)")
741
-
742
-
743
- def plot_marker_recall_pub(marker_results, ct_count, output_dir, min_cells=30):
744
- """Fig5: Marker recall bar chart with random expectation line."""
745
- setup_publication_style()
746
- if not marker_results:
747
- return
748
- cts = [ct for ct in marker_results.keys() if ct_count.get(ct, 0) >= min_cells]
749
- if not cts:
750
- return
751
- recall_50 = [marker_results[ct]["recall_at_50"] for ct in cts]
752
- recall_100 = [marker_results[ct]["recall_at_100"] for ct in cts]
753
- x = np.arange(len(cts))
754
- width = 0.35
755
- fig, ax = plt.subplots(figsize=(max(5, len(cts) * 0.7), 4))
756
- colors_50 = [PANCREAS_COLORS.get(ct, "#999999") for ct in cts]
757
- ax.bar(x - width / 2, recall_50, width, color=colors_50, alpha=0.9, label="Recall@50")
758
- ax.bar(x + width / 2, recall_100, width, color=colors_50, alpha=0.5, label="Recall@100")
759
- # Random expectation
760
- for i, ct in enumerate(cts):
761
- n_genes = marker_results[ct]["n_genes_assigned_to_ct"]
762
- if n_genes > 0:
763
- random_50 = min(50, n_genes) / n_genes
764
- ax.hlines(random_50, i - 0.4, i + 0.4, colors="gray", linestyles=":", linewidth=0.8)
765
- ax.set_xticks(x)
766
- ax.set_xticklabels(cts, rotation=45, ha="right")
767
- ax.set_ylabel("Recall")
768
- ax.set_title("Marker gene recall by cell type")
769
- ax.legend(fontsize=7, frameon=False)
770
- pub_savefig(fig, output_dir / "fig5_marker_recall")
771
- logger.info("Saved fig5_marker_recall (PDF+PNG)")
772
-
773
-
774
- def plot_attention_vs_expression(global_mean, expr_mean, module_gene_names, num_modules, output_dir):
775
- """Fig6: Scatter of mean attention vs mean expression."""
776
- setup_publication_style()
777
- all_attn, all_expr = [], []
778
- for module_i in range(num_modules):
779
- all_attn.extend(global_mean[module_i].tolist())
780
- all_expr.extend(expr_mean[module_i].tolist())
781
- all_attn = np.array(all_attn)
782
- all_expr = np.array(all_expr)
783
- mask = (all_attn > 0) & (all_expr > 0)
784
- fig, ax = plt.subplots(figsize=(4.5, 4.5))
785
- ax.scatter(all_expr[mask], all_attn[mask], s=3, alpha=0.3, c="#377EB8", rasterized=True)
786
- r = np.corrcoef(all_expr[mask], all_attn[mask])[0, 1]
787
- ax.set_xlabel("Mean expression")
788
- ax.set_ylabel("Mean attention weight")
789
- ax.set_title("Attention vs expression")
790
- ax.text(0.05, 0.95, f"Pearson r = {r:.3f}", transform=ax.transAxes,
791
- fontsize=9, va="top", ha="left")
792
- pub_savefig(fig, output_dir / "fig6_attn_vs_expression")
793
- logger.info("Saved fig6_attn_vs_expression (PDF+PNG)")
794
-
795
-
796
- def plot_concentration_pub(concentration_df, baseline_stats, output_dir):
797
- """Fig7: Gini coefficient distribution (learned vs uniform)."""
798
- setup_publication_style()
799
- fig, axes = plt.subplots(1, 2, figsize=(8, 3.5))
800
- # Panel a: Gini histogram
801
- ax = axes[0]
802
- ax.hist(concentration_df["global_gini"], bins=30, alpha=0.7, color="#377EB8", label="Global")
803
- ax.hist(concentration_df["mean_ct_gini"], bins=30, alpha=0.7, color="#E41A1C", label="Per cell-type")
804
- ax.axvline(0, color="gray", linestyle="--", linewidth=0.6)
805
- ax.set_xlabel("Gini coefficient")
806
- ax.set_ylabel("Number of modules")
807
- ax.set_title("Attention concentration")
808
- ax.legend(fontsize=7, frameon=False)
809
- # Panel b: Entropy comparison
810
- ax = axes[1]
811
- labels = ["Learned", "Uniform"]
812
- values = [baseline_stats["mean_normalized_entropy"], 1.0]
813
- bars = ax.bar(labels, values, color=["#377EB8", "#CCCCCC"], width=0.5)
814
- ax.set_ylabel("Normalized entropy")
815
- ax.set_title(f"Entropy reduction: {baseline_stats['entropy_reduction']:.1%}")
816
- ax.set_ylim(0, 1.1)
817
- plt.tight_layout()
818
- pub_savefig(fig, output_dir / "fig7_concentration")
819
- logger.info("Saved fig7_concentration (PDF+PNG)")
820
-
821
-
822
- def plot_baseline_kl(baseline_stats, output_dir):
823
- """Fig8: KL divergence summary."""
824
- setup_publication_style()
825
- fig, ax = plt.subplots(figsize=(3.5, 3.5))
826
- ax.bar(["KL(learned||uniform)"], [baseline_stats["mean_kl_divergence"]],
827
- yerr=[baseline_stats["std_kl_divergence"]], color="#377EB8",
828
- capsize=5, width=0.4)
829
- ax.set_ylabel("KL divergence (nats)")
830
- ax.set_title("Deviation from uniform attention")
831
- pub_savefig(fig, output_dir / "fig8_baseline_kl")
832
- logger.info("Saved fig8_baseline_kl (PDF+PNG)")
833
-
834
-
835
- def plot_deg_correlation_pub(deg_results, output_dir):
836
- """Fig9: DEG correlation with significance."""
837
- setup_publication_style()
838
- if not deg_results:
839
- return
840
- cts = sorted(deg_results.keys(), key=lambda c: deg_results[c]["spearman_r"], reverse=True)
841
- rhos = [deg_results[ct]["spearman_r"] for ct in cts]
842
- pvals = [deg_results[ct]["pval"] for ct in cts]
843
- fig, ax = plt.subplots(figsize=(max(5, len(cts) * 0.6), 4))
844
- colors = ["#377EB8" if r > 0 else "#E41A1C" for r in rhos]
845
- bars = ax.bar(range(len(cts)), rhos, color=colors, width=0.6)
846
- for i, (bar, p) in enumerate(zip(bars, pvals)):
847
- if p < 0.001:
848
- ax.text(i, bar.get_height() + 0.02, "***", ha="center", fontsize=7)
849
- elif p < 0.01:
850
- ax.text(i, bar.get_height() + 0.02, "**", ha="center", fontsize=7)
851
- elif p < 0.05:
852
- ax.text(i, bar.get_height() + 0.02, "*", ha="center", fontsize=7)
853
- ax.set_xticks(range(len(cts)))
854
- ax.set_xticklabels(cts, rotation=45, ha="right")
855
- ax.set_ylabel("Spearman r")
856
- ax.set_title("Attention vs DEG correlation")
857
- ax.axhline(0, color="gray", linewidth=0.5)
858
- pub_savefig(fig, output_dir / "fig9_deg_correlation")
859
- logger.info("Saved fig9_deg_correlation (PDF+PNG)")
860
-
861
-
862
- # ---------------------------------------------------------------------------
863
- # Enrichment analysis (optional)
864
- # ---------------------------------------------------------------------------
865
-
866
- def run_enrichment_analysis(
867
- global_df: pd.DataFrame, top_k: int, output_dir: Path, organism: str = "human",
868
- ) -> Optional[pd.DataFrame]:
869
- """Run GO enrichment on top-K attention genes using gseapy."""
870
- try:
871
- import gseapy as gp
872
- except ImportError:
873
- logger.warning("gseapy not installed, skipping enrichment. Install: pip install gseapy")
874
- return None
875
- top_genes = global_df.groupby("gene")["mean_attention"].mean().nlargest(top_k * 5).index.tolist()
876
- logger.info("Running GO enrichment on top %d genes...", len(top_genes))
877
- try:
878
- enr = gp.enrichr(
879
- gene_list=top_genes,
880
- gene_sets=["GO_Biological_Process_2021", "KEGG_2021_Human"],
881
- organism=organism,
882
- outdir=str(output_dir / "enrichment"),
883
- no_plot=True,
884
- )
885
- results_df = enr.results
886
- results_df.to_csv(output_dir / "enrichment_results.csv", index=False)
887
- logger.info("Enrichment results saved (%d terms)", len(results_df))
888
- return results_df
889
- except Exception as e:
890
- logger.warning("Enrichment analysis failed: %s", e)
891
- return None
892
-
893
-
894
- # ---------------------------------------------------------------------------
895
- # Main
896
- # ---------------------------------------------------------------------------
897
-
898
- def main():
899
- args = parse_args()
900
- seed_all(args.seed)
901
-
902
- _script_dir = Path(__file__).resolve().parent
903
- for attr in ("performer_checkpoint", "config_pkl", "module_data_dir", "gene_list_path",
904
- "gene_mapping_path", "gene2vec_data_path", "gene2ens_file", "data_path"):
905
- val = getattr(args, attr, None)
906
- if val is not None and not Path(val).is_absolute():
907
- setattr(args, attr, str((_script_dir / val).resolve()))
908
-
909
- output_dir = ensure_dir(Path(args.output_dir))
910
- device = torch.device(args.device if torch.cuda.is_available() else "cpu")
911
- logger.info("Device: %s", device)
912
-
913
- # --- Load model ---
914
- logger.info("Loading backbone from %s", args.performer_checkpoint)
915
- backbone, module_data, model_config = load_backbone(args, device)
916
- backbone.eval()
917
-
918
- # --- Build module gene name mapping ---
919
- model_genes_file = args.gene_list_path
920
- with open(model_genes_file) as f:
921
- model_genes = [line.strip() for line in f if line.strip()]
922
-
923
- gene2ens_df = pd.read_csv(args.gene2ens_file)
924
- ens_to_symbol = dict(zip(gene2ens_df["Ensembl gene ID"], gene2ens_df.iloc[:, 0]))
925
-
926
- module_sets = backbone.module_embed.module_sets
927
- num_modules = len(module_sets)
928
- module_sizes = [len(ps) for ps in module_sets]
929
-
930
- module_gene_names: Dict[int, List[str]] = {}
931
- for module_i, gene_idx in enumerate(module_sets):
932
- names = []
933
- for idx in gene_idx.cpu().numpy():
934
- ens_id = model_genes[idx]
935
- symbol = ens_to_symbol.get(ens_id, ens_id)
936
- names.append(symbol)
937
- module_gene_names[module_i] = names
938
-
939
- logger.info("Modules: %d, total genes: %d", num_modules, sum(module_sizes))
940
-
941
- # --- Load and preprocess data ---
942
- logger.info("Loading data from %s", args.data_path)
943
- preprocessor = DataPreprocessor(
944
- gene2ens_file=args.gene2ens_file,
945
- gene2ens_column=args.gene2ens_column,
946
- model_genes_file=model_genes_file,
947
- )
948
- preprocessor.load_gene_mapping()
949
- preprocessor.load_model_genes()
950
-
951
- gene_name_col = args.gene_name_column if args.gene_name_column != "auto" else None
952
- adata, reindex_state = preprocessor.preprocess_adata(
953
- adata_path=args.data_path,
954
- max_cells=args.max_cells,
955
- dataset_name=args.dataset_name,
956
- gene_name_column=gene_name_col,
957
- )
958
- cell_labels = adata.obs[args.celltype_column].values.astype(str)
959
- logger.info("Data: %d cells, %d cell types", len(adata), len(np.unique(cell_labels)))
960
-
961
- # --- DataLoader ---
962
- if hasattr(adata.X, "toarray"):
963
- X = adata.X.toarray().astype(np.float32)
964
- else:
965
- X = np.array(adata.X, dtype=np.float32)
966
-
967
- dataset = MatrixRowDataset(X)
968
- loader = DataLoader(dataset, batch_size=args.batch_size, shuffle=False,
969
- num_workers=args.num_workers, pin_memory=True)
970
-
971
- # --- Extract attention (streaming) ---
972
- logger.info("Extracting module attention (streaming mode)...")
973
- (global_mean, global_std, ct_mean, ct_std, ct_count, n_cells,
974
- expr_mean, attn_expr_prod_sum) = extract_attention_streaming(
975
- backbone, loader, preprocessor, reindex_state, device,
976
- cell_labels, num_modules, module_sizes,
977
- )
978
-
979
- # --- Build DataFrames ---
980
- logger.info("Building analysis DataFrames...")
981
- global_df = build_global_importance_df(global_mean, global_std, module_gene_names, num_modules)
982
- ct_dfs = build_celltype_dfs(ct_mean, ct_count, module_gene_names, num_modules)
983
- specificity_df = compute_specificity_from_means(global_mean, ct_mean, module_gene_names, num_modules)
984
-
985
- # --- New metrics ---
986
- logger.info("Computing z-score specificity...")
987
- zscore_df = compute_specificity_zscore(
988
- global_mean, global_std, ct_mean, ct_count, module_gene_names, num_modules, args.min_cells)
989
-
990
- logger.info("Computing Cohen's d...")
991
- cohens_d_df = compute_cohens_d(
992
- global_mean, ct_mean, ct_std, ct_count, module_gene_names, num_modules, n_cells, args.min_cells)
993
-
994
- logger.info("Computing Welch's t-test...")
995
- welch_df = compute_welch_ttest(
996
- global_mean, global_std, ct_mean, ct_std, ct_count, module_gene_names,
997
- num_modules, n_cells, args.min_cells)
998
-
999
- logger.info("Computing attention-expression correlation...")
1000
- attn_expr_corr = compute_attention_expression_corr(
1001
- global_mean, expr_mean, module_gene_names, num_modules)
1002
-
1003
- logger.info("Computing random baseline comparison...")
1004
- baseline_stats = compute_random_baseline(global_mean, num_modules)
1005
-
1006
- logger.info("Computing attention concentration...")
1007
- concentration_df = compute_attention_concentration(
1008
- global_mean, ct_mean, ct_count, num_modules, args.min_cells)
1009
-
1010
- # --- Marker validation ---
1011
- marker_dict = PANCREAS_MARKERS
1012
- if args.marker_db:
1013
- with open(args.marker_db) as f:
1014
- marker_dict = json.load(f)
1015
-
1016
- logger.info("Computing marker ranks within cell types...")
1017
- marker_rank_df = compute_marker_rank_within_celltype(
1018
- ct_mean, ct_count, module_gene_names, num_modules, marker_dict, args.min_cells)
1019
-
1020
- marker_results_zscore = validate_markers(zscore_df, marker_dict, sort_col="z_score")
1021
- marker_results_fc = validate_markers(specificity_df, marker_dict, sort_col="fold_change")
1022
-
1023
- # --- Save CSVs ---
1024
- logger.info("Saving CSV results...")
1025
- global_df.to_csv(output_dir / "global_gene_importance.csv", index=False)
1026
- specificity_df.to_csv(output_dir / "gene_specificity.csv", index=False)
1027
- zscore_df.to_csv(output_dir / "gene_specificity_zscore.csv", index=False)
1028
- cohens_d_df.to_csv(output_dir / "gene_specificity_cohens_d.csv", index=False)
1029
- welch_df.to_csv(output_dir / "gene_specificity_welch.csv", index=False)
1030
- concentration_df.to_csv(output_dir / "attention_concentration.csv", index=False)
1031
- if not marker_rank_df.empty:
1032
- marker_rank_df.to_csv(output_dir / "marker_ranks_within_celltype.csv", index=False)
1033
- for ct, df in ct_dfs.items():
1034
- safe_name = ct.replace("/", "_").replace(" ", "_")
1035
- df.to_csv(output_dir / f"celltype_{safe_name}_attention.csv", index=False)
1036
-
1037
- with open(output_dir / "marker_validation_zscore.json", "w") as f:
1038
- json.dump(marker_results_zscore, f, indent=2, ensure_ascii=False)
1039
- with open(output_dir / "marker_validation.json", "w") as f:
1040
- json.dump(marker_results_fc, f, indent=2, ensure_ascii=False)
1041
-
1042
- # --- DEG correlation ---
1043
- deg_results = None
1044
- if args.deg_csv:
1045
- deg_df = pd.read_csv(args.deg_csv)
1046
- deg_results = validate_deg_correlation(ct_dfs, deg_df)
1047
- with open(output_dir / "deg_correlation.json", "w") as f:
1048
- json.dump(deg_results, f, indent=2)
1049
-
1050
- # --- Publication figures ---
1051
- logger.info("Generating publication figures...")
1052
- plot_global_importance_pub(global_df, zscore_df, args.top_k_genes, output_dir)
1053
- plot_celltype_heatmap_pub(ct_dfs, ct_count, args.top_k_genes, output_dir, args.min_cells)
1054
- plot_specificity_volcano(welch_df, zscore_df, args.top_k_genes, output_dir)
1055
- plot_marker_dotplot(marker_rank_df, output_dir)
1056
- plot_marker_recall_pub(marker_results_zscore, ct_count, output_dir, args.min_cells)
1057
- plot_attention_vs_expression(global_mean, expr_mean, module_gene_names, num_modules, output_dir)
1058
- plot_concentration_pub(concentration_df, baseline_stats, output_dir)
1059
- plot_baseline_kl(baseline_stats, output_dir)
1060
- if deg_results:
1061
- plot_deg_correlation_pub(deg_results, output_dir)
1062
-
1063
- # --- Enrichment ---
1064
- run_enrichment_analysis(global_df, args.top_k_genes, output_dir)
1065
-
1066
- # --- Summary ---
1067
- filtered_cts = [ct for ct, n in ct_count.items() if n < args.min_cells]
1068
- summary = {
1069
- "n_cells": n_cells,
1070
- "n_modules": num_modules,
1071
- "n_celltypes": len(ct_count),
1072
- "celltypes": {ct: int(n) for ct, n in ct_count.items()},
1073
- "min_cells_filter": args.min_cells,
1074
- "filtered_celltypes": filtered_cts,
1075
- "top_10_global_genes": global_df.groupby("gene")["mean_attention"].mean()
1076
- .nlargest(10).to_dict(),
1077
- "attn_expr_correlation": attn_expr_corr,
1078
- "baseline_comparison": baseline_stats,
1079
- "mean_gini_global": float(concentration_df["global_gini"].mean()),
1080
- "mean_gini_celltype": float(concentration_df["mean_ct_gini"].mean()),
1081
- }
1082
- with open(output_dir / "summary.json", "w") as f:
1083
- json.dump(summary, f, indent=2, ensure_ascii=False)
1084
-
1085
- logger.info("Done. Results saved to %s", output_dir)
1086
-
1087
-
1088
- if __name__ == "__main__":
1089
- main()
1090
-
 
 
 
 
1
+ """ModuleEmbedding Attention-based Gene Interpretability Analysis (Streaming).
2
+
3
+ Interpretability analysis using single-head content-based attention pooling in ModuleEmbedding.
4
+ Streaming computing mode: memory usage O(n_celltypes × n_genes).
5
+
6
+ Analysis content:
7
+ 1. Global gene importance ranking
8
+ 2. Cell type specificity (z-score, Cohen's d, Welch's t)
9
+ 3. Marker gene verification
10
+ 4. Attention-Expression correlation
11
+ 5. Random baseline comparison (KL divergence)
12
+ 6. Attention concentration (Gini coefficient)
13
+ 7. Pathway Enrichment Analysis
14
+ 8. DEG comparison
15
+
16
+ Usage (run from the BioMod repository root):
17
+ python BioMod/interpretability/attention/module_gene_attention.py \
18
+ --performer_checkpoint weight/0810model/mae_tost_70M_75mask_ec2_epoch0.pth \
19
+ --config_pkl weight/0810model/mae_tost_70M_75mask_ec2.pkl \
20
+ --module_data_dir model_dependencies/module_data_all \
21
+ --gene_list_path model_dependencies/selected_genes_27k.txt \
22
+ --gene_mapping_path model_dependencies/gene2ens.csv \
23
+ --gene2vec_data_path model_dependencies/selected_gene2vec_27k.npy \
24
+ --gene2ens_file model_dependencies/gene2ens.csv \
25
+ --data_path data/downstream/final_data/human_pancreas_norm_complexBatch_with_gene_name.h5ad \
26
+ --celltype_column CELL_TYPE_COLUMN \
27
+ --batch_size 64 \
28
+ --output_dir outputs/interpretability/gene_attention/pancreas
29
+
30
+ Replace CELL_TYPE_COLUMN with the cell-type field in the input h5ad metadata.
31
+ """
32
+ from __future__ import annotations
33
+
34
+ import argparse
35
+ import json
36
+ import logging
37
+ import sys
38
+ from pathlib import Path
39
+ from typing import Dict, List, Optional, Tuple
40
+
41
+ import numpy as np
42
+ import pandas as pd
43
+ import torch
44
+ from torch.utils.data import DataLoader
45
+
46
+ import matplotlib
47
+ matplotlib.use("Agg")
48
+ import matplotlib.pyplot as plt
49
+ import seaborn as sns
50
+
51
+ from scipy.stats import spearmanr, t as t_dist
52
+
53
+ HERE = Path(__file__).resolve()
54
+ PERFORMER_MODULE = HERE.parents[2]
55
+ REPO_ROOT = HERE.parents[3]
56
+ ASSET_ROOT = REPO_ROOT / "model_dependencies"
57
+ for p in (str(PERFORMER_MODULE), str(REPO_ROOT)):
58
+ if p not in sys.path:
59
+ sys.path.insert(0, p)
60
+
61
+ # Publication utilities
62
+ MODULE_IMPORTANCE_DIR = PERFORMER_MODULE / "interpretability" / "module_importance"
63
+ if str(MODULE_IMPORTANCE_DIR) not in sys.path:
64
+ sys.path.insert(0, str(MODULE_IMPORTANCE_DIR))
65
+
66
+ from pub_utils import setup_publication_style, savefig as pub_savefig
67
+
68
+ from cell_annotation.performer_module_finetune import (
69
+ DataPreprocessor,
70
+ load_backbone,
71
+ seed_all,
72
+ )
73
+ from interpretability.utils import (
74
+ MatrixRowDataset,
75
+ build_module_metadata,
76
+ ensure_dir,
77
+ load_module_names,
78
+ )
79
+
80
+ logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
81
+ logger = logging.getLogger(__name__)
82
+
83
+
84
+ # ---------------------------------------------------------------------------
85
+ # Argparse
86
+ # ---------------------------------------------------------------------------
87
+
88
+ def parse_args() -> argparse.Namespace:
89
+ p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
90
+
91
+ p.add_argument("--performer_checkpoint", type=str, required=True)
92
+ p.add_argument("--config_pkl", type=str, required=True)
93
+ p.add_argument("--module_data_dir", type=str, default=str(ASSET_ROOT / "module_data_all"))
94
+ p.add_argument("--module_json", type=str, default=None)
95
+ p.add_argument("--gene_list_path", type=str, default=str(ASSET_ROOT / "selected_genes_27k.txt"))
96
+ p.add_argument("--gene_mapping_path", type=str, default=str(ASSET_ROOT / "gene2ens.csv"))
97
+ p.add_argument("--gene2vec_data_path", type=str, default=str(ASSET_ROOT / "selected_gene2vec_27k.npy"))
98
+ p.add_argument("--gene2ens_file", type=str, default=str(ASSET_ROOT / "gene2ens.csv"))
99
+ p.add_argument("--gene2ens_column", type=str, default="Ensembl gene ID")
100
+ p.add_argument("--shuffle_module_genes", action="store_true")
101
+ p.add_argument("--shuffle_seed", type=int, default=2021)
102
+
103
+ p.add_argument("--data_path", type=str, required=True)
104
+ p.add_argument("--celltype_column", type=str, default="celltype")
105
+ p.add_argument("--gene_name_column", type=str, default="auto")
106
+ p.add_argument("--dataset_name", type=str, default="gene_attention")
107
+ p.add_argument("--max_cells", type=int, default=None)
108
+
109
+ p.add_argument("--batch_size", type=int, default=64)
110
+ p.add_argument("--num_workers", type=int, default=0)
111
+ p.add_argument("--device", type=str, default="cuda")
112
+ p.add_argument("--seed", type=int, default=2021)
113
+
114
+ p.add_argument("--top_k_genes", type=int, default=20)
115
+ p.add_argument("--marker_db", type=str, default=None)
116
+ p.add_argument("--deg_csv", type=str, default=None)
117
+ p.add_argument("--min_cells", type=int, default=30)
118
+
119
+ p.add_argument("--output_dir", type=str, required=True)
120
+ return p.parse_args()
121
+
122
+
123
+ # ---------------------------------------------------------------------------
124
+ # Built-in pancreas marker genes (from PanglaoDB + CellMarker)
125
+ # ---------------------------------------------------------------------------
126
+
127
+ PANCREAS_MARKERS: Dict[str, List[str]] = {
128
+ "alpha": ["GCG", "ARX", "IRX2", "TTR", "LOXL4", "GC", "CRYBA2", "DPP4", "IRX1", "PCSK2"],
129
+ "beta": ["INS", "IAPP", "MAFA", "NKX6-1", "PDX1", "HADH", "SLC30A8", "NPTX2", "DLK1", "PCSK1"],
130
+ "delta": ["SST", "HHEX", "RBP4", "LEPR", "PCSK1", "SEC11C"],
131
+ "gamma": ["PPY", "SERTM1", "CARTPT", "MEIS2", "ETV1", "ABCC9"],
132
+ "ductal": ["KRT19", "KRT7", "SPP1", "CFTR", "MUC1", "AMBP", "ANXA4", "BICC1", "SOX9"],
133
+ "acinar": ["PRSS1", "CELA3A", "CELA2A", "CPA1", "CPA2", "PNLIP", "CEL", "CLPS", "REG1A", "CTRB1"],
134
+ "activated_stellate": ["COL1A1", "COL1A2", "COL3A1", "ACTA2", "TAGLN", "RGS5", "PDGFRB"],
135
+ "endothelial": ["PECAM1", "VWF", "CDH5", "KDR", "ENG", "PLVAP", "ESAM"],
136
+ "macrophage": ["CD68", "CD163", "CSF1R", "MARCO", "MSR1", "C1QA", "C1QB"],
137
+ "epsilon": ["GHRL", "PCSK1", "ARX"],
138
+ }
139
+
140
+ PANCREAS_COLORS: Dict[str, str] = {
141
+ "alpha": "#E41A1C", "beta": "#377EB8", "delta": "#4DAF4A",
142
+ "gamma": "#984EA3", "ductal": "#FF7F00", "acinar": "#A65628",
143
+ "activated_stellate": "#F781BF", "endothelial": "#66C2A5",
144
+ "macrophage": "#FC8D62", "epsilon": "#8DA0CB", "t_cell": "#E78AC3",
145
+ "schwann": "#B3B3B3", "mast": "#FDBF6F", "quiescent_stellate": "#CAB2D6",
146
+ }
147
+
148
+
149
+ # ---------------------------------------------------------------------------
150
+ # Streaming attention extraction + statistics accumulation
151
+ # ---------------------------------------------------------------------------
152
+
153
+ @torch.no_grad()
154
+ def extract_attention_streaming(
155
+ backbone, loader, preprocessor, reindex_state, device,
156
+ cell_labels: np.ndarray, num_modules: int, module_sizes: List[int],
157
+ ) -> Tuple:
158
+ """Extract attention and accumulate statistics in streaming fashion.
159
+
160
+ Returns:
161
+ global_mean, global_std, ct_mean, ct_std, ct_count, n_cells,
162
+ expr_mean, attn_expr_prod_sum
163
+ """
164
+ backbone.eval()
165
+ module_embed = backbone.module_embed
166
+
167
+ global_sum = {i: np.zeros(module_sizes[i], dtype=np.float64) for i in range(num_modules)}
168
+ global_sum_sq = {i: np.zeros(module_sizes[i], dtype=np.float64) for i in range(num_modules)}
169
+ expr_sum = {i: np.zeros(module_sizes[i], dtype=np.float64) for i in range(num_modules)}
170
+ attn_expr_prod_sum = {i: np.zeros(module_sizes[i], dtype=np.float64) for i in range(num_modules)}
171
+
172
+ celltypes = np.unique(cell_labels)
173
+ ct_sum = {ct: {i: np.zeros(module_sizes[i], dtype=np.float64) for i in range(num_modules)} for ct in celltypes}
174
+ ct_sum_sq = {ct: {i: np.zeros(module_sizes[i], dtype=np.float64) for i in range(num_modules)} for ct in celltypes}
175
+ ct_count = {ct: 0 for ct in celltypes}
176
+
177
+ n_cells = 0
178
+ cell_idx = 0
179
+
180
+ for batch_idx, x in enumerate(loader):
181
+ x = x.to(device)
182
+ if reindex_state is not None:
183
+ x = preprocessor.reindex_tensor_to_model_space(x, reindex_state=reindex_state)
184
+
185
+ batch_size = x.shape[0]
186
+ batch_labels = cell_labels[cell_idx:cell_idx + batch_size]
187
+ cell_idx += batch_size
188
+
189
+ for module_i, gene_idx in enumerate(module_embed.module_sets):
190
+ gene_idx_dev = gene_idx.to(device)
191
+ expr_slice = torch.index_select(x, dim=1, index=gene_idx_dev)
192
+
193
+ gene_emb = module_embed.gene_embed(gene_idx_dev)
194
+ gene_emb = gene_emb.unsqueeze(0).expand(batch_size, -1, -1)
195
+ gene_tok = module_embed._fuse_expr_and_gene_emb(expr_slice, gene_emb)
196
+ gene_tok = module_embed.proj(gene_tok)
197
+
198
+ attn = module_embed.attn_score(gene_tok).squeeze(-1)
199
+ attn = torch.softmax(attn, dim=1)
200
+ attn_np = attn.cpu().numpy().astype(np.float64)
201
+
202
+ # Expression for correlation
203
+ expr_np = expr_slice.cpu().numpy().astype(np.float64)
204
+ expr_sum[module_i] += expr_np.sum(axis=0)
205
+ attn_expr_prod_sum[module_i] += (attn_np * expr_np).sum(axis=0)
206
+
207
+ # Global accumulation
208
+ global_sum[module_i] += attn_np.sum(axis=0)
209
+ global_sum_sq[module_i] += (attn_np ** 2).sum(axis=0)
210
+
211
+ # Per-celltype accumulation
212
+ for ct in celltypes:
213
+ mask = batch_labels == ct
214
+ if mask.any():
215
+ ct_attn = attn_np[mask]
216
+ ct_sum[ct][module_i] += ct_attn.sum(axis=0)
217
+ ct_sum_sq[ct][module_i] += (ct_attn ** 2).sum(axis=0)
218
+
219
+ for ct in celltypes:
220
+ ct_count[ct] += int((batch_labels == ct).sum())
221
+
222
+ n_cells += batch_size
223
+ if (batch_idx + 1) % 50 == 0:
224
+ logger.info(" [attention] batch %d, %d cells processed", batch_idx + 1, n_cells)
225
+
226
+ # Compute means and stds
227
+ global_mean = {}
228
+ global_std = {}
229
+ for i in range(num_modules):
230
+ global_mean[i] = (global_sum[i] / n_cells).astype(np.float32)
231
+ variance = global_sum_sq[i] / n_cells - global_mean[i].astype(np.float64) ** 2
232
+ global_std[i] = np.sqrt(np.maximum(variance, 0)).astype(np.float32)
233
+
234
+ ct_mean = {}
235
+ ct_std = {}
236
+ for ct in celltypes:
237
+ ct_mean[ct] = {}
238
+ ct_std[ct] = {}
239
+ for i in range(num_modules):
240
+ if ct_count[ct] > 0:
241
+ ct_mean[ct][i] = (ct_sum[ct][i] / ct_count[ct]).astype(np.float32)
242
+ else:
243
+ ct_mean[ct][i] = np.zeros(module_sizes[i], dtype=np.float32)
244
+ if ct_count[ct] > 1:
245
+ var = ct_sum_sq[ct][i] / ct_count[ct] - ct_mean[ct][i].astype(np.float64) ** 2
246
+ ct_std[ct][i] = np.sqrt(np.maximum(var, 0)).astype(np.float32)
247
+ else:
248
+ ct_std[ct][i] = np.zeros(module_sizes[i], dtype=np.float32)
249
+
250
+ expr_mean = {i: (expr_sum[i] / n_cells).astype(np.float32) for i in range(num_modules)}
251
+
252
+ logger.info("Extracted attention for %d cells across %d modules (streaming)", n_cells, num_modules)
253
+ return global_mean, global_std, ct_mean, ct_std, ct_count, n_cells, expr_mean, attn_expr_prod_sum
254
+
255
+
256
+ # ---------------------------------------------------------------------------
257
+ # Analysis functions
258
+ # ---------------------------------------------------------------------------
259
+
260
+ def build_global_importance_df(
261
+ global_mean: Dict[int, np.ndarray],
262
+ global_std: Dict[int, np.ndarray],
263
+ module_gene_names: Dict[int, List[str]],
264
+ num_modules: int,
265
+ ) -> pd.DataFrame:
266
+ """Build global gene importance DataFrame from streaming means."""
267
+ rows = []
268
+ for module_i in range(num_modules):
269
+ mean_attn = global_mean[module_i]
270
+ std_attn = global_std[module_i]
271
+ genes = module_gene_names[module_i]
272
+ rank_order = np.argsort(-mean_attn)
273
+ for g_idx, gene in enumerate(genes):
274
+ rows.append({
275
+ "module_idx": module_i,
276
+ "gene": gene,
277
+ "mean_attention": float(mean_attn[g_idx]),
278
+ "std_attention": float(std_attn[g_idx]),
279
+ "rank_in_module": int(np.where(rank_order == g_idx)[0][0]) + 1,
280
+ "module_size": len(genes),
281
+ })
282
+ df = pd.DataFrame(rows)
283
+ df["normalized_rank"] = df["rank_in_module"] / df["module_size"]
284
+ return df
285
+
286
+
287
+ def build_celltype_dfs(
288
+ ct_mean: Dict[str, Dict[int, np.ndarray]],
289
+ ct_count: Dict[str, int],
290
+ module_gene_names: Dict[int, List[str]],
291
+ num_modules: int,
292
+ ) -> Dict[str, pd.DataFrame]:
293
+ """Build per-celltype DataFrames from streaming means."""
294
+ ct_dfs = {}
295
+ for ct, module_means in ct_mean.items():
296
+ rows = []
297
+ for module_i in range(num_modules):
298
+ mean_attn = module_means[module_i]
299
+ genes = module_gene_names[module_i]
300
+ for g_idx, gene in enumerate(genes):
301
+ rows.append({
302
+ "module_idx": module_i,
303
+ "gene": gene,
304
+ "mean_attention": float(mean_attn[g_idx]),
305
+ "n_cells": ct_count[ct],
306
+ })
307
+ ct_dfs[ct] = pd.DataFrame(rows)
308
+ return ct_dfs
309
+
310
+
311
+ def compute_specificity_from_means(
312
+ global_mean: Dict[int, np.ndarray],
313
+ ct_mean: Dict[str, Dict[int, np.ndarray]],
314
+ module_gene_names: Dict[int, List[str]],
315
+ num_modules: int,
316
+ ) -> pd.DataFrame:
317
+ """Fold-change specificity (legacy): max_ct(mean_attn_ct) / global_mean_attn."""
318
+ celltypes = list(ct_mean.keys())
319
+ rows = []
320
+ for module_i in range(num_modules):
321
+ genes = module_gene_names[module_i]
322
+ g_mean = global_mean[module_i]
323
+ for g_idx, gene in enumerate(genes):
324
+ ct_scores = {ct: float(ct_mean[ct][module_i][g_idx]) for ct in celltypes}
325
+ top_ct = max(ct_scores, key=ct_scores.get)
326
+ top_score = ct_scores[top_ct]
327
+ fold_change = top_score / max(float(g_mean[g_idx]), 1e-10)
328
+ rows.append({
329
+ "module_idx": module_i, "gene": gene, "top_celltype": top_ct,
330
+ "top_ct_mean_attn": top_score, "global_mean_attn": float(g_mean[g_idx]),
331
+ "fold_change": fold_change,
332
+ })
333
+ return pd.DataFrame(rows)
334
+
335
+
336
+ def compute_specificity_zscore(
337
+ global_mean: Dict[int, np.ndarray],
338
+ global_std: Dict[int, np.ndarray],
339
+ ct_mean: Dict[str, Dict[int, np.ndarray]],
340
+ ct_count: Dict[str, int],
341
+ module_gene_names: Dict[int, List[str]],
342
+ num_modules: int,
343
+ min_cells: int = 30,
344
+ ) -> pd.DataFrame:
345
+ """Z-score specificity: (ct_mean - global_mean) / global_std."""
346
+ valid_cts = [ct for ct, n in ct_count.items() if n >= min_cells]
347
+ rows = []
348
+ for module_i in range(num_modules):
349
+ genes = module_gene_names[module_i]
350
+ g_mean = global_mean[module_i]
351
+ g_std = global_std[module_i]
352
+ for g_idx, gene in enumerate(genes):
353
+ std_val = max(float(g_std[g_idx]), 1e-8)
354
+ ct_zscores = {}
355
+ for ct in valid_cts:
356
+ z = (float(ct_mean[ct][module_i][g_idx]) - float(g_mean[g_idx])) / std_val
357
+ ct_zscores[ct] = z
358
+ top_ct = max(ct_zscores, key=ct_zscores.get)
359
+ rows.append({
360
+ "module_idx": module_i, "gene": gene, "top_celltype": top_ct,
361
+ "z_score": ct_zscores[top_ct],
362
+ "global_mean_attn": float(g_mean[g_idx]),
363
+ "ct_mean_attn": float(ct_mean[top_ct][module_i][g_idx]),
364
+ })
365
+ return pd.DataFrame(rows)
366
+
367
+
368
+ def compute_cohens_d(
369
+ global_mean: Dict[int, np.ndarray],
370
+ ct_mean: Dict[str, Dict[int, np.ndarray]],
371
+ ct_std: Dict[str, Dict[int, np.ndarray]],
372
+ ct_count: Dict[str, int],
373
+ module_gene_names: Dict[int, List[str]],
374
+ num_modules: int,
375
+ n_cells: int,
376
+ min_cells: int = 30,
377
+ ) -> pd.DataFrame:
378
+ """Cohen's d effect size for cell-type specificity."""
379
+ valid_cts = [ct for ct, n in ct_count.items() if n >= min_cells]
380
+ rows = []
381
+ for module_i in range(num_modules):
382
+ genes = module_gene_names[module_i]
383
+ for g_idx, gene in enumerate(genes):
384
+ best_d = -np.inf
385
+ best_ct = valid_cts[0]
386
+ for ct in valid_cts:
387
+ n_ct = ct_count[ct]
388
+ n_other = n_cells - n_ct
389
+ ct_m = float(ct_mean[ct][module_i][g_idx])
390
+ ct_s = float(ct_std[ct][module_i][g_idx])
391
+ other_m = (float(global_mean[module_i][g_idx]) * n_cells - ct_m * n_ct) / max(n_other, 1)
392
+ pooled = max(ct_s, 1e-10)
393
+ d = (ct_m - other_m) / pooled
394
+ if d > best_d:
395
+ best_d = d
396
+ best_ct = ct
397
+ rows.append({"module_idx": module_i, "gene": gene, "top_celltype": best_ct, "cohens_d": best_d})
398
+ return pd.DataFrame(rows)
399
+
400
+
401
+ def compute_welch_ttest(
402
+ global_mean: Dict[int, np.ndarray],
403
+ global_std: Dict[int, np.ndarray],
404
+ ct_mean: Dict[str, Dict[int, np.ndarray]],
405
+ ct_std: Dict[str, Dict[int, np.ndarray]],
406
+ ct_count: Dict[str, int],
407
+ module_gene_names: Dict[int, List[str]],
408
+ num_modules: int,
409
+ n_cells: int,
410
+ min_cells: int = 30,
411
+ ) -> pd.DataFrame:
412
+ """Welch's t-test from streaming statistics with BH FDR correction."""
413
+ valid_cts = [ct for ct, n in ct_count.items() if n >= min_cells]
414
+ rows = []
415
+ for module_i in range(num_modules):
416
+ genes = module_gene_names[module_i]
417
+ for g_idx, gene in enumerate(genes):
418
+ best_t = -np.inf
419
+ best_ct = valid_cts[0]
420
+ best_pval = 1.0
421
+ for ct in valid_cts:
422
+ n_ct = ct_count[ct]
423
+ ct_m = float(ct_mean[ct][module_i][g_idx])
424
+ ct_v = float(ct_std[ct][module_i][g_idx]) ** 2
425
+ g_m = float(global_mean[module_i][g_idx])
426
+ g_v = float(global_std[module_i][g_idx]) ** 2
427
+ se = np.sqrt(ct_v / max(n_ct, 1) + g_v / max(n_cells, 1))
428
+ if se < 1e-12:
429
+ continue
430
+ t_stat = (ct_m - g_m) / se
431
+ num = (ct_v / n_ct + g_v / n_cells) ** 2
432
+ denom = (ct_v / n_ct) ** 2 / max(n_ct - 1, 1) + (g_v / n_cells) ** 2 / max(n_cells - 1, 1)
433
+ df = num / max(denom, 1e-12)
434
+ pval = float(t_dist.sf(abs(t_stat), df) * 2)
435
+ if t_stat > best_t:
436
+ best_t = t_stat
437
+ best_ct = ct
438
+ best_pval = pval
439
+ rows.append({"module_idx": module_i, "gene": gene, "top_celltype": best_ct,
440
+ "t_stat": best_t, "pval": best_pval})
441
+
442
+ df = pd.DataFrame(rows)
443
+ # BH FDR correction
444
+ pvals = df["pval"].values
445
+ n = len(pvals)
446
+ sorted_idx = np.argsort(pvals)
447
+ sorted_pvals = pvals[sorted_idx]
448
+ fdr = np.minimum(1.0, sorted_pvals * n / (np.arange(n) + 1))
449
+ for i in range(n - 2, -1, -1):
450
+ fdr[i] = min(fdr[i], fdr[i + 1])
451
+ fdr_all = np.empty(n)
452
+ fdr_all[sorted_idx] = fdr
453
+ df["fdr"] = fdr_all
454
+ return df
455
+
456
+
457
+ def compute_attention_expression_corr(
458
+ global_mean: Dict[int, np.ndarray],
459
+ expr_mean: Dict[int, np.ndarray],
460
+ module_gene_names: Dict[int, List[str]],
461
+ num_modules: int,
462
+ ) -> Dict[str, float]:
463
+ """Correlation between mean attention and mean expression across genes."""
464
+ all_attn = []
465
+ all_expr = []
466
+ for module_i in range(num_modules):
467
+ all_attn.extend(global_mean[module_i].tolist())
468
+ all_expr.extend(expr_mean[module_i].tolist())
469
+ all_attn = np.array(all_attn)
470
+ all_expr = np.array(all_expr)
471
+ mask = (all_attn > 0) & (all_expr > 0)
472
+ if mask.sum() > 10:
473
+ pearson_r = float(np.corrcoef(all_attn[mask], all_expr[mask])[0, 1])
474
+ sp_r, sp_p = spearmanr(all_attn[mask], all_expr[mask])
475
+ else:
476
+ pearson_r, sp_r, sp_p = 0.0, 0.0, 1.0
477
+ return {"pearson_r": pearson_r, "spearman_r": float(sp_r),
478
+ "spearman_pval": float(sp_p), "n_genes": int(mask.sum())}
479
+
480
+
481
+ def compute_random_baseline(
482
+ global_mean: Dict[int, np.ndarray],
483
+ num_modules: int,
484
+ ) -> Dict[str, float]:
485
+ """Compare learned attention to uniform baseline."""
486
+ kl_divs = []
487
+ learned_entropies = []
488
+ for module_i in range(num_modules):
489
+ attn = global_mean[module_i].astype(np.float64)
490
+ attn = attn / max(attn.sum(), 1e-10)
491
+ g_size = len(attn)
492
+ uniform = np.ones(g_size) / g_size
493
+ mask = attn > 1e-12
494
+ kl = float(np.sum(attn[mask] * np.log(attn[mask] / uniform[mask])))
495
+ kl_divs.append(kl)
496
+ h_learned = -float(np.sum(attn[mask] * np.log(attn[mask])))
497
+ h_max = np.log(g_size)
498
+ learned_entropies.append(h_learned / max(h_max, 1e-10))
499
+ return {
500
+ "mean_kl_divergence": float(np.mean(kl_divs)),
501
+ "std_kl_divergence": float(np.std(kl_divs)),
502
+ "mean_normalized_entropy": float(np.mean(learned_entropies)),
503
+ "entropy_reduction": float(1.0 - np.mean(learned_entropies)),
504
+ "n_modules": num_modules,
505
+ }
506
+
507
+
508
+ def compute_attention_concentration(
509
+ global_mean: Dict[int, np.ndarray],
510
+ ct_mean: Dict[str, Dict[int, np.ndarray]],
511
+ ct_count: Dict[str, int],
512
+ num_modules: int,
513
+ min_cells: int = 30,
514
+ ) -> pd.DataFrame:
515
+ """Gini coefficient per module measuring attention selectivity."""
516
+ def gini(arr):
517
+ arr = np.sort(arr.flatten())
518
+ n = len(arr)
519
+ if n == 0 or arr.sum() == 0:
520
+ return 0.0
521
+ index = np.arange(1, n + 1)
522
+ return float((2 * np.sum(index * arr) - (n + 1) * np.sum(arr)) / (n * np.sum(arr)))
523
+
524
+ valid_cts = [ct for ct, n in ct_count.items() if n >= min_cells]
525
+ rows = []
526
+ for module_i in range(num_modules):
527
+ g_attn = global_mean[module_i]
528
+ g_gini = gini(g_attn)
529
+ ct_ginis = [gini(ct_mean[ct][module_i]) for ct in valid_cts]
530
+ rows.append({
531
+ "module_idx": module_i, "global_gini": g_gini,
532
+ "mean_ct_gini": float(np.mean(ct_ginis)) if ct_ginis else 0.0,
533
+ "module_size": len(g_attn),
534
+ })
535
+ return pd.DataFrame(rows)
536
+
537
+
538
+ def compute_marker_rank_within_celltype(
539
+ ct_mean: Dict[str, Dict[int, np.ndarray]],
540
+ ct_count: Dict[str, int],
541
+ module_gene_names: Dict[int, List[str]],
542
+ num_modules: int,
543
+ marker_dict: Dict[str, List[str]],
544
+ min_cells: int = 30,
545
+ ) -> pd.DataFrame:
546
+ """Rank marker genes within their expected cell type by attention."""
547
+ rows = []
548
+ for ct, markers in marker_dict.items():
549
+ if ct not in ct_count or ct_count[ct] < min_cells:
550
+ continue
551
+ gene_attn = {}
552
+ for module_i in range(num_modules):
553
+ genes = module_gene_names[module_i]
554
+ attn = ct_mean[ct][module_i]
555
+ for g_idx, gene in enumerate(genes):
556
+ if gene not in gene_attn or float(attn[g_idx]) > gene_attn[gene]:
557
+ gene_attn[gene] = float(attn[g_idx])
558
+ sorted_genes = sorted(gene_attn.items(), key=lambda x: -x[1])
559
+ gene_rank = {g: rank + 1 for rank, (g, _) in enumerate(sorted_genes)}
560
+ n_total = len(gene_rank)
561
+ for marker in markers:
562
+ if marker in gene_rank:
563
+ rank = gene_rank[marker]
564
+ rows.append({
565
+ "celltype": ct, "marker_gene": marker, "rank": rank,
566
+ "percentile": rank / n_total * 100, "n_total_genes": n_total,
567
+ "attention": gene_attn[marker],
568
+ })
569
+ return pd.DataFrame(rows)
570
+
571
+
572
+ # ---------------------------------------------------------------------------
573
+ # Marker gene validation
574
+ # ---------------------------------------------------------------------------
575
+
576
+ def validate_markers(
577
+ specificity_df: pd.DataFrame,
578
+ marker_dict: Dict[str, List[str]],
579
+ sort_col: str = "fold_change",
580
+ ) -> Dict[str, object]:
581
+ """Validate whether high-attention genes match known markers."""
582
+ results = {}
583
+ all_genes_in_model = set(specificity_df["gene"].unique())
584
+ for ct, markers in marker_dict.items():
585
+ markers_in_model = [g for g in markers if g in all_genes_in_model]
586
+ if not markers_in_model:
587
+ continue
588
+ ct_specific = specificity_df[specificity_df["top_celltype"] == ct].copy()
589
+ ct_specific = ct_specific.sort_values(sort_col, ascending=False).reset_index(drop=True)
590
+ marker_set = set(markers_in_model)
591
+ n_total = len(ct_specific)
592
+ if n_total == 0:
593
+ continue
594
+ top_50_genes = set(ct_specific.head(50)["gene"])
595
+ top_100_genes = set(ct_specific.head(100)["gene"])
596
+ hits_50 = len(marker_set & top_50_genes)
597
+ hits_100 = len(marker_set & top_100_genes)
598
+ marker_rows = ct_specific[ct_specific["gene"].isin(marker_set)]
599
+ if len(marker_rows) > 0:
600
+ marker_ranks = (marker_rows.index + 1).tolist()
601
+ mean_rank = np.mean(marker_ranks)
602
+ median_rank = np.median(marker_ranks)
603
+ else:
604
+ mean_rank = median_rank = n_total
605
+ results[ct] = {
606
+ "n_markers_in_model": len(markers_in_model),
607
+ "markers_in_model": markers_in_model,
608
+ "n_genes_assigned_to_ct": n_total,
609
+ "hits_in_top50": hits_50, "hits_in_top100": hits_100,
610
+ "precision_at_50": hits_50 / min(50, n_total),
611
+ "precision_at_100": hits_100 / min(100, n_total),
612
+ "marker_mean_rank": float(mean_rank),
613
+ "marker_median_rank": float(median_rank),
614
+ "recall_at_50": hits_50 / len(markers_in_model) if markers_in_model else 0,
615
+ "recall_at_100": hits_100 / len(markers_in_model) if markers_in_model else 0,
616
+ }
617
+ return results
618
+
619
+
620
+ def validate_deg_correlation(
621
+ ct_dfs: Dict[str, pd.DataFrame],
622
+ deg_df: pd.DataFrame,
623
+ ) -> Dict[str, Dict[str, float]]:
624
+ """Compare attention ranking with DEG ranking per cell type."""
625
+ results = {}
626
+ for ct, attn_df in ct_dfs.items():
627
+ ct_deg = deg_df[deg_df["cell_type"] == ct].copy()
628
+ if len(ct_deg) == 0:
629
+ continue
630
+ merged = attn_df.merge(ct_deg[["gene", "log2fc"]], on="gene", how="inner")
631
+ if len(merged) < 10:
632
+ continue
633
+ rho, pval = spearmanr(merged["mean_attention"], merged["log2fc"].abs())
634
+ results[ct] = {"spearman_r": float(rho), "pval": float(pval), "n_genes": len(merged)}
635
+ return results
636
+
637
+
638
+ # ---------------------------------------------------------------------------
639
+ # Publication-quality visualization
640
+ # ---------------------------------------------------------------------------
641
+
642
+ def plot_global_importance_pub(global_df, specificity_zscore_df, top_k, output_dir):
643
+ """Fig1: Top-K genes with std error bars, colored by top cell type."""
644
+ setup_publication_style()
645
+ gene_stats = global_df.groupby("gene").agg(
646
+ mean_attn=("mean_attention", "mean"), std_attn=("std_attention", "mean"),
647
+ ).nlargest(top_k, "mean_attn")
648
+ gene_ct = specificity_zscore_df.drop_duplicates("gene").set_index("gene")["top_celltype"]
649
+ fig, ax = plt.subplots(figsize=(4.5, 5))
650
+ genes = gene_stats.index.tolist()
651
+ y_pos = np.arange(len(genes))
652
+ colors = [PANCREAS_COLORS.get(gene_ct.get(g, ""), "#999999") for g in genes]
653
+ ax.barh(y_pos, gene_stats["mean_attn"], xerr=gene_stats["std_attn"],
654
+ color=colors, edgecolor="none", height=0.7, capsize=2, error_kw={"linewidth": 0.8})
655
+ ax.set_yticks(y_pos)
656
+ ax.set_yticklabels(genes)
657
+ ax.set_xlabel("Mean attention weight")
658
+ ax.set_title("Top genes by global attention")
659
+ ax.invert_yaxis()
660
+ pub_savefig(fig, output_dir / "fig1_global_importance")
661
+ logger.info("Saved fig1_global_importance (PDF+PNG)")
662
+
663
+
664
+ def plot_celltype_heatmap_pub(ct_dfs, ct_count, top_k, output_dir, min_cells=30):
665
+ """Fig2: Clustered heatmap of gene attention (z-score normalized)."""
666
+ setup_publication_style()
667
+ valid_cts = {ct for ct, n in ct_count.items() if n >= min_cells}
668
+ all_genes_attn = {}
669
+ for ct, df in ct_dfs.items():
670
+ if ct not in valid_cts:
671
+ continue
672
+ all_genes_attn[ct] = df.groupby("gene")["mean_attention"].mean()
673
+ combined = pd.DataFrame(all_genes_attn)
674
+ gene_var = combined.var(axis=1).nlargest(top_k).index
675
+ heatmap_data = combined.loc[gene_var]
676
+ row_mean = heatmap_data.mean(axis=1)
677
+ row_std = heatmap_data.std(axis=1).replace(0, 1)
678
+ heatmap_z = heatmap_data.sub(row_mean, axis=0).div(row_std, axis=0)
679
+ g = sns.clustermap(heatmap_z, cmap="RdBu_r", center=0, figsize=(6, 7),
680
+ xticklabels=True, yticklabels=True,
681
+ dendrogram_ratio=(0.1, 0.15), cbar_kws={"label": "Z-score"},
682
+ linewidths=0.3)
683
+ g.ax_heatmap.set_xlabel("")
684
+ g.ax_heatmap.set_ylabel("")
685
+ g.fig.suptitle(f"Top {top_k} variable genes (z-scored attention)", y=1.02, fontsize=11)
686
+ pub_savefig(g.fig, output_dir / "fig2_celltype_heatmap")
687
+ logger.info("Saved fig2_celltype_heatmap (PDF+PNG)")
688
+
689
+
690
+ def plot_specificity_volcano(welch_df, zscore_df, top_k, output_dir):
691
+ """Fig3: Volcano plot (z-score vs -log10 FDR)."""
692
+ setup_publication_style()
693
+ merged = zscore_df[["gene", "top_celltype", "z_score"]].drop_duplicates("gene")
694
+ welch_best = welch_df.loc[welch_df.groupby("gene")["t_stat"].idxmax()][["gene", "fdr"]]
695
+ merged = merged.merge(welch_best, on="gene", how="left")
696
+ merged["neg_log10_fdr"] = -np.log10(merged["fdr"].clip(lower=1e-300))
697
+ fig, ax = plt.subplots(figsize=(5.5, 5))
698
+ for ct in merged["top_celltype"].unique():
699
+ subset = merged[merged["top_celltype"] == ct]
700
+ color = PANCREAS_COLORS.get(ct, "#999999")
701
+ ax.scatter(subset["z_score"], subset["neg_log10_fdr"],
702
+ c=color, s=8, alpha=0.5, label=ct, rasterized=True)
703
+ top_genes = merged.nlargest(top_k, "z_score")
704
+ for _, row in top_genes.iterrows():
705
+ ax.annotate(row["gene"], (row["z_score"], row["neg_log10_fdr"]),
706
+ fontsize=6, ha="left", va="bottom")
707
+ ax.axhline(-np.log10(0.05), color="gray", linestyle="--", linewidth=0.6)
708
+ ax.axvline(2, color="gray", linestyle="--", linewidth=0.6)
709
+ ax.set_xlabel("Z-score (cell-type specificity)")
710
+ ax.set_ylabel(r"$-\log_{10}$(FDR)")
711
+ ax.set_title("Gene specificity volcano")
712
+ ax.legend(bbox_to_anchor=(1.02, 1), loc="upper left", fontsize=7, frameon=False, markerscale=2)
713
+ pub_savefig(fig, output_dir / "fig3_specificity_volcano")
714
+ logger.info("Saved fig3_specificity_volcano (PDF+PNG)")
715
+
716
+
717
+ def plot_marker_dotplot(marker_rank_df, output_dir):
718
+ """Fig4: Dot plot of marker gene rank percentile per cell type."""
719
+ setup_publication_style()
720
+ if marker_rank_df.empty:
721
+ return
722
+ fig, ax = plt.subplots(figsize=(6, 4.5))
723
+ celltypes = marker_rank_df["celltype"].unique()
724
+ ct_positions = {ct: i for i, ct in enumerate(celltypes)}
725
+ for _, row in marker_rank_df.iterrows():
726
+ y = ct_positions[row["celltype"]]
727
+ size = max(10, 200 * (1 - row["percentile"] / 100))
728
+ color = PANCREAS_COLORS.get(row["celltype"], "#999999")
729
+ ax.scatter(row["percentile"], y, s=size, c=color, alpha=0.7,
730
+ edgecolors="white", linewidth=0.3)
731
+ if row["percentile"] < 5:
732
+ ax.annotate(row["marker_gene"], (row["percentile"], y),
733
+ fontsize=6, ha="left", va="center", xytext=(3, 0),
734
+ textcoords="offset points")
735
+ ax.set_yticks(range(len(celltypes)))
736
+ ax.set_yticklabels(celltypes)
737
+ ax.set_xlabel("Rank percentile (lower = higher attention)")
738
+ ax.set_title("Marker gene rank within expected cell type")
739
+ ax.axvline(5, color="gray", linestyle="--", linewidth=0.6, alpha=0.5, label="Top 5%")
740
+ ax.legend(fontsize=7, frameon=False)
741
+ ax.set_xlim(-2, 105)
742
+ pub_savefig(fig, output_dir / "fig4_marker_dotplot")
743
+ logger.info("Saved fig4_marker_dotplot (PDF+PNG)")
744
+
745
+
746
+ def plot_marker_recall_pub(marker_results, ct_count, output_dir, min_cells=30):
747
+ """Fig5: Marker recall bar chart with random expectation line."""
748
+ setup_publication_style()
749
+ if not marker_results:
750
+ return
751
+ cts = [ct for ct in marker_results.keys() if ct_count.get(ct, 0) >= min_cells]
752
+ if not cts:
753
+ return
754
+ recall_50 = [marker_results[ct]["recall_at_50"] for ct in cts]
755
+ recall_100 = [marker_results[ct]["recall_at_100"] for ct in cts]
756
+ x = np.arange(len(cts))
757
+ width = 0.35
758
+ fig, ax = plt.subplots(figsize=(max(5, len(cts) * 0.7), 4))
759
+ colors_50 = [PANCREAS_COLORS.get(ct, "#999999") for ct in cts]
760
+ ax.bar(x - width / 2, recall_50, width, color=colors_50, alpha=0.9, label="Recall@50")
761
+ ax.bar(x + width / 2, recall_100, width, color=colors_50, alpha=0.5, label="Recall@100")
762
+ # Random expectation
763
+ for i, ct in enumerate(cts):
764
+ n_genes = marker_results[ct]["n_genes_assigned_to_ct"]
765
+ if n_genes > 0:
766
+ random_50 = min(50, n_genes) / n_genes
767
+ ax.hlines(random_50, i - 0.4, i + 0.4, colors="gray", linestyles=":", linewidth=0.8)
768
+ ax.set_xticks(x)
769
+ ax.set_xticklabels(cts, rotation=45, ha="right")
770
+ ax.set_ylabel("Recall")
771
+ ax.set_title("Marker gene recall by cell type")
772
+ ax.legend(fontsize=7, frameon=False)
773
+ pub_savefig(fig, output_dir / "fig5_marker_recall")
774
+ logger.info("Saved fig5_marker_recall (PDF+PNG)")
775
+
776
+
777
+ def plot_attention_vs_expression(global_mean, expr_mean, module_gene_names, num_modules, output_dir):
778
+ """Fig6: Scatter of mean attention vs mean expression."""
779
+ setup_publication_style()
780
+ all_attn, all_expr = [], []
781
+ for module_i in range(num_modules):
782
+ all_attn.extend(global_mean[module_i].tolist())
783
+ all_expr.extend(expr_mean[module_i].tolist())
784
+ all_attn = np.array(all_attn)
785
+ all_expr = np.array(all_expr)
786
+ mask = (all_attn > 0) & (all_expr > 0)
787
+ fig, ax = plt.subplots(figsize=(4.5, 4.5))
788
+ ax.scatter(all_expr[mask], all_attn[mask], s=3, alpha=0.3, c="#377EB8", rasterized=True)
789
+ r = np.corrcoef(all_expr[mask], all_attn[mask])[0, 1]
790
+ ax.set_xlabel("Mean expression")
791
+ ax.set_ylabel("Mean attention weight")
792
+ ax.set_title("Attention vs expression")
793
+ ax.text(0.05, 0.95, f"Pearson r = {r:.3f}", transform=ax.transAxes,
794
+ fontsize=9, va="top", ha="left")
795
+ pub_savefig(fig, output_dir / "fig6_attn_vs_expression")
796
+ logger.info("Saved fig6_attn_vs_expression (PDF+PNG)")
797
+
798
+
799
+ def plot_concentration_pub(concentration_df, baseline_stats, output_dir):
800
+ """Fig7: Gini coefficient distribution (learned vs uniform)."""
801
+ setup_publication_style()
802
+ fig, axes = plt.subplots(1, 2, figsize=(8, 3.5))
803
+ # Panel a: Gini histogram
804
+ ax = axes[0]
805
+ ax.hist(concentration_df["global_gini"], bins=30, alpha=0.7, color="#377EB8", label="Global")
806
+ ax.hist(concentration_df["mean_ct_gini"], bins=30, alpha=0.7, color="#E41A1C", label="Per cell-type")
807
+ ax.axvline(0, color="gray", linestyle="--", linewidth=0.6)
808
+ ax.set_xlabel("Gini coefficient")
809
+ ax.set_ylabel("Number of modules")
810
+ ax.set_title("Attention concentration")
811
+ ax.legend(fontsize=7, frameon=False)
812
+ # Panel b: Entropy comparison
813
+ ax = axes[1]
814
+ labels = ["Learned", "Uniform"]
815
+ values = [baseline_stats["mean_normalized_entropy"], 1.0]
816
+ bars = ax.bar(labels, values, color=["#377EB8", "#CCCCCC"], width=0.5)
817
+ ax.set_ylabel("Normalized entropy")
818
+ ax.set_title(f"Entropy reduction: {baseline_stats['entropy_reduction']:.1%}")
819
+ ax.set_ylim(0, 1.1)
820
+ plt.tight_layout()
821
+ pub_savefig(fig, output_dir / "fig7_concentration")
822
+ logger.info("Saved fig7_concentration (PDF+PNG)")
823
+
824
+
825
+ def plot_baseline_kl(baseline_stats, output_dir):
826
+ """Fig8: KL divergence summary."""
827
+ setup_publication_style()
828
+ fig, ax = plt.subplots(figsize=(3.5, 3.5))
829
+ ax.bar(["KL(learned||uniform)"], [baseline_stats["mean_kl_divergence"]],
830
+ yerr=[baseline_stats["std_kl_divergence"]], color="#377EB8",
831
+ capsize=5, width=0.4)
832
+ ax.set_ylabel("KL divergence (nats)")
833
+ ax.set_title("Deviation from uniform attention")
834
+ pub_savefig(fig, output_dir / "fig8_baseline_kl")
835
+ logger.info("Saved fig8_baseline_kl (PDF+PNG)")
836
+
837
+
838
+ def plot_deg_correlation_pub(deg_results, output_dir):
839
+ """Fig9: DEG correlation with significance."""
840
+ setup_publication_style()
841
+ if not deg_results:
842
+ return
843
+ cts = sorted(deg_results.keys(), key=lambda c: deg_results[c]["spearman_r"], reverse=True)
844
+ rhos = [deg_results[ct]["spearman_r"] for ct in cts]
845
+ pvals = [deg_results[ct]["pval"] for ct in cts]
846
+ fig, ax = plt.subplots(figsize=(max(5, len(cts) * 0.6), 4))
847
+ colors = ["#377EB8" if r > 0 else "#E41A1C" for r in rhos]
848
+ bars = ax.bar(range(len(cts)), rhos, color=colors, width=0.6)
849
+ for i, (bar, p) in enumerate(zip(bars, pvals)):
850
+ if p < 0.001:
851
+ ax.text(i, bar.get_height() + 0.02, "***", ha="center", fontsize=7)
852
+ elif p < 0.01:
853
+ ax.text(i, bar.get_height() + 0.02, "**", ha="center", fontsize=7)
854
+ elif p < 0.05:
855
+ ax.text(i, bar.get_height() + 0.02, "*", ha="center", fontsize=7)
856
+ ax.set_xticks(range(len(cts)))
857
+ ax.set_xticklabels(cts, rotation=45, ha="right")
858
+ ax.set_ylabel("Spearman r")
859
+ ax.set_title("Attention vs DEG correlation")
860
+ ax.axhline(0, color="gray", linewidth=0.5)
861
+ pub_savefig(fig, output_dir / "fig9_deg_correlation")
862
+ logger.info("Saved fig9_deg_correlation (PDF+PNG)")
863
+
864
+
865
+ # ---------------------------------------------------------------------------
866
+ # Enrichment analysis (optional)
867
+ # ---------------------------------------------------------------------------
868
+
869
+ def run_enrichment_analysis(
870
+ global_df: pd.DataFrame, top_k: int, output_dir: Path, organism: str = "human",
871
+ ) -> Optional[pd.DataFrame]:
872
+ """Run GO enrichment on top-K attention genes using gseapy."""
873
+ try:
874
+ import gseapy as gp
875
+ except ImportError:
876
+ logger.warning("gseapy not installed, skipping enrichment. Install: pip install gseapy")
877
+ return None
878
+ top_genes = global_df.groupby("gene")["mean_attention"].mean().nlargest(top_k * 5).index.tolist()
879
+ logger.info("Running GO enrichment on top %d genes...", len(top_genes))
880
+ try:
881
+ enr = gp.enrichr(
882
+ gene_list=top_genes,
883
+ gene_sets=["GO_Biological_Process_2021", "KEGG_2021_Human"],
884
+ organism=organism,
885
+ outdir=str(output_dir / "enrichment"),
886
+ no_plot=True,
887
+ )
888
+ results_df = enr.results
889
+ results_df.to_csv(output_dir / "enrichment_results.csv", index=False)
890
+ logger.info("Enrichment results saved (%d terms)", len(results_df))
891
+ return results_df
892
+ except Exception as e:
893
+ logger.warning("Enrichment analysis failed: %s", e)
894
+ return None
895
+
896
+
897
+ # ---------------------------------------------------------------------------
898
+ # Main
899
+ # ---------------------------------------------------------------------------
900
+
901
+ def main():
902
+ args = parse_args()
903
+ seed_all(args.seed)
904
+
905
+ _script_dir = Path(__file__).resolve().parent
906
+ for attr in ("performer_checkpoint", "config_pkl", "module_data_dir", "gene_list_path",
907
+ "gene_mapping_path", "gene2vec_data_path", "gene2ens_file", "data_path"):
908
+ val = getattr(args, attr, None)
909
+ if val is not None and not Path(val).is_absolute():
910
+ setattr(args, attr, str((_script_dir / val).resolve()))
911
+
912
+ output_dir = ensure_dir(Path(args.output_dir))
913
+ device = torch.device(args.device if torch.cuda.is_available() else "cpu")
914
+ logger.info("Device: %s", device)
915
+
916
+ # --- Load model ---
917
+ logger.info("Loading backbone from %s", args.performer_checkpoint)
918
+ backbone, module_data, model_config = load_backbone(args, device)
919
+ backbone.eval()
920
+
921
+ # --- Build module gene name mapping ---
922
+ model_genes_file = args.gene_list_path
923
+ with open(model_genes_file) as f:
924
+ model_genes = [line.strip() for line in f if line.strip()]
925
+
926
+ gene2ens_df = pd.read_csv(args.gene2ens_file)
927
+ ens_to_symbol = dict(zip(gene2ens_df["Ensembl gene ID"], gene2ens_df.iloc[:, 0]))
928
+
929
+ module_sets = backbone.module_embed.module_sets
930
+ num_modules = len(module_sets)
931
+ module_sizes = [len(ps) for ps in module_sets]
932
+
933
+ module_gene_names: Dict[int, List[str]] = {}
934
+ for module_i, gene_idx in enumerate(module_sets):
935
+ names = []
936
+ for idx in gene_idx.cpu().numpy():
937
+ ens_id = model_genes[idx]
938
+ symbol = ens_to_symbol.get(ens_id, ens_id)
939
+ names.append(symbol)
940
+ module_gene_names[module_i] = names
941
+
942
+ logger.info("Modules: %d, total genes: %d", num_modules, sum(module_sizes))
943
+
944
+ # --- Load and preprocess data ---
945
+ logger.info("Loading data from %s", args.data_path)
946
+ preprocessor = DataPreprocessor(
947
+ gene2ens_file=args.gene2ens_file,
948
+ gene2ens_column=args.gene2ens_column,
949
+ model_genes_file=model_genes_file,
950
+ )
951
+ preprocessor.load_gene_mapping()
952
+ preprocessor.load_model_genes()
953
+
954
+ gene_name_col = args.gene_name_column if args.gene_name_column != "auto" else None
955
+ adata, reindex_state = preprocessor.preprocess_adata(
956
+ adata_path=args.data_path,
957
+ max_cells=args.max_cells,
958
+ dataset_name=args.dataset_name,
959
+ gene_name_column=gene_name_col,
960
+ )
961
+ cell_labels = adata.obs[args.celltype_column].values.astype(str)
962
+ logger.info("Data: %d cells, %d cell types", len(adata), len(np.unique(cell_labels)))
963
+
964
+ # --- DataLoader ---
965
+ if hasattr(adata.X, "toarray"):
966
+ X = adata.X.toarray().astype(np.float32)
967
+ else:
968
+ X = np.array(adata.X, dtype=np.float32)
969
+
970
+ dataset = MatrixRowDataset(X)
971
+ loader = DataLoader(dataset, batch_size=args.batch_size, shuffle=False,
972
+ num_workers=args.num_workers, pin_memory=True)
973
+
974
+ # --- Extract attention (streaming) ---
975
+ logger.info("Extracting module attention (streaming mode)...")
976
+ (global_mean, global_std, ct_mean, ct_std, ct_count, n_cells,
977
+ expr_mean, attn_expr_prod_sum) = extract_attention_streaming(
978
+ backbone, loader, preprocessor, reindex_state, device,
979
+ cell_labels, num_modules, module_sizes,
980
+ )
981
+
982
+ # --- Build DataFrames ---
983
+ logger.info("Building analysis DataFrames...")
984
+ global_df = build_global_importance_df(global_mean, global_std, module_gene_names, num_modules)
985
+ ct_dfs = build_celltype_dfs(ct_mean, ct_count, module_gene_names, num_modules)
986
+ specificity_df = compute_specificity_from_means(global_mean, ct_mean, module_gene_names, num_modules)
987
+
988
+ # --- New metrics ---
989
+ logger.info("Computing z-score specificity...")
990
+ zscore_df = compute_specificity_zscore(
991
+ global_mean, global_std, ct_mean, ct_count, module_gene_names, num_modules, args.min_cells)
992
+
993
+ logger.info("Computing Cohen's d...")
994
+ cohens_d_df = compute_cohens_d(
995
+ global_mean, ct_mean, ct_std, ct_count, module_gene_names, num_modules, n_cells, args.min_cells)
996
+
997
+ logger.info("Computing Welch's t-test...")
998
+ welch_df = compute_welch_ttest(
999
+ global_mean, global_std, ct_mean, ct_std, ct_count, module_gene_names,
1000
+ num_modules, n_cells, args.min_cells)
1001
+
1002
+ logger.info("Computing attention-expression correlation...")
1003
+ attn_expr_corr = compute_attention_expression_corr(
1004
+ global_mean, expr_mean, module_gene_names, num_modules)
1005
+
1006
+ logger.info("Computing random baseline comparison...")
1007
+ baseline_stats = compute_random_baseline(global_mean, num_modules)
1008
+
1009
+ logger.info("Computing attention concentration...")
1010
+ concentration_df = compute_attention_concentration(
1011
+ global_mean, ct_mean, ct_count, num_modules, args.min_cells)
1012
+
1013
+ # --- Marker validation ---
1014
+ marker_dict = PANCREAS_MARKERS
1015
+ if args.marker_db:
1016
+ with open(args.marker_db) as f:
1017
+ marker_dict = json.load(f)
1018
+
1019
+ logger.info("Computing marker ranks within cell types...")
1020
+ marker_rank_df = compute_marker_rank_within_celltype(
1021
+ ct_mean, ct_count, module_gene_names, num_modules, marker_dict, args.min_cells)
1022
+
1023
+ marker_results_zscore = validate_markers(zscore_df, marker_dict, sort_col="z_score")
1024
+ marker_results_fc = validate_markers(specificity_df, marker_dict, sort_col="fold_change")
1025
+
1026
+ # --- Save CSVs ---
1027
+ logger.info("Saving CSV results...")
1028
+ global_df.to_csv(output_dir / "global_gene_importance.csv", index=False)
1029
+ specificity_df.to_csv(output_dir / "gene_specificity.csv", index=False)
1030
+ zscore_df.to_csv(output_dir / "gene_specificity_zscore.csv", index=False)
1031
+ cohens_d_df.to_csv(output_dir / "gene_specificity_cohens_d.csv", index=False)
1032
+ welch_df.to_csv(output_dir / "gene_specificity_welch.csv", index=False)
1033
+ concentration_df.to_csv(output_dir / "attention_concentration.csv", index=False)
1034
+ if not marker_rank_df.empty:
1035
+ marker_rank_df.to_csv(output_dir / "marker_ranks_within_celltype.csv", index=False)
1036
+ for ct, df in ct_dfs.items():
1037
+ safe_name = ct.replace("/", "_").replace(" ", "_")
1038
+ df.to_csv(output_dir / f"celltype_{safe_name}_attention.csv", index=False)
1039
+
1040
+ with open(output_dir / "marker_validation_zscore.json", "w") as f:
1041
+ json.dump(marker_results_zscore, f, indent=2, ensure_ascii=False)
1042
+ with open(output_dir / "marker_validation.json", "w") as f:
1043
+ json.dump(marker_results_fc, f, indent=2, ensure_ascii=False)
1044
+
1045
+ # --- DEG correlation ---
1046
+ deg_results = None
1047
+ if args.deg_csv:
1048
+ deg_df = pd.read_csv(args.deg_csv)
1049
+ deg_results = validate_deg_correlation(ct_dfs, deg_df)
1050
+ with open(output_dir / "deg_correlation.json", "w") as f:
1051
+ json.dump(deg_results, f, indent=2)
1052
+
1053
+ # --- Publication figures ---
1054
+ logger.info("Generating publication figures...")
1055
+ plot_global_importance_pub(global_df, zscore_df, args.top_k_genes, output_dir)
1056
+ plot_celltype_heatmap_pub(ct_dfs, ct_count, args.top_k_genes, output_dir, args.min_cells)
1057
+ plot_specificity_volcano(welch_df, zscore_df, args.top_k_genes, output_dir)
1058
+ plot_marker_dotplot(marker_rank_df, output_dir)
1059
+ plot_marker_recall_pub(marker_results_zscore, ct_count, output_dir, args.min_cells)
1060
+ plot_attention_vs_expression(global_mean, expr_mean, module_gene_names, num_modules, output_dir)
1061
+ plot_concentration_pub(concentration_df, baseline_stats, output_dir)
1062
+ plot_baseline_kl(baseline_stats, output_dir)
1063
+ if deg_results:
1064
+ plot_deg_correlation_pub(deg_results, output_dir)
1065
+
1066
+ # --- Enrichment ---
1067
+ run_enrichment_analysis(global_df, args.top_k_genes, output_dir)
1068
+
1069
+ # --- Summary ---
1070
+ filtered_cts = [ct for ct, n in ct_count.items() if n < args.min_cells]
1071
+ summary = {
1072
+ "n_cells": n_cells,
1073
+ "n_modules": num_modules,
1074
+ "n_celltypes": len(ct_count),
1075
+ "celltypes": {ct: int(n) for ct, n in ct_count.items()},
1076
+ "min_cells_filter": args.min_cells,
1077
+ "filtered_celltypes": filtered_cts,
1078
+ "top_10_global_genes": global_df.groupby("gene")["mean_attention"].mean()
1079
+ .nlargest(10).to_dict(),
1080
+ "attn_expr_correlation": attn_expr_corr,
1081
+ "baseline_comparison": baseline_stats,
1082
+ "mean_gini_global": float(concentration_df["global_gini"].mean()),
1083
+ "mean_gini_celltype": float(concentration_df["mean_ct_gini"].mean()),
1084
+ }
1085
+ with open(output_dir / "summary.json", "w") as f:
1086
+ json.dump(summary, f, indent=2, ensure_ascii=False)
1087
+
1088
+ logger.info("Done. Results saved to %s", output_dir)
1089
+
1090
+
1091
+ if __name__ == "__main__":
1092
+ main()
1093
+
BioMod/interpretability/module_geometry.py CHANGED
@@ -1,654 +1,655 @@
1
- """Experiment 1 — Intrinsic geometry of module embeddings.
2
-
3
- Question
4
- --------
5
- Does scLong's encoder organize its module latent space by pathway family
6
- without any explicit family-level supervision, If yes, this is a structural
7
- property gene-level baselines (GenePT / scGPT) cannot exhibit because they
8
- do not have a module tier.
9
-
10
- Pipeline
11
- --------
12
- 1. Load pretrained backbone (forward with mask_ratio=0).
13
- 2. Iterate over a population of cells, accumulate the per-module encoder output
14
- in a streaming average -> [num_modules, base_dim].
15
- 3. Parse each module's family from merged_modules.json keys
16
- (Hallmarks / Reactome / Hotspot / Processed) and, where applicable, the
17
- MSigDB hallmark process subcategory (immune / metabolic / signaling / ...).
18
- 4. Quantify family separation:
19
- * silhouette score (cosine + euclidean)
20
- * kNN purity at k=10
21
- * inter-family / intra-family mean-distance ratio
22
- Compare each metric against a permutation null (shuffle family labels K
23
- times) and report empirical p-values.
24
- 5. Visualise with UMAP and PCA, coloured by family / hallmark process.
25
- 6. Save embeddings, metadata, metrics.json, and figures.
26
-
27
- Data modes
28
- ----------
29
- --data_mode h5ad Generic h5ad. Goes through DataPreprocessor and
30
- reindex_tensor_to_model_space. Works for
31
- cell_annotation / zero_shot / any annotated h5ad.
32
- --data_mode pretrain_npz Per-cell npz directory (cell0.npz, cell1.npz, ...),
33
- already in the 27874-dim model gene space. Requires
34
- --cell_num. Skips reindexing.
35
- --data_mode pretrain_h5ad h5ad whose var.index is already the 27874-dim
36
- model gene list (selected_genes_27k.txt). Skips
37
- reindexing for speed.
38
-
39
- DDP
40
- ---
41
- Supports multi-GPU via torchrun. When WORLD_SIZE>1, each rank processes a
42
- shard of cells; embeddings are all_reduced before metrics/plots. Single-GPU
43
- and CPU work unchanged.
44
-
45
- CLI
46
- ---
47
- python -m performer_module.interpretability.module_geometry \\
48
- --performer_checkpoint <ckpt.pth> \\
49
- --config_pkl <config.pkl> \\
50
- --data_path <path.h5ad> \\
51
- --data_mode h5ad \\
52
- --output_dir ./interp/geometry/<run> \\
53
- --max_cells 5000
54
- """
55
- from __future__ import annotations
56
-
57
- import argparse
58
- import json
59
- import logging
60
- import sys
61
- from pathlib import Path
62
- from typing import Dict, List, Optional, Tuple
63
-
64
- import numpy as np
65
- import pandas as pd
66
- import torch
67
- import torch.distributed as dist
68
- from torch.utils.data import DataLoader
69
- from torch.utils.data.distributed import DistributedSampler
70
-
71
- import matplotlib
72
- matplotlib.use("Agg")
73
- import matplotlib.pyplot as plt
74
-
75
- from sklearn.decomposition import PCA
76
- from sklearn.metrics import silhouette_score
77
- from sklearn.metrics.pairwise import cosine_distances, euclidean_distances
78
- from sklearn.neighbors import NearestNeighbors
79
-
80
- # Make sibling modules importable when launched as a script
81
- HERE = Path(__file__).resolve()
82
- PERFORMER_MODULE = HERE.parents[1]
83
- REPO_ROOT = HERE.parents[2]
84
- for p in (str(PERFORMER_MODULE), str(REPO_ROOT)):
85
- if p not in sys.path:
86
- sys.path.insert(0, p)
87
-
88
- from cell_annotation.performer_module_finetune import ( # noqa: E402
89
- DataPreprocessor,
90
- get_rank,
91
- get_world_size,
92
- init_distributed_mode,
93
- is_dist_initialized,
94
- is_main_process,
95
- load_backbone,
96
- reduce_sum,
97
- resolve_path as ca_resolve_path,
98
- seed_all,
99
- )
100
-
101
- from interpretability.utils import ( # noqa: E402
102
- MatrixRowDataset,
103
- NpzCellDataset,
104
- build_module_metadata,
105
- ensure_dir,
106
- load_module_names,
107
- )
108
-
109
- logger = logging.getLogger(__name__)
110
-
111
-
112
- # ---------- argparse ----------
113
-
114
- def parse_args() -> argparse.Namespace:
115
- parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
116
-
117
- # Pretrained backbone (mirrors cell_annotation.performer_module_finetune)
118
- parser.add_argument("--performer_checkpoint", type=str, required=True)
119
- parser.add_argument("--config_pkl", type=str, required=True)
120
- parser.add_argument("--module_data_dir", type=str, default="../module_data_all")
121
- parser.add_argument("--module_json", type=str, default=None,
122
- help="Path to merged_modules.json. Defaults to <module_data_dir>/merged_modules.json.")
123
- parser.add_argument("--gene_list_path", type=str, default="../../selected_genes_27k.txt")
124
- parser.add_argument("--gene_mapping_path", type=str, default="../../gene2ens.csv")
125
- parser.add_argument("--gene2vec_data_path", type=str, default="../../selected_gene2vec_27k.npy")
126
- parser.add_argument("--gene2ens_file", type=str, default="../../gene2ens.csv")
127
- parser.add_argument("--gene2ens_column", type=str, default="Ensembl gene ID")
128
- parser.add_argument("--shuffle_module_genes", action="store_true",
129
- help="Randomly reshuffle gene-to-module assignment (control / ablation).")
130
- parser.add_argument("--shuffle_seed", type=int, default=2021)
131
-
132
- # Data
133
- parser.add_argument("--data_path", type=str, required=True,
134
- help="h5ad file path, or npz directory for pretrain_npz mode.")
135
- parser.add_argument("--data_mode", type=str, default="h5ad",
136
- choices=["h5ad", "pretrain_h5ad", "pretrain_npz"],
137
- help="h5ad: generic, goes through DataPreprocessor reindex. "
138
- "pretrain_h5ad: var.index is already the 27874 model gene list, skip reindex. "
139
- "pretrain_npz: per-cell npz directory (cell0.npz, ...), skip reindex.")
140
- parser.add_argument("--cell_num", type=int, default=None,
141
- help="Total cell count for pretrain_npz sequential naming (cell0.npz..cellN.npz).")
142
- parser.add_argument("--gene_name_column", type=str, default="auto",
143
- help="Which adata.var column holds gene symbols. 'auto' / 'index' / column name. "
144
- "Only used when data_mode=h5ad.")
145
- parser.add_argument("--dataset_name", type=str, default="interpretability")
146
- parser.add_argument("--min_genes", type=int, default=0)
147
- parser.add_argument("--min_cells", type=int, default=0)
148
- parser.add_argument("--max_cells", type=int, default=None,
149
- help="Optional subsample for inference speed. None = use all cells.")
150
-
151
- # Inference
152
- parser.add_argument("--device", type=str, default="cuda")
153
- parser.add_argument("--batch_size", type=int, default=8)
154
- parser.add_argument("--num_workers", type=int, default=0)
155
- parser.add_argument("--seed", type=int, default=2021)
156
-
157
- # Geometry / metrics
158
- parser.add_argument("--knn_k", type=int, default=10,
159
- help="k for kNN-purity metric.")
160
- parser.add_argument("--null_permutations", type=int, default=200,
161
- help="Number of label-permutation samples for null distribution.")
162
- parser.add_argument("--include_other_in_metrics", action="store_true",
163
- help="If false (default), exclude modules whose family is 'Other' from metric computation.")
164
-
165
- # UMAP
166
- parser.add_argument("--umap_n_neighbors", type=int, default=15)
167
- parser.add_argument("--umap_min_dist", type=float, default=0.3)
168
-
169
- # Output
170
- parser.add_argument("--output_dir", type=str, required=True)
171
- parser.add_argument("--run_name", type=str, default="")
172
-
173
- return parser.parse_args()
174
-
175
-
176
- # ---------- dataset construction ----------
177
-
178
- def build_inference_loader(
179
- args: argparse.Namespace,
180
- preprocessor: Optional[DataPreprocessor],
181
- use_ddp: bool,
182
- rank: int,
183
- world_size: int,
184
- ) -> Tuple[DataLoader, Optional[dict], int]:
185
- """Build a DataLoader yielding raw [G_input] expression rows and a reindex_state.
186
-
187
- Returns: (loader, reindex_state_or_None, n_cells_total)
188
- """
189
- if args.data_mode == "h5ad":
190
- import anndata as ad
191
- assert preprocessor is not None
192
- adata, reindex_state = preprocessor.preprocess_adata(
193
- adata_path=args.data_path,
194
- min_genes=args.min_genes,
195
- min_cells=args.min_cells,
196
- max_cells=args.max_cells,
197
- dataset_name=args.dataset_name,
198
- gene_name_column=args.gene_name_column,
199
- )
200
- dataset = MatrixRowDataset(adata.X)
201
- n_cells = adata.n_obs
202
- elif args.data_mode == "pretrain_h5ad":
203
- import anndata as ad
204
- logger.info("Loading pretrain-mode h5ad %s", args.data_path)
205
- adata = ad.read_h5ad(args.data_path)
206
- if args.max_cells is not None and adata.n_obs > args.max_cells:
207
- rng = np.random.default_rng(args.seed)
208
- chosen = np.sort(rng.choice(adata.n_obs, size=args.max_cells, replace=False))
209
- adata = adata[chosen].copy()
210
- dataset = MatrixRowDataset(adata.X)
211
- n_cells = adata.n_obs
212
- reindex_state = None
213
- elif args.data_mode == "pretrain_npz":
214
- dataset = NpzCellDataset(
215
- data_dir=args.data_path,
216
- cell_num=args.cell_num,
217
- max_cells=args.max_cells,
218
- seed=args.seed,
219
- )
220
- n_cells = len(dataset)
221
- reindex_state = None
222
- else:
223
- raise ValueError(f"Unsupported data_mode: {args.data_mode}")
224
-
225
- if use_ddp:
226
- sampler = DistributedSampler(dataset, num_replicas=world_size, rank=rank,
227
- shuffle=False, drop_last=False)
228
- else:
229
- sampler = None
230
-
231
- loader = DataLoader(
232
- dataset,
233
- batch_size=args.batch_size,
234
- shuffle=False,
235
- sampler=sampler,
236
- num_workers=args.num_workers,
237
- pin_memory=False,
238
- )
239
- logger.info("Built inference loader: %d cells total, batch_size=%d, ddp=%s",
240
- n_cells, args.batch_size, use_ddp)
241
- return loader, reindex_state, n_cells
242
-
243
-
244
- # ---------- inference: cell-averaged module embeddings ----------
245
-
246
- @torch.no_grad()
247
- def extract_cell_averaged_module_embeddings(
248
- backbone,
249
- loader: DataLoader,
250
- preprocessor: Optional[DataPreprocessor],
251
- reindex_state: Optional[dict],
252
- device: torch.device,
253
- num_modules: int,
254
- base_dim: int,
255
- expected_input_genes: int,
256
- use_ddp: bool,
257
- ) -> Tuple[np.ndarray, int]:
258
- """Stream forward over the loader and accumulate the cell-mean module embedding.
259
-
260
- Under DDP each rank processes its shard; sum_emb and n_cells are all_reduced
261
- before computing the mean.
262
-
263
- Returns: (mean_embedding [num_modules, base_dim] float32, n_cells_total)
264
- """
265
- backbone.eval()
266
- sum_emb = torch.zeros(num_modules, base_dim, device=device, dtype=torch.float64)
267
- n_cells = 0
268
-
269
- for batch_idx, x in enumerate(loader):
270
- x = x.to(device, non_blocking=False)
271
- if reindex_state is not None:
272
- x_model = preprocessor.reindex_tensor_to_model_space(x, reindex_state=reindex_state)
273
- else:
274
- if x.shape[1] != expected_input_genes:
275
- raise ValueError(
276
- f"pretrain mode expects {expected_input_genes} genes per cell, got {x.shape[1]}. "
277
- f"Use --data_mode h5ad (with reindex) for arbitrary gene spaces."
278
- )
279
- x_model = x
280
-
281
- out = backbone(x_model, mask_ratio=0.0, return_encodings=True)
282
- enc = out["module_encodings"] # (B, num_modules, D)
283
- if enc.shape[1] != num_modules or enc.shape[2] != base_dim:
284
- raise RuntimeError(
285
- f"Encoder output shape mismatch: got {tuple(enc.shape)}, "
286
- f"expected (*, {num_modules}, {base_dim}). "
287
- "Module order between merged_modules.json and the loaded backbone may have drifted."
288
- )
289
- sum_emb += enc.to(torch.float64).sum(dim=0)
290
- n_cells += enc.shape[0]
291
- if (batch_idx + 1) % 50 == 0:
292
- logger.info(" forward batch %d (%d cells on this rank so far)", batch_idx + 1, n_cells)
293
-
294
- if use_ddp:
295
- dist.all_reduce(sum_emb, op=dist.ReduceOp.SUM)
296
- n_cells_tensor = torch.tensor(n_cells, device=device, dtype=torch.long)
297
- dist.all_reduce(n_cells_tensor, op=dist.ReduceOp.SUM)
298
- n_cells = int(n_cells_tensor.item())
299
-
300
- mean_emb = (sum_emb / max(n_cells, 1)).cpu().numpy().astype(np.float32)
301
- return mean_emb, n_cells
302
-
303
-
304
- # ---------- structure metrics ----------
305
-
306
- def _label_to_int(labels: List[str]) -> np.ndarray:
307
- uniq = sorted(set(labels))
308
- mapping = {lab: i for i, lab in enumerate(uniq)}
309
- return np.asarray([mapping[lab] for lab in labels], dtype=np.int64)
310
-
311
-
312
- def _knn_purity(emb: np.ndarray, labels: np.ndarray, k: int, metric: str = "cosine") -> float:
313
- n = emb.shape[0]
314
- k_use = min(k, n - 1)
315
- nn = NearestNeighbors(n_neighbors=k_use + 1, metric=metric)
316
- nn.fit(emb)
317
- _, idx = nn.kneighbors(emb)
318
- # idx[:, 0] == self; drop it
319
- neighbor_labels = labels[idx[:, 1:]]
320
- own_labels = labels[:, None]
321
- purity = (neighbor_labels == own_labels).mean()
322
- return float(purity)
323
-
324
-
325
- def _intra_inter_distance_ratio(distance_matrix: np.ndarray, labels: np.ndarray) -> Dict[str, float]:
326
- """Mean intra-family vs inter-family pairwise distance and inter/intra ratio."""
327
- n = distance_matrix.shape[0]
328
- iu = np.triu_indices(n, k=1)
329
- pair_d = distance_matrix[iu]
330
- same = (labels[iu[0]] == labels[iu[1]])
331
- intra = float(pair_d[same].mean()) if same.any() else float("nan")
332
- inter = float(pair_d[~same].mean()) if (~same).any() else float("nan")
333
- ratio = float(inter / intra) if intra > 0 else float("nan")
334
- return {"intra_mean": intra, "inter_mean": inter, "inter_over_intra": ratio}
335
-
336
-
337
- def compute_structure_metrics(
338
- embedding: np.ndarray,
339
- labels: np.ndarray,
340
- knn_k: int,
341
- ) -> Dict[str, float]:
342
- metrics: Dict[str, float] = {}
343
- n_classes = len(np.unique(labels))
344
- if n_classes < 2 or len(labels) < 3:
345
- return {"silhouette_cosine": float("nan"),
346
- "silhouette_euclidean": float("nan"),
347
- "knn_purity_cosine": float("nan"),
348
- "knn_purity_euclidean": float("nan"),
349
- "inter_over_intra_cosine": float("nan"),
350
- "inter_over_intra_euclidean": float("nan")}
351
-
352
- metrics["silhouette_cosine"] = float(silhouette_score(embedding, labels, metric="cosine"))
353
- metrics["silhouette_euclidean"] = float(silhouette_score(embedding, labels, metric="euclidean"))
354
- metrics["knn_purity_cosine"] = _knn_purity(embedding, labels, knn_k, metric="cosine")
355
- metrics["knn_purity_euclidean"] = _knn_purity(embedding, labels, knn_k, metric="euclidean")
356
-
357
- cos_d = cosine_distances(embedding)
358
- euc_d = euclidean_distances(embedding)
359
- cos_ratio = _intra_inter_distance_ratio(cos_d, labels)
360
- euc_ratio = _intra_inter_distance_ratio(euc_d, labels)
361
- metrics["intra_mean_cosine"] = cos_ratio["intra_mean"]
362
- metrics["inter_mean_cosine"] = cos_ratio["inter_mean"]
363
- metrics["inter_over_intra_cosine"] = cos_ratio["inter_over_intra"]
364
- metrics["intra_mean_euclidean"] = euc_ratio["intra_mean"]
365
- metrics["inter_mean_euclidean"] = euc_ratio["inter_mean"]
366
- metrics["inter_over_intra_euclidean"] = euc_ratio["inter_over_intra"]
367
- return metrics
368
-
369
-
370
- def permutation_null(
371
- embedding: np.ndarray,
372
- labels: np.ndarray,
373
- knn_k: int,
374
- n_permutations: int,
375
- seed: int,
376
- ) -> Dict[str, Dict[str, float]]:
377
- """Shuffle labels K times, recompute metrics, return per-metric (mean, std, p_value).
378
-
379
- p-value: empirical right-tail Pr(null >= observed) under random labeling.
380
- """
381
- observed = compute_structure_metrics(embedding, labels, knn_k)
382
- rng = np.random.default_rng(seed)
383
- null_records: Dict[str, List[float]] = {k: [] for k in observed}
384
-
385
- for _ in range(n_permutations):
386
- permuted = rng.permutation(labels)
387
- m = compute_structure_metrics(embedding, permuted, knn_k)
388
- for k, v in m.items():
389
- null_records[k].append(v)
390
-
391
- summary: Dict[str, Dict[str, float]] = {}
392
- for k, obs_v in observed.items():
393
- arr = np.asarray(null_records[k], dtype=float)
394
- valid = arr[~np.isnan(arr)]
395
- if valid.size == 0 or np.isnan(obs_v):
396
- summary[k] = {"observed": obs_v, "null_mean": float("nan"),
397
- "null_std": float("nan"), "p_right_tail": float("nan")}
398
- continue
399
- p = float((1 + np.sum(valid >= obs_v)) / (valid.size + 1))
400
- summary[k] = {
401
- "observed": float(obs_v),
402
- "null_mean": float(valid.mean()),
403
- "null_std": float(valid.std()),
404
- "p_right_tail": p,
405
- }
406
- return summary
407
-
408
-
409
- # ---------- visualization ----------
410
-
411
- def _color_map(labels: List[str]) -> Dict[str, tuple]:
412
- uniq = sorted(set(labels))
413
- cmap = plt.get_cmap("tab20", max(len(uniq), 1))
414
- return {lab: cmap(i) for i, lab in enumerate(uniq)}
415
-
416
-
417
- def _scatter(coords: np.ndarray, labels: List[str], title: str, out_path: Path) -> None:
418
- fig, ax = plt.subplots(figsize=(7, 6))
419
- cmap = _color_map(labels)
420
- for lab in sorted(set(labels)):
421
- mask = np.asarray([lab == l for l in labels])
422
- ax.scatter(coords[mask, 0], coords[mask, 1],
423
- color=cmap[lab], label=f"{lab} (n={mask.sum()})",
424
- s=20, alpha=0.85, edgecolor="none")
425
- ax.set_title(title)
426
- ax.set_xlabel("dim 1")
427
- ax.set_ylabel("dim 2")
428
- ax.legend(loc="best", fontsize=8, frameon=True)
429
- fig.tight_layout()
430
- fig.savefig(out_path, dpi=160)
431
- plt.close(fig)
432
-
433
-
434
- def visualize(
435
- embedding: np.ndarray,
436
- metadata_df: pd.DataFrame,
437
- output_dir: Path,
438
- umap_n_neighbors: int,
439
- umap_min_dist: float,
440
- ) -> None:
441
- # PCA 2D
442
- pca = PCA(n_components=2, random_state=0)
443
- pca_coords = pca.fit_transform(embedding)
444
- _scatter(pca_coords, metadata_df["family"].tolist(),
445
- "Module embedding PCA — family", output_dir / "pca_family.png")
446
-
447
- has_hallmark = metadata_df["hallmark_process"].notna().any()
448
- if has_hallmark:
449
- # Project only hallmark modules to a separate PCA for readability
450
- hm_mask = metadata_df["hallmark_process"].notna().to_numpy()
451
- if hm_mask.sum() >= 3:
452
- hm_pca = PCA(n_components=2, random_state=0).fit_transform(embedding[hm_mask])
453
- _scatter(hm_pca, metadata_df.loc[hm_mask, "hallmark_process"].tolist(),
454
- "Hallmark modules PCA — process",
455
- output_dir / "pca_hallmark_process.png")
456
-
457
- # UMAP (optional)
458
- try:
459
- import umap # type: ignore
460
- reducer = umap.UMAP(
461
- n_components=2,
462
- n_neighbors=min(umap_n_neighbors, max(2, embedding.shape[0] - 1)),
463
- min_dist=umap_min_dist,
464
- metric="cosine",
465
- random_state=0,
466
- )
467
- umap_coords = reducer.fit_transform(embedding)
468
- _scatter(umap_coords, metadata_df["family"].tolist(),
469
- "Module embedding UMAP — family", output_dir / "umap_family.png")
470
-
471
- if has_hallmark and hm_mask.sum() >= 3:
472
- hm_reducer = umap.UMAP(
473
- n_components=2,
474
- n_neighbors=min(umap_n_neighbors, max(2, hm_mask.sum() - 1)),
475
- min_dist=umap_min_dist,
476
- metric="cosine",
477
- random_state=0,
478
- )
479
- hm_umap = hm_reducer.fit_transform(embedding[hm_mask])
480
- _scatter(hm_umap, metadata_df.loc[hm_mask, "hallmark_process"].tolist(),
481
- "Hallmark modules UMAP — process",
482
- output_dir / "umap_hallmark_process.png")
483
- except ImportError:
484
- logger.warning("umap-learn not installed; skipping UMAP plots. `pip install umap-learn` to enable.")
485
-
486
-
487
- # ---------- driver ----------
488
-
489
- def setup_logging(output_dir: Path) -> None:
490
- log_path = output_dir / "run.log"
491
- logging.basicConfig(
492
- level=logging.INFO,
493
- format="%(asctime)s | %(levelname)s | %(name)s | %(message)s",
494
- handlers=[logging.StreamHandler(sys.stdout), logging.FileHandler(log_path, mode="w")],
495
- force=True,
496
- )
497
-
498
-
499
- def resolve_module_json(args: argparse.Namespace) -> str:
500
- """Resolve relative paths against performer_module/cell_annotation/ to stay
501
- consistent with how `load_backbone` reads `module_data_dir`.
502
- """
503
- if args.module_json is not None:
504
- return str(ca_resolve_path(args.module_json))
505
- return str(ca_resolve_path(args.module_data_dir) / "merged_modules.json")
506
-
507
-
508
- def main() -> None:
509
- args = parse_args()
510
-
511
- # DDP init (auto-detects torchrun env; falls back to single-process)
512
- device, use_ddp, rank, world_size, local_rank = init_distributed_mode(args.device)
513
- seed_all(args.seed + rank)
514
-
515
- if is_main_process():
516
- output_dir = ensure_dir(args.output_dir)
517
- setup_logging(output_dir)
518
- else:
519
- output_dir = Path(args.output_dir)
520
- logging.basicConfig(level=logging.WARNING)
521
- if use_ddp:
522
- dist.barrier()
523
-
524
- logger.info("Args: %s", vars(args))
525
- logger.info("DDP: use_ddp=%s, rank=%d/%d, device=%s", use_ddp, rank, world_size, device)
526
-
527
- # 1. Load backbone
528
- backbone, module_data, model_config = load_backbone(args, device)
529
- backbone.eval()
530
- base_dim = int(model_config["base_dim"])
531
- num_modules = len(module_data)
532
- logger.info("Loaded backbone: num_modules=%d, base_dim=%d", num_modules, base_dim)
533
-
534
- # 2. Load module names (must align with module_data order)
535
- module_json_path = resolve_module_json(args)
536
- module_names = load_module_names(module_json_path)
537
- if len(module_names) != num_modules:
538
- raise RuntimeError(
539
- f"Module name count ({len(module_names)} from {module_json_path}) does not match "
540
- f"backbone num_modules ({num_modules}). The merged_modules.json used to build the "
541
- "model and the one given here must be identical, with no modules dropped during "
542
- "gene-symbol resolution."
543
- )
544
- metadata = build_module_metadata(module_names)
545
- metadata_df = pd.DataFrame(metadata)
546
- if is_main_process():
547
- logger.info("Family distribution:\n%s", metadata_df["family"].value_counts().to_string())
548
-
549
- # 3. Build dataloader
550
- if args.data_mode == "h5ad":
551
- preprocessor = DataPreprocessor(
552
- gene2ens_file=args.gene2ens_file,
553
- gene2ens_column=args.gene2ens_column,
554
- model_genes_file=args.gene_list_path,
555
- )
556
- preprocessor.load_gene_mapping()
557
- preprocessor.load_model_genes()
558
- expected_input_genes = -1
559
- else:
560
- preprocessor = None
561
- expected_input_genes = int(model_config["num_genes"])
562
-
563
- loader, reindex_state, n_cells = build_inference_loader(
564
- args, preprocessor, use_ddp, rank, world_size,
565
- )
566
- if n_cells == 0:
567
- raise RuntimeError("No cells survived preprocessing.")
568
-
569
- # 4. Inference
570
- logger.info("Extracting cell-averaged module embeddings over %d cells...", n_cells)
571
- mean_emb, processed = extract_cell_averaged_module_embeddings(
572
- backbone, loader, preprocessor, reindex_state, device,
573
- num_modules=num_modules, base_dim=base_dim,
574
- expected_input_genes=expected_input_genes,
575
- use_ddp=use_ddp,
576
- )
577
- logger.info("Embedding ready: shape=%s, processed_cells=%d", mean_emb.shape, processed)
578
-
579
- # --- Everything below is rank-0 only ---
580
- if is_main_process():
581
- # 5. Save raw outputs
582
- np.save(output_dir / "module_embeddings.npy", mean_emb)
583
- metadata_df.to_csv(output_dir / "module_metadata.csv", index=False)
584
-
585
- # 6. Metrics — family level
586
- if args.include_other_in_metrics:
587
- keep_mask = np.ones(len(metadata_df), dtype=bool)
588
- else:
589
- keep_mask = (metadata_df["family"] != "Other").to_numpy()
590
- family_labels = metadata_df.loc[keep_mask, "family"].tolist()
591
- family_emb = mean_emb[keep_mask]
592
-
593
- logger.info("Computing structure metrics — family level (n=%d)", keep_mask.sum())
594
- family_summary = permutation_null(
595
- family_emb, _label_to_int(family_labels),
596
- knn_k=args.knn_k,
597
- n_permutations=args.null_permutations,
598
- seed=args.seed,
599
- )
600
-
601
- # 6b. Metrics — hallmark process level
602
- hm_mask = metadata_df["hallmark_process"].notna().to_numpy()
603
- hallmark_summary: Optional[Dict[str, Dict[str, float]]] = None
604
- if hm_mask.sum() >= 3 and metadata_df.loc[hm_mask, "hallmark_process"].nunique() >= 2:
605
- logger.info("Computing structure metrics — hallmark process level (n=%d)", hm_mask.sum())
606
- hallmark_summary = permutation_null(
607
- mean_emb[hm_mask],
608
- _label_to_int(metadata_df.loc[hm_mask, "hallmark_process"].tolist()),
609
- knn_k=args.knn_k,
610
- n_permutations=args.null_permutations,
611
- seed=args.seed,
612
- )
613
-
614
- metrics_payload = {
615
- "n_cells_processed": int(processed),
616
- "num_modules": int(num_modules),
617
- "base_dim": int(base_dim),
618
- "knn_k": int(args.knn_k),
619
- "null_permutations": int(args.null_permutations),
620
- "family_metrics": family_summary,
621
- "hallmark_process_metrics": hallmark_summary,
622
- "family_counts": metadata_df["family"].value_counts().to_dict(),
623
- "hallmark_process_counts": metadata_df["hallmark_process"].dropna().value_counts().to_dict(),
624
- "shuffle_module_genes": bool(args.shuffle_module_genes),
625
- }
626
- with open(output_dir / "metrics.json", "w", encoding="utf-8") as fh:
627
- json.dump(metrics_payload, fh, indent=2)
628
-
629
- logger.info("=== Family-level structure (vs random-shuffled label null) ===")
630
- for k, v in family_summary.items():
631
- logger.info(" %-30s observed=%.4f null=%.4f±%.4f p=%.4f",
632
- k, v["observed"], v["null_mean"], v["null_std"], v["p_right_tail"])
633
- if hallmark_summary is not None:
634
- logger.info("=== Hallmark-process-level structure (Hallmarks subset) ===")
635
- for k, v in hallmark_summary.items():
636
- logger.info(" %-30s observed=%.4f null=%.4f±%.4f p=%.4f",
637
- k, v["observed"], v["null_mean"], v["null_std"], v["p_right_tail"])
638
-
639
- # 7. Visualisation
640
- logger.info("Plotting UMAP / PCA...")
641
- visualize(mean_emb, metadata_df, output_dir,
642
- umap_n_neighbors=args.umap_n_neighbors,
643
- umap_min_dist=args.umap_min_dist)
644
-
645
- logger.info("Done. Outputs saved to %s", output_dir)
646
-
647
- # Teardown
648
- if use_ddp:
649
- dist.barrier()
650
- dist.destroy_process_group()
651
-
652
-
653
- if __name__ == "__main__":
654
- main()
 
 
1
+ """Experiment 1 — Intrinsic geometry of module embeddings.
2
+
3
+ Question
4
+ --------
5
+ Does scLong's encoder organize its module latent space by pathway family
6
+ without any explicit family-level supervision, If yes, this is a structural
7
+ property gene-level baselines (GenePT / scGPT) cannot exhibit because they
8
+ do not have a module tier.
9
+
10
+ Pipeline
11
+ --------
12
+ 1. Load pretrained backbone (forward with mask_ratio=0).
13
+ 2. Iterate over a population of cells, accumulate the per-module encoder output
14
+ in a streaming average -> [num_modules, base_dim].
15
+ 3. Parse each module's family from merged_modules.json keys
16
+ (Hallmarks / Reactome / Hotspot / Processed) and, where applicable, the
17
+ MSigDB hallmark process subcategory (immune / metabolic / signaling / ...).
18
+ 4. Quantify family separation:
19
+ * silhouette score (cosine + euclidean)
20
+ * kNN purity at k=10
21
+ * inter-family / intra-family mean-distance ratio
22
+ Compare each metric against a permutation null (shuffle family labels K
23
+ times) and report empirical p-values.
24
+ 5. Visualise with UMAP and PCA, coloured by family / hallmark process.
25
+ 6. Save embeddings, metadata, metrics.json, and figures.
26
+
27
+ Data modes
28
+ ----------
29
+ --data_mode h5ad Generic h5ad. Goes through DataPreprocessor and
30
+ reindex_tensor_to_model_space. Works for
31
+ cell_annotation / zero_shot / any annotated h5ad.
32
+ --data_mode pretrain_npz Per-cell npz directory (cell0.npz, cell1.npz, ...),
33
+ already in the 27874-dim model gene space. Requires
34
+ --cell_num. Skips reindexing.
35
+ --data_mode pretrain_h5ad h5ad whose var.index is already the 27874-dim
36
+ model gene list (selected_genes_27k.txt). Skips
37
+ reindexing for speed.
38
+
39
+ DDP
40
+ ---
41
+ Supports multi-GPU via torchrun. When WORLD_SIZE>1, each rank processes a
42
+ shard of cells; embeddings are all_reduced before metrics/plots. Single-GPU
43
+ and CPU work unchanged.
44
+
45
+ CLI
46
+ ---
47
+ python -m performer_module.interpretability.module_geometry \\
48
+ --performer_checkpoint <ckpt.pth> \\
49
+ --config_pkl <config.pkl> \\
50
+ --data_path <path.h5ad> \\
51
+ --data_mode h5ad \\
52
+ --output_dir ./interp/geometry/<run> \\
53
+ --max_cells 5000
54
+ """
55
+ from __future__ import annotations
56
+
57
+ import argparse
58
+ import json
59
+ import logging
60
+ import sys
61
+ from pathlib import Path
62
+ from typing import Dict, List, Optional, Tuple
63
+
64
+ import numpy as np
65
+ import pandas as pd
66
+ import torch
67
+ import torch.distributed as dist
68
+ from torch.utils.data import DataLoader
69
+ from torch.utils.data.distributed import DistributedSampler
70
+
71
+ import matplotlib
72
+ matplotlib.use("Agg")
73
+ import matplotlib.pyplot as plt
74
+
75
+ from sklearn.decomposition import PCA
76
+ from sklearn.metrics import silhouette_score
77
+ from sklearn.metrics.pairwise import cosine_distances, euclidean_distances
78
+ from sklearn.neighbors import NearestNeighbors
79
+
80
+ # Make sibling modules importable when launched as a script
81
+ HERE = Path(__file__).resolve()
82
+ PERFORMER_MODULE = HERE.parents[1]
83
+ REPO_ROOT = HERE.parents[2]
84
+ ASSET_ROOT = REPO_ROOT / "model_dependencies"
85
+ for p in (str(PERFORMER_MODULE), str(REPO_ROOT)):
86
+ if p not in sys.path:
87
+ sys.path.insert(0, p)
88
+
89
+ from cell_annotation.performer_module_finetune import ( # noqa: E402
90
+ DataPreprocessor,
91
+ get_rank,
92
+ get_world_size,
93
+ init_distributed_mode,
94
+ is_dist_initialized,
95
+ is_main_process,
96
+ load_backbone,
97
+ reduce_sum,
98
+ resolve_path as ca_resolve_path,
99
+ seed_all,
100
+ )
101
+
102
+ from interpretability.utils import ( # noqa: E402
103
+ MatrixRowDataset,
104
+ NpzCellDataset,
105
+ build_module_metadata,
106
+ ensure_dir,
107
+ load_module_names,
108
+ )
109
+
110
+ logger = logging.getLogger(__name__)
111
+
112
+
113
+ # ---------- argparse ----------
114
+
115
+ def parse_args() -> argparse.Namespace:
116
+ parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
117
+
118
+ # Pretrained backbone (mirrors cell_annotation.performer_module_finetune)
119
+ parser.add_argument("--performer_checkpoint", type=str, required=True)
120
+ parser.add_argument("--config_pkl", type=str, required=True)
121
+ parser.add_argument("--module_data_dir", type=str, default=str(ASSET_ROOT / "module_data_all"))
122
+ parser.add_argument("--module_json", type=str, default=None,
123
+ help="Path to merged_modules.json. Defaults to <module_data_dir>/merged_modules.json.")
124
+ parser.add_argument("--gene_list_path", type=str, default=str(ASSET_ROOT / "selected_genes_27k.txt"))
125
+ parser.add_argument("--gene_mapping_path", type=str, default=str(ASSET_ROOT / "gene2ens.csv"))
126
+ parser.add_argument("--gene2vec_data_path", type=str, default=str(ASSET_ROOT / "selected_gene2vec_27k.npy"))
127
+ parser.add_argument("--gene2ens_file", type=str, default=str(ASSET_ROOT / "gene2ens.csv"))
128
+ parser.add_argument("--gene2ens_column", type=str, default="Ensembl gene ID")
129
+ parser.add_argument("--shuffle_module_genes", action="store_true",
130
+ help="Randomly reshuffle gene-to-module assignment (control / ablation).")
131
+ parser.add_argument("--shuffle_seed", type=int, default=2021)
132
+
133
+ # Data
134
+ parser.add_argument("--data_path", type=str, required=True,
135
+ help="h5ad file path, or npz directory for pretrain_npz mode.")
136
+ parser.add_argument("--data_mode", type=str, default="h5ad",
137
+ choices=["h5ad", "pretrain_h5ad", "pretrain_npz"],
138
+ help="h5ad: generic, goes through DataPreprocessor reindex. "
139
+ "pretrain_h5ad: var.index is already the 27874 model gene list, skip reindex. "
140
+ "pretrain_npz: per-cell npz directory (cell0.npz, ...), skip reindex.")
141
+ parser.add_argument("--cell_num", type=int, default=None,
142
+ help="Total cell count for pretrain_npz sequential naming (cell0.npz..cellN.npz).")
143
+ parser.add_argument("--gene_name_column", type=str, default="auto",
144
+ help="Which adata.var column holds gene symbols. 'auto' / 'index' / column name. "
145
+ "Only used when data_mode=h5ad.")
146
+ parser.add_argument("--dataset_name", type=str, default="interpretability")
147
+ parser.add_argument("--min_genes", type=int, default=0)
148
+ parser.add_argument("--min_cells", type=int, default=0)
149
+ parser.add_argument("--max_cells", type=int, default=None,
150
+ help="Optional subsample for inference speed. None = use all cells.")
151
+
152
+ # Inference
153
+ parser.add_argument("--device", type=str, default="cuda")
154
+ parser.add_argument("--batch_size", type=int, default=8)
155
+ parser.add_argument("--num_workers", type=int, default=0)
156
+ parser.add_argument("--seed", type=int, default=2021)
157
+
158
+ # Geometry / metrics
159
+ parser.add_argument("--knn_k", type=int, default=10,
160
+ help="k for kNN-purity metric.")
161
+ parser.add_argument("--null_permutations", type=int, default=200,
162
+ help="Number of label-permutation samples for null distribution.")
163
+ parser.add_argument("--include_other_in_metrics", action="store_true",
164
+ help="If false (default), exclude modules whose family is 'Other' from metric computation.")
165
+
166
+ # UMAP
167
+ parser.add_argument("--umap_n_neighbors", type=int, default=15)
168
+ parser.add_argument("--umap_min_dist", type=float, default=0.3)
169
+
170
+ # Output
171
+ parser.add_argument("--output_dir", type=str, required=True)
172
+ parser.add_argument("--run_name", type=str, default="")
173
+
174
+ return parser.parse_args()
175
+
176
+
177
+ # ---------- dataset construction ----------
178
+
179
+ def build_inference_loader(
180
+ args: argparse.Namespace,
181
+ preprocessor: Optional[DataPreprocessor],
182
+ use_ddp: bool,
183
+ rank: int,
184
+ world_size: int,
185
+ ) -> Tuple[DataLoader, Optional[dict], int]:
186
+ """Build a DataLoader yielding raw [G_input] expression rows and a reindex_state.
187
+
188
+ Returns: (loader, reindex_state_or_None, n_cells_total)
189
+ """
190
+ if args.data_mode == "h5ad":
191
+ import anndata as ad
192
+ assert preprocessor is not None
193
+ adata, reindex_state = preprocessor.preprocess_adata(
194
+ adata_path=args.data_path,
195
+ min_genes=args.min_genes,
196
+ min_cells=args.min_cells,
197
+ max_cells=args.max_cells,
198
+ dataset_name=args.dataset_name,
199
+ gene_name_column=args.gene_name_column,
200
+ )
201
+ dataset = MatrixRowDataset(adata.X)
202
+ n_cells = adata.n_obs
203
+ elif args.data_mode == "pretrain_h5ad":
204
+ import anndata as ad
205
+ logger.info("Loading pretrain-mode h5ad %s", args.data_path)
206
+ adata = ad.read_h5ad(args.data_path)
207
+ if args.max_cells is not None and adata.n_obs > args.max_cells:
208
+ rng = np.random.default_rng(args.seed)
209
+ chosen = np.sort(rng.choice(adata.n_obs, size=args.max_cells, replace=False))
210
+ adata = adata[chosen].copy()
211
+ dataset = MatrixRowDataset(adata.X)
212
+ n_cells = adata.n_obs
213
+ reindex_state = None
214
+ elif args.data_mode == "pretrain_npz":
215
+ dataset = NpzCellDataset(
216
+ data_dir=args.data_path,
217
+ cell_num=args.cell_num,
218
+ max_cells=args.max_cells,
219
+ seed=args.seed,
220
+ )
221
+ n_cells = len(dataset)
222
+ reindex_state = None
223
+ else:
224
+ raise ValueError(f"Unsupported data_mode: {args.data_mode}")
225
+
226
+ if use_ddp:
227
+ sampler = DistributedSampler(dataset, num_replicas=world_size, rank=rank,
228
+ shuffle=False, drop_last=False)
229
+ else:
230
+ sampler = None
231
+
232
+ loader = DataLoader(
233
+ dataset,
234
+ batch_size=args.batch_size,
235
+ shuffle=False,
236
+ sampler=sampler,
237
+ num_workers=args.num_workers,
238
+ pin_memory=False,
239
+ )
240
+ logger.info("Built inference loader: %d cells total, batch_size=%d, ddp=%s",
241
+ n_cells, args.batch_size, use_ddp)
242
+ return loader, reindex_state, n_cells
243
+
244
+
245
+ # ---------- inference: cell-averaged module embeddings ----------
246
+
247
+ @torch.no_grad()
248
+ def extract_cell_averaged_module_embeddings(
249
+ backbone,
250
+ loader: DataLoader,
251
+ preprocessor: Optional[DataPreprocessor],
252
+ reindex_state: Optional[dict],
253
+ device: torch.device,
254
+ num_modules: int,
255
+ base_dim: int,
256
+ expected_input_genes: int,
257
+ use_ddp: bool,
258
+ ) -> Tuple[np.ndarray, int]:
259
+ """Stream forward over the loader and accumulate the cell-mean module embedding.
260
+
261
+ Under DDP each rank processes its shard; sum_emb and n_cells are all_reduced
262
+ before computing the mean.
263
+
264
+ Returns: (mean_embedding [num_modules, base_dim] float32, n_cells_total)
265
+ """
266
+ backbone.eval()
267
+ sum_emb = torch.zeros(num_modules, base_dim, device=device, dtype=torch.float64)
268
+ n_cells = 0
269
+
270
+ for batch_idx, x in enumerate(loader):
271
+ x = x.to(device, non_blocking=False)
272
+ if reindex_state is not None:
273
+ x_model = preprocessor.reindex_tensor_to_model_space(x, reindex_state=reindex_state)
274
+ else:
275
+ if x.shape[1] != expected_input_genes:
276
+ raise ValueError(
277
+ f"pretrain mode expects {expected_input_genes} genes per cell, got {x.shape[1]}. "
278
+ f"Use --data_mode h5ad (with reindex) for arbitrary gene spaces."
279
+ )
280
+ x_model = x
281
+
282
+ out = backbone(x_model, mask_ratio=0.0, return_encodings=True)
283
+ enc = out["module_encodings"] # (B, num_modules, D)
284
+ if enc.shape[1] != num_modules or enc.shape[2] != base_dim:
285
+ raise RuntimeError(
286
+ f"Encoder output shape mismatch: got {tuple(enc.shape)}, "
287
+ f"expected (*, {num_modules}, {base_dim}). "
288
+ "Module order between merged_modules.json and the loaded backbone may have drifted."
289
+ )
290
+ sum_emb += enc.to(torch.float64).sum(dim=0)
291
+ n_cells += enc.shape[0]
292
+ if (batch_idx + 1) % 50 == 0:
293
+ logger.info(" forward batch %d (%d cells on this rank so far)", batch_idx + 1, n_cells)
294
+
295
+ if use_ddp:
296
+ dist.all_reduce(sum_emb, op=dist.ReduceOp.SUM)
297
+ n_cells_tensor = torch.tensor(n_cells, device=device, dtype=torch.long)
298
+ dist.all_reduce(n_cells_tensor, op=dist.ReduceOp.SUM)
299
+ n_cells = int(n_cells_tensor.item())
300
+
301
+ mean_emb = (sum_emb / max(n_cells, 1)).cpu().numpy().astype(np.float32)
302
+ return mean_emb, n_cells
303
+
304
+
305
+ # ---------- structure metrics ----------
306
+
307
+ def _label_to_int(labels: List[str]) -> np.ndarray:
308
+ uniq = sorted(set(labels))
309
+ mapping = {lab: i for i, lab in enumerate(uniq)}
310
+ return np.asarray([mapping[lab] for lab in labels], dtype=np.int64)
311
+
312
+
313
+ def _knn_purity(emb: np.ndarray, labels: np.ndarray, k: int, metric: str = "cosine") -> float:
314
+ n = emb.shape[0]
315
+ k_use = min(k, n - 1)
316
+ nn = NearestNeighbors(n_neighbors=k_use + 1, metric=metric)
317
+ nn.fit(emb)
318
+ _, idx = nn.kneighbors(emb)
319
+ # idx[:, 0] == self; drop it
320
+ neighbor_labels = labels[idx[:, 1:]]
321
+ own_labels = labels[:, None]
322
+ purity = (neighbor_labels == own_labels).mean()
323
+ return float(purity)
324
+
325
+
326
+ def _intra_inter_distance_ratio(distance_matrix: np.ndarray, labels: np.ndarray) -> Dict[str, float]:
327
+ """Mean intra-family vs inter-family pairwise distance and inter/intra ratio."""
328
+ n = distance_matrix.shape[0]
329
+ iu = np.triu_indices(n, k=1)
330
+ pair_d = distance_matrix[iu]
331
+ same = (labels[iu[0]] == labels[iu[1]])
332
+ intra = float(pair_d[same].mean()) if same.any() else float("nan")
333
+ inter = float(pair_d[~same].mean()) if (~same).any() else float("nan")
334
+ ratio = float(inter / intra) if intra > 0 else float("nan")
335
+ return {"intra_mean": intra, "inter_mean": inter, "inter_over_intra": ratio}
336
+
337
+
338
+ def compute_structure_metrics(
339
+ embedding: np.ndarray,
340
+ labels: np.ndarray,
341
+ knn_k: int,
342
+ ) -> Dict[str, float]:
343
+ metrics: Dict[str, float] = {}
344
+ n_classes = len(np.unique(labels))
345
+ if n_classes < 2 or len(labels) < 3:
346
+ return {"silhouette_cosine": float("nan"),
347
+ "silhouette_euclidean": float("nan"),
348
+ "knn_purity_cosine": float("nan"),
349
+ "knn_purity_euclidean": float("nan"),
350
+ "inter_over_intra_cosine": float("nan"),
351
+ "inter_over_intra_euclidean": float("nan")}
352
+
353
+ metrics["silhouette_cosine"] = float(silhouette_score(embedding, labels, metric="cosine"))
354
+ metrics["silhouette_euclidean"] = float(silhouette_score(embedding, labels, metric="euclidean"))
355
+ metrics["knn_purity_cosine"] = _knn_purity(embedding, labels, knn_k, metric="cosine")
356
+ metrics["knn_purity_euclidean"] = _knn_purity(embedding, labels, knn_k, metric="euclidean")
357
+
358
+ cos_d = cosine_distances(embedding)
359
+ euc_d = euclidean_distances(embedding)
360
+ cos_ratio = _intra_inter_distance_ratio(cos_d, labels)
361
+ euc_ratio = _intra_inter_distance_ratio(euc_d, labels)
362
+ metrics["intra_mean_cosine"] = cos_ratio["intra_mean"]
363
+ metrics["inter_mean_cosine"] = cos_ratio["inter_mean"]
364
+ metrics["inter_over_intra_cosine"] = cos_ratio["inter_over_intra"]
365
+ metrics["intra_mean_euclidean"] = euc_ratio["intra_mean"]
366
+ metrics["inter_mean_euclidean"] = euc_ratio["inter_mean"]
367
+ metrics["inter_over_intra_euclidean"] = euc_ratio["inter_over_intra"]
368
+ return metrics
369
+
370
+
371
+ def permutation_null(
372
+ embedding: np.ndarray,
373
+ labels: np.ndarray,
374
+ knn_k: int,
375
+ n_permutations: int,
376
+ seed: int,
377
+ ) -> Dict[str, Dict[str, float]]:
378
+ """Shuffle labels K times, recompute metrics, return per-metric (mean, std, p_value).
379
+
380
+ p-value: empirical right-tail Pr(null >= observed) under random labeling.
381
+ """
382
+ observed = compute_structure_metrics(embedding, labels, knn_k)
383
+ rng = np.random.default_rng(seed)
384
+ null_records: Dict[str, List[float]] = {k: [] for k in observed}
385
+
386
+ for _ in range(n_permutations):
387
+ permuted = rng.permutation(labels)
388
+ m = compute_structure_metrics(embedding, permuted, knn_k)
389
+ for k, v in m.items():
390
+ null_records[k].append(v)
391
+
392
+ summary: Dict[str, Dict[str, float]] = {}
393
+ for k, obs_v in observed.items():
394
+ arr = np.asarray(null_records[k], dtype=float)
395
+ valid = arr[~np.isnan(arr)]
396
+ if valid.size == 0 or np.isnan(obs_v):
397
+ summary[k] = {"observed": obs_v, "null_mean": float("nan"),
398
+ "null_std": float("nan"), "p_right_tail": float("nan")}
399
+ continue
400
+ p = float((1 + np.sum(valid >= obs_v)) / (valid.size + 1))
401
+ summary[k] = {
402
+ "observed": float(obs_v),
403
+ "null_mean": float(valid.mean()),
404
+ "null_std": float(valid.std()),
405
+ "p_right_tail": p,
406
+ }
407
+ return summary
408
+
409
+
410
+ # ---------- visualization ----------
411
+
412
+ def _color_map(labels: List[str]) -> Dict[str, tuple]:
413
+ uniq = sorted(set(labels))
414
+ cmap = plt.get_cmap("tab20", max(len(uniq), 1))
415
+ return {lab: cmap(i) for i, lab in enumerate(uniq)}
416
+
417
+
418
+ def _scatter(coords: np.ndarray, labels: List[str], title: str, out_path: Path) -> None:
419
+ fig, ax = plt.subplots(figsize=(7, 6))
420
+ cmap = _color_map(labels)
421
+ for lab in sorted(set(labels)):
422
+ mask = np.asarray([lab == l for l in labels])
423
+ ax.scatter(coords[mask, 0], coords[mask, 1],
424
+ color=cmap[lab], label=f"{lab} (n={mask.sum()})",
425
+ s=20, alpha=0.85, edgecolor="none")
426
+ ax.set_title(title)
427
+ ax.set_xlabel("dim 1")
428
+ ax.set_ylabel("dim 2")
429
+ ax.legend(loc="best", fontsize=8, frameon=True)
430
+ fig.tight_layout()
431
+ fig.savefig(out_path, dpi=160)
432
+ plt.close(fig)
433
+
434
+
435
+ def visualize(
436
+ embedding: np.ndarray,
437
+ metadata_df: pd.DataFrame,
438
+ output_dir: Path,
439
+ umap_n_neighbors: int,
440
+ umap_min_dist: float,
441
+ ) -> None:
442
+ # PCA 2D
443
+ pca = PCA(n_components=2, random_state=0)
444
+ pca_coords = pca.fit_transform(embedding)
445
+ _scatter(pca_coords, metadata_df["family"].tolist(),
446
+ "Module embedding PCA — family", output_dir / "pca_family.png")
447
+
448
+ has_hallmark = metadata_df["hallmark_process"].notna().any()
449
+ if has_hallmark:
450
+ # Project only hallmark modules to a separate PCA for readability
451
+ hm_mask = metadata_df["hallmark_process"].notna().to_numpy()
452
+ if hm_mask.sum() >= 3:
453
+ hm_pca = PCA(n_components=2, random_state=0).fit_transform(embedding[hm_mask])
454
+ _scatter(hm_pca, metadata_df.loc[hm_mask, "hallmark_process"].tolist(),
455
+ "Hallmark modules PCA — process",
456
+ output_dir / "pca_hallmark_process.png")
457
+
458
+ # UMAP (optional)
459
+ try:
460
+ import umap # type: ignore
461
+ reducer = umap.UMAP(
462
+ n_components=2,
463
+ n_neighbors=min(umap_n_neighbors, max(2, embedding.shape[0] - 1)),
464
+ min_dist=umap_min_dist,
465
+ metric="cosine",
466
+ random_state=0,
467
+ )
468
+ umap_coords = reducer.fit_transform(embedding)
469
+ _scatter(umap_coords, metadata_df["family"].tolist(),
470
+ "Module embedding UMAP — family", output_dir / "umap_family.png")
471
+
472
+ if has_hallmark and hm_mask.sum() >= 3:
473
+ hm_reducer = umap.UMAP(
474
+ n_components=2,
475
+ n_neighbors=min(umap_n_neighbors, max(2, hm_mask.sum() - 1)),
476
+ min_dist=umap_min_dist,
477
+ metric="cosine",
478
+ random_state=0,
479
+ )
480
+ hm_umap = hm_reducer.fit_transform(embedding[hm_mask])
481
+ _scatter(hm_umap, metadata_df.loc[hm_mask, "hallmark_process"].tolist(),
482
+ "Hallmark modules UMAP — process",
483
+ output_dir / "umap_hallmark_process.png")
484
+ except ImportError:
485
+ logger.warning("umap-learn not installed; skipping UMAP plots. `pip install umap-learn` to enable.")
486
+
487
+
488
+ # ---------- driver ----------
489
+
490
+ def setup_logging(output_dir: Path) -> None:
491
+ log_path = output_dir / "run.log"
492
+ logging.basicConfig(
493
+ level=logging.INFO,
494
+ format="%(asctime)s | %(levelname)s | %(name)s | %(message)s",
495
+ handlers=[logging.StreamHandler(sys.stdout), logging.FileHandler(log_path, mode="w")],
496
+ force=True,
497
+ )
498
+
499
+
500
+ def resolve_module_json(args: argparse.Namespace) -> str:
501
+ """Resolve relative paths against performer_module/cell_annotation/ to stay
502
+ consistent with how `load_backbone` reads `module_data_dir`.
503
+ """
504
+ if args.module_json is not None:
505
+ return str(ca_resolve_path(args.module_json))
506
+ return str(ca_resolve_path(args.module_data_dir) / "merged_modules.json")
507
+
508
+
509
+ def main() -> None:
510
+ args = parse_args()
511
+
512
+ # DDP init (auto-detects torchrun env; falls back to single-process)
513
+ device, use_ddp, rank, world_size, local_rank = init_distributed_mode(args.device)
514
+ seed_all(args.seed + rank)
515
+
516
+ if is_main_process():
517
+ output_dir = ensure_dir(args.output_dir)
518
+ setup_logging(output_dir)
519
+ else:
520
+ output_dir = Path(args.output_dir)
521
+ logging.basicConfig(level=logging.WARNING)
522
+ if use_ddp:
523
+ dist.barrier()
524
+
525
+ logger.info("Args: %s", vars(args))
526
+ logger.info("DDP: use_ddp=%s, rank=%d/%d, device=%s", use_ddp, rank, world_size, device)
527
+
528
+ # 1. Load backbone
529
+ backbone, module_data, model_config = load_backbone(args, device)
530
+ backbone.eval()
531
+ base_dim = int(model_config["base_dim"])
532
+ num_modules = len(module_data)
533
+ logger.info("Loaded backbone: num_modules=%d, base_dim=%d", num_modules, base_dim)
534
+
535
+ # 2. Load module names (must align with module_data order)
536
+ module_json_path = resolve_module_json(args)
537
+ module_names = load_module_names(module_json_path)
538
+ if len(module_names) != num_modules:
539
+ raise RuntimeError(
540
+ f"Module name count ({len(module_names)} from {module_json_path}) does not match "
541
+ f"backbone num_modules ({num_modules}). The merged_modules.json used to build the "
542
+ "model and the one given here must be identical, with no modules dropped during "
543
+ "gene-symbol resolution."
544
+ )
545
+ metadata = build_module_metadata(module_names)
546
+ metadata_df = pd.DataFrame(metadata)
547
+ if is_main_process():
548
+ logger.info("Family distribution:\n%s", metadata_df["family"].value_counts().to_string())
549
+
550
+ # 3. Build dataloader
551
+ if args.data_mode == "h5ad":
552
+ preprocessor = DataPreprocessor(
553
+ gene2ens_file=args.gene2ens_file,
554
+ gene2ens_column=args.gene2ens_column,
555
+ model_genes_file=args.gene_list_path,
556
+ )
557
+ preprocessor.load_gene_mapping()
558
+ preprocessor.load_model_genes()
559
+ expected_input_genes = -1
560
+ else:
561
+ preprocessor = None
562
+ expected_input_genes = int(model_config["num_genes"])
563
+
564
+ loader, reindex_state, n_cells = build_inference_loader(
565
+ args, preprocessor, use_ddp, rank, world_size,
566
+ )
567
+ if n_cells == 0:
568
+ raise RuntimeError("No cells survived preprocessing.")
569
+
570
+ # 4. Inference
571
+ logger.info("Extracting cell-averaged module embeddings over %d cells...", n_cells)
572
+ mean_emb, processed = extract_cell_averaged_module_embeddings(
573
+ backbone, loader, preprocessor, reindex_state, device,
574
+ num_modules=num_modules, base_dim=base_dim,
575
+ expected_input_genes=expected_input_genes,
576
+ use_ddp=use_ddp,
577
+ )
578
+ logger.info("Embedding ready: shape=%s, processed_cells=%d", mean_emb.shape, processed)
579
+
580
+ # --- Everything below is rank-0 only ---
581
+ if is_main_process():
582
+ # 5. Save raw outputs
583
+ np.save(output_dir / "module_embeddings.npy", mean_emb)
584
+ metadata_df.to_csv(output_dir / "module_metadata.csv", index=False)
585
+
586
+ # 6. Metrics — family level
587
+ if args.include_other_in_metrics:
588
+ keep_mask = np.ones(len(metadata_df), dtype=bool)
589
+ else:
590
+ keep_mask = (metadata_df["family"] != "Other").to_numpy()
591
+ family_labels = metadata_df.loc[keep_mask, "family"].tolist()
592
+ family_emb = mean_emb[keep_mask]
593
+
594
+ logger.info("Computing structure metrics — family level (n=%d)", keep_mask.sum())
595
+ family_summary = permutation_null(
596
+ family_emb, _label_to_int(family_labels),
597
+ knn_k=args.knn_k,
598
+ n_permutations=args.null_permutations,
599
+ seed=args.seed,
600
+ )
601
+
602
+ # 6b. Metrics — hallmark process level
603
+ hm_mask = metadata_df["hallmark_process"].notna().to_numpy()
604
+ hallmark_summary: Optional[Dict[str, Dict[str, float]]] = None
605
+ if hm_mask.sum() >= 3 and metadata_df.loc[hm_mask, "hallmark_process"].nunique() >= 2:
606
+ logger.info("Computing structure metrics — hallmark process level (n=%d)", hm_mask.sum())
607
+ hallmark_summary = permutation_null(
608
+ mean_emb[hm_mask],
609
+ _label_to_int(metadata_df.loc[hm_mask, "hallmark_process"].tolist()),
610
+ knn_k=args.knn_k,
611
+ n_permutations=args.null_permutations,
612
+ seed=args.seed,
613
+ )
614
+
615
+ metrics_payload = {
616
+ "n_cells_processed": int(processed),
617
+ "num_modules": int(num_modules),
618
+ "base_dim": int(base_dim),
619
+ "knn_k": int(args.knn_k),
620
+ "null_permutations": int(args.null_permutations),
621
+ "family_metrics": family_summary,
622
+ "hallmark_process_metrics": hallmark_summary,
623
+ "family_counts": metadata_df["family"].value_counts().to_dict(),
624
+ "hallmark_process_counts": metadata_df["hallmark_process"].dropna().value_counts().to_dict(),
625
+ "shuffle_module_genes": bool(args.shuffle_module_genes),
626
+ }
627
+ with open(output_dir / "metrics.json", "w", encoding="utf-8") as fh:
628
+ json.dump(metrics_payload, fh, indent=2)
629
+
630
+ logger.info("=== Family-level structure (vs random-shuffled label null) ===")
631
+ for k, v in family_summary.items():
632
+ logger.info(" %-30s observed=%.4f null=%.4f±%.4f p=%.4f",
633
+ k, v["observed"], v["null_mean"], v["null_std"], v["p_right_tail"])
634
+ if hallmark_summary is not None:
635
+ logger.info("=== Hallmark-process-level structure (Hallmarks subset) ===")
636
+ for k, v in hallmark_summary.items():
637
+ logger.info(" %-30s observed=%.4f null=%.4f±%.4f p=%.4f",
638
+ k, v["observed"], v["null_mean"], v["null_std"], v["p_right_tail"])
639
+
640
+ # 7. Visualisation
641
+ logger.info("Plotting UMAP / PCA...")
642
+ visualize(mean_emb, metadata_df, output_dir,
643
+ umap_n_neighbors=args.umap_n_neighbors,
644
+ umap_min_dist=args.umap_min_dist)
645
+
646
+ logger.info("Done. Outputs saved to %s", output_dir)
647
+
648
+ # Teardown
649
+ if use_ddp:
650
+ dist.barrier()
651
+ dist.destroy_process_group()
652
+
653
+
654
+ if __name__ == "__main__":
655
+ main()
BioMod/interpretability/module_importance/README.md CHANGED
@@ -1,311 +1,319 @@
1
- # Experiment 3: Module Importance and Enrichment Analysis
2
-
3
- This experiment tests whether modules that a finetuned model considers
4
- important are consistent with known cell-type marker genes.
5
-
6
- ## Method
7
-
8
- 1. Load a finetuned cell annotation model: backbone, attention-pooling head, and classifier.
9
- 2. Mask one module at a time on the test set and measure the drop in the correct-class logit (`DeltaLogit`).
10
- 3. Aggregate by cell type to obtain an importance matrix with shape `(num_classes x 660)`.
11
- 4. For the top-k modules of each cell type, run hypergeometric enrichment on their genes.
12
-
13
- ## Masking Modes
14
-
15
- | Mode | Principle | Speed |
16
- | --- | --- | --- |
17
- | `post_encoder` | Zero out the module token after encoder output | Fast, because the encoder runs once |
18
- | `pre_encoder` | Zero out the genes corresponding to the module before encoder input | Slow, because it requires 660 x cells forward passes |
19
-
20
- `pre_encoder` is more accurate because it captures interactions between modules,
21
- but it is much slower. Use `--importance_cells` for subsampling control.
22
-
23
- ## Run
24
-
25
- ```bash
26
- cd ../data/250010166/scLong/performer_module/interpretability/module_importance
27
- torchrun --nproc_per_node=4
28
- python module_importance.py \
29
- --finetune_checkpoint mae_tost_48M_cross_attn_only_pure_memory/attn_pool_full_n1_lrh0.0001_lrb0.0001_seed2021/best_model.pt \
30
- --config_pkl ../../mae_tost_48M_cross_attn_only_pure_memory/mae_tost_48M_cross_attn_only_pure_memory.pkl \
31
- --module_data_dir ../../module_data_all \
32
- --gene_list_path ../../../selected_genes_27k.txt \
33
- --gene_mapping_path ../../../gene2ens.csv \
34
- --gene2vec_data_path ../../../selected_gene2vec_27k.npy \
35
- --gene2ens_file ../../../gene2ens.csv \
36
- --data_path ../../../cell_annotation/Zheng68K.h5ad \
37
- --label_column celltype \
38
- --mask_mode post_encoder \
39
- --batch_size 128 \
40
- --top_k 10 20 \
41
- --output_dir ../results/importance/zheng68k_post
42
- ```
43
-
44
- Pre-encoder mode, more accurate but slower:
45
-
46
- ```bash
47
- python module_importance.py \
48
- --finetune_checkpoint mae_tost_48M_cross_attn_only_pure_memory/attn_pool_full_n1_lrh0.0001_lrb0.0001_seed2021/best_model.pt \
49
- --config_pkl ../../mae_tost_48M_cross_attn_only_pure_memory/mae_tost_48M_cross_attn_only_pure_memory.pkl \
50
- --module_data_dir ../../module_data_all \
51
- --gene_list_path ../../../selected_genes_27k.txt \
52
- --gene_mapping_path ../../../gene2ens.csv \
53
- --gene2vec_data_path ../../../selected_gene2vec_27k.npy \
54
- --gene2ens_file ../../../gene2ens.csv \
55
- --data_path ../../../cell_annotation/Zheng68K.h5ad \
56
- --label_column celltype \
57
- --mask_mode pre_encoder \
58
- --importance_cells 500 \
59
- --batch_size 256 \
60
- --top_k 10 20 \
61
- --output_dir ../results/importance/zheng68k_pre
62
- ```
63
-
64
- Pre-encoder mode on all cells with DDP multi-GPU acceleration:
65
-
66
- ```bash
67
- torchrun --nproc_per_node=4 module_importance.py \
68
- --finetune_checkpoint mae_tost_48M_cross_attn_only_pure_memory/attn_pool_full_n1_lrh0.0001_lrb0.0001_seed2021/best_model.pt \
69
- --config_pkl ../../mae_tost_48M_cross_attn_only_pure_memory/mae_tost_48M_cross_attn_only_pure_memory.pkl \
70
- --module_data_dir ../../module_data_all \
71
- --gene_list_path ../../../selected_genes_27k.txt \
72
- --gene_mapping_path ../../../gene2ens.csv \
73
- --gene2vec_data_path ../../../selected_gene2vec_27k.npy \
74
- --gene2ens_file ../../../gene2ens.csv \
75
- --data_path ../../../cell_annotation/Zheng68K.h5ad \
76
- --label_column celltype \
77
- --mask_mode pre_encoder \
78
- --batch_size 512 \
79
- --top_k 10 20 \
80
- --output_dir ../results/importance/zheng68k_pre_full
81
- ```
82
-
83
- ## Arguments
84
-
85
- | Argument | Default | Description |
86
- | --- | --- | --- |
87
- | `--mask_mode` | post_encoder | Masking strategy |
88
- | `--importance_cells` | 0 | Maximum number of cells for `pre_encoder`; 0 means all cells |
89
- | `--top_k` | 10 20 | Top-k modules used for enrichment analysis |
90
- | `--batch_size` | 64 | Inference batch size |
91
- | `--test_ratio` | 0.2 | Test-set fraction; should match finetuning |
92
- | `--seed` | 2021 | Random seed; should match finetuning |
93
- | `--marker_db_dir` | None | Marker database cache directory |
94
-
95
- ## Outputs
96
-
97
- ```text
98
- output_dir/
99
- |-- importance_matrix.npy # (11, 660) DeltaLogit matrix
100
- |-- importance_matrix.csv # Same matrix with labels
101
- |-- top_modules_per_celltype.json # Top modules and genes for each cell type
102
- |-- enrichment_results.csv # Hypergeometric enrichment results
103
- |-- heatmap_importance.png # Heatmap
104
- |-- top_modules_bar.png # Bar plot of top modules per cell type
105
- |-- metrics.json # Summary metrics
106
- `-- marker_cache/ # PanglaoDB cache
107
- `-- PanglaoDB_markers.tsv
108
- ```
109
-
110
- ## Expected Conclusion
111
-
112
- If the model learns meaningful module importance:
113
-
114
- - Top modules for immune cell types should be enriched for immune-related pathways.
115
- - Enrichment p-values should be significant after Bonferroni correction, below 0.05.
116
- - Marker genes for the corresponding cell types in PanglaoDB should appear in top modules.
117
-
118
- ---
119
-
120
- ## Extended Experiments
121
-
122
- ### 3-A: Masking Ablation Curves
123
-
124
- Progressively mask top, bottom, or random modules and measure the accuracy drop.
125
-
126
- ```bash
127
- python masking_ablation.py \
128
- --finetune_checkpoint mae_tost_48M_cross_attn_only_pure_memory/attn_pool_full_n1_lrh0.0001_lrb0.0001_seed2021/best_model.pt \
129
- --config_pkl ../../mae_tost_48M_cross_attn_only_pure_memory/mae_tost_48M_cross_attn_only_pure_memory.pkl \
130
- --module_data_dir ../../module_data_all \
131
- --gene_list_path ../../../selected_genes_27k.txt \
132
- --gene_mapping_path ../../../gene2ens.csv \
133
- --gene2vec_data_path ../../../selected_gene2vec_27k.npy \
134
- --gene2ens_file ../../../gene2ens.csv \
135
- --data_path ../../../cell_annotation/Zheng68K.h5ad \
136
- --label_column celltype \
137
- --importance_matrix ../results/importance/zheng68k_pre_13690/importance_matrix.npy \
138
- --k_values 1 2 5 10 20 50 100 200 \
139
- --n_random_trials 10 \
140
- --batch_size 360 \
141
- --save_deltas \
142
- --output_dir ../results/importance/ablation/
143
- ```
144
-
145
- ### 3-B: Permutation Tests
146
-
147
- ```bash
148
- # Mode A: shuffle modules, GPU and slow.
149
- python random_baseline.py \
150
- --mode shuffle_modules \
151
- --finetune_checkpoint mae_tost_48M_cross_attn_only_pure_memory/attn_pool_full_n1_lrh0.0001_lrb0.0001_seed2021/best_model.pt \
152
- --config_pkl ../../mae_tost_48M_cross_attn_only_pure_memory/mae_tost_48M_cross_attn_only_pure_memory.pkl \
153
- --module_data_dir ../../module_data_all \
154
- --gene_list_path ../../../selected_genes_27k.txt \
155
- --gene_mapping_path ../../../gene2ens.csv \
156
- --gene2vec_data_path ../../../selected_gene2vec_27k.npy \
157
- --gene2ens_file ../../../gene2ens.csv \
158
- --data_path ../../../cell_annotation/Zheng68K.h5ad \
159
- --label_column celltype \
160
- --importance_matrix ../results/importance/zheng68k_pre_13690/importance_matrix.npy\
161
- --n_permutations 10 \
162
- --output_dir ../results/importance/permutation/
163
-
164
- # Mode B: shuffle labels, CPU and fast, requires per_cell_deltas from ablation.
165
- python random_baseline.py \
166
- --mode shuffle_labels \
167
- --importance_matrix ../results/importance/zheng68k_pre_13690/importance_matrix.npy \
168
- --per_cell_deltas ../results/importance/ablation/per_cell_deltas.npy \
169
- --per_cell_labels ../results/importance/ablation/per_cell_labels.npy \
170
- --n_permutations 1000 \
171
- --output_dir ../results/importance/permutation/
172
- ```
173
-
174
- ### 3-C: Marker Gene Verification, CPU Only
175
-
176
- ```bash
177
- python marker_gene_verification.py \
178
- --importance_matrix ../results/importance/zheng68k_pre_13690/importance_matrix.npy \
179
- --importance_csv <importance_matrix.csv> \
180
- --module_data_dir ../../module_data_all \
181
- --output_dir ../results/importance/marker_verification/
182
- ```
183
-
184
- ### Publication Figures
185
-
186
- ```bash
187
- python module_importance_publication.py \
188
- --results_dir ../results/importance/zheng68k_pre_13690/ \
189
- --ablation_dir ../results/importance/ablation \
190
- --permutation_dir ../results/importance/permutation/ \
191
- --module_data_dir ../../module_data_all \
192
- --output_dir ../results/importance/publication_figures/
193
- ```
194
-
195
- ### 3-E: Family-Level Analysis, CPU Only
196
-
197
- Analyze importance distributions, contribution fractions, and enrichment tests by
198
- module source family: Hallmarks, Reactome, Hotspot, and WGCNA.
199
-
200
- ```bash
201
- python family_importance_analysis.py \
202
- --results_dir ../results/importance/zheng68k_pre \
203
- --output_dir ../results/importance/family_analysis \
204
- --top_k 10 20 50 \
205
- --enrichment_k 50
206
- ```
207
-
208
- Outputs:
209
-
210
- - `fig1_family_importance_distribution.png/pdf`: family-wise importance distribution for each cell type, violin + box.
211
- - `fig2_family_contribution_topk.png/pdf`: family fraction among top-K important modules, stacked bar.
212
- - `fig3_family_enrichment_heatmap.png/pdf`: Fisher exact-test enrichment-ratio heatmap.
213
- - `fig4_family_mean_importance.png/pdf`: family mean-importance comparison, grouped bar.
214
- - `kruskal_wallis_family_test.csv`: Kruskal-Wallis test results.
215
- - `pairwise_mannwhitney_families.csv`: pairwise Mann-Whitney U tests between families.
216
- - `family_enrichment_fisher.csv`: detailed Fisher exact-test results with Bonferroni correction.
217
- - `family_contribution_topk.csv`: numerical family contribution data.
218
- - `family_mean_importance.csv`: family mean-importance matrix.
219
-
220
- ### 3-F: Family-Level Ablation, GPU
221
-
222
- Ablate whole families by removing or keeping one module family and measuring
223
- classification-accuracy changes.
224
-
225
- ```bash
226
- python family_ablation.py \
227
- --finetune_checkpoint mae_tost_48M_cross_attn_only_pure_memory/attn_pool_full_n1_lrh0.0001_lrb0.0001_seed2021/best_model.pt \
228
- --config_pkl ../../mae_tost_48M_cross_attn_only_pure_memory/mae_tost_48M_cross_attn_only_pure_memory.pkl \
229
- --module_data_dir ../../module_data_all \
230
- --gene_list_path ../../../selected_genes_27k.txt \
231
- --gene_mapping_path ../../../gene2ens.csv \
232
- --gene2vec_data_path ../../../selected_gene2vec_27k.npy \
233
- --gene2ens_file ../../../gene2ens.csv \
234
- --data_path ../../../cell_annotation/Zheng68K.h5ad \
235
- --label_column celltype \
236
- --batch_size 256 \
237
- --output_dir ../results/importance/family_analysis_gpu
238
- ```
239
-
240
- Outputs:
241
-
242
- - `family_ablation_results.json`: overall and per-class accuracy under all conditions.
243
- - `fig_family_ablation.png/pdf`: two-panel overall plot for remove-one vs keep-one.
244
- - `fig_family_ablation_per_class.png/pdf`: two-panel heatmap of per-cell-type ablation results.
245
-
246
- ### 3-G: Cross-Class Similarity, CPU Only
247
-
248
- Analyze correlations and similarity between important modules across the 11 cell
249
- classes.
250
-
251
- ```bash
252
- python cross_class_similarity.py \
253
- --results_dir ../results/importance/zheng68k_pre \
254
- --output_dir ../results/importance/publication_figures \
255
- --top_k 50
256
- ```
257
-
258
- ### 3-H: 1B Module Gating + 2B Family Embedding Finetuning Experiment
259
-
260
- Based on module-importance findings, most modules have near-zero importance and
261
- different cell types depend on different families. During finetuning, add two
262
- lightweight modules in a LoRA-style setup: pretrained weights are frozen and new
263
- parameters are trained from zero initialization.
264
-
265
- - **1B, Sparse Module Gate**: `LayerNorm -> Linear(E,1) -> Sigmoid`, learning a 0-1 gate scalar for each module.
266
- - **2B, Family Embedding**: `nn.Parameter(4, E)`, one learnable vector per family, added to `pos_emb`.
267
-
268
- ```bash
269
- # Baseline, no new modules.
270
- python performer_module_finetune.py \
271
- --config_pkl ../../mae_tost_48M_cross_attn_only_pure_memory/mae_tost_48M_cross_attn_only_pure_memory.pkl \
272
- --performer_checkpoint ../../mae_tost_48M_cross_attn_only_pure_memory/mae_tost_48M_cross_attn_only_pure_memory_epoch6.pth \
273
- --module_data_dir ../../module_data_all \
274
- --gene_list_path ../../../selected_genes_27k.txt \
275
- --gene_mapping_path ../../../gene2ens.csv \
276
- --gene2vec_data_path ../../../selected_gene2vec_27k.npy \
277
- --gene2ens_file ../../../gene2ens.csv \
278
- --data_path ../../../cell_annotation/Zheng68K.h5ad \
279
- --label_column celltype \
280
- --head_type attn_pool \
281
- --finetune_mode last_n --unfreeze_last_n 1 \
282
- --lr_head 0.0001 --lr_backbone 0.0001 \
283
- --batch_size 256 --epochs 300 \
284
- --run_name baseline_zheng68k \
285
- --output_dir ../results/family_exp
286
- ```
287
-
288
- The same command can be extended with:
289
-
290
- - `--use_family_emb` to enable family-type embedding, adding `4 x 200 = 800` parameters.
291
- - `--use_module_gate` to enable sparse module gating, adding LayerNorm + Linear, 601 parameters.
292
- - Both flags can be used together.
293
-
294
- The new module parameters use `lr_head`, a higher learning rate, and are
295
- independent of frozen backbone parameters.
296
-
297
- ## File Descriptions
298
-
299
- | File | Description |
300
- | --- | --- |
301
- | `module_importance.py` | Core importance computation with DDP support |
302
- | `masking_ablation.py` | Ablation-curve experiment, GPU |
303
- | `random_baseline.py` | Permutation tests, GPU/CPU |
304
- | `marker_gene_verification.py` | Marker verification, CPU |
305
- | `module_importance_publication.py` | Publication figures |
306
- | `family_importance_analysis.py` | Family-level importance analysis, CPU |
307
- | `family_ablation.py` | Family-level ablation, GPU |
308
- | `cross_class_similarity.py` | Cross-class similarity analysis, CPU |
309
- | `pub_utils.py` | Shared utilities |
310
- | `RESULTS.md` | Result documentation with figure placeholders |
311
- | `TODO.md` | Progress tracker |
 
 
 
 
 
 
 
 
 
1
+ # Experiment 3: Module Importance and Enrichment Analysis
2
+
3
+ This experiment tests whether modules that a finetuned model considers
4
+ important are consistent with known cell-type marker genes.
5
+
6
+ ## Method
7
+
8
+ 1. Load a finetuned cell annotation model: backbone, attention-pooling head, and classifier.
9
+ 2. Mask one module at a time on the test set and measure the drop in the correct-class logit (`DeltaLogit`).
10
+ 3. Aggregate by cell type to obtain an importance matrix with shape `(num_classes x 660)`.
11
+ 4. For the top-k modules of each cell type, run hypergeometric enrichment on their genes.
12
+
13
+ ## Masking Modes
14
+
15
+ | Mode | Principle | Speed |
16
+ | --- | --- | --- |
17
+ | `post_encoder` | Zero out the module token after encoder output | Fast, because the encoder runs once |
18
+ | `pre_encoder` | Zero out the genes corresponding to the module before encoder input | Slow, because it requires 660 x cells forward passes |
19
+
20
+ `pre_encoder` is more accurate because it captures interactions between modules,
21
+ but it is much slower. Use `--importance_cells` for subsampling control.
22
+
23
+ ## Run
24
+
25
+ Run these commands from the BioMod repository root. First train a matching
26
+ cell-annotation model; the example checkpoint path is the output of
27
+ `BioMod/toturials/cell_annotation.ipynb`, not a bundled pretrained weight file.
28
+ Use the matching BioMod configuration, classifier architecture, preprocessing,
29
+ label mapping, and test split for that trained model. Substitute your own
30
+ `--finetune_checkpoint` if its output directory differs. Importance matrices,
31
+ ablation outputs, and permutation results below must be generated by the
32
+ preceding steps; they are not bundled inputs.
33
+
34
+ ```bash
35
+ # Run from the BioMod repository root.
36
+ python BioMod/interpretability/module_importance/module_importance.py \
37
+ --finetune_checkpoint tutorial_runs/cell_annotation_zheng68k_biomod/best_model.pt \
38
+ --config_pkl weight/0810model/mae_tost_70M_75mask_ec2.pkl \
39
+ --module_data_dir model_dependencies/module_data_all \
40
+ --gene_list_path model_dependencies/selected_genes_27k.txt \
41
+ --gene_mapping_path model_dependencies/gene2ens.csv \
42
+ --gene2vec_data_path model_dependencies/selected_gene2vec_27k.npy \
43
+ --gene2ens_file model_dependencies/gene2ens.csv \
44
+ --data_path data/downstream/final_data/Zheng68K.h5ad \
45
+ --label_column celltype \
46
+ --mask_mode post_encoder \
47
+ --batch_size 128 \
48
+ --top_k 10 20 \
49
+ --output_dir outputs/interpretability/importance/zheng68k_post
50
+ ```
51
+
52
+ Pre-encoder mode, more accurate but slower:
53
+
54
+ ```bash
55
+ python BioMod/interpretability/module_importance/module_importance.py \
56
+ --finetune_checkpoint tutorial_runs/cell_annotation_zheng68k_biomod/best_model.pt \
57
+ --config_pkl weight/0810model/mae_tost_70M_75mask_ec2.pkl \
58
+ --module_data_dir model_dependencies/module_data_all \
59
+ --gene_list_path model_dependencies/selected_genes_27k.txt \
60
+ --gene_mapping_path model_dependencies/gene2ens.csv \
61
+ --gene2vec_data_path model_dependencies/selected_gene2vec_27k.npy \
62
+ --gene2ens_file model_dependencies/gene2ens.csv \
63
+ --data_path data/downstream/final_data/Zheng68K.h5ad \
64
+ --label_column celltype \
65
+ --mask_mode pre_encoder \
66
+ --importance_cells 500 \
67
+ --batch_size 256 \
68
+ --top_k 10 20 \
69
+ --output_dir outputs/interpretability/importance/zheng68k_pre
70
+ ```
71
+
72
+ Pre-encoder mode on all cells with DDP multi-GPU acceleration:
73
+
74
+ ```bash
75
+ torchrun --nproc_per_node=4 BioMod/interpretability/module_importance/module_importance.py \
76
+ --finetune_checkpoint tutorial_runs/cell_annotation_zheng68k_biomod/best_model.pt \
77
+ --config_pkl weight/0810model/mae_tost_70M_75mask_ec2.pkl \
78
+ --module_data_dir model_dependencies/module_data_all \
79
+ --gene_list_path model_dependencies/selected_genes_27k.txt \
80
+ --gene_mapping_path model_dependencies/gene2ens.csv \
81
+ --gene2vec_data_path model_dependencies/selected_gene2vec_27k.npy \
82
+ --gene2ens_file model_dependencies/gene2ens.csv \
83
+ --data_path data/downstream/final_data/Zheng68K.h5ad \
84
+ --label_column celltype \
85
+ --mask_mode pre_encoder \
86
+ --batch_size 512 \
87
+ --top_k 10 20 \
88
+ --output_dir outputs/interpretability/importance/zheng68k_pre_full
89
+ ```
90
+
91
+ ## Arguments
92
+
93
+ | Argument | Default | Description |
94
+ | --- | --- | --- |
95
+ | `--mask_mode` | post_encoder | Masking strategy |
96
+ | `--importance_cells` | 0 | Maximum number of cells for `pre_encoder`; 0 means all cells |
97
+ | `--top_k` | 10 20 | Top-k modules used for enrichment analysis |
98
+ | `--batch_size` | 64 | Inference batch size |
99
+ | `--test_ratio` | 0.2 | Test-set fraction; should match finetuning |
100
+ | `--seed` | 2021 | Random seed; should match finetuning |
101
+ | `--marker_db_dir` | None | Marker database cache directory |
102
+
103
+ ## Outputs
104
+
105
+ ```text
106
+ output_dir/
107
+ |-- importance_matrix.npy # (11, 660) DeltaLogit matrix
108
+ |-- importance_matrix.csv # Same matrix with labels
109
+ |-- top_modules_per_celltype.json # Top modules and genes for each cell type
110
+ |-- enrichment_results.csv # Hypergeometric enrichment results
111
+ |-- heatmap_importance.png # Heatmap
112
+ |-- top_modules_bar.png # Bar plot of top modules per cell type
113
+ |-- metrics.json # Summary metrics
114
+ `-- marker_cache/ # PanglaoDB cache
115
+ `-- PanglaoDB_markers.tsv
116
+ ```
117
+
118
+ ## Expected Conclusion
119
+
120
+ If the model learns meaningful module importance:
121
+
122
+ - Top modules for immune cell types should be enriched for immune-related pathways.
123
+ - Enrichment p-values should be significant after Bonferroni correction, below 0.05.
124
+ - Marker genes for the corresponding cell types in PanglaoDB should appear in top modules.
125
+
126
+ ---
127
+
128
+ ## Extended Experiments
129
+
130
+ ### 3-A: Masking Ablation Curves
131
+
132
+ Progressively mask top, bottom, or random modules and measure the accuracy drop.
133
+
134
+ ```bash
135
+ python BioMod/interpretability/module_importance/masking_ablation.py \
136
+ --finetune_checkpoint tutorial_runs/cell_annotation_zheng68k_biomod/best_model.pt \
137
+ --config_pkl weight/0810model/mae_tost_70M_75mask_ec2.pkl \
138
+ --module_data_dir model_dependencies/module_data_all \
139
+ --gene_list_path model_dependencies/selected_genes_27k.txt \
140
+ --gene_mapping_path model_dependencies/gene2ens.csv \
141
+ --gene2vec_data_path model_dependencies/selected_gene2vec_27k.npy \
142
+ --gene2ens_file model_dependencies/gene2ens.csv \
143
+ --data_path data/downstream/final_data/Zheng68K.h5ad \
144
+ --label_column celltype \
145
+ --importance_matrix outputs/interpretability/importance/zheng68k_pre_13690/importance_matrix.npy \
146
+ --k_values 1 2 5 10 20 50 100 200 \
147
+ --n_random_trials 10 \
148
+ --batch_size 360 \
149
+ --save_deltas \
150
+ --output_dir outputs/interpretability/importance/ablation/
151
+ ```
152
+
153
+ ### 3-B: Permutation Tests
154
+
155
+ ```bash
156
+ # Mode A: shuffle modules, GPU and slow.
157
+ python BioMod/interpretability/module_importance/random_baseline.py \
158
+ --mode shuffle_modules \
159
+ --finetune_checkpoint tutorial_runs/cell_annotation_zheng68k_biomod/best_model.pt \
160
+ --config_pkl weight/0810model/mae_tost_70M_75mask_ec2.pkl \
161
+ --module_data_dir model_dependencies/module_data_all \
162
+ --gene_list_path model_dependencies/selected_genes_27k.txt \
163
+ --gene_mapping_path model_dependencies/gene2ens.csv \
164
+ --gene2vec_data_path model_dependencies/selected_gene2vec_27k.npy \
165
+ --gene2ens_file model_dependencies/gene2ens.csv \
166
+ --data_path data/downstream/final_data/Zheng68K.h5ad \
167
+ --label_column celltype \
168
+ --importance_matrix outputs/interpretability/importance/zheng68k_pre_13690/importance_matrix.npy\
169
+ --n_permutations 10 \
170
+ --output_dir outputs/interpretability/importance/permutation/
171
+
172
+ # Mode B: shuffle labels, CPU and fast, requires per_cell_deltas from ablation.
173
+ python BioMod/interpretability/module_importance/random_baseline.py \
174
+ --mode shuffle_labels \
175
+ --importance_matrix outputs/interpretability/importance/zheng68k_pre_13690/importance_matrix.npy \
176
+ --per_cell_deltas outputs/interpretability/importance/ablation/per_cell_deltas.npy \
177
+ --per_cell_labels outputs/interpretability/importance/ablation/per_cell_labels.npy \
178
+ --n_permutations 1000 \
179
+ --output_dir outputs/interpretability/importance/permutation/
180
+ ```
181
+
182
+ ### 3-C: Marker Gene Verification, CPU Only
183
+
184
+ ```bash
185
+ python BioMod/interpretability/module_importance/marker_gene_verification.py \
186
+ --importance_matrix outputs/interpretability/importance/zheng68k_pre_13690/importance_matrix.npy \
187
+ --importance_csv <importance_matrix.csv> \
188
+ --module_data_dir model_dependencies/module_data_all \
189
+ --output_dir outputs/interpretability/importance/marker_verification/
190
+ ```
191
+
192
+ ### Publication Figures
193
+
194
+ ```bash
195
+ python BioMod/interpretability/module_importance/module_importance_publication.py \
196
+ --results_dir outputs/interpretability/importance/zheng68k_pre_13690/ \
197
+ --ablation_dir outputs/interpretability/importance/ablation \
198
+ --permutation_dir outputs/interpretability/importance/permutation/ \
199
+ --module_data_dir model_dependencies/module_data_all \
200
+ --output_dir outputs/interpretability/importance/publication_figures/
201
+ ```
202
+
203
+ ### 3-E: Family-Level Analysis, CPU Only
204
+
205
+ Analyze importance distributions, contribution fractions, and enrichment tests by
206
+ module source family: Hallmarks, Reactome, Hotspot, and WGCNA.
207
+
208
+ ```bash
209
+ python BioMod/interpretability/module_importance/family_importance_analysis.py \
210
+ --results_dir outputs/interpretability/importance/zheng68k_pre \
211
+ --output_dir outputs/interpretability/importance/family_analysis \
212
+ --top_k 10 20 50 \
213
+ --enrichment_k 50
214
+ ```
215
+
216
+ Outputs:
217
+
218
+ - `fig1_family_importance_distribution.png/pdf`: family-wise importance distribution for each cell type, violin + box.
219
+ - `fig2_family_contribution_topk.png/pdf`: family fraction among top-K important modules, stacked bar.
220
+ - `fig3_family_enrichment_heatmap.png/pdf`: Fisher exact-test enrichment-ratio heatmap.
221
+ - `fig4_family_mean_importance.png/pdf`: family mean-importance comparison, grouped bar.
222
+ - `kruskal_wallis_family_test.csv`: Kruskal-Wallis test results.
223
+ - `pairwise_mannwhitney_families.csv`: pairwise Mann-Whitney U tests between families.
224
+ - `family_enrichment_fisher.csv`: detailed Fisher exact-test results with Bonferroni correction.
225
+ - `family_contribution_topk.csv`: numerical family contribution data.
226
+ - `family_mean_importance.csv`: family mean-importance matrix.
227
+
228
+ ### 3-F: Family-Level Ablation, GPU
229
+
230
+ Ablate whole families by removing or keeping one module family and measuring
231
+ classification-accuracy changes.
232
+
233
+ ```bash
234
+ python BioMod/interpretability/module_importance/family_ablation.py \
235
+ --finetune_checkpoint tutorial_runs/cell_annotation_zheng68k_biomod/best_model.pt \
236
+ --config_pkl weight/0810model/mae_tost_70M_75mask_ec2.pkl \
237
+ --module_data_dir model_dependencies/module_data_all \
238
+ --gene_list_path model_dependencies/selected_genes_27k.txt \
239
+ --gene_mapping_path model_dependencies/gene2ens.csv \
240
+ --gene2vec_data_path model_dependencies/selected_gene2vec_27k.npy \
241
+ --gene2ens_file model_dependencies/gene2ens.csv \
242
+ --data_path data/downstream/final_data/Zheng68K.h5ad \
243
+ --label_column celltype \
244
+ --batch_size 256 \
245
+ --output_dir outputs/interpretability/importance/family_analysis_gpu
246
+ ```
247
+
248
+ Outputs:
249
+
250
+ - `family_ablation_results.json`: overall and per-class accuracy under all conditions.
251
+ - `fig_family_ablation.png/pdf`: two-panel overall plot for remove-one vs keep-one.
252
+ - `fig_family_ablation_per_class.png/pdf`: two-panel heatmap of per-cell-type ablation results.
253
+
254
+ ### 3-G: Cross-Class Similarity, CPU Only
255
+
256
+ Analyze correlations and similarity between important modules across the 11 cell
257
+ classes.
258
+
259
+ ```bash
260
+ python BioMod/interpretability/module_importance/cross_class_similarity.py \
261
+ --results_dir outputs/interpretability/importance/zheng68k_pre \
262
+ --output_dir outputs/interpretability/importance/publication_figures \
263
+ --top_k 50
264
+ ```
265
+
266
+ ### 3-H: 1B Module Gating + 2B Family Embedding Finetuning Experiment
267
+
268
+ Based on module-importance findings, most modules have near-zero importance and
269
+ different cell types depend on different families. During finetuning, add two
270
+ lightweight modules in a LoRA-style setup: pretrained weights are frozen and new
271
+ parameters are trained from zero initialization.
272
+
273
+ - **1B, Sparse Module Gate**: `LayerNorm -> Linear(E,1) -> Sigmoid`, learning a 0-1 gate scalar for each module.
274
+ - **2B, Family Embedding**: `nn.Parameter(4, E)`, one learnable vector per family, added to `pos_emb`.
275
+
276
+ ```bash
277
+ # Baseline, no new modules.
278
+ python BioMod/cell_annotation/performer_module_finetune.py \
279
+ --config_pkl weight/0810model/mae_tost_70M_75mask_ec2.pkl \
280
+ --performer_checkpoint weight/0810model/mae_tost_70M_75mask_ec2_epoch0.pth \
281
+ --module_data_dir model_dependencies/module_data_all \
282
+ --gene_list_path model_dependencies/selected_genes_27k.txt \
283
+ --gene_mapping_path model_dependencies/gene2ens.csv \
284
+ --gene2vec_data_path model_dependencies/selected_gene2vec_27k.npy \
285
+ --gene2ens_file model_dependencies/gene2ens.csv \
286
+ --data_path data/downstream/final_data/Zheng68K.h5ad \
287
+ --label_column celltype \
288
+ --head_type attn_pool \
289
+ --finetune_mode last_n --unfreeze_last_n 1 \
290
+ --lr_head 0.0001 --lr_backbone 0.0001 \
291
+ --batch_size 256 --epochs 300 \
292
+ --run_name baseline_zheng68k \
293
+ --output_dir outputs/interpretability/family_exp
294
+ ```
295
+
296
+ The same command can be extended with:
297
+
298
+ - `--use_family_emb` to enable family-type embedding, adding `4 x 200 = 800` parameters.
299
+ - `--use_module_gate` to enable sparse module gating, adding LayerNorm + Linear, 601 parameters.
300
+ - Both flags can be used together.
301
+
302
+ The new module parameters use `lr_head`, a higher learning rate, and are
303
+ independent of frozen backbone parameters.
304
+
305
+ ## File Descriptions
306
+
307
+ | File | Description |
308
+ | --- | --- |
309
+ | `module_importance.py` | Core importance computation with DDP support |
310
+ | `masking_ablation.py` | Ablation-curve experiment, GPU |
311
+ | `random_baseline.py` | Permutation tests, GPU/CPU |
312
+ | `marker_gene_verification.py` | Marker verification, CPU |
313
+ | `module_importance_publication.py` | Publication figures |
314
+ | `family_importance_analysis.py` | Family-level importance analysis, CPU |
315
+ | `family_ablation.py` | Family-level ablation, GPU |
316
+ | `cross_class_similarity.py` | Cross-class similarity analysis, CPU |
317
+ | `pub_utils.py` | Shared utilities |
318
+ | `RESULTS.md` | Result documentation with figure placeholders |
319
+ | `TODO.md` | Progress tracker |
BioMod/interpretability/module_importance/RESULTS.md CHANGED
@@ -1,221 +1,226 @@
1
- # Module Importance: Result Documentation
2
-
3
- > For the manuscript subsection "Model Interpretability: Module Importance Analysis".
4
-
5
- ## Dataset
6
-
7
- - **Zheng68K**: 11 cell types, 68,450 cells, with 13,690 test cells.
8
- - Model: PerformerModuleCellClassifier with attention-pooling head and 48M backbone.
9
- - Baseline test accuracy: **83.1%**.
10
-
11
- ## Data Sources
12
-
13
- All analyses are based on the following core files, generated by `module_importance.py`
14
- with `pre_encoder` over all 13,690 test cells:
15
-
16
- | File | Path | Description |
17
- | --- | --- | --- |
18
- | `importance_matrix.npy` | `results/importance/zheng68k_pre_13690/importance_matrix.npy` | `(11 x 660)` DeltaLogit matrix |
19
- | `importance_matrix.csv` | `results/importance/zheng68k_pre_13690/importance_matrix.csv` | Same matrix with row and column labels |
20
- | `enrichment_results.csv` | `results/importance/zheng68k_pre_13690/enrichment_results.csv` | Enrichment-test results |
21
- | `top_modules_per_celltype.json` | `results/importance/zheng68k_pre_13690/top_modules_per_celltype.json` | Top-module details |
22
- | `merged_modules.json` | `module_data_all/merged_modules.json` | Gene composition of 660 modules |
23
-
24
- Marker verification uses `importance_matrix.npy`, `importance_matrix.csv`, and
25
- `merged_modules.json`.
26
-
27
- Publication figures use all files above plus ablation and permutation results.
28
-
29
- ---
30
-
31
- ## Experiment 3 Core: Module Importance Matrix
32
-
33
- ### Method
34
-
35
- For each test cell, mask one module at a time using `pre_encoder`, which zeros out
36
- the corresponding gene inputs, and measure the drop in the correct-class logit
37
- as the importance score. Aggregate by cell type to obtain an `(11 x 660)`
38
- importance matrix.
39
-
40
- ### Results
41
-
42
- - Masking mode: **pre_encoder** over all 13,690 cells.
43
- - 660 modules: 491 Reactome, 50 Hallmarks, 73 Hotspot, and 46 Processed.
44
-
45
- **Representative top modules per cell type:**
46
-
47
- | Cell Type | #1 Module | #2 Module | #3 Module |
48
- | --- | --- | --- | --- |
49
- | CD14+ Monocyte | Neutrophil Degranulation | TLR4 Cascade | MHC-II Antigen Presentation |
50
- | CD19+ B | MHC-II | IFN-gamma Signaling | CD28 Co-stimulation |
51
- | CD56+ NK | SM_22 (cytotoxic) | module3_2 | Allograft Rejection |
52
- | CD8+ Cytotoxic T | module3_2 | SM_22 | Allograft Rejection |
53
- | Dendritic | MHC-II | Signaling by Interleukins | IFN-gamma |
54
-
55
- ### Figure
56
-
57
- ![Importance Heatmap (clustered)](publication_figures/fig4_importance_heatmap_clustered.png)
58
-
59
- ---
60
-
61
- ## Experiment 3-A: Masking Ablation Curves
62
-
63
- ### Method
64
-
65
- Progressively mask top-k, bottom-k, or random-k modules, with
66
- `k = 1, 2, 5, 10, 20, 50, 100, 200`, and measure classification accuracy. This
67
- uses `post_encoder` masking so the encoder runs only once.
68
-
69
- ### Expected Result
70
-
71
- - Masking top modules should rapidly decrease accuracy.
72
- - Masking bottom modules should have little effect.
73
- - Random masking should fall between the two.
74
-
75
- ### Results, Completed
76
-
77
- - Baseline accuracy: **83.0%** on 13,690 test cells.
78
- - Post-encoder masking with k = [1, 2, 5, 10, 20, 50, 100, 200].
79
-
80
- | Masked k | Top Acc | Bottom Acc | Random Acc |
81
- | --- | --- | --- | --- |
82
- | 1 | 83.0% | 83.0% | 83.0% |
83
- | 2 | 82.6% | 83.0% | 83.0% |
84
- | 5 | 82.0% | 83.0% | 83.0% |
85
- | 10 | 77.1% | 83.0% | 83.0% |
86
- | 20 | 73.7% | 83.1% | 82.7% |
87
- | 50 | 69.5% | 83.2% | 82.4% |
88
- | 100 | 65.1% | 82.7% | 81.7% |
89
- | 200 | 49.7% | 81.9% | 78.6% |
90
-
91
- **Interpretation:** masking the top 200 modules, 30% of all modules, reduces
92
- accuracy from 83% to 50%, a 33 percentage-point drop. Masking the bottom 200
93
- modules has almost no effect, with only a 1.1 point drop. Random masking falls
94
- between the two. This shows that the importance ranking captures modules that
95
- are truly important for classification.
96
-
97
- ![Ablation Curves](publication_figures/fig1_ablation_curves.png)
98
-
99
- ![Per-class Ablation](publication_figures/fig_s1_ablation_per_class.png)
100
-
101
- ---
102
-
103
- ## Experiment 3-B: Permutation Tests, Random Baseline
104
-
105
- ### Method
106
-
107
- **Mode A, Module-Shuffle:** shuffle module token order to break the module-gene
108
- correspondence, then recompute importance. Repeat 10 times.
109
-
110
- **Mode B, Label-Shuffle:** shuffle cell type labels and re-aggregate per-cell
111
- deltas. Repeat 1000 times.
112
-
113
- ### Expected Result
114
-
115
- - The true importance maximum should far exceed the null distribution.
116
- - Empirical p-value should be below 0.001.
117
-
118
- ### Results, Completed
119
-
120
- **Mode A, Module-Shuffle** (10 permutations, GPU):
121
-
122
- | Metric | Value |
123
- | --- | --- |
124
- | Global p-value | 0.091, limited by n=10; minimum possible is 1/11 |
125
- | Effect size | **1474.5** |
126
- | Real global max | 2.046 |
127
- | Null global max | 0.452 +/- 0.001 |
128
-
129
- **Mode B, Label-Shuffle** (1000 permutations, CPU):
130
-
131
- | Metric | Value |
132
- | --- | --- |
133
- | Global p-value | **< 0.001** |
134
- | Per-class p-values | All < 0.001 (11/11) |
135
- | Effect size | **48.1** |
136
- | Real global max | 2.046 |
137
- | Null global max | 0.145 +/- 0.039 |
138
-
139
- **Interpretation:**
140
-
141
- - **Module-Shuffle:** after shuffling the module-gene correspondence, the maximum importance drops from 2.046 to about 0.452. The effect size is extremely large, showing that importance depends on the correct module-gene mapping rather than random noise. The p-value is 0.091 because only 10 permutations were run, making the minimum possible p-value about 1/11; in practice, 0 of 10 null runs exceeded the observed value.
142
- - **Label-Shuffle:** after shuffling cell type labels, the maximum importance drops from 2.046 to about 0.145. All 11 cell types have p < 0.001, showing that importance is cell-type-specific rather than a generic pattern shared by all cells.
143
-
144
- ![Permutation Null](publication_figures/fig2_permutation_null.png)
145
-
146
- ---
147
-
148
- ## Experiment 3-C: Canonical Marker Gene Verification
149
-
150
- ### Method
151
-
152
- Check the importance ranks of modules containing known cell-type canonical
153
- markers such as CD14, CD19, CD8A, and NKG7. Test whether markers are enriched in
154
- top modules.
155
-
156
- ### Results, Completed
157
-
158
- | Metric | Value |
159
- | --- | --- |
160
- | Total markers tested | 59 |
161
- | Markers not found | 2 |
162
- | **Median rank percentile** | **0.9%** |
163
- | Markers in top 5% | **47/59 (80%)** |
164
- | Markers in top 10% | 47/59 (80%) |
165
- | Markers in top 20% | 50/59 (85%) |
166
- | Binomial p-value, top 10% | **< 1e-50** |
167
- | Wilcoxon p-value, <50% | **2.5e-11** |
168
-
169
- **Per cell type:**
170
-
171
- | Cell Type | Median Percentile | Best Marker | Best Rank |
172
- | --- | --- | --- | --- |
173
- | CD14+ Monocyte | 0.0% | LYZ | #1 |
174
- | CD19+ B | 1.3% | MS4A1 | #7 |
175
- | CD34+ | 18.5% | THY1 | #7 |
176
- | CD4+ T Helper2 | 1.7% | CD4 | #1 |
177
- | CD4+/CD25 T Reg | 0.9% | CD4 | #1 |
178
- | CD4+/CD45RA+/CD25- Naive T | 10.5% | CD4 | #1 |
179
- | CD4+/CD45RO+ Memory | 0.0% | CD4 | #1 |
180
- | CD56+ NK | 0.0% | NKG7 | #1 |
181
- | CD8+ Cytotoxic T | 0.0% | CD8A | #1 |
182
- | CD8+/CD45RA+ Naive Cytotoxic | 20.3% | CD8A | #1 |
183
- | Dendritic | 2.1% | ITGAX | #1 |
184
-
185
- **Interpretation:** 80% of canonical markers are in the top 5% importance
186
- modules for the corresponding cell type. In 9 of 11 cell types, the best marker
187
- is in the rank #1 module. This strongly shows that the model learns biologically
188
- meaningful module importance.
189
-
190
- ![Marker Ranks](marker_verification/fig_marker_ranks.png)
191
-
192
- ![Marker Summary Bar](marker_verification/fig_marker_summary_bar.png)
193
-
194
- ---
195
-
196
- ## Experiment 3-D: PanglaoDB Enrichment Analysis
197
-
198
- ### Method
199
-
200
- For genes in each cell type's top-k modules, run a hypergeometric enrichment test
201
- against marker gene sets for 178 PanglaoDB cell types, with Bonferroni
202
- correction.
203
-
204
- ### Results
205
-
206
- | Metric | Value |
207
- | --- | --- |
208
- | Significant enrichment hits, p_adj < 0.05 | **1011 / 3916** |
209
- | Hit rate | **25.8%** |
210
-
211
- ![Enrichment Summary](publication_figures/fig5_enrichment_summary.png)
212
-
213
- ---
214
-
215
- ## Manuscript Writing Points
216
-
217
- 1. **Core conclusion:** the model learns biologically meaningful gene organization through the module-based architecture. Top-importance modules strongly match known cell-type markers, with 80% of markers in the top 5%, and masking these modules causes a large classification-performance drop, from 83% to 50%.
218
- 2. **Methodological contribution:** pre-encoder masking gives a more accurate importance estimate than post-encoder masking because it accounts for interactions between modules. DDP support makes all-cell computation feasible.
219
- 3. **Statistical rigor:** label-shuffle p < 0.001 with effect size 48.1 shows that importance is cell-type-specific; module-shuffle effect size 1474 shows that importance depends on the correct module-gene correspondence. Top-k masking rapidly reduces accuracy while bottom-k masking has almost no effect, and the gap widens as k increases.
220
- 4. **Biological interpretation:** monocytes use Neutrophil Degranulation and TLR4, representing innate immune pathways; B cells use MHC-II, IFN-gamma, and CD28, representing antigen presentation and T-B cooperation; NK/CD8 T share cytotoxic modules; dendritic cells use MHC-II and interleukins, matching professional antigen-presenting cells.
221
- 5. **Key numbers for the main text:** 83.0% baseline accuracy; 80% canonical markers in top 5% importance modules; 9 of 11 cell types have the best marker in the rank #1 module; masking top-200 modules drops accuracy by 33 percentage points; label-shuffle permutation p < 0.001 with effect size 48.1; PanglaoDB enrichment hit rate 25.8% (1011/3916 significant).
 
 
 
 
 
 
1
+ # Module Importance: Result Documentation
2
+
3
+ > For the manuscript subsection "Model Interpretability: Module Importance Analysis".
4
+
5
+ ## Dataset
6
+
7
+ - **Zheng68K**: 11 cell types, 68,450 cells, with 13,690 test cells.
8
+ - Model: PerformerModuleCellClassifier with attention-pooling head and 48M backbone.
9
+ - Baseline test accuracy: **83.1%**.
10
+
11
+ ## Data Sources
12
+
13
+ These are historical experiment results. The analysis output files below are not
14
+ included in the repository; recreate them using the commands in `README.md` or
15
+ point the analysis scripts to the original run directory. The shared module JSON
16
+ is bundled under `model_dependencies/module_data_all/` at the repository root.
17
+
18
+ All analyses are based on the following core files, generated by `module_importance.py`
19
+ with `pre_encoder` over all 13,690 test cells:
20
+
21
+ | File | Path | Description |
22
+ | --- | --- | --- |
23
+ | `importance_matrix.npy` | `outputs/interpretability/importance/zheng68k_pre_13690/importance_matrix.npy` | `(11 x 660)` DeltaLogit matrix |
24
+ | `importance_matrix.csv` | `outputs/interpretability/importance/zheng68k_pre_13690/importance_matrix.csv` | Same matrix with row and column labels |
25
+ | `enrichment_results.csv` | `outputs/interpretability/importance/zheng68k_pre_13690/enrichment_results.csv` | Enrichment-test results |
26
+ | `top_modules_per_celltype.json` | `outputs/interpretability/importance/zheng68k_pre_13690/top_modules_per_celltype.json` | Top-module details |
27
+ | `merged_modules.json` | `model_dependencies/module_data_all/merged_modules.json` | Gene composition of 660 modules |
28
+
29
+ Marker verification uses `importance_matrix.npy`, `importance_matrix.csv`, and
30
+ `merged_modules.json`.
31
+
32
+ Publication figures use all files above plus ablation and permutation results.
33
+
34
+ ---
35
+
36
+ ## Experiment 3 Core: Module Importance Matrix
37
+
38
+ ### Method
39
+
40
+ For each test cell, mask one module at a time using `pre_encoder`, which zeros out
41
+ the corresponding gene inputs, and measure the drop in the correct-class logit
42
+ as the importance score. Aggregate by cell type to obtain an `(11 x 660)`
43
+ importance matrix.
44
+
45
+ ### Results
46
+
47
+ - Masking mode: **pre_encoder** over all 13,690 cells.
48
+ - 660 modules: 491 Reactome, 50 Hallmarks, 73 Hotspot, and 46 Processed.
49
+
50
+ **Representative top modules per cell type:**
51
+
52
+ | Cell Type | #1 Module | #2 Module | #3 Module |
53
+ | --- | --- | --- | --- |
54
+ | CD14+ Monocyte | Neutrophil Degranulation | TLR4 Cascade | MHC-II Antigen Presentation |
55
+ | CD19+ B | MHC-II | IFN-gamma Signaling | CD28 Co-stimulation |
56
+ | CD56+ NK | SM_22 (cytotoxic) | module3_2 | Allograft Rejection |
57
+ | CD8+ Cytotoxic T | module3_2 | SM_22 | Allograft Rejection |
58
+ | Dendritic | MHC-II | Signaling by Interleukins | IFN-gamma |
59
+
60
+ ### Figure
61
+
62
+ Expected generated figure: `publication_figures/fig4_importance_heatmap_clustered.png` (Importance Heatmap (clustered); not bundled).
63
+
64
+ ---
65
+
66
+ ## Experiment 3-A: Masking Ablation Curves
67
+
68
+ ### Method
69
+
70
+ Progressively mask top-k, bottom-k, or random-k modules, with
71
+ `k = 1, 2, 5, 10, 20, 50, 100, 200`, and measure classification accuracy. This
72
+ uses `post_encoder` masking so the encoder runs only once.
73
+
74
+ ### Expected Result
75
+
76
+ - Masking top modules should rapidly decrease accuracy.
77
+ - Masking bottom modules should have little effect.
78
+ - Random masking should fall between the two.
79
+
80
+ ### Results, Completed
81
+
82
+ - Baseline accuracy: **83.0%** on 13,690 test cells.
83
+ - Post-encoder masking with k = [1, 2, 5, 10, 20, 50, 100, 200].
84
+
85
+ | Masked k | Top Acc | Bottom Acc | Random Acc |
86
+ | --- | --- | --- | --- |
87
+ | 1 | 83.0% | 83.0% | 83.0% |
88
+ | 2 | 82.6% | 83.0% | 83.0% |
89
+ | 5 | 82.0% | 83.0% | 83.0% |
90
+ | 10 | 77.1% | 83.0% | 83.0% |
91
+ | 20 | 73.7% | 83.1% | 82.7% |
92
+ | 50 | 69.5% | 83.2% | 82.4% |
93
+ | 100 | 65.1% | 82.7% | 81.7% |
94
+ | 200 | 49.7% | 81.9% | 78.6% |
95
+
96
+ **Interpretation:** masking the top 200 modules, 30% of all modules, reduces
97
+ accuracy from 83% to 50%, a 33 percentage-point drop. Masking the bottom 200
98
+ modules has almost no effect, with only a 1.1 point drop. Random masking falls
99
+ between the two. This shows that the importance ranking captures modules that
100
+ are truly important for classification.
101
+
102
+ Expected generated figure: `publication_figures/fig1_ablation_curves.png` (Ablation Curves; not bundled).
103
+
104
+ Expected generated figure: `publication_figures/fig_s1_ablation_per_class.png` (Per-class Ablation; not bundled).
105
+
106
+ ---
107
+
108
+ ## Experiment 3-B: Permutation Tests, Random Baseline
109
+
110
+ ### Method
111
+
112
+ **Mode A, Module-Shuffle:** shuffle module token order to break the module-gene
113
+ correspondence, then recompute importance. Repeat 10 times.
114
+
115
+ **Mode B, Label-Shuffle:** shuffle cell type labels and re-aggregate per-cell
116
+ deltas. Repeat 1000 times.
117
+
118
+ ### Expected Result
119
+
120
+ - The true importance maximum should far exceed the null distribution.
121
+ - Empirical p-value should be below 0.001.
122
+
123
+ ### Results, Completed
124
+
125
+ **Mode A, Module-Shuffle** (10 permutations, GPU):
126
+
127
+ | Metric | Value |
128
+ | --- | --- |
129
+ | Global p-value | 0.091, limited by n=10; minimum possible is 1/11 |
130
+ | Effect size | **1474.5** |
131
+ | Real global max | 2.046 |
132
+ | Null global max | 0.452 +/- 0.001 |
133
+
134
+ **Mode B, Label-Shuffle** (1000 permutations, CPU):
135
+
136
+ | Metric | Value |
137
+ | --- | --- |
138
+ | Global p-value | **< 0.001** |
139
+ | Per-class p-values | All < 0.001 (11/11) |
140
+ | Effect size | **48.1** |
141
+ | Real global max | 2.046 |
142
+ | Null global max | 0.145 +/- 0.039 |
143
+
144
+ **Interpretation:**
145
+
146
+ - **Module-Shuffle:** after shuffling the module-gene correspondence, the maximum importance drops from 2.046 to about 0.452. The effect size is extremely large, showing that importance depends on the correct module-gene mapping rather than random noise. The p-value is 0.091 because only 10 permutations were run, making the minimum possible p-value about 1/11; in practice, 0 of 10 null runs exceeded the observed value.
147
+ - **Label-Shuffle:** after shuffling cell type labels, the maximum importance drops from 2.046 to about 0.145. All 11 cell types have p < 0.001, showing that importance is cell-type-specific rather than a generic pattern shared by all cells.
148
+
149
+ Expected generated figure: `publication_figures/fig2_permutation_null.png` (Permutation Null; not bundled).
150
+
151
+ ---
152
+
153
+ ## Experiment 3-C: Canonical Marker Gene Verification
154
+
155
+ ### Method
156
+
157
+ Check the importance ranks of modules containing known cell-type canonical
158
+ markers such as CD14, CD19, CD8A, and NKG7. Test whether markers are enriched in
159
+ top modules.
160
+
161
+ ### Results, Completed
162
+
163
+ | Metric | Value |
164
+ | --- | --- |
165
+ | Total markers tested | 59 |
166
+ | Markers not found | 2 |
167
+ | **Median rank percentile** | **0.9%** |
168
+ | Markers in top 5% | **47/59 (80%)** |
169
+ | Markers in top 10% | 47/59 (80%) |
170
+ | Markers in top 20% | 50/59 (85%) |
171
+ | Binomial p-value, top 10% | **< 1e-50** |
172
+ | Wilcoxon p-value, <50% | **2.5e-11** |
173
+
174
+ **Per cell type:**
175
+
176
+ | Cell Type | Median Percentile | Best Marker | Best Rank |
177
+ | --- | --- | --- | --- |
178
+ | CD14+ Monocyte | 0.0% | LYZ | #1 |
179
+ | CD19+ B | 1.3% | MS4A1 | #7 |
180
+ | CD34+ | 18.5% | THY1 | #7 |
181
+ | CD4+ T Helper2 | 1.7% | CD4 | #1 |
182
+ | CD4+/CD25 T Reg | 0.9% | CD4 | #1 |
183
+ | CD4+/CD45RA+/CD25- Naive T | 10.5% | CD4 | #1 |
184
+ | CD4+/CD45RO+ Memory | 0.0% | CD4 | #1 |
185
+ | CD56+ NK | 0.0% | NKG7 | #1 |
186
+ | CD8+ Cytotoxic T | 0.0% | CD8A | #1 |
187
+ | CD8+/CD45RA+ Naive Cytotoxic | 20.3% | CD8A | #1 |
188
+ | Dendritic | 2.1% | ITGAX | #1 |
189
+
190
+ **Interpretation:** 80% of canonical markers are in the top 5% importance
191
+ modules for the corresponding cell type. In 9 of 11 cell types, the best marker
192
+ is in the rank #1 module. This strongly shows that the model learns biologically
193
+ meaningful module importance.
194
+
195
+ Expected generated figure: `marker_verification/fig_marker_ranks.png` (Marker Ranks; not bundled).
196
+
197
+ Expected generated figure: `marker_verification/fig_marker_summary_bar.png` (Marker Summary Bar; not bundled).
198
+
199
+ ---
200
+
201
+ ## Experiment 3-D: PanglaoDB Enrichment Analysis
202
+
203
+ ### Method
204
+
205
+ For genes in each cell type's top-k modules, run a hypergeometric enrichment test
206
+ against marker gene sets for 178 PanglaoDB cell types, with Bonferroni
207
+ correction.
208
+
209
+ ### Results
210
+
211
+ | Metric | Value |
212
+ | --- | --- |
213
+ | Significant enrichment hits, p_adj < 0.05 | **1011 / 3916** |
214
+ | Hit rate | **25.8%** |
215
+
216
+ Expected generated figure: `publication_figures/fig5_enrichment_summary.png` (Enrichment Summary; not bundled).
217
+
218
+ ---
219
+
220
+ ## Manuscript Writing Points
221
+
222
+ 1. **Core conclusion:** the model learns biologically meaningful gene organization through the module-based architecture. Top-importance modules strongly match known cell-type markers, with 80% of markers in the top 5%, and masking these modules causes a large classification-performance drop, from 83% to 50%.
223
+ 2. **Methodological contribution:** pre-encoder masking gives a more accurate importance estimate than post-encoder masking because it accounts for interactions between modules. DDP support makes all-cell computation feasible.
224
+ 3. **Statistical rigor:** label-shuffle p < 0.001 with effect size 48.1 shows that importance is cell-type-specific; module-shuffle effect size 1474 shows that importance depends on the correct module-gene correspondence. Top-k masking rapidly reduces accuracy while bottom-k masking has almost no effect, and the gap widens as k increases.
225
+ 4. **Biological interpretation:** monocytes use Neutrophil Degranulation and TLR4, representing innate immune pathways; B cells use MHC-II, IFN-gamma, and CD28, representing antigen presentation and T-B cooperation; NK/CD8 T share cytotoxic modules; dendritic cells use MHC-II and interleukins, matching professional antigen-presenting cells.
226
+ 5. **Key numbers for the main text:** 83.0% baseline accuracy; 80% canonical markers in top 5% importance modules; 9 of 11 cell types have the best marker in the rank #1 module; masking top-200 modules drops accuracy by 33 percentage points; label-shuffle permutation p < 0.001 with effect size 48.1; PanglaoDB enrichment hit rate 25.8% (1011/3916 significant).
BioMod/interpretability/module_text_alignment/DESIGN.md CHANGED
@@ -1,225 +1,235 @@
1
- # Experiment 2: Module Embedding to Pathway Text Alignment
2
-
3
- ## Overview
4
-
5
- This experiment tests whether the structure of the BioMod module embedding space
6
- significantly aligns with the semantic space of natural-language pathway
7
- descriptions. Gene-level models cannot reproduce this experiment because single
8
- gene names do not carry phrase-level semantics.
9
-
10
- ## Scripts
11
-
12
- | Script | Function |
13
- | --- | --- |
14
- | `module_text_alignment.py` | Main script: RSA/Mantel test, CKA, and leave-one-out Ridge retrieval |
15
- | `module_text_alignment_extended.py` | Extended analysis: Reactome broad-category RSA, Reactome description-based RSA, and Hallmark deep dive |
16
- | `module_text_alignment_publication.py` | Publication-level analysis: six main figures, bootstrap CIs, multi-model comparison, cross-dataset validation, and deep analysis |
17
- | `fetch_reactome_descriptions.py` | Fetch `summaryText` for 491 pathways from the Reactome API |
18
-
19
- ## Commands
20
-
21
- ### 1. Download the Sentence Encoder
22
-
23
- ```bash
24
- cd ../data/250010166/scLong/performer_module/interpretability/module_text_alignment
25
- conda activate sclong
26
-
27
- wget https://hf-mirror.com/hfd/hfd.sh && chmod a+x hfd.sh && export HF_ENDPOINT=https://hf-mirror.com
28
- ./hfd.sh pritamdeka/S-PubMedBert-MS-MARCO --local-dir ./S-PubMedBert-MS-MARCO
29
-
30
- pip install sentence-transformers -i https://pypi.tuna.tsinghua.edu.cn/simple
31
- ```
32
-
33
- ### 2. Main Experiment: RSA + CKA + LOO Retrieval
34
-
35
- ```bash
36
- # pretrain_subset is recommended because averaging over 50k cells is more stable.
37
- python module_text_alignment.py \
38
- --module_embedding_path ../results/geometry/pretrain_subset/module_embeddings.npy \
39
- --module_json ../../module_data_all/merged_modules.json \
40
- --sentence_model ./S-PubMedBert-MS-MARCO \
41
- --n_permutations 1000 \
42
- --output_dir ../results/text_alignment/pretrain_subset_v2
43
-
44
- # pancreas_scib validates cross-dataset consistency.
45
- python module_text_alignment.py \
46
- --module_embedding_path ../results/geometry/pancreas_scib/module_embeddings.npy \
47
- --module_json ../../module_data_all/merged_modules.json \
48
- --sentence_model ./S-PubMedBert-MS-MARCO \
49
- --n_permutations 1000 \
50
- --output_dir ../results/text_alignment/pancreas_scib_v2
51
-
52
- # Quick mode: skip LOO retrieval and run only RSA + CKA.
53
- python module_text_alignment.py \
54
- --module_embedding_path ../results/geometry/pretrain_subset/module_embeddings.npy \
55
- --sentence_model ./S-PubMedBert-MS-MARCO \
56
- --skip_loo \
57
- --output_dir ../results/text_alignment/pretrain_subset_quick
58
- ```
59
-
60
- ### 3. Fetch Official Reactome Descriptions (Completed, 486/491)
61
-
62
- ```bash
63
- # Re-fetch if needed. Requires network access to reactome.org and takes about 2 minutes.
64
- python fetch_reactome_descriptions.py \
65
- --module_json ../../module_data_all/merged_modules.json \
66
- --output reactome_descriptions.json --delay 0.05
67
- ```
68
-
69
- Fetched descriptions are stored in `reactome_descriptions.json`.
70
-
71
- ### 4. Extended Analysis: Reactome Categories, Description-Based RSA, Hallmark Deep Dive
72
-
73
- ```bash
74
- python module_text_alignment_extended.py \
75
- --module_embedding_path ../results/geometry/pretrain_subset/module_embeddings.npy \
76
- --module_json ../../module_data_all/merged_modules.json \
77
- --sentence_model ./S-PubMedBert-MS-MARCO \
78
- --descriptions_file ./reactome_descriptions.json \
79
- --n_permutations 1000 \
80
- --output_dir ../results/text_alignment/pretrain_subset_v2_extended
81
- ```
82
-
83
- ### 5. Publication-Level Analysis
84
-
85
- ```bash
86
- # Basic run: one dataset and one model.
87
- python module_text_alignment_publication.py \
88
- --module_embedding_path ../results/geometry/pretrain_subset/module_embeddings.npy \
89
- --module_json ../../module_data_all/merged_modules.json \
90
- --sentence_model ./S-PubMedBert-MS-MARCO \
91
- --descriptions_file ./reactome_descriptions.json \
92
- --n_permutations 1000 \
93
- --n_bootstrap 2000 \
94
- --output_dir ../results/text_alignment/publication
95
-
96
- # Full run: cross-dataset validation and multi-model comparison.
97
- python module_text_alignment_publication.py \
98
- --module_embedding_path ../results/geometry/pretrain_subset/module_embeddings.npy \
99
- --module_embedding_path2 ../results/geometry/pancreas_scib/module_embeddings.npy \
100
- --module_json ../../module_data_all/merged_modules.json \
101
- --sentence_model ./S-PubMedBert-MS-MARCO \
102
- --sentence_models_compare "sentence-transformers/all-mpnet-base-v2,sentence-transformers/all-MiniLM-L6-v2" \
103
- --descriptions_file ./reactome_descriptions.json \
104
- --n_permutations 1000 \
105
- --n_bootstrap 2000 \
106
- --output_dir ../results/text_alignment/publication_full
107
- ```
108
-
109
- **Arguments:**
110
-
111
- | Argument | Description |
112
- | --- | --- |
113
- | `--module_embedding_path` | `module_embeddings.npy` for the main dataset, from Experiment 1 |
114
- | `--module_embedding_path2` | Optional second dataset for cross-dataset validation |
115
- | `--sentence_models_compare` | Comma-separated model names for comparison, downloaded automatically from HuggingFace |
116
- | `--n_permutations` | Mantel-test permutations; 1000 is standard and 10000 gives more precise p-values |
117
- | `--n_bootstrap` | Bootstrap resampling iterations; 2000 is standard for 95% CIs |
118
- | `--device` | Encoding device, `cpu` or `cuda`; default is CPU |
119
-
120
- ## Dependencies
121
-
122
- ```bash
123
- pip install sentence-transformers -i https://pypi.tuna.tsinghua.edu.cn/simple
124
- ```
125
-
126
- ## Output Files
127
-
128
- ### Main Experiment Outputs (`module_text_alignment.py`)
129
-
130
- | File | Content |
131
- | --- | --- |
132
- | `pathway_descriptions.json` | Text descriptions for 541 modules |
133
- | `text_embeddings.npy` | Sentence-encoder embeddings with shape `[541, 768]` |
134
- | `module_embeddings_subset.npy` | Corresponding model module embeddings with shape `[541, 1024]` |
135
- | `alignment_metrics.json` | RSA, CKA, retrieval recall@k, and p-values |
136
- | `rsa_scatter.png` | Paired-distance scatter plot: module space vs text space |
137
- | `rsa_null_distribution.png` | Mantel test: observed value vs null distribution |
138
- | `retrieval_ranks.png` | Histogram of retrieval ranks |
139
- | `retrieval_examples.csv` | Top-5 retrieval results for representative pathways |
140
- | `run.log` | Full log |
141
-
142
- ### Extended Analysis Outputs (`module_text_alignment_extended.py`)
143
-
144
- | File | Content |
145
- | --- | --- |
146
- | `extended_metrics.json` | Summary metrics for the three analyses |
147
- | `reactome_category_heatmaps.png` | Broad Reactome category distance-matrix comparison, module vs text |
148
- | `hallmarks_deep_dive.png` | Hallmark similarity matrices and per-module alignment-score bar chart |
149
- | `hallmarks_per_module_alignment.csv` | Per-module alignment ranking for 50 Hallmarks |
150
- | `run_extended.log` | Extended-analysis log |
151
-
152
- ### Publication-Level Outputs (`module_text_alignment_publication.py`)
153
-
154
- | File | Content |
155
- | --- | --- |
156
- | `fig1_rsa_overview.{pdf,png}` | 2D hexbin density plus Mantel null distribution with bootstrap CI |
157
- | `fig2_hallmarks_heatmaps.{pdf,png}` | Three 50 x 50 similarity matrices: module, text, and difference; hierarchical clustering order and adaptive color scale |
158
- | `fig3_per_hallmark_alignment.{pdf,png}` | Per-Hallmark alignment-score lollipop plot, colored by MSigDB process category |
159
- | `fig4_category_alignment.{pdf,png}` | Box + strip plot grouped by biological process category with Kruskal-Wallis test |
160
- | `fig5_granularity_effect.{pdf,png}` | Bar plot of RSA across granularity levels: fine-grained, category, superclass, Hallmark |
161
- | `fig6_model_comparison.{pdf,png}` | Optional RSA comparison across models and datasets |
162
- | `fig_alignment_vs_size.{pdf,png}` | Alignment score vs module gene count scatter plot |
163
- | `publication_metrics.json` | Summary of all metrics, including bootstrap 95% CIs |
164
- | `hallmarks_alignment_table.csv` | Alignment scores and process categories for 50 Hallmarks |
165
- | `negative_analysis.csv` | Neighbor comparison for the worst- and best-aligned modules |
166
- | `run_publication.log` | Full log |
167
-
168
- ## Results (v2, pretrain_subset)
169
-
170
- ### Main Experiment
171
-
172
- | Metric | All (541) | Hallmarks (50) | Reactome (491) |
173
- | --- | --- | --- | --- |
174
- | RSA Spearman r | 0.050 | **0.160** | -0.007 |
175
- | Mantel p-value | 0.012 | 0.030 | 0.586 |
176
- | z-score | 2.2 sigma | 2.1 sigma | -0.3 sigma |
177
- | CKA | 0.043 (p=0.000) | - | - |
178
- | Retrieval median rank | 232 (random=270) | - | - |
179
-
180
- ### Interpretation
181
-
182
- - **Hallmark alignment is significant (r=0.16, p=0.03):** among 50 broad biological processes, distance structure in model space significantly correlates with semantic distance in text space.
183
- - **Reactome does not align (r=-0.007, p=0.59):** among 491 fine-grained pathways there is no alignment, likely because fine-grained pathway gene-composition similarity, encoded by the model, and pathway-name language similarity, encoded by text, live on different axes.
184
- - **Global CKA is significant (p=0.000):** the overall representation structure contains an alignment signal.
185
-
186
- ### Improvement History
187
-
188
- **v1, BioLinkBERT + 1-to-1 retrieval:**
189
-
190
- - BioLinkBERT was used as a raw BERT model with mean pooling rather than a sentence-level fine-tuned encoder.
191
- - The main metric was exact 1-to-1 retrieval, which is too strict for a 541-dimensional space.
192
- - CKA=0.047 was statistically significant but weak; retrieval was close to random.
193
-
194
- **v2, S-PubMedBert-MS-MARCO + RSA/Mantel test:**
195
-
196
- 1. Switched to a sentence encoder: PubMedBERT fine-tuned on MS-MARCO sentence similarity.
197
- 2. Added RSA/Mantel test as the main metric, testing structural rather than exact retrieval alignment.
198
- 3. Per-family RSA revealed that the alignment signal is concentrated in Hallmarks.
199
-
200
- ### Extended Analysis Directions
201
-
202
- 1. **Reactome broad-category aggregation:** aggregate the 491 pathways into about 16 top-level categories and retest RSA; broad-category alignment is expected to be significant.
203
- 2. **Reactome descriptions:** use API-fetched `summaryText` as richer text input and test whether alignment improves.
204
- 3. **Hallmark deep dive:** compute per-module alignment scores, kNN overlap, and the best/worst aligned Hallmarks.
205
-
206
- ## Design Decisions
207
-
208
- ### Why Use RSA Instead of 1-to-1 Retrieval,
209
-
210
- 1-to-1 retrieval asks whether the paired text can be found exactly from a module
211
- embedding, requiring a very strong cross-modal mapping. RSA asks whether modules
212
- that are close in model space are also close in text space, which is a more
213
- natural definition of structural alignment.
214
-
215
- ### Why Does Reactome Not Align,
216
-
217
- - Reactome pathways overlap heavily because parent and child pathways share many genes.
218
- - The model encodes gene-composition similarity, while text names reflect hierarchy and naming similarity; the two do not perfectly match.
219
- - The granularity is too fine. For example, text distance between `Signaling by FGFR1` and `Signaling by FGFR2` does not necessarily reflect biological distance.
220
-
221
- ### Why Exclude Hotspot/Processed,
222
-
223
- - Hotspot modules (73) are data-driven co-expression modules with only numeric IDs and no pathway semantics.
224
- - Processed modules (46) are pipeline-derived gene groups such as `SM_2` and have no pathway names.
225
- - Without text pairs, they cannot participate in this alignment experiment.
 
 
 
 
 
 
 
 
 
 
 
1
+ # Experiment 2: Module Embedding to Pathway Text Alignment
2
+
3
+ ## Overview
4
+
5
+ This experiment tests whether the structure of the BioMod module embedding space
6
+ significantly aligns with the semantic space of natural-language pathway
7
+ descriptions. Gene-level models cannot reproduce this experiment because single
8
+ gene names do not carry phrase-level semantics.
9
+
10
+ ## Scripts
11
+
12
+ | Script | Function |
13
+ | --- | --- |
14
+ | `module_text_alignment.py` | Main script: RSA/Mantel test, CKA, and leave-one-out Ridge retrieval |
15
+ | `module_text_alignment_extended.py` | Extended analysis: Reactome broad-category RSA, Reactome description-based RSA, and Hallmark deep dive |
16
+ | `module_text_alignment_publication.py` | Publication-level analysis: six main figures, bootstrap CIs, multi-model comparison, cross-dataset validation, and deep analysis |
17
+ | `fetch_reactome_descriptions.py` | Fetch `summaryText` for 491 pathways from the Reactome API |
18
+
19
+ ## Commands
20
+
21
+ Run from the BioMod repository root. Generate each `module_embeddings.npy` with
22
+ `BioMod/interpretability/module_geometry.py` before using it here. The historical
23
+ `pretrain_subset` output requires a separately supplied pretraining dataset;
24
+ the `pancreas_scib` example uses the bundled pancreas h5ad file. Existing result
25
+ tables below describe historical runs; their raw output directories are not
26
+ included in this repository.
27
+
28
+ ### 1. Download the Sentence Encoder
29
+
30
+ ```bash
31
+ # Run from the BioMod repository root.
32
+ # Activate the environment configured for this project.
33
+
34
+ # The sentence-transformers loader downloads this model on first use.
35
+ # Network access or an existing local model cache is required.
36
+
37
+ pip install sentence-transformers -i https://pypi.tuna.tsinghua.edu.cn/simple
38
+ ```
39
+
40
+ ### 2. Main Experiment: RSA + CKA + LOO Retrieval
41
+
42
+ ```bash
43
+ # pretrain_subset is recommended because averaging over 50k cells is more stable.
44
+ python BioMod/interpretability/module_text_alignment/module_text_alignment.py \
45
+ --module_embedding_path outputs/interpretability/geometry/pretrain_subset/module_embeddings.npy \
46
+ --module_json model_dependencies/module_data_all/merged_modules.json \
47
+ --sentence_model pritamdeka/S-PubMedBert-MS-MARCO \
48
+ --n_permutations 1000 \
49
+ --output_dir outputs/interpretability/text_alignment/pretrain_subset_v2
50
+
51
+ # pancreas_scib validates cross-dataset consistency.
52
+ python BioMod/interpretability/module_text_alignment/module_text_alignment.py \
53
+ --module_embedding_path outputs/interpretability/geometry/pancreas_scib/module_embeddings.npy \
54
+ --module_json model_dependencies/module_data_all/merged_modules.json \
55
+ --sentence_model pritamdeka/S-PubMedBert-MS-MARCO \
56
+ --n_permutations 1000 \
57
+ --output_dir outputs/interpretability/text_alignment/pancreas_scib_v2
58
+
59
+ # Quick mode: skip LOO retrieval and run only RSA + CKA.
60
+ python BioMod/interpretability/module_text_alignment/module_text_alignment.py \
61
+ --module_embedding_path outputs/interpretability/geometry/pretrain_subset/module_embeddings.npy \
62
+ --sentence_model pritamdeka/S-PubMedBert-MS-MARCO \
63
+ --skip_loo \
64
+ --output_dir outputs/interpretability/text_alignment/pretrain_subset_quick
65
+ ```
66
+
67
+ ### 3. Fetch Official Reactome Descriptions (Completed, 486/491)
68
+
69
+ ```bash
70
+ # Re-fetch if needed. Requires network access to reactome.org and takes about 2 minutes.
71
+ python BioMod/interpretability/module_text_alignment/fetch_reactome_descriptions.py \
72
+ --module_json model_dependencies/module_data_all/merged_modules.json \
73
+ --output reactome_descriptions.json --delay 0.05
74
+ ```
75
+
76
+ The command writes refreshed descriptions to `reactome_descriptions.json` in the
77
+ repository root. The analysis examples below use the bundled
78
+ `BioMod/interpretability/module_text_alignment/reactome_descriptions.json` instead;
79
+ set `--descriptions_file reactome_descriptions.json` to use the refreshed output.
80
+
81
+ ### 4. Extended Analysis: Reactome Categories, Description-Based RSA, Hallmark Deep Dive
82
+
83
+ ```bash
84
+ python BioMod/interpretability/module_text_alignment/module_text_alignment_extended.py \
85
+ --module_embedding_path outputs/interpretability/geometry/pretrain_subset/module_embeddings.npy \
86
+ --module_json model_dependencies/module_data_all/merged_modules.json \
87
+ --sentence_model pritamdeka/S-PubMedBert-MS-MARCO \
88
+ --descriptions_file BioMod/interpretability/module_text_alignment/reactome_descriptions.json \
89
+ --n_permutations 1000 \
90
+ --output_dir outputs/interpretability/text_alignment/pretrain_subset_v2_extended
91
+ ```
92
+
93
+ ### 5. Publication-Level Analysis
94
+
95
+ ```bash
96
+ # Basic run: one dataset and one model.
97
+ python BioMod/interpretability/module_text_alignment/module_text_alignment_publication.py \
98
+ --module_embedding_path outputs/interpretability/geometry/pretrain_subset/module_embeddings.npy \
99
+ --module_json model_dependencies/module_data_all/merged_modules.json \
100
+ --sentence_model pritamdeka/S-PubMedBert-MS-MARCO \
101
+ --descriptions_file BioMod/interpretability/module_text_alignment/reactome_descriptions.json \
102
+ --n_permutations 1000 \
103
+ --n_bootstrap 2000 \
104
+ --output_dir outputs/interpretability/text_alignment/publication
105
+
106
+ # Full run: cross-dataset validation and multi-model comparison.
107
+ python BioMod/interpretability/module_text_alignment/module_text_alignment_publication.py \
108
+ --module_embedding_path outputs/interpretability/geometry/pretrain_subset/module_embeddings.npy \
109
+ --module_embedding_path2 outputs/interpretability/geometry/pancreas_scib/module_embeddings.npy \
110
+ --module_json model_dependencies/module_data_all/merged_modules.json \
111
+ --sentence_model pritamdeka/S-PubMedBert-MS-MARCO \
112
+ --sentence_models_compare "sentence-transformers/all-mpnet-base-v2,sentence-transformers/all-MiniLM-L6-v2" \
113
+ --descriptions_file BioMod/interpretability/module_text_alignment/reactome_descriptions.json \
114
+ --n_permutations 1000 \
115
+ --n_bootstrap 2000 \
116
+ --output_dir outputs/interpretability/text_alignment/publication_full
117
+ ```
118
+
119
+ **Arguments:**
120
+
121
+ | Argument | Description |
122
+ | --- | --- |
123
+ | `--module_embedding_path` | `module_embeddings.npy` for the main dataset, from Experiment 1 |
124
+ | `--module_embedding_path2` | Optional second dataset for cross-dataset validation |
125
+ | `--sentence_models_compare` | Comma-separated model names for comparison, downloaded automatically from HuggingFace |
126
+ | `--n_permutations` | Mantel-test permutations; 1000 is standard and 10000 gives more precise p-values |
127
+ | `--n_bootstrap` | Bootstrap resampling iterations; 2000 is standard for 95% CIs |
128
+ | `--device` | Encoding device, `cpu` or `cuda`; default is CPU |
129
+
130
+ ## Dependencies
131
+
132
+ ```bash
133
+ pip install sentence-transformers -i https://pypi.tuna.tsinghua.edu.cn/simple
134
+ ```
135
+
136
+ ## Output Files
137
+
138
+ ### Main Experiment Outputs (`module_text_alignment.py`)
139
+
140
+ | File | Content |
141
+ | --- | --- |
142
+ | `pathway_descriptions.json` | Text descriptions for 541 modules |
143
+ | `text_embeddings.npy` | Sentence-encoder embeddings with shape `[541, 768]` |
144
+ | `module_embeddings_subset.npy` | Corresponding model module embeddings with shape `[541, 1024]` |
145
+ | `alignment_metrics.json` | RSA, CKA, retrieval recall@k, and p-values |
146
+ | `rsa_scatter.png` | Paired-distance scatter plot: module space vs text space |
147
+ | `rsa_null_distribution.png` | Mantel test: observed value vs null distribution |
148
+ | `retrieval_ranks.png` | Histogram of retrieval ranks |
149
+ | `retrieval_examples.csv` | Top-5 retrieval results for representative pathways |
150
+ | `run.log` | Full log |
151
+
152
+ ### Extended Analysis Outputs (`module_text_alignment_extended.py`)
153
+
154
+ | File | Content |
155
+ | --- | --- |
156
+ | `extended_metrics.json` | Summary metrics for the three analyses |
157
+ | `reactome_category_heatmaps.png` | Broad Reactome category distance-matrix comparison, module vs text |
158
+ | `hallmarks_deep_dive.png` | Hallmark similarity matrices and per-module alignment-score bar chart |
159
+ | `hallmarks_per_module_alignment.csv` | Per-module alignment ranking for 50 Hallmarks |
160
+ | `run_extended.log` | Extended-analysis log |
161
+
162
+ ### Publication-Level Outputs (`module_text_alignment_publication.py`)
163
+
164
+ | File | Content |
165
+ | --- | --- |
166
+ | `fig1_rsa_overview.{pdf,png}` | 2D hexbin density plus Mantel null distribution with bootstrap CI |
167
+ | `fig2_hallmarks_heatmaps.{pdf,png}` | Three 50 x 50 similarity matrices: module, text, and difference; hierarchical clustering order and adaptive color scale |
168
+ | `fig3_per_hallmark_alignment.{pdf,png}` | Per-Hallmark alignment-score lollipop plot, colored by MSigDB process category |
169
+ | `fig4_category_alignment.{pdf,png}` | Box + strip plot grouped by biological process category with Kruskal-Wallis test |
170
+ | `fig5_granularity_effect.{pdf,png}` | Bar plot of RSA across granularity levels: fine-grained, category, superclass, Hallmark |
171
+ | `fig6_model_comparison.{pdf,png}` | Optional RSA comparison across models and datasets |
172
+ | `fig_alignment_vs_size.{pdf,png}` | Alignment score vs module gene count scatter plot |
173
+ | `publication_metrics.json` | Summary of all metrics, including bootstrap 95% CIs |
174
+ | `hallmarks_alignment_table.csv` | Alignment scores and process categories for 50 Hallmarks |
175
+ | `negative_analysis.csv` | Neighbor comparison for the worst- and best-aligned modules |
176
+ | `run_publication.log` | Full log |
177
+
178
+ ## Results (v2, pretrain_subset)
179
+
180
+ ### Main Experiment
181
+
182
+ | Metric | All (541) | Hallmarks (50) | Reactome (491) |
183
+ | --- | --- | --- | --- |
184
+ | RSA Spearman r | 0.050 | **0.160** | -0.007 |
185
+ | Mantel p-value | 0.012 | 0.030 | 0.586 |
186
+ | z-score | 2.2 sigma | 2.1 sigma | -0.3 sigma |
187
+ | CKA | 0.043 (p=0.000) | - | - |
188
+ | Retrieval median rank | 232 (random=270) | - | - |
189
+
190
+ ### Interpretation
191
+
192
+ - **Hallmark alignment is significant (r=0.16, p=0.03):** among 50 broad biological processes, distance structure in model space significantly correlates with semantic distance in text space.
193
+ - **Reactome does not align (r=-0.007, p=0.59):** among 491 fine-grained pathways there is no alignment, likely because fine-grained pathway gene-composition similarity, encoded by the model, and pathway-name language similarity, encoded by text, live on different axes.
194
+ - **Global CKA is significant (p=0.000):** the overall representation structure contains an alignment signal.
195
+
196
+ ### Improvement History
197
+
198
+ **v1, BioLinkBERT + 1-to-1 retrieval:**
199
+
200
+ - BioLinkBERT was used as a raw BERT model with mean pooling rather than a sentence-level fine-tuned encoder.
201
+ - The main metric was exact 1-to-1 retrieval, which is too strict for a 541-dimensional space.
202
+ - CKA=0.047 was statistically significant but weak; retrieval was close to random.
203
+
204
+ **v2, S-PubMedBert-MS-MARCO + RSA/Mantel test:**
205
+
206
+ 1. Switched to a sentence encoder: PubMedBERT fine-tuned on MS-MARCO sentence similarity.
207
+ 2. Added RSA/Mantel test as the main metric, testing structural rather than exact retrieval alignment.
208
+ 3. Per-family RSA revealed that the alignment signal is concentrated in Hallmarks.
209
+
210
+ ### Extended Analysis Directions
211
+
212
+ 1. **Reactome broad-category aggregation:** aggregate the 491 pathways into about 16 top-level categories and retest RSA; broad-category alignment is expected to be significant.
213
+ 2. **Reactome descriptions:** use API-fetched `summaryText` as richer text input and test whether alignment improves.
214
+ 3. **Hallmark deep dive:** compute per-module alignment scores, kNN overlap, and the best/worst aligned Hallmarks.
215
+
216
+ ## Design Decisions
217
+
218
+ ### Why Use RSA Instead of 1-to-1 Retrieval,
219
+
220
+ 1-to-1 retrieval asks whether the paired text can be found exactly from a module
221
+ embedding, requiring a very strong cross-modal mapping. RSA asks whether modules
222
+ that are close in model space are also close in text space, which is a more
223
+ natural definition of structural alignment.
224
+
225
+ ### Why Does Reactome Not Align,
226
+
227
+ - Reactome pathways overlap heavily because parent and child pathways share many genes.
228
+ - The model encodes gene-composition similarity, while text names reflect hierarchy and naming similarity; the two do not perfectly match.
229
+ - The granularity is too fine. For example, text distance between `Signaling by FGFR1` and `Signaling by FGFR2` does not necessarily reflect biological distance.
230
+
231
+ ### Why Exclude Hotspot/Processed,
232
+
233
+ - Hotspot modules (73) are data-driven co-expression modules with only numeric IDs and no pathway semantics.
234
+ - Processed modules (46) are pipeline-derived gene groups such as `SM_2` and have no pathway names.
235
+ - Without text pairs, they cannot participate in this alignment experiment.
BioMod/interpretability/module_text_alignment/fetch_reactome_descriptions.py CHANGED
@@ -1,97 +1,97 @@
1
- """Fetch Reactome pathway descriptions via the Content Service REST API.
2
-
3
- Produces a JSON file mapping module_name -> summaryText that can be passed
4
- to module_text_alignment_extended.py via --descriptions_file.
5
-
6
- Usage:
7
- python fetch_reactome_descriptions.py \
8
- --module_json ../../module_data_all/merged_modules.json \
9
- --output reactome_descriptions.json
10
-
11
- Requires internet access to https://reactome.org/ContentService/
12
- """
13
- import argparse
14
- import json
15
- import sys
16
- import time
17
- from pathlib import Path
18
-
19
- try:
20
- import requests
21
- except ImportError:
22
- print("ERROR: 'requests' package required. Install: pip install requests")
23
- sys.exit(1)
24
-
25
- HERE = Path(__file__).resolve().parent
26
- INTERP_DIR = HERE.parent
27
- PERFORMER_MODULE = INTERP_DIR.parent
28
- for p in (str(INTERP_DIR), str(PERFORMER_MODULE)):
29
- if p not in sys.path:
30
- sys.path.insert(0, p)
31
-
32
- from interpretability.utils import load_module_names, parse_module_family
33
-
34
- REACTOME_API = "https://reactome.org/ContentService/data/query"
35
-
36
-
37
- def fetch_summary(rhs_id: str, retries: int = 3) -> str:
38
- """Fetch summaryText for a Reactome pathway."""
39
- url = f"{REACTOME_API}/{rhs_id}"
40
- for attempt in range(retries):
41
- try:
42
- resp = requests.get(url, timeout=10)
43
- if resp.status_code == 200:
44
- data = resp.json()
45
- return data.get("summation", [{}])[0].get("text", "")
46
- elif resp.status_code == 404:
47
- return ""
48
- elif resp.status_code == 429:
49
- time.sleep(2 ** attempt)
50
- else:
51
- return ""
52
- except (requests.RequestException, json.JSONDecodeError):
53
- time.sleep(1)
54
- return ""
55
-
56
-
57
- def main():
58
- parser = argparse.ArgumentParser(description=__doc__)
59
- parser.add_argument("--module_json", type=str, required=True)
60
- parser.add_argument("--output", type=str, default="reactome_descriptions.json")
61
- parser.add_argument("--delay", type=float, default=0.1,
62
- help="Delay between API calls (seconds)")
63
- args = parser.parse_args()
64
-
65
- module_names = load_module_names(args.module_json)
66
- reactome_modules = [n for n in module_names if parse_module_family(n) == "Reactome"]
67
- print(f"Reactome modules to fetch: {len(reactome_modules)}")
68
-
69
- descriptions = {}
70
- for i, pname in enumerate(reactome_modules):
71
- raw = pname.split("__", 1)[1]
72
- parts = raw.rsplit(" R-HSA-", 1)
73
- if len(parts) != 2:
74
- continue
75
- rhs_id = "R-HSA-" + parts[1]
76
-
77
- summary = fetch_summary(rhs_id)
78
- if summary:
79
- # Strip HTML tags
80
- import re
81
- summary = re.sub(r"<[^>]+>", "", summary).strip()
82
- descriptions[pname] = summary
83
-
84
- if (i + 1) % 50 == 0:
85
- print(f" Fetched {i+1}/{len(reactome_modules)}, "
86
- f"got descriptions for {len(descriptions)}")
87
- time.sleep(args.delay)
88
-
89
- print(f"\nDone. Got descriptions for {len(descriptions)}/{len(reactome_modules)} pathways.")
90
-
91
- with open(args.output, "w", encoding="utf-8") as f:
92
- json.dump(descriptions, f, indent=2, ensure_ascii=False)
93
- print(f"Saved: {args.output}")
94
-
95
-
96
- if __name__ == "__main__":
97
- main()
 
1
+ """Fetch Reactome pathway descriptions via the Content Service REST API.
2
+
3
+ Produces a JSON file mapping module_name -> summaryText that can be passed
4
+ to module_text_alignment_extended.py via --descriptions_file.
5
+
6
+ Usage (run from the BioMod repository root):
7
+ python BioMod/interpretability/module_text_alignment/fetch_reactome_descriptions.py \
8
+ --module_json model_dependencies/module_data_all/merged_modules.json \
9
+ --output reactome_descriptions.json
10
+
11
+ Requires internet access to https://reactome.org/ContentService/
12
+ """
13
+ import argparse
14
+ import json
15
+ import sys
16
+ import time
17
+ from pathlib import Path
18
+
19
+ try:
20
+ import requests
21
+ except ImportError:
22
+ print("ERROR: 'requests' package required. Install: pip install requests")
23
+ sys.exit(1)
24
+
25
+ HERE = Path(__file__).resolve().parent
26
+ INTERP_DIR = HERE.parent
27
+ PERFORMER_MODULE = INTERP_DIR.parent
28
+ for p in (str(INTERP_DIR), str(PERFORMER_MODULE)):
29
+ if p not in sys.path:
30
+ sys.path.insert(0, p)
31
+
32
+ from interpretability.utils import load_module_names, parse_module_family
33
+
34
+ REACTOME_API = "https://reactome.org/ContentService/data/query"
35
+
36
+
37
+ def fetch_summary(rhs_id: str, retries: int = 3) -> str:
38
+ """Fetch summaryText for a Reactome pathway."""
39
+ url = f"{REACTOME_API}/{rhs_id}"
40
+ for attempt in range(retries):
41
+ try:
42
+ resp = requests.get(url, timeout=10)
43
+ if resp.status_code == 200:
44
+ data = resp.json()
45
+ return data.get("summation", [{}])[0].get("text", "")
46
+ elif resp.status_code == 404:
47
+ return ""
48
+ elif resp.status_code == 429:
49
+ time.sleep(2 ** attempt)
50
+ else:
51
+ return ""
52
+ except (requests.RequestException, json.JSONDecodeError):
53
+ time.sleep(1)
54
+ return ""
55
+
56
+
57
+ def main():
58
+ parser = argparse.ArgumentParser(description=__doc__)
59
+ parser.add_argument("--module_json", type=str, required=True)
60
+ parser.add_argument("--output", type=str, default="reactome_descriptions.json")
61
+ parser.add_argument("--delay", type=float, default=0.1,
62
+ help="Delay between API calls (seconds)")
63
+ args = parser.parse_args()
64
+
65
+ module_names = load_module_names(args.module_json)
66
+ reactome_modules = [n for n in module_names if parse_module_family(n) == "Reactome"]
67
+ print(f"Reactome modules to fetch: {len(reactome_modules)}")
68
+
69
+ descriptions = {}
70
+ for i, pname in enumerate(reactome_modules):
71
+ raw = pname.split("__", 1)[1]
72
+ parts = raw.rsplit(" R-HSA-", 1)
73
+ if len(parts) != 2:
74
+ continue
75
+ rhs_id = "R-HSA-" + parts[1]
76
+
77
+ summary = fetch_summary(rhs_id)
78
+ if summary:
79
+ # Strip HTML tags
80
+ import re
81
+ summary = re.sub(r"<[^>]+>", "", summary).strip()
82
+ descriptions[pname] = summary
83
+
84
+ if (i + 1) % 50 == 0:
85
+ print(f" Fetched {i+1}/{len(reactome_modules)}, "
86
+ f"got descriptions for {len(descriptions)}")
87
+ time.sleep(args.delay)
88
+
89
+ print(f"\nDone. Got descriptions for {len(descriptions)}/{len(reactome_modules)} pathways.")
90
+
91
+ with open(args.output, "w", encoding="utf-8") as f:
92
+ json.dump(descriptions, f, indent=2, ensure_ascii=False)
93
+ print(f"Saved: {args.output}")
94
+
95
+
96
+ if __name__ == "__main__":
97
+ main()
BioMod/interpretability/module_text_alignment/module_text_alignment.py CHANGED
@@ -1,665 +1,665 @@
1
- """Experiment 2 — Module embedding <-> pathway text representation alignment.
2
-
3
- Question
4
- --------
5
- Do scLong's module embeddings align with the natural-language semantics of
6
- their corresponding pathway names/descriptions, If yes, this demonstrates
7
- that the model has learned biologically meaningful representations at the
8
- module level — something gene-level baselines cannot replicate because
9
- individual gene names lack phrase-level semantics.
10
-
11
- Pipeline
12
- --------
13
- 1. Parse pathway names from merged_modules.json keys for Hallmarks (50) and
14
- Reactome (491) modules. Hotspot/Processed are excluded (no semantic names).
15
- 2. Encode pathway names with a biomedical sentence encoder.
16
- 3. Load cell-averaged module embeddings from experiment 1 output.
17
- 4. Measure alignment:
18
- - RSA (Representational Similarity Analysis) with Mantel test
19
- - CKA (Centered Kernel Alignment) with permutation test
20
- - Ridge regression linear mapping (LOO-CV) for retrieval
21
- 5. Visualize: RSA scatter, similarity distribution, retrieval examples.
22
-
23
- Data requirement
24
- ----------------
25
- Only needs module_embeddings.npy from experiment 1 (any dataset). No raw
26
- expression data needed — this experiment operates entirely in embedding space.
27
-
28
- Sentence encoder
29
- ----------------
30
- Default: pritamdeka/S-PubMedBert-MS-MARCO (a PubMedBERT fine-tuned for
31
- sentence similarity via MS-MARCO). Unlike raw BioLinkBERT (which is just a
32
- masked LM without sentence-level training), this model produces embeddings
33
- optimized for semantic similarity between sentences.
34
-
35
- Alternative: sentence-transformers/all-mpnet-base-v2 (general-domain, strong).
36
- """
37
- from __future__ import annotations
38
-
39
- import argparse
40
- import json
41
- import logging
42
- import sys
43
- from pathlib import Path
44
- from typing import Dict, List, Optional, Tuple
45
-
46
- import numpy as np
47
- import pandas as pd
48
- from scipy.spatial.distance import cosine as cosine_dist
49
- from scipy.spatial.distance import pdist, squareform
50
- from scipy.stats import spearmanr, pearsonr
51
- from sklearn.linear_model import RidgeCV
52
- from sklearn.metrics.pairwise import cosine_similarity
53
- from sklearn.preprocessing import StandardScaler
54
-
55
- import matplotlib
56
- matplotlib.use("Agg")
57
- import matplotlib.pyplot as plt
58
-
59
- HERE = Path(__file__).resolve().parent
60
- INTERP_DIR = HERE.parent
61
- PERFORMER_MODULE = INTERP_DIR.parent
62
- REPO_ROOT = PERFORMER_MODULE.parent
63
- for p in (str(INTERP_DIR), str(PERFORMER_MODULE), str(REPO_ROOT)):
64
- if p not in sys.path:
65
- sys.path.insert(0, p)
66
-
67
- from interpretability.utils import (
68
- MODULE_FAMILY_MAP,
69
- load_module_names,
70
- parse_module_family,
71
- )
72
-
73
- logger = logging.getLogger(__name__)
74
-
75
-
76
- # ============================================================
77
- # 1. Pathway name parsing
78
- # ============================================================
79
-
80
- def parse_pathway_text(module_name: str) -> Optional[str]:
81
- """Extract a human-readable pathway name from a module key."""
82
- family = parse_module_family(module_name)
83
- if family == "Hallmarks":
84
- raw = module_name.split("__", 1)[1]
85
- raw = raw.replace("HALLMARK_", "", 1)
86
- return raw.replace("_", " ").title()
87
- elif family == "Reactome":
88
- raw = module_name.split("__", 1)[1]
89
- parts = raw.rsplit(" R-HSA-", 1)
90
- return parts[0].strip()
91
- return None
92
-
93
-
94
- def build_text_descriptions(
95
- module_names: List[str],
96
- descriptions_file: Optional[str] = None,
97
- ) -> Tuple[List[int], List[str], List[str]]:
98
- """Build text descriptions for modules that have semantic names.
99
-
100
- Returns: (indices, names, texts) — only for Hallmarks + Reactome modules.
101
- """
102
- ext_descriptions = {}
103
- if descriptions_file and Path(descriptions_file).exists():
104
- with open(descriptions_file, "r", encoding="utf-8") as f:
105
- ext_descriptions = json.load(f)
106
-
107
- indices = []
108
- names = []
109
- texts = []
110
- for i, pname in enumerate(module_names):
111
- text = parse_pathway_text(pname)
112
- if text is None:
113
- continue
114
- indices.append(i)
115
- names.append(pname)
116
- if pname in ext_descriptions:
117
- texts.append(f"{text}: {ext_descriptions[pname]}")
118
- else:
119
- texts.append(text)
120
- return indices, names, texts
121
-
122
-
123
- # ============================================================
124
- # 2. Text encoding
125
- # ============================================================
126
-
127
- def encode_texts(
128
- texts: List[str],
129
- model_name: str = "pritamdeka/S-PubMedBert-MS-MARCO",
130
- batch_size: int = 32,
131
- device: str = "cpu",
132
- ) -> np.ndarray:
133
- """Encode texts using a sentence-transformers model."""
134
- try:
135
- from sentence_transformers import SentenceTransformer
136
- except ImportError:
137
- raise ImportError(
138
- "sentence-transformers required. Install: pip install sentence-transformers"
139
- )
140
-
141
- logger.info("Loading sentence encoder: %s", model_name)
142
- model = SentenceTransformer(model_name, device=device)
143
- logger.info("Encoding %d texts...", len(texts))
144
- embeddings = model.encode(
145
- texts, batch_size=batch_size, show_progress_bar=True,
146
- normalize_embeddings=True,
147
- )
148
- return np.asarray(embeddings, dtype=np.float32)
149
-
150
-
151
- # ============================================================
152
- # 3. RSA (Representational Similarity Analysis)
153
- # ============================================================
154
-
155
- def compute_rsa(
156
- module_emb: np.ndarray,
157
- text_emb: np.ndarray,
158
- metric: str = "cosine",
159
- ) -> Tuple[float, float, float, float]:
160
- """Representational Similarity Analysis.
161
-
162
- Computes pairwise distance matrices in both spaces, then correlates them.
163
- This tests: "are modules that are close in model space also close in text space, "
164
-
165
- Returns: (spearman_r, spearman_p, pearson_r, pearson_p)
166
- """
167
- dist_module = pdist(module_emb, metric=metric)
168
- dist_text = pdist(text_emb, metric=metric)
169
-
170
- sp_r, sp_p = spearmanr(dist_module, dist_text)
171
- pe_r, pe_p = pearsonr(dist_module, dist_text)
172
-
173
- return float(sp_r), float(sp_p), float(pe_r), float(pe_p)
174
-
175
-
176
- def mantel_test(
177
- module_emb: np.ndarray,
178
- text_emb: np.ndarray,
179
- metric: str = "cosine",
180
- n_perm: int = 1000,
181
- seed: int = 42,
182
- ) -> Dict[str, float]:
183
- """Mantel test: permutation-based significance for RSA.
184
-
185
- Shuffles rows of one matrix and recomputes the distance-matrix correlation.
186
- More robust than parametric p-values for distance correlations.
187
- """
188
- dist_module = pdist(module_emb, metric=metric)
189
- dist_text = pdist(text_emb, metric=metric)
190
-
191
- observed_r, _ = spearmanr(dist_module, dist_text)
192
-
193
- rng = np.random.default_rng(seed)
194
- null_rs = np.zeros(n_perm)
195
- n = module_emb.shape[0]
196
- for k in range(n_perm):
197
- perm = rng.permutation(n)
198
- dist_text_perm = pdist(text_emb[perm], metric=metric)
199
- null_rs[k], _ = spearmanr(dist_module, dist_text_perm)
200
-
201
- p_value = float(np.mean(null_rs >= observed_r))
202
- return {
203
- "rsa_spearman_observed": float(observed_r),
204
- "rsa_null_mean": float(null_rs.mean()),
205
- "rsa_null_std": float(null_rs.std()),
206
- "rsa_p_value": p_value,
207
- "rsa_effect_size_z": float((observed_r - null_rs.mean()) / max(null_rs.std(), 1e-10)),
208
- }
209
-
210
-
211
- def compute_rsa_by_family(
212
- module_emb: np.ndarray,
213
- text_emb: np.ndarray,
214
- names: List[str],
215
- metric: str = "cosine",
216
- n_perm: int = 500,
217
- seed: int = 42,
218
- ) -> Dict[str, Dict[str, float]]:
219
- """RSA computed separately for each module family (Hallmarks vs Reactome)."""
220
- results = {}
221
- families = [parse_module_family(n) for n in names]
222
- for fam in ["Hallmarks", "Reactome"]:
223
- mask = np.array([f == fam for f in families])
224
- if mask.sum() < 10:
225
- continue
226
- sub_module = module_emb[mask]
227
- sub_text = text_emb[mask]
228
- mantel = mantel_test(sub_module, sub_text, metric=metric,
229
- n_perm=n_perm, seed=seed)
230
- results[fam] = mantel
231
- return results
232
-
233
-
234
- # ============================================================
235
- # 4. CKA
236
- # ============================================================
237
-
238
- def compute_cka(X: np.ndarray, Y: np.ndarray) -> float:
239
- """Centered Kernel Alignment (linear kernel)."""
240
- n = X.shape[0]
241
- H = np.eye(n) - np.ones((n, n)) / n
242
- Kx = H @ (X @ X.T) @ H
243
- Ky = H @ (Y @ Y.T) @ H
244
- hsic = np.sum(Kx * Ky)
245
- norm = np.sqrt(np.sum(Kx * Kx) * np.sum(Ky * Ky))
246
- if norm < 1e-10:
247
- return 0.0
248
- return float(hsic / norm)
249
-
250
-
251
- def permutation_test_cka(
252
- module_emb: np.ndarray,
253
- text_emb: np.ndarray,
254
- n_perm: int = 1000,
255
- seed: int = 42,
256
- ) -> Dict[str, float]:
257
- """CKA with permutation null distribution."""
258
- observed = compute_cka(module_emb, text_emb)
259
- rng = np.random.default_rng(seed)
260
- null_values = np.zeros(n_perm)
261
- for k in range(n_perm):
262
- perm = rng.permutation(text_emb.shape[0])
263
- null_values[k] = compute_cka(module_emb, text_emb[perm])
264
- p_value = float(np.mean(null_values >= observed))
265
- return {
266
- "cka_observed": observed,
267
- "cka_null_mean": float(null_values.mean()),
268
- "cka_null_std": float(null_values.std()),
269
- "cka_p_value": p_value,
270
- }
271
-
272
-
273
- # ============================================================
274
- # 5. Retrieval (Ridge LOO)
275
- # ============================================================
276
-
277
- def compute_retrieval_metrics(
278
- module_emb: np.ndarray,
279
- text_emb: np.ndarray,
280
- ks: List[int] = [1, 5, 10, 20],
281
- ) -> Tuple[Dict[str, float], np.ndarray]:
282
- """Cross-modal retrieval using Ridge LOO projection."""
283
- n = module_emb.shape[0]
284
- scaler_x = StandardScaler()
285
- scaler_y = StandardScaler()
286
- X_scaled = scaler_x.fit_transform(module_emb)
287
- Y_scaled = scaler_y.fit_transform(text_emb)
288
-
289
- ranks = np.zeros(n, dtype=np.int64)
290
- for i in range(n):
291
- train_mask = np.ones(n, dtype=bool)
292
- train_mask[i] = False
293
- ridge = RidgeCV(alphas=np.logspace(-2, 4, 7))
294
- ridge.fit(X_scaled[train_mask], Y_scaled[train_mask])
295
- y_pred = ridge.predict(X_scaled[i:i+1])
296
- sims = cosine_similarity(y_pred, Y_scaled)[0]
297
- sorted_idx = np.argsort(-sims)
298
- ranks[i] = int(np.where(sorted_idx == i)[0][0])
299
-
300
- metrics = {}
301
- for k in ks:
302
- metrics[f"recall@{k}"] = float(np.mean(ranks < k))
303
- metrics["mrr"] = float(np.mean(1.0 / (ranks + 1)))
304
- metrics["median_rank"] = int(np.median(ranks))
305
- metrics["mean_rank"] = float(np.mean(ranks))
306
- return metrics, ranks
307
-
308
-
309
- # ============================================================
310
- # 6. Visualization
311
- # ============================================================
312
-
313
- def plot_rsa_scatter(
314
- module_emb: np.ndarray,
315
- text_emb: np.ndarray,
316
- rsa_r: float,
317
- rsa_p: float,
318
- output_path: str,
319
- metric: str = "cosine",
320
- max_pairs: int = 5000,
321
- ):
322
- """Scatter plot of pairwise distances: module space vs text space."""
323
- dist_module = pdist(module_emb, metric=metric)
324
- dist_text = pdist(text_emb, metric=metric)
325
-
326
- if len(dist_module) > max_pairs:
327
- rng = np.random.default_rng(0)
328
- idx = rng.choice(len(dist_module), size=max_pairs, replace=False)
329
- dist_module = dist_module[idx]
330
- dist_text = dist_text[idx]
331
-
332
- fig, ax = plt.subplots(figsize=(7, 6))
333
- ax.scatter(dist_text, dist_module, alpha=0.1, s=3, color="steelblue")
334
- ax.set_xlabel("Text pairwise distance (cosine)")
335
- ax.set_ylabel("Module embedding pairwise distance (cosine)")
336
- ax.set_title(f"RSA: Spearman r = {rsa_r:.4f} (p = {rsa_p:.4f})")
337
-
338
- z = np.polyfit(dist_text, dist_module, 1)
339
- p_line = np.poly1d(z)
340
- x_range = np.linspace(dist_text.min(), dist_text.max(), 100)
341
- ax.plot(x_range, p_line(x_range), "r-", linewidth=2, alpha=0.8)
342
-
343
- plt.tight_layout()
344
- plt.savefig(output_path, dpi=150, bbox_inches="tight")
345
- plt.close()
346
- logger.info("Saved: %s", output_path)
347
-
348
-
349
- def plot_retrieval_rank_distribution(
350
- ranks: np.ndarray,
351
- n_total: int,
352
- output_path: str,
353
- ):
354
- """Histogram of retrieval ranks."""
355
- fig, ax = plt.subplots(figsize=(8, 5))
356
- ax.hist(ranks, bins=50, alpha=0.7, color="darkorange", edgecolor="white")
357
- ax.axvline(n_total / 2, color="gray", linestyle="--", linewidth=1.5,
358
- label=f"Random baseline = {n_total//2}")
359
- ax.axvline(np.median(ranks), color="blue", linestyle="-", linewidth=1.5,
360
- label=f"Median = {int(np.median(ranks))}")
361
- ax.set_xlabel("Retrieval Rank (0 = perfect)")
362
- ax.set_ylabel("Count")
363
- ax.set_title("Cross-Modal Retrieval: Rank Distribution (LOO Ridge)")
364
- ax.legend()
365
- plt.tight_layout()
366
- plt.savefig(output_path, dpi=150, bbox_inches="tight")
367
- plt.close()
368
- logger.info("Saved: %s", output_path)
369
-
370
-
371
- def plot_rsa_null_distribution(
372
- observed: float,
373
- null_values_mean: float,
374
- null_values_std: float,
375
- p_value: float,
376
- output_path: str,
377
- ):
378
- """Plot observed RSA vs null distribution."""
379
- fig, ax = plt.subplots(figsize=(7, 5))
380
- x = np.linspace(null_values_mean - 4*null_values_std,
381
- max(observed + null_values_std, null_values_mean + 4*null_values_std), 200)
382
- from scipy.stats import norm as norm_dist
383
- y = norm_dist.pdf(x, null_values_mean, null_values_std)
384
- ax.fill_between(x, y, alpha=0.3, color="gray", label="Null distribution")
385
- ax.plot(x, y, color="gray", linewidth=1.5)
386
- ax.axvline(observed, color="red", linewidth=2.5, linestyle="-",
387
- label=f"Observed = {observed:.4f}")
388
- ax.set_xlabel("Spearman r (distance matrix correlation)")
389
- ax.set_ylabel("Density")
390
- ax.set_title(f"Mantel Test: p = {p_value:.4f}")
391
- ax.legend()
392
- plt.tight_layout()
393
- plt.savefig(output_path, dpi=150, bbox_inches="tight")
394
- plt.close()
395
- logger.info("Saved: %s", output_path)
396
-
397
-
398
- def save_retrieval_examples(
399
- module_emb: np.ndarray,
400
- text_emb: np.ndarray,
401
- texts: List[str],
402
- names: List[str],
403
- output_path: str,
404
- top_k: int = 5,
405
- query_indices: Optional[List[int]] = None,
406
- ):
407
- """Save retrieval examples for selected modules."""
408
- n = module_emb.shape[0]
409
- scaler_x = StandardScaler()
410
- scaler_y = StandardScaler()
411
- X = scaler_x.fit_transform(module_emb)
412
- Y = scaler_y.fit_transform(text_emb)
413
-
414
- ridge = RidgeCV(alphas=np.logspace(-2, 4, 7))
415
- ridge.fit(X, Y)
416
-
417
- if query_indices is None:
418
- rng = np.random.default_rng(0)
419
- query_indices = sorted(rng.choice(n, size=min(20, n), replace=False))
420
-
421
- rows = []
422
- for qi in query_indices:
423
- y_pred = ridge.predict(X[qi:qi+1])
424
- sims = cosine_similarity(y_pred, Y)[0]
425
- top_idx = np.argsort(-sims)[:top_k]
426
- for rank, ti in enumerate(top_idx):
427
- rows.append({
428
- "query_module": names[qi],
429
- "query_text": texts[qi],
430
- "retrieved_rank": rank + 1,
431
- "retrieved_text": texts[ti],
432
- "similarity": float(sims[ti]),
433
- "is_self": ti == qi,
434
- })
435
-
436
- df = pd.DataFrame(rows)
437
- df.to_csv(output_path, index=False)
438
- logger.info("Saved retrieval examples: %s", output_path)
439
-
440
-
441
- # ============================================================
442
- # 7. Main
443
- # ============================================================
444
-
445
- def parse_args() -> argparse.Namespace:
446
- parser = argparse.ArgumentParser(
447
- description=__doc__,
448
- formatter_class=argparse.RawDescriptionHelpFormatter,
449
- )
450
- parser.add_argument("--module_embedding_path", type=str, required=True,
451
- help="Path to module_embeddings.npy from experiment 1.")
452
- parser.add_argument("--module_json", type=str,
453
- default=str(PERFORMER_MODULE / "module_data_all" / "merged_modules.json"),
454
- help="Path to merged_modules.json.")
455
- parser.add_argument("--descriptions_file", type=str, default=None,
456
- help="Optional JSON with extended pathway descriptions.")
457
- parser.add_argument("--sentence_model", type=str,
458
- default="pritamdeka/S-PubMedBert-MS-MARCO",
459
- help="HuggingFace sentence-transformers model.")
460
- parser.add_argument("--device", type=str, default="cpu",
461
- help="Device for sentence encoder.")
462
- parser.add_argument("--n_permutations", type=int, default=1000)
463
- parser.add_argument("--seed", type=int, default=42)
464
- parser.add_argument("--output_dir", type=str, required=True)
465
- parser.add_argument("--skip_loo", action="store_true",
466
- help="Skip LOO retrieval (slow). Use RSA/CKA only.")
467
- parser.add_argument("--rsa_metric", type=str, default="cosine",
468
- choices=["cosine", "euclidean"],
469
- help="Distance metric for RSA.")
470
- return parser.parse_args()
471
-
472
-
473
- def main():
474
- args = parse_args()
475
-
476
- output_dir = Path(args.output_dir)
477
- output_dir.mkdir(parents=True, exist_ok=True)
478
-
479
- logging.basicConfig(
480
- level=logging.INFO,
481
- format="%(asctime)s [%(levelname)s] %(message)s",
482
- handlers=[
483
- logging.StreamHandler(),
484
- logging.FileHandler(output_dir / "run.log", mode="w"),
485
- ],
486
- )
487
-
488
- # --- Load module names and embeddings ---
489
- logger.info("Loading module names from %s", args.module_json)
490
- module_names = load_module_names(args.module_json)
491
- logger.info("Total modules: %d", len(module_names))
492
-
493
- logger.info("Loading module embeddings from %s", args.module_embedding_path)
494
- all_embeddings = np.load(args.module_embedding_path)
495
- assert all_embeddings.shape[0] == len(module_names), (
496
- f"Embedding count {all_embeddings.shape[0]} != module count {len(module_names)}"
497
- )
498
-
499
- # --- Build text descriptions (Hallmarks + Reactome only) ---
500
- indices, names, texts = build_text_descriptions(
501
- module_names, args.descriptions_file
502
- )
503
- logger.info("Modules with text descriptions: %d / %d", len(indices), len(module_names))
504
-
505
- module_emb = all_embeddings[indices]
506
- logger.info("Module embedding shape: %s", module_emb.shape)
507
-
508
- # --- Encode texts ---
509
- text_emb = encode_texts(
510
- texts, model_name=args.sentence_model, device=args.device
511
- )
512
- logger.info("Text embedding shape: %s", text_emb.shape)
513
-
514
- # Save intermediate
515
- np.save(output_dir / "text_embeddings.npy", text_emb)
516
- np.save(output_dir / "module_embeddings_subset.npy", module_emb)
517
- with open(output_dir / "pathway_descriptions.json", "w", encoding="utf-8") as f:
518
- json.dump(
519
- [{"index": idx, "module_name": name, "text": text}
520
- for idx, name, text in zip(indices, names, texts)],
521
- f, indent=2, ensure_ascii=False,
522
- )
523
-
524
- # --- RSA / Mantel test (PRIMARY METRIC) ---
525
- logger.info("=" * 50)
526
- logger.info("RSA / Mantel Test (primary alignment metric)")
527
- logger.info("=" * 50)
528
-
529
- sp_r, sp_p, pe_r, pe_p = compute_rsa(module_emb, text_emb, metric=args.rsa_metric)
530
- logger.info("RSA (all modules): Spearman r=%.4f (p=%.2e), Pearson r=%.4f (p=%.2e)",
531
- sp_r, sp_p, pe_r, pe_p)
532
-
533
- mantel_results = mantel_test(
534
- module_emb, text_emb, metric=args.rsa_metric,
535
- n_perm=args.n_permutations, seed=args.seed,
536
- )
537
- logger.info("Mantel test: observed=%.4f, null=%.4f +/- %.4f, p=%.4f, z=%.1f",
538
- mantel_results["rsa_spearman_observed"],
539
- mantel_results["rsa_null_mean"],
540
- mantel_results["rsa_null_std"],
541
- mantel_results["rsa_p_value"],
542
- mantel_results["rsa_effect_size_z"])
543
-
544
- # RSA by family
545
- family_rsa = compute_rsa_by_family(
546
- module_emb, text_emb, names, metric=args.rsa_metric,
547
- n_perm=min(args.n_permutations, 500), seed=args.seed,
548
- )
549
- for fam, res in family_rsa.items():
550
- logger.info("RSA [%s]: r=%.4f, null=%.4f +/- %.4f, p=%.4f, z=%.1f",
551
- fam, res["rsa_spearman_observed"], res["rsa_null_mean"],
552
- res["rsa_null_std"], res["rsa_p_value"], res["rsa_effect_size_z"])
553
-
554
- # --- CKA ---
555
- logger.info("Computing CKA with %d permutations...", args.n_permutations)
556
- cka_results = permutation_test_cka(
557
- module_emb, text_emb, n_perm=args.n_permutations, seed=args.seed
558
- )
559
- logger.info("CKA: observed=%.4f, null=%.4f +/- %.4f, p=%.4f",
560
- cka_results["cka_observed"], cka_results["cka_null_mean"],
561
- cka_results["cka_null_std"], cka_results["cka_p_value"])
562
-
563
- # --- LOO retrieval (optional) ---
564
- retrieval_results = {}
565
- ranks = None
566
- if not args.skip_loo:
567
- logger.info("Computing LOO retrieval metrics (n=%d)...", len(indices))
568
- retrieval_results, ranks = compute_retrieval_metrics(module_emb, text_emb)
569
- logger.info("Retrieval: recall@1=%.3f, @5=%.3f, @10=%.3f, @20=%.3f, MRR=%.3f, median_rank=%d",
570
- retrieval_results.get("recall@1", 0),
571
- retrieval_results.get("recall@5", 0),
572
- retrieval_results.get("recall@10", 0),
573
- retrieval_results.get("recall@20", 0),
574
- retrieval_results.get("mrr", 0),
575
- retrieval_results.get("median_rank", 0))
576
-
577
- # --- Aggregate metrics ---
578
- metrics = {
579
- "n_modules_with_text": len(indices),
580
- "n_modules_total": len(module_names),
581
- "module_emb_dim": int(module_emb.shape[1]),
582
- "text_emb_dim": int(text_emb.shape[1]),
583
- "sentence_model": args.sentence_model,
584
- "rsa_metric": args.rsa_metric,
585
- "n_permutations": args.n_permutations,
586
- "rsa_spearman_r": sp_r,
587
- "rsa_spearman_p": sp_p,
588
- "rsa_pearson_r": pe_r,
589
- "rsa_pearson_p": pe_p,
590
- **mantel_results,
591
- "rsa_by_family": family_rsa,
592
- **cka_results,
593
- **retrieval_results,
594
- }
595
-
596
- with open(output_dir / "alignment_metrics.json", "w") as f:
597
- json.dump(metrics, f, indent=2)
598
- logger.info("Saved alignment_metrics.json")
599
-
600
- # --- Plots ---
601
- plot_rsa_scatter(
602
- module_emb, text_emb, rsa_r=sp_r, rsa_p=mantel_results["rsa_p_value"],
603
- output_path=str(output_dir / "rsa_scatter.png"),
604
- metric=args.rsa_metric,
605
- )
606
-
607
- plot_rsa_null_distribution(
608
- observed=mantel_results["rsa_spearman_observed"],
609
- null_values_mean=mantel_results["rsa_null_mean"],
610
- null_values_std=mantel_results["rsa_null_std"],
611
- p_value=mantel_results["rsa_p_value"],
612
- output_path=str(output_dir / "rsa_null_distribution.png"),
613
- )
614
-
615
- if ranks is not None:
616
- plot_retrieval_rank_distribution(
617
- ranks, n_total=len(indices),
618
- output_path=str(output_dir / "retrieval_ranks.png"),
619
- )
620
-
621
- # --- Retrieval examples ---
622
- representative_keywords = [
623
- "interferon", "cell cycle", "oxidative", "apoptosis",
624
- "immune", "signaling", "metabolism", "dna repair",
625
- "nfkb", "mtor", "wnt", "hedgehog", "notch",
626
- "interleukin", "toll", "mapk",
627
- ]
628
- query_indices = []
629
- for kw in representative_keywords:
630
- for i, t in enumerate(texts):
631
- if kw.lower() in t.lower() and i not in query_indices:
632
- query_indices.append(i)
633
- break
634
- if len(query_indices) < 10:
635
- rng = np.random.default_rng(args.seed)
636
- extra = rng.choice(len(texts), size=15, replace=False)
637
- query_indices = list(set(query_indices) | set(extra.tolist()))[:20]
638
-
639
- save_retrieval_examples(
640
- module_emb, text_emb, texts, names,
641
- output_path=str(output_dir / "retrieval_examples.csv"),
642
- query_indices=sorted(query_indices),
643
- )
644
-
645
- # --- Summary ---
646
- logger.info("=" * 60)
647
- logger.info("EXPERIMENT 2 SUMMARY")
648
- logger.info("=" * 60)
649
- logger.info("Modules with text: %d / %d", len(indices), len(module_names))
650
- logger.info("RSA (Mantel): Spearman r=%.4f, p=%.4f, z=%.1f sigma",
651
- mantel_results["rsa_spearman_observed"],
652
- mantel_results["rsa_p_value"],
653
- mantel_results["rsa_effect_size_z"])
654
- logger.info("CKA: %.4f (p=%.4f)", cka_results["cka_observed"], cka_results["cka_p_value"])
655
- if retrieval_results:
656
- logger.info("Retrieval: recall@1=%.3f, @5=%.3f, @10=%.3f, MRR=%.3f, median_rank=%d",
657
- retrieval_results["recall@1"], retrieval_results["recall@5"],
658
- retrieval_results["recall@10"], retrieval_results["mrr"],
659
- retrieval_results["median_rank"])
660
- logger.info("Output: %s", output_dir)
661
- logger.info("=" * 60)
662
-
663
-
664
- if __name__ == "__main__":
665
- main()
 
1
+ """Experiment 2 — Module embedding <-> pathway text representation alignment.
2
+
3
+ Question
4
+ --------
5
+ Do scLong's module embeddings align with the natural-language semantics of
6
+ their corresponding pathway names/descriptions, If yes, this demonstrates
7
+ that the model has learned biologically meaningful representations at the
8
+ module level — something gene-level baselines cannot replicate because
9
+ individual gene names lack phrase-level semantics.
10
+
11
+ Pipeline
12
+ --------
13
+ 1. Parse pathway names from merged_modules.json keys for Hallmarks (50) and
14
+ Reactome (491) modules. Hotspot/Processed are excluded (no semantic names).
15
+ 2. Encode pathway names with a biomedical sentence encoder.
16
+ 3. Load cell-averaged module embeddings from experiment 1 output.
17
+ 4. Measure alignment:
18
+ - RSA (Representational Similarity Analysis) with Mantel test
19
+ - CKA (Centered Kernel Alignment) with permutation test
20
+ - Ridge regression linear mapping (LOO-CV) for retrieval
21
+ 5. Visualize: RSA scatter, similarity distribution, retrieval examples.
22
+
23
+ Data requirement
24
+ ----------------
25
+ Only needs module_embeddings.npy from experiment 1 (any dataset). No raw
26
+ expression data needed — this experiment operates entirely in embedding space.
27
+
28
+ Sentence encoder
29
+ ----------------
30
+ Default: pritamdeka/S-PubMedBert-MS-MARCO (a PubMedBERT fine-tuned for
31
+ sentence similarity via MS-MARCO). Unlike raw BioLinkBERT (which is just a
32
+ masked LM without sentence-level training), this model produces embeddings
33
+ optimized for semantic similarity between sentences.
34
+
35
+ Alternative: sentence-transformers/all-mpnet-base-v2 (general-domain, strong).
36
+ """
37
+ from __future__ import annotations
38
+
39
+ import argparse
40
+ import json
41
+ import logging
42
+ import sys
43
+ from pathlib import Path
44
+ from typing import Dict, List, Optional, Tuple
45
+
46
+ import numpy as np
47
+ import pandas as pd
48
+ from scipy.spatial.distance import cosine as cosine_dist
49
+ from scipy.spatial.distance import pdist, squareform
50
+ from scipy.stats import spearmanr, pearsonr
51
+ from sklearn.linear_model import RidgeCV
52
+ from sklearn.metrics.pairwise import cosine_similarity
53
+ from sklearn.preprocessing import StandardScaler
54
+
55
+ import matplotlib
56
+ matplotlib.use("Agg")
57
+ import matplotlib.pyplot as plt
58
+
59
+ HERE = Path(__file__).resolve().parent
60
+ INTERP_DIR = HERE.parent
61
+ PERFORMER_MODULE = INTERP_DIR.parent
62
+ REPO_ROOT = PERFORMER_MODULE.parent
63
+ for p in (str(INTERP_DIR), str(PERFORMER_MODULE), str(REPO_ROOT)):
64
+ if p not in sys.path:
65
+ sys.path.insert(0, p)
66
+
67
+ from interpretability.utils import (
68
+ MODULE_FAMILY_MAP,
69
+ load_module_names,
70
+ parse_module_family,
71
+ )
72
+
73
+ logger = logging.getLogger(__name__)
74
+
75
+
76
+ # ============================================================
77
+ # 1. Pathway name parsing
78
+ # ============================================================
79
+
80
+ def parse_pathway_text(module_name: str) -> Optional[str]:
81
+ """Extract a human-readable pathway name from a module key."""
82
+ family = parse_module_family(module_name)
83
+ if family == "Hallmarks":
84
+ raw = module_name.split("__", 1)[1]
85
+ raw = raw.replace("HALLMARK_", "", 1)
86
+ return raw.replace("_", " ").title()
87
+ elif family == "Reactome":
88
+ raw = module_name.split("__", 1)[1]
89
+ parts = raw.rsplit(" R-HSA-", 1)
90
+ return parts[0].strip()
91
+ return None
92
+
93
+
94
+ def build_text_descriptions(
95
+ module_names: List[str],
96
+ descriptions_file: Optional[str] = None,
97
+ ) -> Tuple[List[int], List[str], List[str]]:
98
+ """Build text descriptions for modules that have semantic names.
99
+
100
+ Returns: (indices, names, texts) — only for Hallmarks + Reactome modules.
101
+ """
102
+ ext_descriptions = {}
103
+ if descriptions_file and Path(descriptions_file).exists():
104
+ with open(descriptions_file, "r", encoding="utf-8") as f:
105
+ ext_descriptions = json.load(f)
106
+
107
+ indices = []
108
+ names = []
109
+ texts = []
110
+ for i, pname in enumerate(module_names):
111
+ text = parse_pathway_text(pname)
112
+ if text is None:
113
+ continue
114
+ indices.append(i)
115
+ names.append(pname)
116
+ if pname in ext_descriptions:
117
+ texts.append(f"{text}: {ext_descriptions[pname]}")
118
+ else:
119
+ texts.append(text)
120
+ return indices, names, texts
121
+
122
+
123
+ # ============================================================
124
+ # 2. Text encoding
125
+ # ============================================================
126
+
127
+ def encode_texts(
128
+ texts: List[str],
129
+ model_name: str = "pritamdeka/S-PubMedBert-MS-MARCO",
130
+ batch_size: int = 32,
131
+ device: str = "cpu",
132
+ ) -> np.ndarray:
133
+ """Encode texts using a sentence-transformers model."""
134
+ try:
135
+ from sentence_transformers import SentenceTransformer
136
+ except ImportError:
137
+ raise ImportError(
138
+ "sentence-transformers required. Install: pip install sentence-transformers"
139
+ )
140
+
141
+ logger.info("Loading sentence encoder: %s", model_name)
142
+ model = SentenceTransformer(model_name, device=device)
143
+ logger.info("Encoding %d texts...", len(texts))
144
+ embeddings = model.encode(
145
+ texts, batch_size=batch_size, show_progress_bar=True,
146
+ normalize_embeddings=True,
147
+ )
148
+ return np.asarray(embeddings, dtype=np.float32)
149
+
150
+
151
+ # ============================================================
152
+ # 3. RSA (Representational Similarity Analysis)
153
+ # ============================================================
154
+
155
+ def compute_rsa(
156
+ module_emb: np.ndarray,
157
+ text_emb: np.ndarray,
158
+ metric: str = "cosine",
159
+ ) -> Tuple[float, float, float, float]:
160
+ """Representational Similarity Analysis.
161
+
162
+ Computes pairwise distance matrices in both spaces, then correlates them.
163
+ This tests: "are modules that are close in model space also close in text space, "
164
+
165
+ Returns: (spearman_r, spearman_p, pearson_r, pearson_p)
166
+ """
167
+ dist_module = pdist(module_emb, metric=metric)
168
+ dist_text = pdist(text_emb, metric=metric)
169
+
170
+ sp_r, sp_p = spearmanr(dist_module, dist_text)
171
+ pe_r, pe_p = pearsonr(dist_module, dist_text)
172
+
173
+ return float(sp_r), float(sp_p), float(pe_r), float(pe_p)
174
+
175
+
176
+ def mantel_test(
177
+ module_emb: np.ndarray,
178
+ text_emb: np.ndarray,
179
+ metric: str = "cosine",
180
+ n_perm: int = 1000,
181
+ seed: int = 42,
182
+ ) -> Dict[str, float]:
183
+ """Mantel test: permutation-based significance for RSA.
184
+
185
+ Shuffles rows of one matrix and recomputes the distance-matrix correlation.
186
+ More robust than parametric p-values for distance correlations.
187
+ """
188
+ dist_module = pdist(module_emb, metric=metric)
189
+ dist_text = pdist(text_emb, metric=metric)
190
+
191
+ observed_r, _ = spearmanr(dist_module, dist_text)
192
+
193
+ rng = np.random.default_rng(seed)
194
+ null_rs = np.zeros(n_perm)
195
+ n = module_emb.shape[0]
196
+ for k in range(n_perm):
197
+ perm = rng.permutation(n)
198
+ dist_text_perm = pdist(text_emb[perm], metric=metric)
199
+ null_rs[k], _ = spearmanr(dist_module, dist_text_perm)
200
+
201
+ p_value = float(np.mean(null_rs >= observed_r))
202
+ return {
203
+ "rsa_spearman_observed": float(observed_r),
204
+ "rsa_null_mean": float(null_rs.mean()),
205
+ "rsa_null_std": float(null_rs.std()),
206
+ "rsa_p_value": p_value,
207
+ "rsa_effect_size_z": float((observed_r - null_rs.mean()) / max(null_rs.std(), 1e-10)),
208
+ }
209
+
210
+
211
+ def compute_rsa_by_family(
212
+ module_emb: np.ndarray,
213
+ text_emb: np.ndarray,
214
+ names: List[str],
215
+ metric: str = "cosine",
216
+ n_perm: int = 500,
217
+ seed: int = 42,
218
+ ) -> Dict[str, Dict[str, float]]:
219
+ """RSA computed separately for each module family (Hallmarks vs Reactome)."""
220
+ results = {}
221
+ families = [parse_module_family(n) for n in names]
222
+ for fam in ["Hallmarks", "Reactome"]:
223
+ mask = np.array([f == fam for f in families])
224
+ if mask.sum() < 10:
225
+ continue
226
+ sub_module = module_emb[mask]
227
+ sub_text = text_emb[mask]
228
+ mantel = mantel_test(sub_module, sub_text, metric=metric,
229
+ n_perm=n_perm, seed=seed)
230
+ results[fam] = mantel
231
+ return results
232
+
233
+
234
+ # ============================================================
235
+ # 4. CKA
236
+ # ============================================================
237
+
238
+ def compute_cka(X: np.ndarray, Y: np.ndarray) -> float:
239
+ """Centered Kernel Alignment (linear kernel)."""
240
+ n = X.shape[0]
241
+ H = np.eye(n) - np.ones((n, n)) / n
242
+ Kx = H @ (X @ X.T) @ H
243
+ Ky = H @ (Y @ Y.T) @ H
244
+ hsic = np.sum(Kx * Ky)
245
+ norm = np.sqrt(np.sum(Kx * Kx) * np.sum(Ky * Ky))
246
+ if norm < 1e-10:
247
+ return 0.0
248
+ return float(hsic / norm)
249
+
250
+
251
+ def permutation_test_cka(
252
+ module_emb: np.ndarray,
253
+ text_emb: np.ndarray,
254
+ n_perm: int = 1000,
255
+ seed: int = 42,
256
+ ) -> Dict[str, float]:
257
+ """CKA with permutation null distribution."""
258
+ observed = compute_cka(module_emb, text_emb)
259
+ rng = np.random.default_rng(seed)
260
+ null_values = np.zeros(n_perm)
261
+ for k in range(n_perm):
262
+ perm = rng.permutation(text_emb.shape[0])
263
+ null_values[k] = compute_cka(module_emb, text_emb[perm])
264
+ p_value = float(np.mean(null_values >= observed))
265
+ return {
266
+ "cka_observed": observed,
267
+ "cka_null_mean": float(null_values.mean()),
268
+ "cka_null_std": float(null_values.std()),
269
+ "cka_p_value": p_value,
270
+ }
271
+
272
+
273
+ # ============================================================
274
+ # 5. Retrieval (Ridge LOO)
275
+ # ============================================================
276
+
277
+ def compute_retrieval_metrics(
278
+ module_emb: np.ndarray,
279
+ text_emb: np.ndarray,
280
+ ks: List[int] = [1, 5, 10, 20],
281
+ ) -> Tuple[Dict[str, float], np.ndarray]:
282
+ """Cross-modal retrieval using Ridge LOO projection."""
283
+ n = module_emb.shape[0]
284
+ scaler_x = StandardScaler()
285
+ scaler_y = StandardScaler()
286
+ X_scaled = scaler_x.fit_transform(module_emb)
287
+ Y_scaled = scaler_y.fit_transform(text_emb)
288
+
289
+ ranks = np.zeros(n, dtype=np.int64)
290
+ for i in range(n):
291
+ train_mask = np.ones(n, dtype=bool)
292
+ train_mask[i] = False
293
+ ridge = RidgeCV(alphas=np.logspace(-2, 4, 7))
294
+ ridge.fit(X_scaled[train_mask], Y_scaled[train_mask])
295
+ y_pred = ridge.predict(X_scaled[i:i+1])
296
+ sims = cosine_similarity(y_pred, Y_scaled)[0]
297
+ sorted_idx = np.argsort(-sims)
298
+ ranks[i] = int(np.where(sorted_idx == i)[0][0])
299
+
300
+ metrics = {}
301
+ for k in ks:
302
+ metrics[f"recall@{k}"] = float(np.mean(ranks < k))
303
+ metrics["mrr"] = float(np.mean(1.0 / (ranks + 1)))
304
+ metrics["median_rank"] = int(np.median(ranks))
305
+ metrics["mean_rank"] = float(np.mean(ranks))
306
+ return metrics, ranks
307
+
308
+
309
+ # ============================================================
310
+ # 6. Visualization
311
+ # ============================================================
312
+
313
+ def plot_rsa_scatter(
314
+ module_emb: np.ndarray,
315
+ text_emb: np.ndarray,
316
+ rsa_r: float,
317
+ rsa_p: float,
318
+ output_path: str,
319
+ metric: str = "cosine",
320
+ max_pairs: int = 5000,
321
+ ):
322
+ """Scatter plot of pairwise distances: module space vs text space."""
323
+ dist_module = pdist(module_emb, metric=metric)
324
+ dist_text = pdist(text_emb, metric=metric)
325
+
326
+ if len(dist_module) > max_pairs:
327
+ rng = np.random.default_rng(0)
328
+ idx = rng.choice(len(dist_module), size=max_pairs, replace=False)
329
+ dist_module = dist_module[idx]
330
+ dist_text = dist_text[idx]
331
+
332
+ fig, ax = plt.subplots(figsize=(7, 6))
333
+ ax.scatter(dist_text, dist_module, alpha=0.1, s=3, color="steelblue")
334
+ ax.set_xlabel("Text pairwise distance (cosine)")
335
+ ax.set_ylabel("Module embedding pairwise distance (cosine)")
336
+ ax.set_title(f"RSA: Spearman r = {rsa_r:.4f} (p = {rsa_p:.4f})")
337
+
338
+ z = np.polyfit(dist_text, dist_module, 1)
339
+ p_line = np.poly1d(z)
340
+ x_range = np.linspace(dist_text.min(), dist_text.max(), 100)
341
+ ax.plot(x_range, p_line(x_range), "r-", linewidth=2, alpha=0.8)
342
+
343
+ plt.tight_layout()
344
+ plt.savefig(output_path, dpi=150, bbox_inches="tight")
345
+ plt.close()
346
+ logger.info("Saved: %s", output_path)
347
+
348
+
349
+ def plot_retrieval_rank_distribution(
350
+ ranks: np.ndarray,
351
+ n_total: int,
352
+ output_path: str,
353
+ ):
354
+ """Histogram of retrieval ranks."""
355
+ fig, ax = plt.subplots(figsize=(8, 5))
356
+ ax.hist(ranks, bins=50, alpha=0.7, color="darkorange", edgecolor="white")
357
+ ax.axvline(n_total / 2, color="gray", linestyle="--", linewidth=1.5,
358
+ label=f"Random baseline = {n_total//2}")
359
+ ax.axvline(np.median(ranks), color="blue", linestyle="-", linewidth=1.5,
360
+ label=f"Median = {int(np.median(ranks))}")
361
+ ax.set_xlabel("Retrieval Rank (0 = perfect)")
362
+ ax.set_ylabel("Count")
363
+ ax.set_title("Cross-Modal Retrieval: Rank Distribution (LOO Ridge)")
364
+ ax.legend()
365
+ plt.tight_layout()
366
+ plt.savefig(output_path, dpi=150, bbox_inches="tight")
367
+ plt.close()
368
+ logger.info("Saved: %s", output_path)
369
+
370
+
371
+ def plot_rsa_null_distribution(
372
+ observed: float,
373
+ null_values_mean: float,
374
+ null_values_std: float,
375
+ p_value: float,
376
+ output_path: str,
377
+ ):
378
+ """Plot observed RSA vs null distribution."""
379
+ fig, ax = plt.subplots(figsize=(7, 5))
380
+ x = np.linspace(null_values_mean - 4*null_values_std,
381
+ max(observed + null_values_std, null_values_mean + 4*null_values_std), 200)
382
+ from scipy.stats import norm as norm_dist
383
+ y = norm_dist.pdf(x, null_values_mean, null_values_std)
384
+ ax.fill_between(x, y, alpha=0.3, color="gray", label="Null distribution")
385
+ ax.plot(x, y, color="gray", linewidth=1.5)
386
+ ax.axvline(observed, color="red", linewidth=2.5, linestyle="-",
387
+ label=f"Observed = {observed:.4f}")
388
+ ax.set_xlabel("Spearman r (distance matrix correlation)")
389
+ ax.set_ylabel("Density")
390
+ ax.set_title(f"Mantel Test: p = {p_value:.4f}")
391
+ ax.legend()
392
+ plt.tight_layout()
393
+ plt.savefig(output_path, dpi=150, bbox_inches="tight")
394
+ plt.close()
395
+ logger.info("Saved: %s", output_path)
396
+
397
+
398
+ def save_retrieval_examples(
399
+ module_emb: np.ndarray,
400
+ text_emb: np.ndarray,
401
+ texts: List[str],
402
+ names: List[str],
403
+ output_path: str,
404
+ top_k: int = 5,
405
+ query_indices: Optional[List[int]] = None,
406
+ ):
407
+ """Save retrieval examples for selected modules."""
408
+ n = module_emb.shape[0]
409
+ scaler_x = StandardScaler()
410
+ scaler_y = StandardScaler()
411
+ X = scaler_x.fit_transform(module_emb)
412
+ Y = scaler_y.fit_transform(text_emb)
413
+
414
+ ridge = RidgeCV(alphas=np.logspace(-2, 4, 7))
415
+ ridge.fit(X, Y)
416
+
417
+ if query_indices is None:
418
+ rng = np.random.default_rng(0)
419
+ query_indices = sorted(rng.choice(n, size=min(20, n), replace=False))
420
+
421
+ rows = []
422
+ for qi in query_indices:
423
+ y_pred = ridge.predict(X[qi:qi+1])
424
+ sims = cosine_similarity(y_pred, Y)[0]
425
+ top_idx = np.argsort(-sims)[:top_k]
426
+ for rank, ti in enumerate(top_idx):
427
+ rows.append({
428
+ "query_module": names[qi],
429
+ "query_text": texts[qi],
430
+ "retrieved_rank": rank + 1,
431
+ "retrieved_text": texts[ti],
432
+ "similarity": float(sims[ti]),
433
+ "is_self": ti == qi,
434
+ })
435
+
436
+ df = pd.DataFrame(rows)
437
+ df.to_csv(output_path, index=False)
438
+ logger.info("Saved retrieval examples: %s", output_path)
439
+
440
+
441
+ # ============================================================
442
+ # 7. Main
443
+ # ============================================================
444
+
445
+ def parse_args() -> argparse.Namespace:
446
+ parser = argparse.ArgumentParser(
447
+ description=__doc__,
448
+ formatter_class=argparse.RawDescriptionHelpFormatter,
449
+ )
450
+ parser.add_argument("--module_embedding_path", type=str, required=True,
451
+ help="Path to module_embeddings.npy from experiment 1.")
452
+ parser.add_argument("--module_json", type=str,
453
+ default=str(REPO_ROOT / "model_dependencies" / "module_data_all" / "merged_modules.json"),
454
+ help="Path to merged_modules.json.")
455
+ parser.add_argument("--descriptions_file", type=str, default=None,
456
+ help="Optional JSON with extended pathway descriptions.")
457
+ parser.add_argument("--sentence_model", type=str,
458
+ default="pritamdeka/S-PubMedBert-MS-MARCO",
459
+ help="HuggingFace sentence-transformers model.")
460
+ parser.add_argument("--device", type=str, default="cpu",
461
+ help="Device for sentence encoder.")
462
+ parser.add_argument("--n_permutations", type=int, default=1000)
463
+ parser.add_argument("--seed", type=int, default=42)
464
+ parser.add_argument("--output_dir", type=str, required=True)
465
+ parser.add_argument("--skip_loo", action="store_true",
466
+ help="Skip LOO retrieval (slow). Use RSA/CKA only.")
467
+ parser.add_argument("--rsa_metric", type=str, default="cosine",
468
+ choices=["cosine", "euclidean"],
469
+ help="Distance metric for RSA.")
470
+ return parser.parse_args()
471
+
472
+
473
+ def main():
474
+ args = parse_args()
475
+
476
+ output_dir = Path(args.output_dir)
477
+ output_dir.mkdir(parents=True, exist_ok=True)
478
+
479
+ logging.basicConfig(
480
+ level=logging.INFO,
481
+ format="%(asctime)s [%(levelname)s] %(message)s",
482
+ handlers=[
483
+ logging.StreamHandler(),
484
+ logging.FileHandler(output_dir / "run.log", mode="w"),
485
+ ],
486
+ )
487
+
488
+ # --- Load module names and embeddings ---
489
+ logger.info("Loading module names from %s", args.module_json)
490
+ module_names = load_module_names(args.module_json)
491
+ logger.info("Total modules: %d", len(module_names))
492
+
493
+ logger.info("Loading module embeddings from %s", args.module_embedding_path)
494
+ all_embeddings = np.load(args.module_embedding_path)
495
+ assert all_embeddings.shape[0] == len(module_names), (
496
+ f"Embedding count {all_embeddings.shape[0]} != module count {len(module_names)}"
497
+ )
498
+
499
+ # --- Build text descriptions (Hallmarks + Reactome only) ---
500
+ indices, names, texts = build_text_descriptions(
501
+ module_names, args.descriptions_file
502
+ )
503
+ logger.info("Modules with text descriptions: %d / %d", len(indices), len(module_names))
504
+
505
+ module_emb = all_embeddings[indices]
506
+ logger.info("Module embedding shape: %s", module_emb.shape)
507
+
508
+ # --- Encode texts ---
509
+ text_emb = encode_texts(
510
+ texts, model_name=args.sentence_model, device=args.device
511
+ )
512
+ logger.info("Text embedding shape: %s", text_emb.shape)
513
+
514
+ # Save intermediate
515
+ np.save(output_dir / "text_embeddings.npy", text_emb)
516
+ np.save(output_dir / "module_embeddings_subset.npy", module_emb)
517
+ with open(output_dir / "pathway_descriptions.json", "w", encoding="utf-8") as f:
518
+ json.dump(
519
+ [{"index": idx, "module_name": name, "text": text}
520
+ for idx, name, text in zip(indices, names, texts)],
521
+ f, indent=2, ensure_ascii=False,
522
+ )
523
+
524
+ # --- RSA / Mantel test (PRIMARY METRIC) ---
525
+ logger.info("=" * 50)
526
+ logger.info("RSA / Mantel Test (primary alignment metric)")
527
+ logger.info("=" * 50)
528
+
529
+ sp_r, sp_p, pe_r, pe_p = compute_rsa(module_emb, text_emb, metric=args.rsa_metric)
530
+ logger.info("RSA (all modules): Spearman r=%.4f (p=%.2e), Pearson r=%.4f (p=%.2e)",
531
+ sp_r, sp_p, pe_r, pe_p)
532
+
533
+ mantel_results = mantel_test(
534
+ module_emb, text_emb, metric=args.rsa_metric,
535
+ n_perm=args.n_permutations, seed=args.seed,
536
+ )
537
+ logger.info("Mantel test: observed=%.4f, null=%.4f +/- %.4f, p=%.4f, z=%.1f",
538
+ mantel_results["rsa_spearman_observed"],
539
+ mantel_results["rsa_null_mean"],
540
+ mantel_results["rsa_null_std"],
541
+ mantel_results["rsa_p_value"],
542
+ mantel_results["rsa_effect_size_z"])
543
+
544
+ # RSA by family
545
+ family_rsa = compute_rsa_by_family(
546
+ module_emb, text_emb, names, metric=args.rsa_metric,
547
+ n_perm=min(args.n_permutations, 500), seed=args.seed,
548
+ )
549
+ for fam, res in family_rsa.items():
550
+ logger.info("RSA [%s]: r=%.4f, null=%.4f +/- %.4f, p=%.4f, z=%.1f",
551
+ fam, res["rsa_spearman_observed"], res["rsa_null_mean"],
552
+ res["rsa_null_std"], res["rsa_p_value"], res["rsa_effect_size_z"])
553
+
554
+ # --- CKA ---
555
+ logger.info("Computing CKA with %d permutations...", args.n_permutations)
556
+ cka_results = permutation_test_cka(
557
+ module_emb, text_emb, n_perm=args.n_permutations, seed=args.seed
558
+ )
559
+ logger.info("CKA: observed=%.4f, null=%.4f +/- %.4f, p=%.4f",
560
+ cka_results["cka_observed"], cka_results["cka_null_mean"],
561
+ cka_results["cka_null_std"], cka_results["cka_p_value"])
562
+
563
+ # --- LOO retrieval (optional) ---
564
+ retrieval_results = {}
565
+ ranks = None
566
+ if not args.skip_loo:
567
+ logger.info("Computing LOO retrieval metrics (n=%d)...", len(indices))
568
+ retrieval_results, ranks = compute_retrieval_metrics(module_emb, text_emb)
569
+ logger.info("Retrieval: recall@1=%.3f, @5=%.3f, @10=%.3f, @20=%.3f, MRR=%.3f, median_rank=%d",
570
+ retrieval_results.get("recall@1", 0),
571
+ retrieval_results.get("recall@5", 0),
572
+ retrieval_results.get("recall@10", 0),
573
+ retrieval_results.get("recall@20", 0),
574
+ retrieval_results.get("mrr", 0),
575
+ retrieval_results.get("median_rank", 0))
576
+
577
+ # --- Aggregate metrics ---
578
+ metrics = {
579
+ "n_modules_with_text": len(indices),
580
+ "n_modules_total": len(module_names),
581
+ "module_emb_dim": int(module_emb.shape[1]),
582
+ "text_emb_dim": int(text_emb.shape[1]),
583
+ "sentence_model": args.sentence_model,
584
+ "rsa_metric": args.rsa_metric,
585
+ "n_permutations": args.n_permutations,
586
+ "rsa_spearman_r": sp_r,
587
+ "rsa_spearman_p": sp_p,
588
+ "rsa_pearson_r": pe_r,
589
+ "rsa_pearson_p": pe_p,
590
+ **mantel_results,
591
+ "rsa_by_family": family_rsa,
592
+ **cka_results,
593
+ **retrieval_results,
594
+ }
595
+
596
+ with open(output_dir / "alignment_metrics.json", "w") as f:
597
+ json.dump(metrics, f, indent=2)
598
+ logger.info("Saved alignment_metrics.json")
599
+
600
+ # --- Plots ---
601
+ plot_rsa_scatter(
602
+ module_emb, text_emb, rsa_r=sp_r, rsa_p=mantel_results["rsa_p_value"],
603
+ output_path=str(output_dir / "rsa_scatter.png"),
604
+ metric=args.rsa_metric,
605
+ )
606
+
607
+ plot_rsa_null_distribution(
608
+ observed=mantel_results["rsa_spearman_observed"],
609
+ null_values_mean=mantel_results["rsa_null_mean"],
610
+ null_values_std=mantel_results["rsa_null_std"],
611
+ p_value=mantel_results["rsa_p_value"],
612
+ output_path=str(output_dir / "rsa_null_distribution.png"),
613
+ )
614
+
615
+ if ranks is not None:
616
+ plot_retrieval_rank_distribution(
617
+ ranks, n_total=len(indices),
618
+ output_path=str(output_dir / "retrieval_ranks.png"),
619
+ )
620
+
621
+ # --- Retrieval examples ---
622
+ representative_keywords = [
623
+ "interferon", "cell cycle", "oxidative", "apoptosis",
624
+ "immune", "signaling", "metabolism", "dna repair",
625
+ "nfkb", "mtor", "wnt", "hedgehog", "notch",
626
+ "interleukin", "toll", "mapk",
627
+ ]
628
+ query_indices = []
629
+ for kw in representative_keywords:
630
+ for i, t in enumerate(texts):
631
+ if kw.lower() in t.lower() and i not in query_indices:
632
+ query_indices.append(i)
633
+ break
634
+ if len(query_indices) < 10:
635
+ rng = np.random.default_rng(args.seed)
636
+ extra = rng.choice(len(texts), size=15, replace=False)
637
+ query_indices = list(set(query_indices) | set(extra.tolist()))[:20]
638
+
639
+ save_retrieval_examples(
640
+ module_emb, text_emb, texts, names,
641
+ output_path=str(output_dir / "retrieval_examples.csv"),
642
+ query_indices=sorted(query_indices),
643
+ )
644
+
645
+ # --- Summary ---
646
+ logger.info("=" * 60)
647
+ logger.info("EXPERIMENT 2 SUMMARY")
648
+ logger.info("=" * 60)
649
+ logger.info("Modules with text: %d / %d", len(indices), len(module_names))
650
+ logger.info("RSA (Mantel): Spearman r=%.4f, p=%.4f, z=%.1f sigma",
651
+ mantel_results["rsa_spearman_observed"],
652
+ mantel_results["rsa_p_value"],
653
+ mantel_results["rsa_effect_size_z"])
654
+ logger.info("CKA: %.4f (p=%.4f)", cka_results["cka_observed"], cka_results["cka_p_value"])
655
+ if retrieval_results:
656
+ logger.info("Retrieval: recall@1=%.3f, @5=%.3f, @10=%.3f, MRR=%.3f, median_rank=%d",
657
+ retrieval_results["recall@1"], retrieval_results["recall@5"],
658
+ retrieval_results["recall@10"], retrieval_results["mrr"],
659
+ retrieval_results["median_rank"])
660
+ logger.info("Output: %s", output_dir)
661
+ logger.info("=" * 60)
662
+
663
+
664
+ if __name__ == "__main__":
665
+ main()
BioMod/interpretability/module_text_alignment/module_text_alignment_extended.py CHANGED
@@ -1,537 +1,537 @@
1
- """Extended analysis for Experiment 2 — Module-Text Alignment.
2
-
3
- Three analyses:
4
- 1. Reactome category-level RSA (aggregate 491 pathways into ~16 top-level categories)
5
- 2. Reactome with official descriptions (via descriptions JSON)
6
- 3. Hallmarks deep dive (per-module alignment scores, heatmap)
7
-
8
- Usage:
9
- python module_text_alignment_extended.py \
10
- --module_embedding_path ../results/geometry/pretrain_subset/module_embeddings.npy \
11
- --module_json ../../module_data_all/merged_modules.json \
12
- --sentence_model ./S-PubMedBert-MS-MARCO \
13
- --output_dir ../results/text_alignment/pretrain_subset_v2_extended
14
- """
15
- from __future__ import annotations
16
-
17
- import argparse
18
- import json
19
- import logging
20
- import sys
21
- from collections import Counter, defaultdict
22
- from pathlib import Path
23
- from typing import Dict, List, Optional, Tuple
24
-
25
- import numpy as np
26
- import pandas as pd
27
- from scipy.spatial.distance import pdist, squareform
28
- from scipy.stats import spearmanr
29
- from sklearn.linear_model import RidgeCV
30
- from sklearn.metrics.pairwise import cosine_similarity
31
- from sklearn.preprocessing import StandardScaler
32
-
33
- import matplotlib
34
- matplotlib.use("Agg")
35
- import matplotlib.pyplot as plt
36
-
37
- HERE = Path(__file__).resolve().parent
38
- INTERP_DIR = HERE.parent
39
- PERFORMER_MODULE = INTERP_DIR.parent
40
- REPO_ROOT = PERFORMER_MODULE.parent
41
- for p in (str(INTERP_DIR), str(PERFORMER_MODULE), str(REPO_ROOT)):
42
- if p not in sys.path:
43
- sys.path.insert(0, p)
44
-
45
- from interpretability.utils import load_module_names, parse_module_family
46
- from module_text_alignment.module_text_alignment import (
47
- build_text_descriptions,
48
- compute_cka,
49
- compute_rsa,
50
- encode_texts,
51
- mantel_test,
52
- parse_pathway_text,
53
- )
54
-
55
- logger = logging.getLogger(__name__)
56
-
57
-
58
- # ============================================================
59
- # Reactome top-level category mapping
60
- # ============================================================
61
-
62
- REACTOME_CATEGORIES = {
63
- "Cell Cycle": ["cell cycle", "mitotic", "mitosis", "meiosis", "g1/s", "g2/m",
64
- "s phase", "m phase", "checkpoint", "cdc", "apc/c", "cyclin",
65
- "condensation of", "centrosome", "kinetochore", "spindle",
66
- "cohes", "pre-replicative", "separation of sister", "aurora"],
67
- "DNA Repair": ["dna repair", "base excision", "nucleotide excision",
68
- "mismatch repair", "homologous recombination", "non-homologous",
69
- "double-strand", "single-strand", "fanconi", "dna damage"],
70
- "DNA Replication": ["dna replication", "replication", "okazaki", "telomere"],
71
- "Signal Transduction": ["signaling", "signal transduction", "mapk", "ras",
72
- "raf", "mek", "erk", "pi3k", "akt", "mtor", "wnt",
73
- "notch", "hedgehog", "tgf-beta", "bmp", "fgfr",
74
- "egfr", "vegf", "pdgf", "igf", "insulin", "erbb",
75
- "met receptor", "receptor tyrosine", "gpcr",
76
- "g alpha", "rho gtpase", "rac1", "cdc42",
77
- "ca2+ pathway", "nfkb", "nf-kb", "nmda",
78
- "calcium", "pkc", "plc", "dag", "pip"],
79
- "Immune System": ["immune", "innate", "adaptive", "interferon", "interleukin",
80
- "cytokine", "complement", "toll", "antigen", "mhc", "tcr",
81
- "bcr", "fcr", "neutrophil", "phagocyt", "inflamm",
82
- "antimicrobial", "antiviral", "ifn-stimulated",
83
- "c-type lectin", "nod-like", "rig-i"],
84
- "Metabolism": ["metabolism", "metabolic", "glycolysis", "gluconeogenesis",
85
- "tca cycle", "citric acid", "oxidative phosphorylation",
86
- "fatty acid", "lipid", "cholesterol", "bile", "amino acid",
87
- "nucleotide", "purine", "pyrimidine", "pentose", "urea",
88
- "biological oxidation", "glycosylation", "biosynthesis",
89
- "synthesis of", "respiratory electron"],
90
- "Gene Expression": ["transcription", "translation", "mrna", "rrna", "trna",
91
- "splicing", "polyadenylation", "ribosom",
92
- "eukaryotic translation", "nonsense-mediated",
93
- "deadenylation", "rna polymerase", "processing of capped"],
94
- "Protein Homeostasis": ["ubiquitin", "proteasome", "sumo", "nedd",
95
- "autophagy", "chaperone", "unfolded protein",
96
- "protein folding", "hsp", "degradation of",
97
- "deubiquitin"],
98
- "Transport": ["transport", "vesicle", "endocyt", "exocyt", "golgi",
99
- "er to golgi", "membrane trafficking", "clathrin", "copi",
100
- "copii", "ion channel", "slc-mediated", "abc transporter",
101
- "nuclear pore", "nuclear import", "nuclear export", "cargo"],
102
- "Apoptosis": ["apoptosis", "apoptotic", "programmed cell death", "caspase",
103
- "death receptor", "bcl-2"],
104
- "Cell-Cell Communication": ["cell junction", "gap junction", "tight junction",
105
- "adherens", "cell-cell", "extracellular matrix",
106
- "integrin", "synapse", "neurotransmitter",
107
- "collagen", "laminin", "elastic fibre",
108
- "anchoring"],
109
- "Chromatin & Epigenetics": ["chromatin", "histone", "methylation",
110
- "acetylation", "deacetylation", "epigenetic",
111
- "polycomb", "hox"],
112
- "Hemostasis": ["hemostasis", "platelet", "coagulation", "fibrin", "thrombin"],
113
- "Developmental Biology": ["developmental", "axon", "neural", "muscle",
114
- "myogenesis", "keratinization", "differentiation",
115
- "cardiac conduction"],
116
- "Disease": ["disease", "hiv", "influenza", "sars", "hcmv", "herpes",
117
- "hepatitis", "cancer", "oncogene", "tumor", "amyloid"],
118
- "Cellular Responses": ["cellular response", "oxidative stress", "hypoxia",
119
- "heat shock", "senescence", "ferroptosis",
120
- "detoxification"],
121
- }
122
-
123
-
124
- def categorize_reactome(pathway_name: str) -> str:
125
- name_lower = pathway_name.lower()
126
- for cat, keywords in REACTOME_CATEGORIES.items():
127
- for kw in keywords:
128
- if kw in name_lower:
129
- return cat
130
- return "Other"
131
-
132
-
133
- # ============================================================
134
- # Analysis 1: Reactome category-level RSA
135
- # ============================================================
136
-
137
- def reactome_category_rsa(
138
- module_emb: np.ndarray,
139
- text_emb: np.ndarray,
140
- names: List[str],
141
- texts: List[str],
142
- output_dir: Path,
143
- sentence_model: str,
144
- device: str,
145
- n_perm: int = 1000,
146
- seed: int = 42,
147
- ) -> Dict:
148
- """Aggregate Reactome modules by top-level category, then do RSA."""
149
- logger.info("=" * 50)
150
- logger.info("Analysis 1: Reactome category-level RSA")
151
- logger.info("=" * 50)
152
-
153
- families = [parse_module_family(n) for n in names]
154
- reactome_mask = np.array([f == "Reactome" for f in families])
155
- reactome_idx = np.where(reactome_mask)[0]
156
-
157
- reactome_names = [names[i] for i in reactome_idx]
158
- reactome_texts = [texts[i] for i in reactome_idx]
159
- reactome_module_emb = module_emb[reactome_idx]
160
-
161
- cats = [categorize_reactome(t) for t in reactome_texts]
162
- cat_counts = Counter(cats)
163
- logger.info("Reactome category distribution:")
164
- for cat, count in sorted(cat_counts.items(), key=lambda x: -x[1]):
165
- logger.info(" %s: %d", cat, count)
166
-
167
- # Aggregate: mean embedding per category
168
- unique_cats = sorted(set(c for c in cats if c != "Other"))
169
- n_cats = len(unique_cats)
170
- logger.info("Categories (excl. Other): %d", n_cats)
171
-
172
- cat_module_emb = np.zeros((n_cats, module_emb.shape[1]))
173
- for i, cat in enumerate(unique_cats):
174
- mask = np.array([c == cat for c in cats])
175
- cat_module_emb[i] = reactome_module_emb[mask].mean(axis=0)
176
-
177
- # Encode category names as text
178
- cat_text_emb = encode_texts(unique_cats, model_name=sentence_model, device=device)
179
-
180
- # RSA on category-level
181
- sp_r, sp_p, pe_r, pe_p = compute_rsa(cat_module_emb, cat_text_emb, metric="cosine")
182
- logger.info("Category-level RSA: Spearman r=%.4f (p=%.4f), Pearson r=%.4f",
183
- sp_r, sp_p, pe_r)
184
-
185
- mantel = mantel_test(cat_module_emb, cat_text_emb, metric="cosine",
186
- n_perm=n_perm, seed=seed)
187
- logger.info("Mantel test: observed=%.4f, null=%.4f +/- %.4f, p=%.4f, z=%.1f",
188
- mantel["rsa_spearman_observed"], mantel["rsa_null_mean"],
189
- mantel["rsa_null_std"], mantel["rsa_p_value"],
190
- mantel["rsa_effect_size_z"])
191
-
192
- cka = compute_cka(cat_module_emb, cat_text_emb)
193
- logger.info("Category-level CKA: %.4f", cka)
194
-
195
- # Visualize: distance heatmaps
196
- fig, axes = plt.subplots(1, 2, figsize=(14, 6))
197
-
198
- dist_module = squareform(pdist(cat_module_emb, metric="cosine"))
199
- dist_text = squareform(pdist(cat_text_emb, metric="cosine"))
200
-
201
- im0 = axes[0].imshow(dist_module, cmap="viridis", aspect="auto")
202
- axes[0].set_xticks(range(n_cats))
203
- axes[0].set_xticklabels(unique_cats, rotation=45, ha="right", fontsize=7)
204
- axes[0].set_yticks(range(n_cats))
205
- axes[0].set_yticklabels(unique_cats, fontsize=7)
206
- axes[0].set_title("Module Embedding Distance (cosine)")
207
- plt.colorbar(im0, ax=axes[0], fraction=0.046)
208
-
209
- im1 = axes[1].imshow(dist_text, cmap="viridis", aspect="auto")
210
- axes[1].set_xticks(range(n_cats))
211
- axes[1].set_xticklabels(unique_cats, rotation=45, ha="right", fontsize=7)
212
- axes[1].set_yticks(range(n_cats))
213
- axes[1].set_yticklabels(unique_cats, fontsize=7)
214
- axes[1].set_title("Text Embedding Distance (cosine)")
215
- plt.colorbar(im1, ax=axes[1], fraction=0.046)
216
-
217
- plt.suptitle(f"Reactome Category-Level: RSA r={sp_r:.3f}, p={mantel['rsa_p_value']:.4f}")
218
- plt.tight_layout()
219
- plt.savefig(output_dir / "reactome_category_heatmaps.png", dpi=150, bbox_inches="tight")
220
- plt.close()
221
-
222
- results = {
223
- "n_categories": n_cats,
224
- "category_counts": dict(cat_counts),
225
- "rsa_spearman_r": sp_r,
226
- "rsa_spearman_p": sp_p,
227
- "rsa_pearson_r": pe_r,
228
- "mantel": mantel,
229
- "cka": cka,
230
- "categories": unique_cats,
231
- }
232
- return results
233
-
234
-
235
- # ============================================================
236
- # Analysis 2: Reactome with descriptions
237
- # ============================================================
238
-
239
- def reactome_with_descriptions(
240
- module_emb: np.ndarray,
241
- names: List[str],
242
- texts: List[str],
243
- descriptions_file: str,
244
- output_dir: Path,
245
- sentence_model: str,
246
- device: str,
247
- n_perm: int = 1000,
248
- seed: int = 42,
249
- ) -> Optional[Dict]:
250
- """Re-run RSA using extended descriptions for Reactome pathways."""
251
- if not descriptions_file or not Path(descriptions_file).exists():
252
- logger.info("No descriptions file provided, skipping Analysis 2.")
253
- return None
254
-
255
- logger.info("=" * 50)
256
- logger.info("Analysis 2: Reactome with official descriptions")
257
- logger.info("=" * 50)
258
-
259
- with open(descriptions_file, "r", encoding="utf-8") as f:
260
- descriptions = json.load(f)
261
-
262
- families = [parse_module_family(n) for n in names]
263
- reactome_mask = np.array([f == "Reactome" for f in families])
264
- reactome_idx = np.where(reactome_mask)[0]
265
-
266
- # Build enriched texts
267
- enriched_texts = []
268
- valid_idx = []
269
- for i in reactome_idx:
270
- pname = names[i]
271
- base_text = texts[i]
272
- if pname in descriptions and descriptions[pname]:
273
- enriched_texts.append(f"{base_text}: {descriptions[pname]}")
274
- valid_idx.append(i)
275
- else:
276
- enriched_texts.append(base_text)
277
- valid_idx.append(i)
278
-
279
- logger.info("Reactome modules with descriptions: %d / %d",
280
- sum(1 for i in reactome_idx if names[i] in descriptions),
281
- len(reactome_idx))
282
-
283
- sub_module_emb = module_emb[valid_idx]
284
- text_emb_enriched = encode_texts(enriched_texts, model_name=sentence_model, device=device)
285
-
286
- sp_r, sp_p, pe_r, pe_p = compute_rsa(sub_module_emb, text_emb_enriched, metric="cosine")
287
- logger.info("RSA with descriptions: Spearman r=%.4f (p=%.2e)", sp_r, sp_p)
288
-
289
- mantel = mantel_test(sub_module_emb, text_emb_enriched, metric="cosine",
290
- n_perm=n_perm, seed=seed)
291
- logger.info("Mantel: observed=%.4f, null=%.4f +/- %.4f, p=%.4f, z=%.1f",
292
- mantel["rsa_spearman_observed"], mantel["rsa_null_mean"],
293
- mantel["rsa_null_std"], mantel["rsa_p_value"],
294
- mantel["rsa_effect_size_z"])
295
-
296
- return {
297
- "rsa_spearman_r": sp_r,
298
- "rsa_spearman_p": sp_p,
299
- "mantel": mantel,
300
- }
301
-
302
-
303
- # ============================================================
304
- # Analysis 3: Hallmarks deep dive
305
- # ============================================================
306
-
307
- def hallmarks_deep_dive(
308
- module_emb: np.ndarray,
309
- text_emb: np.ndarray,
310
- names: List[str],
311
- texts: List[str],
312
- output_dir: Path,
313
- seed: int = 42,
314
- ) -> Dict:
315
- """Per-module alignment analysis for the 50 Hallmarks."""
316
- logger.info("=" * 50)
317
- logger.info("Analysis 3: Hallmarks deep dive")
318
- logger.info("=" * 50)
319
-
320
- families = [parse_module_family(n) for n in names]
321
- hallmark_mask = np.array([f == "Hallmarks" for f in families])
322
- hallmark_idx = np.where(hallmark_mask)[0]
323
-
324
- h_module = module_emb[hallmark_idx]
325
- h_text = text_emb[hallmark_idx]
326
- h_names = [names[i] for i in hallmark_idx]
327
- h_texts = [texts[i] for i in hallmark_idx]
328
- n_h = len(hallmark_idx)
329
-
330
- logger.info("Hallmarks: %d modules", n_h)
331
-
332
- # Per-module alignment: for each hallmark, compute cosine sim between
333
- # its module embedding's nearest neighbors and its text embedding's nearest neighbors
334
- # (neighborhood overlap)
335
- module_sim_matrix = cosine_similarity(h_module)
336
- text_sim_matrix = cosine_similarity(h_text)
337
-
338
- # Per-module: correlation between its similarity profile in module space vs text space
339
- per_module_corr = np.zeros(n_h)
340
- for i in range(n_h):
341
- mask = np.ones(n_h, dtype=bool)
342
- mask[i] = False
343
- r, _ = spearmanr(module_sim_matrix[i, mask], text_sim_matrix[i, mask])
344
- per_module_corr[i] = r
345
-
346
- # Sort by alignment quality
347
- sort_idx = np.argsort(-per_module_corr)
348
- logger.info("\nTop-10 best aligned Hallmarks:")
349
- for rank, i in enumerate(sort_idx[:10]):
350
- logger.info(" %2d. %s (r=%.3f)", rank+1, h_texts[i], per_module_corr[i])
351
-
352
- logger.info("\nBottom-10 worst aligned Hallmarks:")
353
- for rank, i in enumerate(sort_idx[-10:]):
354
- logger.info(" %2d. %s (r=%.3f)", rank+1, h_texts[i], per_module_corr[i])
355
-
356
- # Save per-module scores
357
- df = pd.DataFrame({
358
- "module_name": h_names,
359
- "text": h_texts,
360
- "alignment_r": per_module_corr,
361
- }).sort_values("alignment_r", ascending=False)
362
- df.to_csv(output_dir / "hallmarks_per_module_alignment.csv", index=False)
363
-
364
- # Heatmap: module similarity vs text similarity
365
- fig, axes = plt.subplots(1, 3, figsize=(18, 6))
366
-
367
- im0 = axes[0].imshow(module_sim_matrix[sort_idx][:, sort_idx], cmap="RdBu_r",
368
- vmin=-0.5, vmax=0.5, aspect="auto")
369
- axes[0].set_title("Module Embedding Similarity")
370
- axes[0].set_xticks(range(n_h))
371
- axes[0].set_xticklabels([h_texts[i][:15] for i in sort_idx], rotation=90, fontsize=5)
372
- axes[0].set_yticks(range(n_h))
373
- axes[0].set_yticklabels([h_texts[i][:15] for i in sort_idx], fontsize=5)
374
- plt.colorbar(im0, ax=axes[0], fraction=0.046)
375
-
376
- im1 = axes[1].imshow(text_sim_matrix[sort_idx][:, sort_idx], cmap="RdBu_r",
377
- vmin=-0.5, vmax=0.5, aspect="auto")
378
- axes[1].set_title("Text Embedding Similarity")
379
- axes[1].set_xticks(range(n_h))
380
- axes[1].set_xticklabels([h_texts[i][:15] for i in sort_idx], rotation=90, fontsize=5)
381
- axes[1].set_yticks([])
382
- plt.colorbar(im1, ax=axes[1], fraction=0.046)
383
-
384
- # Bar chart of per-module alignment
385
- colors = ["steelblue" if r > 0 else "salmon" for r in per_module_corr[sort_idx]]
386
- axes[2].barh(range(n_h), per_module_corr[sort_idx], color=colors)
387
- axes[2].set_yticks(range(n_h))
388
- axes[2].set_yticklabels([h_texts[i][:20] for i in sort_idx], fontsize=5)
389
- axes[2].set_xlabel("Spearman r (neighborhood alignment)")
390
- axes[2].set_title("Per-Hallmark Alignment Score")
391
- axes[2].axvline(0, color="black", linewidth=0.5)
392
-
393
- plt.tight_layout()
394
- plt.savefig(output_dir / "hallmarks_deep_dive.png", dpi=150, bbox_inches="tight")
395
- plt.close()
396
- logger.info("Saved hallmarks_deep_dive.png")
397
-
398
- # kNN overlap analysis
399
- k = 5
400
- module_knn = np.argsort(-module_sim_matrix, axis=1)[:, 1:k+1]
401
- text_knn = np.argsort(-text_sim_matrix, axis=1)[:, 1:k+1]
402
- knn_overlap = np.zeros(n_h)
403
- for i in range(n_h):
404
- overlap = len(set(module_knn[i]) & set(text_knn[i]))
405
- knn_overlap[i] = overlap / k
406
-
407
- logger.info("\nkNN overlap (k=%d): mean=%.3f, median=%.3f",
408
- k, knn_overlap.mean(), np.median(knn_overlap))
409
-
410
- # Permutation test for kNN overlap
411
- rng = np.random.default_rng(seed)
412
- n_perm = 1000
413
- null_overlaps = np.zeros(n_perm)
414
- for p in range(n_perm):
415
- perm = rng.permutation(n_h)
416
- text_knn_perm = text_knn[perm]
417
- overlaps_perm = np.zeros(n_h)
418
- for i in range(n_h):
419
- overlaps_perm[i] = len(set(module_knn[i]) & set(text_knn_perm[i])) / k
420
- null_overlaps[p] = overlaps_perm.mean()
421
-
422
- knn_p = float(np.mean(null_overlaps >= knn_overlap.mean()))
423
- logger.info("kNN overlap permutation test: observed=%.3f, null=%.3f +/- %.3f, p=%.4f",
424
- knn_overlap.mean(), null_overlaps.mean(), null_overlaps.std(), knn_p)
425
-
426
- results = {
427
- "n_hallmarks": n_h,
428
- "per_module_alignment_mean": float(per_module_corr.mean()),
429
- "per_module_alignment_median": float(np.median(per_module_corr)),
430
- "per_module_alignment_std": float(per_module_corr.std()),
431
- "n_positive_alignment": int((per_module_corr > 0).sum()),
432
- "n_negative_alignment": int((per_module_corr < 0).sum()),
433
- "knn_k": k,
434
- "knn_overlap_mean": float(knn_overlap.mean()),
435
- "knn_overlap_null_mean": float(null_overlaps.mean()),
436
- "knn_overlap_p": knn_p,
437
- }
438
- return results
439
-
440
-
441
- # ============================================================
442
- # Main
443
- # ============================================================
444
-
445
- def parse_args():
446
- parser = argparse.ArgumentParser(description=__doc__,
447
- formatter_class=argparse.RawDescriptionHelpFormatter)
448
- parser.add_argument("--module_embedding_path", type=str, required=True)
449
- parser.add_argument("--module_json", type=str,
450
- default=str(PERFORMER_MODULE / "module_data_all" / "merged_modules.json"))
451
- parser.add_argument("--descriptions_file", type=str, default=None,
452
- help="JSON with Reactome summaryText descriptions.")
453
- parser.add_argument("--sentence_model", type=str,
454
- default="pritamdeka/S-PubMedBert-MS-MARCO")
455
- parser.add_argument("--device", type=str, default="cpu")
456
- parser.add_argument("--n_permutations", type=int, default=1000)
457
- parser.add_argument("--seed", type=int, default=42)
458
- parser.add_argument("--output_dir", type=str, required=True)
459
- return parser.parse_args()
460
-
461
-
462
- def main():
463
- args = parse_args()
464
- output_dir = Path(args.output_dir)
465
- output_dir.mkdir(parents=True, exist_ok=True)
466
-
467
- logging.basicConfig(
468
- level=logging.INFO,
469
- format="%(asctime)s [%(levelname)s] %(message)s",
470
- handlers=[
471
- logging.StreamHandler(),
472
- logging.FileHandler(output_dir / "run_extended.log", mode="w"),
473
- ],
474
- )
475
-
476
- # Load data
477
- module_names = load_module_names(args.module_json)
478
- all_embeddings = np.load(args.module_embedding_path)
479
- indices, names, texts = build_text_descriptions(module_names)
480
- module_emb = all_embeddings[indices]
481
-
482
- # Load or compute text embeddings
483
- text_emb_path = output_dir.parent / "pretrain_subset_v2" / "text_embeddings.npy"
484
- if text_emb_path.exists():
485
- logger.info("Loading cached text embeddings from %s", text_emb_path)
486
- text_emb = np.load(text_emb_path)
487
- else:
488
- text_emb = encode_texts(texts, model_name=args.sentence_model, device=args.device)
489
-
490
- all_results = {}
491
-
492
- # Analysis 1: Reactome category-level
493
- cat_results = reactome_category_rsa(
494
- module_emb, text_emb, names, texts, output_dir,
495
- sentence_model=args.sentence_model, device=args.device,
496
- n_perm=args.n_permutations, seed=args.seed,
497
- )
498
- all_results["reactome_category_rsa"] = cat_results
499
-
500
- # Analysis 2: Reactome with descriptions
501
- desc_results = reactome_with_descriptions(
502
- module_emb, names, texts, args.descriptions_file, output_dir,
503
- sentence_model=args.sentence_model, device=args.device,
504
- n_perm=args.n_permutations, seed=args.seed,
505
- )
506
- if desc_results:
507
- all_results["reactome_with_descriptions"] = desc_results
508
-
509
- # Analysis 3: Hallmarks deep dive
510
- hallmark_results = hallmarks_deep_dive(
511
- module_emb, text_emb, names, texts, output_dir, seed=args.seed,
512
- )
513
- all_results["hallmarks_deep_dive"] = hallmark_results
514
-
515
- # Save all results
516
- with open(output_dir / "extended_metrics.json", "w") as f:
517
- json.dump(all_results, f, indent=2, default=str)
518
- logger.info("Saved extended_metrics.json")
519
-
520
- # Summary
521
- logger.info("=" * 60)
522
- logger.info("EXTENDED ANALYSIS SUMMARY")
523
- logger.info("=" * 60)
524
- logger.info("1. Reactome category-level RSA: r=%.4f, p=%.4f",
525
- cat_results["rsa_spearman_r"], cat_results["mantel"]["rsa_p_value"])
526
- logger.info("2. Hallmarks per-module alignment: mean r=%.3f, %d/%d positive",
527
- hallmark_results["per_module_alignment_mean"],
528
- hallmark_results["n_positive_alignment"],
529
- hallmark_results["n_hallmarks"])
530
- logger.info("3. Hallmarks kNN overlap: %.3f (null=%.3f, p=%.4f)",
531
- hallmark_results["knn_overlap_mean"],
532
- hallmark_results["knn_overlap_null_mean"],
533
- hallmark_results["knn_overlap_p"])
534
-
535
-
536
- if __name__ == "__main__":
537
- main()
 
1
+ """Extended analysis for Experiment 2 — Module-Text Alignment.
2
+
3
+ Three analyses:
4
+ 1. Reactome category-level RSA (aggregate 491 pathways into ~16 top-level categories)
5
+ 2. Reactome with official descriptions (via descriptions JSON)
6
+ 3. Hallmarks deep dive (per-module alignment scores, heatmap)
7
+
8
+ Usage (run from the BioMod repository root):
9
+ python BioMod/interpretability/module_text_alignment/module_text_alignment_extended.py \
10
+ --module_embedding_path outputs/interpretability/geometry/pretrain_subset/module_embeddings.npy \
11
+ --module_json model_dependencies/module_data_all/merged_modules.json \
12
+ --sentence_model pritamdeka/S-PubMedBert-MS-MARCO \
13
+ --output_dir outputs/interpretability/text_alignment/pretrain_subset_v2_extended
14
+ """
15
+ from __future__ import annotations
16
+
17
+ import argparse
18
+ import json
19
+ import logging
20
+ import sys
21
+ from collections import Counter, defaultdict
22
+ from pathlib import Path
23
+ from typing import Dict, List, Optional, Tuple
24
+
25
+ import numpy as np
26
+ import pandas as pd
27
+ from scipy.spatial.distance import pdist, squareform
28
+ from scipy.stats import spearmanr
29
+ from sklearn.linear_model import RidgeCV
30
+ from sklearn.metrics.pairwise import cosine_similarity
31
+ from sklearn.preprocessing import StandardScaler
32
+
33
+ import matplotlib
34
+ matplotlib.use("Agg")
35
+ import matplotlib.pyplot as plt
36
+
37
+ HERE = Path(__file__).resolve().parent
38
+ INTERP_DIR = HERE.parent
39
+ PERFORMER_MODULE = INTERP_DIR.parent
40
+ REPO_ROOT = PERFORMER_MODULE.parent
41
+ for p in (str(INTERP_DIR), str(PERFORMER_MODULE), str(REPO_ROOT)):
42
+ if p not in sys.path:
43
+ sys.path.insert(0, p)
44
+
45
+ from interpretability.utils import load_module_names, parse_module_family
46
+ from module_text_alignment.module_text_alignment import (
47
+ build_text_descriptions,
48
+ compute_cka,
49
+ compute_rsa,
50
+ encode_texts,
51
+ mantel_test,
52
+ parse_pathway_text,
53
+ )
54
+
55
+ logger = logging.getLogger(__name__)
56
+
57
+
58
+ # ============================================================
59
+ # Reactome top-level category mapping
60
+ # ============================================================
61
+
62
+ REACTOME_CATEGORIES = {
63
+ "Cell Cycle": ["cell cycle", "mitotic", "mitosis", "meiosis", "g1/s", "g2/m",
64
+ "s phase", "m phase", "checkpoint", "cdc", "apc/c", "cyclin",
65
+ "condensation of", "centrosome", "kinetochore", "spindle",
66
+ "cohes", "pre-replicative", "separation of sister", "aurora"],
67
+ "DNA Repair": ["dna repair", "base excision", "nucleotide excision",
68
+ "mismatch repair", "homologous recombination", "non-homologous",
69
+ "double-strand", "single-strand", "fanconi", "dna damage"],
70
+ "DNA Replication": ["dna replication", "replication", "okazaki", "telomere"],
71
+ "Signal Transduction": ["signaling", "signal transduction", "mapk", "ras",
72
+ "raf", "mek", "erk", "pi3k", "akt", "mtor", "wnt",
73
+ "notch", "hedgehog", "tgf-beta", "bmp", "fgfr",
74
+ "egfr", "vegf", "pdgf", "igf", "insulin", "erbb",
75
+ "met receptor", "receptor tyrosine", "gpcr",
76
+ "g alpha", "rho gtpase", "rac1", "cdc42",
77
+ "ca2+ pathway", "nfkb", "nf-kb", "nmda",
78
+ "calcium", "pkc", "plc", "dag", "pip"],
79
+ "Immune System": ["immune", "innate", "adaptive", "interferon", "interleukin",
80
+ "cytokine", "complement", "toll", "antigen", "mhc", "tcr",
81
+ "bcr", "fcr", "neutrophil", "phagocyt", "inflamm",
82
+ "antimicrobial", "antiviral", "ifn-stimulated",
83
+ "c-type lectin", "nod-like", "rig-i"],
84
+ "Metabolism": ["metabolism", "metabolic", "glycolysis", "gluconeogenesis",
85
+ "tca cycle", "citric acid", "oxidative phosphorylation",
86
+ "fatty acid", "lipid", "cholesterol", "bile", "amino acid",
87
+ "nucleotide", "purine", "pyrimidine", "pentose", "urea",
88
+ "biological oxidation", "glycosylation", "biosynthesis",
89
+ "synthesis of", "respiratory electron"],
90
+ "Gene Expression": ["transcription", "translation", "mrna", "rrna", "trna",
91
+ "splicing", "polyadenylation", "ribosom",
92
+ "eukaryotic translation", "nonsense-mediated",
93
+ "deadenylation", "rna polymerase", "processing of capped"],
94
+ "Protein Homeostasis": ["ubiquitin", "proteasome", "sumo", "nedd",
95
+ "autophagy", "chaperone", "unfolded protein",
96
+ "protein folding", "hsp", "degradation of",
97
+ "deubiquitin"],
98
+ "Transport": ["transport", "vesicle", "endocyt", "exocyt", "golgi",
99
+ "er to golgi", "membrane trafficking", "clathrin", "copi",
100
+ "copii", "ion channel", "slc-mediated", "abc transporter",
101
+ "nuclear pore", "nuclear import", "nuclear export", "cargo"],
102
+ "Apoptosis": ["apoptosis", "apoptotic", "programmed cell death", "caspase",
103
+ "death receptor", "bcl-2"],
104
+ "Cell-Cell Communication": ["cell junction", "gap junction", "tight junction",
105
+ "adherens", "cell-cell", "extracellular matrix",
106
+ "integrin", "synapse", "neurotransmitter",
107
+ "collagen", "laminin", "elastic fibre",
108
+ "anchoring"],
109
+ "Chromatin & Epigenetics": ["chromatin", "histone", "methylation",
110
+ "acetylation", "deacetylation", "epigenetic",
111
+ "polycomb", "hox"],
112
+ "Hemostasis": ["hemostasis", "platelet", "coagulation", "fibrin", "thrombin"],
113
+ "Developmental Biology": ["developmental", "axon", "neural", "muscle",
114
+ "myogenesis", "keratinization", "differentiation",
115
+ "cardiac conduction"],
116
+ "Disease": ["disease", "hiv", "influenza", "sars", "hcmv", "herpes",
117
+ "hepatitis", "cancer", "oncogene", "tumor", "amyloid"],
118
+ "Cellular Responses": ["cellular response", "oxidative stress", "hypoxia",
119
+ "heat shock", "senescence", "ferroptosis",
120
+ "detoxification"],
121
+ }
122
+
123
+
124
+ def categorize_reactome(pathway_name: str) -> str:
125
+ name_lower = pathway_name.lower()
126
+ for cat, keywords in REACTOME_CATEGORIES.items():
127
+ for kw in keywords:
128
+ if kw in name_lower:
129
+ return cat
130
+ return "Other"
131
+
132
+
133
+ # ============================================================
134
+ # Analysis 1: Reactome category-level RSA
135
+ # ============================================================
136
+
137
+ def reactome_category_rsa(
138
+ module_emb: np.ndarray,
139
+ text_emb: np.ndarray,
140
+ names: List[str],
141
+ texts: List[str],
142
+ output_dir: Path,
143
+ sentence_model: str,
144
+ device: str,
145
+ n_perm: int = 1000,
146
+ seed: int = 42,
147
+ ) -> Dict:
148
+ """Aggregate Reactome modules by top-level category, then do RSA."""
149
+ logger.info("=" * 50)
150
+ logger.info("Analysis 1: Reactome category-level RSA")
151
+ logger.info("=" * 50)
152
+
153
+ families = [parse_module_family(n) for n in names]
154
+ reactome_mask = np.array([f == "Reactome" for f in families])
155
+ reactome_idx = np.where(reactome_mask)[0]
156
+
157
+ reactome_names = [names[i] for i in reactome_idx]
158
+ reactome_texts = [texts[i] for i in reactome_idx]
159
+ reactome_module_emb = module_emb[reactome_idx]
160
+
161
+ cats = [categorize_reactome(t) for t in reactome_texts]
162
+ cat_counts = Counter(cats)
163
+ logger.info("Reactome category distribution:")
164
+ for cat, count in sorted(cat_counts.items(), key=lambda x: -x[1]):
165
+ logger.info(" %s: %d", cat, count)
166
+
167
+ # Aggregate: mean embedding per category
168
+ unique_cats = sorted(set(c for c in cats if c != "Other"))
169
+ n_cats = len(unique_cats)
170
+ logger.info("Categories (excl. Other): %d", n_cats)
171
+
172
+ cat_module_emb = np.zeros((n_cats, module_emb.shape[1]))
173
+ for i, cat in enumerate(unique_cats):
174
+ mask = np.array([c == cat for c in cats])
175
+ cat_module_emb[i] = reactome_module_emb[mask].mean(axis=0)
176
+
177
+ # Encode category names as text
178
+ cat_text_emb = encode_texts(unique_cats, model_name=sentence_model, device=device)
179
+
180
+ # RSA on category-level
181
+ sp_r, sp_p, pe_r, pe_p = compute_rsa(cat_module_emb, cat_text_emb, metric="cosine")
182
+ logger.info("Category-level RSA: Spearman r=%.4f (p=%.4f), Pearson r=%.4f",
183
+ sp_r, sp_p, pe_r)
184
+
185
+ mantel = mantel_test(cat_module_emb, cat_text_emb, metric="cosine",
186
+ n_perm=n_perm, seed=seed)
187
+ logger.info("Mantel test: observed=%.4f, null=%.4f +/- %.4f, p=%.4f, z=%.1f",
188
+ mantel["rsa_spearman_observed"], mantel["rsa_null_mean"],
189
+ mantel["rsa_null_std"], mantel["rsa_p_value"],
190
+ mantel["rsa_effect_size_z"])
191
+
192
+ cka = compute_cka(cat_module_emb, cat_text_emb)
193
+ logger.info("Category-level CKA: %.4f", cka)
194
+
195
+ # Visualize: distance heatmaps
196
+ fig, axes = plt.subplots(1, 2, figsize=(14, 6))
197
+
198
+ dist_module = squareform(pdist(cat_module_emb, metric="cosine"))
199
+ dist_text = squareform(pdist(cat_text_emb, metric="cosine"))
200
+
201
+ im0 = axes[0].imshow(dist_module, cmap="viridis", aspect="auto")
202
+ axes[0].set_xticks(range(n_cats))
203
+ axes[0].set_xticklabels(unique_cats, rotation=45, ha="right", fontsize=7)
204
+ axes[0].set_yticks(range(n_cats))
205
+ axes[0].set_yticklabels(unique_cats, fontsize=7)
206
+ axes[0].set_title("Module Embedding Distance (cosine)")
207
+ plt.colorbar(im0, ax=axes[0], fraction=0.046)
208
+
209
+ im1 = axes[1].imshow(dist_text, cmap="viridis", aspect="auto")
210
+ axes[1].set_xticks(range(n_cats))
211
+ axes[1].set_xticklabels(unique_cats, rotation=45, ha="right", fontsize=7)
212
+ axes[1].set_yticks(range(n_cats))
213
+ axes[1].set_yticklabels(unique_cats, fontsize=7)
214
+ axes[1].set_title("Text Embedding Distance (cosine)")
215
+ plt.colorbar(im1, ax=axes[1], fraction=0.046)
216
+
217
+ plt.suptitle(f"Reactome Category-Level: RSA r={sp_r:.3f}, p={mantel['rsa_p_value']:.4f}")
218
+ plt.tight_layout()
219
+ plt.savefig(output_dir / "reactome_category_heatmaps.png", dpi=150, bbox_inches="tight")
220
+ plt.close()
221
+
222
+ results = {
223
+ "n_categories": n_cats,
224
+ "category_counts": dict(cat_counts),
225
+ "rsa_spearman_r": sp_r,
226
+ "rsa_spearman_p": sp_p,
227
+ "rsa_pearson_r": pe_r,
228
+ "mantel": mantel,
229
+ "cka": cka,
230
+ "categories": unique_cats,
231
+ }
232
+ return results
233
+
234
+
235
+ # ============================================================
236
+ # Analysis 2: Reactome with descriptions
237
+ # ============================================================
238
+
239
+ def reactome_with_descriptions(
240
+ module_emb: np.ndarray,
241
+ names: List[str],
242
+ texts: List[str],
243
+ descriptions_file: str,
244
+ output_dir: Path,
245
+ sentence_model: str,
246
+ device: str,
247
+ n_perm: int = 1000,
248
+ seed: int = 42,
249
+ ) -> Optional[Dict]:
250
+ """Re-run RSA using extended descriptions for Reactome pathways."""
251
+ if not descriptions_file or not Path(descriptions_file).exists():
252
+ logger.info("No descriptions file provided, skipping Analysis 2.")
253
+ return None
254
+
255
+ logger.info("=" * 50)
256
+ logger.info("Analysis 2: Reactome with official descriptions")
257
+ logger.info("=" * 50)
258
+
259
+ with open(descriptions_file, "r", encoding="utf-8") as f:
260
+ descriptions = json.load(f)
261
+
262
+ families = [parse_module_family(n) for n in names]
263
+ reactome_mask = np.array([f == "Reactome" for f in families])
264
+ reactome_idx = np.where(reactome_mask)[0]
265
+
266
+ # Build enriched texts
267
+ enriched_texts = []
268
+ valid_idx = []
269
+ for i in reactome_idx:
270
+ pname = names[i]
271
+ base_text = texts[i]
272
+ if pname in descriptions and descriptions[pname]:
273
+ enriched_texts.append(f"{base_text}: {descriptions[pname]}")
274
+ valid_idx.append(i)
275
+ else:
276
+ enriched_texts.append(base_text)
277
+ valid_idx.append(i)
278
+
279
+ logger.info("Reactome modules with descriptions: %d / %d",
280
+ sum(1 for i in reactome_idx if names[i] in descriptions),
281
+ len(reactome_idx))
282
+
283
+ sub_module_emb = module_emb[valid_idx]
284
+ text_emb_enriched = encode_texts(enriched_texts, model_name=sentence_model, device=device)
285
+
286
+ sp_r, sp_p, pe_r, pe_p = compute_rsa(sub_module_emb, text_emb_enriched, metric="cosine")
287
+ logger.info("RSA with descriptions: Spearman r=%.4f (p=%.2e)", sp_r, sp_p)
288
+
289
+ mantel = mantel_test(sub_module_emb, text_emb_enriched, metric="cosine",
290
+ n_perm=n_perm, seed=seed)
291
+ logger.info("Mantel: observed=%.4f, null=%.4f +/- %.4f, p=%.4f, z=%.1f",
292
+ mantel["rsa_spearman_observed"], mantel["rsa_null_mean"],
293
+ mantel["rsa_null_std"], mantel["rsa_p_value"],
294
+ mantel["rsa_effect_size_z"])
295
+
296
+ return {
297
+ "rsa_spearman_r": sp_r,
298
+ "rsa_spearman_p": sp_p,
299
+ "mantel": mantel,
300
+ }
301
+
302
+
303
+ # ============================================================
304
+ # Analysis 3: Hallmarks deep dive
305
+ # ============================================================
306
+
307
+ def hallmarks_deep_dive(
308
+ module_emb: np.ndarray,
309
+ text_emb: np.ndarray,
310
+ names: List[str],
311
+ texts: List[str],
312
+ output_dir: Path,
313
+ seed: int = 42,
314
+ ) -> Dict:
315
+ """Per-module alignment analysis for the 50 Hallmarks."""
316
+ logger.info("=" * 50)
317
+ logger.info("Analysis 3: Hallmarks deep dive")
318
+ logger.info("=" * 50)
319
+
320
+ families = [parse_module_family(n) for n in names]
321
+ hallmark_mask = np.array([f == "Hallmarks" for f in families])
322
+ hallmark_idx = np.where(hallmark_mask)[0]
323
+
324
+ h_module = module_emb[hallmark_idx]
325
+ h_text = text_emb[hallmark_idx]
326
+ h_names = [names[i] for i in hallmark_idx]
327
+ h_texts = [texts[i] for i in hallmark_idx]
328
+ n_h = len(hallmark_idx)
329
+
330
+ logger.info("Hallmarks: %d modules", n_h)
331
+
332
+ # Per-module alignment: for each hallmark, compute cosine sim between
333
+ # its module embedding's nearest neighbors and its text embedding's nearest neighbors
334
+ # (neighborhood overlap)
335
+ module_sim_matrix = cosine_similarity(h_module)
336
+ text_sim_matrix = cosine_similarity(h_text)
337
+
338
+ # Per-module: correlation between its similarity profile in module space vs text space
339
+ per_module_corr = np.zeros(n_h)
340
+ for i in range(n_h):
341
+ mask = np.ones(n_h, dtype=bool)
342
+ mask[i] = False
343
+ r, _ = spearmanr(module_sim_matrix[i, mask], text_sim_matrix[i, mask])
344
+ per_module_corr[i] = r
345
+
346
+ # Sort by alignment quality
347
+ sort_idx = np.argsort(-per_module_corr)
348
+ logger.info("\nTop-10 best aligned Hallmarks:")
349
+ for rank, i in enumerate(sort_idx[:10]):
350
+ logger.info(" %2d. %s (r=%.3f)", rank+1, h_texts[i], per_module_corr[i])
351
+
352
+ logger.info("\nBottom-10 worst aligned Hallmarks:")
353
+ for rank, i in enumerate(sort_idx[-10:]):
354
+ logger.info(" %2d. %s (r=%.3f)", rank+1, h_texts[i], per_module_corr[i])
355
+
356
+ # Save per-module scores
357
+ df = pd.DataFrame({
358
+ "module_name": h_names,
359
+ "text": h_texts,
360
+ "alignment_r": per_module_corr,
361
+ }).sort_values("alignment_r", ascending=False)
362
+ df.to_csv(output_dir / "hallmarks_per_module_alignment.csv", index=False)
363
+
364
+ # Heatmap: module similarity vs text similarity
365
+ fig, axes = plt.subplots(1, 3, figsize=(18, 6))
366
+
367
+ im0 = axes[0].imshow(module_sim_matrix[sort_idx][:, sort_idx], cmap="RdBu_r",
368
+ vmin=-0.5, vmax=0.5, aspect="auto")
369
+ axes[0].set_title("Module Embedding Similarity")
370
+ axes[0].set_xticks(range(n_h))
371
+ axes[0].set_xticklabels([h_texts[i][:15] for i in sort_idx], rotation=90, fontsize=5)
372
+ axes[0].set_yticks(range(n_h))
373
+ axes[0].set_yticklabels([h_texts[i][:15] for i in sort_idx], fontsize=5)
374
+ plt.colorbar(im0, ax=axes[0], fraction=0.046)
375
+
376
+ im1 = axes[1].imshow(text_sim_matrix[sort_idx][:, sort_idx], cmap="RdBu_r",
377
+ vmin=-0.5, vmax=0.5, aspect="auto")
378
+ axes[1].set_title("Text Embedding Similarity")
379
+ axes[1].set_xticks(range(n_h))
380
+ axes[1].set_xticklabels([h_texts[i][:15] for i in sort_idx], rotation=90, fontsize=5)
381
+ axes[1].set_yticks([])
382
+ plt.colorbar(im1, ax=axes[1], fraction=0.046)
383
+
384
+ # Bar chart of per-module alignment
385
+ colors = ["steelblue" if r > 0 else "salmon" for r in per_module_corr[sort_idx]]
386
+ axes[2].barh(range(n_h), per_module_corr[sort_idx], color=colors)
387
+ axes[2].set_yticks(range(n_h))
388
+ axes[2].set_yticklabels([h_texts[i][:20] for i in sort_idx], fontsize=5)
389
+ axes[2].set_xlabel("Spearman r (neighborhood alignment)")
390
+ axes[2].set_title("Per-Hallmark Alignment Score")
391
+ axes[2].axvline(0, color="black", linewidth=0.5)
392
+
393
+ plt.tight_layout()
394
+ plt.savefig(output_dir / "hallmarks_deep_dive.png", dpi=150, bbox_inches="tight")
395
+ plt.close()
396
+ logger.info("Saved hallmarks_deep_dive.png")
397
+
398
+ # kNN overlap analysis
399
+ k = 5
400
+ module_knn = np.argsort(-module_sim_matrix, axis=1)[:, 1:k+1]
401
+ text_knn = np.argsort(-text_sim_matrix, axis=1)[:, 1:k+1]
402
+ knn_overlap = np.zeros(n_h)
403
+ for i in range(n_h):
404
+ overlap = len(set(module_knn[i]) & set(text_knn[i]))
405
+ knn_overlap[i] = overlap / k
406
+
407
+ logger.info("\nkNN overlap (k=%d): mean=%.3f, median=%.3f",
408
+ k, knn_overlap.mean(), np.median(knn_overlap))
409
+
410
+ # Permutation test for kNN overlap
411
+ rng = np.random.default_rng(seed)
412
+ n_perm = 1000
413
+ null_overlaps = np.zeros(n_perm)
414
+ for p in range(n_perm):
415
+ perm = rng.permutation(n_h)
416
+ text_knn_perm = text_knn[perm]
417
+ overlaps_perm = np.zeros(n_h)
418
+ for i in range(n_h):
419
+ overlaps_perm[i] = len(set(module_knn[i]) & set(text_knn_perm[i])) / k
420
+ null_overlaps[p] = overlaps_perm.mean()
421
+
422
+ knn_p = float(np.mean(null_overlaps >= knn_overlap.mean()))
423
+ logger.info("kNN overlap permutation test: observed=%.3f, null=%.3f +/- %.3f, p=%.4f",
424
+ knn_overlap.mean(), null_overlaps.mean(), null_overlaps.std(), knn_p)
425
+
426
+ results = {
427
+ "n_hallmarks": n_h,
428
+ "per_module_alignment_mean": float(per_module_corr.mean()),
429
+ "per_module_alignment_median": float(np.median(per_module_corr)),
430
+ "per_module_alignment_std": float(per_module_corr.std()),
431
+ "n_positive_alignment": int((per_module_corr > 0).sum()),
432
+ "n_negative_alignment": int((per_module_corr < 0).sum()),
433
+ "knn_k": k,
434
+ "knn_overlap_mean": float(knn_overlap.mean()),
435
+ "knn_overlap_null_mean": float(null_overlaps.mean()),
436
+ "knn_overlap_p": knn_p,
437
+ }
438
+ return results
439
+
440
+
441
+ # ============================================================
442
+ # Main
443
+ # ============================================================
444
+
445
+ def parse_args():
446
+ parser = argparse.ArgumentParser(description=__doc__,
447
+ formatter_class=argparse.RawDescriptionHelpFormatter)
448
+ parser.add_argument("--module_embedding_path", type=str, required=True)
449
+ parser.add_argument("--module_json", type=str,
450
+ default=str(REPO_ROOT / "model_dependencies" / "module_data_all" / "merged_modules.json"))
451
+ parser.add_argument("--descriptions_file", type=str, default=None,
452
+ help="JSON with Reactome summaryText descriptions.")
453
+ parser.add_argument("--sentence_model", type=str,
454
+ default="pritamdeka/S-PubMedBert-MS-MARCO")
455
+ parser.add_argument("--device", type=str, default="cpu")
456
+ parser.add_argument("--n_permutations", type=int, default=1000)
457
+ parser.add_argument("--seed", type=int, default=42)
458
+ parser.add_argument("--output_dir", type=str, required=True)
459
+ return parser.parse_args()
460
+
461
+
462
+ def main():
463
+ args = parse_args()
464
+ output_dir = Path(args.output_dir)
465
+ output_dir.mkdir(parents=True, exist_ok=True)
466
+
467
+ logging.basicConfig(
468
+ level=logging.INFO,
469
+ format="%(asctime)s [%(levelname)s] %(message)s",
470
+ handlers=[
471
+ logging.StreamHandler(),
472
+ logging.FileHandler(output_dir / "run_extended.log", mode="w"),
473
+ ],
474
+ )
475
+
476
+ # Load data
477
+ module_names = load_module_names(args.module_json)
478
+ all_embeddings = np.load(args.module_embedding_path)
479
+ indices, names, texts = build_text_descriptions(module_names)
480
+ module_emb = all_embeddings[indices]
481
+
482
+ # Load or compute text embeddings
483
+ text_emb_path = output_dir.parent / "pretrain_subset_v2" / "text_embeddings.npy"
484
+ if text_emb_path.exists():
485
+ logger.info("Loading cached text embeddings from %s", text_emb_path)
486
+ text_emb = np.load(text_emb_path)
487
+ else:
488
+ text_emb = encode_texts(texts, model_name=args.sentence_model, device=args.device)
489
+
490
+ all_results = {}
491
+
492
+ # Analysis 1: Reactome category-level
493
+ cat_results = reactome_category_rsa(
494
+ module_emb, text_emb, names, texts, output_dir,
495
+ sentence_model=args.sentence_model, device=args.device,
496
+ n_perm=args.n_permutations, seed=args.seed,
497
+ )
498
+ all_results["reactome_category_rsa"] = cat_results
499
+
500
+ # Analysis 2: Reactome with descriptions
501
+ desc_results = reactome_with_descriptions(
502
+ module_emb, names, texts, args.descriptions_file, output_dir,
503
+ sentence_model=args.sentence_model, device=args.device,
504
+ n_perm=args.n_permutations, seed=args.seed,
505
+ )
506
+ if desc_results:
507
+ all_results["reactome_with_descriptions"] = desc_results
508
+
509
+ # Analysis 3: Hallmarks deep dive
510
+ hallmark_results = hallmarks_deep_dive(
511
+ module_emb, text_emb, names, texts, output_dir, seed=args.seed,
512
+ )
513
+ all_results["hallmarks_deep_dive"] = hallmark_results
514
+
515
+ # Save all results
516
+ with open(output_dir / "extended_metrics.json", "w") as f:
517
+ json.dump(all_results, f, indent=2, default=str)
518
+ logger.info("Saved extended_metrics.json")
519
+
520
+ # Summary
521
+ logger.info("=" * 60)
522
+ logger.info("EXTENDED ANALYSIS SUMMARY")
523
+ logger.info("=" * 60)
524
+ logger.info("1. Reactome category-level RSA: r=%.4f, p=%.4f",
525
+ cat_results["rsa_spearman_r"], cat_results["mantel"]["rsa_p_value"])
526
+ logger.info("2. Hallmarks per-module alignment: mean r=%.3f, %d/%d positive",
527
+ hallmark_results["per_module_alignment_mean"],
528
+ hallmark_results["n_positive_alignment"],
529
+ hallmark_results["n_hallmarks"])
530
+ logger.info("3. Hallmarks kNN overlap: %.3f (null=%.3f, p=%.4f)",
531
+ hallmark_results["knn_overlap_mean"],
532
+ hallmark_results["knn_overlap_null_mean"],
533
+ hallmark_results["knn_overlap_p"])
534
+
535
+
536
+ if __name__ == "__main__":
537
+ main()
BioMod/interpretability/module_text_alignment/module_text_alignment_publication.py CHANGED
The diff for this file is too large to render. See raw diff
 
BioMod/models_performer_module.py CHANGED
The diff for this file is too large to render. See raw diff
 
BioMod/module_data_all/create_uncovered_modules.py CHANGED
@@ -1,146 +1,151 @@
1
- #!/usr/bin/env python
2
- # -*- coding: utf-8 -*-
3
- """
4
- Create additional modules for genes not covered by existing modules
5
- Ensure that all model genes can participate in training
6
-
7
- How to use:
8
- python create_uncovered_modules.py \
9
- --merged_modules module_data_all/merged_modules.json \
10
- --output module_data_all/merged_modules_full.json \
11
- --module_size 50
12
- """
13
-
14
- import json
15
- import os
16
- import csv
17
- import argparse
18
-
19
- def create_uncovered_modules(
20
- merged_modules_path,
21
- gene_list_path,
22
- gene_mapping_path,
23
- output_path,
24
- module_size=50,
25
- verbose=True
26
- ):
27
- """
28
- Create additional modules for uncovered genes
29
-
30
- Args:
31
- merged_modules_path: merged module file path
32
- gene_list_path: model gene list path
33
- gene_mapping_path: Gene mapping file path
34
- output_path: output file path
35
- module_size: the size of the new module
36
- verbose: whether to print detailed information
37
- """
38
- # Load model gene list
39
- with open(gene_list_path, 'r') as f:
40
- model_genes = set(line.strip() for line in f)
41
-
42
- # Load existing modules
43
- with open(merged_modules_path, 'r') as f:
44
- modules = json.load(f)
45
-
46
- # Load gene map (symbol -> ensembl)
47
- symbol_to_ens = {}
48
- ens_to_symbol = {}
49
- with open(gene_mapping_path, 'r') as f:
50
- reader = csv.DictReader(f)
51
- for row in reader:
52
- symbol = row.get('Approved symbol', '').strip()
53
- ens_id = row.get('Ensembl gene ID', '').strip()
54
- if symbol and ens_id:
55
- symbol_to_ens[symbol] = ens_id
56
- ens_to_symbol[ens_id] = symbol
57
-
58
- # Find genes that are covered
59
- covered_genes_ens = set()
60
- for module_name, genes in modules.items():
61
- for g in genes:
62
- ens_id = symbol_to_ens.get(g)
63
- if ens_id and ens_id in model_genes:
64
- covered_genes_ens.add(ens_id)
65
- elif g in model_genes:
66
- covered_genes_ens.add(g)
67
-
68
- # Find uncovered genes
69
- uncovered_genes = model_genes - covered_genes_ens
70
- uncovered_genes = sorted(list(uncovered_genes))
71
-
72
- if verbose:
73
- print(f"Total number of model genes: {len(model_genes)}")
74
- print(f"Genes covered: {len(covered_genes_ens)} ({len(covered_genes_ens)/len(model_genes)*100:.1f}%)")
75
- print(f"Genes not covered: {len(uncovered_genes)} ({len(uncovered_genes)/len(model_genes)*100:.1f}%)")
76
-
77
- # Create new modules for uncovered genes
78
- # Use Ensembl ID (because the model uses Ensembl ID)
79
- new_modules = {}
80
- num_new_modules = (len(uncovered_genes) + module_size - 1) // module_size
81
-
82
- for i in range(num_new_modules):
83
- start_idx = i * module_size
84
- end_idx = min((i + 1) * module_size, len(uncovered_genes))
85
- module_genes = uncovered_genes[start_idx:end_idx]
86
-
87
- # Try converting to symbol if there is a mapping, otherwise keep the Ensembl ID
88
- module_genes_final = []
89
- for g in module_genes:
90
- symbol = ens_to_symbol.get(g)
91
- if symbol:
92
- module_genes_final.append(symbol)
93
- else:
94
- module_genes_final.append(g)
95
-
96
- new_modules[f"uncovered_module_{i+1:04d}"] = module_genes_final
97
-
98
- if verbose:
99
- print(f"\nCreated {len(new_modules)} new modules (each approx. {module_size} genes)")
100
-
101
- # Merge existing modules and new modules
102
- merged_modules = {**modules, **new_modules}
103
-
104
- # save
105
- os.makedirs(os.path.dirname(output_path) if os.path.dirname(output_path) else '.', exist_ok=True)
106
- with open(output_path, 'w', encoding='utf-8') as f:
107
- json.dump(merged_modules, f, indent=2)
108
-
109
- if verbose:
110
- print(f"\nFinal result:")
111
- print(f" Original modules: {len(modules)}")
112
- print(f" New modules: {len(new_modules)}")
113
- print(f" Total modules: {len(merged_modules)}")
114
- print(f" Save to: {output_path}")
115
-
116
- return merged_modules
117
-
118
- def main():
119
- parser = argparse.ArgumentParser(description='Create modules for uncovered genes')
120
- parser.add_argument('--merged_modules', type=str,
121
- default='module_data_all/merged_modules.json',
122
- help='Path to merged modules JSON file')
123
- parser.add_argument('--gene_list', type=str,
124
- default='../selected_genes_27k.txt',
125
- help='Path to model gene list file')
126
- parser.add_argument('--gene_mapping', type=str,
127
- default='../gene2ens.csv',
128
- help='Path to gene mapping CSV file')
129
- parser.add_argument('--output', '-o', type=str,
130
- default='module_data_all/merged_modules_full.json',
131
- help='Output file path')
132
- parser.add_argument('--module_size', type=int, default=50,
133
- help='Size of new modules for uncovered genes')
134
-
135
- args = parser.parse_args()
136
-
137
- create_uncovered_modules(
138
- merged_modules_path=args.merged_modules,
139
- gene_list_path=args.gene_list,
140
- gene_mapping_path=args.gene_mapping,
141
- output_path=args.output,
142
- module_size=args.module_size
143
- )
144
-
145
- if __name__ == '__main__':
146
- main()
 
 
 
 
 
 
1
+ #!/usr/bin/env python
2
+ # -*- coding: utf-8 -*-
3
+ """
4
+ Create additional modules for genes not covered by existing modules
5
+ Ensure that all model genes can participate in training
6
+
7
+ Run from the repository root:
8
+ python BioMod/module_data_all/create_uncovered_modules.py \
9
+ --merged_modules model_dependencies/module_data_all/merged_modules.json \
10
+ --output tutorial_runs/module_data/merged_modules_full.json \
11
+ --module_size 50
12
+ """
13
+
14
+ import json
15
+ import os
16
+ import csv
17
+ import argparse
18
+ from pathlib import Path
19
+
20
+ PROJECT_ROOT = Path(__file__).resolve().parents[2]
21
+ ASSET_ROOT = PROJECT_ROOT / 'model_dependencies'
22
+ MODULE_DIR = Path(__file__).resolve().parent
23
+
24
+ def create_uncovered_modules(
25
+ merged_modules_path,
26
+ gene_list_path,
27
+ gene_mapping_path,
28
+ output_path,
29
+ module_size=50,
30
+ verbose=True
31
+ ):
32
+ """
33
+ Create additional modules for uncovered genes
34
+
35
+ Args:
36
+ merged_modules_path: merged module file path
37
+ gene_list_path: model gene list path
38
+ gene_mapping_path: Gene mapping file path
39
+ output_path: output file path
40
+ module_size: the size of the new module
41
+ verbose: whether to print detailed information
42
+ """
43
+ # Load model gene list
44
+ with open(gene_list_path, 'r') as f:
45
+ model_genes = set(line.strip() for line in f)
46
+
47
+ # Load existing modules
48
+ with open(merged_modules_path, 'r') as f:
49
+ modules = json.load(f)
50
+
51
+ # Load gene map (symbol -> ensembl)
52
+ symbol_to_ens = {}
53
+ ens_to_symbol = {}
54
+ with open(gene_mapping_path, 'r') as f:
55
+ reader = csv.DictReader(f)
56
+ for row in reader:
57
+ symbol = row.get('Approved symbol', '').strip()
58
+ ens_id = row.get('Ensembl gene ID', '').strip()
59
+ if symbol and ens_id:
60
+ symbol_to_ens[symbol] = ens_id
61
+ ens_to_symbol[ens_id] = symbol
62
+
63
+ # Find genes that are covered
64
+ covered_genes_ens = set()
65
+ for module_name, genes in modules.items():
66
+ for g in genes:
67
+ ens_id = symbol_to_ens.get(g)
68
+ if ens_id and ens_id in model_genes:
69
+ covered_genes_ens.add(ens_id)
70
+ elif g in model_genes:
71
+ covered_genes_ens.add(g)
72
+
73
+ # Find uncovered genes
74
+ uncovered_genes = model_genes - covered_genes_ens
75
+ uncovered_genes = sorted(list(uncovered_genes))
76
+
77
+ if verbose:
78
+ print(f"Total number of model genes: {len(model_genes)}")
79
+ print(f"Genes covered: {len(covered_genes_ens)} ({len(covered_genes_ens)/len(model_genes)*100:.1f}%)")
80
+ print(f"Genes not covered: {len(uncovered_genes)} ({len(uncovered_genes)/len(model_genes)*100:.1f}%)")
81
+
82
+ # Create new modules for uncovered genes
83
+ # Use Ensembl ID (because the model uses Ensembl ID)
84
+ new_modules = {}
85
+ num_new_modules = (len(uncovered_genes) + module_size - 1) // module_size
86
+
87
+ for i in range(num_new_modules):
88
+ start_idx = i * module_size
89
+ end_idx = min((i + 1) * module_size, len(uncovered_genes))
90
+ module_genes = uncovered_genes[start_idx:end_idx]
91
+
92
+ # Try converting to symbol if there is a mapping, otherwise keep the Ensembl ID
93
+ module_genes_final = []
94
+ for g in module_genes:
95
+ symbol = ens_to_symbol.get(g)
96
+ if symbol:
97
+ module_genes_final.append(symbol)
98
+ else:
99
+ module_genes_final.append(g)
100
+
101
+ new_modules[f"uncovered_module_{i+1:04d}"] = module_genes_final
102
+
103
+ if verbose:
104
+ print(f"\nCreated {len(new_modules)} new modules (each approx. {module_size} genes)")
105
+
106
+ # Merge existing modules and new modules
107
+ merged_modules = {**modules, **new_modules}
108
+
109
+ # save
110
+ os.makedirs(os.path.dirname(output_path) if os.path.dirname(output_path) else '.', exist_ok=True)
111
+ with open(output_path, 'w', encoding='utf-8') as f:
112
+ json.dump(merged_modules, f, indent=2)
113
+
114
+ if verbose:
115
+ print(f"\nFinal result:")
116
+ print(f" Original modules: {len(modules)}")
117
+ print(f" New modules: {len(new_modules)}")
118
+ print(f" Total modules: {len(merged_modules)}")
119
+ print(f" Save to: {output_path}")
120
+
121
+ return merged_modules
122
+
123
+ def main():
124
+ parser = argparse.ArgumentParser(description='Create modules for uncovered genes')
125
+ parser.add_argument('--merged_modules', type=str,
126
+ default=str(MODULE_DIR / 'merged_modules.json'),
127
+ help='Path to merged modules JSON file')
128
+ parser.add_argument('--gene_list', type=str,
129
+ default=str(ASSET_ROOT / 'selected_genes_27k.txt'),
130
+ help='Path to model gene list file')
131
+ parser.add_argument('--gene_mapping', type=str,
132
+ default=str(ASSET_ROOT / 'gene2ens.csv'),
133
+ help='Path to gene mapping CSV file')
134
+ parser.add_argument('--output', '-o', type=str,
135
+ default=str(MODULE_DIR / 'merged_modules_full.json'),
136
+ help='Output file path')
137
+ parser.add_argument('--module_size', type=int, default=50,
138
+ help='Size of new modules for uncovered genes')
139
+
140
+ args = parser.parse_args()
141
+
142
+ create_uncovered_modules(
143
+ merged_modules_path=args.merged_modules,
144
+ gene_list_path=args.gene_list,
145
+ gene_mapping_path=args.gene_mapping,
146
+ output_path=args.output,
147
+ module_size=args.module_size
148
+ )
149
+
150
+ if __name__ == '__main__':
151
+ main()
BioMod/module_data_all/merge_modules.py CHANGED
@@ -1,96 +1,96 @@
1
- #!/usr/bin/env python
2
- # -*- coding: utf-8 -*-
3
- """
4
- Merge multiple module JSON files into one file
5
- Used to combine module data from different sources for training
6
-
7
- How to use:
8
- python merge_modules.py --output module_data_all/merged_modules.json
9
- """
10
-
11
- import json
12
- import os
13
- import argparse
14
-
15
- def merge_module_files(input_files, output_file, verbose=True):
16
- """
17
- Merge multiple module JSON files
18
-
19
- Args:
20
- input_files: list of input file paths
21
- output_file: output file path
22
- verbose: whether to print detailed information
23
- """
24
- merged_modules = {}
25
- total_modules = 0
26
- all_genes = set()
27
-
28
- for input_file in input_files:
29
- if not os.path.exists(input_file):
30
- print(f"Warning: File not found: {input_file}")
31
- continue
32
-
33
- with open(input_file, 'r', encoding='utf-8') as f:
34
- data = json.load(f)
35
-
36
- file_modules = 0
37
- file_genes = set()
38
-
39
- # Get the file name as a prefix to avoid key conflicts
40
- basename = os.path.splitext(os.path.basename(input_file))[0]
41
-
42
- for key, genes in data.items():
43
- if isinstance(genes, (list, tuple)):
44
- # Add file name prefix to avoid key conflicts
45
- new_key = f"{basename}__{key}"
46
- merged_modules[new_key] = list(genes)
47
- file_modules += 1
48
- file_genes.update(genes)
49
-
50
- total_modules += file_modules
51
- all_genes.update(file_genes)
52
-
53
- if verbose:
54
- print(f"{os.path.basename(input_file)}:")
55
- print(f" Modules: {file_modules}")
56
- print(f" Unique genes: {len(file_genes)}")
57
-
58
- # Save the merged file
59
- os.makedirs(os.path.dirname(output_file) if os.path.dirname(output_file) else '.', exist_ok=True)
60
- with open(output_file, 'w', encoding='utf-8') as f:
61
- json.dump(merged_modules, f, indent=2)
62
-
63
- if verbose:
64
- print("=" * 50)
65
- print(f"Total modules: {total_modules}")
66
- print(f"Total unique genes: {len(all_genes)}")
67
- print(f"Saved to: {output_file}")
68
-
69
- return merged_modules
70
-
71
- def main():
72
- parser = argparse.ArgumentParser(description='Merge multiple module JSON files')
73
- parser.add_argument('--input', '-i', nargs='+',
74
- default=[
75
- 'module_data/module1_hallmarks.json',
76
- 'module_data/module2_reactome_2022_filtered.json',
77
- 'module_data_2/module3_hotspot_260k.json',
78
- # Use absolute paths when adding external files
79
- ],
80
- help='Input JSON files to merge')
81
- parser.add_argument('--output', '-o',
82
- default='module_data_all/merged_modules.json',
83
- help='Output merged JSON file')
84
- parser.add_argument('--add-external', type=str, default=None,
85
- help='Additional external module file to include')
86
-
87
- args = parser.parse_args()
88
-
89
- input_files = args.input
90
- if args.add_external:
91
- input_files.append(args.add_external)
92
-
93
- merge_module_files(input_files, args.output)
94
-
95
- if __name__ == '__main__':
96
- main()
 
1
+ #!/usr/bin/env python
2
+ # -*- coding: utf-8 -*-
3
+ """Merge explicitly selected module JSON files.
4
+
5
+ Run from the repository root, choosing the module sources for your experiment:
6
+ python BioMod/module_data_all/merge_modules.py --input first.json second.json \
7
+ --output tutorial_runs/module_data/merged_modules.json
8
+
9
+ The example input filenames are placeholders for your module definitions.
10
+ """
11
+
12
+ import json
13
+ import os
14
+ import argparse
15
+ from pathlib import Path
16
+
17
+ PROJECT_ROOT = Path(__file__).resolve().parents[2]
18
+ ASSET_ROOT = PROJECT_ROOT / 'model_dependencies'
19
+ MODULE_DIR = Path(__file__).resolve().parent
20
+
21
+ def merge_module_files(input_files, output_file, verbose=True):
22
+ """
23
+ Merge multiple module JSON files
24
+
25
+ Args:
26
+ input_files: list of input file paths
27
+ output_file: output file path
28
+ verbose: whether to print detailed information
29
+ """
30
+ merged_modules = {}
31
+ total_modules = 0
32
+ all_genes = set()
33
+
34
+ for input_file in input_files:
35
+ if not os.path.exists(input_file):
36
+ print(f"Warning: File not found: {input_file}")
37
+ continue
38
+
39
+ with open(input_file, 'r', encoding='utf-8') as f:
40
+ data = json.load(f)
41
+
42
+ file_modules = 0
43
+ file_genes = set()
44
+
45
+ # Get the file name as a prefix to avoid key conflicts
46
+ basename = os.path.splitext(os.path.basename(input_file))[0]
47
+
48
+ for key, genes in data.items():
49
+ if isinstance(genes, (list, tuple)):
50
+ # Add file name prefix to avoid key conflicts
51
+ new_key = f"{basename}__{key}"
52
+ merged_modules[new_key] = list(genes)
53
+ file_modules += 1
54
+ file_genes.update(genes)
55
+
56
+ total_modules += file_modules
57
+ all_genes.update(file_genes)
58
+
59
+ if verbose:
60
+ print(f"{os.path.basename(input_file)}:")
61
+ print(f" Modules: {file_modules}")
62
+ print(f" Unique genes: {len(file_genes)}")
63
+
64
+ # Save the merged file
65
+ os.makedirs(os.path.dirname(output_file) if os.path.dirname(output_file) else '.', exist_ok=True)
66
+ with open(output_file, 'w', encoding='utf-8') as f:
67
+ json.dump(merged_modules, f, indent=2)
68
+
69
+ if verbose:
70
+ print("=" * 50)
71
+ print(f"Total modules: {total_modules}")
72
+ print(f"Total unique genes: {len(all_genes)}")
73
+ print(f"Saved to: {output_file}")
74
+
75
+ return merged_modules
76
+
77
+ def main():
78
+ parser = argparse.ArgumentParser(description='Merge multiple module JSON files')
79
+ parser.add_argument('--input', '-i', nargs='+', required=True,
80
+ help='Explicit input module JSON files; historical source collections are not all bundled.')
81
+ parser.add_argument('--output', '-o',
82
+ default=str(MODULE_DIR / 'merged_modules.json'),
83
+ help='Output merged JSON file')
84
+ parser.add_argument('--add-external', type=str, default=None,
85
+ help='Additional external module file to include')
86
+
87
+ args = parser.parse_args()
88
+
89
+ input_files = args.input
90
+ if args.add_external:
91
+ input_files.append(args.add_external)
92
+
93
+ merge_module_files(input_files, args.output)
94
+
95
+ if __name__ == '__main__':
96
+ main()
BioMod/performer_module.py CHANGED
@@ -8,7 +8,7 @@
8
  Main features:
9
  1. Use module data input (refer to MAE)
10
  2. Mask the module (refer to MAE’s random_masking)
11
- 3. Only use encoder (no decoder)
12
  4. Use Performer as encoder (instead of standard Transformer Block)
13
 
14
  ================================================================================
@@ -56,7 +56,13 @@
56
  from torch.nn.parallel import DistributedDataParallel as DDP
57
  import torch.distributed as dist
58
  import sys
59
- sys.path.append('../')
 
 
 
 
 
 
60
  from models_performer_module import (
61
  PerformerEncoderOnly_Module,
62
  sanitize_performer_module_config,
@@ -82,7 +88,7 @@
82
  parser = argparse.ArgumentParser()
83
  parser.add_argument("--local-rank", type=int, default=-1, help='Local process rank.')
84
  parser.add_argument("--gene_list_path", type=str,
85
- default = "../selected_genes_27k.txt")
86
  parser.add_argument("--gene_num", type=int, default=27874, help='Number of genes.')
87
  parser.add_argument("--cell_num", type=int, default=49738024, help='Number of cells.')
88
  parser.add_argument("--epoch", type=int, default=100, help='Number of epochs.')
@@ -118,11 +124,10 @@
118
  parser.add_argument("--dec_depth", type=int, default=4, help='Decoder layer number (valid when performer/cross_attn)')
119
  parser.add_argument("--dec_heads", type=int, default=8, help='Number of decoder attention heads (valid when performer/cross_attn)')
120
  parser.add_argument("--dec_dim_head", type=int, default=64, help='Performer decoder dim per head (ignored when cross_attn)')
121
- parser.add_argument("--ckpt_dir", type=str, default='./ckpts_performer_module/', help='Directory of checkpoint to save.')
122
  parser.add_argument("--model_name", type=str, default='performer_module_27k', help='Pretrained model name.')
123
- parser.add_argument("--data_path", type=str,
124
- default='./cell_files',
125
- help='Path of data for pretraining. When npz/txt is the directory, when h5/h5ad is the file path.')
126
  parser.add_argument("--data_format", type=str, default="npz", choices=["npz", "h5", "h5ad", "txt"],
127
  help='Pre-training data format: npz=one npz per cell; h5=single HDF5 file; h5ad=AnnData file; txt=one txt per cell')
128
  parser.add_argument("--h5_dataset_key", type=str, default="expression",
@@ -130,15 +135,15 @@
130
  # Module related parameters
131
  parser.add_argument("--module_data_path", type=str, default=None, help='Path to module JSON file (single file).')
132
  parser.add_argument("--module_data_dir", type=str, default=None, help='Path to directory containing module JSON files.')
133
- parser.add_argument("--gene_mapping_path", type=str, default='../gene2ens.csv', help='Path to gene symbol to Ensembl ID mapping CSV.')
134
- parser.add_argument("--gene2vec_data_path", type=str, default='../selected_gene2vec_27k.npy')
135
  parser.add_argument("--fusion_method", type=str, default="multiply",
136
  choices=["multiply", "add", "gated_residual"],
137
  help='Fusion method: "multiply"=multiply; "add"=add to gene2vec after expressing MLP; "gated_residual"=inject expression information into the gated residual')
138
  parser.add_argument("-use_wandb", action='store_true', default=False)
139
  parser.add_argument("-compile", action='store_true')
140
  parser.add_argument("--model_path", type=str, default='None',
141
- help='Directory of checkpoint to load.')
142
  parser.add_argument(
143
  "--model_config_path",
144
  type=str,
@@ -963,8 +968,9 @@ def any_rank_true(local_flag: bool) -> bool:
963
  wandb.log(wandb_payload)# train every LOG_EVERY step train_log_step train_loss, learning_rate
964
  train_log_step += 1
965
  if args.latest_ckpt_every > 0 and (index % args.latest_ckpt_every == 0):
966
- os.makedirs('./latest_checkpoint_performer_module/', exist_ok=True)
967
- save_scaler_ckpt(i, model, optimizer, scheduler, scaler, loss.item(), model_name, './latest_checkpoint_performer_module/')
 
968
 
969
  running_loss += loss.item()
970
 
 
8
  Main features:
9
  1. Use module data input (refer to MAE)
10
  2. Mask the module (refer to MAE’s random_masking)
11
+ 3. Use a configurable reconstruction decoder or direct prediction heads
12
  4. Use Performer as encoder (instead of standard Transformer Block)
13
 
14
  ================================================================================
 
56
  from torch.nn.parallel import DistributedDataParallel as DDP
57
  import torch.distributed as dist
58
  import sys
59
+ from pathlib import Path
60
+
61
+ SOURCE_DIR = Path(__file__).resolve().parent
62
+ PROJECT_ROOT = SOURCE_DIR.parent
63
+ ASSET_ROOT = PROJECT_ROOT / 'model_dependencies'
64
+ if str(SOURCE_DIR) not in sys.path:
65
+ sys.path.insert(0, str(SOURCE_DIR))
66
  from models_performer_module import (
67
  PerformerEncoderOnly_Module,
68
  sanitize_performer_module_config,
 
88
  parser = argparse.ArgumentParser()
89
  parser.add_argument("--local-rank", type=int, default=-1, help='Local process rank.')
90
  parser.add_argument("--gene_list_path", type=str,
91
+ default=str(ASSET_ROOT / 'selected_genes_27k.txt'))
92
  parser.add_argument("--gene_num", type=int, default=27874, help='Number of genes.')
93
  parser.add_argument("--cell_num", type=int, default=49738024, help='Number of cells.')
94
  parser.add_argument("--epoch", type=int, default=100, help='Number of epochs.')
 
124
  parser.add_argument("--dec_depth", type=int, default=4, help='Decoder layer number (valid when performer/cross_attn)')
125
  parser.add_argument("--dec_heads", type=int, default=8, help='Number of decoder attention heads (valid when performer/cross_attn)')
126
  parser.add_argument("--dec_dim_head", type=int, default=64, help='Performer decoder dim per head (ignored when cross_attn)')
127
+ parser.add_argument("--ckpt_dir", type=str, default=str(PROJECT_ROOT / 'tutorial_runs' / 'pretraining'), help='Directory for generated checkpoints; user relative paths use the current working directory.')
128
  parser.add_argument("--model_name", type=str, default='performer_module_27k', help='Pretrained model name.')
129
+ parser.add_argument("--data_path", type=str, required=True,
130
+ help='Pretraining input: a directory for npz/txt or a file for h5/h5ad. Supply your own pretraining data; it is not bundled.')
 
131
  parser.add_argument("--data_format", type=str, default="npz", choices=["npz", "h5", "h5ad", "txt"],
132
  help='Pre-training data format: npz=one npz per cell; h5=single HDF5 file; h5ad=AnnData file; txt=one txt per cell')
133
  parser.add_argument("--h5_dataset_key", type=str, default="expression",
 
135
  # Module related parameters
136
  parser.add_argument("--module_data_path", type=str, default=None, help='Path to module JSON file (single file).')
137
  parser.add_argument("--module_data_dir", type=str, default=None, help='Path to directory containing module JSON files.')
138
+ parser.add_argument("--gene_mapping_path", type=str, default=str(ASSET_ROOT / 'gene2ens.csv'), help='Path to gene symbol to Ensembl ID mapping CSV.')
139
+ parser.add_argument("--gene2vec_data_path", type=str, default=str(ASSET_ROOT / 'selected_gene2vec_27k.npy'))
140
  parser.add_argument("--fusion_method", type=str, default="multiply",
141
  choices=["multiply", "add", "gated_residual"],
142
  help='Fusion method: "multiply"=multiply; "add"=add to gene2vec after expressing MLP; "gated_residual"=inject expression information into the gated residual')
143
  parser.add_argument("-use_wandb", action='store_true', default=False)
144
  parser.add_argument("-compile", action='store_true')
145
  parser.add_argument("--model_path", type=str, default='None',
146
+ help='Checkpoint file to load; use its matching configuration for continued pretraining.')
147
  parser.add_argument(
148
  "--model_config_path",
149
  type=str,
 
968
  wandb.log(wandb_payload)# train every LOG_EVERY step train_log_step train_loss, learning_rate
969
  train_log_step += 1
970
  if args.latest_ckpt_every > 0 and (index % args.latest_ckpt_every == 0):
971
+ latest_ckpt_dir = os.path.join(ckpt_dir, 'latest')
972
+ os.makedirs(latest_ckpt_dir, exist_ok=True)
973
+ save_scaler_ckpt(i, model, optimizer, scheduler, scaler, loss.item(), model_name, latest_ckpt_dir)
974
 
975
  running_loss += loss.item()
976
 
BioMod/zero_shot_batch/README.md CHANGED
@@ -1,193 +1,159 @@
1
- # BioMod Zero-Shot Embeddings and Batch-Effect Evaluation
2
-
3
- This directory contains code for extracting BioMod cell embeddings and
4
- evaluating batch effects without fine-tuning. Use the
5
- [BioMod model weights and matching configuration](../../README.md#biomod-model-weights).
6
-
7
- ## Features
8
-
9
- - **Task objective**: evaluate batch effects in cell embeddings extracted with the BioMod model weights.
10
- - **Evaluation metrics**: compute ASW_label/batch metrics with the scib package.
11
- - **Data format**: supports single-cell data in h5ad format.
12
-
13
- ## Usage
14
-
15
- ### Basic Usage
16
-
17
- ```bash
18
- python performer_module_zero_shot.py \
19
- --scfm_hyper_params_path <path_to_model_hyper_params.pkl> \
20
- --scfm_ckpt_path <path_to_model_checkpoint.pth> \
21
- --module_data_path <path_to_module_json_file.json> \
22
- --data_path <path_to_input_data.h5ad>
23
- ```
24
-
25
- ### Full Argument Example
26
-
27
- ```bash
28
- python performer_module_zero_shot.py \
29
- --gene2vec_path ../../selected_gene2vec_27k.npy \
30
- --scfm_hyper_params_path ../performer_module_multiply_all_module_48Mcell/performer_module_multiply_all_module_48Mcell.pkl \
31
- --scfm_ckpt_path ../performer_module_multiply_all_module_48Mcell/performer_module_multiply_all_module_48Mcell_epoch2.pth \
32
- --module_data_dir ../module_data_all \
33
- --gene_mapping_path ../../gene2ens.csv \
34
- --scfm_genes_file ../../selected_genes_27k.txt \
35
- --data_path ../../zero-shot-batch/pancreas_scib.h5ad \
36
- --output_key X_sclong_module \
37
- --n_cells all \
38
- --batch_size 256 \
39
- --fusion_method multiply \
40
- --output_dir res/performer_module_multiply_all_module_48Mcell_first_module \
41
- --metrics_mode simple \
42
- --pool_method first_module
43
- ```
44
-
45
- ```bash
46
- python performer_module_zero_shot.py \
47
- --gene2vec_path ../../selected_gene2vec_27k.npy \
48
- --scfm_hyper_params_path ../performer_module_multiply_all_module_48Mcell/performer_module_multiply_all_module_48Mcell.pkl \
49
- --scfm_ckpt_path ../performer_module_multiply_all_module_48Mcell/performer_module_multiply_all_module_48Mcell_epoch2.pth \
50
- --module_data_dir ../module_data_all \
51
- --gene_mapping_path ../../gene2ens.csv \
52
- --scfm_genes_file ../../selected_genes_27k.txt \
53
- --dataset_preset lung \
54
- --data_path Kim2020_Lung.h5ad \
55
- --output_key X_sclong_module \
56
- --batch_size 128 \
57
- --fusion_method multiply \
58
- --metrics_mode scgpt \
59
- --output_dir res_Lung/performer_module_multiply_all_module_48Mcell_max \
60
- --pool_method max
61
- ```
62
-
63
- ### Arguments
64
-
65
- | Argument | Description | Default |
66
- | --- | --- | --- |
67
- | `--scfm_hyper_params_path` | Path to the matching BioMod model configuration; required | - |
68
- | `--scfm_ckpt_path` | Path to the BioMod model weights; required | - |
69
- | `--module_data_path` | Path to one module JSON file | None |
70
- | `--module_data_dir` | Directory containing multiple module JSON files | None |
71
- | `--gene2vec_path` | Path to the Gene2Vec file | ../selected_gene2vec_27k.npy |
72
- | `--gene_mapping_path` | CSV mapping gene symbols to Ensembl IDs | ../gene2ens.csv |
73
- | `--scfm_genes_file` | BioMod gene list file | ../selected_genes_27k.txt |
74
- | `--data_path` | Input h5ad data file | ../zero-shot-batch/pancreas_scib.h5ad |
75
- | `--output_key` | Key name for embeddings in `adata.obsm` | X_sclong_module |
76
- | `--n_cells` | Number of cells to process; `all` means all cells | all |
77
- | `--batch_size` | Inference batch size | 4 |
78
- | `--output_dir` | Output directory | ./res |
79
-
80
- ## Outputs
81
-
82
- After the run finishes, the following files are generated under `--output_dir`:
83
-
84
- 1. **`sclong_module_batch_cell_emb_mode.csv`**: evaluation metrics in CSV format.
85
- - Contains three columns: metric, label, and value.
86
- - Main metric: ASW_label/batch. Larger values are better and indicate stronger batch-correction performance.
87
-
88
- 2. **`sclong_module_batch_cell_emb_mode_ce.npy`**: NumPy array of cell embeddings.
89
- - Shape: `(n_cells, embed_dim)`.
90
- - Can be used for downstream analysis or visualization.
91
-
92
- ## Workflow
93
-
94
- 1. **Data loading**: load single-cell data in h5ad format.
95
- 2. **Gene mapping**: convert input gene symbols to Ensembl IDs and map them to the BioMod gene list.
96
- 3. **Model inference**:
97
- - Reindex gene expression values to the BioMod gene order.
98
- - Obtain encodings with the BioMod model.
99
- - Extract cell embeddings by averaging `module_encodings`.
100
- 4. **Evaluation**: compute batch-correction metrics with scib.
101
-
102
- ## Cell Embedding Extraction
103
-
104
- The code extracts cell embeddings in the following priority order:
105
-
106
- 1. **`module_encodings`** (preferred): average the module encodings from the encoder output.
107
- ```python
108
- cell_emb = out['module_encodings'].mean(dim=1) # (batch_size, embed_dim)
109
- ```
110
-
111
- 2. **`merged_decodings`**: if `module_encodings` is absent, use the first position of decoder output.
112
- ```python
113
- cell_emb = out['merged_decodings'][:, 0, :] # (batch_size, embed_dim)
114
- ```
115
-
116
- 3. **`module_decodings`**: average module decodings.
117
- ```python
118
- cell_emb = out['module_decodings'].mean(dim=1) # (batch_size, embed_dim)
119
- ```
120
-
121
- ## Comparison with Original scLong
122
-
123
- | Feature | scLong | BioMod |
124
- | --- | --- | --- |
125
- | Model architecture | DualEncoder (mini + large) | Single Encoder |
126
- | Input format | Flat gene expression | Module-based |
127
- | Embedding extraction | `merged_decodings[:, 0, :]` | `module_encodings.mean(dim=1)` |
128
- | Gene handling | Directly uses 27k genes | Aggregates through modules |
129
-
130
- ## Notes
131
-
132
- 1. **Module data**: make sure the correct module data path is provided with `--module_data_path` or `--module_data_dir`.
133
- 2. **Gene mapping**: make sure the mapping file (`gene2ens.csv`) contains the required gene-symbol to Ensembl-ID mappings.
134
- 3. **Memory usage**: for large datasets, set `--n_cells` to limit the number of processed cells.
135
- 4. **Batch size**: adjust `--batch_size` according to GPU memory.
136
-
137
- ## Example Output
138
-
139
- The following legacy log illustrates the output format; it does not document a
140
- validated run with the bundled BioMod model weights.
141
-
142
- ```text
143
- Using device: cuda
144
- Loading hyper parameters from ./checkpoints/model_params.pkl
145
- Loading module data...
146
- Creating sclong_module model...
147
- Loading checkpoint from ./checkpoints/model_epoch6.pth
148
- Total model parameters: 16,140,337
149
- Processing 1000 cells with 2000 genes
150
- Input genes and scfm genes intersect: 1800 genes
151
- Extracting cell embeddings...
152
- Cell embeddings shape: (1000, 512)
153
- Evaluating batch correction...
154
- Using batch_key: batch
155
- Using label_keys: ['celltype']
156
-
157
- Evaluation Results:
158
- ==================================================
159
- metric label value
160
- 0 ASW_label/batch celltype 0.8234
161
- ==================================================
162
- Zero-shot evaluation completed!
163
- ```
164
-
165
- ## Troubleshooting
166
-
167
- ### Issue 1: Module Data Not Found
168
-
169
- **Error**: `KeyError: 'module_sets'`
170
-
171
- **Fix**: make sure `--module_data_path` or `--module_data_dir` is provided.
172
-
173
- ### Issue 2: Gene Mapping Failed
174
-
175
- **Error**: `KeyError: 'gene_symbol'`
176
-
177
- **Fix**: check the format of `--gene_mapping_path` and make sure it contains the `Ensembl gene ID` column.
178
-
179
- ### Issue 3: Model Output Key Missing
180
-
181
- **Error**: `KeyError: 'module_encodings'`
182
-
183
- **Fix**: use the BioMod model weights and their matching configuration, and
184
- check that the model provides the required output.
185
-
186
- ### Issue 4: Out of Memory
187
-
188
- **Error**: `RuntimeError: CUDA out of memory`
189
-
190
- **Fix**:
191
-
192
- - Decrease `--batch_size`.
193
- - Use `--n_cells` to limit the number of processed cells.
 
1
+ # BioMod Zero-Shot Embeddings and Batch-Effect Evaluation
2
+
3
+ This directory contains code for extracting BioMod cell embeddings and
4
+ evaluating batch effects without fine-tuning. Use the
5
+ [BioMod model weights and matching configuration](../../README.md#biomod-model-weights).
6
+
7
+ ## Features
8
+
9
+ - **Task objective**: evaluate batch effects in cell embeddings extracted with the BioMod model weights.
10
+ - **Evaluation metrics**: compute ASW_label/batch metrics with the scib package.
11
+ - **Data format**: supports single-cell data in h5ad format.
12
+
13
+ ## Usage
14
+
15
+ Run these commands from the repository root. Built-in resource defaults resolve from the script location; explicit relative paths resolve from the working directory.
16
+
17
+ ### Basic Usage
18
+
19
+ ```bash
20
+ python BioMod/zero_shot_batch/performer_module_zero_shot.py \
21
+ --scfm_hyper_params_path weight/0810model/mae_tost_70M_75mask_ec2.pkl \
22
+ --scfm_ckpt_path weight/0810model/mae_tost_70M_75mask_ec2_epoch0.pth \
23
+ --module_data_path model_dependencies/module_data_all/merged_modules.json \
24
+ --data_path data/downstream/final_data/Kim2020_Lung.h5ad \
25
+ --dataset_preset lung
26
+ ```
27
+
28
+ ### Full Argument Example
29
+
30
+ ```bash
31
+ python BioMod/zero_shot_batch/performer_module_zero_shot.py \
32
+ --gene2vec_path model_dependencies/selected_gene2vec_27k.npy \
33
+ --scfm_hyper_params_path weight/0810model/mae_tost_70M_75mask_ec2.pkl \
34
+ --scfm_ckpt_path weight/0810model/mae_tost_70M_75mask_ec2_epoch0.pth \
35
+ --module_data_dir model_dependencies/module_data_all \
36
+ --gene_mapping_path model_dependencies/gene2ens.csv \
37
+ --scfm_genes_file model_dependencies/selected_genes_27k.txt \
38
+ --data_path data/downstream/final_data/Kim2020_Lung.h5ad \
39
+ --dataset_preset lung \
40
+ --output_key X_sclong_module \
41
+ --n_cells all \
42
+ --batch_size 256 \
43
+ --output_dir tutorial_runs/zero_shot_first_module \
44
+ --metrics_mode simple \
45
+ --pool_method first_module
46
+ ```
47
+
48
+ ```bash
49
+ python BioMod/zero_shot_batch/performer_module_zero_shot.py \
50
+ --gene2vec_path model_dependencies/selected_gene2vec_27k.npy \
51
+ --scfm_hyper_params_path weight/0810model/mae_tost_70M_75mask_ec2.pkl \
52
+ --scfm_ckpt_path weight/0810model/mae_tost_70M_75mask_ec2_epoch0.pth \
53
+ --module_data_dir model_dependencies/module_data_all \
54
+ --gene_mapping_path model_dependencies/gene2ens.csv \
55
+ --scfm_genes_file model_dependencies/selected_genes_27k.txt \
56
+ --dataset_preset lung \
57
+ --data_path data/downstream/final_data/Kim2020_Lung.h5ad \
58
+ --output_key X_sclong_module \
59
+ --batch_size 128 \
60
+ --metrics_mode scgpt \
61
+ --output_dir tutorial_runs/zero_shot_max \
62
+ --pool_method max
63
+ ```
64
+
65
+ ### Arguments
66
+
67
+ | Argument | Description | Default |
68
+ | --- | --- | --- |
69
+ | `--scfm_hyper_params_path` | Path to the matching BioMod model configuration; required | - |
70
+ | `--scfm_ckpt_path` | Path to the BioMod model weights; required | - |
71
+ | `--module_data_path` | Path to one module JSON file | None |
72
+ | `--module_data_dir` | Directory containing multiple module JSON files | None |
73
+ | `--gene2vec_path` | Path to the Gene2Vec file | model_dependencies/selected_gene2vec_27k.npy |
74
+ | `--gene_mapping_path` | CSV mapping gene symbols to Ensembl IDs | model_dependencies/gene2ens.csv |
75
+ | `--scfm_genes_file` | BioMod gene list file | model_dependencies/selected_genes_27k.txt |
76
+ | `--data_path` | Input h5ad data file | Required |
77
+ | `--output_key` | Key name for embeddings in `adata.obsm` | X_sclong_module |
78
+ | `--n_cells` | Number of cells to process; `all` means all cells | all |
79
+ | `--batch_size` | Inference batch size | 4 |
80
+ | `--output_dir` | Output directory | `BioMod/zero_shot_batch/res/` under the repository |
81
+
82
+ ## Outputs
83
+
84
+ After the run finishes, the following files are generated under `--output_dir`:
85
+
86
+ 1. **`sclong_module_batch_cell_emb_mode.csv`**: evaluation metrics in CSV format.
87
+ - Contains three columns: metric, label, and value.
88
+ - Main metric: ASW_label/batch. Larger values are better and indicate stronger batch-correction performance.
89
+
90
+ 2. **`sclong_module_batch_cell_emb_mode_ce.npy`**: NumPy array of cell embeddings.
91
+ - Shape: `(n_cells, embed_dim)`.
92
+ - Can be used for downstream analysis or visualization.
93
+
94
+ ## Workflow
95
+
96
+ 1. **Data loading**: load single-cell data in h5ad format.
97
+ 2. **Gene mapping**: convert input gene symbols to Ensembl IDs and map them to the BioMod gene list.
98
+ 3. **Model inference**:
99
+ - Reindex gene expression values to the BioMod gene order.
100
+ - Obtain encodings with the BioMod model.
101
+ - Extract cell embeddings using the selected `--pool_method`.
102
+ 4. **Evaluation**: compute batch-correction metrics with scib.
103
+
104
+ ## Cell Embedding Extraction
105
+
106
+ The model output must contain `module_encodings`. Pooling is selected explicitly:
107
+
108
+ - `auto`: use `cls_encoding` when present, otherwise mean-pool module encodings.
109
+ - `cls`: use `cls_encoding`; if absent, log a warning and use mean pooling.
110
+ - `mean` or `max`: pool module encodings along the module dimension.
111
+ - `first_module`: restore the original module order with `ids_restore` and select the first module.
112
+
113
+ The current script does not fall back to decoder outputs when module encodings are absent.
114
+
115
+ ## Comparison with Original scLong
116
+
117
+ | Feature | scLong | BioMod |
118
+ | --- | --- | --- |
119
+ | Model architecture | DualEncoder (mini + large) | Single Encoder |
120
+ | Input format | Flat gene expression | Module-based |
121
+ | Embedding extraction | `merged_decodings[:, 0, :]` | `module_encodings.mean(dim=1)` |
122
+ | Gene handling | Directly uses 27k genes | Aggregates through modules |
123
+
124
+ ## Notes
125
+
126
+ 1. **Module data**: make sure the correct module data path is provided with `--module_data_path` or `--module_data_dir`.
127
+ 2. **Gene mapping**: make sure the mapping file (`gene2ens.csv`) contains the required gene-symbol to Ensembl-ID mappings.
128
+ 3. **Memory usage**: for large datasets, set `--n_cells` to limit the number of processed cells.
129
+ 4. **Batch size**: adjust `--batch_size` according to GPU memory.
130
+
131
+ ## Troubleshooting
132
+
133
+ ### Issue 1: Module Data Not Found
134
+
135
+ **Error**: `KeyError: 'module_sets'`
136
+
137
+ **Fix**: make sure `--module_data_path` or `--module_data_dir` is provided.
138
+
139
+ ### Issue 2: Gene Mapping Failed
140
+
141
+ **Error**: `KeyError: 'gene_symbol'`
142
+
143
+ **Fix**: check the format of `--gene_mapping_path` and make sure it contains the `Ensembl gene ID` column.
144
+
145
+ ### Issue 3: Model Output Key Missing
146
+
147
+ **Error**: `KeyError: 'module_encodings'`
148
+
149
+ **Fix**: use the BioMod model weights and their matching configuration, and
150
+ check that the model provides the required output.
151
+
152
+ ### Issue 4: Out of Memory
153
+
154
+ **Error**: `RuntimeError: CUDA out of memory`
155
+
156
+ **Fix**:
157
+
158
+ - Decrease `--batch_size`.
159
+ - Use `--n_cells` to limit the number of processed cells.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
BioMod/zero_shot_batch/performer_module_zero_shot.py CHANGED
@@ -1,689 +1,694 @@
1
- """
2
- Adaptation code of the PerformerEncoderOnly_Module model on the zero-shot batch correction task.
3
-
4
- The current script mainly supports:
5
- 1. Use the PerformerEncoderOnly_Module model (encoder-only) to extract cell embeddings.
6
- 2. Use mean pooling of module encodings as cell embedding.
7
- 3. Supports evaluation based on scIB indicators consistent with the scGPT tutorial.
8
- 4. Supports Lung data set preset and retains all genes by default without going through the HVG 3000 process.
9
- """
10
-
11
- import argparse
12
- import importlib.util
13
- import os
14
- import pickle
15
- import sys
16
- import tempfile
17
- from copy import deepcopy
18
- from typing import Dict, List, Optional, Sequence, Set, Union
19
-
20
- os.environ.setdefault("NUMBA_DISABLE_JIT", "1")
21
- os.environ.setdefault("NUMBA_CACHE_DIR", tempfile.mkdtemp(prefix="numba_cache_"))
22
-
23
- import numpy as np
24
- import pandas as pd
25
- import scanpy as sc
26
- import scib
27
- import torch
28
- from scanpy import AnnData
29
- from tqdm import tqdm
30
-
31
- sys.path.append("../../")
32
- sys.path.append("../")
33
-
34
-
35
- DATASET_PRESETS = {
36
- "auto": {},
37
- "lung": {
38
- "gene_col": "gene_name",
39
- "batch_key": "sample",
40
- "label_keys": ["cell_type"],
41
- },
42
- }
43
-
44
-
45
- def print_sys(message: str) -> None:
46
- print(message, flush=True, file=sys.stderr)
47
-
48
-
49
- def strip_gene_version(gene_id: str) -> str:
50
- return gene_id.split(".", 1)[0]
51
-
52
-
53
- def canonicalize_target_gene_id(
54
- raw_gene: object,
55
- symbol_to_ensembl: Dict[str, str],
56
- target_gene_set: Set[str],
57
- ) -> Optional[str]:
58
- if pd.isna(raw_gene):
59
- return None
60
-
61
- raw_gene_str = str(raw_gene).strip()
62
- if not raw_gene_str:
63
- return None
64
-
65
- direct_candidates = [raw_gene_str, strip_gene_version(raw_gene_str)]
66
- for candidate in direct_candidates:
67
- if candidate in target_gene_set:
68
- return candidate
69
-
70
- mapped_gene = symbol_to_ensembl.get(raw_gene_str)
71
- if mapped_gene is None:
72
- mapped_gene = symbol_to_ensembl.get(strip_gene_version(raw_gene_str))
73
-
74
- if mapped_gene is None or pd.isna(mapped_gene):
75
- return None
76
-
77
- mapped_gene_str = strip_gene_version(str(mapped_gene).strip())
78
- return mapped_gene_str or None
79
-
80
-
81
- def resolve_input_gene_ids(
82
- adata: AnnData,
83
- gene_col: Optional[str],
84
- symbol_to_ensembl: Dict[str, str],
85
- target_gene_set: Set[str],
86
- ) -> List[str]:
87
- if gene_col is None:
88
- raw_genes = [str(gene) for gene in adata.var.index.tolist()]
89
- source_name = "adata.var.index"
90
- else:
91
- if gene_col not in adata.var.columns:
92
- raise KeyError(f"gene_col '{gene_col}' not found in adata.var.columns")
93
- raw_genes = [str(gene) if not pd.isna(gene) else "" for gene in adata.var[gene_col].tolist()]
94
- source_name = f"adata.var['{gene_col}']"
95
-
96
- resolved_gene_ids: List[str] = []
97
- direct_hits = 0
98
- mapped_hits = 0
99
- unresolved = 0
100
-
101
- for index, raw_gene in enumerate(raw_genes):
102
- resolved_gene = canonicalize_target_gene_id(raw_gene, symbol_to_ensembl, target_gene_set)
103
- if resolved_gene is None:
104
- resolved_gene = f"unknown{index}"
105
- unresolved += 1
106
- elif strip_gene_version(raw_gene) == resolved_gene:
107
- direct_hits += 1
108
- else:
109
- mapped_hits += 1
110
- resolved_gene_ids.append(resolved_gene)
111
-
112
- print_sys(f"Using genes from {source_name}")
113
- print_sys(
114
- "Gene ID resolution summary: "
115
- f"direct_hits={direct_hits}, mapped_hits={mapped_hits}, unresolved={unresolved}"
116
- )
117
- return resolved_gene_ids
118
-
119
-
120
- def clear_neighbors(adata: AnnData) -> None:
121
- neighbors = adata.uns.pop("neighbors", None)
122
- if neighbors is not None:
123
- dist_key = neighbors.get("distances_key", "distances")
124
- conn_key = neighbors.get("connectivities_key", "connectivities")
125
- adata.obsp.pop(dist_key, None)
126
- adata.obsp.pop(conn_key, None)
127
- adata.obsp.pop("distances", None)
128
- adata.obsp.pop("connectivities", None)
129
-
130
-
131
- def prepare_eval_adata(
132
- adata: AnnData,
133
- batch_key: str,
134
- label_key: str,
135
- embedding_key: str,
136
- build_neighbors: bool = False,
137
- ) -> AnnData:
138
- if embedding_key not in adata.obsm:
139
- raise ValueError(f"Embeddings '{embedding_key}' not found in adata.obsm")
140
-
141
- if batch_key not in adata.obs.columns:
142
- raise ValueError(f"batch_key '{batch_key}' not found in adata.obs")
143
-
144
- if label_key not in adata.obs.columns:
145
- raise ValueError(f"label_key '{label_key}' not found in adata.obs")
146
-
147
- valid_mask = adata.obs[batch_key].notna() & adata.obs[label_key].notna()
148
- dropped = int((~valid_mask).sum())
149
- if dropped > 0:
150
- print_sys(
151
- f"Dropping {dropped} cells with missing batch/label values "
152
- f"for evaluation (batch_key={batch_key}, label_key={label_key})."
153
- )
154
-
155
- if int(valid_mask.sum()) == 0:
156
- raise ValueError(
157
- f"No labeled cells remain after filtering batch_key={batch_key}, label_key={label_key}"
158
- )
159
-
160
- eval_adata = adata[valid_mask].copy()
161
- eval_adata.obs[batch_key] = eval_adata.obs[batch_key].astype("category")
162
- eval_adata.obs[label_key] = eval_adata.obs[label_key].astype("category")
163
-
164
- if build_neighbors:
165
- clear_neighbors(eval_adata)
166
- sc.pp.neighbors(eval_adata, use_rep=embedding_key)
167
- return eval_adata
168
-
169
-
170
- def eval_scib_metrics_simple(
171
- adata: AnnData,
172
- batch_key: str,
173
- label_key: str,
174
- embedding_key: str,
175
- ) -> Dict[str, float]:
176
- eval_adata = prepare_eval_adata(
177
- adata,
178
- batch_key,
179
- label_key,
180
- embedding_key,
181
- build_neighbors=False,
182
- )
183
-
184
- results_dict: Dict[str, float] = {}
185
- if len(eval_adata.obs[batch_key].unique()) > 1:
186
- results_dict["ASW_label/batch"] = scib.metrics.silhouette_batch(
187
- eval_adata,
188
- batch_key,
189
- label_key,
190
- embed=embedding_key,
191
- metric="euclidean",
192
- return_all=False,
193
- verbose=False,
194
- )
195
-
196
- results_dict = {key: value for key, value in results_dict.items() if not np.isnan(value)}
197
- return results_dict
198
-
199
-
200
- def eval_scib_metrics_scgpt(
201
- adata: AnnData,
202
- batch_key: str,
203
- label_key: str,
204
- embedding_key: str,
205
- ) -> Dict[str, float]:
206
- eval_adata = prepare_eval_adata(
207
- adata,
208
- batch_key,
209
- label_key,
210
- embedding_key,
211
- build_neighbors=True,
212
- )
213
-
214
- results = scib.metrics.metrics(
215
- eval_adata,
216
- adata_int=eval_adata,
217
- batch_key=batch_key,
218
- label_key=label_key,
219
- embed=embedding_key,
220
- isolated_labels_asw_=False,
221
- silhouette_=True,
222
- hvg_score_=False,
223
- graph_conn_=True,
224
- pcr_=True,
225
- isolated_labels_f1_=False,
226
- trajectory_=False,
227
- nmi_=True,
228
- ari_=True,
229
- cell_cycle_=False,
230
- kBET_=False,
231
- ilisi_=False,
232
- clisi_=False,
233
- )
234
- result_dict = results[0].to_dict()
235
-
236
- result_dict["avg_bio"] = np.mean(
237
- [
238
- result_dict["NMI_cluster/label"],
239
- result_dict["ARI_cluster/label"],
240
- result_dict["ASW_label"],
241
- ]
242
- )
243
- result_dict["avg_batch"] = np.mean(
244
- [
245
- result_dict["graph_conn"],
246
- result_dict["ASW_label/batch"],
247
- ]
248
- )
249
-
250
- result_dict = {key: value for key, value in result_dict.items() if not np.isnan(value)}
251
- return result_dict
252
-
253
-
254
- def evaluate(
255
- adata: AnnData,
256
- batch_key: str,
257
- label_keys: Sequence[str],
258
- embedding_key: str,
259
- metrics_mode: str,
260
- res_path: str,
261
- ) -> pd.DataFrame:
262
- met_df = pd.DataFrame(columns=["metric", "label", "value"])
263
-
264
- unique_label_keys: List[str] = []
265
- for key in label_keys:
266
- if key not in unique_label_keys:
267
- unique_label_keys.append(key)
268
-
269
- if not unique_label_keys:
270
- raise ValueError("No label keys were provided for evaluation")
271
-
272
- for label_key in unique_label_keys:
273
- if metrics_mode == "scgpt":
274
- metrics = eval_scib_metrics_scgpt(
275
- adata=adata,
276
- batch_key=batch_key,
277
- label_key=label_key,
278
- embedding_key=embedding_key,
279
- )
280
- elif metrics_mode == "simple":
281
- metrics = eval_scib_metrics_simple(
282
- adata=adata,
283
- batch_key=batch_key,
284
- label_key=label_key,
285
- embedding_key=embedding_key,
286
- )
287
- else:
288
- raise ValueError(f"Unknown metrics_mode: {metrics_mode}")
289
-
290
- print_sys(
291
- "\n".join(
292
- [f"[{label_key}] {metric_name}: {metric_value:.4f}" for metric_name, metric_value in metrics.items()]
293
- )
294
- )
295
- for metric_name, metric_value in metrics.items():
296
- met_df.loc[len(met_df)] = [metric_name, label_key, metric_value]
297
-
298
- met_df.to_csv(res_path, index=False)
299
- np.save(f"{res_path[:-4]}_ce.npy", adata.obsm[embedding_key])
300
- return met_df
301
-
302
-
303
- def resolve_with_preset(
304
- cli_value: Optional[object],
305
- preset_value: Optional[object],
306
- ) -> Optional[object]:
307
- return cli_value if cli_value is not None else preset_value
308
-
309
-
310
- def resolve_batch_key(adata: AnnData, requested_batch_key: Optional[str]) -> str:
311
- if requested_batch_key is not None:
312
- if requested_batch_key not in adata.obs.columns:
313
- raise KeyError(f"batch_key '{requested_batch_key}' not found in adata.obs.columns")
314
- return requested_batch_key
315
-
316
- fallback_keys = ["batch", "str_batch", "sample", "Sample Characteristic[individual]"]
317
- for key in fallback_keys:
318
- if key in adata.obs.columns:
319
- print_sys(f"Auto-selected batch_key='{key}'")
320
- return key
321
-
322
- raise ValueError(
323
- "Unable to infer batch_key. Please pass --batch_key explicitly."
324
- )
325
-
326
-
327
- def resolve_label_keys(adata: AnnData, requested_label_keys: Optional[Sequence[str]]) -> List[str]:
328
- if requested_label_keys:
329
- missing_keys = [key for key in requested_label_keys if key not in adata.obs.columns]
330
- if missing_keys:
331
- raise KeyError(f"label_key(s) not found in adata.obs.columns: {missing_keys}")
332
- return list(requested_label_keys)
333
-
334
- fallback_keys = ["celltype", "cell_type", "celltype_original"]
335
- resolved_keys = [key for key in fallback_keys if key in adata.obs.columns]
336
- if resolved_keys:
337
- print_sys(f"Auto-selected label_keys={resolved_keys}")
338
- return resolved_keys
339
-
340
- raise ValueError(
341
- "Unable to infer label_key. Please pass --label_key explicitly."
342
- )
343
-
344
-
345
- def reindex_tensor_universal(
346
- tensor: torch.Tensor,
347
- index_positions: Sequence[int],
348
- dim: int,
349
- filler: str = "0",
350
- device: Union[str, torch.device] = "cpu",
351
- ) -> torch.Tensor:
352
- index_tensor = torch.tensor(index_positions, device=device)
353
- tensor_shape = list(tensor.shape)
354
- tensor_shape[dim] = len(index_positions)
355
- index_shape = [1] * len(tensor_shape)
356
- index_shape[dim] = len(index_positions)
357
- index_tensor = index_tensor.view(*index_shape).expand(*tensor_shape)
358
-
359
- if filler == "0":
360
- padder_shape = deepcopy(tensor_shape)
361
- padder_shape[dim] = 1
362
- padder = torch.zeros(padder_shape, dtype=tensor.dtype, device=device)
363
- elif filler == "mean":
364
- padder = tensor.mean(dim=dim, keepdim=True)
365
- else:
366
- raise ValueError("filler should be 0 or mean")
367
- expanded_tensor = torch.cat((tensor, padder), dim=dim)
368
- return torch.gather(expanded_tensor, dim, index_tensor)
369
-
370
-
371
- project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
372
- sys.path.insert(0, project_root)
373
- sys.path.insert(0, os.path.dirname(project_root))
374
-
375
- performer_module_dir = project_root
376
-
377
- def load_module_modules():
378
- models_path = os.path.join(performer_module_dir, "models_performer_module.py")
379
- if not os.path.exists(models_path):
380
- raise FileNotFoundError(f"Model file not found: {models_path}")
381
- models_spec = importlib.util.spec_from_file_location("models_performer_module", models_path)
382
- models_performer_module = importlib.util.module_from_spec(models_spec)
383
- models_spec.loader.exec_module(models_performer_module)
384
-
385
- dataset_path = os.path.join(performer_module_dir, "module_dataset.py")
386
- if not os.path.exists(dataset_path):
387
- raise FileNotFoundError(f"Dataset file not found: {dataset_path}")
388
- dataset_spec = importlib.util.spec_from_file_location("module_dataset", dataset_path)
389
- module_dataset = importlib.util.module_from_spec(dataset_spec)
390
- dataset_spec.loader.exec_module(module_dataset)
391
-
392
- return (
393
- models_performer_module.PerformerEncoderOnly_Module,
394
- models_performer_module.sanitize_performer_module_config,
395
- module_dataset.load_module_data,
396
- )
397
-
398
- OUTPUT_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "res")
399
- os.makedirs(OUTPUT_DIR, exist_ok=True)
400
-
401
- parser = argparse.ArgumentParser()
402
- parser.add_argument("--gene2vec_path", type=str, default="../selected_gene2vec_27k.npy",
403
- help="Path to gene2vec file")
404
- parser.add_argument("--scfm_hyper_params_path", type=str, required=True,
405
- help="Path to sclong_module model hyper parameters pkl file")
406
- parser.add_argument("--scfm_ckpt_path", type=str, required=True,
407
- help="Path to sclong_module model checkpoint")
408
- parser.add_argument("--module_data_path", type=str, default=None,
409
- help="Path to module JSON file (single file)")
410
- parser.add_argument("--module_data_dir", type=str, default=None,
411
- help="Path to directory containing module JSON files")
412
- parser.add_argument("--gene_mapping_path", type=str, default="../gene2ens.csv",
413
- help="Path to gene symbol to Ensembl ID mapping CSV")
414
- parser.add_argument("--scfm_genes_file", type=str, default="../selected_genes_27k.txt",
415
- help="Path to scfm genes list file")
416
- parser.add_argument("--data_path", type=str, default="../zero-shot-batch/pancreas_scib.h5ad",
417
- help="Path to input h5ad file")
418
- parser.add_argument("--dataset_preset", type=str, default="auto", choices=sorted(DATASET_PRESETS.keys()),
419
- help="Dataset preset for batch/label/gene column defaults")
420
- parser.add_argument("--gene_col", type=str, default=None,
421
- help="Column in adata.var used as the input gene symbol column. "
422
- "If not set, use adata.var.index.")
423
- parser.add_argument("--batch_key", type=str, default=None,
424
- help="Column in adata.obs used as the batch label")
425
- parser.add_argument("--label_key", nargs="+", default=None,
426
- help="One or more columns in adata.obs used as evaluation labels")
427
- parser.add_argument("--output_key", type=str, default="X_sclong_module",
428
- help="Key name for storing embeddings in adata.obsm")
429
- parser.add_argument("--pool_method", type=str, default="auto", choices=["auto", "mean", "cls", "max", "first_module"],
430
- help="Cell embedding pooling. auto prefers cls_encoding when the checkpoint has a trained CLS token; "
431
- "first_module uses the first module after restoring original module order.")
432
- parser.add_argument("--n_cells", type=str, default="all",
433
- help="Number of cells to process (default: all)")
434
- parser.add_argument("--batch_size", type=int, default=4,
435
- help="Batch size for inference")
436
- parser.add_argument("--output_dir", type=str, default=OUTPUT_DIR,
437
- help="Output directory for results")
438
- parser.add_argument("--metrics_mode", type=str, default="scgpt", choices=["scgpt", "simple"],
439
- help="Evaluation metrics mode. Default uses the scGPT tutorial metric bundle.")
440
- parser.add_argument("--min_genes_per_cell", type=int, default=0,
441
- help="Optional Scanpy cell filter. Default 0 keeps all cells.")
442
- parser.add_argument("--min_cells_per_gene", type=int, default=0,
443
- help="Optional Scanpy gene filter. Default 0 keeps all genes.")
444
- parser.add_argument("--shuffle_module_genes", action="store_true",
445
- help="Optional downstream-only extra reshuffle on top of module_sets restored from PKL. "
446
- "Do not pass this when reproducing the pretrained backbone as-is.")
447
- parser.add_argument("--shuffle_seed", type=int, default=2021,
448
- help="Random seed for the downstream extra reshuffle when --shuffle_module_genes is enabled.")
449
- parser.add_argument("--save_raw_module_emb", action="store_true",
450
- help="Save raw module_encodings with shape (num_cells, num_modules, embed_dim)")
451
-
452
- args = parser.parse_args()
453
- os.makedirs(args.output_dir, exist_ok=True)
454
-
455
- device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
456
- print_sys(f"Using device: {device}")
457
-
458
- preset_config = DATASET_PRESETS[args.dataset_preset]
459
- resolved_gene_col = resolve_with_preset(args.gene_col, preset_config.get("gene_col"))
460
- resolved_batch_key = resolve_with_preset(args.batch_key, preset_config.get("batch_key"))
461
- resolved_label_keys = resolve_with_preset(args.label_key, preset_config.get("label_keys"))
462
-
463
- print_sys(
464
- f"dataset_preset={args.dataset_preset}, "
465
- f"gene_col={resolved_gene_col}, batch_key={resolved_batch_key}, "
466
- f"label_keys={resolved_label_keys}, metrics_mode={args.metrics_mode}, "
467
- f"pool_method={args.pool_method}"
468
- )
469
-
470
- PerformerEncoderOnly_Module, sanitize_performer_module_config, load_module_data = load_module_modules()
471
-
472
- print_sys(f"Loading hyper parameters from {args.scfm_hyper_params_path}")
473
- with open(args.scfm_hyper_params_path, "rb") as handle:
474
- scfm_hyper_params = pickle.load(handle)
475
- scfm_hyper_params.pop("module_embed_chunk_padded_gene_budget", None)
476
- scfm_hyper_params = sanitize_performer_module_config(
477
- scfm_hyper_params,
478
- source=args.scfm_hyper_params_path,
479
- log_fn=print_sys,
480
- )
481
-
482
- scfm_hyper_params["gene2vec_file"] = args.gene2vec_path
483
- # Read fusion_method from pkl to ensure consistency with pre-training
484
- fusion_method = scfm_hyper_params.get("fusion_method", "multiply")
485
- print_sys(f"Using fusion_method from pkl: {fusion_method}")
486
-
487
- if "module_sets" not in scfm_hyper_params or scfm_hyper_params.get("module_sets") is None:
488
- if args.module_data_path or args.module_data_dir:
489
- print_sys("Loading module data...")
490
- module_sets = load_module_data(
491
- module_data_path=args.module_data_path,
492
- module_data_dir=args.module_data_dir,
493
- gene_list_path=args.scfm_genes_file,
494
- gene_mapping_path=args.gene_mapping_path,
495
- verbose=True,
496
- )
497
- scfm_hyper_params["module_sets"] = module_sets
498
- else:
499
- raise ValueError("Model requires module_sets, but neither --module_data_path nor --module_data_dir was provided")
500
- else:
501
- module_sets = scfm_hyper_params["module_sets"]
502
- print_sys(f"Using module_sets from hyper parameters ({len(module_sets)} modules)")
503
-
504
- if args.shuffle_module_genes:
505
- import random
506
-
507
- print_sys(
508
- f"Applying downstream-only extra module reshuffle (seed={args.shuffle_seed}). "
509
- f"Pretraining-time module_sets from PKL will be reshuffled again."
510
- )
511
- all_gene_indices: List[int] = []
512
- module_sizes: List[int] = []
513
- for module in module_sets:
514
- all_gene_indices.extend(module)
515
- module_sizes.append(len(module))
516
-
517
- rng = random.Random(args.shuffle_seed)
518
- rng.shuffle(all_gene_indices)
519
-
520
- shuffled_module_sets = []
521
- start_idx = 0
522
- for size in module_sizes:
523
- shuffled_module_sets.append(all_gene_indices[start_idx:start_idx + size])
524
- start_idx += size
525
- module_sets = shuffled_module_sets
526
- scfm_hyper_params["module_sets"] = module_sets
527
- print_sys(f"Shuffled {len(all_gene_indices)} genes across {len(module_sets)} modules")
528
-
529
- print_sys(f"Model hyper parameters: {list(scfm_hyper_params.keys())}")
530
-
531
- pkl_use_gene2vec = scfm_hyper_params.get("use_gene2vec", True)
532
- scfm_hyper_params["use_gene2vec"] = pkl_use_gene2vec
533
- print_sys(f"use_gene2vec={pkl_use_gene2vec} (from PKL)")
534
-
535
- print_sys("Creating PerformerEncoderOnly_Module model...")
536
- model = PerformerEncoderOnly_Module(**scfm_hyper_params).to(device)
537
-
538
- print_sys(f"Loading checkpoint from {args.scfm_ckpt_path}")
539
- ckpt = torch.load(args.scfm_ckpt_path, map_location=device)
540
- model_state_dict = ckpt.get("model_state_dict", ckpt)
541
- filtered_state_dict = {
542
- key: value
543
- for key, value in model_state_dict.items()
544
- if "go_conv" not in key and "G_go" not in key.lower()
545
- }
546
- model.load_state_dict(filtered_state_dict, strict=True)
547
-
548
- for parameter in model.parameters():
549
- parameter.requires_grad = False
550
- model.eval()
551
-
552
- all_params = sum(parameter.numel() for parameter in model.parameters())
553
- trainable_params = sum(parameter.numel() for parameter in model.parameters() if parameter.requires_grad)
554
- print_sys(f"Total model parameters: {all_params:,}")
555
- print_sys(f"Trainable parameters during zero-shot: {trainable_params:,}")
556
-
557
- gene2ens = pd.read_csv(args.gene_mapping_path, index_col=0)
558
- symbol_to_ensembl = gene2ens["Ensembl gene ID"].to_dict()
559
-
560
- with open(args.scfm_genes_file, "r") as handle:
561
- scfm_genes = [line.rstrip("\n") for line in handle.readlines()]
562
- scfm_gene_set = set(scfm_genes)
563
-
564
- print_sys(f"Loading data from {args.data_path}")
565
- adata = sc.read_h5ad(args.data_path)
566
-
567
- if args.min_genes_per_cell > 0:
568
- before_cells = adata.n_obs
569
- sc.pp.filter_cells(adata, min_genes=args.min_genes_per_cell)
570
- print_sys(f"Filtered cells by min_genes={args.min_genes_per_cell}: {before_cells} -> {adata.n_obs}")
571
-
572
- if args.min_cells_per_gene > 0:
573
- before_genes = adata.n_vars
574
- sc.pp.filter_genes(adata, min_cells=args.min_cells_per_gene)
575
- print_sys(f"Filtered genes by min_cells={args.min_cells_per_gene}: {before_genes} -> {adata.n_vars}")
576
-
577
- if args.n_cells == "all":
578
- n_cells = adata.n_obs
579
- else:
580
- n_cells = int(args.n_cells)
581
- if adata.n_obs > n_cells:
582
- print_sys(f"Subsampling to {n_cells} cells")
583
- sc.pp.subsample(adata, n_obs=n_cells, copy=False)
584
- n_cells = adata.n_obs
585
-
586
- batch_key = resolve_batch_key(adata, resolved_batch_key)
587
- label_keys = resolve_label_keys(adata, resolved_label_keys)
588
-
589
- print_sys(f"Processing {n_cells} cells with {adata.n_vars} genes")
590
- print_sys(f"Evaluation batch_key={batch_key}, label_keys={label_keys}")
591
-
592
- input_gene_ids = resolve_input_gene_ids(
593
- adata=adata,
594
- gene_col=resolved_gene_col,
595
- symbol_to_ensembl=symbol_to_ensembl,
596
- target_gene_set=scfm_gene_set,
597
- )
598
-
599
- input_mapping = {gene_id: index for index, gene_id in enumerate(input_gene_ids)}
600
- input_gene_num = len(input_gene_ids)
601
-
602
- intersect_genes = np.intersect1d(input_gene_ids, scfm_genes)
603
- print_sys(f"Input genes and scfm genes intersect: {len(intersect_genes)} genes")
604
-
605
- scfm_genes_pad = scfm_genes + ["PAD"]
606
- scfm_seq_len = len(scfm_genes_pad)
607
- scfm_mapping = {gene_id: index for index, gene_id in enumerate(scfm_genes_pad)}
608
-
609
- scfm_index_positions = [input_mapping.get(gene_id, input_gene_num) for gene_id in scfm_genes_pad]
610
- _ = [scfm_mapping.get(gene_id, scfm_seq_len) for gene_id in input_gene_ids]
611
-
612
- print_sys("Extracting cell embeddings...")
613
- cell_embeddings = []
614
- raw_module_embeddings = [] if args.save_raw_module_emb else None
615
- embedding_key = args.output_key
616
-
617
- with torch.no_grad():
618
- for start in tqdm(range(0, n_cells, args.batch_size)):
619
- batch_end = min(start + args.batch_size, n_cells)
620
- batch_matrix = adata.X[start:batch_end, :]
621
- if hasattr(batch_matrix, "toarray"):
622
- batch_matrix = batch_matrix.toarray()
623
- x = torch.tensor(batch_matrix, dtype=torch.float32).to(device)
624
-
625
- x_scfm = reindex_tensor_universal(x, scfm_index_positions, dim=1, filler="0", device=device)
626
- out = model(x_scfm, mask_ratio=0.0, return_encodings=True)
627
-
628
- if "module_encodings" not in out:
629
- raise ValueError(f"Expected 'module_encodings' in output, got: {list(out.keys())}")
630
-
631
- module_encodings = out["module_encodings"]
632
- ids_restore = out.get("ids_restore")
633
- if args.pool_method == "cls":
634
- if "cls_encoding" not in out:
635
- print_sys("Warning: pool_method='cls' but checkpoint has no cls_encoding, using mean pooling instead")
636
- cell_emb = module_encodings.mean(dim=1)
637
- else:
638
- cell_emb = out["cls_encoding"]
639
- elif args.pool_method == "mean":
640
- cell_emb = module_encodings.mean(dim=1)
641
- elif args.pool_method == "max":
642
- cell_emb = module_encodings.max(dim=1)[0]
643
- elif args.pool_method == "first_module":
644
- if ids_restore is None:
645
- raise ValueError("pool_method='first_module' requires ids_restore in model outputs")
646
- restore_index = ids_restore.unsqueeze(-1).expand(-1, -1, module_encodings.shape[-1])
647
- restored_module_encodings = torch.gather(module_encodings, dim=1, index=restore_index)
648
- cell_emb = restored_module_encodings[:, 0, :]
649
- else:
650
- cell_emb = out["cls_encoding"] if "cls_encoding" in out else module_encodings.mean(dim=1)
651
-
652
- if args.save_raw_module_emb:
653
- raw_module_embeddings.append(module_encodings.cpu().numpy())
654
-
655
- cell_embeddings.append(cell_emb.cpu().numpy())
656
-
657
- cell_embeddings = np.concatenate(cell_embeddings, axis=0)
658
- print_sys(f"Cell embeddings shape: {cell_embeddings.shape}")
659
-
660
- if args.save_raw_module_emb and raw_module_embeddings is not None:
661
- raw_module_embeddings = np.concatenate(raw_module_embeddings, axis=0)
662
- print_sys(f"Raw module embeddings shape: {raw_module_embeddings.shape}")
663
- raw_module_path = os.path.join(args.output_dir, "sclong_module_raw_module_emb.npy")
664
- np.save(raw_module_path, raw_module_embeddings)
665
- print_sys(f"Raw module embeddings saved to {raw_module_path}")
666
-
667
- adata.obsm[embedding_key] = cell_embeddings
668
-
669
- print_sys("Evaluating batch correction...")
670
- res = evaluate(
671
- adata=adata,
672
- batch_key=batch_key,
673
- label_keys=label_keys,
674
- embedding_key=embedding_key,
675
- metrics_mode=args.metrics_mode,
676
- res_path=os.path.join(args.output_dir, "sclong_module_batch_cell_emb_mode.csv"),
677
- )
678
-
679
- print_sys("\nEvaluation Results:")
680
- print_sys("=" * 50)
681
- print_sys(res.to_string(index=False))
682
- print_sys("=" * 50)
683
-
684
- np.save(os.path.join(args.output_dir, "sclong_module_batch_cell_emb_mode_ce.npy"), cell_embeddings)
685
- print_sys(
686
- f"Cell embeddings saved to "
687
- f"{os.path.join(args.output_dir, 'sclong_module_batch_cell_emb_mode_ce.npy')}"
688
- )
689
- print_sys("Zero-shot evaluation completed!")
 
 
 
 
 
 
1
+ """
2
+ Adaptation code of the PerformerEncoderOnly_Module model on the zero-shot batch correction task.
3
+
4
+ The current script mainly supports:
5
+ 1. Use the PerformerEncoderOnly_Module model (encoder-only) to extract cell embeddings.
6
+ 2. Use mean pooling of module encodings as cell embedding.
7
+ 3. Supports evaluation based on scIB indicators consistent with the scGPT tutorial.
8
+ 4. Supports Lung data set preset and retains all genes by default without going through the HVG 3000 process.
9
+ """
10
+
11
+ import argparse
12
+ import importlib.util
13
+ import os
14
+ import pickle
15
+ import sys
16
+ import tempfile
17
+ from pathlib import Path
18
+
19
+ REPO_ROOT = Path(__file__).resolve().parents[2]
20
+ ASSET_DIR = REPO_ROOT / "model_dependencies"
21
+ DOWNSTREAM_DATA_DIR = REPO_ROOT / "data" / "downstream" / "final_data"
22
+ WEIGHTS_DIR = REPO_ROOT / "weight" / "0810model"
23
+ from copy import deepcopy
24
+ from typing import Dict, List, Optional, Sequence, Set, Union
25
+
26
+ os.environ.setdefault("NUMBA_DISABLE_JIT", "1")
27
+ os.environ.setdefault("NUMBA_CACHE_DIR", tempfile.mkdtemp(prefix="numba_cache_"))
28
+
29
+ import numpy as np
30
+ import pandas as pd
31
+ import scanpy as sc
32
+ import scib
33
+ import torch
34
+ from scanpy import AnnData
35
+ from tqdm import tqdm
36
+
37
+ # Model imports below are anchored to this file, independent of the working directory.
38
+
39
+
40
+ DATASET_PRESETS = {
41
+ "auto": {},
42
+ "lung": {
43
+ "gene_col": "gene_name",
44
+ "batch_key": "sample",
45
+ "label_keys": ["cell_type"],
46
+ },
47
+ }
48
+
49
+
50
+ def print_sys(message: str) -> None:
51
+ print(message, flush=True, file=sys.stderr)
52
+
53
+
54
+ def strip_gene_version(gene_id: str) -> str:
55
+ return gene_id.split(".", 1)[0]
56
+
57
+
58
+ def canonicalize_target_gene_id(
59
+ raw_gene: object,
60
+ symbol_to_ensembl: Dict[str, str],
61
+ target_gene_set: Set[str],
62
+ ) -> Optional[str]:
63
+ if pd.isna(raw_gene):
64
+ return None
65
+
66
+ raw_gene_str = str(raw_gene).strip()
67
+ if not raw_gene_str:
68
+ return None
69
+
70
+ direct_candidates = [raw_gene_str, strip_gene_version(raw_gene_str)]
71
+ for candidate in direct_candidates:
72
+ if candidate in target_gene_set:
73
+ return candidate
74
+
75
+ mapped_gene = symbol_to_ensembl.get(raw_gene_str)
76
+ if mapped_gene is None:
77
+ mapped_gene = symbol_to_ensembl.get(strip_gene_version(raw_gene_str))
78
+
79
+ if mapped_gene is None or pd.isna(mapped_gene):
80
+ return None
81
+
82
+ mapped_gene_str = strip_gene_version(str(mapped_gene).strip())
83
+ return mapped_gene_str or None
84
+
85
+
86
+ def resolve_input_gene_ids(
87
+ adata: AnnData,
88
+ gene_col: Optional[str],
89
+ symbol_to_ensembl: Dict[str, str],
90
+ target_gene_set: Set[str],
91
+ ) -> List[str]:
92
+ if gene_col is None:
93
+ raw_genes = [str(gene) for gene in adata.var.index.tolist()]
94
+ source_name = "adata.var.index"
95
+ else:
96
+ if gene_col not in adata.var.columns:
97
+ raise KeyError(f"gene_col '{gene_col}' not found in adata.var.columns")
98
+ raw_genes = [str(gene) if not pd.isna(gene) else "" for gene in adata.var[gene_col].tolist()]
99
+ source_name = f"adata.var['{gene_col}']"
100
+
101
+ resolved_gene_ids: List[str] = []
102
+ direct_hits = 0
103
+ mapped_hits = 0
104
+ unresolved = 0
105
+
106
+ for index, raw_gene in enumerate(raw_genes):
107
+ resolved_gene = canonicalize_target_gene_id(raw_gene, symbol_to_ensembl, target_gene_set)
108
+ if resolved_gene is None:
109
+ resolved_gene = f"unknown{index}"
110
+ unresolved += 1
111
+ elif strip_gene_version(raw_gene) == resolved_gene:
112
+ direct_hits += 1
113
+ else:
114
+ mapped_hits += 1
115
+ resolved_gene_ids.append(resolved_gene)
116
+
117
+ print_sys(f"Using genes from {source_name}")
118
+ print_sys(
119
+ "Gene ID resolution summary: "
120
+ f"direct_hits={direct_hits}, mapped_hits={mapped_hits}, unresolved={unresolved}"
121
+ )
122
+ return resolved_gene_ids
123
+
124
+
125
+ def clear_neighbors(adata: AnnData) -> None:
126
+ neighbors = adata.uns.pop("neighbors", None)
127
+ if neighbors is not None:
128
+ dist_key = neighbors.get("distances_key", "distances")
129
+ conn_key = neighbors.get("connectivities_key", "connectivities")
130
+ adata.obsp.pop(dist_key, None)
131
+ adata.obsp.pop(conn_key, None)
132
+ adata.obsp.pop("distances", None)
133
+ adata.obsp.pop("connectivities", None)
134
+
135
+
136
+ def prepare_eval_adata(
137
+ adata: AnnData,
138
+ batch_key: str,
139
+ label_key: str,
140
+ embedding_key: str,
141
+ build_neighbors: bool = False,
142
+ ) -> AnnData:
143
+ if embedding_key not in adata.obsm:
144
+ raise ValueError(f"Embeddings '{embedding_key}' not found in adata.obsm")
145
+
146
+ if batch_key not in adata.obs.columns:
147
+ raise ValueError(f"batch_key '{batch_key}' not found in adata.obs")
148
+
149
+ if label_key not in adata.obs.columns:
150
+ raise ValueError(f"label_key '{label_key}' not found in adata.obs")
151
+
152
+ valid_mask = adata.obs[batch_key].notna() & adata.obs[label_key].notna()
153
+ dropped = int((~valid_mask).sum())
154
+ if dropped > 0:
155
+ print_sys(
156
+ f"Dropping {dropped} cells with missing batch/label values "
157
+ f"for evaluation (batch_key={batch_key}, label_key={label_key})."
158
+ )
159
+
160
+ if int(valid_mask.sum()) == 0:
161
+ raise ValueError(
162
+ f"No labeled cells remain after filtering batch_key={batch_key}, label_key={label_key}"
163
+ )
164
+
165
+ eval_adata = adata[valid_mask].copy()
166
+ eval_adata.obs[batch_key] = eval_adata.obs[batch_key].astype("category")
167
+ eval_adata.obs[label_key] = eval_adata.obs[label_key].astype("category")
168
+
169
+ if build_neighbors:
170
+ clear_neighbors(eval_adata)
171
+ sc.pp.neighbors(eval_adata, use_rep=embedding_key)
172
+ return eval_adata
173
+
174
+
175
+ def eval_scib_metrics_simple(
176
+ adata: AnnData,
177
+ batch_key: str,
178
+ label_key: str,
179
+ embedding_key: str,
180
+ ) -> Dict[str, float]:
181
+ eval_adata = prepare_eval_adata(
182
+ adata,
183
+ batch_key,
184
+ label_key,
185
+ embedding_key,
186
+ build_neighbors=False,
187
+ )
188
+
189
+ results_dict: Dict[str, float] = {}
190
+ if len(eval_adata.obs[batch_key].unique()) > 1:
191
+ results_dict["ASW_label/batch"] = scib.metrics.silhouette_batch(
192
+ eval_adata,
193
+ batch_key,
194
+ label_key,
195
+ embed=embedding_key,
196
+ metric="euclidean",
197
+ return_all=False,
198
+ verbose=False,
199
+ )
200
+
201
+ results_dict = {key: value for key, value in results_dict.items() if not np.isnan(value)}
202
+ return results_dict
203
+
204
+
205
+ def eval_scib_metrics_scgpt(
206
+ adata: AnnData,
207
+ batch_key: str,
208
+ label_key: str,
209
+ embedding_key: str,
210
+ ) -> Dict[str, float]:
211
+ eval_adata = prepare_eval_adata(
212
+ adata,
213
+ batch_key,
214
+ label_key,
215
+ embedding_key,
216
+ build_neighbors=True,
217
+ )
218
+
219
+ results = scib.metrics.metrics(
220
+ eval_adata,
221
+ adata_int=eval_adata,
222
+ batch_key=batch_key,
223
+ label_key=label_key,
224
+ embed=embedding_key,
225
+ isolated_labels_asw_=False,
226
+ silhouette_=True,
227
+ hvg_score_=False,
228
+ graph_conn_=True,
229
+ pcr_=True,
230
+ isolated_labels_f1_=False,
231
+ trajectory_=False,
232
+ nmi_=True,
233
+ ari_=True,
234
+ cell_cycle_=False,
235
+ kBET_=False,
236
+ ilisi_=False,
237
+ clisi_=False,
238
+ )
239
+ result_dict = results[0].to_dict()
240
+
241
+ result_dict["avg_bio"] = np.mean(
242
+ [
243
+ result_dict["NMI_cluster/label"],
244
+ result_dict["ARI_cluster/label"],
245
+ result_dict["ASW_label"],
246
+ ]
247
+ )
248
+ result_dict["avg_batch"] = np.mean(
249
+ [
250
+ result_dict["graph_conn"],
251
+ result_dict["ASW_label/batch"],
252
+ ]
253
+ )
254
+
255
+ result_dict = {key: value for key, value in result_dict.items() if not np.isnan(value)}
256
+ return result_dict
257
+
258
+
259
+ def evaluate(
260
+ adata: AnnData,
261
+ batch_key: str,
262
+ label_keys: Sequence[str],
263
+ embedding_key: str,
264
+ metrics_mode: str,
265
+ res_path: str,
266
+ ) -> pd.DataFrame:
267
+ met_df = pd.DataFrame(columns=["metric", "label", "value"])
268
+
269
+ unique_label_keys: List[str] = []
270
+ for key in label_keys:
271
+ if key not in unique_label_keys:
272
+ unique_label_keys.append(key)
273
+
274
+ if not unique_label_keys:
275
+ raise ValueError("No label keys were provided for evaluation")
276
+
277
+ for label_key in unique_label_keys:
278
+ if metrics_mode == "scgpt":
279
+ metrics = eval_scib_metrics_scgpt(
280
+ adata=adata,
281
+ batch_key=batch_key,
282
+ label_key=label_key,
283
+ embedding_key=embedding_key,
284
+ )
285
+ elif metrics_mode == "simple":
286
+ metrics = eval_scib_metrics_simple(
287
+ adata=adata,
288
+ batch_key=batch_key,
289
+ label_key=label_key,
290
+ embedding_key=embedding_key,
291
+ )
292
+ else:
293
+ raise ValueError(f"Unknown metrics_mode: {metrics_mode}")
294
+
295
+ print_sys(
296
+ "\n".join(
297
+ [f"[{label_key}] {metric_name}: {metric_value:.4f}" for metric_name, metric_value in metrics.items()]
298
+ )
299
+ )
300
+ for metric_name, metric_value in metrics.items():
301
+ met_df.loc[len(met_df)] = [metric_name, label_key, metric_value]
302
+
303
+ met_df.to_csv(res_path, index=False)
304
+ np.save(f"{res_path[:-4]}_ce.npy", adata.obsm[embedding_key])
305
+ return met_df
306
+
307
+
308
+ def resolve_with_preset(
309
+ cli_value: Optional[object],
310
+ preset_value: Optional[object],
311
+ ) -> Optional[object]:
312
+ return cli_value if cli_value is not None else preset_value
313
+
314
+
315
+ def resolve_batch_key(adata: AnnData, requested_batch_key: Optional[str]) -> str:
316
+ if requested_batch_key is not None:
317
+ if requested_batch_key not in adata.obs.columns:
318
+ raise KeyError(f"batch_key '{requested_batch_key}' not found in adata.obs.columns")
319
+ return requested_batch_key
320
+
321
+ fallback_keys = ["batch", "str_batch", "sample", "Sample Characteristic[individual]"]
322
+ for key in fallback_keys:
323
+ if key in adata.obs.columns:
324
+ print_sys(f"Auto-selected batch_key='{key}'")
325
+ return key
326
+
327
+ raise ValueError(
328
+ "Unable to infer batch_key. Please pass --batch_key explicitly."
329
+ )
330
+
331
+
332
+ def resolve_label_keys(adata: AnnData, requested_label_keys: Optional[Sequence[str]]) -> List[str]:
333
+ if requested_label_keys:
334
+ missing_keys = [key for key in requested_label_keys if key not in adata.obs.columns]
335
+ if missing_keys:
336
+ raise KeyError(f"label_key(s) not found in adata.obs.columns: {missing_keys}")
337
+ return list(requested_label_keys)
338
+
339
+ fallback_keys = ["celltype", "cell_type", "celltype_original"]
340
+ resolved_keys = [key for key in fallback_keys if key in adata.obs.columns]
341
+ if resolved_keys:
342
+ print_sys(f"Auto-selected label_keys={resolved_keys}")
343
+ return resolved_keys
344
+
345
+ raise ValueError(
346
+ "Unable to infer label_key. Please pass --label_key explicitly."
347
+ )
348
+
349
+
350
+ def reindex_tensor_universal(
351
+ tensor: torch.Tensor,
352
+ index_positions: Sequence[int],
353
+ dim: int,
354
+ filler: str = "0",
355
+ device: Union[str, torch.device] = "cpu",
356
+ ) -> torch.Tensor:
357
+ index_tensor = torch.tensor(index_positions, device=device)
358
+ tensor_shape = list(tensor.shape)
359
+ tensor_shape[dim] = len(index_positions)
360
+ index_shape = [1] * len(tensor_shape)
361
+ index_shape[dim] = len(index_positions)
362
+ index_tensor = index_tensor.view(*index_shape).expand(*tensor_shape)
363
+
364
+ if filler == "0":
365
+ padder_shape = deepcopy(tensor_shape)
366
+ padder_shape[dim] = 1
367
+ padder = torch.zeros(padder_shape, dtype=tensor.dtype, device=device)
368
+ elif filler == "mean":
369
+ padder = tensor.mean(dim=dim, keepdim=True)
370
+ else:
371
+ raise ValueError("filler should be 0 or mean")
372
+ expanded_tensor = torch.cat((tensor, padder), dim=dim)
373
+ return torch.gather(expanded_tensor, dim, index_tensor)
374
+
375
+
376
+ project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
377
+ sys.path.insert(0, project_root)
378
+ sys.path.insert(0, os.path.dirname(project_root))
379
+
380
+ performer_module_dir = project_root
381
+
382
+ def load_module_modules():
383
+ models_path = os.path.join(performer_module_dir, "models_performer_module.py")
384
+ if not os.path.exists(models_path):
385
+ raise FileNotFoundError(f"Model file not found: {models_path}")
386
+ models_spec = importlib.util.spec_from_file_location("models_performer_module", models_path)
387
+ models_performer_module = importlib.util.module_from_spec(models_spec)
388
+ models_spec.loader.exec_module(models_performer_module)
389
+
390
+ dataset_path = os.path.join(performer_module_dir, "module_dataset.py")
391
+ if not os.path.exists(dataset_path):
392
+ raise FileNotFoundError(f"Dataset file not found: {dataset_path}")
393
+ dataset_spec = importlib.util.spec_from_file_location("module_dataset", dataset_path)
394
+ module_dataset = importlib.util.module_from_spec(dataset_spec)
395
+ dataset_spec.loader.exec_module(module_dataset)
396
+
397
+ return (
398
+ models_performer_module.PerformerEncoderOnly_Module,
399
+ models_performer_module.sanitize_performer_module_config,
400
+ module_dataset.load_module_data,
401
+ )
402
+
403
+ OUTPUT_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "res")
404
+ os.makedirs(OUTPUT_DIR, exist_ok=True)
405
+
406
+ parser = argparse.ArgumentParser()
407
+ parser.add_argument("--gene2vec_path", type=str, default=str(ASSET_DIR / "selected_gene2vec_27k.npy"),
408
+ help="Path to gene2vec file")
409
+ parser.add_argument("--scfm_hyper_params_path", type=str, required=True,
410
+ help="Path to the matching BioMod model configuration pkl file")
411
+ parser.add_argument("--scfm_ckpt_path", type=str, required=True,
412
+ help="Path to the BioMod model weights")
413
+ parser.add_argument("--module_data_path", type=str, default=None,
414
+ help="Path to module JSON file (single file)")
415
+ parser.add_argument("--module_data_dir", type=str, default=None,
416
+ help="Path to directory containing module JSON files")
417
+ parser.add_argument("--gene_mapping_path", type=str, default=str(ASSET_DIR / "gene2ens.csv"),
418
+ help="Path to gene symbol to Ensembl ID mapping CSV")
419
+ parser.add_argument("--scfm_genes_file", type=str, default=str(ASSET_DIR / "selected_genes_27k.txt"),
420
+ help="Path to scfm genes list file")
421
+ parser.add_argument("--data_path", type=str, required=True,
422
+ help="Path to input h5ad file")
423
+ parser.add_argument("--dataset_preset", type=str, default="auto", choices=sorted(DATASET_PRESETS.keys()),
424
+ help="Dataset preset for batch/label/gene column defaults")
425
+ parser.add_argument("--gene_col", type=str, default=None,
426
+ help="Column in adata.var used as the input gene symbol column. "
427
+ "If not set, use adata.var.index.")
428
+ parser.add_argument("--batch_key", type=str, default=None,
429
+ help="Column in adata.obs used as the batch label")
430
+ parser.add_argument("--label_key", nargs="+", default=None,
431
+ help="One or more columns in adata.obs used as evaluation labels")
432
+ parser.add_argument("--output_key", type=str, default="X_sclong_module",
433
+ help="Key name for storing embeddings in adata.obsm")
434
+ parser.add_argument("--pool_method", type=str, default="auto", choices=["auto", "mean", "cls", "max", "first_module"],
435
+ help="Cell embedding pooling. auto prefers cls_encoding when the checkpoint has a trained CLS token; "
436
+ "first_module uses the first module after restoring original module order.")
437
+ parser.add_argument("--n_cells", type=str, default="all",
438
+ help="Number of cells to process (default: all)")
439
+ parser.add_argument("--batch_size", type=int, default=4,
440
+ help="Batch size for inference")
441
+ parser.add_argument("--output_dir", type=str, default=OUTPUT_DIR,
442
+ help="Output directory for results")
443
+ parser.add_argument("--metrics_mode", type=str, default="scgpt", choices=["scgpt", "simple"],
444
+ help="Evaluation metrics mode. Default uses the scGPT tutorial metric bundle.")
445
+ parser.add_argument("--min_genes_per_cell", type=int, default=0,
446
+ help="Optional Scanpy cell filter. Default 0 keeps all cells.")
447
+ parser.add_argument("--min_cells_per_gene", type=int, default=0,
448
+ help="Optional Scanpy gene filter. Default 0 keeps all genes.")
449
+ parser.add_argument("--shuffle_module_genes", action="store_true",
450
+ help="Optional downstream-only extra reshuffle on top of module_sets restored from PKL. "
451
+ "Do not pass this when reproducing the pretrained backbone as-is.")
452
+ parser.add_argument("--shuffle_seed", type=int, default=2021,
453
+ help="Random seed for the downstream extra reshuffle when --shuffle_module_genes is enabled.")
454
+ parser.add_argument("--save_raw_module_emb", action="store_true",
455
+ help="Save raw module_encodings with shape (num_cells, num_modules, embed_dim)")
456
+
457
+ args = parser.parse_args()
458
+ os.makedirs(args.output_dir, exist_ok=True)
459
+
460
+ device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
461
+ print_sys(f"Using device: {device}")
462
+
463
+ preset_config = DATASET_PRESETS[args.dataset_preset]
464
+ resolved_gene_col = resolve_with_preset(args.gene_col, preset_config.get("gene_col"))
465
+ resolved_batch_key = resolve_with_preset(args.batch_key, preset_config.get("batch_key"))
466
+ resolved_label_keys = resolve_with_preset(args.label_key, preset_config.get("label_keys"))
467
+
468
+ print_sys(
469
+ f"dataset_preset={args.dataset_preset}, "
470
+ f"gene_col={resolved_gene_col}, batch_key={resolved_batch_key}, "
471
+ f"label_keys={resolved_label_keys}, metrics_mode={args.metrics_mode}, "
472
+ f"pool_method={args.pool_method}"
473
+ )
474
+
475
+ PerformerEncoderOnly_Module, sanitize_performer_module_config, load_module_data = load_module_modules()
476
+
477
+ print_sys(f"Loading hyper parameters from {args.scfm_hyper_params_path}")
478
+ with open(args.scfm_hyper_params_path, "rb") as handle:
479
+ scfm_hyper_params = pickle.load(handle)
480
+ scfm_hyper_params.pop("module_embed_chunk_padded_gene_budget", None)
481
+ scfm_hyper_params = sanitize_performer_module_config(
482
+ scfm_hyper_params,
483
+ source=args.scfm_hyper_params_path,
484
+ log_fn=print_sys,
485
+ )
486
+
487
+ scfm_hyper_params["gene2vec_file"] = args.gene2vec_path
488
+ # Read fusion_method from pkl to ensure consistency with pre-training
489
+ fusion_method = scfm_hyper_params.get("fusion_method", "multiply")
490
+ print_sys(f"Using fusion_method from pkl: {fusion_method}")
491
+
492
+ if "module_sets" not in scfm_hyper_params or scfm_hyper_params.get("module_sets") is None:
493
+ if args.module_data_path or args.module_data_dir:
494
+ print_sys("Loading module data...")
495
+ module_sets = load_module_data(
496
+ module_data_path=args.module_data_path,
497
+ module_data_dir=args.module_data_dir,
498
+ gene_list_path=args.scfm_genes_file,
499
+ gene_mapping_path=args.gene_mapping_path,
500
+ verbose=True,
501
+ )
502
+ scfm_hyper_params["module_sets"] = module_sets
503
+ else:
504
+ raise ValueError("Model requires module_sets, but neither --module_data_path nor --module_data_dir was provided")
505
+ else:
506
+ module_sets = scfm_hyper_params["module_sets"]
507
+ print_sys(f"Using module_sets from hyper parameters ({len(module_sets)} modules)")
508
+
509
+ if args.shuffle_module_genes:
510
+ import random
511
+
512
+ print_sys(
513
+ f"Applying downstream-only extra module reshuffle (seed={args.shuffle_seed}). "
514
+ f"Pretraining-time module_sets from PKL will be reshuffled again."
515
+ )
516
+ all_gene_indices: List[int] = []
517
+ module_sizes: List[int] = []
518
+ for module in module_sets:
519
+ all_gene_indices.extend(module)
520
+ module_sizes.append(len(module))
521
+
522
+ rng = random.Random(args.shuffle_seed)
523
+ rng.shuffle(all_gene_indices)
524
+
525
+ shuffled_module_sets = []
526
+ start_idx = 0
527
+ for size in module_sizes:
528
+ shuffled_module_sets.append(all_gene_indices[start_idx:start_idx + size])
529
+ start_idx += size
530
+ module_sets = shuffled_module_sets
531
+ scfm_hyper_params["module_sets"] = module_sets
532
+ print_sys(f"Shuffled {len(all_gene_indices)} genes across {len(module_sets)} modules")
533
+
534
+ print_sys(f"Model hyper parameters: {list(scfm_hyper_params.keys())}")
535
+
536
+ pkl_use_gene2vec = scfm_hyper_params.get("use_gene2vec", True)
537
+ scfm_hyper_params["use_gene2vec"] = pkl_use_gene2vec
538
+ print_sys(f"use_gene2vec={pkl_use_gene2vec} (from PKL)")
539
+
540
+ print_sys("Creating PerformerEncoderOnly_Module model...")
541
+ model = PerformerEncoderOnly_Module(**scfm_hyper_params).to(device)
542
+
543
+ print_sys(f"Loading checkpoint from {args.scfm_ckpt_path}")
544
+ ckpt = torch.load(args.scfm_ckpt_path, map_location=device)
545
+ model_state_dict = ckpt.get("model_state_dict", ckpt)
546
+ filtered_state_dict = {
547
+ key: value
548
+ for key, value in model_state_dict.items()
549
+ if "go_conv" not in key and "G_go" not in key.lower()
550
+ }
551
+ model.load_state_dict(filtered_state_dict, strict=True)
552
+
553
+ for parameter in model.parameters():
554
+ parameter.requires_grad = False
555
+ model.eval()
556
+
557
+ all_params = sum(parameter.numel() for parameter in model.parameters())
558
+ trainable_params = sum(parameter.numel() for parameter in model.parameters() if parameter.requires_grad)
559
+ print_sys(f"Total model parameters: {all_params:,}")
560
+ print_sys(f"Trainable parameters during zero-shot: {trainable_params:,}")
561
+
562
+ gene2ens = pd.read_csv(args.gene_mapping_path, index_col=0)
563
+ symbol_to_ensembl = gene2ens["Ensembl gene ID"].to_dict()
564
+
565
+ with open(args.scfm_genes_file, "r") as handle:
566
+ scfm_genes = [line.rstrip("\n") for line in handle.readlines()]
567
+ scfm_gene_set = set(scfm_genes)
568
+
569
+ print_sys(f"Loading data from {args.data_path}")
570
+ adata = sc.read_h5ad(args.data_path)
571
+
572
+ if args.min_genes_per_cell > 0:
573
+ before_cells = adata.n_obs
574
+ sc.pp.filter_cells(adata, min_genes=args.min_genes_per_cell)
575
+ print_sys(f"Filtered cells by min_genes={args.min_genes_per_cell}: {before_cells} -> {adata.n_obs}")
576
+
577
+ if args.min_cells_per_gene > 0:
578
+ before_genes = adata.n_vars
579
+ sc.pp.filter_genes(adata, min_cells=args.min_cells_per_gene)
580
+ print_sys(f"Filtered genes by min_cells={args.min_cells_per_gene}: {before_genes} -> {adata.n_vars}")
581
+
582
+ if args.n_cells == "all":
583
+ n_cells = adata.n_obs
584
+ else:
585
+ n_cells = int(args.n_cells)
586
+ if adata.n_obs > n_cells:
587
+ print_sys(f"Subsampling to {n_cells} cells")
588
+ sc.pp.subsample(adata, n_obs=n_cells, copy=False)
589
+ n_cells = adata.n_obs
590
+
591
+ batch_key = resolve_batch_key(adata, resolved_batch_key)
592
+ label_keys = resolve_label_keys(adata, resolved_label_keys)
593
+
594
+ print_sys(f"Processing {n_cells} cells with {adata.n_vars} genes")
595
+ print_sys(f"Evaluation batch_key={batch_key}, label_keys={label_keys}")
596
+
597
+ input_gene_ids = resolve_input_gene_ids(
598
+ adata=adata,
599
+ gene_col=resolved_gene_col,
600
+ symbol_to_ensembl=symbol_to_ensembl,
601
+ target_gene_set=scfm_gene_set,
602
+ )
603
+
604
+ input_mapping = {gene_id: index for index, gene_id in enumerate(input_gene_ids)}
605
+ input_gene_num = len(input_gene_ids)
606
+
607
+ intersect_genes = np.intersect1d(input_gene_ids, scfm_genes)
608
+ print_sys(f"Input genes and scfm genes intersect: {len(intersect_genes)} genes")
609
+
610
+ scfm_genes_pad = scfm_genes + ["PAD"]
611
+ scfm_seq_len = len(scfm_genes_pad)
612
+ scfm_mapping = {gene_id: index for index, gene_id in enumerate(scfm_genes_pad)}
613
+
614
+ scfm_index_positions = [input_mapping.get(gene_id, input_gene_num) for gene_id in scfm_genes_pad]
615
+ _ = [scfm_mapping.get(gene_id, scfm_seq_len) for gene_id in input_gene_ids]
616
+
617
+ print_sys("Extracting cell embeddings...")
618
+ cell_embeddings = []
619
+ raw_module_embeddings = [] if args.save_raw_module_emb else None
620
+ embedding_key = args.output_key
621
+
622
+ with torch.no_grad():
623
+ for start in tqdm(range(0, n_cells, args.batch_size)):
624
+ batch_end = min(start + args.batch_size, n_cells)
625
+ batch_matrix = adata.X[start:batch_end, :]
626
+ if hasattr(batch_matrix, "toarray"):
627
+ batch_matrix = batch_matrix.toarray()
628
+ x = torch.tensor(batch_matrix, dtype=torch.float32).to(device)
629
+
630
+ x_scfm = reindex_tensor_universal(x, scfm_index_positions, dim=1, filler="0", device=device)
631
+ out = model(x_scfm, mask_ratio=0.0, return_encodings=True)
632
+
633
+ if "module_encodings" not in out:
634
+ raise ValueError(f"Expected 'module_encodings' in output, got: {list(out.keys())}")
635
+
636
+ module_encodings = out["module_encodings"]
637
+ ids_restore = out.get("ids_restore")
638
+ if args.pool_method == "cls":
639
+ if "cls_encoding" not in out:
640
+ print_sys("Warning: pool_method='cls' but checkpoint has no cls_encoding, using mean pooling instead")
641
+ cell_emb = module_encodings.mean(dim=1)
642
+ else:
643
+ cell_emb = out["cls_encoding"]
644
+ elif args.pool_method == "mean":
645
+ cell_emb = module_encodings.mean(dim=1)
646
+ elif args.pool_method == "max":
647
+ cell_emb = module_encodings.max(dim=1)[0]
648
+ elif args.pool_method == "first_module":
649
+ if ids_restore is None:
650
+ raise ValueError("pool_method='first_module' requires ids_restore in model outputs")
651
+ restore_index = ids_restore.unsqueeze(-1).expand(-1, -1, module_encodings.shape[-1])
652
+ restored_module_encodings = torch.gather(module_encodings, dim=1, index=restore_index)
653
+ cell_emb = restored_module_encodings[:, 0, :]
654
+ else:
655
+ cell_emb = out["cls_encoding"] if "cls_encoding" in out else module_encodings.mean(dim=1)
656
+
657
+ if args.save_raw_module_emb:
658
+ raw_module_embeddings.append(module_encodings.cpu().numpy())
659
+
660
+ cell_embeddings.append(cell_emb.cpu().numpy())
661
+
662
+ cell_embeddings = np.concatenate(cell_embeddings, axis=0)
663
+ print_sys(f"Cell embeddings shape: {cell_embeddings.shape}")
664
+
665
+ if args.save_raw_module_emb and raw_module_embeddings is not None:
666
+ raw_module_embeddings = np.concatenate(raw_module_embeddings, axis=0)
667
+ print_sys(f"Raw module embeddings shape: {raw_module_embeddings.shape}")
668
+ raw_module_path = os.path.join(args.output_dir, "sclong_module_raw_module_emb.npy")
669
+ np.save(raw_module_path, raw_module_embeddings)
670
+ print_sys(f"Raw module embeddings saved to {raw_module_path}")
671
+
672
+ adata.obsm[embedding_key] = cell_embeddings
673
+
674
+ print_sys("Evaluating batch correction...")
675
+ res = evaluate(
676
+ adata=adata,
677
+ batch_key=batch_key,
678
+ label_keys=label_keys,
679
+ embedding_key=embedding_key,
680
+ metrics_mode=args.metrics_mode,
681
+ res_path=os.path.join(args.output_dir, "sclong_module_batch_cell_emb_mode.csv"),
682
+ )
683
+
684
+ print_sys("\nEvaluation Results:")
685
+ print_sys("=" * 50)
686
+ print_sys(res.to_string(index=False))
687
+ print_sys("=" * 50)
688
+
689
+ np.save(os.path.join(args.output_dir, "sclong_module_batch_cell_emb_mode_ce.npy"), cell_embeddings)
690
+ print_sys(
691
+ f"Cell embeddings saved to "
692
+ f"{os.path.join(args.output_dir, 'sclong_module_batch_cell_emb_mode_ce.npy')}"
693
+ )
694
+ print_sys("Zero-shot evaluation completed!")
BioMod/zero_shot_batch/sequencing_depth/cmd.txt CHANGED
@@ -1,41 +1,42 @@
1
- python make_low_depth_h5ad.py \
2
- --input_path ../Kim2020_Lung.h5ad \
3
- --output_dir ./data/Kim2020_Lung \
4
- --keep_ratios 0.5,0.25,0.1,0.05 \
5
- --seeds 0,1,2
6
-
7
- python run_zero_shot_low_depth.py \
8
- --data_paths \
9
- ../Kim2020_Lung.h5ad \
10
- ./data/Kim2020_Lung/*.h5ad \
11
- --scfm_ckpt_path ../../init_models/performer_module_multiply_all_module_init.pth \
12
- --scfm_hyper_params_path ../../init_models/performer_module_multiply_all_module_init.pkl \
13
- --output_root ./results \
14
- --gene_mapping_path ../../gene2ens.csv \
15
- --scfm_genes_file ../../../selected_genes_27k.txt \
16
- --module_data_dir ../../../module_data_all \
17
- --dataset_preset lung \
18
- --output_key X_sclong_module \
19
- --batch_size 256 \
20
- --metrics_mode scgpt \
21
- --pool_method max
22
-
23
-
24
- python run_zero_shot_low_depth.py \
25
- --data_paths \
26
- ../Kim2020_Lung.h5ad \
27
- ./data/Kim2020_Lung/Kim2020_Lung_ld0p5_seed0.h5ad \
28
- ./data/Kim2020_Lung/Kim2020_Lung_ld0p25_seed0.h5ad \
29
- ./data/Kim2020_Lung/Kim2020_Lung_ld0p1_seed0.h5ad \
30
- ./data/Kim2020_Lung/Kim2020_Lung_ld0p05_seed0.h5ad \
31
- --scfm_ckpt_path ../../init_models/performer_module_multiply_all_module_init.pth \
32
- --scfm_hyper_params_path ../../init_models/performer_module_multiply_all_module_init.pkl \
33
- --output_root ./results \
34
- --gene_mapping_path ../../../gene2ens.csv \
35
- --scfm_genes_file ../../../selected_genes_27k.txt \
36
- --module_data_dir ../../module_data_all \
37
- --dataset_preset lung \
38
- --output_key X_sclong_module \
39
- --batch_size 256 \
40
- --metrics_mode scgpt \
 
41
  --pool_method max
 
1
+ # Run from the repository root. The selected input/layer must contain raw integer counts.
2
+ python BioMod/zero_shot_batch/sequencing_depth/make_low_depth_h5ad.py \
3
+ --input_path data/downstream/final_data/Kim2020_Lung.h5ad \
4
+ --output_dir tutorial_runs/sequencing_depth/inputs \
5
+ --keep_ratios 0.5,0.25,0.1,0.05 \
6
+ --seeds 0,1,2
7
+
8
+ python BioMod/zero_shot_batch/sequencing_depth/run_zero_shot_low_depth.py \
9
+ --data_paths \
10
+ data/downstream/final_data/Kim2020_Lung.h5ad \
11
+ tutorial_runs/sequencing_depth/inputs/*.h5ad \
12
+ --scfm_ckpt_path weight/0810model/mae_tost_70M_75mask_ec2_epoch0.pth \
13
+ --scfm_hyper_params_path weight/0810model/mae_tost_70M_75mask_ec2.pkl \
14
+ --output_root tutorial_runs/sequencing_depth/results \
15
+ --gene_mapping_path model_dependencies/gene2ens.csv \
16
+ --scfm_genes_file model_dependencies/selected_genes_27k.txt \
17
+ --module_data_dir model_dependencies/module_data_all \
18
+ --dataset_preset lung \
19
+ --output_key X_sclong_module \
20
+ --batch_size 256 \
21
+ --metrics_mode scgpt \
22
+ --pool_method max
23
+
24
+
25
+ python BioMod/zero_shot_batch/sequencing_depth/run_zero_shot_low_depth.py \
26
+ --data_paths \
27
+ data/downstream/final_data/Kim2020_Lung.h5ad \
28
+ tutorial_runs/sequencing_depth/inputs/Kim2020_Lung_ld0p5_seed0.h5ad \
29
+ tutorial_runs/sequencing_depth/inputs/Kim2020_Lung_ld0p25_seed0.h5ad \
30
+ tutorial_runs/sequencing_depth/inputs/Kim2020_Lung_ld0p1_seed0.h5ad \
31
+ tutorial_runs/sequencing_depth/inputs/Kim2020_Lung_ld0p05_seed0.h5ad \
32
+ --scfm_ckpt_path weight/0810model/mae_tost_70M_75mask_ec2_epoch0.pth \
33
+ --scfm_hyper_params_path weight/0810model/mae_tost_70M_75mask_ec2.pkl \
34
+ --output_root tutorial_runs/sequencing_depth/results \
35
+ --gene_mapping_path model_dependencies/gene2ens.csv \
36
+ --scfm_genes_file model_dependencies/selected_genes_27k.txt \
37
+ --module_data_dir model_dependencies/module_data_all \
38
+ --dataset_preset lung \
39
+ --output_key X_sclong_module \
40
+ --batch_size 256 \
41
+ --metrics_mode scgpt \
42
  --pool_method max
BioMod/zero_shot_batch/sequencing_depth/run_zero_shot_low_depth.py CHANGED
@@ -1,196 +1,196 @@
1
- #!/usr/bin/env python
2
- # -*- coding: utf-8 -*-
3
-
4
- """Batch-run performer_module zero-shot on full/low-depth h5ad files and aggregate metrics."""
5
-
6
- import argparse
7
- import re
8
- import subprocess
9
- import sys
10
- from pathlib import Path
11
- from typing import Iterable, List, Optional
12
-
13
- import pandas as pd
14
-
15
-
16
- def parse_args() -> argparse.Namespace:
17
- parser = argparse.ArgumentParser(
18
- description="Run performer_module zero-shot on multiple full/low-depth h5ad files and aggregate metrics."
19
- )
20
- parser.add_argument("--script_path", type=str, default=None, help="Path to performer_module_zero_shot.py")
21
- parser.add_argument("--data_paths", nargs="+", required=True, help="One or more full/low-depth h5ad files")
22
- parser.add_argument("--output_root", type=str, required=True, help="Root directory for per-run outputs")
23
- parser.add_argument("--scfm_hyper_params_path", type=str, required=True)
24
- parser.add_argument("--scfm_ckpt_path", type=str, required=True)
25
- parser.add_argument("--module_data_path", type=str, default=None)
26
- parser.add_argument("--module_data_dir", type=str, default=None)
27
- parser.add_argument("--gene_mapping_path", type=str, required=True)
28
- parser.add_argument("--scfm_genes_file", type=str, required=True)
29
- parser.add_argument("--gene2vec_path", type=str, default="selected_gene2vec_27k.npy")
30
- parser.add_argument("--dataset_preset", type=str, default="lung")
31
- parser.add_argument("--gene_col", type=str, default=None)
32
- parser.add_argument("--batch_key", type=str, default=None)
33
- parser.add_argument("--label_key", nargs="+", default=None)
34
- parser.add_argument("--output_key", type=str, default="X_sclong_module")
35
- parser.add_argument("--pool_method", type=str, default="auto")
36
- parser.add_argument("--n_cells", type=str, default="all")
37
- parser.add_argument("--batch_size", type=int, default=4)
38
- parser.add_argument("--metrics_mode", type=str, default="scgpt")
39
- parser.add_argument("--min_genes_per_cell", type=int, default=0)
40
- parser.add_argument("--min_cells_per_gene", type=int, default=0)
41
- parser.add_argument("--shuffle_module_genes", action="store_true")
42
- parser.add_argument("--shuffle_seed", type=int, default=2021)
43
- parser.add_argument("--save_raw_module_emb", action="store_true")
44
- parser.add_argument("--python_bin", type=str, default=sys.executable)
45
- parser.add_argument("--overwrite", action="store_true", help="Re-run even if result CSV already exists")
46
- return parser.parse_args()
47
-
48
-
49
- def resolve_script_path(script_path: Optional[str]) -> Path:
50
- if script_path is not None:
51
- return Path(script_path).resolve()
52
- return (Path(__file__).resolve().parent.parent / "performer_module_zero_shot.py").resolve()
53
-
54
-
55
- def slugify_path(path: Path) -> str:
56
- return re.sub(r"[^A-Za-z0-9._-]+", "_", path.stem)
57
-
58
-
59
- def build_command(args: argparse.Namespace, script_path: Path, data_path: Path, output_dir: Path) -> List[str]:
60
- cmd = [
61
- args.python_bin,
62
- str(script_path),
63
- "--data_path",
64
- str(data_path),
65
- "--output_dir",
66
- str(output_dir),
67
- "--scfm_hyper_params_path",
68
- args.scfm_hyper_params_path,
69
- "--scfm_ckpt_path",
70
- args.scfm_ckpt_path,
71
- "--gene_mapping_path",
72
- args.gene_mapping_path,
73
- "--scfm_genes_file",
74
- args.scfm_genes_file,
75
- "--gene2vec_path",
76
- args.gene2vec_path,
77
- "--dataset_preset",
78
- args.dataset_preset,
79
- "--output_key",
80
- args.output_key,
81
- "--pool_method",
82
- args.pool_method,
83
- "--n_cells",
84
- args.n_cells,
85
- "--batch_size",
86
- str(args.batch_size),
87
- "--metrics_mode",
88
- args.metrics_mode,
89
- "--min_genes_per_cell",
90
- str(args.min_genes_per_cell),
91
- "--min_cells_per_gene",
92
- str(args.min_cells_per_gene),
93
- ]
94
-
95
- if args.module_data_path:
96
- cmd.extend(["--module_data_path", args.module_data_path])
97
- if args.module_data_dir:
98
- cmd.extend(["--module_data_dir", args.module_data_dir])
99
- if args.gene_col:
100
- cmd.extend(["--gene_col", args.gene_col])
101
- if args.batch_key:
102
- cmd.extend(["--batch_key", args.batch_key])
103
- if args.label_key:
104
- cmd.extend(["--label_key", *args.label_key])
105
- if args.shuffle_module_genes:
106
- cmd.append("--shuffle_module_genes")
107
- cmd.extend(["--shuffle_seed", str(args.shuffle_seed)])
108
- if args.save_raw_module_emb:
109
- cmd.append("--save_raw_module_emb")
110
-
111
- return cmd
112
-
113
-
114
- def parse_low_depth_tags(path: Path) -> dict:
115
- stem = path.stem
116
- keep_ratio = None
117
- seed = None
118
- match = re.search(r"_ld([0-9]+p[0-9]+|[0-9]+)_seed([0-9]+)", stem)
119
- if match:
120
- keep_ratio = float(match.group(1).replace("p", "."))
121
- seed = int(match.group(2))
122
- return {
123
- "input_file": path.name,
124
- "input_stem": stem,
125
- "keep_ratio": keep_ratio,
126
- "seed": seed,
127
- "is_low_depth": keep_ratio is not None,
128
- }
129
-
130
-
131
- def load_metrics(csv_path: Path) -> pd.DataFrame:
132
- df = pd.read_csv(csv_path)
133
- required_columns = {"metric", "label", "value"}
134
- missing = required_columns - set(df.columns)
135
- if missing:
136
- raise ValueError(f"Metrics CSV missing required columns {sorted(missing)}: {csv_path}")
137
- return df
138
-
139
-
140
- def build_summary(records: Iterable[pd.DataFrame]) -> pd.DataFrame:
141
- frames = list(records)
142
- if not frames:
143
- return pd.DataFrame()
144
- summary = pd.concat(frames, ignore_index=True)
145
-
146
- baseline = summary[summary["is_low_depth"] == False][["metric", "label", "value"]].copy()
147
- baseline = baseline.rename(columns={"value": "baseline_value"})
148
- baseline = baseline.drop_duplicates(subset=["metric", "label"])
149
- summary = summary.merge(baseline, on=["metric", "label"], how="left")
150
- summary["relative_to_full"] = summary["value"] / summary["baseline_value"]
151
- return summary
152
-
153
-
154
- def main() -> None:
155
- args = parse_args()
156
- script_path = resolve_script_path(args.script_path)
157
- output_root = Path(args.output_root).resolve()
158
- output_root.mkdir(parents=True, exist_ok=True)
159
-
160
- if not script_path.exists():
161
- raise FileNotFoundError(f"Zero-shot script not found: {script_path}")
162
-
163
- per_run_tables: List[pd.DataFrame] = []
164
- for raw_data_path in args.data_paths:
165
- data_path = Path(raw_data_path).resolve()
166
- if not data_path.exists():
167
- raise FileNotFoundError(f"Input h5ad not found: {data_path}")
168
-
169
- run_name = slugify_path(data_path)
170
- run_dir = output_root / run_name
171
- run_dir.mkdir(parents=True, exist_ok=True)
172
- metrics_csv = run_dir / "sclong_module_batch_cell_emb_mode.csv"
173
-
174
- if args.overwrite or (not metrics_csv.exists()):
175
- cmd = build_command(args, script_path, data_path, run_dir)
176
- print(f"[runner] running {' '.join(cmd)}", flush=True)
177
- subprocess.run(cmd, check=True)
178
- else:
179
- print(f"[runner] skip existing run: {run_dir}", flush=True)
180
-
181
- metrics_df = load_metrics(metrics_csv)
182
- tag_info = parse_low_depth_tags(data_path)
183
- metrics_df["run_name"] = run_name
184
- metrics_df["data_path"] = str(data_path)
185
- for key, value in tag_info.items():
186
- metrics_df[key] = value
187
- per_run_tables.append(metrics_df)
188
-
189
- summary = build_summary(per_run_tables)
190
- summary_csv = output_root / "zero_shot_low_depth_summary.csv"
191
- summary.to_csv(summary_csv, index=False)
192
- print(f"[runner] saved summary -> {summary_csv}", flush=True)
193
-
194
-
195
- if __name__ == "__main__":
196
- main()
 
1
+ #!/usr/bin/env python
2
+ # -*- coding: utf-8 -*-
3
+
4
+ """Batch-run performer_module zero-shot on full/low-depth h5ad files and aggregate metrics."""
5
+
6
+ import argparse
7
+ import re
8
+ import subprocess
9
+ import sys
10
+ from pathlib import Path
11
+ from typing import Iterable, List, Optional
12
+
13
+ import pandas as pd
14
+
15
+
16
+ def parse_args() -> argparse.Namespace:
17
+ parser = argparse.ArgumentParser(
18
+ description="Run performer_module zero-shot on multiple full/low-depth h5ad files and aggregate metrics."
19
+ )
20
+ parser.add_argument("--script_path", type=str, default=None, help="Path to performer_module_zero_shot.py")
21
+ parser.add_argument("--data_paths", nargs="+", required=True, help="One or more full/low-depth h5ad files")
22
+ parser.add_argument("--output_root", type=str, required=True, help="Root directory for per-run outputs")
23
+ parser.add_argument("--scfm_hyper_params_path", type=str, required=True)
24
+ parser.add_argument("--scfm_ckpt_path", type=str, required=True)
25
+ parser.add_argument("--module_data_path", type=str, default=None)
26
+ parser.add_argument("--module_data_dir", type=str, default=None)
27
+ parser.add_argument("--gene_mapping_path", type=str, required=True)
28
+ parser.add_argument("--scfm_genes_file", type=str, required=True)
29
+ parser.add_argument("--gene2vec_path", type=str, default=str(Path(__file__).resolve().parents[3] / "model_dependencies" / "selected_gene2vec_27k.npy"))
30
+ parser.add_argument("--dataset_preset", type=str, default="lung")
31
+ parser.add_argument("--gene_col", type=str, default=None)
32
+ parser.add_argument("--batch_key", type=str, default=None)
33
+ parser.add_argument("--label_key", nargs="+", default=None)
34
+ parser.add_argument("--output_key", type=str, default="X_sclong_module")
35
+ parser.add_argument("--pool_method", type=str, default="auto")
36
+ parser.add_argument("--n_cells", type=str, default="all")
37
+ parser.add_argument("--batch_size", type=int, default=4)
38
+ parser.add_argument("--metrics_mode", type=str, default="scgpt")
39
+ parser.add_argument("--min_genes_per_cell", type=int, default=0)
40
+ parser.add_argument("--min_cells_per_gene", type=int, default=0)
41
+ parser.add_argument("--shuffle_module_genes", action="store_true")
42
+ parser.add_argument("--shuffle_seed", type=int, default=2021)
43
+ parser.add_argument("--save_raw_module_emb", action="store_true")
44
+ parser.add_argument("--python_bin", type=str, default=sys.executable)
45
+ parser.add_argument("--overwrite", action="store_true", help="Re-run even if result CSV already exists")
46
+ return parser.parse_args()
47
+
48
+
49
+ def resolve_script_path(script_path: Optional[str]) -> Path:
50
+ if script_path is not None:
51
+ return Path(script_path).resolve()
52
+ return (Path(__file__).resolve().parent.parent / "performer_module_zero_shot.py").resolve()
53
+
54
+
55
+ def slugify_path(path: Path) -> str:
56
+ return re.sub(r"[^A-Za-z0-9._-]+", "_", path.stem)
57
+
58
+
59
+ def build_command(args: argparse.Namespace, script_path: Path, data_path: Path, output_dir: Path) -> List[str]:
60
+ cmd = [
61
+ args.python_bin,
62
+ str(script_path),
63
+ "--data_path",
64
+ str(data_path),
65
+ "--output_dir",
66
+ str(output_dir),
67
+ "--scfm_hyper_params_path",
68
+ args.scfm_hyper_params_path,
69
+ "--scfm_ckpt_path",
70
+ args.scfm_ckpt_path,
71
+ "--gene_mapping_path",
72
+ args.gene_mapping_path,
73
+ "--scfm_genes_file",
74
+ args.scfm_genes_file,
75
+ "--gene2vec_path",
76
+ args.gene2vec_path,
77
+ "--dataset_preset",
78
+ args.dataset_preset,
79
+ "--output_key",
80
+ args.output_key,
81
+ "--pool_method",
82
+ args.pool_method,
83
+ "--n_cells",
84
+ args.n_cells,
85
+ "--batch_size",
86
+ str(args.batch_size),
87
+ "--metrics_mode",
88
+ args.metrics_mode,
89
+ "--min_genes_per_cell",
90
+ str(args.min_genes_per_cell),
91
+ "--min_cells_per_gene",
92
+ str(args.min_cells_per_gene),
93
+ ]
94
+
95
+ if args.module_data_path:
96
+ cmd.extend(["--module_data_path", args.module_data_path])
97
+ if args.module_data_dir:
98
+ cmd.extend(["--module_data_dir", args.module_data_dir])
99
+ if args.gene_col:
100
+ cmd.extend(["--gene_col", args.gene_col])
101
+ if args.batch_key:
102
+ cmd.extend(["--batch_key", args.batch_key])
103
+ if args.label_key:
104
+ cmd.extend(["--label_key", *args.label_key])
105
+ if args.shuffle_module_genes:
106
+ cmd.append("--shuffle_module_genes")
107
+ cmd.extend(["--shuffle_seed", str(args.shuffle_seed)])
108
+ if args.save_raw_module_emb:
109
+ cmd.append("--save_raw_module_emb")
110
+
111
+ return cmd
112
+
113
+
114
+ def parse_low_depth_tags(path: Path) -> dict:
115
+ stem = path.stem
116
+ keep_ratio = None
117
+ seed = None
118
+ match = re.search(r"_ld([0-9]+p[0-9]+|[0-9]+)_seed([0-9]+)", stem)
119
+ if match:
120
+ keep_ratio = float(match.group(1).replace("p", "."))
121
+ seed = int(match.group(2))
122
+ return {
123
+ "input_file": path.name,
124
+ "input_stem": stem,
125
+ "keep_ratio": keep_ratio,
126
+ "seed": seed,
127
+ "is_low_depth": keep_ratio is not None,
128
+ }
129
+
130
+
131
+ def load_metrics(csv_path: Path) -> pd.DataFrame:
132
+ df = pd.read_csv(csv_path)
133
+ required_columns = {"metric", "label", "value"}
134
+ missing = required_columns - set(df.columns)
135
+ if missing:
136
+ raise ValueError(f"Metrics CSV missing required columns {sorted(missing)}: {csv_path}")
137
+ return df
138
+
139
+
140
+ def build_summary(records: Iterable[pd.DataFrame]) -> pd.DataFrame:
141
+ frames = list(records)
142
+ if not frames:
143
+ return pd.DataFrame()
144
+ summary = pd.concat(frames, ignore_index=True)
145
+
146
+ baseline = summary[summary["is_low_depth"] == False][["metric", "label", "value"]].copy()
147
+ baseline = baseline.rename(columns={"value": "baseline_value"})
148
+ baseline = baseline.drop_duplicates(subset=["metric", "label"])
149
+ summary = summary.merge(baseline, on=["metric", "label"], how="left")
150
+ summary["relative_to_full"] = summary["value"] / summary["baseline_value"]
151
+ return summary
152
+
153
+
154
+ def main() -> None:
155
+ args = parse_args()
156
+ script_path = resolve_script_path(args.script_path)
157
+ output_root = Path(args.output_root).resolve()
158
+ output_root.mkdir(parents=True, exist_ok=True)
159
+
160
+ if not script_path.exists():
161
+ raise FileNotFoundError(f"Zero-shot script not found: {script_path}")
162
+
163
+ per_run_tables: List[pd.DataFrame] = []
164
+ for raw_data_path in args.data_paths:
165
+ data_path = Path(raw_data_path).resolve()
166
+ if not data_path.exists():
167
+ raise FileNotFoundError(f"Input h5ad not found: {data_path}")
168
+
169
+ run_name = slugify_path(data_path)
170
+ run_dir = output_root / run_name
171
+ run_dir.mkdir(parents=True, exist_ok=True)
172
+ metrics_csv = run_dir / "sclong_module_batch_cell_emb_mode.csv"
173
+
174
+ if args.overwrite or (not metrics_csv.exists()):
175
+ cmd = build_command(args, script_path, data_path, run_dir)
176
+ print(f"[runner] running {' '.join(cmd)}", flush=True)
177
+ subprocess.run(cmd, check=True)
178
+ else:
179
+ print(f"[runner] skip existing run: {run_dir}", flush=True)
180
+
181
+ metrics_df = load_metrics(metrics_csv)
182
+ tag_info = parse_low_depth_tags(data_path)
183
+ metrics_df["run_name"] = run_name
184
+ metrics_df["data_path"] = str(data_path)
185
+ for key, value in tag_info.items():
186
+ metrics_df[key] = value
187
+ per_run_tables.append(metrics_df)
188
+
189
+ summary = build_summary(per_run_tables)
190
+ summary_csv = output_root / "zero_shot_low_depth_summary.csv"
191
+ summary.to_csv(summary_csv, index=False)
192
+ print(f"[runner] saved summary -> {summary_csv}", flush=True)
193
+
194
+
195
+ if __name__ == "__main__":
196
+ main()
BioMod/zero_shot_batch/visualize_results.py CHANGED
@@ -1,294 +1,294 @@
1
- """
2
- Zero-shot batch correction visual code
3
-
4
- Used to generate the following images:
5
- 1. umap_cell_embeddings.png - UMAP visualization of cell embeddings (colored by cell type and batch)
6
- 2. umap_pca50.png - PCA dimensionality reduction to 50 dimensions and then UMAP visualization
7
- 3. batch_ASW_comparison.png - Batch ASW score comparison bar chart for different models/settings
8
-
9
- Enter data:
10
- - sclong_module_batch_cell_emb_mode_ce.npy: cell embedding vector
11
- - sclong_module_batch_cell_emb_mode.csv: ASW score
12
- - pancreas_scib.h5ad: original data (including celltype and batch information)
13
-
14
- Calculation method:
15
- 1. umap_cell_embeddings: directly perform UMAP dimensionality reduction on cell embedding vectors
16
- 2. umap_pca50: First use PCA to reduce the dimension to 50 dimensions, and then perform UMAP dimension reduction.
17
- 3. batch_ASW: Use scib.metrics.silhouette_batch to calculate the batch ASW score
18
- """
19
-
20
- import os
21
- import numpy as np
22
- import pandas as pd
23
- import scanpy as sc
24
- import matplotlib.pyplot as plt
25
- import seaborn as sns
26
- from sklearn.decomposition import PCA
27
- import argparse
28
-
29
- # Set drawing style
30
- plt.rcParams['font.family'] = 'DejaVu Sans'
31
- plt.rcParams['font.size'] = 10
32
- plt.rcParams['figure.dpi'] = 150
33
-
34
- def plot_umap_cell_embeddings(
35
- cell_embeddings: np.ndarray,
36
- adata: sc.AnnData,
37
- output_path: str,
38
- embedding_key: str = "X_sclong_module"
39
- ):
40
- """
41
- Draw cell-embedded UMAP visualization
42
-
43
- parameter:
44
- - cell_embeddings: cell embedding vector (n_cells, embed_dim)
45
- - adata: AnnData object, containing celltype and batch information
46
- - output_path: output image path
47
- - embedding_key: the key name embedded in adata.obsm
48
- """
49
- print(f"Plotting UMAP of cell embeddings...")
50
-
51
- # Store embedding vector into adata
52
- adata.obsm[embedding_key] = cell_embeddings
53
-
54
- # Calculate neighbors and UMAP
55
- sc.pp.neighbors(adata, use_rep=embedding_key, n_neighbors=30, n_pcs=None)
56
- sc.tl.umap(adata, min_dist=0.1)
57
-
58
- # Create graph - two subgraphs: color by celltype and by batch
59
- fig, axes = plt.subplots(1, 2, figsize=(14, 6))
60
-
61
- # Subfigure 1: Coloring by celltype
62
- sc.pl.umap(
63
- adata,
64
- color='celltype',
65
- ax=axes[0],
66
- show=False,
67
- title='Cell Type',
68
- frameon=True
69
- )
70
-
71
- # Subpicture 2: Coloring by batch
72
- sc.pl.umap(
73
- adata,
74
- color='batch',
75
- ax=axes[1],
76
- show=False,
77
- title='Batch',
78
- frameon=True
79
- )
80
-
81
- plt.tight_layout()
82
- plt.savefig(output_path, dpi=150, bbox_inches='tight')
83
- plt.close()
84
-
85
- print(f"Saved: {output_path}")
86
-
87
-
88
- def plot_umap_pca50(
89
- cell_embeddings: np.ndarray,
90
- adata: sc.AnnData,
91
- output_path: str,
92
- embedding_key: str = "X_sclong_module_pca50"
93
- ):
94
- """
95
- Plot PCA50 + UMAP visualization
96
-
97
- parameter:
98
- - cell_embeddings: cell embedding vector (n_cells, embed_dim)
99
- - adata: AnnData object
100
- - output_path: output image path
101
- - embedding_key: the key name embedded in adata.obsm
102
-
103
- Calculation method:
104
- 1. Perform PCA dimensionality reduction on the cell embedding vector to 50 dimensions.
105
- 2. Perform UMAP dimensionality reduction on the vector after PCA dimensionality reduction.
106
- """
107
- print(f"Plotting UMAP after PCA50...")
108
-
109
- # PCA reduces dimensionality to 50 dimensions
110
- n_components = min(50, cell_embeddings.shape[1], cell_embeddings.shape[0] - 1)
111
- print(f" PCA components: {n_components}")
112
-
113
- pca = PCA(n_components=n_components)
114
- cell_embeddings_pca50 = pca.fit_transform(cell_embeddings)
115
-
116
- # Calculate proportion of explained variance
117
- explained_variance = np.sum(pca.explained_variance_ratio_) * 100
118
- print(f" Explained variance: {explained_variance:.2f}%")
119
-
120
- # Store PCA50 embedding into adata
121
- adata.obsm[embedding_key] = cell_embeddings_pca50
122
-
123
- # Calculate neighbors and UMAP
124
- sc.pp.neighbors(adata, use_rep=embedding_key, n_neighbors=15, n_pcs=None)
125
- sc.tl.umap(adata, min_dist=0.1)
126
-
127
- # Create a graph - two subplots
128
- fig, axes = plt.subplots(1, 2, figsize=(14, 6))
129
-
130
- # Subfigure 1: Coloring by celltype
131
- sc.pl.umap(
132
- adata,
133
- color='celltype',
134
- ax=axes[0],
135
- show=False,
136
- title=f'Cell Type (PCA{n_components})',
137
- frameon=True
138
- )
139
-
140
- # Subpicture 2: Coloring by batch
141
- sc.pl.umap(
142
- adata,
143
- color='batch',
144
- ax=axes[1],
145
- show=False,
146
- title=f'Batch (PCA{n_components})',
147
- frameon=True
148
- )
149
-
150
- plt.tight_layout()
151
- plt.savefig(output_path, dpi=150, bbox_inches='tight')
152
- plt.close()
153
-
154
- print(f"Saved: {output_path}")
155
-
156
-
157
- def plot_batch_asw_comparison(
158
- results_dict: dict,
159
- output_path: str,
160
- output_path_svg: str = None
161
- ):
162
- """
163
- Plot a bar chart comparing batch ASW scores for different models/setups
164
-
165
- parameter:
166
- - results_dict: dictionary, key is the model name, value is the ASW score
167
- - output_path: output image path (.png)
168
- - output_path_svg: output SVG image path (optional)
169
-
170
- Example results_dict:
171
- {
172
- 'scLong (multiply_27k)': 0.982,
173
- 'scLong (no_gene2vec)': 0.778,
174
- 'scGPT': 0.95,
175
- ...
176
- }
177
- """
178
- print(f"Plotting batch ASW comparison...")
179
-
180
- # Color settings
181
- color_dict = {
182
- 'Raw': '#6FD1B2',
183
- 'HVG': '#FF99CC',
184
- 'scVI': '#FFCC99',
185
- 'Geneformer': '#99CC99',
186
- 'scGPT': '#FF9999',
187
- 'scFoundation': '#D1B26F',
188
- 'UCE': '#99FFCC',
189
- 'scLong': '#CC99FF',
190
- 'scLong (multiply_27k)': '#CC99FF',
191
- 'scLong (no_gene2vec)': '#9966CC',
192
- }
193
-
194
- models = list(results_dict.keys())
195
- values = list(results_dict.values())
196
- colors = [color_dict.get(model, '#888888') for model in models]
197
-
198
- fig, ax = plt.subplots(1, 1, figsize=(max(6, len(models) * 0.8), 4))
199
-
200
- # Draw a bar chart
201
- bars = ax.bar(models, values, color=colors, width=0.6)
202
-
203
- # Display values on bars
204
- for bar, value in zip(bars, values):
205
- height = bar.get_height()
206
- ax.text(bar.get_x() + bar.get_width()/2., height + 0.01,
207
- f'{value:.4f}',
208
- ha='center', va='bottom', fontsize=9)
209
-
210
- ax.set_ylim((min(values) - 0.05, 1.02))
211
- ax.set_ylabel('batch ASW', fontsize=12)
212
- ax.set_title('Batch ASW Comparison', fontsize=14)
213
-
214
- # Rotate x-axis label
215
- plt.xticks(rotation=30, ha='right', fontsize=10)
216
-
217
- # Remove top and right borders
218
- ax.spines['right'].set_visible(False)
219
- ax.spines['top'].set_visible(False)
220
-
221
- plt.tight_layout()
222
- plt.savefig(output_path, dpi=150, bbox_inches='tight')
223
-
224
- if output_path_svg:
225
- plt.savefig(output_path_svg, format='svg', bbox_inches='tight')
226
-
227
- plt.close()
228
-
229
- print(f"Saved: {output_path}")
230
- if output_path_svg:
231
- print(f"Saved: {output_path_svg}")
232
-
233
-
234
- def main():
235
- parser = argparse.ArgumentParser(description='Zero-shot batch correction visualization')
236
- parser.add_argument('--res_dir', type=str, required=True,
237
- help='Path to result directory (e.g., res/multiply_27k)')
238
- parser.add_argument('--data_path', type=str, default='../zero-shot-batch/pancreas_scib.h5ad',
239
- help='Path to original h5ad data file')
240
- parser.add_argument('--embedding_file', type=str, default='sclong_module_batch_cell_emb_mode_ce.npy',
241
- help='Name of embedding numpy file')
242
- parser.add_argument('--csv_file', type=str, default='sclong_module_batch_cell_emb_mode.csv',
243
- help='Name of ASW score csv file')
244
- parser.add_argument('--embedding_key', type=str, default='X_sclong_module',
245
- help='Key name for storing embeddings')
246
-
247
- args = parser.parse_args()
248
-
249
- # Load data
250
- print(f"Loading data from {args.data_path}...")
251
- adata = sc.read_h5ad(args.data_path)
252
-
253
- # Load cell embedding
254
- embedding_path = os.path.join(args.res_dir, args.embedding_file)
255
- print(f"Loading embeddings from {embedding_path}...")
256
- cell_embeddings = np.load(embedding_path)
257
- print(f" Cell embeddings shape: {cell_embeddings.shape}")
258
-
259
- # Load ASW scores
260
- csv_path = os.path.join(args.res_dir, args.csv_file)
261
- print(f"Loading ASW scores from {csv_path}...")
262
- asw_df = pd.read_csv(csv_path)
263
- asw_score = asw_df['value'].values[0]
264
- print(f" ASW score: {asw_score:.4f}")
265
-
266
- # Make sure the number of cells in adata matches the number of embeddings
267
- if adata.n_obs != cell_embeddings.shape[0]:
268
- print(f"Warning: adata has {adata.n_obs} cells, but embeddings have {cell_embeddings.shape[0]} cells")
269
- # If there are more cells in adata, only the first n are taken.
270
- if adata.n_obs > cell_embeddings.shape[0]:
271
- print(f" Subsampling adata to {cell_embeddings.shape[0]} cells")
272
- adata = adata[:cell_embeddings.shape[0]].copy()
273
-
274
- # 1. Draw UMAP of cell embeddings
275
- plot_umap_cell_embeddings(
276
- cell_embeddings=cell_embeddings,
277
- adata=adata,
278
- output_path=os.path.join(args.res_dir, 'umap_cell_embeddings.png'),
279
- embedding_key=args.embedding_key
280
- )
281
-
282
- # 2. Draw UMAP after PCA50
283
- plot_umap_pca50(
284
- cell_embeddings=cell_embeddings,
285
- adata=adata,
286
- output_path=os.path.join(args.res_dir, 'umap_pca50.png'),
287
- embedding_key=f"{args.embedding_key}_pca50"
288
- )
289
-
290
- print("\nVisualization completed!")
291
-
292
-
293
- if __name__ == '__main__':
294
  main()
 
1
+ """
2
+ Zero-shot batch correction visual code
3
+
4
+ Used to generate the following images:
5
+ 1. umap_cell_embeddings.png - UMAP visualization of cell embeddings (colored by cell type and batch)
6
+ 2. umap_pca50.png - PCA dimensionality reduction to 50 dimensions and then UMAP visualization
7
+ 3. batch_ASW_comparison.png - Batch ASW score comparison bar chart for different models/settings
8
+
9
+ Enter data:
10
+ - sclong_module_batch_cell_emb_mode_ce.npy: cell embedding vector
11
+ - sclong_module_batch_cell_emb_mode.csv: ASW score
12
+ - Caller-supplied h5ad: the original data corresponding to the saved embeddings.
13
+
14
+ Calculation method:
15
+ 1. umap_cell_embeddings: directly perform UMAP dimensionality reduction on cell embedding vectors
16
+ 2. umap_pca50: First use PCA to reduce the dimension to 50 dimensions, and then perform UMAP dimension reduction.
17
+ 3. batch_ASW: Use scib.metrics.silhouette_batch to calculate the batch ASW score
18
+ """
19
+
20
+ import os
21
+ import numpy as np
22
+ import pandas as pd
23
+ import scanpy as sc
24
+ import matplotlib.pyplot as plt
25
+ import seaborn as sns
26
+ from sklearn.decomposition import PCA
27
+ import argparse
28
+
29
+ # Set drawing style
30
+ plt.rcParams['font.family'] = 'DejaVu Sans'
31
+ plt.rcParams['font.size'] = 10
32
+ plt.rcParams['figure.dpi'] = 150
33
+
34
+ def plot_umap_cell_embeddings(
35
+ cell_embeddings: np.ndarray,
36
+ adata: sc.AnnData,
37
+ output_path: str,
38
+ embedding_key: str = "X_sclong_module"
39
+ ):
40
+ """
41
+ Draw cell-embedded UMAP visualization
42
+
43
+ parameter:
44
+ - cell_embeddings: cell embedding vector (n_cells, embed_dim)
45
+ - adata: AnnData object, containing celltype and batch information
46
+ - output_path: output image path
47
+ - embedding_key: the key name embedded in adata.obsm
48
+ """
49
+ print(f"Plotting UMAP of cell embeddings...")
50
+
51
+ # Store embedding vector into adata
52
+ adata.obsm[embedding_key] = cell_embeddings
53
+
54
+ # Calculate neighbors and UMAP
55
+ sc.pp.neighbors(adata, use_rep=embedding_key, n_neighbors=30, n_pcs=None)
56
+ sc.tl.umap(adata, min_dist=0.1)
57
+
58
+ # Create graph - two subgraphs: color by celltype and by batch
59
+ fig, axes = plt.subplots(1, 2, figsize=(14, 6))
60
+
61
+ # Subfigure 1: Coloring by celltype
62
+ sc.pl.umap(
63
+ adata,
64
+ color='celltype',
65
+ ax=axes[0],
66
+ show=False,
67
+ title='Cell Type',
68
+ frameon=True
69
+ )
70
+
71
+ # Subpicture 2: Coloring by batch
72
+ sc.pl.umap(
73
+ adata,
74
+ color='batch',
75
+ ax=axes[1],
76
+ show=False,
77
+ title='Batch',
78
+ frameon=True
79
+ )
80
+
81
+ plt.tight_layout()
82
+ plt.savefig(output_path, dpi=150, bbox_inches='tight')
83
+ plt.close()
84
+
85
+ print(f"Saved: {output_path}")
86
+
87
+
88
+ def plot_umap_pca50(
89
+ cell_embeddings: np.ndarray,
90
+ adata: sc.AnnData,
91
+ output_path: str,
92
+ embedding_key: str = "X_sclong_module_pca50"
93
+ ):
94
+ """
95
+ Plot PCA50 + UMAP visualization
96
+
97
+ parameter:
98
+ - cell_embeddings: cell embedding vector (n_cells, embed_dim)
99
+ - adata: AnnData object
100
+ - output_path: output image path
101
+ - embedding_key: the key name embedded in adata.obsm
102
+
103
+ Calculation method:
104
+ 1. Perform PCA dimensionality reduction on the cell embedding vector to 50 dimensions.
105
+ 2. Perform UMAP dimensionality reduction on the vector after PCA dimensionality reduction.
106
+ """
107
+ print(f"Plotting UMAP after PCA50...")
108
+
109
+ # PCA reduces dimensionality to 50 dimensions
110
+ n_components = min(50, cell_embeddings.shape[1], cell_embeddings.shape[0] - 1)
111
+ print(f" PCA components: {n_components}")
112
+
113
+ pca = PCA(n_components=n_components)
114
+ cell_embeddings_pca50 = pca.fit_transform(cell_embeddings)
115
+
116
+ # Calculate proportion of explained variance
117
+ explained_variance = np.sum(pca.explained_variance_ratio_) * 100
118
+ print(f" Explained variance: {explained_variance:.2f}%")
119
+
120
+ # Store PCA50 embedding into adata
121
+ adata.obsm[embedding_key] = cell_embeddings_pca50
122
+
123
+ # Calculate neighbors and UMAP
124
+ sc.pp.neighbors(adata, use_rep=embedding_key, n_neighbors=15, n_pcs=None)
125
+ sc.tl.umap(adata, min_dist=0.1)
126
+
127
+ # Create a graph - two subplots
128
+ fig, axes = plt.subplots(1, 2, figsize=(14, 6))
129
+
130
+ # Subfigure 1: Coloring by celltype
131
+ sc.pl.umap(
132
+ adata,
133
+ color='celltype',
134
+ ax=axes[0],
135
+ show=False,
136
+ title=f'Cell Type (PCA{n_components})',
137
+ frameon=True
138
+ )
139
+
140
+ # Subpicture 2: Coloring by batch
141
+ sc.pl.umap(
142
+ adata,
143
+ color='batch',
144
+ ax=axes[1],
145
+ show=False,
146
+ title=f'Batch (PCA{n_components})',
147
+ frameon=True
148
+ )
149
+
150
+ plt.tight_layout()
151
+ plt.savefig(output_path, dpi=150, bbox_inches='tight')
152
+ plt.close()
153
+
154
+ print(f"Saved: {output_path}")
155
+
156
+
157
+ def plot_batch_asw_comparison(
158
+ results_dict: dict,
159
+ output_path: str,
160
+ output_path_svg: str = None
161
+ ):
162
+ """
163
+ Plot a bar chart comparing batch ASW scores for different models/setups
164
+
165
+ parameter:
166
+ - results_dict: dictionary, key is the model name, value is the ASW score
167
+ - output_path: output image path (.png)
168
+ - output_path_svg: output SVG image path (optional)
169
+
170
+ Example results_dict:
171
+ {
172
+ 'scLong (multiply_27k)': 0.982,
173
+ 'scLong (no_gene2vec)': 0.778,
174
+ 'scGPT': 0.95,
175
+ ...
176
+ }
177
+ """
178
+ print(f"Plotting batch ASW comparison...")
179
+
180
+ # Color settings
181
+ color_dict = {
182
+ 'Raw': '#6FD1B2',
183
+ 'HVG': '#FF99CC',
184
+ 'scVI': '#FFCC99',
185
+ 'Geneformer': '#99CC99',
186
+ 'scGPT': '#FF9999',
187
+ 'scFoundation': '#D1B26F',
188
+ 'UCE': '#99FFCC',
189
+ 'scLong': '#CC99FF',
190
+ 'scLong (multiply_27k)': '#CC99FF',
191
+ 'scLong (no_gene2vec)': '#9966CC',
192
+ }
193
+
194
+ models = list(results_dict.keys())
195
+ values = list(results_dict.values())
196
+ colors = [color_dict.get(model, '#888888') for model in models]
197
+
198
+ fig, ax = plt.subplots(1, 1, figsize=(max(6, len(models) * 0.8), 4))
199
+
200
+ # Draw a bar chart
201
+ bars = ax.bar(models, values, color=colors, width=0.6)
202
+
203
+ # Display values on bars
204
+ for bar, value in zip(bars, values):
205
+ height = bar.get_height()
206
+ ax.text(bar.get_x() + bar.get_width()/2., height + 0.01,
207
+ f'{value:.4f}',
208
+ ha='center', va='bottom', fontsize=9)
209
+
210
+ ax.set_ylim((min(values) - 0.05, 1.02))
211
+ ax.set_ylabel('batch ASW', fontsize=12)
212
+ ax.set_title('Batch ASW Comparison', fontsize=14)
213
+
214
+ # Rotate x-axis label
215
+ plt.xticks(rotation=30, ha='right', fontsize=10)
216
+
217
+ # Remove top and right borders
218
+ ax.spines['right'].set_visible(False)
219
+ ax.spines['top'].set_visible(False)
220
+
221
+ plt.tight_layout()
222
+ plt.savefig(output_path, dpi=150, bbox_inches='tight')
223
+
224
+ if output_path_svg:
225
+ plt.savefig(output_path_svg, format='svg', bbox_inches='tight')
226
+
227
+ plt.close()
228
+
229
+ print(f"Saved: {output_path}")
230
+ if output_path_svg:
231
+ print(f"Saved: {output_path_svg}")
232
+
233
+
234
+ def main():
235
+ parser = argparse.ArgumentParser(description='Zero-shot batch correction visualization')
236
+ parser.add_argument('--res_dir', type=str, required=True,
237
+ help='Path to result directory (e.g., res/multiply_27k)')
238
+ parser.add_argument('--data_path', type=str, required=True,
239
+ help='Path to original h5ad data file')
240
+ parser.add_argument('--embedding_file', type=str, default='sclong_module_batch_cell_emb_mode_ce.npy',
241
+ help='Name of embedding numpy file')
242
+ parser.add_argument('--csv_file', type=str, default='sclong_module_batch_cell_emb_mode.csv',
243
+ help='Name of ASW score csv file')
244
+ parser.add_argument('--embedding_key', type=str, default='X_sclong_module',
245
+ help='Key name for storing embeddings')
246
+
247
+ args = parser.parse_args()
248
+
249
+ # Load data
250
+ print(f"Loading data from {args.data_path}...")
251
+ adata = sc.read_h5ad(args.data_path)
252
+
253
+ # Load cell embedding
254
+ embedding_path = os.path.join(args.res_dir, args.embedding_file)
255
+ print(f"Loading embeddings from {embedding_path}...")
256
+ cell_embeddings = np.load(embedding_path)
257
+ print(f" Cell embeddings shape: {cell_embeddings.shape}")
258
+
259
+ # Load ASW scores
260
+ csv_path = os.path.join(args.res_dir, args.csv_file)
261
+ print(f"Loading ASW scores from {csv_path}...")
262
+ asw_df = pd.read_csv(csv_path)
263
+ asw_score = asw_df['value'].values[0]
264
+ print(f" ASW score: {asw_score:.4f}")
265
+
266
+ # Make sure the number of cells in adata matches the number of embeddings
267
+ if adata.n_obs != cell_embeddings.shape[0]:
268
+ print(f"Warning: adata has {adata.n_obs} cells, but embeddings have {cell_embeddings.shape[0]} cells")
269
+ # If there are more cells in adata, only the first n are taken.
270
+ if adata.n_obs > cell_embeddings.shape[0]:
271
+ print(f" Subsampling adata to {cell_embeddings.shape[0]} cells")
272
+ adata = adata[:cell_embeddings.shape[0]].copy()
273
+
274
+ # 1. Draw UMAP of cell embeddings
275
+ plot_umap_cell_embeddings(
276
+ cell_embeddings=cell_embeddings,
277
+ adata=adata,
278
+ output_path=os.path.join(args.res_dir, 'umap_cell_embeddings.png'),
279
+ embedding_key=args.embedding_key
280
+ )
281
+
282
+ # 2. Draw UMAP after PCA50
283
+ plot_umap_pca50(
284
+ cell_embeddings=cell_embeddings,
285
+ adata=adata,
286
+ output_path=os.path.join(args.res_dir, 'umap_pca50.png'),
287
+ embedding_key=f"{args.embedding_key}_pca50"
288
+ )
289
+
290
+ print("\nVisualization completed!")
291
+
292
+
293
+ if __name__ == '__main__':
294
  main()
README.md CHANGED
@@ -97,10 +97,10 @@ The repository also includes the following supporting workflows.
97
  | Pretraining and continued pretraining | [Pretraining script](BioMod/performer_module.py) | [Source guide](BioMod/README.md), [launcher reference](BioMod/command.txt) |
98
  | Module interpretability analysis | [Module geometry](BioMod/interpretability/module_geometry.py), [module importance](BioMod/interpretability/module_importance/module_importance.py), [text alignment](BioMod/interpretability/module_text_alignment/module_text_alignment.py) | [Overview](BioMod/interpretability/README.md), [attention](BioMod/interpretability/attention/README.md), [module importance guide](BioMod/interpretability/module_importance/README.md) |
99
 
100
- Tutorials and launcher references retain some historical checkpoint paths,
101
- server paths, and arguments. Adapt them to the current assets above and check
102
- the selected script's `--help` before running. Some notebook plotting steps
103
- also refer to external scripts or reference results that are not bundled here.
104
 
105
  ## Usage notes
106
 
@@ -111,22 +111,27 @@ The project root contains `BioMod/`, `model_dependencies/`, `weight/`, and
111
  may assume different working directories; adjust relative paths when following
112
  a guide.
113
 
114
- Before running a notebook, update its configuration cells:
 
 
 
115
 
116
  | Setting | Location |
117
  | --- | --- |
118
- | `BASE` or `ASSET_ROOT` | Project root |
119
- | `PROJECT_ROOT` | Inner `BioMod/` source directory |
120
- | `MODULE_MODEL` or `LEGACY_MODULE_MODEL` | `model_dependencies/` under the project root |
121
  | Weight and configuration paths | The BioMod model weights and matching configuration listed above |
122
- | `MODULE_DATA_DIR` | `BioMod/module_data_all/` under the project root |
123
  | Data and output paths | Your task inputs and a writable output directory |
124
 
125
  Checkpoint variable names differ by notebook: `CKPT_PATH` / `CONFIG_PKL`
126
  for cell annotation, drug response, and zero-shot integration;
127
  `SCFM_CKPT` / `SCFM_HYPER` for gene perturbation; and
128
  `CHECKPOINT_PATH` / `HYPER_PARAMS_PATH` for drug combination.
129
- Updating environment variables alone does not replace hard-coded notebook paths.
 
 
130
 
131
  ### Checkpoint arguments
132
 
 
97
  | Pretraining and continued pretraining | [Pretraining script](BioMod/performer_module.py) | [Source guide](BioMod/README.md), [launcher reference](BioMod/command.txt) |
98
  | Module interpretability analysis | [Module geometry](BioMod/interpretability/module_geometry.py), [module importance](BioMod/interpretability/module_importance/module_importance.py), [text alignment](BioMod/interpretability/module_text_alignment/module_text_alignment.py) | [Overview](BioMod/interpretability/README.md), [attention](BioMod/interpretability/attention/README.md), [module importance guide](BioMod/interpretability/module_importance/README.md) |
99
 
100
+ Bundled input defaults resolve from each script's location. Command examples
101
+ state their working directory; explicit relative command-line paths are resolved
102
+ from that directory. Pretraining corpora, legacy reference checkpoints, and
103
+ generated analysis results must be supplied separately where indicated.
104
 
105
  ## Usage notes
106
 
 
111
  may assume different working directories; adjust relative paths when following
112
  a guide.
113
 
114
+ Start Jupyter from `BioMod/toturials/`. Each notebook sets
115
+ `PROJECT_ROOT = Path('../..').resolve()` and derives absolute runtime paths from
116
+ the outer repository root. If Jupyter starts at the repository root, use
117
+ `Path('.').resolve()` instead. Review the configuration cells before running:
118
 
119
  | Setting | Location |
120
  | --- | --- |
121
+ | `PROJECT_ROOT` | Outer project root containing `BioMod/`, `model_dependencies/`, `weight/`, and `data/` |
122
+ | `SOURCE_DIR` | `PROJECT_ROOT / 'BioMod'` |
123
+ | Asset or dependency directory | `PROJECT_ROOT / 'model_dependencies'` |
124
  | Weight and configuration paths | The BioMod model weights and matching configuration listed above |
125
+ | `MODULE_DATA_DIR` | `model_dependencies/module_data_all/` under the project root |
126
  | Data and output paths | Your task inputs and a writable output directory |
127
 
128
  Checkpoint variable names differ by notebook: `CKPT_PATH` / `CONFIG_PKL`
129
  for cell annotation, drug response, and zero-shot integration;
130
  `SCFM_CKPT` / `SCFM_HYPER` for gene perturbation; and
131
  `CHECKPOINT_PATH` / `HYPER_PARAMS_PATH` for drug combination.
132
+ Training defaults to disabled (`RUN_TRAINING = False`). The zero-shot notebook
133
+ uses `RUN_EVALUATION = False` and enables UMAP export separately with `MAKE_UMAP`.
134
+ Output directories are created when the corresponding operation is enabled.
135
 
136
  ### Checkpoint arguments
137
 
data/downstream/final_data/drug_combination_synergy/README.md CHANGED
@@ -1,48 +1,50 @@
1
- # Drug-combination synergy data
2
-
3
- This folder is a cleaned data layout for drug-combination synergy prediction.
4
- The original collaborator package is kept unchanged at:
5
-
6
- ```text
7
- C:\Users\JXZ\Desktop\文章的相关材料\data\downstream\final_data\drug_response_pytorch\drug_combination
8
- ```
9
-
10
- ## Runnable data path
11
-
12
- Use this directory as `--data_dir` for `drug_combination/train_drug_combination_performer_patch.py`:
13
-
14
- ```text
15
- C:\Users\JXZ\Desktop\文章的相关材料\data\downstream\final_data\drug_combination_synergy\data_summary
16
- ```
17
-
18
- Required files used by the current script:
19
-
20
- ```text
21
- data_summary/
22
- |-- new_labels_0_10.csv
23
- |-- new_cell_features_954.csv
24
- |-- smiles_0_10.csv
25
- `-- test_data/
26
- |-- independent_input.csv
27
- `-- independent_cell_features_954.csv
28
- ```
29
-
30
- ## Checked statistics
31
-
32
- - Training labels: (12415, 4)
33
- - Training cell features: (32, 955)
34
- - SMILES table: (35, 2)
35
- - Independent test labels: (668, 4)
36
- - Independent test cell features: (1020, 955)
37
- - Training label counts: {'0': 6299, '1': 6116}
38
- - Test label counts: {'1': 480, '0': 188}
39
- - Unique training cells: 31
40
- - Unique test cells: 24
41
- - Unique training drug columns: drug1=35, drug2=35
42
- - Unique test drug columns: drug1=44, drug2=39
43
-
44
- Additional files in `data_summary/` are retained for provenance and regeneration of train/test labels.
45
-
46
- ## Files
47
-
48
- See `MANIFEST.csv` for file sizes and checksums. See `SUMMARY.json` for machine-readable shape checks.
 
 
 
1
+ # Drug-combination synergy data
2
+
3
+ This folder is a cleaned data layout for drug-combination synergy prediction.
4
+ The bundled legacy collaborator reference package is located at this repository-relative path:
5
+
6
+ ```text
7
+ data/downstream/final_data/drug_response_pytorch/drug_combination
8
+ ```
9
+
10
+ ## Runnable data path
11
+
12
+ From the repository root, use this directory as `--data_dir` for `BioMod/drug_combination/train_drug_combination_performer_module.py`:
13
+
14
+ ```text
15
+ data/downstream/final_data/drug_combination_synergy/data_summary
16
+ ```
17
+
18
+ Required files used by the current script:
19
+
20
+ ```text
21
+ data_summary/
22
+ |-- new_labels_0_10.csv
23
+ |-- new_cell_features_954.csv
24
+ |-- smiles_0_10.csv
25
+ `-- test_data/
26
+ |-- independent_input.csv
27
+ `-- independent_cell_features_954.csv
28
+ ```
29
+
30
+ ## Previously recorded statistics
31
+
32
+ The figures below are retained from the existing dataset documentation; this path/comment audit did not recompute them.
33
+
34
+ - Training labels: (12415, 4)
35
+ - Training cell features: (32, 955)
36
+ - SMILES table: (35, 2)
37
+ - Independent test labels: (668, 4)
38
+ - Independent test cell features: (1020, 955)
39
+ - Training label counts: {'0': 6299, '1': 6116}
40
+ - Test label counts: {'1': 480, '0': 188}
41
+ - Unique training cells: 31
42
+ - Unique test cells: 24
43
+ - Unique training drug columns: drug1=35, drug2=35
44
+ - Unique test drug columns: drug1=44, drug2=39
45
+
46
+ Additional files in `data_summary/` are retained for provenance and regeneration of train/test labels.
47
+
48
+ ## Files
49
+
50
+ See `MANIFEST.csv` for file sizes and checksums. See `SUMMARY.json` for machine-readable shape checks.
data/downstream/final_data/drug_combination_synergy/data_summary/README.md CHANGED
@@ -1,109 +1,108 @@
1
- # DeepDDS 核心数据文件说明
2
 
3
- ## 数据来源
 
4
 
5
- - **药物协同数据**:来自 O'Neil et al. (2016) 的高通量药物组合筛选实验,涵盖 38 种抗癌药物在 39 个人类癌症细胞系上的两两组合。
6
- - **协同评分**:基于 4×4 剂量-响应矩阵,使用 Combenefit 工具计算 Loewe Additivity Score。同一药物组合若有多个重复实验,取平均值作为最终协同评分。
7
- - **细胞系表达数据**:来自 CCLE (Cancer Cell Line Encyclopedia) 2018年发布的 RNA-seq TPM 数据 (`CCLE_RNAseq_rsem_transcripts_tpm_20180929`),为细胞系在**未经药物处理、正常培养条件**下的基线基因表达谱。
8
 
9
- ## 标签定义(二值化阈值)
 
 
10
 
11
- | 条件 | 标签 | 含义 |
12
- |------|------|------|
13
- | Loewe score > 10 | 1 (正样本) | 协同 (synergistic) |
14
- | Loewe score < 0 | 0 (负样本) | 拮抗 (antagonistic) |
15
- | 0 ≤ score ≤ 10 | 排除 | 模糊区间,不参与建模 |
16
 
17
- 使用严格阈值(>10 而非 >0)的目的是:(1) 减少噪声附近样本的干扰;(2) 获得正负样本基本平衡的数据集。
 
 
 
 
18
 
19
- ## 最终数据规模
20
 
21
- - **12,415** 个唯一的 drug pair–cell line 组合
22
- - **36** 种抗癌药物
23
- - **31** 个人类癌症细胞系
24
- - 正样本 6,116 个,负样本 6,299 个
25
 
26
- ---
 
 
27
 
28
- ## 文件清单
29
 
30
- ### 1. new_labels_0_10.csv — 二值化标签文件(模型训练核心输入)
31
 
32
- - **行数**:12,415 条记录 + 1 行表头
33
- - **列**:
34
- | 列名 | 类型 | 说明 |
35
- |------|------|------|
36
- | drug1 | string | 药物 A 的 SMILES 分子式 |
37
- | drug2 | string | 药物 B 的 SMILES 分子式 |
38
- | cell | string | 细胞系名称(如 A375, HCT116) |
39
- | label | int | 0 = 拮抗 (score < 0),1 = 协同 (score > 10) |
40
 
41
- ### 2. score_0_10.csv — 连续协同评分文件
 
 
 
 
 
42
 
43
- - **行数**:12,415 条记录 + 1 行表头(与 new_labels_0_10.csv 一一对应)
44
- - **列**:
45
- | 列名 | 类型 | 说明 |
46
- |------|------|------|
47
- | drug1 | string | 药物 A 名称(如 5-FU) |
48
- | drug2 | string | 药物 B 名称(如 ABT-888) |
49
- | cell | string | 细胞系名称 |
50
- | score | float | Loewe Additivity 连续协同评分 |
51
- | nor | float | 归一化后的评分 |
52
- | index | int | 样本索引 |
53
- | true_label | int | 真实二值标签 |
54
- | predict_label | int | 模型预测标签(作者预训练结果) |
55
- | predict_score | float | 模型预测概率(作者预训练结果) |
56
- | smile1 | string | 药物 A 的 SMILES |
57
- | smile2 | string | 药物 B 的 SMILES |
58
 
59
- ### 3. new_cell_features_954.csv — 细胞系基因表达矩阵
60
 
61
- - **维度**:31 个细胞系 × 954 个基因
62
- - **数据来源**:CCLE RNA-seq baseline TPM(未经药物处理的基线表达)
63
- - **格式**:
64
- - 第 1 行:gene_id(ENSG 编号)
65
- - 第 2 行:transcript_id(ENST 编号)
66
- - 第 3~33 行:31 个细胞系的表达值,第 1 列为细胞系名称
67
- - **用途**:作为模型中细胞系特征的输入,经全连接网络编码后与药物图特征融合
 
 
 
 
 
 
68
 
69
- ### 4. cell_gene954.csv — 954 基因 ID 对照表
70
 
71
- - **行数**:954 条记录 + 1 行表头
72
- - **列**:gene_id (ENSG), transcript_id (ENST)
73
- - **用途**:记录 new_cell_features_954.csv 中 954 个基因的 Ensembl 编号
74
 
75
- ### 5. smiles_0_10.csv — 药物 SMILES 对照表
 
 
 
 
 
76
 
77
- - **行数**:36 条(无表头)
78
- - **列**:药物名称, SMILES 分子式
79
- - **用途**:药物名称与 SMILES 的映射关系,SMILES 用于构建分子图作为 GNN 输入
80
 
81
- ### 6. drugs_0_10.csv — 药物信息表
 
 
82
 
83
- - **行数**:36 条(无表头)
84
- - **列**:药物名称, SMILES 分子式, 数值编码
85
- - **用途**:药物列表及其编码索引
86
 
87
- ### 7. cell_line_0_10.csv — 细胞系信息表
 
 
88
 
89
- - **行数**:31 条(无表头)
90
- - **列**:细胞系名称, 癌症类型(如 melanoma, ovarian, colon)
91
- - **用途**:记录 31 个细胞系对应的癌症类型
92
 
93
- ---
 
 
94
 
95
- ## DeepDDS 模型输入结构
96
 
97
- ```
98
- Drug A (SMILES) ──→ 分子图 ──→ GAT/GCN ──→ Drug A embedding ─┐
99
- ├──→ 拼接 ──→ 全连接层 ──→ 协同/拮抗预测
100
- Drug B (SMILES) ──→ 分子图 ──→ GAT/GCN ──→ Drug B embedding ─┤
101
- │
102
- Cell line (954 genes TPM) ──→ FC layers ──→ Cell embedding ────┘
 
 
 
 
 
 
103
  ```
104
 
105
- ## 参考文献
106
 
107
  - Wang et al. "DeepDDS: deep graph neural network with attention mechanism to predict synergistic drug combinations." Briefings in Bioinformatics, 2022.
108
  - O'Neil et al. "An unbiased oncology compound screen to identify novel combination strategies." Molecular Cancer Therapeutics, 2016.
109
- - CCLE: Cancer Cell Line Encyclopedia (Broad Institute)
 
1
+ # DeepDDS core data files
2
 
3
+ The provenance and counts below are translated from the existing dataset notes.
4
+ This path/comment audit did not recompute counts or validate the source studies.
5
 
6
+ ## Data sources
 
 
7
 
8
+ - **Drug synergy:** the O'Neil et al. (2016) high-throughput screen of pairwise combinations of 38 anticancer drugs across 39 human cancer cell lines.
9
+ - **Synergy scores:** Loewe Additivity Scores computed with Combenefit from 4 x 4 dose-response matrices. Replicate measurements for the same combination were averaged.
10
+ - **Cell-line expression:** CCLE RNA-seq TPM data released in 2018 (`CCLE_RNAseq_rsem_transcripts_tpm_20180929`), describing baseline expression under standard culture conditions without drug treatment.
11
 
12
+ ## Binary labels
 
 
 
 
13
 
14
+ | Condition | Label | Meaning |
15
+ |---|---|---|
16
+ | Loewe score > 10 | 1 | Synergistic |
17
+ | Loewe score < 0 | 0 | Antagonistic |
18
+ | 0 <= score <= 10 | Excluded | Ambiguous range excluded from modeling |
19
 
20
+ The original notes describe the stricter positive threshold as intended to reduce near-zero noise and obtain roughly balanced classes.
21
 
22
+ ## Previously recorded dataset size
 
 
 
23
 
24
+ - 12,415 unique drug-pair/cell-line combinations.
25
+ - 36 anticancer drugs and 31 human cancer cell lines.
26
+ - 6,116 positive and 6,299 negative samples.
27
 
28
+ ## File descriptions
29
 
30
+ ### `new_labels_0_10.csv`: binary training labels
31
 
32
+ The recorded layout is 12,415 records plus one header row.
 
 
 
 
 
 
 
33
 
34
+ | Column | Type | Description |
35
+ |---|---|---|
36
+ | drug1 | string | SMILES representation of drug A |
37
+ | drug2 | string | SMILES representation of drug B |
38
+ | cell | string | Cell-line name, such as A375 or HCT116 |
39
+ | label | int | 0 = antagonistic (score < 0); 1 = synergistic (score > 10) |
40
 
41
+ ### `score_0_10.csv`: continuous synergy scores
 
 
 
 
 
 
 
 
 
 
 
 
 
 
42
 
43
+ The existing notes record 12,415 records plus a header, corresponding to `new_labels_0_10.csv`.
44
 
45
+ | Column | Type | Description |
46
+ |---|---|---|
47
+ | drug1 | string | Drug A name, such as 5-FU |
48
+ | drug2 | string | Drug B name, such as ABT-888 |
49
+ | cell | string | Cell-line name |
50
+ | score | float | Continuous Loewe Additivity Score |
51
+ | nor | float | Normalized score |
52
+ | index | int | Sample index |
53
+ | true_label | int | Binary ground-truth label |
54
+ | predict_label | int | Legacy predicted label |
55
+ | predict_score | float | Legacy prediction probability |
56
+ | smile1 | string | Drug A SMILES |
57
+ | smile2 | string | Drug B SMILES |
58
 
59
+ The stored prediction columns are historical reference results, not results from the current BioMod checkout.
60
 
61
+ ### `new_cell_features_954.csv`: cell-line expression
 
 
62
 
63
+ - Recorded dimensions: 31 cell lines x 954 genes.
64
+ - Source: CCLE baseline RNA-seq TPM, without drug treatment.
65
+ - Row 1: gene IDs (ENSG).
66
+ - Row 2: transcript IDs (ENST).
67
+ - Rows 3-33: expression values, with the cell-line name in the first column.
68
+ - Purpose: cell features encoded by fully connected layers and combined with drug graph features.
69
 
70
+ ### `cell_gene954.csv`: gene identifier table
 
 
71
 
72
+ - Recorded layout: 954 records plus a header.
73
+ - Columns: `gene_id` (ENSG), `transcript_id` (ENST).
74
+ - Purpose: identify the genes in `new_cell_features_954.csv`.
75
 
76
+ ### `smiles_0_10.csv`: drug SMILES table
 
 
77
 
78
+ - The legacy notes record 36 headerless rows; counts may differ from the parser-based summary in the parent directory.
79
+ - Columns: drug name and SMILES representation.
80
+ - Purpose: map drug names to SMILES used for molecular-graph construction.
81
 
82
+ ### `drugs_0_10.csv`: drug table
 
 
83
 
84
+ - Recorded layout: 36 headerless rows.
85
+ - Columns: drug name, SMILES, numeric encoding.
86
+ - Purpose: enumerate drugs and their encoded indices.
87
 
88
+ ### `cell_line_0_10.csv`: cell-line table
89
 
90
+ - Recorded layout: 31 headerless rows.
91
+ - Columns: cell-line name and cancer type, such as melanoma, ovarian, or colon.
92
+ - Purpose: associate cell lines with cancer types.
93
+
94
+ ## DeepDDS inputs
95
+
96
+ ```text
97
+ Drug A SMILES -> Molecular graph -> GAT/GCN -> Drug A embedding --+
98
+ |
99
+ Drug B SMILES -> Molecular graph -> GAT/GCN -> Drug B embedding --+-> Concatenate -> FC layers -> Synergy prediction
100
+ |
101
+ Cell line (954 genes, TPM) -> FC layers -> Cell embedding -------+
102
  ```
103
 
104
+ ## References retained from the original notes
105
 
106
  - Wang et al. "DeepDDS: deep graph neural network with attention mechanism to predict synergistic drug combinations." Briefings in Bioinformatics, 2022.
107
  - O'Neil et al. "An unbiased oncology compound screen to identify novel combination strategies." Molecular Cancer Therapeutics, 2016.
108
+ - CCLE: Cancer Cell Line Encyclopedia, Broad Institute.
data/downstream/final_data/drug_combination_synergy/data_summary/test_data/README.md CHANGED
@@ -1,62 +1,54 @@
1
- # DeepDDS 独立测试集(AstraZeneca-Sanger DREAM Challenge)
2
 
3
- ## 数据来源
4
 
5
- 来自 AstraZeneca-Sanger Drug Combination DREAM Challenge 数据集(Menden et al., Nature Communications, 2019)。
6
- 原始数据文件:`41467_2019_9799_MOESM3_ESM.xlsx`
7
 
8
- 该数据集与训练集(O'Neil et al. 2016)完全独立,用于评估模型的泛化能力。
9
 
10
- ## 数据规模
11
 
12
- - **668** 个 drug pair–cell line 组合
13
- - **52** 种药物(部分为训练集中未见过的新药)
14
- - **24** 个细胞系(部分为训练集中未见过的新细胞系)
15
- - 正样本 480 个,负样本 188 个
16
 
17
- ---
18
 
19
- ## 文件清单
 
 
 
20
 
21
- ### 核心输入文件
22
 
23
- | 文件 | 说明 |
24
- |------|------|
25
- | `independent_input.csv` | 最终格式化的模型输入文件(668 条),列:drug1(SMILES), drug2(SMILES), cell, label |
26
- | `independent_cell_features_954.csv` | 细胞系 954 基因 CCLE 基线表达矩阵(1019 个细胞系,含训练集外的新细胞系) |
 
 
 
27
 
28
- ### 原始数据与中间处理文件
29
 
30
- | 文件 | 说明 |
31
- |------|------|
32
- | `41467_2019_9799_MOESM3_ESM.xlsx` | 原始论文附件(AZ-DREAM Challenge 全量数据) |
33
- | `independent_allin.csv` | 从原始数据提取的全量组合(76,607 条),列:drug1, drug2, cell |
34
- | `independent_allin_smiles.csv` | 附加 SMILES 信息的全量组合 |
35
- | `independent_set_combinatioin.csv` | 原始组合详情,含 synergy score、IC50、Hill slope、Einf 等药理学参数 |
36
- | `independent_set_newcombinatioin.csv` | 筛选后的组合(去除与训练集重叠的部分) |
37
 
38
- ### 辅助信息文件
39
 
40
- | 文件 | 说明 |
41
- |------|------|
42
- | `independent_set_drug.csv` | 药物信息表(药物名、靶点、通路、SMILES、分子量、LogP、Lipinski 等) |
43
- | `independent_set_cell_line.csv` | 144 个细胞系信息(COSMIC ID、组织类型、TCGA 标签、MSI 状态、是否在 AZ-DREAM/O'Neil/ALMANAC 中) |
44
- | `independent_drug.csv` | 筛选后可用于测试的药物列表 |
45
- | `independent_cell.csv` | 筛选后可用于测试的细胞系列表 |
46
- | `independent_input_drug_cell.csv` | 输入数据中的药物-细胞系映射关系 |
47
 
48
- ### 子集文件(按药物组合拆分)
49
-
50
- | 文件 | 说明 |
51
- |------|------|
52
- | `AZD2014_6244.csv` | AZD2014 + AZD6244 组合的子集 |
53
- | `AZD8931-AZD5363.csv` | AZD8931 + AZD5363 组合的子集 |
54
- | `Afatinib-MK2206.csv` | Afatinib + MK2206 组合的子集 |
55
- | `GDC0941-AZD6244.csv` | GDC0941 + AZD6244 组合的子集 |
56
- | `GDC0941-MK2206.csv` | GDC0941 + MK2206 组合的子集 |
57
-
58
- ---
59
-
60
- ## ��考文献
61
 
62
  - Menden et al. "Community assessment to advance computational prediction of cancer drug combinations in a pharmacogenomic screen." Nature Communications, 2019.
 
1
+ # DeepDDS independent test data
2
 
3
+ ## Source
4
 
5
+ The original notes attribute these data to the AstraZeneca-Sanger Drug Combination DREAM Challenge (Menden et al., Nature Communications, 2019), with source supplement `41467_2019_9799_MOESM3_ESM.xlsx`.
 
6
 
7
+ This collection is intended for evaluation against the separate O'Neil et al. (2016) training collection. The path/comment audit did not verify overlap, counts, or the original source study; those checks are required before making claims about benchmark independence.
8
 
9
+ ## Previously recorded size
10
 
11
+ - 668 drug-pair/cell-line combinations.
12
+ - 52 drugs and 24 cell lines, including entries described in the original notes as absent from training.
13
+ - 480 positive and 188 negative samples.
 
14
 
15
+ ## Core inputs
16
 
17
+ | File | Purpose |
18
+ |---|---|
19
+ | `independent_input.csv` | Formatted model inputs; recorded as 668 rows with drug1 (SMILES), drug2 (SMILES), cell, and label |
20
+ | `independent_cell_features_954.csv` | CCLE baseline expression for 954 genes; recorded as 1,019 cell lines, including lines outside the training collection |
21
 
22
+ ## Source and intermediate files
23
 
24
+ | File | Purpose |
25
+ |---|---|
26
+ | `41467_2019_9799_MOESM3_ESM.xlsx` | Original publication supplement with the full AZ-DREAM data |
27
+ | `independent_allin.csv` | Extracted combinations; recorded as 76,607 rows with drug1, drug2, and cell |
28
+ | `independent_allin_smiles.csv` | Extracted combinations with SMILES annotations |
29
+ | `independent_set_combinatioin.csv` | Combination details, including synergy scores, IC50, Hill slope, and Einf |
30
+ | `independent_set_newcombinatioin.csv` | Filtered combinations described in the original notes as excluding overlap with training |
31
 
32
+ ## Supporting metadata
33
 
34
+ | File | Purpose |
35
+ |---|---|
36
+ | `independent_set_drug.csv` | Drug names, targets, pathways, SMILES, molecular weights, LogP, and Lipinski descriptors |
37
+ | `independent_set_cell_line.csv` | Metadata recorded for 144 cell lines: COSMIC IDs, tissue, TCGA labels, MSI status, and AZ-DREAM/O'Neil/ALMANAC membership |
38
+ | `independent_drug.csv` | Filtered drugs available for testing |
39
+ | `independent_cell.csv` | Filtered cell lines available for testing |
40
+ | `independent_input_drug_cell.csv` | Drug-to-cell-line mapping for the formatted inputs |
41
 
42
+ ## Drug-pair subsets
43
 
44
+ | File | Drug pair |
45
+ |---|---|
46
+ | `AZD2014_6244.csv` | AZD2014 + AZD6244 |
47
+ | `AZD8931-AZD5363.csv` | AZD8931 + AZD5363 |
48
+ | `Afatinib-MK2206.csv` | Afatinib + MK2206 |
49
+ | `GDC0941-AZD6244.csv` | GDC0941 + AZD6244 |
50
+ | `GDC0941-MK2206.csv` | GDC0941 + MK2206 |
51
 
52
+ ## Reference retained from the original notes
 
 
 
 
 
 
 
 
 
 
 
 
53
 
54
  - Menden et al. "Community assessment to advance computational prediction of cancer drug combinations in a pharmacogenomic screen." Nature Communications, 2019.
data/downstream/final_data/drug_response_pytorch/drug_combination/README.md CHANGED
@@ -1,9 +1,40 @@
1
- # DeepDDs
2
- tips
3
- Step1: create dataset for input
4
- creat_data_DC.py could deal with the file like data/new_labels_0_10.csv
5
-
6
- Step2: input the train data to the modle
7
- training_GAT.py is a train model by GAT \\
8
- trianing_GCN.py is a train model by GCN \\
9
- training_validation.py is a validation model for predicting novel drug pairs
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Legacy DeepDDS reference code
2
+
3
+ This directory preserves the collaborator's DeepDDS/scLong/Geneformer reference
4
+ workflows. It is separate from the current BioMod drug-combination implementation
5
+ in `BioMod/drug_combination` at the repository root. Its model names and training
6
+ settings are retained; BioMod model weights are not interchangeable with these
7
+ legacy artifacts.
8
+
9
+ ## Paths
10
+
11
+ Input defaults resolve from the source directory, independent of the shell's
12
+ working directory. `data_summary/` contains the bundled CSV inputs. The legacy
13
+ `independent_set/` input name maps to `data_summary/test_data/`.
14
+
15
+ - `DEEPDDS_DATA_DIR`: optional directory containing compatible input CSVs.
16
+ - `DEEPDDS_ARTIFACT_DIR`: generated cache, embedding, result, and figure directory;
17
+ defaults to `outputs/reference_deepdds` at the repository root.
18
+ - Shared gene IDs resolve from the repository's `model_dependencies/` directory.
19
+
20
+ The legacy scripts also require artifacts that are not bundled: a `smiles.csv`
21
+ with a `smile` column, matching scLong cell embeddings, optional Geneformer
22
+ embeddings under `geneformer/`, and saved benchmark results for plotting.
23
+ `smiles_0_10.csv` has a different schema and is not silently substituted for
24
+ `smiles.csv`. Missing inputs raise a path-specific error. Generate or supply
25
+ these artifacts with the corresponding reference model before running.
26
+
27
+ ## Entry points
28
+
29
+ - `creat_data_DC.py`: prepare PyTorch Geometric datasets from compatible CSVs.
30
+ - `training_GAT.py` and `training_GCN.py`: legacy graph-model training workflows.
31
+ - `training_validation.py`: evaluate held-out drug combinations.
32
+ - `process.py` and `process.ipynb`: align previously generated scLong embeddings
33
+ to the DeepDDS gene order.
34
+ - `plot.ipynb`: plot previously generated ROC results. Its example filenames must
35
+ be adjusted to the chosen run and checkpoint identifier.
36
+
37
+ Start the reference notebooks with this directory as the Jupyter working
38
+ directory. The stored execution outputs have been cleared because they refer to
39
+ the original machine and do not validate this checkout. This path/comment audit
40
+ did not run the reference training workflows or verify their numerical behavior.
data/downstream/final_data/drug_response_pytorch/drug_combination/_reference_paths.py ADDED
@@ -0,0 +1,64 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Portable paths for the bundled legacy DeepDDS reference scripts.
2
+
3
+ DEEPDDS_DATA_DIR can override the bundled data_summary directory.
4
+ DEEPDDS_ARTIFACT_DIR stores generated caches, embeddings, metrics, and figures.
5
+ External legacy artifacts must be supplied by the user; BioMod weights are not
6
+ substituted for scLong or Geneformer artifacts.
7
+ """
8
+
9
+ import os
10
+ from pathlib import Path
11
+
12
+ REFERENCE_DIR = Path(__file__).resolve().parent
13
+ PROJECT_ROOT = REFERENCE_DIR.parents[4]
14
+ DATA_DIR = Path(os.environ.get("DEEPDDS_DATA_DIR", str(REFERENCE_DIR / "data_summary"))).expanduser().resolve()
15
+ ARTIFACT_DIR = Path(os.environ.get("DEEPDDS_ARTIFACT_DIR", str(PROJECT_ROOT / "outputs" / "reference_deepdds"))).expanduser().resolve()
16
+ ASSET_DIR = PROJECT_ROOT / "model_dependencies"
17
+
18
+
19
+ def data_path(relative):
20
+ """Resolve a bundled input, preserving the legacy independent_set layout."""
21
+ relative = relative.replace("\\", "/")
22
+ supplied_path = DATA_DIR / relative
23
+ if supplied_path.is_file():
24
+ return str(supplied_path)
25
+ if relative.startswith("independent_set/"):
26
+ relative = "test_data/" + relative[len("independent_set/"):]
27
+ path = DATA_DIR / relative
28
+ if not path.is_file():
29
+ raise FileNotFoundError(
30
+ "Legacy DeepDDS input is not bundled at {}. Set DEEPDDS_DATA_DIR to "
31
+ "a compatible data directory; retain the expected CSV schema.".format(path)
32
+ )
33
+ return str(path)
34
+
35
+
36
+ def artifact_path(relative):
37
+ """Resolve a generated or externally supplied reference artifact."""
38
+ return str(ARTIFACT_DIR / relative)
39
+
40
+
41
+ def output_path(relative):
42
+ """Resolve an output and create only its containing directory when run."""
43
+ path = ARTIFACT_DIR / relative
44
+ path.parent.mkdir(parents=True, exist_ok=True)
45
+ return str(path)
46
+
47
+
48
+ def output_dir(relative):
49
+ """Return a generated-output directory with a trailing path separator."""
50
+ path = ARTIFACT_DIR / relative
51
+ path.mkdir(parents=True, exist_ok=True)
52
+ return str(path) + os.sep
53
+
54
+
55
+ def required_artifact(relative):
56
+ """Locate an external legacy result without loading its contents."""
57
+ path = Path(artifact_path(relative))
58
+ if not path.is_file():
59
+ raise FileNotFoundError(
60
+ "Legacy reference artifact is missing: {}. Generate it with the "
61
+ "matching reference model or set DEEPDDS_ARTIFACT_DIR to its directory."
62
+ .format(path)
63
+ )
64
+ return str(path)
data/downstream/final_data/drug_response_pytorch/drug_combination/cluster.py CHANGED
@@ -1,25 +1,26 @@
1
- from sklearn.feature_extraction.text import TfidfVectorizer
2
- from sklearn.cluster import KMeans
3
- from sklearn.cluster import AffinityPropagation
4
- from sklearn.cluster import DBSCAN
5
- from sklearn.cluster import MeanShift, estimate_bandwidth
6
- from sklearn.cluster import AgglomerativeClustering
7
- from sklearn.cluster import Birch
8
- from sklearn.cluster import SpectralClustering
9
- from sklearn.mixture import GaussianMixture
10
- from sklearn.metrics.cluster import normalized_mutual_info_score
11
- import pandas as pd
12
- import numpy as np
13
-
14
- AZD6244_weight = pd.read_csv('data/result/AZD6244_weight.csv', header=None)
15
- AZD6244_weight = np.array(AZD6244_weight.T)
16
- print('AZD6244_weight.shape', AZD6244_weight.shape)
17
-
18
- AZD2014_weight = pd.read_csv('data/result/AZD2014_weight.csv', header=None)
19
- AZD2014_weight = np.array(AZD2014_weight.T)
20
- print('AZD2014_weight.shape', AZD2014_weight.shape)
21
-
22
- #AffinityPropagation
23
- af = AffinityPropagation().fit(AZD6244_weight)
24
- result = af.labels_
25
- print(result)
 
 
1
+ from _reference_paths import ASSET_DIR, artifact_path, data_path, output_dir, output_path, required_artifact
2
+ from sklearn.feature_extraction.text import TfidfVectorizer
3
+ from sklearn.cluster import KMeans
4
+ from sklearn.cluster import AffinityPropagation
5
+ from sklearn.cluster import DBSCAN
6
+ from sklearn.cluster import MeanShift, estimate_bandwidth
7
+ from sklearn.cluster import AgglomerativeClustering
8
+ from sklearn.cluster import Birch
9
+ from sklearn.cluster import SpectralClustering
10
+ from sklearn.mixture import GaussianMixture
11
+ from sklearn.metrics.cluster import normalized_mutual_info_score
12
+ import pandas as pd
13
+ import numpy as np
14
+
15
+ AZD6244_weight = pd.read_csv(output_path('result/AZD6244_weight.csv'), header=None)
16
+ AZD6244_weight = np.array(AZD6244_weight.T)
17
+ print('AZD6244_weight.shape', AZD6244_weight.shape)
18
+
19
+ AZD2014_weight = pd.read_csv(output_path('result/AZD2014_weight.csv'), header=None)
20
+ AZD2014_weight = np.array(AZD2014_weight.T)
21
+ print('AZD2014_weight.shape', AZD2014_weight.shape)
22
+
23
+ #AffinityPropagation
24
+ af = AffinityPropagation().fit(AZD6244_weight)
25
+ result = af.labels_
26
+ print(result)
data/downstream/final_data/drug_response_pytorch/drug_combination/creat_data_DC.py CHANGED
@@ -1,123 +1,124 @@
1
- import csv
2
- from itertools import islice
3
-
4
- import pandas as pd
5
- import numpy as np
6
- import os
7
- import json, pickle
8
- from collections import OrderedDict
9
- from rdkit import Chem
10
- # from rdkit.Chem import MolFromSmiles
11
- import networkx as nx
12
- from utils_test import *
13
-
14
-
15
-
16
- def get_cell_feature(cellId, cell_features):
17
- for row in islice(cell_features, 0, None):
18
- if row[0] == cellId:
19
- return row[1: ]
20
-
21
- def atom_features(atom):
22
- return np.array(one_of_k_encoding_unk(atom.GetSymbol(),
23
- ['C', 'N', 'O', 'S', 'F', 'Si', 'P', 'Cl', 'Br', 'Mg', 'Na', 'Ca', 'Fe', 'As',
24
- 'Al', 'I', 'B', 'V', 'K', 'Tl', 'Yb', 'Sb', 'Sn', 'Ag', 'Pd', 'Co', 'Se',
25
- 'Ti', 'Zn', 'H', 'Li', 'Ge', 'Cu', 'Au', 'Ni', 'Cd', 'In', 'Mn', 'Zr', 'Cr',
26
- 'Pt', 'Hg', 'Pb', 'Unknown']) +
27
- one_of_k_encoding(atom.GetDegree(), [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) +
28
- one_of_k_encoding_unk(atom.GetTotalNumHs(), [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) +
29
- one_of_k_encoding_unk(atom.GetImplicitValence(), [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) +
30
- [atom.GetIsAromatic()])
31
-
32
-
33
- def one_of_k_encoding(x, allowable_set):
34
- if x not in allowable_set:
35
- raise Exception("input {0} not in allowable set{1}:".format(x, allowable_set))
36
- return list(map(lambda s: x == s, allowable_set))
37
-
38
-
39
- def one_of_k_encoding_unk(x, allowable_set):
40
- """Maps inputs not in the allowable set to the last element."""
41
- if x not in allowable_set:
42
- x = allowable_set[-1]
43
- return list(map(lambda s: x == s, allowable_set))
44
-
45
-
46
- def smile_to_graph(smile):
47
- mol = Chem.MolFromSmiles(smile)
48
-
49
- c_size = mol.GetNumAtoms()
50
-
51
- features = []
52
- for atom in mol.GetAtoms():
53
- feature = atom_features(atom)
54
- features.append(feature / sum(feature))
55
-
56
- edges = []
57
- for bond in mol.GetBonds():
58
- edges.append([bond.GetBeginAtomIdx(), bond.GetEndAtomIdx()])
59
- g = nx.Graph(edges).to_directed()
60
- edge_index = []
61
- for e1, e2 in g.edges:
62
- edge_index.append([e1, e2])
63
-
64
- return c_size, features, edge_index
65
-
66
-
67
-
68
- def creat_data(datafile, cellfile):
69
- file2 = cellfile
70
- cell_features = []
71
- with open(file2) as csvfile:
72
- csv_reader = csv.reader(csvfile) # 使用csv.reader读取csvfile中的文件
73
- for row in csv_reader:
74
- cell_features.append(row)
75
- cell_features = np.array(cell_features)
76
- print('cell_features',len(cell_features), cell_features)
77
- #exit()
78
- compound_iso_smiles = []
79
- df = pd.read_csv('/data1/ruiyi/deepdds/data/smiles.csv')
80
- compound_iso_smiles += list(df['smile'])
81
- compound_iso_smiles = set(compound_iso_smiles)
82
- smile_graph = {}
83
- print('compound_iso_smiles', compound_iso_smiles)
84
- for smile in compound_iso_smiles:
85
- print('smiles', smile)
86
- g = smile_to_graph(smile)
87
- smile_graph[smile] = g
88
-
89
- datasets = datafile
90
- # convert to PyTorch data format
91
- processed_data_file_train = '/data1/ruiyi/deepdds/data/processed/' + datasets + '_train.pt'
92
-
93
- if ((not os.path.isfile(processed_data_file_train))):
94
- df = pd.read_csv('/data1/ruiyi/deepdds/data/' + datasets + '.csv')
95
- drug1, drug2, cell, label = list(df['drug1']), list(df['drug2']), list(df['cell']), list(df['label'])
96
- print(cell)
97
-
98
- drug1, drug2, cell, label = np.asarray(drug1), np.asarray(drug2), np.asarray(cell), np.asarray(label)
99
- # make data PyTorch Geometric ready
100
-
101
- print('开始创建数据')
102
- TestbedDataset(root='/data1/ruiyi/deepdds/data', dataset=datafile + '_drug1_combined_{}'.format(ckpt), xd=drug1, xt=cell, xt_featrue=cell_features, y=label,smile_graph=smile_graph)
103
- TestbedDataset(root='/data1/ruiyi/deepdds/data', dataset=datafile + '_drug2_combined_{}'.format(ckpt), xd=drug2, xt=cell, xt_featrue=cell_features, y=label,smile_graph=smile_graph)
104
- print('创建数据成功')
105
- print('preparing ', datasets + '_.pt in pytorch format!')
106
- #
107
- # print(processed_data_file_train, ' have been created')
108
- #
109
- # else:
110
- # print(processed_data_file_train, ' are already created')
111
-
112
- if __name__ == "__main__":
113
- # datafile = 'prostate'
114
-
115
- if ind:
116
- cellfile = '/data1/ruiyi/deepdds/data/independent_set/independent_cell_features_954.csv'
117
- da = ['independent_set/independent_input']
118
- else:
119
- cellfile = '/data1/ruiyi/deepdds/data/new_cell_features_954.csv'
120
- da = ['new_labels_0_10']
121
-
122
- for datafile in da:
123
- creat_data(datafile, cellfile)
 
 
1
+ from _reference_paths import ASSET_DIR, artifact_path, data_path, output_dir, output_path, required_artifact
2
+ import csv
3
+ from itertools import islice
4
+
5
+ import pandas as pd
6
+ import numpy as np
7
+ import os
8
+ import json, pickle
9
+ from collections import OrderedDict
10
+ from rdkit import Chem
11
+ # from rdkit.Chem import MolFromSmiles
12
+ import networkx as nx
13
+ from utils_test import *
14
+
15
+
16
+
17
+ def get_cell_feature(cellId, cell_features):
18
+ for row in islice(cell_features, 0, None):
19
+ if row[0] == cellId:
20
+ return row[1: ]
21
+
22
+ def atom_features(atom):
23
+ return np.array(one_of_k_encoding_unk(atom.GetSymbol(),
24
+ ['C', 'N', 'O', 'S', 'F', 'Si', 'P', 'Cl', 'Br', 'Mg', 'Na', 'Ca', 'Fe', 'As',
25
+ 'Al', 'I', 'B', 'V', 'K', 'Tl', 'Yb', 'Sb', 'Sn', 'Ag', 'Pd', 'Co', 'Se',
26
+ 'Ti', 'Zn', 'H', 'Li', 'Ge', 'Cu', 'Au', 'Ni', 'Cd', 'In', 'Mn', 'Zr', 'Cr',
27
+ 'Pt', 'Hg', 'Pb', 'Unknown']) +
28
+ one_of_k_encoding(atom.GetDegree(), [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) +
29
+ one_of_k_encoding_unk(atom.GetTotalNumHs(), [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) +
30
+ one_of_k_encoding_unk(atom.GetImplicitValence(), [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) +
31
+ [atom.GetIsAromatic()])
32
+
33
+
34
+ def one_of_k_encoding(x, allowable_set):
35
+ if x not in allowable_set:
36
+ raise Exception("input {0} not in allowable set{1}:".format(x, allowable_set))
37
+ return list(map(lambda s: x == s, allowable_set))
38
+
39
+
40
+ def one_of_k_encoding_unk(x, allowable_set):
41
+ """Maps inputs not in the allowable set to the last element."""
42
+ if x not in allowable_set:
43
+ x = allowable_set[-1]
44
+ return list(map(lambda s: x == s, allowable_set))
45
+
46
+
47
+ def smile_to_graph(smile):
48
+ mol = Chem.MolFromSmiles(smile)
49
+
50
+ c_size = mol.GetNumAtoms()
51
+
52
+ features = []
53
+ for atom in mol.GetAtoms():
54
+ feature = atom_features(atom)
55
+ features.append(feature / sum(feature))
56
+
57
+ edges = []
58
+ for bond in mol.GetBonds():
59
+ edges.append([bond.GetBeginAtomIdx(), bond.GetEndAtomIdx()])
60
+ g = nx.Graph(edges).to_directed()
61
+ edge_index = []
62
+ for e1, e2 in g.edges:
63
+ edge_index.append([e1, e2])
64
+
65
+ return c_size, features, edge_index
66
+
67
+
68
+
69
+ def creat_data(datafile, cellfile):
70
+ file2 = cellfile
71
+ cell_features = []
72
+ with open(file2) as csvfile:
73
+ csv_reader = csv.reader(csvfile) # Read CSV rows with csv.reader.
74
+ for row in csv_reader:
75
+ cell_features.append(row)
76
+ cell_features = np.array(cell_features)
77
+ print('cell_features',len(cell_features), cell_features)
78
+ #exit()
79
+ compound_iso_smiles = []
80
+ df = pd.read_csv(data_path('smiles.csv'))
81
+ compound_iso_smiles += list(df['smile'])
82
+ compound_iso_smiles = set(compound_iso_smiles)
83
+ smile_graph = {}
84
+ print('compound_iso_smiles', compound_iso_smiles)
85
+ for smile in compound_iso_smiles:
86
+ print('smiles', smile)
87
+ g = smile_to_graph(smile)
88
+ smile_graph[smile] = g
89
+
90
+ datasets = datafile
91
+ # convert to PyTorch data format
92
+ processed_data_file_train = output_dir("cache/processed") + datasets + '_train.pt'
93
+
94
+ if ((not os.path.isfile(processed_data_file_train))):
95
+ df = pd.read_csv(data_path(datasets + '.csv'))
96
+ drug1, drug2, cell, label = list(df['drug1']), list(df['drug2']), list(df['cell']), list(df['label'])
97
+ print(cell)
98
+
99
+ drug1, drug2, cell, label = np.asarray(drug1), np.asarray(drug2), np.asarray(cell), np.asarray(label)
100
+ # make data PyTorch Geometric ready
101
+
102
+ print('Creating datasets')
103
+ TestbedDataset(root=artifact_path("cache"), dataset=datafile + '_drug1_combined_{}'.format(ckpt), xd=drug1, xt=cell, xt_featrue=cell_features, y=label,smile_graph=smile_graph)
104
+ TestbedDataset(root=artifact_path("cache"), dataset=datafile + '_drug2_combined_{}'.format(ckpt), xd=drug2, xt=cell, xt_featrue=cell_features, y=label,smile_graph=smile_graph)
105
+ print('Datasets created')
106
+ print('preparing ', datasets + '_.pt in pytorch format!')
107
+ #
108
+ # print(processed_data_file_train, ' have been created')
109
+ #
110
+ # else:
111
+ # print(processed_data_file_train, ' are already created')
112
+
113
+ if __name__ == "__main__":
114
+ # datafile = 'prostate'
115
+
116
+ if ind:
117
+ cellfile = data_path('independent_set/independent_cell_features_954.csv')
118
+ da = ['independent_set/independent_input']
119
+ else:
120
+ cellfile = data_path('new_cell_features_954.csv')
121
+ da = ['new_labels_0_10']
122
+
123
+ for datafile in da:
124
+ creat_data(datafile, cellfile)
data/downstream/final_data/drug_response_pytorch/drug_combination/data_summary/README.md CHANGED
@@ -1,109 +1,108 @@
1
- # DeepDDS 核心数据文件说明
2
 
3
- ## 数据来源
 
4
 
5
- - **药物协同数据**:来自 O'Neil et al. (2016) 的高通量药物组合筛选实验,涵盖 38 种抗癌药物在 39 个人类癌症细胞系上的两两组合。
6
- - **协同评分**:基于 4×4 剂量-响应矩阵,使用 Combenefit 工具计算 Loewe Additivity Score。同一药物组合若有多个重复实验,取平均值作为最终协同评分。
7
- - **细胞系表达数据**:来自 CCLE (Cancer Cell Line Encyclopedia) 2018年发布的 RNA-seq TPM 数据 (`CCLE_RNAseq_rsem_transcripts_tpm_20180929`),为细胞系在**未经药物处理、正常培养条件**下的基线基因表达谱。
8
 
9
- ## 标签定义(二值化阈值)
 
 
10
 
11
- | 条件 | 标签 | 含义 |
12
- |------|------|------|
13
- | Loewe score > 10 | 1 (正样本) | 协同 (synergistic) |
14
- | Loewe score < 0 | 0 (负样本) | 拮抗 (antagonistic) |
15
- | 0 ≤ score ≤ 10 | 排除 | 模糊区间,不参与建模 |
16
 
17
- 使用严格阈值(>10 而非 >0)的目的是:(1) 减少噪声附近样本的干扰;(2) 获得正负样本基本平衡的数据集。
 
 
 
 
18
 
19
- ## 最终数据规模
20
 
21
- - **12,415** 个唯一的 drug pair–cell line 组合
22
- - **36** 种抗癌药物
23
- - **31** 个人类癌症细胞系
24
- - 正样本 6,116 个,负样本 6,299 个
25
 
26
- ---
 
 
27
 
28
- ## 文件清单
29
 
30
- ### 1. new_labels_0_10.csv — 二值化标签文件(模型训练核心输入)
31
 
32
- - **行数**:12,415 条记录 + 1 行表头
33
- - **列**:
34
- | 列名 | 类型 | 说明 |
35
- |------|------|------|
36
- | drug1 | string | 药物 A 的 SMILES 分子式 |
37
- | drug2 | string | 药物 B 的 SMILES 分子式 |
38
- | cell | string | 细胞系名称(如 A375, HCT116) |
39
- | label | int | 0 = 拮抗 (score < 0),1 = 协同 (score > 10) |
40
 
41
- ### 2. score_0_10.csv — 连续协同评分文件
 
 
 
 
 
42
 
43
- - **行数**:12,415 条记录 + 1 行表头(与 new_labels_0_10.csv 一一对应)
44
- - **列**:
45
- | 列名 | 类型 | 说明 |
46
- |------|------|------|
47
- | drug1 | string | 药物 A 名称(如 5-FU) |
48
- | drug2 | string | 药物 B 名称(如 ABT-888) |
49
- | cell | string | 细胞系名称 |
50
- | score | float | Loewe Additivity 连续协同评分 |
51
- | nor | float | 归一化后的评分 |
52
- | index | int | 样本索引 |
53
- | true_label | int | 真实二值标签 |
54
- | predict_label | int | 模型预测标签(作者预训练结果) |
55
- | predict_score | float | 模型预测概率(作者预训练结果) |
56
- | smile1 | string | 药物 A 的 SMILES |
57
- | smile2 | string | 药物 B 的 SMILES |
58
 
59
- ### 3. new_cell_features_954.csv — 细胞系基因表达矩阵
60
 
61
- - **维度**:31 个细胞系 × 954 个基因
62
- - **数据来源**:CCLE RNA-seq baseline TPM(未经药物处理的基线表达)
63
- - **格式**:
64
- - 第 1 行:gene_id(ENSG 编号)
65
- - 第 2 行:transcript_id(ENST 编号)
66
- - 第 3~33 行:31 个细胞系的表达值,第 1 列为细胞系名称
67
- - **用途**:作为模型中细胞系特征的输入,经全连接网络编码后与药物图特征融合
 
 
 
 
 
 
68
 
69
- ### 4. cell_gene954.csv — 954 基因 ID 对照表
70
 
71
- - **行数**:954 条记录 + 1 行表头
72
- - **列**:gene_id (ENSG), transcript_id (ENST)
73
- - **用途**:记录 new_cell_features_954.csv 中 954 个基因的 Ensembl 编号
74
 
75
- ### 5. smiles_0_10.csv — 药物 SMILES 对照表
 
 
 
 
 
76
 
77
- - **行数**:36 条(无表头)
78
- - **列**:药物名称, SMILES 分子式
79
- - **用途**:药物名称与 SMILES 的映射关系,SMILES 用于构建分子图作为 GNN 输入
80
 
81
- ### 6. drugs_0_10.csv — 药物信息表
 
 
82
 
83
- - **行数**:36 条(无表头)
84
- - **列**:药物名称, SMILES 分子式, 数值编码
85
- - **用途**:药物列表及其编码索引
86
 
87
- ### 7. cell_line_0_10.csv — 细胞系信息表
 
 
88
 
89
- - **行数**:31 条(无表头)
90
- - **列**:细胞系名称, 癌症类型(如 melanoma, ovarian, colon)
91
- - **用途**:记录 31 个细胞系对应的癌症类型
92
 
93
- ---
 
 
94
 
95
- ## DeepDDS 模型输入结构
96
 
97
- ```
98
- Drug A (SMILES) ──→ 分子图 ──→ GAT/GCN ──→ Drug A embedding ─┐
99
- ├──→ 拼接 ──→ 全连接层 ──→ 协同/拮抗预测
100
- Drug B (SMILES) ──→ 分子图 ──→ GAT/GCN ──→ Drug B embedding ─┤
101
- │
102
- Cell line (954 genes TPM) ──→ FC layers ──→ Cell embedding ────┘
 
 
 
 
 
 
103
  ```
104
 
105
- ## 参考文献
106
 
107
  - Wang et al. "DeepDDS: deep graph neural network with attention mechanism to predict synergistic drug combinations." Briefings in Bioinformatics, 2022.
108
  - O'Neil et al. "An unbiased oncology compound screen to identify novel combination strategies." Molecular Cancer Therapeutics, 2016.
109
- - CCLE: Cancer Cell Line Encyclopedia (Broad Institute)
 
1
+ # DeepDDS core data files
2
 
3
+ The provenance and counts below are translated from the existing dataset notes.
4
+ This path/comment audit did not recompute counts or validate the source studies.
5
 
6
+ ## Data sources
 
 
7
 
8
+ - **Drug synergy:** the O'Neil et al. (2016) high-throughput screen of pairwise combinations of 38 anticancer drugs across 39 human cancer cell lines.
9
+ - **Synergy scores:** Loewe Additivity Scores computed with Combenefit from 4 x 4 dose-response matrices. Replicate measurements for the same combination were averaged.
10
+ - **Cell-line expression:** CCLE RNA-seq TPM data released in 2018 (`CCLE_RNAseq_rsem_transcripts_tpm_20180929`), describing baseline expression under standard culture conditions without drug treatment.
11
 
12
+ ## Binary labels
 
 
 
 
13
 
14
+ | Condition | Label | Meaning |
15
+ |---|---|---|
16
+ | Loewe score > 10 | 1 | Synergistic |
17
+ | Loewe score < 0 | 0 | Antagonistic |
18
+ | 0 <= score <= 10 | Excluded | Ambiguous range excluded from modeling |
19
 
20
+ The original notes describe the stricter positive threshold as intended to reduce near-zero noise and obtain roughly balanced classes.
21
 
22
+ ## Previously recorded dataset size
 
 
 
23
 
24
+ - 12,415 unique drug-pair/cell-line combinations.
25
+ - 36 anticancer drugs and 31 human cancer cell lines.
26
+ - 6,116 positive and 6,299 negative samples.
27
 
28
+ ## File descriptions
29
 
30
+ ### `new_labels_0_10.csv`: binary training labels
31
 
32
+ The recorded layout is 12,415 records plus one header row.
 
 
 
 
 
 
 
33
 
34
+ | Column | Type | Description |
35
+ |---|---|---|
36
+ | drug1 | string | SMILES representation of drug A |
37
+ | drug2 | string | SMILES representation of drug B |
38
+ | cell | string | Cell-line name, such as A375 or HCT116 |
39
+ | label | int | 0 = antagonistic (score < 0); 1 = synergistic (score > 10) |
40
 
41
+ ### `score_0_10.csv`: continuous synergy scores
 
 
 
 
 
 
 
 
 
 
 
 
 
 
42
 
43
+ The existing notes record 12,415 records plus a header, corresponding to `new_labels_0_10.csv`.
44
 
45
+ | Column | Type | Description |
46
+ |---|---|---|
47
+ | drug1 | string | Drug A name, such as 5-FU |
48
+ | drug2 | string | Drug B name, such as ABT-888 |
49
+ | cell | string | Cell-line name |
50
+ | score | float | Continuous Loewe Additivity Score |
51
+ | nor | float | Normalized score |
52
+ | index | int | Sample index |
53
+ | true_label | int | Binary ground-truth label |
54
+ | predict_label | int | Legacy predicted label |
55
+ | predict_score | float | Legacy prediction probability |
56
+ | smile1 | string | Drug A SMILES |
57
+ | smile2 | string | Drug B SMILES |
58
 
59
+ The stored prediction columns are historical reference results, not results from the current BioMod checkout.
60
 
61
+ ### `new_cell_features_954.csv`: cell-line expression
 
 
62
 
63
+ - Recorded dimensions: 31 cell lines x 954 genes.
64
+ - Source: CCLE baseline RNA-seq TPM, without drug treatment.
65
+ - Row 1: gene IDs (ENSG).
66
+ - Row 2: transcript IDs (ENST).
67
+ - Rows 3-33: expression values, with the cell-line name in the first column.
68
+ - Purpose: cell features encoded by fully connected layers and combined with drug graph features.
69
 
70
+ ### `cell_gene954.csv`: gene identifier table
 
 
71
 
72
+ - Recorded layout: 954 records plus a header.
73
+ - Columns: `gene_id` (ENSG), `transcript_id` (ENST).
74
+ - Purpose: identify the genes in `new_cell_features_954.csv`.
75
 
76
+ ### `smiles_0_10.csv`: drug SMILES table
 
 
77
 
78
+ - The legacy notes record 36 headerless rows; counts may differ from the parser-based summary in the parent directory.
79
+ - Columns: drug name and SMILES representation.
80
+ - Purpose: map drug names to SMILES used for molecular-graph construction.
81
 
82
+ ### `drugs_0_10.csv`: drug table
 
 
83
 
84
+ - Recorded layout: 36 headerless rows.
85
+ - Columns: drug name, SMILES, numeric encoding.
86
+ - Purpose: enumerate drugs and their encoded indices.
87
 
88
+ ### `cell_line_0_10.csv`: cell-line table
89
 
90
+ - Recorded layout: 31 headerless rows.
91
+ - Columns: cell-line name and cancer type, such as melanoma, ovarian, or colon.
92
+ - Purpose: associate cell lines with cancer types.
93
+
94
+ ## DeepDDS inputs
95
+
96
+ ```text
97
+ Drug A SMILES -> Molecular graph -> GAT/GCN -> Drug A embedding --+
98
+ |
99
+ Drug B SMILES -> Molecular graph -> GAT/GCN -> Drug B embedding --+-> Concatenate -> FC layers -> Synergy prediction
100
+ |
101
+ Cell line (954 genes, TPM) -> FC layers -> Cell embedding -------+
102
  ```
103
 
104
+ ## References retained from the original notes
105
 
106
  - Wang et al. "DeepDDS: deep graph neural network with attention mechanism to predict synergistic drug combinations." Briefings in Bioinformatics, 2022.
107
  - O'Neil et al. "An unbiased oncology compound screen to identify novel combination strategies." Molecular Cancer Therapeutics, 2016.
108
+ - CCLE: Cancer Cell Line Encyclopedia, Broad Institute.
data/downstream/final_data/drug_response_pytorch/drug_combination/data_summary/test_data/README.md CHANGED
@@ -1,62 +1,54 @@
1
- # DeepDDS 独立测试集(AstraZeneca-Sanger DREAM Challenge)
2
 
3
- ## 数据来源
4
 
5
- 来自 AstraZeneca-Sanger Drug Combination DREAM Challenge 数据集(Menden et al., Nature Communications, 2019)。
6
- 原始数据文件:`41467_2019_9799_MOESM3_ESM.xlsx`
7
 
8
- 该数据集与训练集(O'Neil et al. 2016)完全独立,用于评估模型的泛化能力。
9
 
10
- ## 数据规模
11
 
12
- - **668** 个 drug pair–cell line 组合
13
- - **52** 种药物(部分为训练集中未见过的新药)
14
- - **24** 个细胞系(部分为训练集中未见过的新细胞系)
15
- - 正样本 480 个,负样本 188 个
16
 
17
- ---
18
 
19
- ## 文件清单
 
 
 
20
 
21
- ### 核心输入文件
22
 
23
- | 文件 | 说明 |
24
- |------|------|
25
- | `independent_input.csv` | 最终格式化的模型输入文件(668 条),列:drug1(SMILES), drug2(SMILES), cell, label |
26
- | `independent_cell_features_954.csv` | 细胞系 954 基因 CCLE 基线表达矩阵(1019 个细胞系,含训练集外的新细胞系) |
 
 
 
27
 
28
- ### 原始数据与中间处理文件
29
 
30
- | 文件 | 说明 |
31
- |------|------|
32
- | `41467_2019_9799_MOESM3_ESM.xlsx` | 原始论文附件(AZ-DREAM Challenge 全量数据) |
33
- | `independent_allin.csv` | 从原始数据提取的全量组合(76,607 条),列:drug1, drug2, cell |
34
- | `independent_allin_smiles.csv` | 附加 SMILES 信息的全量组合 |
35
- | `independent_set_combinatioin.csv` | 原始组合详情,含 synergy score、IC50、Hill slope、Einf 等药理学参数 |
36
- | `independent_set_newcombinatioin.csv` | 筛选后的组合(去除与训练集重叠的部分) |
37
 
38
- ### 辅助信息文件
39
 
40
- | 文件 | 说明 |
41
- |------|------|
42
- | `independent_set_drug.csv` | 药物信息表(药物名、靶点、通路、SMILES、分子量、LogP、Lipinski 等) |
43
- | `independent_set_cell_line.csv` | 144 个细胞系信息(COSMIC ID、组织类型、TCGA 标签、MSI 状态、是否在 AZ-DREAM/O'Neil/ALMANAC 中) |
44
- | `independent_drug.csv` | 筛选后可用于测试的药物列表 |
45
- | `independent_cell.csv` | 筛选后可用于测试的细胞系列表 |
46
- | `independent_input_drug_cell.csv` | 输入数据中的药物-细胞系映射关系 |
47
 
48
- ### 子集文件(按药物组合拆分)
49
-
50
- | 文件 | 说明 |
51
- |------|------|
52
- | `AZD2014_6244.csv` | AZD2014 + AZD6244 组合的子集 |
53
- | `AZD8931-AZD5363.csv` | AZD8931 + AZD5363 组合的子集 |
54
- | `Afatinib-MK2206.csv` | Afatinib + MK2206 组合的子集 |
55
- | `GDC0941-AZD6244.csv` | GDC0941 + AZD6244 组合的子集 |
56
- | `GDC0941-MK2206.csv` | GDC0941 + MK2206 组合的子集 |
57
-
58
- ---
59
-
60
- ## ��考文献
61
 
62
  - Menden et al. "Community assessment to advance computational prediction of cancer drug combinations in a pharmacogenomic screen." Nature Communications, 2019.
 
1
+ # DeepDDS independent test data
2
 
3
+ ## Source
4
 
5
+ The original notes attribute these data to the AstraZeneca-Sanger Drug Combination DREAM Challenge (Menden et al., Nature Communications, 2019), with source supplement `41467_2019_9799_MOESM3_ESM.xlsx`.
 
6
 
7
+ This collection is intended for evaluation against the separate O'Neil et al. (2016) training collection. The path/comment audit did not verify overlap, counts, or the original source study; those checks are required before making claims about benchmark independence.
8
 
9
+ ## Previously recorded size
10
 
11
+ - 668 drug-pair/cell-line combinations.
12
+ - 52 drugs and 24 cell lines, including entries described in the original notes as absent from training.
13
+ - 480 positive and 188 negative samples.
 
14
 
15
+ ## Core inputs
16
 
17
+ | File | Purpose |
18
+ |---|---|
19
+ | `independent_input.csv` | Formatted model inputs; recorded as 668 rows with drug1 (SMILES), drug2 (SMILES), cell, and label |
20
+ | `independent_cell_features_954.csv` | CCLE baseline expression for 954 genes; recorded as 1,019 cell lines, including lines outside the training collection |
21
 
22
+ ## Source and intermediate files
23
 
24
+ | File | Purpose |
25
+ |---|---|
26
+ | `41467_2019_9799_MOESM3_ESM.xlsx` | Original publication supplement with the full AZ-DREAM data |
27
+ | `independent_allin.csv` | Extracted combinations; recorded as 76,607 rows with drug1, drug2, and cell |
28
+ | `independent_allin_smiles.csv` | Extracted combinations with SMILES annotations |
29
+ | `independent_set_combinatioin.csv` | Combination details, including synergy scores, IC50, Hill slope, and Einf |
30
+ | `independent_set_newcombinatioin.csv` | Filtered combinations described in the original notes as excluding overlap with training |
31
 
32
+ ## Supporting metadata
33
 
34
+ | File | Purpose |
35
+ |---|---|
36
+ | `independent_set_drug.csv` | Drug names, targets, pathways, SMILES, molecular weights, LogP, and Lipinski descriptors |
37
+ | `independent_set_cell_line.csv` | Metadata recorded for 144 cell lines: COSMIC IDs, tissue, TCGA labels, MSI status, and AZ-DREAM/O'Neil/ALMANAC membership |
38
+ | `independent_drug.csv` | Filtered drugs available for testing |
39
+ | `independent_cell.csv` | Filtered cell lines available for testing |
40
+ | `independent_input_drug_cell.csv` | Drug-to-cell-line mapping for the formatted inputs |
41
 
42
+ ## Drug-pair subsets
43
 
44
+ | File | Drug pair |
45
+ |---|---|
46
+ | `AZD2014_6244.csv` | AZD2014 + AZD6244 |
47
+ | `AZD8931-AZD5363.csv` | AZD8931 + AZD5363 |
48
+ | `Afatinib-MK2206.csv` | Afatinib + MK2206 |
49
+ | `GDC0941-AZD6244.csv` | GDC0941 + AZD6244 |
50
+ | `GDC0941-MK2206.csv` | GDC0941 + MK2206 |
51
 
52
+ ## Reference retained from the original notes
 
 
 
 
 
 
 
 
 
 
 
 
53
 
54
  - Menden et al. "Community assessment to advance computational prediction of cancer drug combinations in a pharmacogenomic screen." Nature Communications, 2019.
data/downstream/final_data/drug_response_pytorch/drug_combination/heatmap.py CHANGED
@@ -1,113 +1,114 @@
1
- import matplotlib.pyplot as plt
2
- import numpy as np
3
- import csv
4
- import numpy as np
5
- from numpy import random
6
- import pandas as pd
7
-
8
- import matplotlib.pyplot as plt
9
-
10
- import seaborn as sns
11
-
12
-
13
- def get_map(AZD6244_weight, file):
14
- confusion = AZD6244_weight
15
- # print(confusion.shape)
16
- # 热度图,后面是指定的颜色块,可设置其他的不同颜色
17
- plt.figure(figsize=(10, 8), dpi=1000)
18
-
19
- plt.imshow(confusion, cmap=plt.cm.Blues)
20
- # ticks 坐标轴的坐标点
21
- # label 坐标轴标签说明
22
- indices = range(len(confusion[0]))
23
- # print(indices)
24
- # 第一个是迭代对象,表示坐标的显示顺序,第二个参数是坐标轴显示列表
25
- #plt.xticks(indices, [0, 1, 2])
26
- #plt.yticks(indices, [0, 1, 2])
27
- idx = range(len(confusion[0]))
28
- # print(idx)
29
- plt.xticks(indices, idx, fontsize=25)
30
- plt.yticks(indices, idx, fontsize=25)
31
-
32
- plt.colorbar()
33
-
34
- font = {'family' : 'Times New Roman',
35
- 'weight' : 'normal',
36
- 'size' : 30,
37
- }
38
-
39
- plt.xlabel('Predict', font)
40
- plt.ylabel('True', font)
41
- plt.title(file, font)
42
-
43
- # plt.rcParams两行是用于解决标签不能显示汉字的问题
44
- plt.rcParams['font.sans-serif']=['SimHei']
45
- plt.rcParams['axes.unicode_minus'] = False
46
-
47
- # 显示数据
48
- # for first_index in range(len(confusion)): #第几行
49
- # for second_index in range(len(confusion[first_index])): #第几列
50
- # plt.text(first_index, second_index, confusion[second_index][first_index], fontsize=30)
51
- # 在matlab里面可以对矩阵直接imagesc(confusion)
52
- # 显示
53
- plt.savefig('fig/' + file + '.png')
54
- plt.close()
55
- # plt.show()
56
- if __name__ == '__main__':
57
-
58
- tem = np.array(([[126, 62], [289, 191]]))
59
- get_map(tem, 'DeepSynergy')
60
-
61
- # AZD6244_weight = []
62
- # csvFile = open('data/result/AZD2014_begin_x1.csv', 'r')
63
- # reader = csv.reader(csvFile)
64
- # for it in reader:
65
- # for num in range(len(it)):
66
- # it[num] = round(float(it[num]), 3)
67
- # AZD6244_weight.append(it)
68
- # AZD6244_weight = np.array(AZD6244_weight)
69
- #
70
- # AZD2014_weight = []
71
- # csvFile = open('data/result/AZD2014_fin_x1.csv', 'r')
72
- # reader = csv.reader(csvFile)
73
- # for it in reader:
74
- # for num in range(len(it)):
75
- # it[num] = round(float(it[num]), 3)
76
- # AZD2014_weight.append(it)
77
- # AZD2014_weight = np.array(AZD2014_weight)
78
- #
79
- # print(AZD6244_weight)
80
- # print(AZD2014_weight)
81
- #
82
- # dt = pd.DataFrame(AZD6244_weight.T)
83
- # dt = np.array(dt.corr(method='pearson')) # 默认为'pearson'检验,可选'kendall','spearman'
84
- # dt = np.around(dt, decimals=2)
85
- # get_map(dt, '1')
86
- #
87
- # dt = pd.DataFrame(AZD2014_weight.T)
88
- # dt = np.array(dt.corr(method='pearson')) # 默认为'pearson'检验,可选'kendall','spearman'
89
- # dt = np.around(dt, decimals=2)
90
- # get_map(dt, '2')
91
-
92
- # datafile = 'data/independent_set/independent_cell.csv'
93
- # independent_cell = pd.read_csv(datafile)
94
- # independent_cell = np.array(independent_cell)[:, 0]
95
- # print(independent_cell)
96
- #
97
- # datafile = 'data/independent_set/independent_drug.csv'
98
- # independent_drug = pd.read_csv(datafile)
99
- # independent_drug = np.array(independent_drug)[:, 0]
100
- # print(independent_drug)
101
- #
102
- # lc = len(independent_cell)
103
- # ld = len(independent_drug)
104
- #
105
- # tem = []
106
- # for i in range(ld):
107
- # for j in range(ld):
108
- # for k in range(lc):
109
- # if i != j:
110
- # tem.append([independent_drug[i], independent_drug[j], independent_cell[k]])
111
- #
112
- # df = pd.DataFrame(tem)
113
- # df.to_csv('data/independent_set/independent_allin.csv', header=None, index=None)
 
 
1
+ from _reference_paths import ASSET_DIR, artifact_path, data_path, output_dir, output_path, required_artifact
2
+ import matplotlib.pyplot as plt
3
+ import numpy as np
4
+ import csv
5
+ import numpy as np
6
+ from numpy import random
7
+ import pandas as pd
8
+
9
+ import matplotlib.pyplot as plt
10
+
11
+ import seaborn as sns
12
+
13
+
14
+ def get_map(AZD6244_weight, file):
15
+ confusion = AZD6244_weight
16
+ # print(confusion.shape)
17
+ # Draw a heatmap using the selected color map.
18
+ plt.figure(figsize=(10, 8), dpi=1000)
19
+
20
+ plt.imshow(confusion, cmap=plt.cm.Blues)
21
+ # ticks: axis tick positions.
22
+ # label: axis tick labels.
23
+ indices = range(len(confusion[0]))
24
+ # print(indices)
25
+ # The first argument sets tick positions; the second supplies their labels.
26
+ #plt.xticks(indices, [0, 1, 2])
27
+ #plt.yticks(indices, [0, 1, 2])
28
+ idx = range(len(confusion[0]))
29
+ # print(idx)
30
+ plt.xticks(indices, idx, fontsize=25)
31
+ plt.yticks(indices, idx, fontsize=25)
32
+
33
+ plt.colorbar()
34
+
35
+ font = {'family' : 'Times New Roman',
36
+ 'weight' : 'normal',
37
+ 'size' : 30,
38
+ }
39
+
40
+ plt.xlabel('Predict', font)
41
+ plt.ylabel('True', font)
42
+ plt.title(file, font)
43
+
44
+ # These rcParams settings support Chinese glyphs in labels.
45
+ plt.rcParams['font.sans-serif']=['SimHei']
46
+ plt.rcParams['axes.unicode_minus'] = False
47
+
48
+ # Display values.
49
+ # for first_index in range(len(confusion)): #Row index.
50
+ # for second_index in range(len(confusion[first_index])): #Column index.
51
+ # plt.text(first_index, second_index, confusion[second_index][first_index], fontsize=30)
52
+ # MATLAB offers imagesc(confusion) for the same matrix view.
53
+ # Display.
54
+ plt.savefig(output_dir("fig") + file + '.png')
55
+ plt.close()
56
+ # plt.show()
57
+ if __name__ == '__main__':
58
+
59
+ tem = np.array(([[126, 62], [289, 191]]))
60
+ get_map(tem, 'DeepSynergy')
61
+
62
+ # AZD6244_weight = []
63
+ # csvFile = open('data/result/AZD2014_begin_x1.csv', 'r')
64
+ # reader = csv.reader(csvFile)
65
+ # for it in reader:
66
+ # for num in range(len(it)):
67
+ # it[num] = round(float(it[num]), 3)
68
+ # AZD6244_weight.append(it)
69
+ # AZD6244_weight = np.array(AZD6244_weight)
70
+ #
71
+ # AZD2014_weight = []
72
+ # csvFile = open('data/result/AZD2014_fin_x1.csv', 'r')
73
+ # reader = csv.reader(csvFile)
74
+ # for it in reader:
75
+ # for num in range(len(it)):
76
+ # it[num] = round(float(it[num]), 3)
77
+ # AZD2014_weight.append(it)
78
+ # AZD2014_weight = np.array(AZD2014_weight)
79
+ #
80
+ # print(AZD6244_weight)
81
+ # print(AZD2014_weight)
82
+ #
83
+ # dt = pd.DataFrame(AZD6244_weight.T)
84
+ # dt = np.array(dt.corr(method='pearson')) # Use Pearson correlation; Kendall and Spearman are alternatives.
85
+ # dt = np.around(dt, decimals=2)
86
+ # get_map(dt, '1')
87
+ #
88
+ # dt = pd.DataFrame(AZD2014_weight.T)
89
+ # dt = np.array(dt.corr(method='pearson')) # Use Pearson correlation; Kendall and Spearman are alternatives.
90
+ # dt = np.around(dt, decimals=2)
91
+ # get_map(dt, '2')
92
+
93
+ # datafile = 'data/independent_set/independent_cell.csv'
94
+ # independent_cell = pd.read_csv(datafile)
95
+ # independent_cell = np.array(independent_cell)[:, 0]
96
+ # print(independent_cell)
97
+ #
98
+ # datafile = 'data/independent_set/independent_drug.csv'
99
+ # independent_drug = pd.read_csv(datafile)
100
+ # independent_drug = np.array(independent_drug)[:, 0]
101
+ # print(independent_drug)
102
+ #
103
+ # lc = len(independent_cell)
104
+ # ld = len(independent_drug)
105
+ #
106
+ # tem = []
107
+ # for i in range(ld):
108
+ # for j in range(ld):
109
+ # for k in range(lc):
110
+ # if i != j:
111
+ # tem.append([independent_drug[i], independent_drug[j], independent_cell[k]])
112
+ #
113
+ # df = pd.DataFrame(tem)
114
+ # df.to_csv('data/independent_set/independent_allin.csv', header=None, index=None)