diff --git a/.gitignore b/.gitignore index cff96f77d197d56e732a80193590c28ab4dcf81e..cd9f9d0011bdd9d987d790f972dacb3b7633629a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,42 +1,42 @@ -# ===== 大文件:不进 GitHub,单独托管(HF / OSS)===== -# weight/ 4.1GB 训练权重 checkpoint -# data/ 17GB 下游公开 benchmark 数据集 -weight/ -data/ -!data/ -!data/downstream/ -!data/downstream/final_data/ -!data/downstream/final_data/norman/ -!data/downstream/final_data/norman/data_pyg/ -!data/downstream/final_data/norman/data_pyg/cell_graphs.pkl -!data/downstream/final_data/norman/splits/ -!data/downstream/final_data/norman/splits/*.pkl - -# ===== 二进制 / 打包 / 缓存 ===== -*.zip -!data/downstream/final_data/drug_response_pytorch/drug_response_pytorch/data/GDSC/drug_graph_feat.zip -!data/downstream/final_data/single_drug_response/optional_reference/GDSC/drug_graph_feat.zip -*.tar -*.tar.gz -*.pth -*.pkl -!data/downstream/final_data/norman/data_pyg/cell_graphs.pkl -!data/downstream/final_data/norman/splits/*.pkl -*.ckpt -__pycache__/ -*.pyc -.ipynb_checkpoints/ - -# ===== 本地环境与密钥 ===== -.env -*.local - -# Dated release checkpoints are intentionally versioned on Hugging Face. -!weight/ -weight/* -!weight/0629model/ -!weight/0629model/*.pkl -!weight/0629model/*.pth -!weight/0810model/ -!weight/0810model/*.pkl -!weight/0810model/*.pth +# Large assets are hosted on Hugging Face or separate object storage. +# weight/ contains pretrained checkpoints and paired configurations. +# data/ contains downstream benchmark datasets and reference code. +weight/ +data/ +!data/ +!data/downstream/ +!data/downstream/final_data/ +!data/downstream/final_data/norman/ +!data/downstream/final_data/norman/data_pyg/ +!data/downstream/final_data/norman/data_pyg/cell_graphs.pkl +!data/downstream/final_data/norman/splits/ +!data/downstream/final_data/norman/splits/*.pkl + +# Binary files, archives, and caches. +*.zip +!data/downstream/final_data/drug_response_pytorch/drug_response_pytorch/data/GDSC/drug_graph_feat.zip +!data/downstream/final_data/single_drug_response/optional_reference/GDSC/drug_graph_feat.zip +*.tar +*.tar.gz +*.pth +*.pkl +!data/downstream/final_data/norman/data_pyg/cell_graphs.pkl +!data/downstream/final_data/norman/splits/*.pkl +*.ckpt +__pycache__/ +*.pyc +.ipynb_checkpoints/ + +# Local environment files and credentials. +.env +*.local + +# Dated release checkpoints are intentionally versioned on Hugging Face. +!weight/ +weight/* +!weight/0629model/ +!weight/0629model/*.pkl +!weight/0629model/*.pth +!weight/0810model/ +!weight/0810model/*.pkl +!weight/0810model/*.pth diff --git a/BioMod/README.md b/BioMod/README.md index f4d3373952dfc6a7b532b9300109f318a5c69ce9..c0b45ecfed7181545da2bee95a4671ff603ed160 100644 --- a/BioMod/README.md +++ b/BioMod/README.md @@ -1,399 +1,409 @@ -# BioMod / PerformerModule - -Source code and tutorial notebooks for BioMod single-cell pretraining and -downstream experiments using PerformerModule representations. - -This source directory contains the pretraining entry point and shared training -utilities, alongside downstream task runners, tutorial notebooks, environment -files, and plotting helpers. - -## Highlights - -| Area | What is included | -| --- | --- | -| Representation | Module-based single-cell encoder utilities and module metadata | -| Annotation | Supervised cell-type annotation workflows | -| Integration | Zero-shot batch integration and UMAP/scIB-style reporting | -| Perturbation | Norman gene-perturbation tutorial and smoke workflow | -| Drug response | Single-drug response prediction with GDSC/CCLE data and validation-based model selection | -| Drug combinations | Drug-combination synergy prediction with validation-based model selection | -| Figures | Notebook-level PDF/PNG/SVG plotting cells for paper and GitHub display | - -## Repository Layout - -This is the `BioMod/` source directory inside the project root. See the -[full project layout](../README.md#repository-layout) for sibling -`model_dependencies/` and `weight/` directories; `data/` and `.cache/` are omitted -from these repository trees. - -```text -BioMod/ # Task code: training, evaluation, interpretability and tutorials -├── cell_annotation/ # Cell-type annotation and model fine-tuning -│ ├── module_data/ # Hallmark and Reactome modules with loading utilities -│ │ ├── load_module.py -│ │ ├── module1_hallmarks.json -│ │ └── module2_reactome_2022_filtered.json -│ ├── cmd_finetune.md -│ └── performer_module_finetune.py -├── drug_combination/ # Drug-combination synergy prediction -│ ├── README.md -│ └── train_drug_combination_performer_module.py -├── drug_response/ # Single-drug response prediction and training -│ ├── cmd.txt -│ └── train_drug_performer_module.py -├── gene_perturbation/ # Perturbation response prediction: processing, training and inference -│ ├── data_utils_local.py -│ ├── gears_reduce_local.py -│ ├── inference_local.py -│ ├── model_module.py -│ ├── pertdata_local.py -│ ├── README.md -│ ├── scfm_utils_local.py -│ ├── train_module.py -│ └── utils.py -├── interpretability/ # Module representations and biological interpretability -│ ├── attention/ # Module crosstalk and within-module gene attention -│ │ ├── __init__.py -│ │ ├── module_crosstalk.py -│ │ ├── module_gene_attention.py -│ │ ├── README.md -│ │ └── README_gene_attention.md -│ ├── module_importance/ # Module importance, ablation studies and marker-gene validation -│ │ ├── __init__.py -│ │ ├── cross_class_similarity.py -│ │ ├── family_ablation.py -│ │ ├── family_importance_analysis.py -│ │ ├── marker_gene_verification.py -│ │ ├── masking_ablation.py -│ │ ├── module_importance.py -│ │ ├── module_importance_publication.py -│ │ ├── pub_utils.py -│ │ ├── random_baseline.py -│ │ ├── README.md -│ │ ├── RESULTS.md -│ │ ├── tcell_reactome_analysis.py -│ │ └── TODO.md -│ ├── module_text_alignment/ # Semantic alignment of module representations and pathway text -│ │ ├── __init__.py -│ │ ├── DESIGN.md -│ │ ├── fetch_reactome_descriptions.py -│ │ ├── MANUSCRIPT_SECTION.md -│ │ ├── module_text_alignment.py -│ │ ├── module_text_alignment_extended.py -│ │ ├── module_text_alignment_publication.py -│ │ └── reactome_descriptions.json -│ ├── __init__.py -│ ├── module_geometry.py -│ ├── README.md -│ └── utils.py -├── module_data_all/ # Module definitions, merging and uncovered-gene tools -│ ├── create_uncovered_modules.py -│ ├── merge_modules.py -│ └── merged_modules.json -├── tests/ # CPU tests for shared training utilities -│ └── test_training_utils.py -├── toturials/ # Jupyter tutorials for five downstream tasks -│ ├── cell_annotation.ipynb -│ ├── drug_combination.ipynb -│ ├── drug_response_prediction.ipynb -│ ├── gene_perturbation.ipynb -│ └── zero_shot_batch_integration.ipynb -├── zero_shot_batch/ # Zero-shot batch integration, evaluation and visualization -│ ├── sequencing_depth/ # Low-depth data generation and zero-shot evaluation -│ │ ├── cmd.txt -│ │ ├── make_low_depth_h5ad.py -│ │ └── run_zero_shot_low_depth.py -│ ├── performer_module_zero_shot.py -│ ├── README.md -│ └── visualize_results.py -├── .env.example -├── .gitattributes -├── .gitignore -├── command.txt -├── environment.yml -├── models_performer_module.py -├── module_dataset.py -├── performer_module.py -├── README.md -├── requirements.txt -└── training_utils.py -``` - -The notebook directory is intentionally named `toturials` to preserve existing -paths used by the project. - -## Quick Start - -Create a fresh environment: - -```bash -cd /path/to/BioMod/BioMod -conda env create -f environment.yml -conda activate performer_module -``` - -Or install Python packages into an existing CUDA/PyTorch environment: - -```bash -python -m pip install -r requirements.txt -``` - -For the original server runs, the tutorials assume the `sclong` environment: - -```bash -conda activate sclong -jupyter lab /path/to/BioMod/BioMod/toturials -``` - -### Training utility checks - -The pretraining entry point imports `training_utils.py` explicitly. This local -module supplies random seeding, a sequential distributed validation sampler, -cosine learning-rate scheduling with warmup and restarts, scalar metric -reduction, and resumable checkpoint saving. It does not depend on a separate -package named `utils` or on a downstream task's utility module. - -From this source directory, run the CPU utility tests and check entry-point -imports after installing the dependencies: - -```bash -python -B -m unittest discover -s tests -p test_training_utils.py -v -python -B performer_module.py --help -``` - -The utility tests use synthetic tensors and temporary checkpoints; they do not -load the supplied pretrained weights. Passing these checks does not validate a -full training run. The pretraining entry point still requires CUDA and an NCCL -process group, with training data and paths configured for the target machine. - -## Shared Assets - -The BioMod model weights and matching configuration are stored in -`weight/0810model/`. Use the supplied `.pth` and `.pkl` together, and update -server-specific paths for the local environment. - -Tutorials expect a pretrained BioMod/PerformerModule checkpoint and shared -gene assets. Set `BIOMODULEFM_ASSET_ROOT` to the directory containing -`weight/0810model/` and `model_dependencies/` (the outer `BioMod/` project directory). - -```bash -export BIOMODULEFM_ASSET_ROOT=/path/to/BioMod -``` - -Current local asset files: - -```text -$BIOMODULEFM_ASSET_ROOT/weight/0810model/ # BioMod model weights and matching configuration -├── mae_tost_70M_75mask_ec2.pkl -└── mae_tost_70M_75mask_ec2_epoch0.pth - -$BIOMODULEFM_ASSET_ROOT/model_dependencies/ # Shared gene mappings, gene lists, embeddings and model components -├── module_data_all/ # Shared module definitions and construction tools -│ ├── create_uncovered_modules.py -│ ├── merge_modules.py -│ └── merged_modules.json -├── performer_pytorch_cont/ # Performer attention and model components -│ ├── performer_pytorch_cont/ # Nested package containing copies of the model component source files -│ │ ├── __init__.py -│ │ ├── ding_models.py -│ │ ├── performer_pytorch.py -│ │ ├── performer_pytorch_old.py -│ │ └── reversible.py -│ ├── __init__.py -│ ├── ding_models.py -│ ├── performer_pytorch.py -│ ├── performer_pytorch_old.py -│ └── reversible.py -├── gene2ens.csv -├── selected_gene2vec_27k.npy -└── selected_genes_27k.txt -``` - -Project-local module definitions are kept in: - -```text -BioMod/module_data_all/ # Module definitions, merging and uncovered-gene tools -├── create_uncovered_modules.py -├── merge_modules.py -└── merged_modules.json -``` - -## BioMod model weights - -Use `mae_tost_70M_75mask_ec2_epoch0.pth` with -`mae_tost_70M_75mask_ec2.pkl` from the shared-assets directory above. The supplied -configuration specifies a 2-layer encoder, a 27,874-gene axis, and 660 module -definitions. Keep this configuration paired with the weights so that task -runners can restore the corresponding architecture settings. - -## Tutorials - -| Task | Notebook | Main entry point | Default output style | -| --- | --- | --- | --- | -| Cell-type annotation | `toturials/cell_annotation.ipynb` | task script under `cell_annotation/` | metrics, classification report, confusion matrix | -| Zero-shot batch integration | `toturials/zero_shot_batch_integration.ipynb` | `zero_shot_batch/` runner | UMAP PDF/PNG, avgBatch, avgBio | -| Gene perturbation | `toturials/gene_perturbation.ipynb` | `gene_perturbation/` runner | GEARS-style metrics and smoke outputs | -| Drug response prediction | `toturials/drug_response_prediction.ipynb` | `drug_response/train_drug_performer_module.py` | scatter, residuals, ROC, per-drug and per-cancer tables | -| Drug-combination synergy | `toturials/drug_combination.ipynb` | `drug_combination/train_drug_combination_performer_module.py` | label balance, loss, AUROC, PR-AUC, confusion matrix when predictions exist | - -Each notebook uses absolute paths through configurable roots and keeps full -training disabled by default when data are not present. Flip the notebook -`RUN_TRAINING` flag only after checking the path table at the top of the -notebook. - -## Data Layout - -The Hugging Face repository includes datasets under the project-root `data/` -directory. Individual tasks may require additional inputs or a different data -layout. Check each task's required files, then configure its data paths for the -local environment; the presence of `data/` does not imply that every tutorial -has all of its inputs. - -### Single-Drug Response - -Expected clean data root: - -```text -single_drug_response/data/ -|-- CCLE/ -| |-- Cell_lines_annotations_20181226.txt -| |-- GDSC_IC50.csv -| `-- genomic_expression_561celllines_697genes_demap_features.csv -`-- GDSC/ - |-- 1.Drug_listMon Jun 24 09_00_55 2019.csv - `-- drug_graph_feat/ -``` - -### Drug-Combination Synergy - -Expected clean data root: - -```text -drug_combination/data_summary/ -|-- new_labels_0_10.csv -|-- new_cell_features_954.csv -|-- smiles_0_10.csv -`-- test_data/ - |-- independent_input.csv - `-- independent_cell_features_954.csv -``` - -The drug-combination tutorial also supports old DeepDDS-style prediction CSV -files for ROC, PR, and confusion-matrix visualization. - -## Example Commands - -Drug response prediction with the BioMod model weights: - -```bash -python drug_response/train_drug_performer_module.py \ - --gpu_id 0 \ - --data_path /path/to/single_drug_response/data \ - --checkpoint_path $BIOMODULEFM_ASSET_ROOT/weight/0810model/mae_tost_70M_75mask_ec2_epoch0.pth \ - --hyper_params_path $BIOMODULEFM_ASSET_ROOT/weight/0810model/mae_tost_70M_75mask_ec2.pkl \ - --module_data_dir ./module_data_all \ - --gene_list_path $BIOMODULEFM_ASSET_ROOT/model_dependencies/selected_genes_27k.txt \ - --gene_mapping_path $BIOMODULEFM_ASSET_ROOT/model_dependencies/gene2ens.csv \ - --gene2vec_path $BIOMODULEFM_ASSET_ROOT/model_dependencies/selected_gene2vec_27k.npy \ - --symbol2ens_path $BIOMODULEFM_ASSET_ROOT/model_dependencies/gene2ens.csv \ - --save_dir ./results/drug_response \ - --epochs 300 \ - --batch_size 32 \ - --pool_method max -``` - -Drug-combination synergy prediction: - -```bash -python drug_combination/train_drug_combination_performer_module.py \ - --checkpoint_path $BIOMODULEFM_ASSET_ROOT/weight/0810model/mae_tost_70M_75mask_ec2_epoch0.pth \ - --hyper_params_path $BIOMODULEFM_ASSET_ROOT/weight/0810model/mae_tost_70M_75mask_ec2.pkl \ - --module_data_dir ./module_data_all \ - --gene_list_path $BIOMODULEFM_ASSET_ROOT/model_dependencies/selected_genes_27k.txt \ - --gene_mapping_path $BIOMODULEFM_ASSET_ROOT/model_dependencies/gene2ens.csv \ - --gene2vec_path $BIOMODULEFM_ASSET_ROOT/model_dependencies/selected_gene2vec_27k.npy \ - --symbol2ens_path $BIOMODULEFM_ASSET_ROOT/model_dependencies/gene2ens.csv \ - --data_dir ./drug_combination/data_summary \ - --output_dir ./results/drug_combination \ - --epochs 500 \ - --batch_size 256 \ - --lr 1e-4 \ - --n_folds 1 \ - --pool_method max -``` - -## Display Outputs - -The notebooks regenerate figure files in task output directories. Typical files -include: - -| Task | Generated display files | -| --- | --- | -| Zero-shot integration | A4 UMAP PDF/PNG panels, combined cell-type and batch views | -| Drug response | `drug_response_overall_scatter.pdf`, `drug_response_median_threshold_roc.svg`, per-drug and per-cancer CSV summaries | -| Drug combination | `drug_combination_metric_curves.pdf`, `drug_combination_roc_curves.pdf`, `drug_combination_label_balance.pdf`, optional PR and confusion-matrix panels | -| Cell annotation | confusion matrices, classification reports, UMAP/prediction diagnostics | -| Gene perturbation | smoke-run logs, perturbation metrics, downstream result tables | - -For GitHub display, use PNG files. For manuscripts, use the PDF/SVG exports. -The plotting cells use embedded, editable PDF fonts where supported. - -## Evaluation Results - -This README does not provide verified benchmark scores for the bundled BioMod -model weights. Run the relevant task with a recorded data split and configuration -before reporting performance. Notebook plotting cells and historical outputs -alone do not establish performance for the supplied weights. - -## Reproducibility Conventions - -- Record the exact command line, checkpoint, hyper-parameter pkl, data version, - random seed, and output directory for every final run. -- Keep raw data, cached embeddings, model checkpoints, metric tables, and paper - figures in separate directories. -- Prefer shared storage for large `.h5ad`, `.npy`, and checkpoint outputs. -- Do not commit private data, raw model weights, API keys, or personal absolute - paths. - -## Troubleshooting - -### Missing data - -Open the relevant notebook and run the path-check cell. It prints all required -files and shows which entries are missing. - -### Missing model assets - -Check `BIOMODULEFM_ASSET_ROOT` and verify that the BioMod model weights, matching -`.pkl` configuration, gene2vec file, gene list, gene mapping, and -`performer_pytorch_cont/` directory exist at the paths listed under Shared Assets. - -### No space left on device - -Large `.h5ad`, `.npy`, checkpoint, and figure outputs can fill local disks. -Move result directories to shared storage and remove partial failed outputs only -after confirming they are not needed. - -### Matplotlib font warnings - -The notebooks request Arial first and fall back to DejaVu Sans when Arial is not -installed. PDF font embedding is enabled through `pdf.fonttype = 42`. - -## GitHub Release Checklist - -- Keep `README.md`, `requirements.txt`, `environment.yml`, `.gitignore`, and - `.gitattributes` at the repository root. -- Confirm tutorial notebooks open without personal paths. -- Keep large data and checkpoints outside Git, then document download or - placement instructions. -- Add final figure examples under a small `docs/` or `assets/` directory if the - repository needs rendered images on the GitHub landing page. -- Add license, citation, and data-availability text before public release. - -## Changelog - -| Date | Change | -| --- | --- | -| 2026-06-15 | Added paper-style plotting cells for single-drug and drug-combination tutorials. | -| 2026-06-15 | Removed `stage2_alignment/` from the active repository scope. | -| 2026-06-15 | Added root environment files and GitHub-ready repository documentation. | +# BioMod / PerformerModule + +Source code and tutorial notebooks for BioMod single-cell pretraining and +downstream experiments using PerformerModule representations. + +This source directory contains the pretraining entry point and shared training +utilities, alongside downstream task runners, tutorial notebooks, environment +files, and plotting helpers. + +## Highlights + +| Area | What is included | +| --- | --- | +| Representation | Module-based single-cell encoder utilities and module metadata | +| Annotation | Supervised cell-type annotation workflows | +| Integration | Zero-shot batch integration and UMAP/scIB-style reporting | +| Perturbation | Norman gene-perturbation training and evaluation tutorial | +| Drug response | Single-drug response prediction with GDSC/CCLE data and validation-based model selection | +| Drug combinations | Drug-combination synergy prediction with validation-based model selection | +| Figures | Notebook-level PDF/PNG/SVG plotting cells for paper and GitHub display | + +## Repository Layout + +This is the `BioMod/` source directory inside the project root. See the +[full project layout](../README.md#repository-layout) for sibling +`model_dependencies/` and `weight/` directories; `data/` and `.cache/` are omitted +from these repository trees. + +```text +BioMod/ # Task code: training, evaluation, interpretability and tutorials +├── cell_annotation/ # Cell-type annotation and model fine-tuning +│ ├── module_data/ # Hallmark and Reactome modules with loading utilities +│ │ ├── load_module.py +│ │ ├── module1_hallmarks.json +│ │ └── module2_reactome_2022_filtered.json +│ ├── cmd_finetune.md +│ └── performer_module_finetune.py +├── drug_combination/ # Drug-combination synergy prediction +│ ├── README.md +│ └── train_drug_combination_performer_module.py +├── drug_response/ # Single-drug response prediction and training +│ ├── cmd.txt +│ └── train_drug_performer_module.py +├── gene_perturbation/ # Perturbation response prediction: processing, training and inference +│ ├── data_utils_local.py +│ ├── gears_reduce_local.py +│ ├── inference_local.py +│ ├── model_module.py +│ ├── pertdata_local.py +│ ├── README.md +│ ├── scfm_utils_local.py +│ ├── train_module.py +│ └── utils.py +├── interpretability/ # Module representations and biological interpretability +│ ├── attention/ # Module crosstalk and within-module gene attention +│ │ ├── __init__.py +│ │ ├── module_crosstalk.py +│ │ ├── module_gene_attention.py +│ │ ├── README.md +│ │ └── README_gene_attention.md +│ ├── module_importance/ # Module importance, ablation studies and marker-gene validation +│ │ ├── __init__.py +│ │ ├── cross_class_similarity.py +│ │ ├── family_ablation.py +│ │ ├── family_importance_analysis.py +│ │ ├── marker_gene_verification.py +│ │ ├── masking_ablation.py +│ │ ├── module_importance.py +│ │ ├── module_importance_publication.py +│ │ ├── pub_utils.py +│ │ ├── random_baseline.py +│ │ ├── README.md +│ │ ├── RESULTS.md +│ │ ├── tcell_reactome_analysis.py +│ │ └── TODO.md +│ ├── module_text_alignment/ # Semantic alignment of module representations and pathway text +│ │ ├── __init__.py +│ │ ├── DESIGN.md +│ │ ├── fetch_reactome_descriptions.py +│ │ ├── MANUSCRIPT_SECTION.md +│ │ ├── module_text_alignment.py +│ │ ├── module_text_alignment_extended.py +│ │ ├── module_text_alignment_publication.py +│ │ └── reactome_descriptions.json +│ ├── __init__.py +│ ├── module_geometry.py +│ ├── README.md +│ └── utils.py +├── module_data_all/ # Module definitions, merging and uncovered-gene tools +│ ├── create_uncovered_modules.py +│ ├── merge_modules.py +│ └── merged_modules.json +├── tests/ # CPU tests for shared training utilities +│ └── test_training_utils.py +├── toturials/ # Jupyter tutorials for five downstream tasks +│ ├── cell_annotation.ipynb +│ ├── drug_combination.ipynb +│ ├── drug_response_prediction.ipynb +│ ├── gene_perturbation.ipynb +│ └── zero_shot_batch_integration.ipynb +├── zero_shot_batch/ # Zero-shot batch integration, evaluation and visualization +│ ├── sequencing_depth/ # Low-depth data generation and zero-shot evaluation +│ │ ├── cmd.txt +│ │ ├── make_low_depth_h5ad.py +│ │ └── run_zero_shot_low_depth.py +│ ├── performer_module_zero_shot.py +│ ├── README.md +│ └── visualize_results.py +├── .env.example +├── .gitattributes +├── .gitignore +├── command.txt +├── environment.yml +├── models_performer_module.py +├── module_dataset.py +├── performer_module.py +├── README.md +├── requirements.txt +└── training_utils.py +``` + +The notebook directory is intentionally named `toturials` to preserve existing +paths used by the project. + +## Quick Start + +From the outer repository root, enter the source directory and create a fresh +environment: + +```bash +cd BioMod +conda env create -f environment.yml +conda activate performer_module +``` + +Or install Python packages into an existing CUDA/PyTorch environment: + +```bash +python -m pip install -r requirements.txt +``` + +From this source directory, activate the supplied environment and start Jupyter +with the tutorial directory as its working directory: + +```bash +conda activate performer_module +cd toturials +jupyter lab +``` + +### Training utility checks + +The pretraining entry point imports `training_utils.py` explicitly. This local +module supplies random seeding, a sequential distributed validation sampler, +cosine learning-rate scheduling with warmup and restarts, scalar metric +reduction, and resumable checkpoint saving. It does not depend on a separate +package named `utils` or on a downstream task's utility module. + +From this source directory, run the CPU utility tests and check entry-point +imports after installing the dependencies: + +```bash +python -B -m unittest discover -s tests -p test_training_utils.py -v +python -B performer_module.py --help +``` + +The utility tests use synthetic tensors and temporary checkpoints; they do not +load the supplied pretrained weights. Passing these checks does not validate a +full training run. The pretraining entry point requires CUDA and an NCCL +process group. Supply `--data_path` explicitly: the pretraining corpus is not +bundled. Configure module inputs with `--module_data_dir` or `--module_data_path`. +Generated checkpoints default to `tutorial_runs/pretraining/` under the outer +repository root, with periodic latest checkpoints under its `latest/` subdirectory. + +## Shared Assets + +The BioMod model weights and matching configuration are stored in +the outer repository's `weight/0810model/`. Use the supplied `.pth` and `.pkl` +together. + +Tutorials expect a pretrained BioMod/PerformerModule checkpoint and shared +gene assets. For the shell examples below, set `BIOMOD_ROOT` to the directory containing +`weight/0810model/` and `model_dependencies/` (the outer `BioMod/` project directory). + +```bash +# Run this from the inner BioMod/ source directory. +export BIOMOD_ROOT="$(cd .. && pwd)" +``` + +Current local asset files: + +```text +$BIOMOD_ROOT/weight/0810model/ # BioMod model weights and matching configuration +├── mae_tost_70M_75mask_ec2.pkl +└── mae_tost_70M_75mask_ec2_epoch0.pth + +$BIOMOD_ROOT/model_dependencies/ # Shared gene mappings, gene lists, embeddings and model components +├── module_data_all/ # Shared module definitions and construction tools +│ ├── create_uncovered_modules.py +│ ├── merge_modules.py +│ └── merged_modules.json +├── performer_pytorch_cont/ # Performer attention and model components +│ ├── performer_pytorch_cont/ # Nested package containing copies of the model component source files +│ │ ├── __init__.py +│ │ ├── ding_models.py +│ │ ├── performer_pytorch.py +│ │ ├── performer_pytorch_old.py +│ │ └── reversible.py +│ ├── __init__.py +│ ├── ding_models.py +│ ├── performer_pytorch.py +│ ├── performer_pytorch_old.py +│ └── reversible.py +├── gene2ens.csv +├── selected_gene2vec_27k.npy +└── selected_genes_27k.txt +``` + +Project-local module definitions are kept in: + +```text +BioMod/module_data_all/ # Module definitions, merging and uncovered-gene tools +├── create_uncovered_modules.py +├── merge_modules.py +└── merged_modules.json +``` + +## BioMod model weights + +Use `mae_tost_70M_75mask_ec2_epoch0.pth` with +`mae_tost_70M_75mask_ec2.pkl` from the shared-assets directory above. The supplied +configuration specifies a 2-layer encoder, a 27,874-gene axis, and 660 module +definitions. Keep this configuration paired with the weights so that task +runners can restore the corresponding architecture settings. + +## Tutorials + +| Task | Notebook | Main entry point | Default output style | +| --- | --- | --- | --- | +| Cell-type annotation | `toturials/cell_annotation.ipynb` | task script under `cell_annotation/` | metrics, classification report, confusion matrix | +| Zero-shot batch integration | `toturials/zero_shot_batch_integration.ipynb` | `zero_shot_batch/` runner | UMAP PDF/PNG, avgBatch, avgBio | +| Gene perturbation | `toturials/gene_perturbation.ipynb` | `gene_perturbation/` runner | selected model, configuration, parameters, and training log | +| Drug response prediction | `toturials/drug_response_prediction.ipynb` | `drug_response/train_drug_performer_module.py` | scatter, residuals, ROC, per-drug and per-cancer tables | +| Drug-combination synergy | `toturials/drug_combination.ipynb` | `drug_combination/train_drug_combination_performer_module.py` | label balance, loss, validation metrics, and ROC curves | + +Each notebook derives paths from `PROJECT_ROOT = Path('../..').resolve()` when +Jupyter runs from `BioMod/toturials/`. From the repository root, change this to +`Path('.').resolve()`. Training is disabled by default with `RUN_TRAINING = False`; +zero-shot inference uses `RUN_EVALUATION = False`. Review the configuration and +path checks before enabling execution. The shell variable `BIOMOD_ROOT` is for +the command examples only and does not override notebook configuration cells. + +## Data Layout + +The Hugging Face repository includes datasets under the project-root `data/` +directory. Individual tasks may require additional inputs or a different data +layout. Check each task's required files, then configure its data paths for the +local environment; the presence of `data/` does not imply that every tutorial +has all of its inputs. + +### Single-Drug Response + +Expected clean data root: + +```text +data/downstream/final_data/single_drug_response/data/ +|-- CCLE/ +| |-- Cell_lines_annotations_20181226.txt +| |-- GDSC_IC50.csv +| `-- genomic_expression_561celllines_697genes_demap_features.csv +`-- GDSC/ + |-- 1.Drug_listMon Jun 24 09_00_55 2019.csv + `-- drug_graph_feat/ +``` + +### Drug-Combination Synergy + +Expected clean data root: + +```text +data/downstream/final_data/drug_combination_synergy/data_summary/ +|-- new_labels_0_10.csv +|-- new_cell_features_954.csv +|-- smiles_0_10.csv +`-- test_data/ + |-- independent_input.csv + `-- independent_cell_features_954.csv +``` + +The drug-combination tutorial reads the metric tables and ROC artifacts produced +by its training script. The script does not export per-example prediction CSVs. + +## Example Commands + +Run these commands from the inner `BioMod/` source directory after setting +`BIOMOD_ROOT` as shown above. Drug response prediction with the BioMod model weights: + +```bash +python drug_response/train_drug_performer_module.py \ + --gpu_id 0 \ + --data_path "$BIOMOD_ROOT/data/downstream/final_data/single_drug_response/data" \ + --checkpoint_path "$BIOMOD_ROOT/weight/0810model/mae_tost_70M_75mask_ec2_epoch0.pth" \ + --hyper_params_path "$BIOMOD_ROOT/weight/0810model/mae_tost_70M_75mask_ec2.pkl" \ + --module_data_dir "$BIOMOD_ROOT/model_dependencies/module_data_all" \ + --gene_list_path "$BIOMOD_ROOT/model_dependencies/selected_genes_27k.txt" \ + --gene_mapping_path "$BIOMOD_ROOT/model_dependencies/gene2ens.csv" \ + --gene2vec_path "$BIOMOD_ROOT/model_dependencies/selected_gene2vec_27k.npy" \ + --symbol2ens_path "$BIOMOD_ROOT/model_dependencies/gene2ens.csv" \ + --save_dir "$BIOMOD_ROOT/tutorial_runs/drug_response_prediction" \ + --epochs 300 \ + --batch_size 32 \ + --pool_method max +``` + +Drug-combination synergy prediction: + +```bash +python drug_combination/train_drug_combination_performer_module.py \ + --checkpoint_path "$BIOMOD_ROOT/weight/0810model/mae_tost_70M_75mask_ec2_epoch0.pth" \ + --hyper_params_path "$BIOMOD_ROOT/weight/0810model/mae_tost_70M_75mask_ec2.pkl" \ + --module_data_dir "$BIOMOD_ROOT/model_dependencies/module_data_all" \ + --gene_list_path "$BIOMOD_ROOT/model_dependencies/selected_genes_27k.txt" \ + --gene_mapping_path "$BIOMOD_ROOT/model_dependencies/gene2ens.csv" \ + --gene2vec_path "$BIOMOD_ROOT/model_dependencies/selected_gene2vec_27k.npy" \ + --symbol2ens_path "$BIOMOD_ROOT/model_dependencies/gene2ens.csv" \ + --data_dir "$BIOMOD_ROOT/data/downstream/final_data/drug_combination_synergy/data_summary" \ + --output_dir "$BIOMOD_ROOT/tutorial_runs/drug_combination/results" \ + --epochs 500 \ + --batch_size 256 \ + --lr 1e-4 \ + --n_folds 1 \ + --pool_method max +``` + +## Display Outputs + +The notebooks regenerate figure files in task output directories. Typical files +include: + +| Task | Generated display files | +| --- | --- | +| Zero-shot integration | A4 UMAP PDF/PNG panels, combined cell-type and batch views | +| Drug response | `drug_response_overall_scatter.pdf`, `drug_response_median_threshold_roc.svg`, per-drug and per-cancer CSV summaries | +| Drug combination | `drug_combination_metric_curves.pdf`, `drug_combination_roc_curves.pdf`, `drug_combination_label_balance.pdf` | +| Cell annotation | confusion matrices and classification reports | +| Gene perturbation | training logs containing perturbation evaluation metrics | + +For GitHub display, use PNG files. For manuscripts, use the PDF/SVG exports. +The plotting cells use embedded, editable PDF fonts where supported. + +## Evaluation Results + +This README does not provide verified benchmark scores for the bundled BioMod +model weights. Run the relevant task with a recorded data split and configuration +before reporting performance. Notebook plotting cells and historical outputs +alone do not establish performance for the supplied weights. + +## Reproducibility Conventions + +- Record the exact command line, checkpoint, hyper-parameter pkl, data version, + random seed, and output directory for every final run. +- Keep raw data, cached embeddings, model checkpoints, metric tables, and paper + figures in separate directories. +- Prefer shared storage for large `.h5ad`, `.npy`, and checkpoint outputs. +- Do not commit private data, raw model weights, API keys, or personal absolute + paths. + +## Troubleshooting + +### Missing data + +Open the relevant notebook and run the path-check cell. It prints all required +files and shows which entries are missing. + +### Missing model assets + +Check `BIOMOD_ROOT` for shell commands or `PROJECT_ROOT` in a notebook, and verify that the BioMod model weights, matching +`.pkl` configuration, gene2vec file, gene list, gene mapping, and +`performer_pytorch_cont/` directory exist at the paths listed under Shared Assets. + +### No space left on device + +Large `.h5ad`, `.npy`, checkpoint, and figure outputs can fill local disks. +Move result directories to shared storage and remove partial failed outputs only +after confirming they are not needed. + +### Matplotlib font warnings + +The notebooks request Arial first and fall back to DejaVu Sans when Arial is not +installed. PDF font embedding is enabled through `pdf.fonttype = 42`. + +## GitHub Release Checklist + +- Keep `README.md`, `requirements.txt`, `environment.yml`, `.gitignore`, and + `.gitattributes` at the repository root. +- Confirm tutorial notebooks open without personal paths. +- Keep large data and checkpoints outside Git, then document download or + placement instructions. +- Add final figure examples under a small `docs/` or `assets/` directory if the + repository needs rendered images on the GitHub landing page. +- Add license, citation, and data-availability text before public release. + +## Changelog + +| Date | Change | +| --- | --- | +| 2026-06-15 | Added paper-style plotting cells for single-drug and drug-combination tutorials. | +| 2026-06-15 | Removed `stage2_alignment/` from the active repository scope. | +| 2026-06-15 | Added root environment files and GitHub-ready repository documentation. | diff --git a/BioMod/cell_annotation/cmd_finetune.md b/BioMod/cell_annotation/cmd_finetune.md index 10b6d3d0d7fab8e2c245cd7935326adff3966192..c6b2c1f7e0034ca01b917b3549dc9385e3e751a9 100644 --- a/BioMod/cell_annotation/cmd_finetune.md +++ b/BioMod/cell_annotation/cmd_finetune.md @@ -1,15 +1,21 @@ -torchrun --nproc_per_node=4 performer_module_finetune.py \ ---config_pkl ../ckpts_performer_module_multiply_all_module/performer_module_multiply_all_module.pkl \ ---performer_checkpoint ../ckpts_performer_module_multiply_all_module/performer_module_multiply_all_module_epoch6.pth \ ---data_path ../../cell_annotation/Zheng68K.h5ad \ ---label_column celltype \ ---head_type attn_pool \ ---finetune_mode last_n \ ---unfreeze_last_n 1 \ ---cls_decoder_layers 3 \ ---lr_head 1e-4 \ ---lr_backbone 1e-5 \ ---batch_size 256 \ ---epochs 300 \ ---module_data_dir ../module_data_all \ ---output_dir ../../../../share/extracted_feature/ckpts_performer_module_fintune/performer_module_multiply_all_module +# Cell annotation command + +Run this command from the repository root in a CUDA/NCCL environment. Resource defaults resolve from the script location; explicit relative paths resolve from the working directory. + +```bash +torchrun --nproc_per_node=4 BioMod/cell_annotation/performer_module_finetune.py \ +--config_pkl weight/0810model/mae_tost_70M_75mask_ec2.pkl \ +--performer_checkpoint weight/0810model/mae_tost_70M_75mask_ec2_epoch0.pth \ +--data_path data/downstream/final_data/Zheng68K.h5ad \ +--label_column celltype \ +--head_type attn_pool \ +--finetune_mode last_n \ +--unfreeze_last_n 1 \ +--cls_decoder_layers 3 \ +--lr_head 1e-4 \ +--lr_backbone 1e-5 \ +--batch_size 256 \ +--epochs 300 \ +--module_data_dir model_dependencies/module_data_all \ +--output_dir tutorial_runs/cell_annotation_zheng68k +``` diff --git a/BioMod/cell_annotation/module_data/load_module.py b/BioMod/cell_annotation/module_data/load_module.py index f1d358572fa881c3623b2588b7e989f884152515..21d40e9703067e398ff2308e62d699495667fa32 100644 --- a/BioMod/cell_annotation/module_data/load_module.py +++ b/BioMod/cell_annotation/module_data/load_module.py @@ -1,33 +1,36 @@ -def load_json_files(file1_path, file2_path): - """ - - Args: - file1_path: hallmarks file path - file2_path: reactome file path - - Returns: - tuple: (hallmark_dict, filtered_reactome_dict) - """ - # Read HALLMARKS - with open(file1_path, 'r', encoding='utf-8') as f: - hallmark_dict = json.load(f) - - # Read reactome data - with open(file2_path, 'r', encoding='utf-8') as f: - reactome_full = json.load(f) - - # Filter reactome data - filtered_reactome = { - k: v for k, v in reactome_full.items() - if 50 <= len(v) <= 500 - } - - return hallmark_dict, filtered_reactome - - -# hallmark_data, reactome_data = load_json_files( -# 'module1_hallmarks.json', -# 'module2_reactome.json' -# ) -# len(hallmark_data.keys()) +import json + + +def load_json_files(file1_path, file2_path): + """ + + Args: + file1_path: hallmarks file path + file2_path: reactome file path + + Returns: + tuple: (hallmark_dict, filtered_reactome_dict) + """ + # Read HALLMARKS + with open(file1_path, 'r', encoding='utf-8') as f: + hallmark_dict = json.load(f) + + # Read reactome data + with open(file2_path, 'r', encoding='utf-8') as f: + reactome_full = json.load(f) + + # Filter reactome data + filtered_reactome = { + k: v for k, v in reactome_full.items() + if 50 <= len(v) <= 500 + } + + return hallmark_dict, filtered_reactome + + +# hallmark_data, reactome_data = load_json_files( +# 'module1_hallmarks.json', +# 'module2_reactome_2022_filtered.json' +# ) +# len(hallmark_data.keys()) # len(reactome_data.keys()) \ No newline at end of file diff --git a/BioMod/cell_annotation/performer_module_finetune.py b/BioMod/cell_annotation/performer_module_finetune.py index d3a0afc82e34d66f14cc2336a7e6d6b287fc46ba..025bbc8ede900cf5cda7410503337e411f8bf652 100644 --- a/BioMod/cell_annotation/performer_module_finetune.py +++ b/BioMod/cell_annotation/performer_module_finetune.py @@ -1,1591 +1,1598 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -""" -End-to-end cell annotation fine-tuning for Performer Module. - -This script is intentionally independent from the legacy two-stage pipeline: -1. No feature extraction to .npy files. -2. No imports from other scripts in performer_module/cell_annotation. -3. Reuses the pretrained Performer Module backbone directly. - -Supported cell embedding heads: -- module_mean: mean over restored module tokens -- max: max pooling over restored module tokens -- gene_mean: module -> gene -> mean over genes -- attn_pool: learnable attention pooling over restored module tokens -- cls: use pretrained CLS token if the checkpoint contains one - -Supported backbone finetune modes: -- frozen: freeze the full backbone -- last_n: unfreeze only the last N encoder blocks -- full: full-parameter finetuning -""" - -import argparse -import contextlib -import json -import logging -import os -import pickle -import random -import sys -from pathlib import Path -from typing import TYPE_CHECKING, Dict, List, Optional, Sequence, Tuple - -import numpy as np -import pandas as pd -import torch -import anndata as ad -import torch.distributed as dist -from scipy import sparse -from sklearn.metrics import accuracy_score, classification_report, confusion_matrix, f1_score -from sklearn.model_selection import StratifiedShuffleSplit -from sklearn.preprocessing import LabelEncoder -from torch.nn.parallel import DistributedDataParallel as DDP -from torch import nn -from torch.utils.data import DataLoader, Dataset, Sampler -from torch.utils.data.distributed import DistributedSampler - -SCRIPT_DIR = Path(__file__).resolve().parent -PERFORMER_MODULE_ROOT = SCRIPT_DIR.parent -SCLONG_ROOT = PERFORMER_MODULE_ROOT.parent - -if str(PERFORMER_MODULE_ROOT) not in sys.path: - sys.path.insert(0, str(PERFORMER_MODULE_ROOT)) -if str(SCLONG_ROOT) not in sys.path: - sys.path.insert(0, str(SCLONG_ROOT)) - -from models_performer_module import ( - migrate_legacy_module_config, - migrate_legacy_module_state_dict, -) - -if TYPE_CHECKING: - from models_performer_module import PerformerEncoderOnly_Module - - -logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s") -logger = logging.getLogger(__name__) - - -def is_dist_initialized() -> bool: - return dist.is_available() and dist.is_initialized() - - -def get_rank() -> int: - return dist.get_rank() if is_dist_initialized() else 0 - - -def get_world_size() -> int: - return dist.get_world_size() if is_dist_initialized() else 1 - - -def is_main_process() -> bool: - return get_rank() == 0 - - -def unwrap_model(model: nn.Module) -> nn.Module: - return model.module if hasattr(model, "module") else model - - -def init_distributed_mode(device_arg: str) -> Tuple[torch.device, bool, int, int, int]: - use_ddp = int(os.environ.get("WORLD_SIZE", "1")) > 1 - if not use_ddp: - if device_arg == "cuda" and not torch.cuda.is_available(): - logger.warning("CUDA requested but not available. Falling back to CPU.") - return torch.device("cpu"), False, 0, 1, 0 - return torch.device(device_arg), False, 0, 1, 0 - - local_rank = int(os.environ["LOCAL_RANK"]) - rank = int(os.environ["RANK"]) - world_size = int(os.environ["WORLD_SIZE"]) - - if device_arg != "cuda": - backend = "gloo" - device = torch.device(device_arg) - else: - if not torch.cuda.is_available(): - raise RuntimeError("DDP with device='cuda' requires CUDA to be available") - backend = "nccl" - torch.cuda.set_device(local_rank) - device = torch.device("cuda", local_rank) - - dist.init_process_group(backend=backend) - return device, True, rank, world_size, local_rank - - -def all_gather_numpy(array: np.ndarray) -> np.ndarray: - if not is_dist_initialized(): - return array - gathered = [None for _ in range(get_world_size())] - dist.all_gather_object(gathered, array) - non_empty = [arr for arr in gathered if arr is not None and len(arr) > 0] - if not non_empty: - return np.empty((0,), dtype=array.dtype) - return np.concatenate(non_empty, axis=0) - - -def reduce_sum(value: float, device: torch.device) -> float: - if not is_dist_initialized(): - return float(value) - tensor = torch.tensor(float(value), device=device, dtype=torch.float64) - dist.all_reduce(tensor, op=dist.ReduceOp.SUM) - return float(tensor.item()) - - -def resolve_path(path_str: str) -> Path: - path = Path(path_str) - if path.is_absolute(): - return path - return (SCRIPT_DIR / path).resolve() - - -def seed_all(seed: int) -> None: - random.seed(seed) - np.random.seed(seed) - torch.manual_seed(seed) - torch.cuda.manual_seed_all(seed) - torch.backends.cudnn.deterministic = True - torch.backends.cudnn.benchmark = False - - -class DataPreprocessor: - """Map input genes to the pretrained model gene space.""" - - def __init__( - self, - gene2ens_file: str, - gene2ens_column: str = "Ensembl gene ID", - model_genes_file: Optional[str] = None, - ): - self.gene2ens_file = gene2ens_file - self.gene2ens_column = gene2ens_column - self.model_genes_file = model_genes_file - self.gene2ens_dict: Optional[Dict[str, str]] = None - self.model_genes: Optional[List[str]] = None - self._index_cache: Dict[str, torch.Tensor] = {} - - def load_gene_mapping(self) -> None: - gene2ens = pd.read_csv(self.gene2ens_file, index_col=0) - if self.gene2ens_column not in gene2ens.columns: - raise ValueError(f"Column '{self.gene2ens_column}' not found in {self.gene2ens_file}") - self.gene2ens_dict = gene2ens[self.gene2ens_column].to_dict() - logger.info("Loaded gene2ens mapping: %d entries", len(self.gene2ens_dict)) - - def load_model_genes(self) -> None: - if self.model_genes_file is None: - raise ValueError("model_genes_file must be provided") - with open(self.model_genes_file, "r", encoding="utf-8") as handle: - self.model_genes = [line.rstrip("\n") for line in handle if line.strip()] - logger.info("Loaded %d model genes from %s", len(self.model_genes), self.model_genes_file) - - def _extract_input_genes( - self, - adata: ad.AnnData, - gene_name_column: Optional[str] = None, - ) -> Tuple[List[str], str]: - if gene_name_column is not None and gene_name_column.lower() == "index": - return adata.var.index.astype(str).tolist(), "var.index" - - if gene_name_column is not None and gene_name_column.lower() != "auto": - if gene_name_column not in adata.var.columns: - raise ValueError(f"Gene column '{gene_name_column}' not found in adata.var") - return adata.var[gene_name_column].astype(str).tolist(), f"var['{gene_name_column}']" - - if "gene_name" in adata.var.columns: - return adata.var["gene_name"].astype(str).tolist(), "var['gene_name']" - return adata.var.index.astype(str).tolist(), "var.index" - - def _build_reindex_state( - self, - adata: ad.AnnData, - dataset_name: str, - gene_name_column: Optional[str] = None, - ) -> dict: - input_genes, gene_source = self._extract_input_genes(adata, gene_name_column=gene_name_column) - if self.gene2ens_dict is not None: - input_genes = [self.gene2ens_dict.get(symbol, symbol) for symbol in input_genes] - logger.info("[%s] Mapped input genes to Ensembl IDs", dataset_name) - - input_mapping = {gene: i for i, gene in enumerate(input_genes)} - input_gene_num = len(input_genes) - - if self.model_genes is None: - raise RuntimeError("Model genes must be loaded before preprocess_adata") - - model_index_positions = [input_mapping.get(gene, input_gene_num) for gene in self.model_genes] - overlap = sum(pos < input_gene_num for pos in model_index_positions) - logger.info( - "[%s] Gene source: %s | overlap with model space: %d / %d (model gene space: %d)", - dataset_name, - gene_source, - overlap, - input_gene_num, - len(self.model_genes), - ) - return { - "cache_key": f"{dataset_name}:{gene_source}:{input_gene_num}", - "dataset_name": dataset_name, - "gene_source": gene_source, - "model_index_positions": model_index_positions, - "input_gene_num": input_gene_num, - "overlap": overlap, - } - - def preprocess_adata( - self, - adata_path: str, - min_genes: int = 10, - min_cells: int = 10, - max_cells: Optional[int] = None, - dataset_name: str = "dataset", - gene_name_column: Optional[str] = None, - ) -> Tuple[ad.AnnData, dict]: - logger.info("Loading data from %s", adata_path) - adata = ad.read_h5ad(adata_path) - matrix = adata.X - - if sparse.issparse(matrix): - cell_nonzero = np.asarray((matrix > 0).sum(axis=1)).ravel() - gene_nonzero = np.asarray((matrix > 0).sum(axis=0)).ravel() - else: - cell_nonzero = np.count_nonzero(np.asarray(matrix), axis=1) - gene_nonzero = np.count_nonzero(np.asarray(matrix), axis=0) - - cell_mask = cell_nonzero >= min_genes - gene_mask = gene_nonzero >= min_cells - adata = adata[cell_mask, gene_mask].copy() - - if max_cells is not None and adata.n_obs > max_cells: - logger.info("Subsampling %d cells from %d", max_cells, adata.n_obs) - rng = np.random.default_rng(0) - chosen = np.sort(rng.choice(adata.n_obs, size=max_cells, replace=False)) - adata = adata[chosen].copy() - - reindex_state = self._build_reindex_state( - adata, - dataset_name=dataset_name, - gene_name_column=gene_name_column, - ) - return adata, reindex_state - - def _get_index_tensor(self, reindex_state: dict, device: torch.device) -> torch.Tensor: - key = f"{str(device)}::{reindex_state['cache_key']}" - if key not in self._index_cache: - self._index_cache[key] = torch.as_tensor( - reindex_state["model_index_positions"], - dtype=torch.long, - device=device, - ) - return self._index_cache[key] - - def reindex_tensor_to_model_space( - self, - tensor: torch.Tensor, - reindex_state: dict, - filler: float = 0.0, - ) -> torch.Tensor: - if tensor.dim() != 2: - raise ValueError(f"Expected 2D tensor [B, G], got {tuple(tensor.shape)}") - index_tensor = self._get_index_tensor(reindex_state, tensor.device) - filler_col = torch.full( - (tensor.shape[0], 1), - float(filler), - dtype=tensor.dtype, - device=tensor.device, - ) - expanded_tensor = torch.cat([tensor, filler_col], dim=1) - return torch.index_select(expanded_tensor, 1, index_tensor) - - -class ExpressionDataset(Dataset): - def __init__(self, matrix, labels: np.ndarray, indices: Sequence[int]): - self.matrix = matrix.tocsr() if sparse.issparse(matrix) and not sparse.isspmatrix_csr(matrix) else matrix - self.labels = labels.astype(np.int64, copy=False) - self.indices = np.asarray(indices, dtype=np.int64) - - def __len__(self) -> int: - return len(self.indices) - - def __getitem__(self, idx: int): - real_idx = int(self.indices[idx]) - row = self.matrix[real_idx] - if sparse.issparse(row): - row = row.toarray().ravel() - else: - row = np.asarray(row).reshape(-1) - x = torch.from_numpy(row.astype(np.float32, copy=False)) - y = torch.tensor(self.labels[real_idx], dtype=torch.long) - return x, y - - -class DistributedEvalSampler(Sampler[int]): - """Shard evaluation data across ranks without padding or duplication.""" - - def __init__(self, dataset_len: int, rank: int, world_size: int): - self.indices = list(range(rank, dataset_len, world_size)) - - def __iter__(self): - return iter(self.indices) - - def __len__(self) -> int: - return len(self.indices) - - -class GatedAttentionPool(nn.Module): - def __init__(self, embed_dim: int, hidden_dim: int): - super().__init__() - self.score = nn.Sequential( - nn.LayerNorm(embed_dim), - nn.Linear(embed_dim, hidden_dim), - nn.Tanh(), - nn.Linear(hidden_dim, 1), - ) - - def forward(self, tokens: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: - scores = self.score(tokens).squeeze(-1) - weights = torch.softmax(scores, dim=1) - pooled = torch.sum(tokens * weights.unsqueeze(-1), dim=1) - return pooled, weights - - -class ScGPTClsDecoder(nn.Module): - """Classifier head aligned with scGPT's ClsDecoder.""" - - def __init__(self, d_model: int, n_cls: int, nlayers: int = 3): - super().__init__() - if nlayers < 1: - raise ValueError(f"nlayers must be >= 1, got {nlayers}") - self.layers = nn.ModuleList() - for _ in range(nlayers - 1): - self.layers.append(nn.Linear(d_model, d_model)) - self.layers.append(nn.ReLU()) - self.layers.append(nn.LayerNorm(d_model)) - self.out_layer = nn.Linear(d_model, n_cls) - - def forward(self, x: torch.Tensor) -> torch.Tensor: - for layer in self.layers: - x = layer(x) - return self.out_layer(x) - - -class ModuleSelector(nn.Module): - """Per-module scorer: soft weighting during training, hard threshold during inference.""" - - def __init__(self, embed_dim: int): - super().__init__() - self.scorer = nn.Sequential( - nn.LayerNorm(embed_dim), - nn.Linear(embed_dim, 1), - nn.Sigmoid(), - ) - - def forward( - self, x: torch.Tensor, tau: float = 0.5, hard: bool = False, - ) -> Tuple[torch.Tensor, torch.Tensor]: - scores = self.scorer(x) # (B, L, 1) - if hard: - x = x * (scores > tau).float() - else: - x = x * scores - return x, scores - - -class PerformerModuleCellClassifier(nn.Module): - def __init__( - self, - backbone: "PerformerEncoderOnly_Module", - module_sets: List[List[int]], - num_classes: int, - head_type: str, - pool_hidden_dim: int, - dropout: float, - cls_decoder_layers: int, - use_module_selector: bool = False, - ): - super().__init__() - self.backbone = backbone - self.module_sets = module_sets - self.num_classes = num_classes - self.head_type = head_type - self.base_module_count = len(module_sets) - self.embed_dim = backbone.base_dim - self.num_genes = backbone.num_genes - self.use_cls = bool(backbone.use_cls) - - if self.head_type == "cls" and not self.use_cls: - raise ValueError("head_type='cls' requires a checkpoint with cls_token") - - if self.head_type == "attn_pool": - hidden_dim = pool_hidden_dim if pool_hidden_dim > 0 else self.embed_dim - self.pooler = GatedAttentionPool(self.embed_dim, hidden_dim) - else: - self.pooler = None - self.output_dropout = nn.Dropout(dropout) if dropout > 0 else nn.Identity() - - self.use_module_selector = use_module_selector - self.selector_tau = 0.5 - if use_module_selector: - self.module_selector = ModuleSelector(self.embed_dim) - else: - self.module_selector = None - self._selector_scores = None - - if self.head_type == "gene_mean": - module_idx, gene_idx, gene_counts = self._build_module_gene_index() - self.register_buffer("agg_module_idx", module_idx) - self.register_buffer("agg_gene_idx", gene_idx) - self.register_buffer("agg_gene_counts", gene_counts) - else: - self.agg_module_idx = None - self.agg_gene_idx = None - self.agg_gene_counts = None - - self.classifier = ScGPTClsDecoder( - d_model=self.embed_dim, - n_cls=num_classes, - nlayers=cls_decoder_layers, - ) - - def load_state_dict(self, state_dict, strict: bool = True, assign: bool = False): - """Accept current module names and older serialized backbone names.""" - return super().load_state_dict( - migrate_legacy_module_state_dict(state_dict), - strict=strict, - assign=assign, - ) - - def _build_module_gene_index(self) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - module_indices = [] - gene_indices = [] - gene_counts = torch.zeros(self.num_genes, dtype=torch.long) - for module_idx, module_genes in enumerate(self.module_sets): - for gene_idx in module_genes: - if 0 <= gene_idx < self.num_genes: - module_indices.append(module_idx) - gene_indices.append(gene_idx) - gene_counts[gene_idx] += 1 - return ( - torch.tensor(module_indices, dtype=torch.long), - torch.tensor(gene_indices, dtype=torch.long), - gene_counts, - ) - - def _restore_module_order(self, latent: torch.Tensor, ids_restore: torch.Tensor) -> torch.Tensor: - if self.use_cls: - module_tokens = latent[:, 1:, :] - else: - module_tokens = latent - gather_index = ids_restore.unsqueeze(-1).expand(-1, -1, module_tokens.shape[-1]) - return torch.gather(module_tokens, dim=1, index=gather_index) - - def _module_to_gene(self, module_tokens: torch.Tensor) -> torch.Tensor: - batch_size, _, dim = module_tokens.shape - gene_feats = torch.zeros( - batch_size, - self.num_genes, - dim, - device=module_tokens.device, - dtype=module_tokens.dtype, - ) - global_mean = module_tokens.mean(dim=1) - - base_counts = self.agg_gene_counts.to(device=module_tokens.device, dtype=module_tokens.dtype) - gene_counts = base_counts.unsqueeze(0).expand(batch_size, -1).clone() - - static_module_tokens = module_tokens[:, :self.base_module_count, :] - if self.agg_module_idx.numel() > 0: - pair_module_feats = static_module_tokens[:, self.agg_module_idx, :] - gene_idx_expand = self.agg_gene_idx.view(1, -1, 1).expand(batch_size, -1, dim) - gene_feats.scatter_add_(1, gene_idx_expand, pair_module_feats) - - zero_mask = gene_counts == 0 - gene_feats = gene_feats / gene_counts.clamp(min=1.0).unsqueeze(-1) - gene_feats = torch.where(zero_mask.unsqueeze(-1), global_mean.unsqueeze(1), gene_feats) - return gene_feats - - def get_selector_sparsity_loss(self) -> torch.Tensor: - if self._selector_scores is not None: - return self._selector_scores.mean() - return torch.tensor(0.0) - - def forward(self, x_model: torch.Tensor) -> torch.Tensor: - latent, _, ids_restore, _ = self.backbone.forward_encoder(x_model, mask_ratio=0.0) - - if self.head_type == "cls": - cell_emb = latent[:, 0, :] - else: - module_tokens = self._restore_module_order(latent, ids_restore) - if self.module_selector is not None: - module_tokens, self._selector_scores = self.module_selector( - module_tokens, tau=self.selector_tau, hard=not self.training, - ) - if self.head_type == "module_mean": - cell_emb = module_tokens.mean(dim=1) - elif self.head_type == "max": - cell_emb = module_tokens.amax(dim=1) - elif self.head_type == "attn_pool": - cell_emb, _ = self.pooler(module_tokens) - elif self.head_type == "gene_mean": - gene_tokens = self._module_to_gene(module_tokens) - cell_emb = gene_tokens.mean(dim=1) - else: - raise ValueError(f"Unsupported head_type: {self.head_type}") - - cell_emb = self.output_dropout(cell_emb) - return self.classifier(cell_emb) - - -def get_encoder_blocks(backbone: "PerformerEncoderOnly_Module") -> List[nn.Module]: - net = backbone.encoder.net - if hasattr(net, "layers"): - return list(net.layers) - if hasattr(net, "blocks"): - return list(net.blocks) - raise AttributeError("Cannot find encoder blocks on Performer backbone") - - -def set_requires_grad(module: nn.Module, flag: bool) -> None: - for param in module.parameters(): - param.requires_grad = flag - - -def configure_backbone_trainability( - backbone: "PerformerEncoderOnly_Module", - finetune_mode: str, - unfreeze_last_n: int, - train_pos_emb: bool, - train_module_embed: bool, -) -> Dict[str, int]: - set_requires_grad(backbone, False) - - if finetune_mode == "full": - # "full" here means full finetuning of the encoder pathway used by - # downstream classification, not the MAE-style reconstruction decoder. - set_requires_grad(backbone.module_embed, True) - set_requires_grad(backbone.encoder, True) - set_requires_grad(backbone.encoder_norm, True) - backbone.pos_emb.requires_grad = True - if getattr(backbone, "cls_token", None) is not None: - backbone.cls_token.requires_grad = True - elif finetune_mode == "last_n": - blocks = get_encoder_blocks(backbone) - if unfreeze_last_n <= 0: - raise ValueError("unfreeze_last_n must be > 0 when finetune_mode='last_n'") - for block in blocks[-min(unfreeze_last_n, len(blocks)):]: - set_requires_grad(block, True) - set_requires_grad(backbone.encoder_norm, True) - elif finetune_mode != "frozen": - raise ValueError(f"Unsupported finetune_mode: {finetune_mode}") - - if train_pos_emb: - backbone.pos_emb.requires_grad = True - if getattr(backbone, "cls_token", None) is not None: - backbone.cls_token.requires_grad = True - - if train_module_embed: - set_requires_grad(backbone.module_embed, True) - - if getattr(backbone, 'family_emb', None) is not None: - backbone.family_emb.requires_grad = True - if getattr(backbone, 'module_gate', None) is not None: - set_requires_grad(backbone.module_gate, True) - - total_params = sum(p.numel() for p in backbone.parameters()) - trainable_params = sum(p.numel() for p in backbone.parameters() if p.requires_grad) - return { - "total": total_params, - "trainable": trainable_params, - } - - -def build_optimizer( - model: PerformerModuleCellClassifier, - lr_head: float, - lr_backbone: float, - weight_decay: float, -) -> torch.optim.Optimizer: - backbone_params = [] - head_params = [] - new_module_params = [] - for name, param in model.named_parameters(): - if not param.requires_grad: - continue - if 'family_emb' in name or 'module_gate' in name or 'module_selector' in name: - new_module_params.append(param) - elif name.startswith("backbone."): - backbone_params.append(param) - else: - head_params.append(param) - - param_groups = [] - if backbone_params: - param_groups.append({"params": backbone_params, "lr": lr_backbone}) - if head_params: - param_groups.append({"params": head_params, "lr": lr_head}) - if new_module_params: - param_groups.append({"params": new_module_params, "lr": lr_head}) - - if not param_groups: - raise RuntimeError("No trainable parameters found") - - return torch.optim.AdamW(param_groups, weight_decay=weight_decay, betas=(0.9, 0.999)) - - -def resolve_batch_sizes(args, world_size: int) -> Tuple[int, int, int]: - if args.global_batch_size is not None: - denom = world_size * args.grad_accum_steps - if args.global_batch_size % denom != 0: - raise ValueError( - f"global_batch_size={args.global_batch_size} must be divisible by " - f"world_size({world_size}) * grad_accum_steps({args.grad_accum_steps})" - ) - train_batch_size = args.global_batch_size // denom - else: - train_batch_size = args.batch_size - - if train_batch_size <= 0: - raise ValueError(f"Resolved per-device train batch size must be > 0, got {train_batch_size}") - - if args.global_eval_batch_size is not None: - if args.global_eval_batch_size % world_size != 0: - raise ValueError( - f"global_eval_batch_size={args.global_eval_batch_size} must be divisible by world_size({world_size})" - ) - eval_batch_size = args.global_eval_batch_size // world_size - elif args.eval_batch_size is not None: - eval_batch_size = args.eval_batch_size - else: - eval_batch_size = train_batch_size - - if eval_batch_size <= 0: - raise ValueError(f"Resolved per-device eval batch size must be > 0, got {eval_batch_size}") - - effective_global_batch = train_batch_size * world_size * args.grad_accum_steps - return train_batch_size, eval_batch_size, effective_global_batch - - -def infer_model_config(args, checkpoint: dict, pkl_config: Optional[dict], module_data: List[List[int]]) -> dict: - from models_performer_module import sanitize_performer_module_config - - if pkl_config is not None and isinstance(pkl_config, dict): - config = sanitize_performer_module_config( - pkl_config, - source=str(args.config_pkl), - log_fn=logger.info, - ) - logger.info("Using config from PKL: %s", args.config_pkl) - if "module_sets" in config: - module_data = config["module_sets"] - elif isinstance(checkpoint, dict) and "config" in checkpoint: - config = sanitize_performer_module_config( - checkpoint["config"], - source=str(resolve_path(args.performer_checkpoint)), - log_fn=logger.info, - ) - logger.info("Using config from checkpoint") - else: - raise ValueError( - "Backbone config was not found in config_pkl or checkpoint['config']. " - "This script now expects pretrained structural args to be restored automatically." - ) - - return { - "module_sets": module_data, - "num_genes": config["num_genes"], - "base_dim": config["base_dim"], - "enc_depth": config["enc_depth"], - "enc_heads": config["enc_heads"], - "enc_dim_head": config["enc_dim_head"], - "gene_embed_dim": config.get("gene_embed_dim", 200), - "fusion_method": config.get("fusion_method", "multiply"), - "use_gene2vec": config.get("use_gene2vec", True), - "loss_mode": config.get("loss_mode", "gene_mean"), - "recon_target_level": config.get("recon_target_level", "gene"), - "pretrain_arch": config.get("pretrain_arch", "mae"), - "mask_style": config.get("mask_style", "drop"), - "mask_token_mode": config.get("mask_token_mode", "learned"), - "pos_embed_mode": config.get("pos_embed_mode", "module_id"), - "decoder_type": config.get("decoder_type", "simple"), - "decoder_gene_agg": config.get("decoder_gene_agg", False), - "dec_embed_dim": config.get("dec_embed_dim", config["base_dim"]), - "dec_depth": config.get("dec_depth", 4), - "dec_heads": config.get("dec_heads", 8), - "dec_dim_head": config.get("dec_dim_head", 64), - "use_cls_token": bool(config.get("use_cls_token", False)), - "cls_loss_weight": float(config.get("cls_loss_weight", 0.0)), - "cls_head_hidden_dim": int(config.get("cls_head_hidden_dim", 0)), - } - - -def compute_module_family_ids(module_data_dir: str) -> List[int]: - """Read merged_modules.json, return family ID (0-3) for each module.""" - merged_path = Path(module_data_dir) / "merged_modules.json" - with open(merged_path) as f: - modules = json.load(f) - PREFIXES = [ - ("module1_hallmarks", 0), - ("module2_reactome", 1), - ("module3_hotspot", 2), - ("module3_processed", 3), - ] - family_ids = [] - for name in modules.keys(): - fid = 0 - for prefix, idx in PREFIXES: - if name.startswith(prefix): - fid = idx - break - family_ids.append(fid) - return family_ids - - -def load_backbone(args, device: torch.device) -> Tuple["PerformerEncoderOnly_Module", List[List[int]], dict]: - from models_performer_module import PerformerEncoderOnly_Module - from module_dataset import load_module_data - - config_pkl = None - config_path = resolve_path(args.config_pkl) - if config_path.exists(): - with open(config_path, "rb") as handle: - config_pkl = migrate_legacy_module_config(pickle.load(handle)) - else: - logger.warning("Config PKL not found at %s; fallback to checkpoint config", config_path) - - ckpt_path = resolve_path(args.performer_checkpoint) - checkpoint = torch.load(ckpt_path, map_location=device) - - module_data = None - if isinstance(config_pkl, dict) and "module_sets" in config_pkl: - module_data = config_pkl["module_sets"] - logger.info("Using module_sets from PKL (%d modules)", len(module_data)) - elif isinstance(checkpoint, dict) and isinstance(checkpoint.get("config"), dict) and "module_sets" in checkpoint["config"]: - module_data = checkpoint["config"]["module_sets"] - logger.info("Using module_sets from checkpoint config (%d modules)", len(module_data)) - else: - if not args.module_data_dir: - raise ValueError("module_data_dir is required when module_sets are not stored in the PKL/checkpoint config") - module_data = load_module_data( - module_data_dir=str(resolve_path(args.module_data_dir)), - gene_list_path=str(resolve_path(args.gene_list_path)), - gene_mapping_path=str(resolve_path(args.gene_mapping_path)), - verbose=True, - ) - - if args.shuffle_module_genes: - logger.info("Shuffling genes across modules with seed=%d", args.shuffle_seed) - rng = random.Random(args.shuffle_seed) - module_sizes = [len(module) for module in module_data] - all_genes = [gene for module in module_data for gene in module] - rng.shuffle(all_genes) - shuffled = [] - start = 0 - for size in module_sizes: - shuffled.append(all_genes[start:start + size]) - start += size - module_data = shuffled - - model_config = infer_model_config(args, checkpoint, config_pkl, module_data) - module_data = model_config["module_sets"] - - if "model_state_dict" in checkpoint: - state_dict = checkpoint["model_state_dict"] - elif "model" in checkpoint: - state_dict = checkpoint["model"] - else: - state_dict = checkpoint - - has_cls_in_checkpoint = "cls_token" in state_dict - use_cls_token = has_cls_in_checkpoint or model_config["use_cls_token"] - - logger.info( - "Backbone config: base_dim=%d depth=%d heads=%d modules=%d cls=%s", - model_config["base_dim"], - model_config["enc_depth"], - model_config["enc_heads"], - len(module_data), - use_cls_token, - ) - - module_family_ids = None - if getattr(args, "use_family_emb", False): - pdd = getattr(args, "module_data_dir", None) - if pdd: - module_family_ids = compute_module_family_ids(str(resolve_path(pdd))) - logger.info("Family embedding enabled: %d module family IDs computed", len(module_family_ids)) - - backbone = PerformerEncoderOnly_Module( - module_sets=module_data, - num_genes=model_config["num_genes"], - max_seq_len=len(module_data), - base_dim=model_config["base_dim"], - enc_depth=model_config["enc_depth"], - enc_heads=model_config["enc_heads"], - enc_dim_head=model_config["enc_dim_head"], - mask_ratio=0.75, - gene2vec_file=str(resolve_path(args.gene2vec_data_path)), - gene_embed_dim=model_config["gene_embed_dim"], - fusion_method=model_config["fusion_method"], - use_cls_token=use_cls_token, - cls_loss_weight=model_config["cls_loss_weight"] if use_cls_token else 0.0, - cls_head_hidden_dim=model_config["cls_head_hidden_dim"], - use_gene2vec=model_config["use_gene2vec"], - loss_mode=model_config["loss_mode"], - recon_target_level=model_config["recon_target_level"], - pretrain_arch=model_config["pretrain_arch"], - mask_style=model_config["mask_style"], - mask_token_mode=model_config["mask_token_mode"], - pos_embed_mode=model_config["pos_embed_mode"], - decoder_type=model_config["decoder_type"], - decoder_gene_agg=model_config["decoder_gene_agg"], - dec_embed_dim=model_config["dec_embed_dim"], - dec_depth=model_config["dec_depth"], - dec_heads=model_config["dec_heads"], - dec_dim_head=model_config["dec_dim_head"], - use_family_emb=getattr(args, "use_family_emb", False), - module_family_ids=module_family_ids, - use_module_gate=getattr(args, "use_module_gate", False), - ) - - filtered_state_dict = { - key: value - for key, value in state_dict.items() - if "go_conv" not in key and "G_go" not in key.lower() - } - missing_keys, unexpected_keys = backbone.load_state_dict(filtered_state_dict, strict=False) - expected_new = {"family_emb", "module_family_ids", "module_gate."} - real_missing = [k for k in missing_keys - if not any(k == e or k.startswith(e) for e in expected_new)] - if real_missing: - logger.warning("Unexpected missing keys: %s", real_missing[:10]) - if unexpected_keys: - logger.warning("Unexpected keys: %s", unexpected_keys[:10]) - - backbone.to(device) - return backbone, module_data, model_config - - -def stratified_split( - labels: np.ndarray, - seed: int, - val_ratio: float, - test_ratio: float, -) -> Tuple[np.ndarray, np.ndarray, np.ndarray]: - indices = np.arange(len(labels)) - test_split = StratifiedShuffleSplit(n_splits=1, test_size=test_ratio, random_state=seed) - train_val_idx, test_idx = next(test_split.split(indices, labels)) - - remaining_ratio = 1.0 - test_ratio - if remaining_ratio <= 0.0: - raise ValueError("test_ratio must be < 1") - val_ratio_adjusted = val_ratio / remaining_ratio - - train_val_labels = labels[train_val_idx] - val_split = StratifiedShuffleSplit(n_splits=1, test_size=val_ratio_adjusted, random_state=seed) - train_sub_idx, val_sub_idx = next(val_split.split(train_val_idx, train_val_labels)) - - train_idx = train_val_idx[train_sub_idx] - val_idx = train_val_idx[val_sub_idx] - return train_idx, val_idx, test_idx - - -def stratified_train_val_split( - labels: np.ndarray, - seed: int, - val_ratio: float, -) -> Tuple[np.ndarray, np.ndarray]: - if not 0.0 < val_ratio < 1.0: - raise ValueError(f"val_ratio must be in (0, 1), got {val_ratio}") - indices = np.arange(len(labels)) - val_split = StratifiedShuffleSplit(n_splits=1, test_size=val_ratio, random_state=seed) - train_idx, val_idx = next(val_split.split(indices, labels)) - return train_idx, val_idx - - -@torch.no_grad() -def evaluate( - model: PerformerModuleCellClassifier, - data_loader: DataLoader, - preprocessor: DataPreprocessor, - reindex_state: dict, - criterion: nn.Module, - device: torch.device, -) -> dict: - model.eval() - loss_sum = 0.0 - sample_count = 0 - preds_all = [] - labels_all = [] - - for x, y in data_loader: - x = x.to(device) - y = y.to(device) - x_model = preprocessor.reindex_tensor_to_model_space(x, reindex_state=reindex_state) - logits = model(x_model) - loss = criterion(logits, y) - loss_sum += loss.item() * y.size(0) - sample_count += y.size(0) - preds_all.append(logits.argmax(dim=-1).cpu().numpy()) - labels_all.append(y.cpu().numpy()) - - preds = np.concatenate(preds_all) if preds_all else np.empty((0,), dtype=np.int64) - labels = np.concatenate(labels_all) if labels_all else np.empty((0,), dtype=np.int64) - preds = all_gather_numpy(preds) - labels = all_gather_numpy(labels) - loss_sum = reduce_sum(loss_sum, device) - sample_count = int(reduce_sum(sample_count, device)) - return { - "loss": float(loss_sum / max(sample_count, 1)), - "acc": float(accuracy_score(labels, preds)), - "macro_f1": float(f1_score(labels, preds, average="macro")), - "preds": preds, - "labels": labels, - } - - -def train_one_epoch( - model: PerformerModuleCellClassifier, - data_loader: DataLoader, - preprocessor: DataPreprocessor, - reindex_state: dict, - criterion: nn.Module, - optimizer: torch.optim.Optimizer, - device: torch.device, - grad_clip: float, - grad_accum_steps: int, - use_ddp: bool, - selector_sparsity_weight: float = 0.0, -) -> dict: - model.train() - running_loss = 0.0 - sample_count = 0 - preds_all = [] - labels_all = [] - optimizer.zero_grad(set_to_none=True) - - num_steps = len(data_loader) - for step, (x, y) in enumerate(data_loader, start=1): - x = x.to(device) - y = y.to(device) - x_model = preprocessor.reindex_tensor_to_model_space(x, reindex_state=reindex_state) - - sync_context = ( - model.no_sync() - if use_ddp and grad_accum_steps > 1 and step % grad_accum_steps != 0 - else contextlib.nullcontext() - ) - with sync_context: - logits = model(x_model) - loss = criterion(logits, y) - if selector_sparsity_weight > 0: - model_raw = model.module if use_ddp else model - loss = loss + selector_sparsity_weight * model_raw.get_selector_sparsity_loss() - (loss / grad_accum_steps).backward() - - if step % grad_accum_steps == 0 or step == num_steps: - if grad_clip > 0: - torch.nn.utils.clip_grad_norm_(model.parameters(), grad_clip) - optimizer.step() - optimizer.zero_grad(set_to_none=True) - - running_loss += loss.item() * y.size(0) - sample_count += y.size(0) - preds_all.append(logits.argmax(dim=-1).detach().cpu().numpy()) - labels_all.append(y.detach().cpu().numpy()) - - preds = np.concatenate(preds_all) if preds_all else np.empty((0,), dtype=np.int64) - labels = np.concatenate(labels_all) if labels_all else np.empty((0,), dtype=np.int64) - preds = all_gather_numpy(preds) - labels = all_gather_numpy(labels) - running_loss = reduce_sum(running_loss, device) - sample_count = int(reduce_sum(sample_count, device)) - return { - "loss": float(running_loss / max(sample_count, 1)), - "acc": float(accuracy_score(labels, preds)), - "macro_f1": float(f1_score(labels, preds, average="macro")), - } - - -def save_json(path: Path, payload: dict) -> None: - with open(path, "w", encoding="utf-8") as handle: - json.dump(payload, handle, indent=2, ensure_ascii=False) - - -def build_confusion_tables( - labels: np.ndarray, - preds: np.ndarray, - class_names: Sequence[str], -) -> Tuple[np.ndarray, np.ndarray, pd.DataFrame, pd.DataFrame]: - label_ids = np.arange(len(class_names), dtype=np.int64) - conf = confusion_matrix(labels, preds, labels=label_ids) - row_sums = conf.sum(axis=1, keepdims=True) - conf_norm = np.divide( - conf.astype(np.float64), - row_sums, - out=np.zeros_like(conf, dtype=np.float64), - where=row_sums != 0, - ) - conf_df = pd.DataFrame(conf, index=class_names, columns=class_names) - conf_norm_df = pd.DataFrame(conf_norm, index=class_names, columns=class_names) - return conf, conf_norm, conf_df, conf_norm_df - - -def apply_dataset_preset(args) -> None: - if args.dataset_preset == "none": - return - - if args.dataset_preset == "ms": - default_single_data_path = "../../cell_annotation/cellxgene80k_balanced_10types.h5ad" - default_train_path = "../../cell_annotation/data/c_data.h5ad" - default_test_path = "../../cell_annotation/data/filtered_ms_adata.h5ad" - - if args.data_path == default_single_data_path and args.train_data_path is None and args.test_data_path is None: - args.train_data_path = default_train_path - args.test_data_path = default_test_path - - if args.label_column == "cell_type_anno": - args.label_column = "celltype" - if args.test_label_column is None: - args.test_label_column = "celltype" - if args.gene_name_column == "auto": - args.gene_name_column = "gene_name" - if args.min_genes == 10: - args.min_genes = 0 - if args.min_cells == 10: - args.min_cells = 0 - - logger.info( - "Applied dataset preset 'ms': train_data_path=%s test_data_path=%s label_column=%s " - "test_label_column=%s gene_name_column=%s min_genes=%d min_cells=%d", - args.train_data_path, - args.test_data_path, - args.label_column, - args.test_label_column, - args.gene_name_column, - args.min_genes, - args.min_cells, - ) - return - - raise ValueError(f"Unsupported dataset_preset: {args.dataset_preset}") - - -def parse_args(): - parser = argparse.ArgumentParser(description="End-to-end fine-tuning for Performer Module cell annotation") - - parser.add_argument("--seed", type=int, default=2021) - parser.add_argument("--device", type=str, default="cuda") - parser.add_argument("--batch_size", type=int, default=32, help="Per-device train batch size.") - parser.add_argument("--eval_batch_size", type=int, default=None, help="Per-device eval batch size.") - parser.add_argument("--global_batch_size", type=int, default=None, help="Effective global train batch size.") - parser.add_argument("--global_eval_batch_size", type=int, default=None, help="Effective global eval batch size.") - parser.add_argument("--grad_accum_steps", type=int, default=1) - parser.add_argument("--num_workers", type=int, default=4) - parser.add_argument("--prefetch_factor", type=int, default=2) - parser.add_argument("--epochs", type=int, default=50) - parser.add_argument("--patience", type=int, default=10) - parser.add_argument("--grad_clip", type=float, default=1.0) - - parser.add_argument("--lr_head", type=float, default=1e-4) - parser.add_argument("--lr_backbone", type=float, default=1e-5) - parser.add_argument("--weight_decay", type=float, default=0.01) - parser.add_argument("--label_smoothing", type=float, default=0.1) - parser.add_argument( - "--dropout", - type=float, - default=0.2, - help="Shared dropout applied to pooled cell embeddings before classification.", - ) - parser.add_argument("--pool_hidden_dim", type=int, default=0) - parser.add_argument("--cls_decoder_layers", type=int, default=3) - - parser.add_argument( - "--head_type", - type=str, - default="attn_pool", - choices=["module_mean", "max", "gene_mean", "attn_pool", "cls"], - ) - parser.add_argument( - "--finetune_mode", - type=str, - default="last_n", - choices=["frozen", "last_n", "full"], - ) - parser.add_argument("--unfreeze_last_n", type=int, default=1) - parser.add_argument("--train_pos_emb", action="store_true") - parser.add_argument("--train_module_embed", action="store_true") - parser.add_argument("--use_family_emb", action="store_true", - help="Add family-type embedding (Hallmarks/Reactome/Hotspot/WGCNA)") - parser.add_argument("--use_module_gate", action="store_true", - help="Add learnable sparse gate per module") - parser.add_argument("--use_module_selector", action="store_true", - help="Dynamic module selector with sparsity loss (soft train, hard inference)") - parser.add_argument("--selector_sparsity_weight", type=float, default=0.1, - help="Lambda for sparsity loss on module selector scores") - parser.add_argument("--selector_tau", type=float, default=0.5, - help="Score threshold for hard module selection during inference") - parser.add_argument("--dataset_preset", type=str, default="none", choices=["none", "ms"]) - - parser.add_argument( - "--data_path", - type=str, - default="../../cell_annotation/cellxgene80k_balanced_10types.h5ad", - ) - parser.add_argument("--train_data_path", type=str, default=None) - parser.add_argument("--test_data_path", type=str, default=None) - parser.add_argument("--label_column", type=str, default="cell_type_anno") - parser.add_argument("--test_label_column", type=str, default=None) - parser.add_argument( - "--gene_name_column", - type=str, - default="auto", - help="Gene-name source in adata.var: 'auto', 'index', or a column name such as 'gene_name'.", - ) - parser.add_argument("--val_ratio", type=float, default=0.1) - parser.add_argument("--test_ratio", type=float, default=0.2) - parser.add_argument("--min_genes", type=int, default=10) - parser.add_argument("--min_cells", type=int, default=10) - parser.add_argument("--max_cells", type=int, default=None) - - parser.add_argument("--config_pkl", type=str, default="../ckpts_performer_module_test/performer_module_test.pkl") - parser.add_argument( - "--performer_checkpoint", - type=str, - default="../ckpts_performer_module_test/performer_module_test_epoch7.pth", - ) - parser.add_argument("--module_data_dir", type=str, default="../module_data_all") - parser.add_argument("--gene_list_path", type=str, default="../../selected_genes_27k.txt") - parser.add_argument("--gene_mapping_path", type=str, default="../../gene2ens.csv") - parser.add_argument("--gene2vec_data_path", type=str, default="../../selected_gene2vec_27k.npy") - parser.add_argument("--gene2ens_file", type=str, default="../../gene2ens.csv") - parser.add_argument("--gene2ens_column", type=str, default="Ensembl gene ID") - parser.add_argument( - "--shuffle_module_genes", - action="store_true", - help="Apply an extra downstream reshuffle on top of module_sets restored from PKL. " - "Do not pass this when reproducing the pretrained backbone as-is.", - ) - parser.add_argument( - "--shuffle_seed", - type=int, - default=2021, - help="Random seed for the downstream extra reshuffle when --shuffle_module_genes is enabled.", - ) - - parser.add_argument("--output_dir", type=str, default="./ckpts_performer_module_finetune") - parser.add_argument("--run_name", type=str, default="") - parser.add_argument( - "--save_test_artifacts", - action="store_true", - help="Save test classification report, confusion matrices, and metrics summary.", - ) - - return parser.parse_args() - - -def main(): - args = parse_args() - apply_dataset_preset(args) - device, use_ddp, rank, world_size, local_rank = init_distributed_mode(args.device) - if not is_main_process(): - logger.setLevel(logging.WARNING) - seed_all(args.seed + rank) - - run_name = args.run_name or ( - f"{args.head_type}_{args.finetune_mode}" - f"_n{args.unfreeze_last_n}_lrh{args.lr_head:g}_lrb{args.lr_backbone:g}_seed{args.seed}" - ) - output_dir = resolve_path(args.output_dir) / run_name - output_dir.mkdir(parents=True, exist_ok=True) - - train_batch_size, eval_batch_size, effective_global_batch = resolve_batch_sizes(args, world_size) - logger.info("Run directory: %s", output_dir) - logger.info( - "Device: %s | DDP=%s rank=%d world_size=%d local_rank=%d", - device, - use_ddp, - rank, - world_size, - local_rank, - ) - logger.info( - "Batch config: per_device_train=%d per_device_eval=%d grad_accum=%d effective_global_train=%d", - train_batch_size, - eval_batch_size, - args.grad_accum_steps, - effective_global_batch, - ) - - preprocessor = DataPreprocessor( - gene2ens_file=str(resolve_path(args.gene2ens_file)), - gene2ens_column=args.gene2ens_column, - model_genes_file=str(resolve_path(args.gene_list_path)), - ) - preprocessor.load_gene_mapping() - preprocessor.load_model_genes() - paired_data_mode = bool(args.train_data_path or args.test_data_path) - if bool(args.train_data_path) != bool(args.test_data_path): - raise ValueError("train_data_path and test_data_path must be provided together") - - if paired_data_mode: - if args.test_ratio != 0.2: - logger.info("External test set provided; ignoring test_ratio=%s", args.test_ratio) - - train_adata, train_reindex_state = preprocessor.preprocess_adata( - str(resolve_path(args.train_data_path)), - min_genes=args.min_genes, - min_cells=args.min_cells, - max_cells=args.max_cells, - dataset_name="train", - gene_name_column=args.gene_name_column, - ) - test_adata, test_reindex_state = preprocessor.preprocess_adata( - str(resolve_path(args.test_data_path)), - min_genes=args.min_genes, - min_cells=args.min_cells, - max_cells=None, - dataset_name="test", - gene_name_column=args.gene_name_column, - ) - - train_label_column = args.label_column - test_label_column = args.test_label_column or args.label_column - label_encoder = LabelEncoder() - train_labels_raw = train_adata.obs[train_label_column].astype(str).to_numpy() - label_encoder.fit(train_labels_raw) - labels_train = label_encoder.transform(train_labels_raw) - - test_labels_raw = test_adata.obs[test_label_column].astype(str).to_numpy() - unseen_test_labels = sorted(set(test_labels_raw) - set(label_encoder.classes_)) - if unseen_test_labels: - raise ValueError( - "External test set contains labels unseen in training data: " - + ", ".join(unseen_test_labels[:10]) - ) - labels_test = label_encoder.transform(test_labels_raw) - num_classes = len(label_encoder.classes_) - - train_idx, val_idx = stratified_train_val_split( - labels=labels_train, - seed=args.seed, - val_ratio=args.val_ratio, - ) - logger.info( - "Using external test split | train=%s test=%s", - resolve_path(args.train_data_path), - resolve_path(args.test_data_path), - ) - logger.info( - "Split sizes: train=%d val=%d external_test=%d classes=%d", - len(train_idx), - len(val_idx), - test_adata.n_obs, - num_classes, - ) - - train_dataset = ExpressionDataset(train_adata.X, labels_train, train_idx) - val_dataset = ExpressionDataset(train_adata.X, labels_train, val_idx) - test_dataset = ExpressionDataset(test_adata.X, labels_test, np.arange(test_adata.n_obs)) - val_reindex_state = train_reindex_state - else: - adata, train_reindex_state = preprocessor.preprocess_adata( - str(resolve_path(args.data_path)), - min_genes=args.min_genes, - min_cells=args.min_cells, - max_cells=args.max_cells, - dataset_name="data", - gene_name_column=args.gene_name_column, - ) - - label_encoder = LabelEncoder() - labels = label_encoder.fit_transform(adata.obs[args.label_column].astype(str).to_numpy()) - num_classes = len(label_encoder.classes_) - train_idx, val_idx, test_idx = stratified_split( - labels=labels, - seed=args.seed, - val_ratio=args.val_ratio, - test_ratio=args.test_ratio, - ) - logger.info( - "Split sizes: train=%d val=%d test=%d classes=%d", - len(train_idx), - len(val_idx), - len(test_idx), - num_classes, - ) - - matrix = adata.X - train_dataset = ExpressionDataset(matrix, labels, train_idx) - val_dataset = ExpressionDataset(matrix, labels, val_idx) - test_dataset = ExpressionDataset(matrix, labels, test_idx) - val_reindex_state = train_reindex_state - test_reindex_state = train_reindex_state - - train_sampler = ( - DistributedSampler(train_dataset, num_replicas=world_size, rank=rank, shuffle=True, drop_last=False) - if use_ddp - else None - ) - val_sampler = ( - DistributedEvalSampler(len(val_dataset), rank=rank, world_size=world_size) - if use_ddp - else None - ) - test_sampler = ( - DistributedEvalSampler(len(test_dataset), rank=rank, world_size=world_size) - if use_ddp - else None - ) - - loader_kwargs = { - "num_workers": args.num_workers, - "pin_memory": device.type == "cuda", - } - if args.num_workers > 0: - loader_kwargs["persistent_workers"] = True - loader_kwargs["prefetch_factor"] = args.prefetch_factor - - train_loader = DataLoader( - train_dataset, - batch_size=train_batch_size, - shuffle=train_sampler is None, - sampler=train_sampler, - **loader_kwargs, - ) - val_loader = DataLoader( - val_dataset, - batch_size=eval_batch_size, - shuffle=False, - sampler=val_sampler, - **loader_kwargs, - ) - test_loader = DataLoader( - test_dataset, - batch_size=eval_batch_size, - shuffle=False, - sampler=test_sampler, - **loader_kwargs, - ) - - backbone, module_sets, backbone_config = load_backbone(args, device=device) - trainability = configure_backbone_trainability( - backbone, - finetune_mode=args.finetune_mode, - unfreeze_last_n=args.unfreeze_last_n, - train_pos_emb=args.train_pos_emb, - train_module_embed=args.train_module_embed, - ) - - model = PerformerModuleCellClassifier( - backbone=backbone, - module_sets=module_sets, - num_classes=num_classes, - head_type=args.head_type, - pool_hidden_dim=args.pool_hidden_dim, - dropout=args.dropout, - cls_decoder_layers=args.cls_decoder_layers, - use_module_selector=getattr(args, "use_module_selector", False), - ).to(device) - model_to_save = model - if getattr(args, "use_module_selector", False): - model_to_save.selector_tau = getattr(args, "selector_tau", 0.5) - - head_total = sum(p.numel() for name, p in model.named_parameters() if not name.startswith("backbone.")) - head_trainable = sum( - p.numel() for name, p in model.named_parameters() if not name.startswith("backbone.") and p.requires_grad - ) - logger.info( - "Backbone params: trainable=%d total=%d | Head params: trainable=%d total=%d", - trainability["trainable"], - trainability["total"], - head_trainable, - head_total, - ) - - optimizer = build_optimizer( - model=model, - lr_head=args.lr_head, - lr_backbone=args.lr_backbone, - weight_decay=args.weight_decay, - ) - if use_ddp: - model = DDP(model, device_ids=[local_rank] if device.type == "cuda" else None) - scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=max(args.epochs, 1)) - criterion = nn.CrossEntropyLoss(label_smoothing=args.label_smoothing) - - if is_main_process(): - save_json( - output_dir / "run_config.json", - { - "args": vars(args), - "run_name": run_name, - "num_classes": num_classes, - "label_classes": label_encoder.classes_.tolist(), - "backbone_config": backbone_config, - "trainable_backbone_params": trainability, - "head_type": args.head_type, - "data_mode": "paired_train_test" if paired_data_mode else "single_split", - "gene_name_column": args.gene_name_column, - "world_size": world_size, - "effective_global_batch": effective_global_batch, - "per_device_batch_size": train_batch_size, - "per_device_eval_batch_size": eval_batch_size, - }, - ) - if use_ddp: - dist.barrier() - - best_val_f1 = -1.0 - best_metrics = None - best_epoch = -1 - patience_counter = 0 - - for epoch in range(1, args.epochs + 1): - if train_sampler is not None: - train_sampler.set_epoch(epoch) - train_metrics = train_one_epoch( - model=model, - data_loader=train_loader, - preprocessor=preprocessor, - reindex_state=train_reindex_state, - criterion=criterion, - optimizer=optimizer, - device=device, - grad_clip=args.grad_clip, - grad_accum_steps=args.grad_accum_steps, - use_ddp=use_ddp, - selector_sparsity_weight=getattr(args, "selector_sparsity_weight", 0.0) if getattr(args, "use_module_selector", False) else 0.0, - ) - val_metrics = evaluate( - model=model, - data_loader=val_loader, - preprocessor=preprocessor, - reindex_state=val_reindex_state, - criterion=criterion, - device=device, - ) - scheduler.step() - - logger.info( - "Epoch %d | train loss %.4f acc %.4f f1 %.4f | val loss %.4f acc %.4f f1 %.4f", - epoch, - train_metrics["loss"], - train_metrics["acc"], - train_metrics["macro_f1"], - val_metrics["loss"], - val_metrics["acc"], - val_metrics["macro_f1"], - ) - - if val_metrics["macro_f1"] > best_val_f1: - best_val_f1 = val_metrics["macro_f1"] - best_epoch = epoch - patience_counter = 0 - - best_metrics = { - "epoch": epoch, - "train": train_metrics, - "val": {k: v for k, v in val_metrics.items() if k not in {"preds", "labels"}}, - } - if is_main_process(): - torch.save( - { - "epoch": epoch, - "model_state_dict": unwrap_model(model_to_save).state_dict(), - "optimizer_state_dict": optimizer.state_dict(), - "scheduler_state_dict": scheduler.state_dict(), - "best_val_f1": best_val_f1, - "args": vars(args), - "label_classes": label_encoder.classes_.tolist(), - "backbone_config": backbone_config, - "world_size": world_size, - "effective_global_batch": effective_global_batch, - }, - output_dir / "best_model.pt", - ) - else: - patience_counter += 1 - if patience_counter >= args.patience: - logger.info("Early stopping at epoch %d", epoch) - break - if use_ddp: - dist.barrier() - - if best_epoch < 0: - raise RuntimeError("Training finished without a valid checkpoint") - - if use_ddp: - dist.barrier() - ckpt = torch.load(output_dir / "best_model.pt", map_location=device) - unwrap_model(model).load_state_dict(ckpt["model_state_dict"], strict=True) - - test_metrics = evaluate( - model=model, - data_loader=test_loader, - preprocessor=preprocessor, - reindex_state=test_reindex_state, - criterion=criterion, - device=device, - ) - logger.info( - "Best epoch %d | test loss %.4f acc %.4f f1 %.4f", - best_epoch, - test_metrics["loss"], - test_metrics["acc"], - test_metrics["macro_f1"], - ) - - report = classification_report( - test_metrics["labels"], - test_metrics["preds"], - target_names=label_encoder.classes_.tolist(), - digits=4, - ) - conf, conf_norm, conf_df, conf_norm_df = build_confusion_tables( - test_metrics["labels"], - test_metrics["preds"], - label_encoder.classes_.tolist(), - ) - - summary = { - "best_epoch": best_epoch, - "best_val_f1": best_val_f1, - "train": best_metrics["train"] if best_metrics is not None else None, - "val": best_metrics["val"] if best_metrics is not None else None, - "test": {k: v for k, v in test_metrics.items() if k not in {"preds", "labels"}}, - } - if is_main_process(): - logger.info("Test classification report:\n%s", report) - logger.info( - "Test confusion matrix (rows=true, cols=pred):\n%s", - conf_df.to_string(), - ) - logger.info( - "Test confusion matrix normalized by true class:\n%s", - conf_norm_df.round(4).to_string(), - ) - if args.save_test_artifacts: - with open(output_dir / "test_classification_report.txt", "w", encoding="utf-8") as handle: - handle.write(report) - with open(output_dir / "test_confusion_matrix.txt", "w", encoding="utf-8") as handle: - handle.write(conf_df.to_string()) - handle.write("\n") - with open(output_dir / "test_confusion_matrix_normalized.txt", "w", encoding="utf-8") as handle: - handle.write(conf_norm_df.round(4).to_string()) - handle.write("\n") - np.save(output_dir / "test_confusion_matrix.npy", conf) - np.save(output_dir / "test_confusion_matrix_normalized.npy", conf_norm) - conf_df.to_csv(output_dir / "test_confusion_matrix.csv", encoding="utf-8") - conf_norm_df.to_csv(output_dir / "test_confusion_matrix_normalized.csv", encoding="utf-8") - save_json(output_dir / "metrics_summary.json", summary) - logger.info("Saved test artifacts to %s", output_dir) - else: - logger.info("Test artifacts were not saved. Use --save_test_artifacts to enable file output.") - if use_ddp: - dist.barrier() - dist.destroy_process_group() - - -if __name__ == "__main__": - main() +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +""" +End-to-end cell annotation fine-tuning for Performer Module. + +This script is intentionally independent from the legacy two-stage pipeline: +1. No feature extraction to .npy files. +2. No imports from other scripts in performer_module/cell_annotation. +3. Reuses the pretrained Performer Module backbone directly. + +Supported cell embedding heads: +- module_mean: mean over restored module tokens +- max: max pooling over restored module tokens +- gene_mean: module -> gene -> mean over genes +- attn_pool: learnable attention pooling over restored module tokens +- cls: use pretrained CLS token if the checkpoint contains one + +Supported backbone finetune modes: +- frozen: freeze the full backbone +- last_n: unfreeze only the last N encoder blocks +- full: full-parameter finetuning +""" + +import argparse +import contextlib +import json +import logging +import os +import pickle +import random +import sys +from pathlib import Path +from typing import TYPE_CHECKING, Dict, List, Optional, Sequence, Tuple + +import numpy as np +import pandas as pd +import torch +import anndata as ad +import torch.distributed as dist +from scipy import sparse +from sklearn.metrics import accuracy_score, classification_report, confusion_matrix, f1_score +from sklearn.model_selection import StratifiedShuffleSplit +from sklearn.preprocessing import LabelEncoder +from torch.nn.parallel import DistributedDataParallel as DDP +from torch import nn +from torch.utils.data import DataLoader, Dataset, Sampler +from torch.utils.data.distributed import DistributedSampler + +SCRIPT_DIR = Path(__file__).resolve().parent +PERFORMER_MODULE_ROOT = SCRIPT_DIR.parent +SCLONG_ROOT = PERFORMER_MODULE_ROOT.parent + +REPO_ROOT = Path(__file__).resolve().parents[2] +ASSET_DIR = REPO_ROOT / "model_dependencies" +DOWNSTREAM_DATA_DIR = REPO_ROOT / "data" / "downstream" / "final_data" +WEIGHTS_DIR = REPO_ROOT / "weight" / "0810model" + +if str(PERFORMER_MODULE_ROOT) not in sys.path: + sys.path.insert(0, str(PERFORMER_MODULE_ROOT)) +if str(SCLONG_ROOT) not in sys.path: + sys.path.insert(0, str(SCLONG_ROOT)) + +from models_performer_module import ( + migrate_legacy_module_config, + migrate_legacy_module_state_dict, +) + +if TYPE_CHECKING: + from models_performer_module import PerformerEncoderOnly_Module + + +logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s") +logger = logging.getLogger(__name__) + + +def is_dist_initialized() -> bool: + return dist.is_available() and dist.is_initialized() + + +def get_rank() -> int: + return dist.get_rank() if is_dist_initialized() else 0 + + +def get_world_size() -> int: + return dist.get_world_size() if is_dist_initialized() else 1 + + +def is_main_process() -> bool: + return get_rank() == 0 + + +def unwrap_model(model: nn.Module) -> nn.Module: + return model.module if hasattr(model, "module") else model + + +def init_distributed_mode(device_arg: str) -> Tuple[torch.device, bool, int, int, int]: + use_ddp = int(os.environ.get("WORLD_SIZE", "1")) > 1 + if not use_ddp: + if device_arg == "cuda" and not torch.cuda.is_available(): + logger.warning("CUDA requested but not available. Falling back to CPU.") + return torch.device("cpu"), False, 0, 1, 0 + return torch.device(device_arg), False, 0, 1, 0 + + local_rank = int(os.environ["LOCAL_RANK"]) + rank = int(os.environ["RANK"]) + world_size = int(os.environ["WORLD_SIZE"]) + + if device_arg != "cuda": + backend = "gloo" + device = torch.device(device_arg) + else: + if not torch.cuda.is_available(): + raise RuntimeError("DDP with device='cuda' requires CUDA to be available") + backend = "nccl" + torch.cuda.set_device(local_rank) + device = torch.device("cuda", local_rank) + + dist.init_process_group(backend=backend) + return device, True, rank, world_size, local_rank + + +def all_gather_numpy(array: np.ndarray) -> np.ndarray: + if not is_dist_initialized(): + return array + gathered = [None for _ in range(get_world_size())] + dist.all_gather_object(gathered, array) + non_empty = [arr for arr in gathered if arr is not None and len(arr) > 0] + if not non_empty: + return np.empty((0,), dtype=array.dtype) + return np.concatenate(non_empty, axis=0) + + +def reduce_sum(value: float, device: torch.device) -> float: + if not is_dist_initialized(): + return float(value) + tensor = torch.tensor(float(value), device=device, dtype=torch.float64) + dist.all_reduce(tensor, op=dist.ReduceOp.SUM) + return float(tensor.item()) + + +def resolve_path(path_str: str) -> Path: + """Resolve user-supplied paths relative to the caller's working directory.""" + return Path(path_str).expanduser().resolve() + + +def seed_all(seed: int) -> None: + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + torch.backends.cudnn.deterministic = True + torch.backends.cudnn.benchmark = False + + +class DataPreprocessor: + """Map input genes to the pretrained model gene space.""" + + def __init__( + self, + gene2ens_file: str, + gene2ens_column: str = "Ensembl gene ID", + model_genes_file: Optional[str] = None, + ): + self.gene2ens_file = gene2ens_file + self.gene2ens_column = gene2ens_column + self.model_genes_file = model_genes_file + self.gene2ens_dict: Optional[Dict[str, str]] = None + self.model_genes: Optional[List[str]] = None + self._index_cache: Dict[str, torch.Tensor] = {} + + def load_gene_mapping(self) -> None: + gene2ens = pd.read_csv(self.gene2ens_file, index_col=0) + if self.gene2ens_column not in gene2ens.columns: + raise ValueError(f"Column '{self.gene2ens_column}' not found in {self.gene2ens_file}") + self.gene2ens_dict = gene2ens[self.gene2ens_column].to_dict() + logger.info("Loaded gene2ens mapping: %d entries", len(self.gene2ens_dict)) + + def load_model_genes(self) -> None: + if self.model_genes_file is None: + raise ValueError("model_genes_file must be provided") + with open(self.model_genes_file, "r", encoding="utf-8") as handle: + self.model_genes = [line.rstrip("\n") for line in handle if line.strip()] + logger.info("Loaded %d model genes from %s", len(self.model_genes), self.model_genes_file) + + def _extract_input_genes( + self, + adata: ad.AnnData, + gene_name_column: Optional[str] = None, + ) -> Tuple[List[str], str]: + if gene_name_column is not None and gene_name_column.lower() == "index": + return adata.var.index.astype(str).tolist(), "var.index" + + if gene_name_column is not None and gene_name_column.lower() != "auto": + if gene_name_column not in adata.var.columns: + raise ValueError(f"Gene column '{gene_name_column}' not found in adata.var") + return adata.var[gene_name_column].astype(str).tolist(), f"var['{gene_name_column}']" + + if "gene_name" in adata.var.columns: + return adata.var["gene_name"].astype(str).tolist(), "var['gene_name']" + return adata.var.index.astype(str).tolist(), "var.index" + + def _build_reindex_state( + self, + adata: ad.AnnData, + dataset_name: str, + gene_name_column: Optional[str] = None, + ) -> dict: + input_genes, gene_source = self._extract_input_genes(adata, gene_name_column=gene_name_column) + if self.gene2ens_dict is not None: + input_genes = [self.gene2ens_dict.get(symbol, symbol) for symbol in input_genes] + logger.info("[%s] Mapped input genes to Ensembl IDs", dataset_name) + + input_mapping = {gene: i for i, gene in enumerate(input_genes)} + input_gene_num = len(input_genes) + + if self.model_genes is None: + raise RuntimeError("Model genes must be loaded before preprocess_adata") + + model_index_positions = [input_mapping.get(gene, input_gene_num) for gene in self.model_genes] + overlap = sum(pos < input_gene_num for pos in model_index_positions) + logger.info( + "[%s] Gene source: %s | overlap with model space: %d / %d (model gene space: %d)", + dataset_name, + gene_source, + overlap, + input_gene_num, + len(self.model_genes), + ) + return { + "cache_key": f"{dataset_name}:{gene_source}:{input_gene_num}", + "dataset_name": dataset_name, + "gene_source": gene_source, + "model_index_positions": model_index_positions, + "input_gene_num": input_gene_num, + "overlap": overlap, + } + + def preprocess_adata( + self, + adata_path: str, + min_genes: int = 10, + min_cells: int = 10, + max_cells: Optional[int] = None, + dataset_name: str = "dataset", + gene_name_column: Optional[str] = None, + ) -> Tuple[ad.AnnData, dict]: + logger.info("Loading data from %s", adata_path) + adata = ad.read_h5ad(adata_path) + matrix = adata.X + + if sparse.issparse(matrix): + cell_nonzero = np.asarray((matrix > 0).sum(axis=1)).ravel() + gene_nonzero = np.asarray((matrix > 0).sum(axis=0)).ravel() + else: + cell_nonzero = np.count_nonzero(np.asarray(matrix), axis=1) + gene_nonzero = np.count_nonzero(np.asarray(matrix), axis=0) + + cell_mask = cell_nonzero >= min_genes + gene_mask = gene_nonzero >= min_cells + adata = adata[cell_mask, gene_mask].copy() + + if max_cells is not None and adata.n_obs > max_cells: + logger.info("Subsampling %d cells from %d", max_cells, adata.n_obs) + rng = np.random.default_rng(0) + chosen = np.sort(rng.choice(adata.n_obs, size=max_cells, replace=False)) + adata = adata[chosen].copy() + + reindex_state = self._build_reindex_state( + adata, + dataset_name=dataset_name, + gene_name_column=gene_name_column, + ) + return adata, reindex_state + + def _get_index_tensor(self, reindex_state: dict, device: torch.device) -> torch.Tensor: + key = f"{str(device)}::{reindex_state['cache_key']}" + if key not in self._index_cache: + self._index_cache[key] = torch.as_tensor( + reindex_state["model_index_positions"], + dtype=torch.long, + device=device, + ) + return self._index_cache[key] + + def reindex_tensor_to_model_space( + self, + tensor: torch.Tensor, + reindex_state: dict, + filler: float = 0.0, + ) -> torch.Tensor: + if tensor.dim() != 2: + raise ValueError(f"Expected 2D tensor [B, G], got {tuple(tensor.shape)}") + index_tensor = self._get_index_tensor(reindex_state, tensor.device) + filler_col = torch.full( + (tensor.shape[0], 1), + float(filler), + dtype=tensor.dtype, + device=tensor.device, + ) + expanded_tensor = torch.cat([tensor, filler_col], dim=1) + return torch.index_select(expanded_tensor, 1, index_tensor) + + +class ExpressionDataset(Dataset): + def __init__(self, matrix, labels: np.ndarray, indices: Sequence[int]): + self.matrix = matrix.tocsr() if sparse.issparse(matrix) and not sparse.isspmatrix_csr(matrix) else matrix + self.labels = labels.astype(np.int64, copy=False) + self.indices = np.asarray(indices, dtype=np.int64) + + def __len__(self) -> int: + return len(self.indices) + + def __getitem__(self, idx: int): + real_idx = int(self.indices[idx]) + row = self.matrix[real_idx] + if sparse.issparse(row): + row = row.toarray().ravel() + else: + row = np.asarray(row).reshape(-1) + x = torch.from_numpy(row.astype(np.float32, copy=False)) + y = torch.tensor(self.labels[real_idx], dtype=torch.long) + return x, y + + +class DistributedEvalSampler(Sampler[int]): + """Shard evaluation data across ranks without padding or duplication.""" + + def __init__(self, dataset_len: int, rank: int, world_size: int): + self.indices = list(range(rank, dataset_len, world_size)) + + def __iter__(self): + return iter(self.indices) + + def __len__(self) -> int: + return len(self.indices) + + +class GatedAttentionPool(nn.Module): + def __init__(self, embed_dim: int, hidden_dim: int): + super().__init__() + self.score = nn.Sequential( + nn.LayerNorm(embed_dim), + nn.Linear(embed_dim, hidden_dim), + nn.Tanh(), + nn.Linear(hidden_dim, 1), + ) + + def forward(self, tokens: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + scores = self.score(tokens).squeeze(-1) + weights = torch.softmax(scores, dim=1) + pooled = torch.sum(tokens * weights.unsqueeze(-1), dim=1) + return pooled, weights + + +class ScGPTClsDecoder(nn.Module): + """Classifier head aligned with scGPT's ClsDecoder.""" + + def __init__(self, d_model: int, n_cls: int, nlayers: int = 3): + super().__init__() + if nlayers < 1: + raise ValueError(f"nlayers must be >= 1, got {nlayers}") + self.layers = nn.ModuleList() + for _ in range(nlayers - 1): + self.layers.append(nn.Linear(d_model, d_model)) + self.layers.append(nn.ReLU()) + self.layers.append(nn.LayerNorm(d_model)) + self.out_layer = nn.Linear(d_model, n_cls) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + for layer in self.layers: + x = layer(x) + return self.out_layer(x) + + +class ModuleSelector(nn.Module): + """Per-module scorer: soft weighting during training, hard threshold during inference.""" + + def __init__(self, embed_dim: int): + super().__init__() + self.scorer = nn.Sequential( + nn.LayerNorm(embed_dim), + nn.Linear(embed_dim, 1), + nn.Sigmoid(), + ) + + def forward( + self, x: torch.Tensor, tau: float = 0.5, hard: bool = False, + ) -> Tuple[torch.Tensor, torch.Tensor]: + scores = self.scorer(x) # (B, L, 1) + if hard: + x = x * (scores > tau).float() + else: + x = x * scores + return x, scores + + +class PerformerModuleCellClassifier(nn.Module): + def __init__( + self, + backbone: "PerformerEncoderOnly_Module", + module_sets: List[List[int]], + num_classes: int, + head_type: str, + pool_hidden_dim: int, + dropout: float, + cls_decoder_layers: int, + use_module_selector: bool = False, + ): + super().__init__() + self.backbone = backbone + self.module_sets = module_sets + self.num_classes = num_classes + self.head_type = head_type + self.base_module_count = len(module_sets) + self.embed_dim = backbone.base_dim + self.num_genes = backbone.num_genes + self.use_cls = bool(backbone.use_cls) + + if self.head_type == "cls" and not self.use_cls: + raise ValueError("head_type='cls' requires a checkpoint with cls_token") + + if self.head_type == "attn_pool": + hidden_dim = pool_hidden_dim if pool_hidden_dim > 0 else self.embed_dim + self.pooler = GatedAttentionPool(self.embed_dim, hidden_dim) + else: + self.pooler = None + self.output_dropout = nn.Dropout(dropout) if dropout > 0 else nn.Identity() + + self.use_module_selector = use_module_selector + self.selector_tau = 0.5 + if use_module_selector: + self.module_selector = ModuleSelector(self.embed_dim) + else: + self.module_selector = None + self._selector_scores = None + + if self.head_type == "gene_mean": + module_idx, gene_idx, gene_counts = self._build_module_gene_index() + self.register_buffer("agg_module_idx", module_idx) + self.register_buffer("agg_gene_idx", gene_idx) + self.register_buffer("agg_gene_counts", gene_counts) + else: + self.agg_module_idx = None + self.agg_gene_idx = None + self.agg_gene_counts = None + + self.classifier = ScGPTClsDecoder( + d_model=self.embed_dim, + n_cls=num_classes, + nlayers=cls_decoder_layers, + ) + + def load_state_dict(self, state_dict, strict: bool = True, assign: bool = False): + """Accept current module names and older serialized backbone names.""" + return super().load_state_dict( + migrate_legacy_module_state_dict(state_dict), + strict=strict, + assign=assign, + ) + + def _build_module_gene_index(self) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + module_indices = [] + gene_indices = [] + gene_counts = torch.zeros(self.num_genes, dtype=torch.long) + for module_idx, module_genes in enumerate(self.module_sets): + for gene_idx in module_genes: + if 0 <= gene_idx < self.num_genes: + module_indices.append(module_idx) + gene_indices.append(gene_idx) + gene_counts[gene_idx] += 1 + return ( + torch.tensor(module_indices, dtype=torch.long), + torch.tensor(gene_indices, dtype=torch.long), + gene_counts, + ) + + def _restore_module_order(self, latent: torch.Tensor, ids_restore: torch.Tensor) -> torch.Tensor: + if self.use_cls: + module_tokens = latent[:, 1:, :] + else: + module_tokens = latent + gather_index = ids_restore.unsqueeze(-1).expand(-1, -1, module_tokens.shape[-1]) + return torch.gather(module_tokens, dim=1, index=gather_index) + + def _module_to_gene(self, module_tokens: torch.Tensor) -> torch.Tensor: + batch_size, _, dim = module_tokens.shape + gene_feats = torch.zeros( + batch_size, + self.num_genes, + dim, + device=module_tokens.device, + dtype=module_tokens.dtype, + ) + global_mean = module_tokens.mean(dim=1) + + base_counts = self.agg_gene_counts.to(device=module_tokens.device, dtype=module_tokens.dtype) + gene_counts = base_counts.unsqueeze(0).expand(batch_size, -1).clone() + + static_module_tokens = module_tokens[:, :self.base_module_count, :] + if self.agg_module_idx.numel() > 0: + pair_module_feats = static_module_tokens[:, self.agg_module_idx, :] + gene_idx_expand = self.agg_gene_idx.view(1, -1, 1).expand(batch_size, -1, dim) + gene_feats.scatter_add_(1, gene_idx_expand, pair_module_feats) + + zero_mask = gene_counts == 0 + gene_feats = gene_feats / gene_counts.clamp(min=1.0).unsqueeze(-1) + gene_feats = torch.where(zero_mask.unsqueeze(-1), global_mean.unsqueeze(1), gene_feats) + return gene_feats + + def get_selector_sparsity_loss(self) -> torch.Tensor: + if self._selector_scores is not None: + return self._selector_scores.mean() + return torch.tensor(0.0) + + def forward(self, x_model: torch.Tensor) -> torch.Tensor: + latent, _, ids_restore, _ = self.backbone.forward_encoder(x_model, mask_ratio=0.0) + + if self.head_type == "cls": + cell_emb = latent[:, 0, :] + else: + module_tokens = self._restore_module_order(latent, ids_restore) + if self.module_selector is not None: + module_tokens, self._selector_scores = self.module_selector( + module_tokens, tau=self.selector_tau, hard=not self.training, + ) + if self.head_type == "module_mean": + cell_emb = module_tokens.mean(dim=1) + elif self.head_type == "max": + cell_emb = module_tokens.amax(dim=1) + elif self.head_type == "attn_pool": + cell_emb, _ = self.pooler(module_tokens) + elif self.head_type == "gene_mean": + gene_tokens = self._module_to_gene(module_tokens) + cell_emb = gene_tokens.mean(dim=1) + else: + raise ValueError(f"Unsupported head_type: {self.head_type}") + + cell_emb = self.output_dropout(cell_emb) + return self.classifier(cell_emb) + + +def get_encoder_blocks(backbone: "PerformerEncoderOnly_Module") -> List[nn.Module]: + net = backbone.encoder.net + if hasattr(net, "layers"): + return list(net.layers) + if hasattr(net, "blocks"): + return list(net.blocks) + raise AttributeError("Cannot find encoder blocks on Performer backbone") + + +def set_requires_grad(module: nn.Module, flag: bool) -> None: + for param in module.parameters(): + param.requires_grad = flag + + +def configure_backbone_trainability( + backbone: "PerformerEncoderOnly_Module", + finetune_mode: str, + unfreeze_last_n: int, + train_pos_emb: bool, + train_module_embed: bool, +) -> Dict[str, int]: + set_requires_grad(backbone, False) + + if finetune_mode == "full": + # "full" here means full finetuning of the encoder pathway used by + # downstream classification, not the MAE-style reconstruction decoder. + set_requires_grad(backbone.module_embed, True) + set_requires_grad(backbone.encoder, True) + set_requires_grad(backbone.encoder_norm, True) + backbone.pos_emb.requires_grad = True + if getattr(backbone, "cls_token", None) is not None: + backbone.cls_token.requires_grad = True + elif finetune_mode == "last_n": + blocks = get_encoder_blocks(backbone) + if unfreeze_last_n <= 0: + raise ValueError("unfreeze_last_n must be > 0 when finetune_mode='last_n'") + for block in blocks[-min(unfreeze_last_n, len(blocks)):]: + set_requires_grad(block, True) + set_requires_grad(backbone.encoder_norm, True) + elif finetune_mode != "frozen": + raise ValueError(f"Unsupported finetune_mode: {finetune_mode}") + + if train_pos_emb: + backbone.pos_emb.requires_grad = True + if getattr(backbone, "cls_token", None) is not None: + backbone.cls_token.requires_grad = True + + if train_module_embed: + set_requires_grad(backbone.module_embed, True) + + if getattr(backbone, 'family_emb', None) is not None: + backbone.family_emb.requires_grad = True + if getattr(backbone, 'module_gate', None) is not None: + set_requires_grad(backbone.module_gate, True) + + total_params = sum(p.numel() for p in backbone.parameters()) + trainable_params = sum(p.numel() for p in backbone.parameters() if p.requires_grad) + return { + "total": total_params, + "trainable": trainable_params, + } + + +def build_optimizer( + model: PerformerModuleCellClassifier, + lr_head: float, + lr_backbone: float, + weight_decay: float, +) -> torch.optim.Optimizer: + backbone_params = [] + head_params = [] + new_module_params = [] + for name, param in model.named_parameters(): + if not param.requires_grad: + continue + if 'family_emb' in name or 'module_gate' in name or 'module_selector' in name: + new_module_params.append(param) + elif name.startswith("backbone."): + backbone_params.append(param) + else: + head_params.append(param) + + param_groups = [] + if backbone_params: + param_groups.append({"params": backbone_params, "lr": lr_backbone}) + if head_params: + param_groups.append({"params": head_params, "lr": lr_head}) + if new_module_params: + param_groups.append({"params": new_module_params, "lr": lr_head}) + + if not param_groups: + raise RuntimeError("No trainable parameters found") + + return torch.optim.AdamW(param_groups, weight_decay=weight_decay, betas=(0.9, 0.999)) + + +def resolve_batch_sizes(args, world_size: int) -> Tuple[int, int, int]: + if args.global_batch_size is not None: + denom = world_size * args.grad_accum_steps + if args.global_batch_size % denom != 0: + raise ValueError( + f"global_batch_size={args.global_batch_size} must be divisible by " + f"world_size({world_size}) * grad_accum_steps({args.grad_accum_steps})" + ) + train_batch_size = args.global_batch_size // denom + else: + train_batch_size = args.batch_size + + if train_batch_size <= 0: + raise ValueError(f"Resolved per-device train batch size must be > 0, got {train_batch_size}") + + if args.global_eval_batch_size is not None: + if args.global_eval_batch_size % world_size != 0: + raise ValueError( + f"global_eval_batch_size={args.global_eval_batch_size} must be divisible by world_size({world_size})" + ) + eval_batch_size = args.global_eval_batch_size // world_size + elif args.eval_batch_size is not None: + eval_batch_size = args.eval_batch_size + else: + eval_batch_size = train_batch_size + + if eval_batch_size <= 0: + raise ValueError(f"Resolved per-device eval batch size must be > 0, got {eval_batch_size}") + + effective_global_batch = train_batch_size * world_size * args.grad_accum_steps + return train_batch_size, eval_batch_size, effective_global_batch + + +def infer_model_config(args, checkpoint: dict, pkl_config: Optional[dict], module_data: List[List[int]]) -> dict: + from models_performer_module import sanitize_performer_module_config + + if pkl_config is not None and isinstance(pkl_config, dict): + config = sanitize_performer_module_config( + pkl_config, + source=str(args.config_pkl), + log_fn=logger.info, + ) + logger.info("Using config from PKL: %s", args.config_pkl) + if "module_sets" in config: + module_data = config["module_sets"] + elif isinstance(checkpoint, dict) and "config" in checkpoint: + config = sanitize_performer_module_config( + checkpoint["config"], + source=str(resolve_path(args.performer_checkpoint)), + log_fn=logger.info, + ) + logger.info("Using config from checkpoint") + else: + raise ValueError( + "Backbone config was not found in config_pkl or checkpoint['config']. " + "This script now expects pretrained structural args to be restored automatically." + ) + + return { + "module_sets": module_data, + "num_genes": config["num_genes"], + "base_dim": config["base_dim"], + "enc_depth": config["enc_depth"], + "enc_heads": config["enc_heads"], + "enc_dim_head": config["enc_dim_head"], + "gene_embed_dim": config.get("gene_embed_dim", 200), + "fusion_method": config.get("fusion_method", "multiply"), + "use_gene2vec": config.get("use_gene2vec", True), + "loss_mode": config.get("loss_mode", "gene_mean"), + "recon_target_level": config.get("recon_target_level", "gene"), + "pretrain_arch": config.get("pretrain_arch", "mae"), + "mask_style": config.get("mask_style", "drop"), + "mask_token_mode": config.get("mask_token_mode", "learned"), + "pos_embed_mode": config.get("pos_embed_mode", "module_id"), + "decoder_type": config.get("decoder_type", "simple"), + "decoder_gene_agg": config.get("decoder_gene_agg", False), + "dec_embed_dim": config.get("dec_embed_dim", config["base_dim"]), + "dec_depth": config.get("dec_depth", 4), + "dec_heads": config.get("dec_heads", 8), + "dec_dim_head": config.get("dec_dim_head", 64), + "use_cls_token": bool(config.get("use_cls_token", False)), + "cls_loss_weight": float(config.get("cls_loss_weight", 0.0)), + "cls_head_hidden_dim": int(config.get("cls_head_hidden_dim", 0)), + } + + +def compute_module_family_ids(module_data_dir: str) -> List[int]: + """Read merged_modules.json, return family ID (0-3) for each module.""" + merged_path = Path(module_data_dir) / "merged_modules.json" + with open(merged_path) as f: + modules = json.load(f) + PREFIXES = [ + ("module1_hallmarks", 0), + ("module2_reactome", 1), + ("module3_hotspot", 2), + ("module3_processed", 3), + ] + family_ids = [] + for name in modules.keys(): + fid = 0 + for prefix, idx in PREFIXES: + if name.startswith(prefix): + fid = idx + break + family_ids.append(fid) + return family_ids + + +def load_backbone(args, device: torch.device) -> Tuple["PerformerEncoderOnly_Module", List[List[int]], dict]: + from models_performer_module import PerformerEncoderOnly_Module + from module_dataset import load_module_data + + config_pkl = None + config_path = resolve_path(args.config_pkl) + if config_path.exists(): + with open(config_path, "rb") as handle: + config_pkl = migrate_legacy_module_config(pickle.load(handle)) + else: + logger.warning("Config PKL not found at %s; fallback to checkpoint config", config_path) + + ckpt_path = resolve_path(args.performer_checkpoint) + checkpoint = torch.load(ckpt_path, map_location=device) + + module_data = None + if isinstance(config_pkl, dict) and "module_sets" in config_pkl: + module_data = config_pkl["module_sets"] + logger.info("Using module_sets from PKL (%d modules)", len(module_data)) + elif isinstance(checkpoint, dict) and isinstance(checkpoint.get("config"), dict) and "module_sets" in checkpoint["config"]: + module_data = checkpoint["config"]["module_sets"] + logger.info("Using module_sets from checkpoint config (%d modules)", len(module_data)) + else: + if not args.module_data_dir: + raise ValueError("module_data_dir is required when module_sets are not stored in the PKL/checkpoint config") + module_data = load_module_data( + module_data_dir=str(resolve_path(args.module_data_dir)), + gene_list_path=str(resolve_path(args.gene_list_path)), + gene_mapping_path=str(resolve_path(args.gene_mapping_path)), + verbose=True, + ) + + if args.shuffle_module_genes: + logger.info("Shuffling genes across modules with seed=%d", args.shuffle_seed) + rng = random.Random(args.shuffle_seed) + module_sizes = [len(module) for module in module_data] + all_genes = [gene for module in module_data for gene in module] + rng.shuffle(all_genes) + shuffled = [] + start = 0 + for size in module_sizes: + shuffled.append(all_genes[start:start + size]) + start += size + module_data = shuffled + + model_config = infer_model_config(args, checkpoint, config_pkl, module_data) + module_data = model_config["module_sets"] + + if "model_state_dict" in checkpoint: + state_dict = checkpoint["model_state_dict"] + elif "model" in checkpoint: + state_dict = checkpoint["model"] + else: + state_dict = checkpoint + + has_cls_in_checkpoint = "cls_token" in state_dict + use_cls_token = has_cls_in_checkpoint or model_config["use_cls_token"] + + logger.info( + "Backbone config: base_dim=%d depth=%d heads=%d modules=%d cls=%s", + model_config["base_dim"], + model_config["enc_depth"], + model_config["enc_heads"], + len(module_data), + use_cls_token, + ) + + module_family_ids = None + if getattr(args, "use_family_emb", False): + pdd = getattr(args, "module_data_dir", None) + if pdd: + module_family_ids = compute_module_family_ids(str(resolve_path(pdd))) + logger.info("Family embedding enabled: %d module family IDs computed", len(module_family_ids)) + + backbone = PerformerEncoderOnly_Module( + module_sets=module_data, + num_genes=model_config["num_genes"], + max_seq_len=len(module_data), + base_dim=model_config["base_dim"], + enc_depth=model_config["enc_depth"], + enc_heads=model_config["enc_heads"], + enc_dim_head=model_config["enc_dim_head"], + mask_ratio=0.75, + gene2vec_file=str(resolve_path(args.gene2vec_data_path)), + gene_embed_dim=model_config["gene_embed_dim"], + fusion_method=model_config["fusion_method"], + use_cls_token=use_cls_token, + cls_loss_weight=model_config["cls_loss_weight"] if use_cls_token else 0.0, + cls_head_hidden_dim=model_config["cls_head_hidden_dim"], + use_gene2vec=model_config["use_gene2vec"], + loss_mode=model_config["loss_mode"], + recon_target_level=model_config["recon_target_level"], + pretrain_arch=model_config["pretrain_arch"], + mask_style=model_config["mask_style"], + mask_token_mode=model_config["mask_token_mode"], + pos_embed_mode=model_config["pos_embed_mode"], + decoder_type=model_config["decoder_type"], + decoder_gene_agg=model_config["decoder_gene_agg"], + dec_embed_dim=model_config["dec_embed_dim"], + dec_depth=model_config["dec_depth"], + dec_heads=model_config["dec_heads"], + dec_dim_head=model_config["dec_dim_head"], + use_family_emb=getattr(args, "use_family_emb", False), + module_family_ids=module_family_ids, + use_module_gate=getattr(args, "use_module_gate", False), + ) + + filtered_state_dict = { + key: value + for key, value in state_dict.items() + if "go_conv" not in key and "G_go" not in key.lower() + } + missing_keys, unexpected_keys = backbone.load_state_dict(filtered_state_dict, strict=False) + expected_new = {"family_emb", "module_family_ids", "module_gate."} + real_missing = [k for k in missing_keys + if not any(k == e or k.startswith(e) for e in expected_new)] + if real_missing: + logger.warning("Unexpected missing keys: %s", real_missing[:10]) + if unexpected_keys: + logger.warning("Unexpected keys: %s", unexpected_keys[:10]) + + backbone.to(device) + return backbone, module_data, model_config + + +def stratified_split( + labels: np.ndarray, + seed: int, + val_ratio: float, + test_ratio: float, +) -> Tuple[np.ndarray, np.ndarray, np.ndarray]: + indices = np.arange(len(labels)) + test_split = StratifiedShuffleSplit(n_splits=1, test_size=test_ratio, random_state=seed) + train_val_idx, test_idx = next(test_split.split(indices, labels)) + + remaining_ratio = 1.0 - test_ratio + if remaining_ratio <= 0.0: + raise ValueError("test_ratio must be < 1") + val_ratio_adjusted = val_ratio / remaining_ratio + + train_val_labels = labels[train_val_idx] + val_split = StratifiedShuffleSplit(n_splits=1, test_size=val_ratio_adjusted, random_state=seed) + train_sub_idx, val_sub_idx = next(val_split.split(train_val_idx, train_val_labels)) + + train_idx = train_val_idx[train_sub_idx] + val_idx = train_val_idx[val_sub_idx] + return train_idx, val_idx, test_idx + + +def stratified_train_val_split( + labels: np.ndarray, + seed: int, + val_ratio: float, +) -> Tuple[np.ndarray, np.ndarray]: + if not 0.0 < val_ratio < 1.0: + raise ValueError(f"val_ratio must be in (0, 1), got {val_ratio}") + indices = np.arange(len(labels)) + val_split = StratifiedShuffleSplit(n_splits=1, test_size=val_ratio, random_state=seed) + train_idx, val_idx = next(val_split.split(indices, labels)) + return train_idx, val_idx + + +@torch.no_grad() +def evaluate( + model: PerformerModuleCellClassifier, + data_loader: DataLoader, + preprocessor: DataPreprocessor, + reindex_state: dict, + criterion: nn.Module, + device: torch.device, +) -> dict: + model.eval() + loss_sum = 0.0 + sample_count = 0 + preds_all = [] + labels_all = [] + + for x, y in data_loader: + x = x.to(device) + y = y.to(device) + x_model = preprocessor.reindex_tensor_to_model_space(x, reindex_state=reindex_state) + logits = model(x_model) + loss = criterion(logits, y) + loss_sum += loss.item() * y.size(0) + sample_count += y.size(0) + preds_all.append(logits.argmax(dim=-1).cpu().numpy()) + labels_all.append(y.cpu().numpy()) + + preds = np.concatenate(preds_all) if preds_all else np.empty((0,), dtype=np.int64) + labels = np.concatenate(labels_all) if labels_all else np.empty((0,), dtype=np.int64) + preds = all_gather_numpy(preds) + labels = all_gather_numpy(labels) + loss_sum = reduce_sum(loss_sum, device) + sample_count = int(reduce_sum(sample_count, device)) + return { + "loss": float(loss_sum / max(sample_count, 1)), + "acc": float(accuracy_score(labels, preds)), + "macro_f1": float(f1_score(labels, preds, average="macro")), + "preds": preds, + "labels": labels, + } + + +def train_one_epoch( + model: PerformerModuleCellClassifier, + data_loader: DataLoader, + preprocessor: DataPreprocessor, + reindex_state: dict, + criterion: nn.Module, + optimizer: torch.optim.Optimizer, + device: torch.device, + grad_clip: float, + grad_accum_steps: int, + use_ddp: bool, + selector_sparsity_weight: float = 0.0, +) -> dict: + model.train() + running_loss = 0.0 + sample_count = 0 + preds_all = [] + labels_all = [] + optimizer.zero_grad(set_to_none=True) + + num_steps = len(data_loader) + for step, (x, y) in enumerate(data_loader, start=1): + x = x.to(device) + y = y.to(device) + x_model = preprocessor.reindex_tensor_to_model_space(x, reindex_state=reindex_state) + + sync_context = ( + model.no_sync() + if use_ddp and grad_accum_steps > 1 and step % grad_accum_steps != 0 + else contextlib.nullcontext() + ) + with sync_context: + logits = model(x_model) + loss = criterion(logits, y) + if selector_sparsity_weight > 0: + model_raw = model.module if use_ddp else model + loss = loss + selector_sparsity_weight * model_raw.get_selector_sparsity_loss() + (loss / grad_accum_steps).backward() + + if step % grad_accum_steps == 0 or step == num_steps: + if grad_clip > 0: + torch.nn.utils.clip_grad_norm_(model.parameters(), grad_clip) + optimizer.step() + optimizer.zero_grad(set_to_none=True) + + running_loss += loss.item() * y.size(0) + sample_count += y.size(0) + preds_all.append(logits.argmax(dim=-1).detach().cpu().numpy()) + labels_all.append(y.detach().cpu().numpy()) + + preds = np.concatenate(preds_all) if preds_all else np.empty((0,), dtype=np.int64) + labels = np.concatenate(labels_all) if labels_all else np.empty((0,), dtype=np.int64) + preds = all_gather_numpy(preds) + labels = all_gather_numpy(labels) + running_loss = reduce_sum(running_loss, device) + sample_count = int(reduce_sum(sample_count, device)) + return { + "loss": float(running_loss / max(sample_count, 1)), + "acc": float(accuracy_score(labels, preds)), + "macro_f1": float(f1_score(labels, preds, average="macro")), + } + + +def save_json(path: Path, payload: dict) -> None: + with open(path, "w", encoding="utf-8") as handle: + json.dump(payload, handle, indent=2, ensure_ascii=False) + + +def build_confusion_tables( + labels: np.ndarray, + preds: np.ndarray, + class_names: Sequence[str], +) -> Tuple[np.ndarray, np.ndarray, pd.DataFrame, pd.DataFrame]: + label_ids = np.arange(len(class_names), dtype=np.int64) + conf = confusion_matrix(labels, preds, labels=label_ids) + row_sums = conf.sum(axis=1, keepdims=True) + conf_norm = np.divide( + conf.astype(np.float64), + row_sums, + out=np.zeros_like(conf, dtype=np.float64), + where=row_sums != 0, + ) + conf_df = pd.DataFrame(conf, index=class_names, columns=class_names) + conf_norm_df = pd.DataFrame(conf_norm, index=class_names, columns=class_names) + return conf, conf_norm, conf_df, conf_norm_df + + +def apply_dataset_preset(args) -> None: + if args.dataset_preset == "none": + return + + if args.dataset_preset == "ms": + default_single_data_path = None + default_train_path = str(DOWNSTREAM_DATA_DIR / "MS" / "c_data.h5ad") + default_test_path = str(DOWNSTREAM_DATA_DIR / "MS" / "filtered_ms_adata.h5ad") + + if args.data_path == default_single_data_path and args.train_data_path is None and args.test_data_path is None: + args.train_data_path = default_train_path + args.test_data_path = default_test_path + + if args.label_column == "cell_type_anno": + args.label_column = "celltype" + if args.test_label_column is None: + args.test_label_column = "celltype" + if args.gene_name_column == "auto": + args.gene_name_column = "gene_name" + if args.min_genes == 10: + args.min_genes = 0 + if args.min_cells == 10: + args.min_cells = 0 + + logger.info( + "Applied dataset preset 'ms': train_data_path=%s test_data_path=%s label_column=%s " + "test_label_column=%s gene_name_column=%s min_genes=%d min_cells=%d", + args.train_data_path, + args.test_data_path, + args.label_column, + args.test_label_column, + args.gene_name_column, + args.min_genes, + args.min_cells, + ) + return + + raise ValueError(f"Unsupported dataset_preset: {args.dataset_preset}") + + +def parse_args(): + parser = argparse.ArgumentParser(description="End-to-end fine-tuning for Performer Module cell annotation") + + parser.add_argument("--seed", type=int, default=2021) + parser.add_argument("--device", type=str, default="cuda") + parser.add_argument("--batch_size", type=int, default=32, help="Per-device train batch size.") + parser.add_argument("--eval_batch_size", type=int, default=None, help="Per-device eval batch size.") + parser.add_argument("--global_batch_size", type=int, default=None, help="Effective global train batch size.") + parser.add_argument("--global_eval_batch_size", type=int, default=None, help="Effective global eval batch size.") + parser.add_argument("--grad_accum_steps", type=int, default=1) + parser.add_argument("--num_workers", type=int, default=4) + parser.add_argument("--prefetch_factor", type=int, default=2) + parser.add_argument("--epochs", type=int, default=50) + parser.add_argument("--patience", type=int, default=10) + parser.add_argument("--grad_clip", type=float, default=1.0) + + parser.add_argument("--lr_head", type=float, default=1e-4) + parser.add_argument("--lr_backbone", type=float, default=1e-5) + parser.add_argument("--weight_decay", type=float, default=0.01) + parser.add_argument("--label_smoothing", type=float, default=0.1) + parser.add_argument( + "--dropout", + type=float, + default=0.2, + help="Shared dropout applied to pooled cell embeddings before classification.", + ) + parser.add_argument("--pool_hidden_dim", type=int, default=0) + parser.add_argument("--cls_decoder_layers", type=int, default=3) + + parser.add_argument( + "--head_type", + type=str, + default="attn_pool", + choices=["module_mean", "max", "gene_mean", "attn_pool", "cls"], + ) + parser.add_argument( + "--finetune_mode", + type=str, + default="last_n", + choices=["frozen", "last_n", "full"], + ) + parser.add_argument("--unfreeze_last_n", type=int, default=1) + parser.add_argument("--train_pos_emb", action="store_true") + parser.add_argument("--train_module_embed", action="store_true") + parser.add_argument("--use_family_emb", action="store_true", + help="Add family-type embedding (Hallmarks/Reactome/Hotspot/WGCNA)") + parser.add_argument("--use_module_gate", action="store_true", + help="Add learnable sparse gate per module") + parser.add_argument("--use_module_selector", action="store_true", + help="Dynamic module selector with sparsity loss (soft train, hard inference)") + parser.add_argument("--selector_sparsity_weight", type=float, default=0.1, + help="Lambda for sparsity loss on module selector scores") + parser.add_argument("--selector_tau", type=float, default=0.5, + help="Score threshold for hard module selection during inference") + parser.add_argument("--dataset_preset", type=str, default="none", choices=["none", "ms"]) + + parser.add_argument( + "--data_path", + type=str, + default=None, + help="Input h5ad file; required unless paired train/test files or the MS preset are used.", + ) + parser.add_argument("--train_data_path", type=str, default=None) + parser.add_argument("--test_data_path", type=str, default=None) + parser.add_argument("--label_column", type=str, default="cell_type_anno") + parser.add_argument("--test_label_column", type=str, default=None) + parser.add_argument( + "--gene_name_column", + type=str, + default="auto", + help="Gene-name source in adata.var: 'auto', 'index', or a column name such as 'gene_name'.", + ) + parser.add_argument("--val_ratio", type=float, default=0.1) + parser.add_argument("--test_ratio", type=float, default=0.2) + parser.add_argument("--min_genes", type=int, default=10) + parser.add_argument("--min_cells", type=int, default=10) + parser.add_argument("--max_cells", type=int, default=None) + + parser.add_argument("--config_pkl", type=str, default=str(WEIGHTS_DIR / "mae_tost_70M_75mask_ec2.pkl")) + parser.add_argument( + "--performer_checkpoint", + type=str, + default=str(WEIGHTS_DIR / "mae_tost_70M_75mask_ec2_epoch0.pth"), + ) + parser.add_argument("--module_data_dir", type=str, default=str(ASSET_DIR / "module_data_all")) + parser.add_argument("--gene_list_path", type=str, default=str(ASSET_DIR / "selected_genes_27k.txt")) + parser.add_argument("--gene_mapping_path", type=str, default=str(ASSET_DIR / "gene2ens.csv")) + parser.add_argument("--gene2vec_data_path", type=str, default=str(ASSET_DIR / "selected_gene2vec_27k.npy")) + parser.add_argument("--gene2ens_file", type=str, default=str(ASSET_DIR / "gene2ens.csv")) + parser.add_argument("--gene2ens_column", type=str, default="Ensembl gene ID") + parser.add_argument( + "--shuffle_module_genes", + action="store_true", + help="Apply an extra downstream reshuffle on top of module_sets restored from PKL. " + "Do not pass this when reproducing the pretrained backbone as-is.", + ) + parser.add_argument( + "--shuffle_seed", + type=int, + default=2021, + help="Random seed for the downstream extra reshuffle when --shuffle_module_genes is enabled.", + ) + + parser.add_argument("--output_dir", type=str, default="./ckpts_performer_module_finetune") + parser.add_argument("--run_name", type=str, default="") + parser.add_argument( + "--save_test_artifacts", + action="store_true", + help="Save test classification report, confusion matrices, and metrics summary.", + ) + + args = parser.parse_args() + if args.dataset_preset == "none" and args.data_path is None and args.train_data_path is None: + parser.error("Provide --data_path, paired --train_data_path/--test_data_path, or --dataset_preset ms.") + return args + + +def main(): + args = parse_args() + apply_dataset_preset(args) + device, use_ddp, rank, world_size, local_rank = init_distributed_mode(args.device) + if not is_main_process(): + logger.setLevel(logging.WARNING) + seed_all(args.seed + rank) + + run_name = args.run_name or ( + f"{args.head_type}_{args.finetune_mode}" + f"_n{args.unfreeze_last_n}_lrh{args.lr_head:g}_lrb{args.lr_backbone:g}_seed{args.seed}" + ) + output_dir = resolve_path(args.output_dir) / run_name + output_dir.mkdir(parents=True, exist_ok=True) + + train_batch_size, eval_batch_size, effective_global_batch = resolve_batch_sizes(args, world_size) + logger.info("Run directory: %s", output_dir) + logger.info( + "Device: %s | DDP=%s rank=%d world_size=%d local_rank=%d", + device, + use_ddp, + rank, + world_size, + local_rank, + ) + logger.info( + "Batch config: per_device_train=%d per_device_eval=%d grad_accum=%d effective_global_train=%d", + train_batch_size, + eval_batch_size, + args.grad_accum_steps, + effective_global_batch, + ) + + preprocessor = DataPreprocessor( + gene2ens_file=str(resolve_path(args.gene2ens_file)), + gene2ens_column=args.gene2ens_column, + model_genes_file=str(resolve_path(args.gene_list_path)), + ) + preprocessor.load_gene_mapping() + preprocessor.load_model_genes() + paired_data_mode = bool(args.train_data_path or args.test_data_path) + if bool(args.train_data_path) != bool(args.test_data_path): + raise ValueError("train_data_path and test_data_path must be provided together") + + if paired_data_mode: + if args.test_ratio != 0.2: + logger.info("External test set provided; ignoring test_ratio=%s", args.test_ratio) + + train_adata, train_reindex_state = preprocessor.preprocess_adata( + str(resolve_path(args.train_data_path)), + min_genes=args.min_genes, + min_cells=args.min_cells, + max_cells=args.max_cells, + dataset_name="train", + gene_name_column=args.gene_name_column, + ) + test_adata, test_reindex_state = preprocessor.preprocess_adata( + str(resolve_path(args.test_data_path)), + min_genes=args.min_genes, + min_cells=args.min_cells, + max_cells=None, + dataset_name="test", + gene_name_column=args.gene_name_column, + ) + + train_label_column = args.label_column + test_label_column = args.test_label_column or args.label_column + label_encoder = LabelEncoder() + train_labels_raw = train_adata.obs[train_label_column].astype(str).to_numpy() + label_encoder.fit(train_labels_raw) + labels_train = label_encoder.transform(train_labels_raw) + + test_labels_raw = test_adata.obs[test_label_column].astype(str).to_numpy() + unseen_test_labels = sorted(set(test_labels_raw) - set(label_encoder.classes_)) + if unseen_test_labels: + raise ValueError( + "External test set contains labels unseen in training data: " + + ", ".join(unseen_test_labels[:10]) + ) + labels_test = label_encoder.transform(test_labels_raw) + num_classes = len(label_encoder.classes_) + + train_idx, val_idx = stratified_train_val_split( + labels=labels_train, + seed=args.seed, + val_ratio=args.val_ratio, + ) + logger.info( + "Using external test split | train=%s test=%s", + resolve_path(args.train_data_path), + resolve_path(args.test_data_path), + ) + logger.info( + "Split sizes: train=%d val=%d external_test=%d classes=%d", + len(train_idx), + len(val_idx), + test_adata.n_obs, + num_classes, + ) + + train_dataset = ExpressionDataset(train_adata.X, labels_train, train_idx) + val_dataset = ExpressionDataset(train_adata.X, labels_train, val_idx) + test_dataset = ExpressionDataset(test_adata.X, labels_test, np.arange(test_adata.n_obs)) + val_reindex_state = train_reindex_state + else: + adata, train_reindex_state = preprocessor.preprocess_adata( + str(resolve_path(args.data_path)), + min_genes=args.min_genes, + min_cells=args.min_cells, + max_cells=args.max_cells, + dataset_name="data", + gene_name_column=args.gene_name_column, + ) + + label_encoder = LabelEncoder() + labels = label_encoder.fit_transform(adata.obs[args.label_column].astype(str).to_numpy()) + num_classes = len(label_encoder.classes_) + train_idx, val_idx, test_idx = stratified_split( + labels=labels, + seed=args.seed, + val_ratio=args.val_ratio, + test_ratio=args.test_ratio, + ) + logger.info( + "Split sizes: train=%d val=%d test=%d classes=%d", + len(train_idx), + len(val_idx), + len(test_idx), + num_classes, + ) + + matrix = adata.X + train_dataset = ExpressionDataset(matrix, labels, train_idx) + val_dataset = ExpressionDataset(matrix, labels, val_idx) + test_dataset = ExpressionDataset(matrix, labels, test_idx) + val_reindex_state = train_reindex_state + test_reindex_state = train_reindex_state + + train_sampler = ( + DistributedSampler(train_dataset, num_replicas=world_size, rank=rank, shuffle=True, drop_last=False) + if use_ddp + else None + ) + val_sampler = ( + DistributedEvalSampler(len(val_dataset), rank=rank, world_size=world_size) + if use_ddp + else None + ) + test_sampler = ( + DistributedEvalSampler(len(test_dataset), rank=rank, world_size=world_size) + if use_ddp + else None + ) + + loader_kwargs = { + "num_workers": args.num_workers, + "pin_memory": device.type == "cuda", + } + if args.num_workers > 0: + loader_kwargs["persistent_workers"] = True + loader_kwargs["prefetch_factor"] = args.prefetch_factor + + train_loader = DataLoader( + train_dataset, + batch_size=train_batch_size, + shuffle=train_sampler is None, + sampler=train_sampler, + **loader_kwargs, + ) + val_loader = DataLoader( + val_dataset, + batch_size=eval_batch_size, + shuffle=False, + sampler=val_sampler, + **loader_kwargs, + ) + test_loader = DataLoader( + test_dataset, + batch_size=eval_batch_size, + shuffle=False, + sampler=test_sampler, + **loader_kwargs, + ) + + backbone, module_sets, backbone_config = load_backbone(args, device=device) + trainability = configure_backbone_trainability( + backbone, + finetune_mode=args.finetune_mode, + unfreeze_last_n=args.unfreeze_last_n, + train_pos_emb=args.train_pos_emb, + train_module_embed=args.train_module_embed, + ) + + model = PerformerModuleCellClassifier( + backbone=backbone, + module_sets=module_sets, + num_classes=num_classes, + head_type=args.head_type, + pool_hidden_dim=args.pool_hidden_dim, + dropout=args.dropout, + cls_decoder_layers=args.cls_decoder_layers, + use_module_selector=getattr(args, "use_module_selector", False), + ).to(device) + model_to_save = model + if getattr(args, "use_module_selector", False): + model_to_save.selector_tau = getattr(args, "selector_tau", 0.5) + + head_total = sum(p.numel() for name, p in model.named_parameters() if not name.startswith("backbone.")) + head_trainable = sum( + p.numel() for name, p in model.named_parameters() if not name.startswith("backbone.") and p.requires_grad + ) + logger.info( + "Backbone params: trainable=%d total=%d | Head params: trainable=%d total=%d", + trainability["trainable"], + trainability["total"], + head_trainable, + head_total, + ) + + optimizer = build_optimizer( + model=model, + lr_head=args.lr_head, + lr_backbone=args.lr_backbone, + weight_decay=args.weight_decay, + ) + if use_ddp: + model = DDP(model, device_ids=[local_rank] if device.type == "cuda" else None) + scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=max(args.epochs, 1)) + criterion = nn.CrossEntropyLoss(label_smoothing=args.label_smoothing) + + if is_main_process(): + save_json( + output_dir / "run_config.json", + { + "args": vars(args), + "run_name": run_name, + "num_classes": num_classes, + "label_classes": label_encoder.classes_.tolist(), + "backbone_config": backbone_config, + "trainable_backbone_params": trainability, + "head_type": args.head_type, + "data_mode": "paired_train_test" if paired_data_mode else "single_split", + "gene_name_column": args.gene_name_column, + "world_size": world_size, + "effective_global_batch": effective_global_batch, + "per_device_batch_size": train_batch_size, + "per_device_eval_batch_size": eval_batch_size, + }, + ) + if use_ddp: + dist.barrier() + + best_val_f1 = -1.0 + best_metrics = None + best_epoch = -1 + patience_counter = 0 + + for epoch in range(1, args.epochs + 1): + if train_sampler is not None: + train_sampler.set_epoch(epoch) + train_metrics = train_one_epoch( + model=model, + data_loader=train_loader, + preprocessor=preprocessor, + reindex_state=train_reindex_state, + criterion=criterion, + optimizer=optimizer, + device=device, + grad_clip=args.grad_clip, + grad_accum_steps=args.grad_accum_steps, + use_ddp=use_ddp, + selector_sparsity_weight=getattr(args, "selector_sparsity_weight", 0.0) if getattr(args, "use_module_selector", False) else 0.0, + ) + val_metrics = evaluate( + model=model, + data_loader=val_loader, + preprocessor=preprocessor, + reindex_state=val_reindex_state, + criterion=criterion, + device=device, + ) + scheduler.step() + + logger.info( + "Epoch %d | train loss %.4f acc %.4f f1 %.4f | val loss %.4f acc %.4f f1 %.4f", + epoch, + train_metrics["loss"], + train_metrics["acc"], + train_metrics["macro_f1"], + val_metrics["loss"], + val_metrics["acc"], + val_metrics["macro_f1"], + ) + + if val_metrics["macro_f1"] > best_val_f1: + best_val_f1 = val_metrics["macro_f1"] + best_epoch = epoch + patience_counter = 0 + + best_metrics = { + "epoch": epoch, + "train": train_metrics, + "val": {k: v for k, v in val_metrics.items() if k not in {"preds", "labels"}}, + } + if is_main_process(): + torch.save( + { + "epoch": epoch, + "model_state_dict": unwrap_model(model_to_save).state_dict(), + "optimizer_state_dict": optimizer.state_dict(), + "scheduler_state_dict": scheduler.state_dict(), + "best_val_f1": best_val_f1, + "args": vars(args), + "label_classes": label_encoder.classes_.tolist(), + "backbone_config": backbone_config, + "world_size": world_size, + "effective_global_batch": effective_global_batch, + }, + output_dir / "best_model.pt", + ) + else: + patience_counter += 1 + if patience_counter >= args.patience: + logger.info("Early stopping at epoch %d", epoch) + break + if use_ddp: + dist.barrier() + + if best_epoch < 0: + raise RuntimeError("Training finished without a valid checkpoint") + + if use_ddp: + dist.barrier() + ckpt = torch.load(output_dir / "best_model.pt", map_location=device) + unwrap_model(model).load_state_dict(ckpt["model_state_dict"], strict=True) + + test_metrics = evaluate( + model=model, + data_loader=test_loader, + preprocessor=preprocessor, + reindex_state=test_reindex_state, + criterion=criterion, + device=device, + ) + logger.info( + "Best epoch %d | test loss %.4f acc %.4f f1 %.4f", + best_epoch, + test_metrics["loss"], + test_metrics["acc"], + test_metrics["macro_f1"], + ) + + report = classification_report( + test_metrics["labels"], + test_metrics["preds"], + target_names=label_encoder.classes_.tolist(), + digits=4, + ) + conf, conf_norm, conf_df, conf_norm_df = build_confusion_tables( + test_metrics["labels"], + test_metrics["preds"], + label_encoder.classes_.tolist(), + ) + + summary = { + "best_epoch": best_epoch, + "best_val_f1": best_val_f1, + "train": best_metrics["train"] if best_metrics is not None else None, + "val": best_metrics["val"] if best_metrics is not None else None, + "test": {k: v for k, v in test_metrics.items() if k not in {"preds", "labels"}}, + } + if is_main_process(): + logger.info("Test classification report:\n%s", report) + logger.info( + "Test confusion matrix (rows=true, cols=pred):\n%s", + conf_df.to_string(), + ) + logger.info( + "Test confusion matrix normalized by true class:\n%s", + conf_norm_df.round(4).to_string(), + ) + if args.save_test_artifacts: + with open(output_dir / "test_classification_report.txt", "w", encoding="utf-8") as handle: + handle.write(report) + with open(output_dir / "test_confusion_matrix.txt", "w", encoding="utf-8") as handle: + handle.write(conf_df.to_string()) + handle.write("\n") + with open(output_dir / "test_confusion_matrix_normalized.txt", "w", encoding="utf-8") as handle: + handle.write(conf_norm_df.round(4).to_string()) + handle.write("\n") + np.save(output_dir / "test_confusion_matrix.npy", conf) + np.save(output_dir / "test_confusion_matrix_normalized.npy", conf_norm) + conf_df.to_csv(output_dir / "test_confusion_matrix.csv", encoding="utf-8") + conf_norm_df.to_csv(output_dir / "test_confusion_matrix_normalized.csv", encoding="utf-8") + save_json(output_dir / "metrics_summary.json", summary) + logger.info("Saved test artifacts to %s", output_dir) + else: + logger.info("Test artifacts were not saved. Use --save_test_artifacts to enable file output.") + if use_ddp: + dist.barrier() + dist.destroy_process_group() + + +if __name__ == "__main__": + main() diff --git a/BioMod/command.txt b/BioMod/command.txt index 1ee514f15c27213ad9b74f927b231ad52596871a..35da82fc602192e5ef95aad354a780dcd5cdea03 100644 --- a/BioMod/command.txt +++ b/BioMod/command.txt @@ -1,31 +1,31 @@ -source activate base -conda activate sclong -cd ../data/250010166/scLong/performer_module -torchrun --nproc_per_node $SENSECORE_ACCELERATE_DEVICE_COUNT --nnodes $SENSECORE_PYTORCH_NNODES --node_rank $SENSECORE_PYTORCH_NODE_RANK --master_addr $MASTER_ADDR --master_port $MASTER_PORT \ -performer_module.py \ - --gene_list_path ../selected_genes_27k.txt \ - --cell_num 4000000 \ - --epoch 15 \ - --batch_size 60 \ - --valid_batch_size 120 \ - --learning_rate 1e-4 \ - --grad_acc 5 \ - --log_every 100 \ - --mask_ratio 0.75 \ - --data_path ../../../share/cell_files \ - --module_data_dir module_data_all/ \ - --gene_mapping_path ../gene2ens.csv \ - --gene2vec_data_path ../selected_gene2vec_27k.npy \ - --ckpt_dir ./performer_module_4M_all_module_hvg_600_singleton/ \ - --model_name performer_module_4M_all_module_hvg_600_singleton \ - --fusion_method multiply \ - -use_wandb \ - --api_key "$WANDB_API_KEY" \ - --loss_mode gene_mean \ - --decoder_type simple \ - --num_workers 12 \ - --prefetch_factor 4 \ - --extra_gene_rank_path hvg_rankings/hvg_seurat_v3_top4096_first4000000cells_gene27874.npz \ - --extra_gene_num 600 \ - --extra_gene_module_strategy singleton \ - --dec_embed_dim 1024 +# Run from the outer repository root in a Linux CUDA/NCCL environment. +# Set BIOMOD_PRETRAIN_DATA to your preprocessed per-cell NPZ directory. +# The pretraining corpus is not included in this repository. +conda activate performer_module +: "${BIOMOD_PRETRAIN_DATA:?Set BIOMOD_PRETRAIN_DATA to the pretraining NPZ directory}" +torchrun --standalone --nproc_per_node="${NPROC_PER_NODE:-1}" \ + BioMod/performer_module.py \ + --gene_list_path model_dependencies/selected_genes_27k.txt \ + --cell_num 4000000 \ + --epoch 15 \ + --batch_size 60 \ + --valid_batch_size 120 \ + --learning_rate 1e-4 \ + --grad_acc 5 \ + --log_every 100 \ + --mask_ratio 0.75 \ + --data_path "$BIOMOD_PRETRAIN_DATA" \ + --module_data_dir model_dependencies/module_data_all \ + --gene_mapping_path model_dependencies/gene2ens.csv \ + --gene2vec_data_path model_dependencies/selected_gene2vec_27k.npy \ + --ckpt_dir tutorial_runs/pretraining \ + --model_name performer_module_pretraining \ + --fusion_method multiply \ + --loss_mode gene_mean \ + --decoder_type simple \ + --num_workers 12 \ + --prefetch_factor 4 \ + --dec_embed_dim 1024 + +# Adjust cell count and optimization settings to your own dataset and experiment. +# Add -use_wandb --api_key "$WANDB_API_KEY" to enable optional experiment logging. diff --git a/BioMod/drug_combination/README.md b/BioMod/drug_combination/README.md index 48e4a1e1ef0b488928519447ba2255811a51e7fb..c50393bbba008bc76515dd7c0bd13186619e5afb 100644 --- a/BioMod/drug_combination/README.md +++ b/BioMod/drug_combination/README.md @@ -1,143 +1,145 @@ -# Drug Combination Synergy Prediction with BioMod - -This task predicts drug-combination synergy using BioMod cell embeddings from -the PerformerModule encoder and GCN-based drug molecular graph encoders. Use the -[BioMod model weights and matching configuration](../../README.md#biomod-model-weights). - -It is adapted from [scLong/drug_combination](../../../drug_combination) -(DeepDDs), replacing the scfm/geneformer cell-line representation with -PerformerModule. - -## Model Architecture - -| Component | Description | -| --- | --- | -| Drug branch | Three PyG GCNConv layers (78 -> 78 -> 156 -> 312), `global_max_pool`, and two fully connected layers to 128 dimensions | -| Cell branch | Pretrained PerformerModule encoder, module mean pooling, and an MLP to 128 dimensions | -| Fusion layer | `concat(drug1, drug2, cell) = 384 -> MLP -> 2-class prediction` | - -The drug1 and drug2 branches share weights, matching scLong. - -## Data - -Place the data under `data_summary/`: - -```text -data_summary/ -|-- new_labels_0_10.csv # Training labels (drug1, drug2, cell, label) -|-- new_cell_features_954.csv # Training cell features (30 cell lines x 954 genes) -|-- smiles_0_10.csv # Drug SMILES -`-- test_data/ - |-- independent_input.csv # Validation labels (668 samples) - `-- independent_cell_features_954.csv # Validation cell features (1020 cell lines x 954 genes) -``` - -The directory name `test_data/` is retained for compatibility. These files are -used as the validation set for checkpoint selection. - -## Dependencies - -- PyTorch -- PyTorch Geometric (`torch_geometric`) -- RDKit -- networkx -- scikit-learn -- pandas, numpy - -## Usage - -```bash -pip install RDKit -i https://pypi.tuna.tsinghua.edu.cn/simple -``` - -### Basic Run (Domain Generalization Mode, Default) - -```bash -python train_drug_combination_performer_module.py \ - --checkpoint_path /path/to/performer_module_checkpoint.pt \ - --hyper_params_path /path/to/hyper_params.pkl \ - --module_data_dir ../module_data_all \ - --data_dir ./data_summary -``` - -### Custom Paths - -```bash -python train_drug_combination_performer_module.py \ - --checkpoint_path ../mae_tost_48M_cross_attn_only_pure_memory/mae_tost_48M_cross_attn_only_pure_memory_epoch12.pth \ - --hyper_params_path ../mae_tost_48M_cross_attn_only_pure_memory/mae_tost_48M_cross_attn_only_pure_memory.pkl \ - --module_data_dir ../module_data_all \ - --gene_list_path ../../selected_genes_27k.txt\ - --gene_mapping_path ../gene_mapping.pkl \ - --gene2vec_path ../../selected_gene2vec_27k.npy \ - --symbol2ens_path ../../gene2ens.csv \ - --data_dir ./data_summary \ - --output_dir ./results \ - --epochs 500 \ - --batch_size 256 \ - --lr 1e-4 \ - --n_folds 1 \ - --pool_method max -``` - -Reference note: `1e-4 500 0.6824301861702127`. - -### Five-Fold Cross-Validation (Disable Domain Generalization) - -```bash -python train_drug_combination_performer_module.py \ - --checkpoint_path /path/to/checkpoint.pt \ - --hyper_params_path /path/to/hyper_params.pkl \ - --module_data_dir ../module_data_all \ - --no_domain_gen -``` - -### Force Reprocessing After Changing the Model or Data - -```bash -python train_drug_combination_performer_module.py \ - --checkpoint_path /path/to/checkpoint.pt \ - --hyper_params_path /path/to/hyper_params.pkl \ - --module_data_dir ../module_data_all \ - --force_preprocess -``` - -## Main Arguments - -| Argument | Default | Description | -| --- | --- | --- | -| `--checkpoint_path` | Required | Path to the BioMod model weights | -| `--hyper_params_path` | Required | Path to the PerformerModule hyper-parameter pkl file | -| `--module_data_dir` | None | Directory containing module data files | -| `--data_dir` | `./data_summary` | Data directory | -| `--validation_label_file` | `test_data/independent_input.csv` | Validation labels for checkpoint selection; legacy alias: `--test_label_file` | -| `--validation_cell_file` | `test_data/independent_cell_features_954.csv` | Validation cell features; legacy alias: `--test_cell_file` | -| `--batch_size` | 256 | Training batch size | -| `--lr` | 1e-4 | Learning rate | -| `--epochs` | 100 | Number of training epochs | -| `--n_folds` | 5 | Number of folds | -| `--domain_gen` | True | Domain generalization mode: train on the full training set and select checkpoints using a separate validation set | -| `--eval_interval` | 5 | Evaluate every N epochs | -| `--pool_method` | mean | PerformerModule pooling method (`mean`, `max`, or `concat`) | -| `--cache_emb` | True | Cache cell embeddings to a `.npy` file | -| `--force_preprocess` | False | Force regeneration of the PyG dataset | - -## Outputs - -Results are saved under `--output_dir` (default: `./results/`): - -- `best_model_fold{i}.pt` - model weights selected by the highest validation AUC for each fold -- `fpr_tpr_fold{i}.pkl` - validation FPR/TPR curve data for drawing ROC curves -- `cell_emb_cache_{pool_method}.npy` - cached PerformerModule cell embeddings - -## Evaluation Metrics - -Every `--eval_interval` epochs, the script evaluates AUC, PR_AUC, ACC, BACC, -PREC, TPR, and KAPPA on the validation set, matching scLong's metric definitions. -The checkpoint with the highest validation AUC is saved for each fold. In -`--no_domain_gen` mode, each held-out fold serves as the validation set. - -This naming update preserves the data, splits, checkpoint-selection behavior, -output filenames, metric column names, and legacy dataset cache IDs. It does not -create or evaluate an independent final test set; the reported values are -validation results used during model selection, not final test estimates. +# Drug Combination Synergy Prediction with BioMod + +This task predicts drug-combination synergy using BioMod cell embeddings from +the PerformerModule encoder and GCN-based drug molecular graph encoders. Use the +[BioMod model weights and matching configuration](../../README.md#biomod-model-weights). + +It is adapted from scLong's drug-combination implementation +(DeepDDs), replacing the scfm/geneformer cell-line representation with +PerformerModule. + +## Model Architecture + +| Component | Description | +| --- | --- | +| Drug branch | Three PyG GCNConv layers (78 -> 78 -> 156 -> 312), `global_max_pool`, and two fully connected layers to 128 dimensions | +| Cell branch | Pretrained PerformerModule encoder, module mean pooling, and an MLP to 128 dimensions | +| Fusion layer | `concat(drug1, drug2, cell) = 384 -> MLP -> 2-class prediction` | + +The drug1 and drug2 branches share weights, matching scLong. + +## Data + +The bundled data are under `data/downstream/final_data/drug_combination_synergy/data_summary/`: + +```text +data_summary/ +|-- new_labels_0_10.csv # Training labels (drug1, drug2, cell, label) +|-- new_cell_features_954.csv # Training cell features (30 cell lines x 954 genes) +|-- smiles_0_10.csv # Drug SMILES +`-- test_data/ + |-- independent_input.csv # Validation labels (668 samples) + `-- independent_cell_features_954.csv # Validation cell features (1020 cell lines x 954 genes) +``` + +The directory name `test_data/` is retained for compatibility. These files are +used as the validation set for checkpoint selection. + +## Dependencies + +- PyTorch +- PyTorch Geometric (`torch_geometric`) +- RDKit +- networkx +- scikit-learn +- pandas, numpy + +## Usage + +Run all commands below from the repository root. Built-in resource defaults resolve from the script location. Explicit relative paths resolve from the working directory. + +```bash +pip install RDKit -i https://pypi.tuna.tsinghua.edu.cn/simple +``` + +### Basic Run (Domain Generalization Mode, Default) + +```bash +python BioMod/drug_combination/train_drug_combination_performer_module.py \ + --checkpoint_path weight/0810model/mae_tost_70M_75mask_ec2_epoch0.pth \ + --hyper_params_path weight/0810model/mae_tost_70M_75mask_ec2.pkl \ + --module_data_dir model_dependencies/module_data_all \ + --data_dir data/downstream/final_data/drug_combination_synergy/data_summary +``` + +### Custom Paths + +```bash +python BioMod/drug_combination/train_drug_combination_performer_module.py \ + --checkpoint_path weight/0810model/mae_tost_70M_75mask_ec2_epoch0.pth \ + --hyper_params_path weight/0810model/mae_tost_70M_75mask_ec2.pkl \ + --module_data_dir model_dependencies/module_data_all \ + --gene_list_path model_dependencies/selected_genes_27k.txt\ + --gene_mapping_path model_dependencies/gene2ens.csv \ + --gene2vec_path model_dependencies/selected_gene2vec_27k.npy \ + --symbol2ens_path model_dependencies/gene2ens.csv \ + --data_dir data/downstream/final_data/drug_combination_synergy/data_summary \ + --output_dir tutorial_runs/drug_combination \ + --epochs 500 \ + --batch_size 256 \ + --lr 1e-4 \ + --n_folds 1 \ + --pool_method max +``` + +Reference note: `1e-4 500 0.6824301861702127`. + +### Five-Fold Cross-Validation (Disable Domain Generalization) + +```bash +python BioMod/drug_combination/train_drug_combination_performer_module.py \ + --checkpoint_path weight/0810model/mae_tost_70M_75mask_ec2_epoch0.pth \ + --hyper_params_path weight/0810model/mae_tost_70M_75mask_ec2.pkl \ + --module_data_dir model_dependencies/module_data_all \ + --no_domain_gen +``` + +### Force Reprocessing After Changing the Model or Data + +```bash +python BioMod/drug_combination/train_drug_combination_performer_module.py \ + --checkpoint_path weight/0810model/mae_tost_70M_75mask_ec2_epoch0.pth \ + --hyper_params_path weight/0810model/mae_tost_70M_75mask_ec2.pkl \ + --module_data_dir model_dependencies/module_data_all \ + --force_preprocess +``` + +## Main Arguments + +| Argument | Default | Description | +| --- | --- | --- | +| `--checkpoint_path` | Required | Path to the BioMod model weights | +| `--hyper_params_path` | Required | Path to the PerformerModule hyper-parameter pkl file | +| `--module_data_dir` | None | Directory containing module data files | +| `--data_dir` | `data/downstream/final_data/drug_combination_synergy/data_summary` | Data directory | +| `--validation_label_file` | `test_data/independent_input.csv` | Validation labels for checkpoint selection; legacy alias: `--test_label_file` | +| `--validation_cell_file` | `test_data/independent_cell_features_954.csv` | Validation cell features; legacy alias: `--test_cell_file` | +| `--batch_size` | 256 | Training batch size | +| `--lr` | 1e-4 | Learning rate | +| `--epochs` | 100 | Number of training epochs | +| `--n_folds` | 5 | Number of folds | +| `--domain_gen` | True | Domain generalization mode: train on the full training set and select checkpoints using a separate validation set | +| `--eval_interval` | 5 | Evaluate every N epochs | +| `--pool_method` | mean | PerformerModule pooling method (`mean`, `max`, or `concat`) | +| `--cache_emb` | True | Cache cell embeddings to a `.npy` file | +| `--force_preprocess` | False | Force regeneration of the PyG dataset | + +## Outputs + +Results are saved under `--output_dir` (default: `BioMod/drug_combination/results/` relative to the repository root): + +- `best_model_fold{i}.pt` - model weights selected by the highest validation AUC for each fold +- `fpr_tpr_fold{i}.pkl` - validation FPR/TPR curve data for drawing ROC curves +- `cell_emb_cache_{pool_method}.npy` - cached PerformerModule cell embeddings + +## Evaluation Metrics + +Every `--eval_interval` epochs, the script evaluates AUC, PR_AUC, ACC, BACC, +PREC, TPR, and KAPPA on the validation set, matching scLong's metric definitions. +The checkpoint with the highest validation AUC is saved for each fold. In +`--no_domain_gen` mode, each held-out fold serves as the validation set. + +This naming update preserves the data, splits, checkpoint-selection behavior, +output filenames, metric column names, and legacy dataset cache IDs. It does not +create or evaluate an independent final test set; the reported values are +validation results used during model selection, not final test estimates. diff --git a/BioMod/drug_combination/train_drug_combination_performer_module.py b/BioMod/drug_combination/train_drug_combination_performer_module.py index 126ab1751aaf68876b0a2c1a95c79c545ee21b94..ddcb98f2e8ed45171a9f40b9a8a0b1abfcad41ba 100644 --- a/BioMod/drug_combination/train_drug_combination_performer_module.py +++ b/BioMod/drug_combination/train_drug_combination_performer_module.py @@ -1,900 +1,906 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" -Drug Combination Synergy Prediction with PerformerModule cell embeddings. - -Aligned with scLong/drug_combination (DeepDDs) architecture: - - Drug branch: PyG GCNConv + global_max_pool (shared weights for drug1 & drug2) - - Cell branch: PerformerModule pretrained encoder -> MLP - - Fusion: concat(drug1, drug2, cell) -> MLP -> 2-class classification - -Usage: - python train_drug_combination_performer_module.py \ - --checkpoint_path --hyper_params_path \ - --module_data_dir ../module_data_all \ - --data_dir ./data_summary -""" - -import argparse -import os -import pickle -import random -import sys -from pathlib import Path -from typing import Dict, List, Optional, Tuple - -import matplotlib -matplotlib.use('Agg') -import matplotlib.pyplot as plt -import networkx as nx -import numpy as np -import pandas as pd -import torch -import torch.nn as nn -import torch.nn.functional as F -from rdkit import Chem -from sklearn import metrics -from tqdm import tqdm -from sklearn.metrics import ( - accuracy_score, - balanced_accuracy_score, - cohen_kappa_score, - confusion_matrix, - precision_score, - recall_score, - roc_auc_score, -) -from torch.optim.lr_scheduler import StepLR - -from torch_geometric import data as DATA -from torch_geometric.data import InMemoryDataset -from torch_geometric.loader import DataLoader -from torch_geometric.nn import GCNConv, global_max_pool as gmp - -# ===================== Path setup ===================== -SCRIPT_DIR = Path(__file__).resolve().parent -PERFORMER_MODULE_ROOT = SCRIPT_DIR.parent -SCLONG_ROOT = PERFORMER_MODULE_ROOT.parent - -if str(PERFORMER_MODULE_ROOT) not in sys.path: - sys.path.insert(0, str(PERFORMER_MODULE_ROOT)) -if str(SCLONG_ROOT) not in sys.path: - sys.path.insert(0, str(SCLONG_ROOT)) - -import importlib.util - -# Import PerformerModule model -_models_path = os.path.join(PERFORMER_MODULE_ROOT, 'models_performer_module.py') -_spec = importlib.util.spec_from_file_location("models_performer_module", _models_path) -_models_mod = importlib.util.module_from_spec(_spec) -_spec.loader.exec_module(_models_mod) -PerformerEncoderOnly_Module = _models_mod.PerformerEncoderOnly_Module -sanitize_performer_module_config = _models_mod.sanitize_performer_module_config - -# Import module_dataset -_dataset_path = os.path.join(PERFORMER_MODULE_ROOT, 'module_dataset.py') -_spec2 = importlib.util.spec_from_file_location("module_dataset", _dataset_path) -_module_mod = importlib.util.module_from_spec(_spec2) -_spec2.loader.exec_module(_module_mod) -load_module_data = _module_mod.load_module_data - - -# ===================== Utilities ===================== - -def print_sys(*args, **kwargs): - print(*args, **kwargs, flush=True) - - -def reindex_tensor_universal(tensor, index_positions, dim, filler="0", device="cpu"): - index_tensor = torch.tensor(index_positions, device=device) - tensor_shape = list(tensor.shape) - tensor_shape[dim] = len(index_positions) - index_shape = [1] * len(tensor_shape) - index_shape[dim] = len(index_positions) - index_tensor = index_tensor.view(*index_shape).expand(*tensor_shape) - if filler == "0": - padder_shape = list(tensor_shape) - padder_shape[dim] = 1 - padder = torch.zeros(padder_shape, dtype=tensor.dtype, device=device) - elif filler == "mean": - padder = tensor.mean(dim=dim, keepdim=True) - else: - raise ValueError("filler should be '0' or 'mean'") - expanded_tensor = torch.cat((tensor, padder), dim=dim) - reindexed = torch.gather(expanded_tensor, dim, index_tensor) - return reindexed - - -# ===================== PerformerModule Loading ===================== - -def load_performer_module_model( - checkpoint_path, hyper_params_path, module_data_dir, module_data_path, - gene_list_path, gene_mapping_path, gene2vec_path, device, - shuffle_module_genes=False, shuffle_seed=2021, -): - print_sys(f"Loading PerformerModule model from: {checkpoint_path}") - - with open(hyper_params_path, 'rb') as f: - model_params = pickle.load(f) - model_params.pop('module_embed_chunk_padded_gene_budget', None) - model_params = sanitize_performer_module_config( - model_params, source=hyper_params_path, log_fn=print_sys, - ) - - if gene2vec_path: - model_params['gene2vec_file'] = gene2vec_path - model_params['use_gene2vec'] = model_params.get('use_gene2vec', True) - - if 'module_sets' not in model_params or model_params.get('module_sets') is None: - if module_data_dir or module_data_path: - module_sets = load_module_data( - module_data_path=module_data_path, - module_data_dir=module_data_dir, - gene_list_path=gene_list_path, - gene_mapping_path=gene_mapping_path, - verbose=True, - ) - model_params['module_sets'] = module_sets - else: - raise ValueError( - "Model needs module_sets but neither --module_data_dir " - "nor --module_data_path provided" - ) - - if shuffle_module_genes: - print_sys(f"Shuffling genes across modules with seed={shuffle_seed}") - all_gene_indices = [] - module_sizes = [] - for module in model_params['module_sets']: - all_gene_indices.extend(module) - module_sizes.append(len(module)) - rng = random.Random(shuffle_seed) - rng.shuffle(all_gene_indices) - shuffled_module_sets = [] - start_idx = 0 - for size in module_sizes: - shuffled_module_sets.append(all_gene_indices[start_idx:start_idx + size]) - start_idx += size - model_params['module_sets'] = shuffled_module_sets - print_sys(f"Shuffled {len(all_gene_indices)} genes across {len(shuffled_module_sets)} modules") - - model = PerformerEncoderOnly_Module(**model_params).to(device) - - checkpoint = torch.load(checkpoint_path, map_location=device, weights_only=False) - state_dict = checkpoint.get('model_state_dict', checkpoint) - filtered = { - k: v for k, v in state_dict.items() - if 'go_conv' not in k and 'G_go' not in k.lower() - } - model.load_state_dict(filtered, strict=True) - print_sys("Loaded pretrained PerformerModule weights.") - - model.eval() - for p in model.parameters(): - p.requires_grad = False - - return model, len(model_params['module_sets']), model_params.get('base_dim', 1024) - - -def build_performer_module_gene_mapping(symbol2ens_path, gene_list_path, expr_genes): - gene2ens = pd.read_csv(symbol2ens_path, index_col=0) - symbol2ens = gene2ens["Ensembl gene ID"].to_dict() - - input_genes = [symbol2ens.get(sym, sym) for sym in expr_genes] - input_gene_num = len(input_genes) - input_mapping = {gid: i for i, gid in enumerate(input_genes)} - - with open(gene_list_path, "r") as f: - performer_genes = [line.rstrip("\n") for line in f.readlines()] - performer_genes_pad = performer_genes + ["PAD"] - - performer_index_positions = [ - input_mapping.get(gid, input_gene_num) for gid in performer_genes_pad - ] - intersect = np.intersect1d(input_genes, performer_genes) - print_sys(f"Target and PerformerModule genes intersect: {len(intersect)}") - return input_mapping, performer_index_positions - - -def build_cell_embeddings( - cell_features_df, symbol2ens_path, gene_list_path, gene_mapping_path, - gene2vec_path, checkpoint_path, hyper_params_path, module_data_dir, - module_data_path, device, pool_method="mean", - shuffle_module_genes=False, shuffle_seed=2021, -): - model, num_modules, encoder_embed_dim = load_performer_module_model( - checkpoint_path, hyper_params_path, module_data_dir, module_data_path, - gene_list_path, gene_mapping_path, gene2vec_path, device, - shuffle_module_genes=shuffle_module_genes, shuffle_seed=shuffle_seed, - ) - - gene_columns = list(cell_features_df.columns) - _, performer_index_positions = build_performer_module_gene_mapping( - symbol2ens_path, gene_list_path, gene_columns, - ) - - cell_emb_dict: Dict[str, np.ndarray] = {} - first_emb_dim = -1 - - with torch.no_grad(): - for cell_line_id, row in cell_features_df.iterrows(): - x = torch.tensor(row.values[None, :], dtype=torch.float32, device=device) - x_performer = reindex_tensor_universal( - x, performer_index_positions, dim=1, filler="0", device=device, - ) - out = model(x_performer, mask_ratio=0.0, return_encodings=True) - - if 'module_encodings' in out: - module_enc = out['module_encodings'] - if pool_method == "mean": - cell_emb = module_enc.mean(dim=1) - elif pool_method == "max": - cell_emb = module_enc.max(dim=1)[0] - elif pool_method == "concat": - cell_emb = module_enc.reshape(1, -1) - else: - cell_emb = module_enc.mean(dim=1) - else: - raise ValueError( - f"Expected 'module_encodings' in output, got: {list(out.keys())}" - ) - - cell_emb_np = cell_emb.squeeze(0).cpu().numpy() - cell_emb_dict[cell_line_id] = cell_emb_np - if first_emb_dim < 0: - first_emb_dim = cell_emb_np.shape[0] - - print_sys( - f"Built PerformerModule embeddings for {len(cell_emb_dict)} cell lines, " - f"dim={first_emb_dim}, pool_method={pool_method}" - ) - return cell_emb_dict, first_emb_dim - - -# ===================== Drug Graph (scLong-aligned: edge_index format) ===================== - -def one_of_k_encoding(x, allowable_set): - if x not in allowable_set: - raise Exception(f"input {x} not in allowable set {allowable_set}") - return list(map(lambda s: x == s, allowable_set)) - - -def one_of_k_encoding_unk(x, allowable_set): - if x not in allowable_set: - x = allowable_set[-1] - return list(map(lambda s: x == s, allowable_set)) - - -def atom_features(atom): - return np.array( - one_of_k_encoding_unk(atom.GetSymbol(), [ - 'C', 'N', 'O', 'S', 'F', 'Si', 'P', 'Cl', 'Br', 'Mg', 'Na', 'Ca', - 'Fe', 'As', 'Al', 'I', 'B', 'V', 'K', 'Tl', 'Yb', 'Sb', 'Sn', 'Ag', - 'Pd', 'Co', 'Se', 'Ti', 'Zn', 'H', 'Li', 'Ge', 'Cu', 'Au', 'Ni', 'Cd', - 'In', 'Mn', 'Zr', 'Cr', 'Pt', 'Hg', 'Pb', 'Unknown', - ]) + - one_of_k_encoding(atom.GetDegree(), [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) + - one_of_k_encoding_unk(atom.GetTotalNumHs(), [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) + - one_of_k_encoding_unk(atom.GetImplicitValence(), [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) + - [atom.GetIsAromatic()] - ) - - -ATOM_FEAT_DIM = 78 # 44 + 11 + 11 + 11 + 1 - - -def smile_to_graph(smile): - """Convert SMILES to (c_size, features, edge_index) — identical to scLong's creat_data_DC.py.""" - mol = Chem.MolFromSmiles(smile) - if mol is None: - return None - - c_size = mol.GetNumAtoms() - - features = [] - for atom in mol.GetAtoms(): - feature = atom_features(atom) - features.append(feature / (sum(feature) + 1e-8)) - - edges = [] - for bond in mol.GetBonds(): - edges.append([bond.GetBeginAtomIdx(), bond.GetEndAtomIdx()]) - g = nx.Graph(edges).to_directed() - edge_index = [] - for e1, e2 in g.edges: - edge_index.append([e1, e2]) - - return c_size, features, edge_index - - -# ===================== TestbedDataset (PyG InMemoryDataset, scLong-aligned) ===================== - -class TestbedDataset(InMemoryDataset): - """ - PyG InMemoryDataset for drug combination prediction. - Same pattern as scLong/drug_combination/utils_test.py::TestbedDataset, - but uses PerformerModule cell embeddings instead of scfm/geneformer. - """ - - def __init__( - self, root='data_summary', dataset='drug1', - xd=None, xt=None, y=None, - cell_emb_dict=None, smile_graph=None, - transform=None, pre_transform=None, - ): - self.dataset = dataset - self._xd = xd - self._xt = xt - self._y = y - self._cell_emb_dict = cell_emb_dict - self._smile_graph = smile_graph - super(TestbedDataset, self).__init__(root, transform, pre_transform) - - if os.path.isfile(self.processed_paths[0]): - print_sys(f'Loading preprocessed data: {self.processed_paths[0]}') - self.data, self.slices = torch.load(self.processed_paths[0], weights_only=False) - else: - print_sys(f'Preprocessing data: {self.processed_paths[0]}') - self.process(xd, xt, y, cell_emb_dict, smile_graph) - self.data, self.slices = torch.load(self.processed_paths[0], weights_only=False) - - @property - def raw_file_names(self): - return [] - - @property - def processed_file_names(self): - return [self.dataset + '.pt'] - - def download(self): - pass - - def _download(self): - pass - - def _process(self): - if not os.path.exists(self.processed_dir): - os.makedirs(self.processed_dir) - - def process(self, xd, xt, y, cell_emb_dict, smile_graph): - assert len(xd) == len(xt) == len(y), "xd, xt, y must have the same length" - data_list = [] - skipped = 0 - - for i in range(len(xd)): - smiles = xd[i] - cell_name = xt[i] - label = y[i] - - if smiles not in smile_graph: - skipped += 1 - continue - if cell_name not in cell_emb_dict: - skipped += 1 - continue - - c_size, features, edge_index = smile_graph[smiles] - - GCNData = DATA.Data( - x=torch.Tensor(features), - edge_index=torch.LongTensor(edge_index).transpose(1, 0), - y=torch.Tensor([label]), - ) - cell_emb = cell_emb_dict[cell_name] - GCNData.cell = torch.FloatTensor([cell_emb]) - GCNData.__setitem__('c_size', torch.LongTensor([c_size])) - data_list.append(GCNData) - - if skipped > 0: - print_sys(f'Skipped {skipped} samples (missing graph or cell embedding)') - - if self.pre_filter is not None: - data_list = [d for d in data_list if self.pre_filter(d)] - if self.pre_transform is not None: - data_list = [self.pre_transform(d) for d in data_list] - - print_sys(f'Processed {len(data_list)} samples. Saving to {self.processed_paths[0]}') - data, slices = self.collate(data_list) - torch.save((data, slices), self.processed_paths[0]) - - -# ===================== Model: GCNNet (PyG, scLong-aligned) ===================== - -class GCNNet(torch.nn.Module): - """ - GCN-based drug combination model aligned with scLong/drug_combination/models/gcn.py. - - Drug branch: 3-layer GCNConv (78->78->156->312) + global_max_pool + 2 FC layers - Drug1 and drug2 share weights (drug2 uses drug1_conv* layers, same as scLong). - Cell branch: PerformerModule embedding -> MLP -> output_dim - Fusion: concat(drug1, drug2, cell) -> fc1(256) -> fc2(128) -> out(2) - """ - - def __init__( - self, n_output=2, num_features_xd=78, cell_emb_dim=1024, - output_dim=128, dropout=0.2, - ): - super(GCNNet, self).__init__() - self.relu = nn.ReLU() - self.dropout = nn.Dropout(dropout) - self.n_output = n_output - - # Drug graph branch (shared weights for drug1 & drug2) - self.drug1_conv1 = GCNConv(num_features_xd, num_features_xd) - self.drug1_conv2 = GCNConv(num_features_xd, num_features_xd * 2) - self.drug1_conv3 = GCNConv(num_features_xd * 2, num_features_xd * 4) - self.drug1_fc_g1 = nn.Linear(num_features_xd * 4, num_features_xd * 2) - self.drug1_fc_g2 = nn.Linear(num_features_xd * 2, output_dim) - - # Cell branch (PerformerModule embedding -> MLP) - self.reduction = nn.Sequential( - nn.Linear(cell_emb_dim, 512), - nn.ReLU(), - nn.Linear(512, 256), - nn.ReLU(), - nn.Linear(256, output_dim), - ) - - # Fusion layers - self.fc1 = nn.Linear(3 * output_dim, 256) - self.fc2 = nn.Linear(256, 128) - self.out = nn.Linear(128, self.n_output) - - def forward(self, data1, data2): - x1, edge_index1, batch1, cell = ( - data1.x, data1.edge_index, data1.batch, data1.cell - ) - x2, edge_index2, batch2 = data2.x, data2.edge_index, data2.batch - - # Drug1 branch - x1 = self.relu(self.drug1_conv1(x1, edge_index1)) - x1 = self.relu(self.drug1_conv2(x1, edge_index1)) - x1 = self.relu(self.drug1_conv3(x1, edge_index1)) - x1 = gmp(x1, batch1) - x1 = self.relu(self.drug1_fc_g1(x1)) - x1 = self.dropout(x1) - x1 = self.drug1_fc_g2(x1) - x1 = self.dropout(x1) - - # Drug2 branch (shared weights — uses drug1_conv* layers, same as scLong) - x2 = self.relu(self.drug1_conv1(x2, edge_index2)) - x2 = self.relu(self.drug1_conv2(x2, edge_index2)) - x2 = self.relu(self.drug1_conv3(x2, edge_index2)) - x2 = gmp(x2, batch2) - x2 = self.relu(self.drug1_fc_g1(x2)) - x2 = self.dropout(x2) - x2 = self.drug1_fc_g2(x2) - x2 = self.dropout(x2) - - # Cell branch - cell_vector = F.normalize(cell, 2, 1) - cell_vector = self.reduction(cell_vector) - - # Fusion - xc = torch.cat((x1, x2, cell_vector), 1) - xc = self.relu(self.fc1(xc)) - xc = self.dropout(xc) - xc = self.relu(self.fc2(xc)) - xc = self.dropout(xc) - out = self.out(xc) - return out - - -# ===================== Training & Evaluation ===================== - -def train_epoch(model, device, drug1_loader, drug2_loader, optimizer, epoch): - """Train one epoch. Returns average loss.""" - model.train() - total_loss = 0.0 - n_batches = 0 - pbar = tqdm( - zip(drug1_loader, drug2_loader), - total=len(drug1_loader), - desc=f'Epoch {epoch}', - leave=True, - ) - for data1, data2 in pbar: - data1 = data1.to(device) - data2 = data2.to(device) - y = data1.y.view(-1, 1).long().to(device).squeeze(1) - optimizer.zero_grad() - output = model(data1, data2) - loss = F.cross_entropy(output, y) - loss.backward() - optimizer.step() - total_loss += loss.item() - n_batches += 1 - pbar.set_postfix(loss=f'{loss.item():.4f}') - avg_loss = total_loss / max(n_batches, 1) - return avg_loss - - -def predict(model, device, drug1_loader, drug2_loader): - model.eval() - total_preds = torch.Tensor() - total_labels = torch.Tensor() - total_prelabels = torch.Tensor() - print_sys(f'Make prediction for {len(drug1_loader.dataset)} samples...') - - with torch.no_grad(): - for data in zip(drug1_loader, drug2_loader): - data1 = data[0].to(device) - data2 = data[1].to(device) - output = model(data1, data2) - ys = F.softmax(output, 1).to('cpu').data.numpy() - predicted_labels = list(map(lambda x: np.argmax(x), ys)) - predicted_scores = list(map(lambda x: x[1], ys)) - total_preds = torch.cat((total_preds, torch.Tensor(predicted_scores)), 0) - total_prelabels = torch.cat( - (total_prelabels, torch.Tensor(predicted_labels)), 0 - ) - total_labels = torch.cat((total_labels, data1.y.view(-1, 1).cpu()), 0) - - return ( - total_labels.numpy().flatten(), - total_preds.numpy().flatten(), - total_prelabels.numpy().flatten(), - ) - - -def evaluate_metrics(T, S, Y): - """Compute all metrics matching scLong's evaluation.""" - try: - AUC = roc_auc_score(T, S) - except Exception: - AUC = 0.0 - precision_arr, recall_arr, _ = metrics.precision_recall_curve(T, S) - PR_AUC = metrics.auc(recall_arr, precision_arr) - BACC = balanced_accuracy_score(T, Y) - tn, fp, fn, tp = confusion_matrix(T, Y).ravel() - TPR = tp / (tp + fn) - PREC = precision_score(T, Y) - ACC = accuracy_score(T, Y) - KAPPA = cohen_kappa_score(T, Y) - RECALL = recall_score(T, Y) - return { - 'AUC': AUC, 'PR_AUC': PR_AUC, 'ACC': ACC, 'BACC': BACC, - 'PREC': PREC, 'TPR': TPR, 'KAPPA': KAPPA, 'RECALL': RECALL, - } - - -# ===================== Cell Feature Loading ===================== - -def load_cell_features(cell_file): - """Load cell features CSV. First row is header (gene_id), second is transcript_id (skipped).""" - df = pd.read_csv(cell_file) - df = df.iloc[1:].copy() # skip transcript_id row - df = df.set_index(df.columns[0]) - df = df.astype(float) - # Clean cell names: "X22RV1_PROSTATE" -> "22RV1" - df.index = [ - idx.lstrip('X').split('_')[0] if '_' in idx else idx - for idx in df.index - ] - return df - - -# ===================== Main ===================== - -def main(): - parser = argparse.ArgumentParser( - description='Drug Combination Synergy Prediction with PerformerModule' - ) - - # PerformerModule model args - parser.add_argument('--checkpoint_path', type=str, required=True, - help='Path to PerformerModule pretrained checkpoint') - parser.add_argument('--hyper_params_path', type=str, required=True, - help='Path to PerformerModule hyper parameters pkl') - parser.add_argument('--module_data_dir', type=str, default=None, - help='Directory containing module data files') - parser.add_argument('--module_data_path', type=str, default=None, - help='Path to single module data file') - parser.add_argument('--gene_list_path', type=str, - default=os.path.join(PERFORMER_MODULE_ROOT, 'gene_list_27k.txt')) - parser.add_argument('--gene_mapping_path', type=str, - default=os.path.join(PERFORMER_MODULE_ROOT, 'gene_mapping.pkl')) - parser.add_argument('--gene2vec_path', type=str, - default=os.path.join(PERFORMER_MODULE_ROOT, 'gene2vec_27k.npy')) - parser.add_argument('--symbol2ens_path', type=str, - default=os.path.join(SCLONG_ROOT, 'gene2ensemble_ID.csv')) - parser.add_argument('--pool_method', type=str, default='mean', - choices=['mean', 'max', 'concat']) - parser.add_argument('--shuffle_module_genes', action='store_true', - help='Disrupt the gene allocation within the module (used for comparative experiments to verify the effectiveness of module division)') - parser.add_argument('--shuffle_seed', type=int, default=2021, - help='The random seed used when scrambling the module gene (needs to be consistent with the value used during pre-training)') - - # Data args - parser.add_argument('--data_dir', type=str, - default=os.path.join(SCRIPT_DIR, 'data_summary')) - parser.add_argument('--label_file', type=str, default='new_labels_0_10.csv') - parser.add_argument('--cell_file', type=str, default='new_cell_features_954.csv') - parser.add_argument('--validation_label_file', '--test_label_file', - dest='validation_label_file', type=str, - default='test_data/independent_input.csv', - help='Validation labels used for checkpoint selection; --test_label_file is a legacy alias') - parser.add_argument('--validation_cell_file', '--test_cell_file', - dest='validation_cell_file', type=str, - default='test_data/independent_cell_features_954.csv', - help='Validation cell features; --test_cell_file is a legacy alias') - - # Training args - parser.add_argument('--batch_size', type=int, default=256) - parser.add_argument('--lr', type=float, default=1e-4) - parser.add_argument('--epochs', type=int, default=100) - parser.add_argument('--n_folds', type=int, default=5) - parser.add_argument('--domain_gen', action='store_true', default=True, - help='Domain generalization: train full set, validate on a separate set for checkpoint selection') - parser.add_argument('--no_domain_gen', dest='domain_gen', action='store_false') - parser.add_argument('--eval_interval', type=int, default=5) - parser.add_argument('--output_dir', type=str, - default=os.path.join(SCRIPT_DIR, 'results')) - parser.add_argument('--seed', type=int, default=42) - parser.add_argument('--cache_emb', action='store_true', default=True, - help='Cache PerformerModule cell embeddings to .npy') - parser.add_argument('--no_cache_emb', dest='cache_emb', action='store_false') - parser.add_argument('--force_preprocess', action='store_true', default=False, - help='Force re-creation of PyG .pt datasets') - - args = parser.parse_args() - - # Device - device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') - print_sys(f'Using device: {device}') - - # Seed - random.seed(args.seed) - np.random.seed(args.seed) - torch.manual_seed(args.seed) - - os.makedirs(args.output_dir, exist_ok=True) - - # -------- Step 1: Build PerformerModule cell embeddings -------- - train_cell_file = os.path.join(args.data_dir, args.cell_file) - train_cell_df = load_cell_features(train_cell_file) - print_sys(f'Loaded train cell features: {train_cell_df.shape}') - - validation_cell_file = os.path.join(args.data_dir, args.validation_cell_file) - validation_cell_df = load_cell_features(validation_cell_file) - print_sys(f'Loaded validation cell features: {validation_cell_df.shape}') - - # Combine all cell lines and remove duplicates - all_cell_df = pd.concat([train_cell_df, validation_cell_df]) - all_cell_df = all_cell_df[~all_cell_df.index.duplicated(keep='first')] - print_sys(f'Total unique cell lines: {len(all_cell_df)}') - - # Cache embeddings - cache_path = os.path.join(args.output_dir, f'cell_emb_cache_{args.pool_method}.npy') - cache_dim_path = os.path.join( - args.output_dir, f'cell_emb_dim_{args.pool_method}.txt' - ) - - if args.cache_emb and os.path.exists(cache_path) and not args.force_preprocess: - print_sys(f'Loading cached cell embeddings from {cache_path}') - cell_emb_dict = np.load(cache_path, allow_pickle=True).item() - with open(cache_dim_path) as f: - cell_emb_dim = int(f.read().strip()) - else: - cell_emb_dict, cell_emb_dim = build_cell_embeddings( - all_cell_df, args.symbol2ens_path, args.gene_list_path, - args.gene_mapping_path, args.gene2vec_path, args.checkpoint_path, - args.hyper_params_path, args.module_data_dir, args.module_data_path, - device, args.pool_method, - shuffle_module_genes=args.shuffle_module_genes, - shuffle_seed=args.shuffle_seed, - ) - if args.cache_emb: - np.save(cache_path, cell_emb_dict) - with open(cache_dim_path, 'w') as f: - f.write(str(cell_emb_dim)) - print_sys(f'Cached cell embeddings to {cache_path}') - - # -------- Step 2: Build SMILES graphs -------- - train_label_path = os.path.join(args.data_dir, args.label_file) - train_labels = pd.read_csv(train_label_path) - print_sys(f'Train labels: {len(train_labels)} samples') - - validation_label_path = os.path.join(args.data_dir, args.validation_label_file) - validation_labels = pd.read_csv(validation_label_path) - print_sys(f'Validation labels: {len(validation_labels)} samples') - - all_smiles = set( - train_labels['drug1'].tolist() + train_labels['drug2'].tolist() - + validation_labels['drug1'].tolist() + validation_labels['drug2'].tolist() - ) - smile_graph = {} - failed = [] - for smi in all_smiles: - result = smile_to_graph(smi) - if result is not None: - smile_graph[smi] = result - else: - failed.append(smi) - if failed: - print_sys(f'Warning: {len(failed)} SMILES failed to parse') - print_sys(f'Built graphs for {len(smile_graph)} unique SMILES') - - # -------- Step 3: Create PyG datasets -------- - # Delete cached .pt files if force_preprocess - if args.force_preprocess: - processed_dir = os.path.join(args.data_dir, 'processed') - if os.path.exists(processed_dir): - import shutil - shutil.rmtree(processed_dir) - print_sys(f'Removed cached processed data: {processed_dir}') - - # Train datasets (drug1 and drug2 separate, scLong pattern) - drug1_train = train_labels['drug1'].tolist() - drug2_train = train_labels['drug2'].tolist() - cells_train = train_labels['cell'].tolist() - labels_train = train_labels['label'].tolist() - - train_d1_dataset = TestbedDataset( - root=args.data_dir, dataset='train_drug1_performer_module', - xd=drug1_train, xt=cells_train, y=labels_train, - cell_emb_dict=cell_emb_dict, smile_graph=smile_graph, - ) - train_d2_dataset = TestbedDataset( - root=args.data_dir, dataset='train_drug2_performer_module', - xd=drug2_train, xt=cells_train, y=labels_train, - cell_emb_dict=cell_emb_dict, smile_graph=smile_graph, - ) - - # Validation datasets (separate set used for checkpoint selection) - # Preserve legacy test_* cache IDs to reuse existing preprocessed datasets. - drug1_validation = validation_labels['drug1'].tolist() - drug2_validation = validation_labels['drug2'].tolist() - cells_validation = validation_labels['cell'].tolist() - labels_validation = validation_labels['label'].tolist() - - validation_d1_dataset = TestbedDataset( - root=args.data_dir, dataset='test_drug1_performer_module', - xd=drug1_validation, xt=cells_validation, y=labels_validation, - cell_emb_dict=cell_emb_dict, smile_graph=smile_graph, - ) - validation_d2_dataset = TestbedDataset( - root=args.data_dir, dataset='test_drug2_performer_module', - xd=drug2_validation, xt=cells_validation, y=labels_validation, - cell_emb_dict=cell_emb_dict, smile_graph=smile_graph, - ) - - # -------- Step 4: Training (5-fold, scLong-aligned) -------- - lenth = len(train_d1_dataset) - pot = lenth // args.n_folds - random_num = random.sample(range(lenth), lenth) - all_fold_aucs = [] - - for fold_i in range(args.n_folds): - print_sys(f'\n{"=" * 60}') - print_sys(f'Fold {fold_i + 1}/{args.n_folds}') - print_sys(f'{"=" * 60}') - - if args.domain_gen: - # Domain generalization: train on full training set, select on validation set - d1_train_data = train_d1_dataset - d1_validation_data = validation_d1_dataset - d2_train_data = train_d2_dataset - d2_validation_data = validation_d2_dataset - else: - # Standard k-fold cross-validation - validation_num = random_num[pot * fold_i : pot * (fold_i + 1)] - train_num = ( - random_num[: pot * fold_i] + random_num[pot * (fold_i + 1) :] - ) - d1_train_data = train_d1_dataset[train_num] - d1_validation_data = train_d1_dataset[validation_num] - d2_train_data = train_d2_dataset[train_num] - d2_validation_data = train_d2_dataset[validation_num] - - d1_loader_train = DataLoader( - d1_train_data, batch_size=args.batch_size, shuffle=None, pin_memory=True, - ) - d1_loader_validation = DataLoader( - d1_validation_data, batch_size=args.batch_size, shuffle=None, pin_memory=True, - ) - d2_loader_train = DataLoader( - d2_train_data, batch_size=args.batch_size, shuffle=None, pin_memory=True, - ) - d2_loader_validation = DataLoader( - d2_validation_data, batch_size=args.batch_size, shuffle=None, pin_memory=True, - ) - - model = GCNNet(cell_emb_dim=cell_emb_dim).to(device) - - if fold_i == 0: - for name, param in model.named_parameters(): - print_sys(f" Parameter: {name}, Shape: {param.size()}") - total_params = sum(p.numel() for p in model.parameters()) - print_sys(f'Total model parameters: {total_params}') - - optimizer = torch.optim.Adam(model.parameters(), lr=args.lr) - scheduler = StepLR(optimizer, step_size=1000, gamma=0.9) - - best_auc = 0 - loss_history = [] - metric_history = [] - - for epoch in range(1, args.epochs + 1): - avg_loss = train_epoch( - model, device, d1_loader_train, d2_loader_train, - optimizer, epoch, - ) - scheduler.step() - loss_history.append(avg_loss) - - if epoch % args.eval_interval == 0: - T, S, Y = predict(model, device, d1_loader_validation, d2_loader_validation) - m = evaluate_metrics(T, S, Y) - metric_history.append({'epoch': epoch, **m}) - - if m['AUC'] > best_auc: - best_auc = m['AUC'] - best_fpr, best_tpr = metrics.roc_curve(T, S)[:2] - save_path = os.path.join( - args.output_dir, f'best_model_fold{fold_i}.pt' - ) - torch.save(model.state_dict(), save_path) - print_sys(f'** New best validation AUROC: {best_auc:.4f} (saved)') - - print_sys( - f'Epoch {epoch} | Validation | ' - f'AUROC: {m["AUC"]:.4f} PR_AUC: {m["PR_AUC"]:.4f} ' - f'ACC: {m["ACC"]:.4f} BACC: {m["BACC"]:.4f} ' - f'PREC: {m["PREC"]:.4f} TPR: {m["TPR"]:.4f} ' - f'KAPPA: {m["KAPPA"]:.4f}' - ) - - # Save loss curve - loss_csv_path = os.path.join(args.output_dir, f'loss_fold{fold_i}.csv') - loss_df = pd.DataFrame({'epoch': range(1, len(loss_history) + 1), 'avg_loss': loss_history}) - loss_df.to_csv(loss_csv_path, index=False) - - # Plot loss curve - fig, ax = plt.subplots(figsize=(8, 5)) - ax.plot(loss_df['epoch'], loss_df['avg_loss'], linewidth=1.5) - ax.set_xlabel('Epoch') - ax.set_ylabel('Average Loss') - ax.set_title(f'Training Loss - Fold {fold_i + 1}') - ax.grid(True, alpha=0.3) - fig.tight_layout() - fig.savefig(os.path.join(args.output_dir, f'loss_curve_fold{fold_i}.png'), dpi=150) - plt.close(fig) - - # Save validation metrics history (retain the existing CSV schema) - if metric_history: - metric_df = pd.DataFrame(metric_history) - metric_df.to_csv(os.path.join(args.output_dir, f'metrics_fold{fold_i}.csv'), index=False) - - # Plot validation ROC curve (using the selected model's predictions) - fig, ax = plt.subplots(figsize=(6, 6)) - ax.plot(best_fpr, best_tpr, linewidth=2, label=f'AUROC = {best_auc:.4f}') - ax.plot([0, 1], [0, 1], 'k--', linewidth=1, alpha=0.5) - ax.set_xlim([0, 1]) - ax.set_ylim([0, 1.02]) - ax.set_xlabel('False Positive Rate') - ax.set_ylabel('True Positive Rate') - ax.set_title(f'Validation ROC Curve - Fold {fold_i + 1}') - ax.legend(loc='lower right') - ax.grid(True, alpha=0.3) - fig.tight_layout() - fig.savefig(os.path.join(args.output_dir, f'roc_curve_fold{fold_i}.png'), dpi=150) - plt.close(fig) - - # Save fpr/tpr data - fpr_tpr_path = os.path.join(args.output_dir, f'fpr_tpr_fold{fold_i}.pkl') - with open(fpr_tpr_path, 'wb') as fp: - pickle.dump((best_fpr, best_tpr), fp) - - print_sys(f'Fold {fold_i + 1} best validation AUROC: {best_auc:.4f}') - all_fold_aucs.append(best_auc) - - # Summary - print_sys(f'\n{"=" * 60}') - print_sys(f'All folds validation AUROC: {all_fold_aucs}') - print_sys(f'Mean validation AUROC: {np.mean(all_fold_aucs):.4f} +/- {np.std(all_fold_aucs):.4f}') - print_sys('Training complete.') - - -if __name__ == '__main__': - main() +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +Drug Combination Synergy Prediction with PerformerModule cell embeddings. + +Aligned with scLong/drug_combination (DeepDDs) architecture: + - Drug branch: PyG GCNConv + global_max_pool (shared weights for drug1 & drug2) + - Cell branch: PerformerModule pretrained encoder -> MLP + - Fusion: concat(drug1, drug2, cell) -> MLP -> 2-class classification + +Usage: + # Run from the repository root. + python BioMod/drug_combination/train_drug_combination_performer_module.py \ + --checkpoint_path --hyper_params_path \ + --module_data_dir model_dependencies/module_data_all \ + --data_dir data/downstream/final_data/drug_combination_synergy/data_summary +""" + +import argparse +import os +import pickle +import random +import sys +from pathlib import Path +from typing import Dict, List, Optional, Tuple + +import matplotlib +matplotlib.use('Agg') +import matplotlib.pyplot as plt +import networkx as nx +import numpy as np +import pandas as pd +import torch +import torch.nn as nn +import torch.nn.functional as F +from rdkit import Chem +from sklearn import metrics +from tqdm import tqdm +from sklearn.metrics import ( + accuracy_score, + balanced_accuracy_score, + cohen_kappa_score, + confusion_matrix, + precision_score, + recall_score, + roc_auc_score, +) +from torch.optim.lr_scheduler import StepLR + +from torch_geometric import data as DATA +from torch_geometric.data import InMemoryDataset +from torch_geometric.loader import DataLoader +from torch_geometric.nn import GCNConv, global_max_pool as gmp + +# ===================== Path setup ===================== +SCRIPT_DIR = Path(__file__).resolve().parent +PERFORMER_MODULE_ROOT = SCRIPT_DIR.parent +SCLONG_ROOT = PERFORMER_MODULE_ROOT.parent + +REPO_ROOT = Path(__file__).resolve().parents[2] +ASSET_DIR = REPO_ROOT / "model_dependencies" +DOWNSTREAM_DATA_DIR = REPO_ROOT / "data" / "downstream" / "final_data" +WEIGHTS_DIR = REPO_ROOT / "weight" / "0810model" + +if str(PERFORMER_MODULE_ROOT) not in sys.path: + sys.path.insert(0, str(PERFORMER_MODULE_ROOT)) +if str(SCLONG_ROOT) not in sys.path: + sys.path.insert(0, str(SCLONG_ROOT)) + +import importlib.util + +# Import PerformerModule model +_models_path = os.path.join(PERFORMER_MODULE_ROOT, 'models_performer_module.py') +_spec = importlib.util.spec_from_file_location("models_performer_module", _models_path) +_models_mod = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(_models_mod) +PerformerEncoderOnly_Module = _models_mod.PerformerEncoderOnly_Module +sanitize_performer_module_config = _models_mod.sanitize_performer_module_config + +# Import module_dataset +_dataset_path = os.path.join(PERFORMER_MODULE_ROOT, 'module_dataset.py') +_spec2 = importlib.util.spec_from_file_location("module_dataset", _dataset_path) +_module_mod = importlib.util.module_from_spec(_spec2) +_spec2.loader.exec_module(_module_mod) +load_module_data = _module_mod.load_module_data + + +# ===================== Utilities ===================== + +def print_sys(*args, **kwargs): + print(*args, **kwargs, flush=True) + + +def reindex_tensor_universal(tensor, index_positions, dim, filler="0", device="cpu"): + index_tensor = torch.tensor(index_positions, device=device) + tensor_shape = list(tensor.shape) + tensor_shape[dim] = len(index_positions) + index_shape = [1] * len(tensor_shape) + index_shape[dim] = len(index_positions) + index_tensor = index_tensor.view(*index_shape).expand(*tensor_shape) + if filler == "0": + padder_shape = list(tensor_shape) + padder_shape[dim] = 1 + padder = torch.zeros(padder_shape, dtype=tensor.dtype, device=device) + elif filler == "mean": + padder = tensor.mean(dim=dim, keepdim=True) + else: + raise ValueError("filler should be '0' or 'mean'") + expanded_tensor = torch.cat((tensor, padder), dim=dim) + reindexed = torch.gather(expanded_tensor, dim, index_tensor) + return reindexed + + +# ===================== PerformerModule Loading ===================== + +def load_performer_module_model( + checkpoint_path, hyper_params_path, module_data_dir, module_data_path, + gene_list_path, gene_mapping_path, gene2vec_path, device, + shuffle_module_genes=False, shuffle_seed=2021, +): + print_sys(f"Loading PerformerModule model from: {checkpoint_path}") + + with open(hyper_params_path, 'rb') as f: + model_params = pickle.load(f) + model_params.pop('module_embed_chunk_padded_gene_budget', None) + model_params = sanitize_performer_module_config( + model_params, source=hyper_params_path, log_fn=print_sys, + ) + + if gene2vec_path: + model_params['gene2vec_file'] = gene2vec_path + model_params['use_gene2vec'] = model_params.get('use_gene2vec', True) + + if 'module_sets' not in model_params or model_params.get('module_sets') is None: + if module_data_dir or module_data_path: + module_sets = load_module_data( + module_data_path=module_data_path, + module_data_dir=module_data_dir, + gene_list_path=gene_list_path, + gene_mapping_path=gene_mapping_path, + verbose=True, + ) + model_params['module_sets'] = module_sets + else: + raise ValueError( + "Model needs module_sets but neither --module_data_dir " + "nor --module_data_path provided" + ) + + if shuffle_module_genes: + print_sys(f"Shuffling genes across modules with seed={shuffle_seed}") + all_gene_indices = [] + module_sizes = [] + for module in model_params['module_sets']: + all_gene_indices.extend(module) + module_sizes.append(len(module)) + rng = random.Random(shuffle_seed) + rng.shuffle(all_gene_indices) + shuffled_module_sets = [] + start_idx = 0 + for size in module_sizes: + shuffled_module_sets.append(all_gene_indices[start_idx:start_idx + size]) + start_idx += size + model_params['module_sets'] = shuffled_module_sets + print_sys(f"Shuffled {len(all_gene_indices)} genes across {len(shuffled_module_sets)} modules") + + model = PerformerEncoderOnly_Module(**model_params).to(device) + + checkpoint = torch.load(checkpoint_path, map_location=device, weights_only=False) + state_dict = checkpoint.get('model_state_dict', checkpoint) + filtered = { + k: v for k, v in state_dict.items() + if 'go_conv' not in k and 'G_go' not in k.lower() + } + model.load_state_dict(filtered, strict=True) + print_sys("Loaded pretrained PerformerModule weights.") + + model.eval() + for p in model.parameters(): + p.requires_grad = False + + return model, len(model_params['module_sets']), model_params.get('base_dim', 1024) + + +def build_performer_module_gene_mapping(symbol2ens_path, gene_list_path, expr_genes): + gene2ens = pd.read_csv(symbol2ens_path, index_col=0) + symbol2ens = gene2ens["Ensembl gene ID"].to_dict() + + input_genes = [symbol2ens.get(sym, sym) for sym in expr_genes] + input_gene_num = len(input_genes) + input_mapping = {gid: i for i, gid in enumerate(input_genes)} + + with open(gene_list_path, "r") as f: + performer_genes = [line.rstrip("\n") for line in f.readlines()] + performer_genes_pad = performer_genes + ["PAD"] + + performer_index_positions = [ + input_mapping.get(gid, input_gene_num) for gid in performer_genes_pad + ] + intersect = np.intersect1d(input_genes, performer_genes) + print_sys(f"Target and PerformerModule genes intersect: {len(intersect)}") + return input_mapping, performer_index_positions + + +def build_cell_embeddings( + cell_features_df, symbol2ens_path, gene_list_path, gene_mapping_path, + gene2vec_path, checkpoint_path, hyper_params_path, module_data_dir, + module_data_path, device, pool_method="mean", + shuffle_module_genes=False, shuffle_seed=2021, +): + model, num_modules, encoder_embed_dim = load_performer_module_model( + checkpoint_path, hyper_params_path, module_data_dir, module_data_path, + gene_list_path, gene_mapping_path, gene2vec_path, device, + shuffle_module_genes=shuffle_module_genes, shuffle_seed=shuffle_seed, + ) + + gene_columns = list(cell_features_df.columns) + _, performer_index_positions = build_performer_module_gene_mapping( + symbol2ens_path, gene_list_path, gene_columns, + ) + + cell_emb_dict: Dict[str, np.ndarray] = {} + first_emb_dim = -1 + + with torch.no_grad(): + for cell_line_id, row in cell_features_df.iterrows(): + x = torch.tensor(row.values[None, :], dtype=torch.float32, device=device) + x_performer = reindex_tensor_universal( + x, performer_index_positions, dim=1, filler="0", device=device, + ) + out = model(x_performer, mask_ratio=0.0, return_encodings=True) + + if 'module_encodings' in out: + module_enc = out['module_encodings'] + if pool_method == "mean": + cell_emb = module_enc.mean(dim=1) + elif pool_method == "max": + cell_emb = module_enc.max(dim=1)[0] + elif pool_method == "concat": + cell_emb = module_enc.reshape(1, -1) + else: + cell_emb = module_enc.mean(dim=1) + else: + raise ValueError( + f"Expected 'module_encodings' in output, got: {list(out.keys())}" + ) + + cell_emb_np = cell_emb.squeeze(0).cpu().numpy() + cell_emb_dict[cell_line_id] = cell_emb_np + if first_emb_dim < 0: + first_emb_dim = cell_emb_np.shape[0] + + print_sys( + f"Built PerformerModule embeddings for {len(cell_emb_dict)} cell lines, " + f"dim={first_emb_dim}, pool_method={pool_method}" + ) + return cell_emb_dict, first_emb_dim + + +# ===================== Drug Graph (scLong-aligned: edge_index format) ===================== + +def one_of_k_encoding(x, allowable_set): + if x not in allowable_set: + raise Exception(f"input {x} not in allowable set {allowable_set}") + return list(map(lambda s: x == s, allowable_set)) + + +def one_of_k_encoding_unk(x, allowable_set): + if x not in allowable_set: + x = allowable_set[-1] + return list(map(lambda s: x == s, allowable_set)) + + +def atom_features(atom): + return np.array( + one_of_k_encoding_unk(atom.GetSymbol(), [ + 'C', 'N', 'O', 'S', 'F', 'Si', 'P', 'Cl', 'Br', 'Mg', 'Na', 'Ca', + 'Fe', 'As', 'Al', 'I', 'B', 'V', 'K', 'Tl', 'Yb', 'Sb', 'Sn', 'Ag', + 'Pd', 'Co', 'Se', 'Ti', 'Zn', 'H', 'Li', 'Ge', 'Cu', 'Au', 'Ni', 'Cd', + 'In', 'Mn', 'Zr', 'Cr', 'Pt', 'Hg', 'Pb', 'Unknown', + ]) + + one_of_k_encoding(atom.GetDegree(), [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) + + one_of_k_encoding_unk(atom.GetTotalNumHs(), [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) + + one_of_k_encoding_unk(atom.GetImplicitValence(), [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) + + [atom.GetIsAromatic()] + ) + + +ATOM_FEAT_DIM = 78 # 44 + 11 + 11 + 11 + 1 + + +def smile_to_graph(smile): + """Convert SMILES to (c_size, features, edge_index) — identical to scLong's creat_data_DC.py.""" + mol = Chem.MolFromSmiles(smile) + if mol is None: + return None + + c_size = mol.GetNumAtoms() + + features = [] + for atom in mol.GetAtoms(): + feature = atom_features(atom) + features.append(feature / (sum(feature) + 1e-8)) + + edges = [] + for bond in mol.GetBonds(): + edges.append([bond.GetBeginAtomIdx(), bond.GetEndAtomIdx()]) + g = nx.Graph(edges).to_directed() + edge_index = [] + for e1, e2 in g.edges: + edge_index.append([e1, e2]) + + return c_size, features, edge_index + + +# ===================== TestbedDataset (PyG InMemoryDataset, scLong-aligned) ===================== + +class TestbedDataset(InMemoryDataset): + """ + PyG InMemoryDataset for drug combination prediction. + Same pattern as scLong/drug_combination/utils_test.py::TestbedDataset, + but uses PerformerModule cell embeddings instead of scfm/geneformer. + """ + + def __init__( + self, root=str(DOWNSTREAM_DATA_DIR / 'drug_combination_synergy' / 'data_summary'), dataset='drug1', + xd=None, xt=None, y=None, + cell_emb_dict=None, smile_graph=None, + transform=None, pre_transform=None, + ): + self.dataset = dataset + self._xd = xd + self._xt = xt + self._y = y + self._cell_emb_dict = cell_emb_dict + self._smile_graph = smile_graph + super(TestbedDataset, self).__init__(root, transform, pre_transform) + + if os.path.isfile(self.processed_paths[0]): + print_sys(f'Loading preprocessed data: {self.processed_paths[0]}') + self.data, self.slices = torch.load(self.processed_paths[0], weights_only=False) + else: + print_sys(f'Preprocessing data: {self.processed_paths[0]}') + self.process(xd, xt, y, cell_emb_dict, smile_graph) + self.data, self.slices = torch.load(self.processed_paths[0], weights_only=False) + + @property + def raw_file_names(self): + return [] + + @property + def processed_file_names(self): + return [self.dataset + '.pt'] + + def download(self): + pass + + def _download(self): + pass + + def _process(self): + if not os.path.exists(self.processed_dir): + os.makedirs(self.processed_dir) + + def process(self, xd, xt, y, cell_emb_dict, smile_graph): + assert len(xd) == len(xt) == len(y), "xd, xt, y must have the same length" + data_list = [] + skipped = 0 + + for i in range(len(xd)): + smiles = xd[i] + cell_name = xt[i] + label = y[i] + + if smiles not in smile_graph: + skipped += 1 + continue + if cell_name not in cell_emb_dict: + skipped += 1 + continue + + c_size, features, edge_index = smile_graph[smiles] + + GCNData = DATA.Data( + x=torch.Tensor(features), + edge_index=torch.LongTensor(edge_index).transpose(1, 0), + y=torch.Tensor([label]), + ) + cell_emb = cell_emb_dict[cell_name] + GCNData.cell = torch.FloatTensor([cell_emb]) + GCNData.__setitem__('c_size', torch.LongTensor([c_size])) + data_list.append(GCNData) + + if skipped > 0: + print_sys(f'Skipped {skipped} samples (missing graph or cell embedding)') + + if self.pre_filter is not None: + data_list = [d for d in data_list if self.pre_filter(d)] + if self.pre_transform is not None: + data_list = [self.pre_transform(d) for d in data_list] + + print_sys(f'Processed {len(data_list)} samples. Saving to {self.processed_paths[0]}') + data, slices = self.collate(data_list) + torch.save((data, slices), self.processed_paths[0]) + + +# ===================== Model: GCNNet (PyG, scLong-aligned) ===================== + +class GCNNet(torch.nn.Module): + """ + GCN-based drug combination model aligned with scLong/drug_combination/models/gcn.py. + + Drug branch: 3-layer GCNConv (78->78->156->312) + global_max_pool + 2 FC layers + Drug1 and drug2 share weights (drug2 uses drug1_conv* layers, same as scLong). + Cell branch: PerformerModule embedding -> MLP -> output_dim + Fusion: concat(drug1, drug2, cell) -> fc1(256) -> fc2(128) -> out(2) + """ + + def __init__( + self, n_output=2, num_features_xd=78, cell_emb_dim=1024, + output_dim=128, dropout=0.2, + ): + super(GCNNet, self).__init__() + self.relu = nn.ReLU() + self.dropout = nn.Dropout(dropout) + self.n_output = n_output + + # Drug graph branch (shared weights for drug1 & drug2) + self.drug1_conv1 = GCNConv(num_features_xd, num_features_xd) + self.drug1_conv2 = GCNConv(num_features_xd, num_features_xd * 2) + self.drug1_conv3 = GCNConv(num_features_xd * 2, num_features_xd * 4) + self.drug1_fc_g1 = nn.Linear(num_features_xd * 4, num_features_xd * 2) + self.drug1_fc_g2 = nn.Linear(num_features_xd * 2, output_dim) + + # Cell branch (PerformerModule embedding -> MLP) + self.reduction = nn.Sequential( + nn.Linear(cell_emb_dim, 512), + nn.ReLU(), + nn.Linear(512, 256), + nn.ReLU(), + nn.Linear(256, output_dim), + ) + + # Fusion layers + self.fc1 = nn.Linear(3 * output_dim, 256) + self.fc2 = nn.Linear(256, 128) + self.out = nn.Linear(128, self.n_output) + + def forward(self, data1, data2): + x1, edge_index1, batch1, cell = ( + data1.x, data1.edge_index, data1.batch, data1.cell + ) + x2, edge_index2, batch2 = data2.x, data2.edge_index, data2.batch + + # Drug1 branch + x1 = self.relu(self.drug1_conv1(x1, edge_index1)) + x1 = self.relu(self.drug1_conv2(x1, edge_index1)) + x1 = self.relu(self.drug1_conv3(x1, edge_index1)) + x1 = gmp(x1, batch1) + x1 = self.relu(self.drug1_fc_g1(x1)) + x1 = self.dropout(x1) + x1 = self.drug1_fc_g2(x1) + x1 = self.dropout(x1) + + # Drug2 branch (shared weights — uses drug1_conv* layers, same as scLong) + x2 = self.relu(self.drug1_conv1(x2, edge_index2)) + x2 = self.relu(self.drug1_conv2(x2, edge_index2)) + x2 = self.relu(self.drug1_conv3(x2, edge_index2)) + x2 = gmp(x2, batch2) + x2 = self.relu(self.drug1_fc_g1(x2)) + x2 = self.dropout(x2) + x2 = self.drug1_fc_g2(x2) + x2 = self.dropout(x2) + + # Cell branch + cell_vector = F.normalize(cell, 2, 1) + cell_vector = self.reduction(cell_vector) + + # Fusion + xc = torch.cat((x1, x2, cell_vector), 1) + xc = self.relu(self.fc1(xc)) + xc = self.dropout(xc) + xc = self.relu(self.fc2(xc)) + xc = self.dropout(xc) + out = self.out(xc) + return out + + +# ===================== Training & Evaluation ===================== + +def train_epoch(model, device, drug1_loader, drug2_loader, optimizer, epoch): + """Train one epoch. Returns average loss.""" + model.train() + total_loss = 0.0 + n_batches = 0 + pbar = tqdm( + zip(drug1_loader, drug2_loader), + total=len(drug1_loader), + desc=f'Epoch {epoch}', + leave=True, + ) + for data1, data2 in pbar: + data1 = data1.to(device) + data2 = data2.to(device) + y = data1.y.view(-1, 1).long().to(device).squeeze(1) + optimizer.zero_grad() + output = model(data1, data2) + loss = F.cross_entropy(output, y) + loss.backward() + optimizer.step() + total_loss += loss.item() + n_batches += 1 + pbar.set_postfix(loss=f'{loss.item():.4f}') + avg_loss = total_loss / max(n_batches, 1) + return avg_loss + + +def predict(model, device, drug1_loader, drug2_loader): + model.eval() + total_preds = torch.Tensor() + total_labels = torch.Tensor() + total_prelabels = torch.Tensor() + print_sys(f'Make prediction for {len(drug1_loader.dataset)} samples...') + + with torch.no_grad(): + for data in zip(drug1_loader, drug2_loader): + data1 = data[0].to(device) + data2 = data[1].to(device) + output = model(data1, data2) + ys = F.softmax(output, 1).to('cpu').data.numpy() + predicted_labels = list(map(lambda x: np.argmax(x), ys)) + predicted_scores = list(map(lambda x: x[1], ys)) + total_preds = torch.cat((total_preds, torch.Tensor(predicted_scores)), 0) + total_prelabels = torch.cat( + (total_prelabels, torch.Tensor(predicted_labels)), 0 + ) + total_labels = torch.cat((total_labels, data1.y.view(-1, 1).cpu()), 0) + + return ( + total_labels.numpy().flatten(), + total_preds.numpy().flatten(), + total_prelabels.numpy().flatten(), + ) + + +def evaluate_metrics(T, S, Y): + """Compute all metrics matching scLong's evaluation.""" + try: + AUC = roc_auc_score(T, S) + except Exception: + AUC = 0.0 + precision_arr, recall_arr, _ = metrics.precision_recall_curve(T, S) + PR_AUC = metrics.auc(recall_arr, precision_arr) + BACC = balanced_accuracy_score(T, Y) + tn, fp, fn, tp = confusion_matrix(T, Y).ravel() + TPR = tp / (tp + fn) + PREC = precision_score(T, Y) + ACC = accuracy_score(T, Y) + KAPPA = cohen_kappa_score(T, Y) + RECALL = recall_score(T, Y) + return { + 'AUC': AUC, 'PR_AUC': PR_AUC, 'ACC': ACC, 'BACC': BACC, + 'PREC': PREC, 'TPR': TPR, 'KAPPA': KAPPA, 'RECALL': RECALL, + } + + +# ===================== Cell Feature Loading ===================== + +def load_cell_features(cell_file): + """Load cell features CSV. First row is header (gene_id), second is transcript_id (skipped).""" + df = pd.read_csv(cell_file) + df = df.iloc[1:].copy() # skip transcript_id row + df = df.set_index(df.columns[0]) + df = df.astype(float) + # Clean cell names: "X22RV1_PROSTATE" -> "22RV1" + df.index = [ + idx.lstrip('X').split('_')[0] if '_' in idx else idx + for idx in df.index + ] + return df + + +# ===================== Main ===================== + +def main(): + parser = argparse.ArgumentParser( + description='Drug Combination Synergy Prediction with PerformerModule' + ) + + # PerformerModule model args + parser.add_argument('--checkpoint_path', type=str, required=True, + help='Path to PerformerModule pretrained checkpoint') + parser.add_argument('--hyper_params_path', type=str, required=True, + help='Path to PerformerModule hyper parameters pkl') + parser.add_argument('--module_data_dir', type=str, default=None, + help='Directory containing module data files') + parser.add_argument('--module_data_path', type=str, default=None, + help='Path to single module data file') + parser.add_argument('--gene_list_path', type=str, + default=str(ASSET_DIR / 'selected_genes_27k.txt')) + parser.add_argument('--gene_mapping_path', type=str, + default=str(ASSET_DIR / 'gene2ens.csv')) + parser.add_argument('--gene2vec_path', type=str, + default=str(ASSET_DIR / 'selected_gene2vec_27k.npy')) + parser.add_argument('--symbol2ens_path', type=str, + default=str(ASSET_DIR / 'gene2ens.csv')) + parser.add_argument('--pool_method', type=str, default='mean', + choices=['mean', 'max', 'concat']) + parser.add_argument('--shuffle_module_genes', action='store_true', + help='Disrupt the gene allocation within the module (used for comparative experiments to verify the effectiveness of module division)') + parser.add_argument('--shuffle_seed', type=int, default=2021, + help='The random seed used when scrambling the module gene (needs to be consistent with the value used during pre-training)') + + # Data args + parser.add_argument('--data_dir', type=str, + default=str(DOWNSTREAM_DATA_DIR / 'drug_combination_synergy' / 'data_summary')) + parser.add_argument('--label_file', type=str, default='new_labels_0_10.csv') + parser.add_argument('--cell_file', type=str, default='new_cell_features_954.csv') + parser.add_argument('--validation_label_file', '--test_label_file', + dest='validation_label_file', type=str, + default='test_data/independent_input.csv', + help='Validation labels used for checkpoint selection; --test_label_file is a legacy alias') + parser.add_argument('--validation_cell_file', '--test_cell_file', + dest='validation_cell_file', type=str, + default='test_data/independent_cell_features_954.csv', + help='Validation cell features; --test_cell_file is a legacy alias') + + # Training args + parser.add_argument('--batch_size', type=int, default=256) + parser.add_argument('--lr', type=float, default=1e-4) + parser.add_argument('--epochs', type=int, default=100) + parser.add_argument('--n_folds', type=int, default=5) + parser.add_argument('--domain_gen', action='store_true', default=True, + help='Domain generalization: train full set, validate on a separate set for checkpoint selection') + parser.add_argument('--no_domain_gen', dest='domain_gen', action='store_false') + parser.add_argument('--eval_interval', type=int, default=5) + parser.add_argument('--output_dir', type=str, + default=os.path.join(SCRIPT_DIR, 'results')) + parser.add_argument('--seed', type=int, default=42) + parser.add_argument('--cache_emb', action='store_true', default=True, + help='Cache PerformerModule cell embeddings to .npy') + parser.add_argument('--no_cache_emb', dest='cache_emb', action='store_false') + parser.add_argument('--force_preprocess', action='store_true', default=False, + help='Force re-creation of PyG .pt datasets') + + args = parser.parse_args() + + # Device + device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') + print_sys(f'Using device: {device}') + + # Seed + random.seed(args.seed) + np.random.seed(args.seed) + torch.manual_seed(args.seed) + + os.makedirs(args.output_dir, exist_ok=True) + + # -------- Step 1: Build PerformerModule cell embeddings -------- + train_cell_file = os.path.join(args.data_dir, args.cell_file) + train_cell_df = load_cell_features(train_cell_file) + print_sys(f'Loaded train cell features: {train_cell_df.shape}') + + validation_cell_file = os.path.join(args.data_dir, args.validation_cell_file) + validation_cell_df = load_cell_features(validation_cell_file) + print_sys(f'Loaded validation cell features: {validation_cell_df.shape}') + + # Combine all cell lines and remove duplicates + all_cell_df = pd.concat([train_cell_df, validation_cell_df]) + all_cell_df = all_cell_df[~all_cell_df.index.duplicated(keep='first')] + print_sys(f'Total unique cell lines: {len(all_cell_df)}') + + # Cache embeddings + cache_path = os.path.join(args.output_dir, f'cell_emb_cache_{args.pool_method}.npy') + cache_dim_path = os.path.join( + args.output_dir, f'cell_emb_dim_{args.pool_method}.txt' + ) + + if args.cache_emb and os.path.exists(cache_path) and not args.force_preprocess: + print_sys(f'Loading cached cell embeddings from {cache_path}') + cell_emb_dict = np.load(cache_path, allow_pickle=True).item() + with open(cache_dim_path) as f: + cell_emb_dim = int(f.read().strip()) + else: + cell_emb_dict, cell_emb_dim = build_cell_embeddings( + all_cell_df, args.symbol2ens_path, args.gene_list_path, + args.gene_mapping_path, args.gene2vec_path, args.checkpoint_path, + args.hyper_params_path, args.module_data_dir, args.module_data_path, + device, args.pool_method, + shuffle_module_genes=args.shuffle_module_genes, + shuffle_seed=args.shuffle_seed, + ) + if args.cache_emb: + np.save(cache_path, cell_emb_dict) + with open(cache_dim_path, 'w') as f: + f.write(str(cell_emb_dim)) + print_sys(f'Cached cell embeddings to {cache_path}') + + # -------- Step 2: Build SMILES graphs -------- + train_label_path = os.path.join(args.data_dir, args.label_file) + train_labels = pd.read_csv(train_label_path) + print_sys(f'Train labels: {len(train_labels)} samples') + + validation_label_path = os.path.join(args.data_dir, args.validation_label_file) + validation_labels = pd.read_csv(validation_label_path) + print_sys(f'Validation labels: {len(validation_labels)} samples') + + all_smiles = set( + train_labels['drug1'].tolist() + train_labels['drug2'].tolist() + + validation_labels['drug1'].tolist() + validation_labels['drug2'].tolist() + ) + smile_graph = {} + failed = [] + for smi in all_smiles: + result = smile_to_graph(smi) + if result is not None: + smile_graph[smi] = result + else: + failed.append(smi) + if failed: + print_sys(f'Warning: {len(failed)} SMILES failed to parse') + print_sys(f'Built graphs for {len(smile_graph)} unique SMILES') + + # -------- Step 3: Create PyG datasets -------- + # Delete cached .pt files if force_preprocess + if args.force_preprocess: + processed_dir = os.path.join(args.data_dir, 'processed') + if os.path.exists(processed_dir): + import shutil + shutil.rmtree(processed_dir) + print_sys(f'Removed cached processed data: {processed_dir}') + + # Train datasets (drug1 and drug2 separate, scLong pattern) + drug1_train = train_labels['drug1'].tolist() + drug2_train = train_labels['drug2'].tolist() + cells_train = train_labels['cell'].tolist() + labels_train = train_labels['label'].tolist() + + train_d1_dataset = TestbedDataset( + root=args.data_dir, dataset='train_drug1_performer_module', + xd=drug1_train, xt=cells_train, y=labels_train, + cell_emb_dict=cell_emb_dict, smile_graph=smile_graph, + ) + train_d2_dataset = TestbedDataset( + root=args.data_dir, dataset='train_drug2_performer_module', + xd=drug2_train, xt=cells_train, y=labels_train, + cell_emb_dict=cell_emb_dict, smile_graph=smile_graph, + ) + + # Validation datasets (separate set used for checkpoint selection) + # Preserve legacy test_* cache IDs to reuse existing preprocessed datasets. + drug1_validation = validation_labels['drug1'].tolist() + drug2_validation = validation_labels['drug2'].tolist() + cells_validation = validation_labels['cell'].tolist() + labels_validation = validation_labels['label'].tolist() + + validation_d1_dataset = TestbedDataset( + root=args.data_dir, dataset='test_drug1_performer_module', + xd=drug1_validation, xt=cells_validation, y=labels_validation, + cell_emb_dict=cell_emb_dict, smile_graph=smile_graph, + ) + validation_d2_dataset = TestbedDataset( + root=args.data_dir, dataset='test_drug2_performer_module', + xd=drug2_validation, xt=cells_validation, y=labels_validation, + cell_emb_dict=cell_emb_dict, smile_graph=smile_graph, + ) + + # -------- Step 4: Training (5-fold, scLong-aligned) -------- + lenth = len(train_d1_dataset) + pot = lenth // args.n_folds + random_num = random.sample(range(lenth), lenth) + all_fold_aucs = [] + + for fold_i in range(args.n_folds): + print_sys(f'\n{"=" * 60}') + print_sys(f'Fold {fold_i + 1}/{args.n_folds}') + print_sys(f'{"=" * 60}') + + if args.domain_gen: + # Domain generalization: train on full training set, select on validation set + d1_train_data = train_d1_dataset + d1_validation_data = validation_d1_dataset + d2_train_data = train_d2_dataset + d2_validation_data = validation_d2_dataset + else: + # Standard k-fold cross-validation + validation_num = random_num[pot * fold_i : pot * (fold_i + 1)] + train_num = ( + random_num[: pot * fold_i] + random_num[pot * (fold_i + 1) :] + ) + d1_train_data = train_d1_dataset[train_num] + d1_validation_data = train_d1_dataset[validation_num] + d2_train_data = train_d2_dataset[train_num] + d2_validation_data = train_d2_dataset[validation_num] + + d1_loader_train = DataLoader( + d1_train_data, batch_size=args.batch_size, shuffle=None, pin_memory=True, + ) + d1_loader_validation = DataLoader( + d1_validation_data, batch_size=args.batch_size, shuffle=None, pin_memory=True, + ) + d2_loader_train = DataLoader( + d2_train_data, batch_size=args.batch_size, shuffle=None, pin_memory=True, + ) + d2_loader_validation = DataLoader( + d2_validation_data, batch_size=args.batch_size, shuffle=None, pin_memory=True, + ) + + model = GCNNet(cell_emb_dim=cell_emb_dim).to(device) + + if fold_i == 0: + for name, param in model.named_parameters(): + print_sys(f" Parameter: {name}, Shape: {param.size()}") + total_params = sum(p.numel() for p in model.parameters()) + print_sys(f'Total model parameters: {total_params}') + + optimizer = torch.optim.Adam(model.parameters(), lr=args.lr) + scheduler = StepLR(optimizer, step_size=1000, gamma=0.9) + + best_auc = 0 + loss_history = [] + metric_history = [] + + for epoch in range(1, args.epochs + 1): + avg_loss = train_epoch( + model, device, d1_loader_train, d2_loader_train, + optimizer, epoch, + ) + scheduler.step() + loss_history.append(avg_loss) + + if epoch % args.eval_interval == 0: + T, S, Y = predict(model, device, d1_loader_validation, d2_loader_validation) + m = evaluate_metrics(T, S, Y) + metric_history.append({'epoch': epoch, **m}) + + if m['AUC'] > best_auc: + best_auc = m['AUC'] + best_fpr, best_tpr = metrics.roc_curve(T, S)[:2] + save_path = os.path.join( + args.output_dir, f'best_model_fold{fold_i}.pt' + ) + torch.save(model.state_dict(), save_path) + print_sys(f'** New best validation AUROC: {best_auc:.4f} (saved)') + + print_sys( + f'Epoch {epoch} | Validation | ' + f'AUROC: {m["AUC"]:.4f} PR_AUC: {m["PR_AUC"]:.4f} ' + f'ACC: {m["ACC"]:.4f} BACC: {m["BACC"]:.4f} ' + f'PREC: {m["PREC"]:.4f} TPR: {m["TPR"]:.4f} ' + f'KAPPA: {m["KAPPA"]:.4f}' + ) + + # Save loss curve + loss_csv_path = os.path.join(args.output_dir, f'loss_fold{fold_i}.csv') + loss_df = pd.DataFrame({'epoch': range(1, len(loss_history) + 1), 'avg_loss': loss_history}) + loss_df.to_csv(loss_csv_path, index=False) + + # Plot loss curve + fig, ax = plt.subplots(figsize=(8, 5)) + ax.plot(loss_df['epoch'], loss_df['avg_loss'], linewidth=1.5) + ax.set_xlabel('Epoch') + ax.set_ylabel('Average Loss') + ax.set_title(f'Training Loss - Fold {fold_i + 1}') + ax.grid(True, alpha=0.3) + fig.tight_layout() + fig.savefig(os.path.join(args.output_dir, f'loss_curve_fold{fold_i}.png'), dpi=150) + plt.close(fig) + + # Save validation metrics history (retain the existing CSV schema) + if metric_history: + metric_df = pd.DataFrame(metric_history) + metric_df.to_csv(os.path.join(args.output_dir, f'metrics_fold{fold_i}.csv'), index=False) + + # Plot validation ROC curve (using the selected model's predictions) + fig, ax = plt.subplots(figsize=(6, 6)) + ax.plot(best_fpr, best_tpr, linewidth=2, label=f'AUROC = {best_auc:.4f}') + ax.plot([0, 1], [0, 1], 'k--', linewidth=1, alpha=0.5) + ax.set_xlim([0, 1]) + ax.set_ylim([0, 1.02]) + ax.set_xlabel('False Positive Rate') + ax.set_ylabel('True Positive Rate') + ax.set_title(f'Validation ROC Curve - Fold {fold_i + 1}') + ax.legend(loc='lower right') + ax.grid(True, alpha=0.3) + fig.tight_layout() + fig.savefig(os.path.join(args.output_dir, f'roc_curve_fold{fold_i}.png'), dpi=150) + plt.close(fig) + + # Save fpr/tpr data + fpr_tpr_path = os.path.join(args.output_dir, f'fpr_tpr_fold{fold_i}.pkl') + with open(fpr_tpr_path, 'wb') as fp: + pickle.dump((best_fpr, best_tpr), fp) + + print_sys(f'Fold {fold_i + 1} best validation AUROC: {best_auc:.4f}') + all_fold_aucs.append(best_auc) + + # Summary + print_sys(f'\n{"=" * 60}') + print_sys(f'All folds validation AUROC: {all_fold_aucs}') + print_sys(f'Mean validation AUROC: {np.mean(all_fold_aucs):.4f} +/- {np.std(all_fold_aucs):.4f}') + print_sys('Training complete.') + + +if __name__ == '__main__': + main() diff --git a/BioMod/drug_response/cmd.txt b/BioMod/drug_response/cmd.txt index 77d31e810268f1215c6ea0f3ae6cf616d14d1e34..88b89a1c07ecd1132eab366c1389dc73735f6b87 100644 --- a/BioMod/drug_response/cmd.txt +++ b/BioMod/drug_response/cmd.txt @@ -1,10 +1,10 @@ -python train_drug_performer_module.py \ - --gpu_id 0 \ - --data_path ../../drug_response_pytorch/data \ - --checkpoint_path ../performer_module_multiply_all_module_48Mcell/performer_module_multiply_all_module_48Mcell_epoch2.pth \ - --hyper_params_path ../performer_module_multiply_all_module_48Mcell/performer_module_multiply_all_module_48Mcell.pkl \ - --epochs 300 \ - --module_data_dir ../module_data_all \ - --save_dir ./performer_module_multiply_all_module_48Mcell \ - --fusion_method multiply \ - --pool_method max +# Run from the repository root. +python BioMod/drug_response/train_drug_performer_module.py \ + --gpu_id 0 \ + --data_path data/downstream/final_data/single_drug_response/data \ + --checkpoint_path weight/0810model/mae_tost_70M_75mask_ec2_epoch0.pth \ + --hyper_params_path weight/0810model/mae_tost_70M_75mask_ec2.pkl \ + --epochs 300 \ + --module_data_dir model_dependencies/module_data_all \ + --save_dir tutorial_runs/drug_response \ + --pool_method max diff --git a/BioMod/drug_response/train_drug_performer_module.py b/BioMod/drug_response/train_drug_performer_module.py index 1b0e0cbb5c92c44311035a048b927ee4140a492b..9d049e28baa14805b4060d15605135e14964279d 100644 --- a/BioMod/drug_response/train_drug_performer_module.py +++ b/BioMod/drug_response/train_drug_performer_module.py @@ -1,1065 +1,1075 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" -Drug response prediction: using drug molecular structure + PerformerModule cell embedding - -Architecture: -- Drug branch: 3-layer GCN + residual connection + double pooling -- Cell branch: extract module-level features from pre-trained PerformerModule -- Fusion layer: attention mechanism + deep MLP -- Optimization strategy: AdamW + cosine annealing + early stopping - -How to use: - python train_drug_performer_module.py --gpu_id 0 --data_path ../drug_response_pytorch/data --epochs 300 -""" - -import argparse -import os -import random -import sys -import pickle -from typing import Dict, List, Tuple, Union, Optional - -import hickle as hkl -import numpy as np -import pandas as pd -import scipy.sparse as sp -import torch -import torch.nn as nn -from scipy.stats import pearsonr -from torch.utils.data import DataLoader, Dataset - -# Import PerformerModule model -project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -if project_root not in sys.path: - sys.path.insert(0, project_root) -sys.path.insert(0, os.path.dirname(project_root)) # Add scLong root directory - -# Use direct path import to avoid conflicts with files with the same name -import importlib.util -performer_module_dir = project_root - -# import models_performer_module -models_path = os.path.join(performer_module_dir, 'models_performer_module.py') -if not os.path.exists(models_path): - raise FileNotFoundError(f"Model file not found: {models_path}") -spec = importlib.util.spec_from_file_location("models_performer_module", models_path) -models_performer_module = importlib.util.module_from_spec(spec) -spec.loader.exec_module(models_performer_module) -PerformerEncoderOnly_Module = models_performer_module.PerformerEncoderOnly_Module -sanitize_performer_module_config = models_performer_module.sanitize_performer_module_config - -# Import module_dataset -dataset_path = os.path.join(performer_module_dir, 'module_dataset.py') -if not os.path.exists(dataset_path): - raise FileNotFoundError(f"Dataset file not found: {dataset_path}") -spec = importlib.util.spec_from_file_location("module_dataset", dataset_path) -module_dataset_module = importlib.util.module_from_spec(spec) -spec.loader.exec_module(module_dataset_module) -load_module_data = module_dataset_module.load_module_data - - -# ===================== Utility functions ===================== - -def print_sys(*args, **kwargs): - """Print and refresh output""" - print(*args, **kwargs, flush=True) - - -def reindex_tensor_universal( - tensor: torch.Tensor, - index_positions: List[int], - dim: int, - filler: str = "0", - device: Union[torch.device, str] = "cpu", -) -> torch.Tensor: - """Universal reordering operation""" - index_tensor = torch.tensor(index_positions, device=device) - tensor_shape = list(tensor.shape) - tensor_shape[dim] = len(index_positions) - index_shape = [1] * len(tensor_shape) - index_shape[dim] = len(index_positions) - index_tensor = index_tensor.view(*index_shape).expand(*tensor_shape) - - if filler == "0": - padder_shape = list(tensor_shape) - padder_shape[dim] = 1 - padder = torch.zeros(padder_shape, dtype=tensor.dtype, device=device) - elif filler == "mean": - padder = tensor.mean(dim=dim, keepdim=True) - else: - raise ValueError("filler should be '0' or 'mean'") - - expanded_tensor = torch.cat((tensor, padder), dim=dim) - reindexed = torch.gather(expanded_tensor, dim, index_tensor) - return reindexed - - -# ===================== PerformerModule related ===================== - -def load_performer_module_model( - checkpoint_path: str, - hyper_params_path: str, - module_data_dir: Optional[str], - module_data_path: Optional[str], - gene_list_path: str, - gene_mapping_path: str, - gene2vec_path: str, - device: torch.device, - shuffle_module_genes: bool = False, - seed: int = 0, -) -> PerformerEncoderOnly_Module: - """Load a pretrained PerformerModule model""" - print_sys(f"Loading PerformerModule model from: {checkpoint_path}") - - # Load model hyperparameters - print_sys(f"Loading hyper parameters from: {hyper_params_path}") - with open(hyper_params_path, 'rb') as f: - model_params = pickle.load(f) - model_params.pop('module_embed_chunk_padded_gene_budget', None) - model_params = sanitize_performer_module_config( - model_params, - source=hyper_params_path, - log_fn=print_sys, - ) - - # Update gene2vec path - if gene2vec_path: - model_params['gene2vec_file'] = gene2vec_path - - # Read fusion_method from pkl to ensure consistency with pre-training - fusion_method = model_params.get('fusion_method', 'multiply') - print_sys(f"Using fusion_method from pkl: {fusion_method}") - - # use_gene2vec strictly follows the pre-training pkl to avoid conflicts caused by downstream coverage. - pkl_use_gene2vec = model_params.get('use_gene2vec', True) - model_params['use_gene2vec'] = pkl_use_gene2vec - print_sys(f"use_gene2vec={pkl_use_gene2vec} (from PKL)") - - # Load module data (if not present in hyperparameters) - if 'module_sets' not in model_params or model_params.get('module_sets') is None: - if module_data_dir or module_data_path: - print_sys("Loading module data...") - module_sets = load_module_data( - module_data_path=module_data_path, - module_data_dir=module_data_dir, - gene_list_path=gene_list_path, - gene_mapping_path=gene_mapping_path, - verbose=True - ) - model_params['module_sets'] = module_sets - else: - raise ValueError("Model requires module_sets, but --module_data_dir or --module_data_path was not provided") - else: - module_sets = model_params['module_sets'] - print_sys(f"Using module_sets from hyper parameters ({len(module_sets)} modules)") - - # If shuffle_module_genes is enabled, disrupt the gene distribution within the module. - if shuffle_module_genes: - print_sys( - f"Applying downstream-only extra module reshuffle (seed={seed}). " - f"Pretraining-time module_sets from PKL will be reshuffled again." - ) - # Collect all gene indices - all_gene_indices = [] - module_sizes = [] - for module in module_sets: - all_gene_indices.extend(module) - module_sizes.append(len(module)) - - # Scramble the gene index using a fixed random seed - rng = random.Random(seed) - rng.shuffle(all_gene_indices) - - # Redistribute genes according to original module size - shuffled_module_sets = [] - start_idx = 0 - for size in module_sizes: - shuffled_module_sets.append(all_gene_indices[start_idx:start_idx + size]) - start_idx += size - module_sets = shuffled_module_sets - model_params['module_sets'] = module_sets - print_sys(f"Shuffled {len(all_gene_indices)} genes across {len(module_sets)} modules") - - print_sys(f"PerformerModule config: base_dim={model_params.get('base_dim', 'N/A')}, " - f"num_modules={len(module_sets)}") - - # Build model - model = PerformerEncoderOnly_Module(**model_params).to(device) - - # Load weights - print_sys(f"Loading checkpoint from: {checkpoint_path}") - checkpoint = torch.load(checkpoint_path, map_location=device) - - # Handle parameters in checkpoint - if 'model_state_dict' in checkpoint: - state_dict = checkpoint['model_state_dict'] - else: - state_dict = checkpoint - - # Filter out possible GO related parameters - filtered_state_dict = {} - for k, v in state_dict.items(): - if 'go_conv' not in k and 'G_go' not in k.lower(): - filtered_state_dict[k] = v - - # Use strict=True to ensure the weights match exactly (except for filtered GO parameters) - model.load_state_dict(filtered_state_dict, strict=True) - - print_sys("Loaded pretrained PerformerModule weights.") - - # freeze model - model.eval() - for p in model.parameters(): - p.requires_grad = False - - return model, len(module_sets), model_params.get('base_dim', 1024) - - -def build_performer_module_gene_mapping( - symbol2ens_path: str, - gene_list_path: str, - expr_genes: List[str], -) -> Tuple[Dict[str, int], List[int]]: - """Build a gene map""" - gene2ens = pd.read_csv(symbol2ens_path, index_col=0) - symbol2ens = gene2ens["Ensembl gene ID"].to_dict() - - input_genes = [symbol2ens.get(sym, sym) for sym in expr_genes] - input_gene_num = len(input_genes) - input_mapping = {gid: i for i, gid in enumerate(input_genes)} - - with open(gene_list_path, "r") as f: - performer_genes = [line.rstrip("\n") for line in f.readlines()] - performer_genes_pad = performer_genes + ["PAD"] - - performer_index_positions = [ - input_mapping.get(gid, input_gene_num) for gid in performer_genes_pad - ] - intersect = np.intersect1d(input_genes, performer_genes) - print_sys(f"Target and PerformerModule genes intersect: {len(intersect)}") - return input_mapping, performer_index_positions - - -def build_cell_embeddings_with_performer_module( - gexpr_feature: pd.DataFrame, - symbol2ens_path: str, - gene_list_path: str, - gene_mapping_path: str, - gene2vec_path: str, - checkpoint_path: str, - hyper_params_path: str, - module_data_dir: Optional[str], - module_data_path: Optional[str], - device: torch.device, - pool_method: str = "mean", - shuffle_module_genes: bool = False, - seed: int = 0, -) -> Tuple[Dict[str, np.ndarray], int]: - """ - Use PerformerModule to construct embedding for each cell line - - Args: - pool_method: pooling method, 'mean', 'max', 'cls', or 'concat' - - 'mean': average all modules (default) - - 'max': take the maximum value for all modules - - 'cls': only use CLS token (if available) - - 'concat': concatenate all modules (the dimension will be very large) - use_gene2vec: automatically restored by pre-trained PKL, no longer covered downstream - """ - model, num_modules, encoder_embed_dim = load_performer_module_model( - checkpoint_path=checkpoint_path, - hyper_params_path=hyper_params_path, - module_data_dir=module_data_dir, - module_data_path=module_data_path, - gene_list_path=gene_list_path, - gene_mapping_path=gene_mapping_path, - gene2vec_path=gene2vec_path, - device=device, - shuffle_module_genes=shuffle_module_genes, - seed=seed, - ) - - _, performer_index_positions = build_performer_module_gene_mapping( - symbol2ens_path=symbol2ens_path, - gene_list_path=gene_list_path, - expr_genes=list(gexpr_feature.columns), - ) - - cell_emb_dict: Dict[str, np.ndarray] = {} - first_emb_dim: int = -1 - - with torch.no_grad(): - for cell_line_id, row in gexpr_feature.iterrows(): - x = torch.tensor(row.values[None, :], dtype=torch.float32, device=device) - x_performer = reindex_tensor_universal( - x, performer_index_positions, dim=1, filler="0", device=device - ) - - # Extract features through encoder (without masking, mask_ratio=0.0) - out = model(x_performer, mask_ratio=0.0, return_encodings=True) - - # Extract module encodings - if 'module_encodings' in out: - module_encodings = out['module_encodings'] # (1, num_modules, encoder_embed_dim) - - # Processed according to pooling method - if pool_method == "cls": - if model.use_cls: - # Use CLS token (in latent, but module_encodings does not contain CLS) - # Need to get the complete latent from forward_encoder - enc_out = model.forward_encoder(x_performer, mask_ratio=0.0) - latent = enc_out[0] - if model.use_cls: - cell_emb = latent[:, 0, :] # (1, encoder_embed_dim) - else: - print_sys("Warning: pool_method='cls' but model has no CLS token, using 'mean' instead") - cell_emb = module_encodings.mean(dim=1) - else: - print_sys("Warning: pool_method='cls' but model has no CLS token, using 'mean' instead") - cell_emb = module_encodings.mean(dim=1) - elif pool_method == "mean": - cell_emb = module_encodings.mean(dim=1) # (1, encoder_embed_dim) - elif pool_method == "max": - cell_emb = module_encodings.max(dim=1)[0] # (1, encoder_embed_dim) - elif pool_method == "concat": - cell_emb = module_encodings.reshape(1, -1) # (1, num_modules * encoder_embed_dim) - else: - raise ValueError(f"Unknown pool_method: {pool_method}") - else: - raise ValueError(f"Expected 'module_encodings' in output, got: {list(out.keys())}") - - cell_emb_np = cell_emb.squeeze(0).cpu().numpy() - cell_emb_dict[cell_line_id] = cell_emb_np - if first_emb_dim < 0: - first_emb_dim = cell_emb_np.shape[0] - - print_sys( - f"Built PerformerModule embeddings for {len(cell_emb_dict)} cell lines, " - f"dim={first_emb_dim}, pool_method={pool_method}" - ) - return cell_emb_dict, first_emb_dim - - -# ===================== Data processing ===================== - -TCGA_LABEL_SET = [ - "ALL", "BLCA", "BRCA", "CESC", "DLBC", "LIHC", "LUAD", "ESCA", "GBM", - "HNSC", "KIRC", "LAML", "LCML", "LGG", "LUSC", "MESO", "MM", "NB", - "OV", "PAAD", "SCLC", "SKCM", "STAD", "THCA", "COAD/READ", -] - - -def normalize_adj(adj: np.ndarray) -> np.ndarray: - """Normalized adjacency matrix""" - adj = adj + np.eye(adj.shape[0]) - d = sp.diags(np.power(np.array(adj.sum(1)), -0.5).flatten(), 0).toarray() - a_norm = adj.dot(d).transpose().dot(d) - return a_norm - - -def calculate_graph_feat( - feat_mat: np.ndarray, - adj_list: List[List[int]], - max_atoms: int, -) -> Tuple[np.ndarray, np.ndarray]: - """Compute drug graph features""" - assert feat_mat.shape[0] == len(adj_list) - feat = np.zeros((max_atoms, feat_mat.shape[-1]), dtype=np.float32) - adj_mat = np.zeros((max_atoms, max_atoms), dtype=np.float32) - - feat[: feat_mat.shape[0], :] = feat_mat - for i, nodes in enumerate(adj_list): - for each in nodes: - adj_mat[i, int(each)] = 1 - - assert np.allclose(adj_mat, adj_mat.T) - adj_ = adj_mat[: len(adj_list), : len(adj_list)] - norm_adj_ = normalize_adj(adj_) - adj_mat[: len(adj_list), : len(adj_list)] = norm_adj_ - return feat, adj_mat - - -def metadata_generate( - drug_info_file: str, - cell_line_info_file: str, - drug_feature_dir: str, - gene_expression_file: str, -) -> Tuple[dict, pd.DataFrame, list]: - """Generate metadata""" - # drug_id --> pubchem_id - reader = pd.read_csv(drug_info_file) - drugid2pubchemid = {# Build drug_id → pubchem_id mapping - str(row.iloc[0]): str(row.iloc[5])# Mapping from column 0 of drug information CSV to column 5 (pubchem_id) - for _, row in reader.iterrows() - if str(row.iloc[5]).isdigit() - } - - # cellline --> cancer type # Read from the TSV file, the first column is cellline_id, the last column is the cancer type - cellline2cancertype: Dict[str, str] = {} - with open(cell_line_info_file, "r") as f: - for line in f.readlines()[1:]: - cellline_id = line.split("\t")[1] - tcga_label = line.strip().split("\t")[-1] - cellline2cancertype[cellline_id] = tcga_label - - gexpr_feature = pd.read_csv(# Gene expression matrix for cell lines - gene_expression_file, sep=",", header=0, index_col=0 - ) - - # load drug graph features - drug_feature: Dict[str, list] = {} - drug_pubchem_id_set = [] - for fname in os.listdir(drug_feature_dir): - if not fname.endswith(".hkl"): - continue - pubchem_id = fname.split(".")[0] - feat_mat, adj_list, degree_list = hkl.load( - os.path.join(drug_feature_dir, fname) - ) - drug_feature[pubchem_id] = [feat_mat, adj_list, degree_list]# [Feature matrix, adjacency list, degree list] - drug_pubchem_id_set.append(pubchem_id) - - # IC50 - # Read IC50 response data from GDSC database - # Filter out drug-cell line pairs with corresponding pubchem_id and gene expression data - cancer_response_exp_file = os.path.join( - os.path.dirname(gene_expression_file), "GDSC_IC50.csv" - ) - experiment_data = pd.read_csv( - cancer_response_exp_file, sep=",", header=0, index_col=0 - ) - drug_match_list = [ - item - for item in experiment_data.index - if item.split(":")[1] in drugid2pubchemid.keys() - ] - experiment_data_filtered = experiment_data.loc[drug_match_list] - - data_idx: list = []# Each entry contains: cell line ID, drug pubchem ID, IC50 value, cancer type. - for each_drug in experiment_data_filtered.index: - for each_cellline in experiment_data_filtered.columns: - pubchem_id = drugid2pubchemid[each_drug.split(":")[-1]] - if ( - str(pubchem_id) in drug_pubchem_id_set - and each_cellline in gexpr_feature.index - ): - if ( - not np.isnan( - experiment_data_filtered.loc[each_drug, each_cellline] - ) - and each_cellline in cellline2cancertype.keys() - ): - ln_ic50 = float( - experiment_data_filtered.loc[each_drug, each_cellline] - ) - data_idx.append( - ( - each_cellline, - pubchem_id, - ln_ic50, - cellline2cancertype[each_cellline], - ) - ) - - nb_celllines = len(set([item[0] for item in data_idx])) - nb_drugs = len(set([item[1] for item in data_idx])) - print_sys( - f"{len(data_idx)} instances across {nb_celllines} cell lines and {nb_drugs} drugs." - ) - return drug_feature, gexpr_feature, data_idx # Drug molecular graph features, cell line gene expression features, effective (cell line, drug, IC50, cancer type) four-tuple list - - -def data_split(data_idx: list, ratio: float = 0.95): - """Return training and validation splits stratified by cancer type.""" - train_idx, validation_idx = [], [] - for each_type in TCGA_LABEL_SET: - data_subtype_idx = [item for item in data_idx if item[-1] == each_type] - if len(data_subtype_idx) == 0: - continue - k = max(1, int(ratio * len(data_subtype_idx))) - train_list = random.sample(data_subtype_idx, k) - validation_list = [item for item in data_subtype_idx if item not in train_list] - train_idx += train_list - validation_idx += validation_list - return train_idx, validation_idx - - -def feature_extract( - data_idx: list, - drug_feature: dict, - max_atoms: int, -) -> Tuple[list, np.ndarray, list, list]: - """Extract drug map features""" - nb_instance = len(data_idx) - - drug_data = [[] for _ in range(nb_instance)] - target = np.zeros(nb_instance, dtype=np.float32) - cancer_type_list = [] - - for idx in range(nb_instance): - cell_line_id, pubchem_id, ln_ic50, cancer_type = data_idx[idx] - - # Drug map features - feat_mat, adj_list, _ = drug_feature[str(pubchem_id)] - feat, adj = calculate_graph_feat(feat_mat, adj_list, max_atoms) - drug_data[idx] = [feat, adj] - - # target value - target[idx] = ln_ic50 - - cancer_type_list.append([cancer_type, cell_line_id, pubchem_id]) - - return drug_data, target, cancer_type_list, data_idx - - -# ===================== Dataset ===================== - -class DrugPerformerModuleDataset(Dataset): - """Drug + PerformerModule Dataset""" - - def __init__( - self, - drug_data: list, - targets: np.ndarray, - index_list: list, - cell_emb_dict: Dict[str, np.ndarray], - ): - super().__init__() - self.drug_data = drug_data - self.targets = targets.astype(np.float32) - self.index_list = index_list - self.cell_emb_dict = cell_emb_dict - - def __len__(self) -> int: - return int(self.targets.shape[0]) - - def __getitem__(self, idx: int): - # Drug map features - feat, adj = self.drug_data[idx] - drug_feat = torch.from_numpy(feat).float() - drug_adj = torch.from_numpy(adj).float() - - # PerformerModule cell embedding - cell_line_id = self.index_list[idx][0] - cell_emb_np = self.cell_emb_dict[cell_line_id] - cell_emb = torch.from_numpy(cell_emb_np).float() - - # target value - target = torch.tensor(self.targets[idx], dtype=torch.float32) - - return { - "drug_feat": drug_feat, - "drug_adj": drug_adj, - "cell_emb": cell_emb, - "target": target, - } - - -# ===================== Model Definition ===================== - -class GraphConv(nn.Module): - """graph convolution layer""" - - def __init__(self, in_features: int, out_features: int, bias: bool = True): - super().__init__() - self.in_features = in_features - self.out_features = out_features - self.weight = nn.Parameter(torch.Tensor(in_features, out_features)) - if bias: - self.bias = nn.Parameter(torch.Tensor(out_features)) - else: - self.bias = None - self.reset_parameters() - - def reset_parameters(self): - nn.init.xavier_uniform_(self.weight) - if self.bias is not None: - nn.init.zeros_(self.bias) - - def forward(self, x: torch.Tensor, adj: torch.Tensor) -> torch.Tensor: - xw = torch.matmul(x, self.weight) - if self.bias is not None: - xw = xw + self.bias - out = torch.bmm(adj.transpose(1, 2), xw) - return out - - -class DrugPerformerModuleModel(nn.Module): - """ - Drugs + PerformerModule Predictive Model - - Architecture: - - Drug branch: 3-layer GraphConv + residual connection + double pooling - - Cell branch: 2-layer MLP (processing PerformerModule module embeddings) - - Fusion layer: 4-layer MLP + attention mechanism - """ - - def __init__( - self, - drug_feat_dim: int, - cell_emb_dim: int, - hidden_dim: int = 256, - dropout: float = 0.1, - ): - super().__init__() - self.relu = nn.ReLU() - self.dropout = nn.Dropout(dropout) - - # Drug branch: enhanced 3-layer GraphConv + residual connection - self.graph_conv1 = GraphConv(drug_feat_dim, 256) - self.graph_conv2 = GraphConv(256, 256) - self.graph_conv3 = GraphConv(256, 128) - self.drug_bn1 = nn.BatchNorm1d(256) - self.drug_bn2 = nn.BatchNorm1d(256) - self.drug_bn3 = nn.BatchNorm1d(128) - - # Cell branch: 2-layer MLP (processing PerformerModule embeddings) - self.cell_fc1 = nn.Linear(cell_emb_dim, 256) - self.cell_bn1 = nn.BatchNorm1d(256) - self.cell_fc2 = nn.Linear(256, 128) - self.cell_bn2 = nn.BatchNorm1d(128) - - # Fusion dimension: 128 (drug) + 128 (cell) = 256 - fusion_dim = 256 - - # attention mechanism - self.attention = nn.Sequential( - nn.Linear(fusion_dim, fusion_dim // 2), - nn.ReLU(), - nn.Linear(fusion_dim // 2, fusion_dim), - nn.Sigmoid() - ) - - # Fusion layer: 4-layer deep MLP - self.fusion_fc1 = nn.Linear(fusion_dim, hidden_dim * 2) - self.fusion_bn1 = nn.BatchNorm1d(hidden_dim * 2) - self.fusion_fc2 = nn.Linear(hidden_dim * 2, hidden_dim) - self.fusion_bn2 = nn.BatchNorm1d(hidden_dim) - self.fusion_fc3 = nn.Linear(hidden_dim, hidden_dim // 2) - self.fusion_bn3 = nn.BatchNorm1d(hidden_dim // 2) - self.output_fc = nn.Linear(hidden_dim // 2, 1) - - print_sys(f"Model initialized: drug_feat_dim={drug_feat_dim}, " - f"cell_emb_dim={cell_emb_dim}, hidden_dim={hidden_dim}") - - def forward( - self, - drug_feat: torch.Tensor, - drug_adj: torch.Tensor, - cell_emb: torch.Tensor, - ) -> torch.Tensor: - # Drug branch (enhanced version) - # first floor - x_drug = self.graph_conv1(drug_feat, drug_adj) # (B, N, 256) - x_drug = x_drug.transpose(1, 2) # (B, 256, N) for BatchNorm - x_drug = self.drug_bn1(x_drug) - x_drug = x_drug.transpose(1, 2) # (B, N, 256) - x_drug = self.relu(x_drug) - x_drug = self.dropout(x_drug) - - # Second layer + residual connection - x_drug_res = x_drug - x_drug = self.graph_conv2(x_drug, drug_adj) # (B, N, 256) - x_drug = x_drug.transpose(1, 2) - x_drug = self.drug_bn2(x_drug) - x_drug = x_drug.transpose(1, 2) - x_drug = self.relu(x_drug + x_drug_res) # residual connection - x_drug = self.dropout(x_drug) - - # third floor - x_drug = self.graph_conv3(x_drug, drug_adj) # (B, N, 128) - x_drug = x_drug.transpose(1, 2) - x_drug = self.drug_bn3(x_drug) - x_drug = x_drug.transpose(1, 2) - x_drug = self.relu(x_drug) - - # Aggregation: max pooling + mean pooling - x_drug_max = x_drug.max(dim=1)[0] # (B, 128) - x_drug_mean = x_drug.mean(dim=1) # (B, 128) - x_drug = x_drug_max + x_drug_mean # Fusion - - # Cell branching (handling PerformerModule embeddings) - x_cell = self.cell_fc1(cell_emb) # (B, 256) - x_cell = self.cell_bn1(x_cell) - x_cell = self.relu(x_cell) - x_cell = self.dropout(x_cell) - - x_cell = self.cell_fc2(x_cell) # (B, 128) - x_cell = self.cell_bn2(x_cell) - x_cell = self.relu(x_cell) - - # Feature fusion + attention - h = torch.cat([x_drug, x_cell], dim=1) # (B, 256) - - # Apply attention mechanism - att_weights = self.attention(h) - h = h * att_weights # weighted features - - # Deep Fusion MLP - h = self.fusion_fc1(h) # (B, hidden_dim * 2) - h = self.fusion_bn1(h) - h = self.relu(h) - h = self.dropout(h) - - h = self.fusion_fc2(h) # (B, hidden_dim) - h = self.fusion_bn2(h) - h = self.relu(h) - h = self.dropout(h) - - h = self.fusion_fc3(h) # (B, hidden_dim // 2) - h = self.fusion_bn3(h) - h = self.relu(h) - h = self.dropout(h) - - # output layer - out = self.output_fc(h) # (B, 1) - return out - - -# ===================== Training and Evaluation ===================== - -def train_epoch(model, loader, optimizer, criterion, device): - """train for one epoch""" - model.train() - total_loss = 0.0 - total = 0 - for batch in loader: - drug_feat = batch["drug_feat"].to(device) - drug_adj = batch["drug_adj"].to(device) - cell_emb = batch["cell_emb"].to(device) - target = batch["target"].to(device) - - optimizer.zero_grad(set_to_none=True) - pred = model(drug_feat, drug_adj, cell_emb).squeeze(-1) - - loss = criterion(pred, target) - loss.backward() - - # gradient crop - torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0) - - optimizer.step() - - total_loss += float(loss.item()) * target.size(0) - total += target.size(0) - return total_loss / total if total > 0 else 0.0 - - -def eval_epoch(model, loader, device): - """Evaluate an epoch""" - model.eval() - preds = [] - trues = [] - with torch.no_grad(): - for batch in loader: - drug_feat = batch["drug_feat"].to(device) - drug_adj = batch["drug_adj"].to(device) - cell_emb = batch["cell_emb"].to(device) - target = batch["target"].to(device) - - pred = model(drug_feat, drug_adj, cell_emb).squeeze(-1) - - preds.append(pred.cpu().numpy()) - trues.append(target.cpu().numpy()) - - preds = np.concatenate(preds, axis=0) - trues = np.concatenate(trues, axis=0) - pcc = pearsonr(preds, trues)[0] - return preds, trues, pcc - - -# ===================== Main ===================== - -def main(): - parser = argparse.ArgumentParser( - description="Drug response prediction using Drug + PerformerModule" - ) - parser.add_argument("--gpu_id", type=str, default="0") - parser.add_argument("--data_path", type=str, default="../drug_response_pytorch/data") - parser.add_argument("--epochs", type=int, default=300) - parser.add_argument("--batch_size", type=int, default=32) - parser.add_argument("--max_atoms", type=int, default=100) - parser.add_argument("--hidden_dim", type=int, default=256) - parser.add_argument("--learning_rate", type=float, default=1e-3) - parser.add_argument("--dropout", type=float, default=0.1) - parser.add_argument("--patience", type=int, default=30) - - # PerformerModule related paths - parser.add_argument( - "--checkpoint_path", - type=str, - required=True, - help="Path to pre-trained PerformerModule checkpoint" - ) - parser.add_argument( - "--hyper_params_path", - type=str, - required=True, - help="Path to PerformerModule model hyper parameters pkl file" - ) - parser.add_argument( - "--module_data_dir", - type=str, - default=None, - help="Directory containing module JSON files" - ) - parser.add_argument( - "--module_data_path", - type=str, - default=None, - help="Path to module JSON file (single file)" - ) - parser.add_argument( - "--gene_list_path", - type=str, - default="../../selected_genes_27k.txt", - help="Path to gene list file" - ) - parser.add_argument( - "--gene_mapping_path", - type=str, - default="../../gene2ens.csv", - help="Path to gene symbol to Ensembl ID mapping" - ) - parser.add_argument( - "--gene2vec_path", - type=str, - default="../../selected_gene2vec_27k.npy", - help="Path to gene2vec file" - ) - parser.add_argument( - "--symbol2ens_path", - type=str, - default="../../gene2ens.csv", - help="Path to symbol to Ensembl mapping (for data processing)" - ) - parser.add_argument( - "--pool_method", - type=str, - default="mean", - choices=["mean", "max", "cls", "concat"], - help="Pooling method for PerformerModule module embeddings" - ) - parser.add_argument( - "--shuffle_module_genes", - action="store_true", - help="Optional downstream-only extra reshuffle on top of module_sets restored from PKL. " - "Do not pass this when reproducing the pretrained backbone as-is." - ) - parser.add_argument( - "--shuffle_seed", - type=int, - default=2021, - help="Random seed for the downstream extra reshuffle when --shuffle_module_genes is enabled." - ) - # Output path - parser.add_argument( - "--save_dir", - type=str, - default="./models_drug_performer_module", - help="Directory to save the model" - ) - - args = parser.parse_args() - - os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu_id - device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - - print_sys("=" * 70) - print_sys("Drug Response Prediction: Drug + PerformerModule") - print_sys("=" * 70) - print_sys(f"Device: {device}") - print_sys(f"Epochs: {args.epochs}") - print_sys(f"Batch size: {args.batch_size}") - print_sys(f"Learning rate: {args.learning_rate}") - print_sys(f"Hidden dim: {args.hidden_dim}") - print_sys(f"Dropout: {args.dropout}") - print_sys(f"Pool method: {args.pool_method}") - print_sys(f"Early stopping patience: {args.patience}") - print_sys("=" * 70) - - # Create save directory - os.makedirs(args.save_dir, exist_ok=True) - - dpath = args.data_path - drug_info_file = f"{dpath}/GDSC/1.Drug_listMon Jun 24 09_00_55 2019.csv" - cell_line_info_file = f"{dpath}/CCLE/Cell_lines_annotations_20181226.txt" - drug_feature_dir = f"{dpath}/GDSC/drug_graph_feat" - gene_expression_file = ( - f"{dpath}/CCLE/genomic_expression_561celllines_697genes_demap_features.csv" - ) - - # fixed random seed - random.seed(0) - np.random.seed(0) - torch.manual_seed(0) - if torch.cuda.is_available(): - torch.cuda.manual_seed_all(0) - - # 1) Read metadata - print_sys("\n[Step 1/6] Loading metadata...") - drug_feature, gexpr_feature, data_idx = metadata_generate( - drug_info_file, - cell_line_info_file, - drug_feature_dir, - gene_expression_file, - ) - - # 2) Build PerformerModule embeddings - print_sys("\n[Step 2/6] Building PerformerModule embeddings...") - cell_emb_dict, cell_emb_dim = build_cell_embeddings_with_performer_module( # cell_emb_dict │ Dict[str, np.ndarray] │ {cell_line_id: embedding} mapping dictionary - gexpr_feature=gexpr_feature, - symbol2ens_path=args.symbol2ens_path, - gene_list_path=args.gene_list_path, - gene_mapping_path=args.gene_mapping_path, - gene2vec_path=args.gene2vec_path, - checkpoint_path=args.checkpoint_path, - hyper_params_path=args.hyper_params_path, - module_data_dir=args.module_data_dir, - module_data_path=args.module_data_path, - device=device, - pool_method=args.pool_method, - shuffle_module_genes=args.shuffle_module_genes, - seed=args.shuffle_seed, - ) - - # 3) Split training and validation data; validation is used for model selection - print_sys("\n[Step 3/6] Splitting dataset...") - train_idx, validation_idx = data_split(data_idx) - print_sys(f"Train samples: {len(train_idx)}, Validation samples: {len(validation_idx)}") - - # 4) Extract features - print_sys("\n[Step 4/6] Extracting features...") - X_drug_train, Y_train, train_cancer_info, train_index_list = feature_extract( - train_idx, drug_feature, args.max_atoms - ) - X_drug_validation, Y_validation, validation_cancer_info, validation_index_list = feature_extract( - validation_idx, drug_feature, args.max_atoms - ) - - # 5) Build a data set - print_sys("\n[Step 5/6] Creating datasets...") - train_ds = DrugPerformerModuleDataset( - X_drug_train, Y_train, train_index_list, cell_emb_dict - ) - validation_ds = DrugPerformerModuleDataset( - X_drug_validation, Y_validation, validation_index_list, cell_emb_dict - ) - - drug_dim = X_drug_train[0][0].shape[-1] - - num_workers = min(4, os.cpu_count() or 1) - train_loader = DataLoader( - train_ds, - batch_size=args.batch_size, - shuffle=True, - num_workers=num_workers, - pin_memory=torch.cuda.is_available(), - ) - validation_loader = DataLoader( - validation_ds, - batch_size=args.batch_size, - shuffle=False, - num_workers=num_workers, - pin_memory=torch.cuda.is_available(), - ) - - # 6) Build the model - print_sys("\n[Step 6/6] Building model...") - model = DrugPerformerModuleModel( - drug_feat_dim=drug_dim, - cell_emb_dim=cell_emb_dim, - hidden_dim=args.hidden_dim, - dropout=args.dropout, - ).to(device) - - # Statistical parameters - total_params = sum(p.numel() for p in model.parameters()) - trainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad) - print_sys(f"Total parameters: {total_params:,}") - print_sys(f"Trainable parameters: {trainable_params:,}") - - # Optimizers and schedulers - optimizer = torch.optim.AdamW( - model.parameters(), lr=args.learning_rate, weight_decay=1e-5 - ) - scheduler = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts( - optimizer, T_0=20, T_mult=2, eta_min=1e-6 - ) - criterion = torch.nn.HuberLoss(delta=1.0) - - # train - print_sys("\n" + "=" * 70) - print_sys("Starting training...") - print_sys("=" * 70) - - best_pcc = -1.0 - best_pred = None - best_true = None - best_epoch = 0 - patience_counter = 0 - - for epoch in range(1, args.epochs + 1): - train_loss = train_epoch(model, train_loader, optimizer, criterion, device) - preds, trues, pcc = eval_epoch(model, validation_loader, device) - - scheduler.step() - current_lr = optimizer.param_groups[0]["lr"] - - print_sys( - f"Epoch {epoch:03d}/{args.epochs} | " - f"train_loss={train_loss:.4f} | " - f"validation_pcc={pcc:.4f} | " - f"lr={current_lr:.6f}" - ) - - if pcc > best_pcc: - best_pcc = pcc - best_pred = preds - best_true = trues - best_epoch = epoch - patience_counter = 0 - - # Save the best model selected by validation PCC - model_save_path = os.path.join(args.save_dir, "best_model.pth") - torch.save( - { - "epoch": epoch, - "model_state_dict": model.state_dict(), - "optimizer_state_dict": optimizer.state_dict(), - "scheduler_state_dict": scheduler.state_dict(), - "best_pcc": best_pcc, - "train_loss": train_loss, - "args": vars(args), - }, - model_save_path, - ) - print_sys(f" → Best model saved! Validation PCC improved to {best_pcc:.4f}") - else: - patience_counter += 1 - if patience_counter >= args.patience: - print_sys( - f"\n[Early Stopping] No improvement for {args.patience} epochs." - ) - break - - # Save final result - print_sys("\n" + "=" * 70) - print_sys("Training completed!") - print_sys("=" * 70) - print_sys(f"Best epoch: {best_epoch}") - print_sys(f"Best validation PCC: {best_pcc:.4f}") - - if best_pred is not None: - result_path = os.path.join(args.save_dir, "predictions.npz") - np.savez(result_path, pred=best_pred, validation=best_true) - print_sys(f"Validation predictions saved to: {result_path}") - - print_sys(f"Best model saved to: {os.path.join(args.save_dir, 'best_model.pth')}") - print_sys("=" * 70) - - -if __name__ == "__main__": - main() - +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +Drug response prediction: using drug molecular structure + PerformerModule cell embedding + +Architecture: +- Drug branch: 3-layer GCN + residual connection + double pooling +- Cell branch: extract module-level features from pre-trained PerformerModule +- Fusion layer: attention mechanism + deep MLP +- Optimization strategy: AdamW + cosine annealing + early stopping + +How to use: + # Run from the repository root; see drug_response/cmd.txt for a full example. + python BioMod/drug_response/train_drug_performer_module.py \ + --gpu_id 0 --epochs 300 \ + --checkpoint_path weight/0810model/mae_tost_70M_75mask_ec2_epoch0.pth \ + --hyper_params_path weight/0810model/mae_tost_70M_75mask_ec2.pkl +""" + +import argparse +import os +import random +import sys +import pickle +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +ASSET_DIR = REPO_ROOT / "model_dependencies" +DOWNSTREAM_DATA_DIR = REPO_ROOT / "data" / "downstream" / "final_data" +WEIGHTS_DIR = REPO_ROOT / "weight" / "0810model" +from typing import Dict, List, Tuple, Union, Optional + +import hickle as hkl +import numpy as np +import pandas as pd +import scipy.sparse as sp +import torch +import torch.nn as nn +from scipy.stats import pearsonr +from torch.utils.data import DataLoader, Dataset + +# Import PerformerModule model +project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +if project_root not in sys.path: + sys.path.insert(0, project_root) +sys.path.insert(0, os.path.dirname(project_root)) # Add the repository root directory + +# Use direct path import to avoid conflicts with files with the same name +import importlib.util +performer_module_dir = project_root + +# import models_performer_module +models_path = os.path.join(performer_module_dir, 'models_performer_module.py') +if not os.path.exists(models_path): + raise FileNotFoundError(f"Model file not found: {models_path}") +spec = importlib.util.spec_from_file_location("models_performer_module", models_path) +models_performer_module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(models_performer_module) +PerformerEncoderOnly_Module = models_performer_module.PerformerEncoderOnly_Module +sanitize_performer_module_config = models_performer_module.sanitize_performer_module_config + +# Import module_dataset +dataset_path = os.path.join(performer_module_dir, 'module_dataset.py') +if not os.path.exists(dataset_path): + raise FileNotFoundError(f"Dataset file not found: {dataset_path}") +spec = importlib.util.spec_from_file_location("module_dataset", dataset_path) +module_dataset_module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module_dataset_module) +load_module_data = module_dataset_module.load_module_data + + +# ===================== Utility functions ===================== + +def print_sys(*args, **kwargs): + """Print and refresh output""" + print(*args, **kwargs, flush=True) + + +def reindex_tensor_universal( + tensor: torch.Tensor, + index_positions: List[int], + dim: int, + filler: str = "0", + device: Union[torch.device, str] = "cpu", +) -> torch.Tensor: + """Universal reordering operation""" + index_tensor = torch.tensor(index_positions, device=device) + tensor_shape = list(tensor.shape) + tensor_shape[dim] = len(index_positions) + index_shape = [1] * len(tensor_shape) + index_shape[dim] = len(index_positions) + index_tensor = index_tensor.view(*index_shape).expand(*tensor_shape) + + if filler == "0": + padder_shape = list(tensor_shape) + padder_shape[dim] = 1 + padder = torch.zeros(padder_shape, dtype=tensor.dtype, device=device) + elif filler == "mean": + padder = tensor.mean(dim=dim, keepdim=True) + else: + raise ValueError("filler should be '0' or 'mean'") + + expanded_tensor = torch.cat((tensor, padder), dim=dim) + reindexed = torch.gather(expanded_tensor, dim, index_tensor) + return reindexed + + +# ===================== PerformerModule related ===================== + +def load_performer_module_model( + checkpoint_path: str, + hyper_params_path: str, + module_data_dir: Optional[str], + module_data_path: Optional[str], + gene_list_path: str, + gene_mapping_path: str, + gene2vec_path: str, + device: torch.device, + shuffle_module_genes: bool = False, + seed: int = 0, +) -> PerformerEncoderOnly_Module: + """Load a pretrained PerformerModule model""" + print_sys(f"Loading PerformerModule model from: {checkpoint_path}") + + # Load model hyperparameters + print_sys(f"Loading hyper parameters from: {hyper_params_path}") + with open(hyper_params_path, 'rb') as f: + model_params = pickle.load(f) + model_params.pop('module_embed_chunk_padded_gene_budget', None) + model_params = sanitize_performer_module_config( + model_params, + source=hyper_params_path, + log_fn=print_sys, + ) + + # Update gene2vec path + if gene2vec_path: + model_params['gene2vec_file'] = gene2vec_path + + # Read fusion_method from pkl to ensure consistency with pre-training + fusion_method = model_params.get('fusion_method', 'multiply') + print_sys(f"Using fusion_method from pkl: {fusion_method}") + + # use_gene2vec strictly follows the pre-training pkl to avoid conflicts caused by downstream coverage. + pkl_use_gene2vec = model_params.get('use_gene2vec', True) + model_params['use_gene2vec'] = pkl_use_gene2vec + print_sys(f"use_gene2vec={pkl_use_gene2vec} (from PKL)") + + # Load module data (if not present in hyperparameters) + if 'module_sets' not in model_params or model_params.get('module_sets') is None: + if module_data_dir or module_data_path: + print_sys("Loading module data...") + module_sets = load_module_data( + module_data_path=module_data_path, + module_data_dir=module_data_dir, + gene_list_path=gene_list_path, + gene_mapping_path=gene_mapping_path, + verbose=True + ) + model_params['module_sets'] = module_sets + else: + raise ValueError("Model requires module_sets, but --module_data_dir or --module_data_path was not provided") + else: + module_sets = model_params['module_sets'] + print_sys(f"Using module_sets from hyper parameters ({len(module_sets)} modules)") + + # If shuffle_module_genes is enabled, disrupt the gene distribution within the module. + if shuffle_module_genes: + print_sys( + f"Applying downstream-only extra module reshuffle (seed={seed}). " + f"Pretraining-time module_sets from PKL will be reshuffled again." + ) + # Collect all gene indices + all_gene_indices = [] + module_sizes = [] + for module in module_sets: + all_gene_indices.extend(module) + module_sizes.append(len(module)) + + # Scramble the gene index using a fixed random seed + rng = random.Random(seed) + rng.shuffle(all_gene_indices) + + # Redistribute genes according to original module size + shuffled_module_sets = [] + start_idx = 0 + for size in module_sizes: + shuffled_module_sets.append(all_gene_indices[start_idx:start_idx + size]) + start_idx += size + module_sets = shuffled_module_sets + model_params['module_sets'] = module_sets + print_sys(f"Shuffled {len(all_gene_indices)} genes across {len(module_sets)} modules") + + print_sys(f"PerformerModule config: base_dim={model_params.get('base_dim', 'N/A')}, " + f"num_modules={len(module_sets)}") + + # Build model + model = PerformerEncoderOnly_Module(**model_params).to(device) + + # Load weights + print_sys(f"Loading checkpoint from: {checkpoint_path}") + checkpoint = torch.load(checkpoint_path, map_location=device) + + # Handle parameters in checkpoint + if 'model_state_dict' in checkpoint: + state_dict = checkpoint['model_state_dict'] + else: + state_dict = checkpoint + + # Filter out possible GO related parameters + filtered_state_dict = {} + for k, v in state_dict.items(): + if 'go_conv' not in k and 'G_go' not in k.lower(): + filtered_state_dict[k] = v + + # Use strict=True to ensure the weights match exactly (except for filtered GO parameters) + model.load_state_dict(filtered_state_dict, strict=True) + + print_sys("Loaded pretrained PerformerModule weights.") + + # freeze model + model.eval() + for p in model.parameters(): + p.requires_grad = False + + return model, len(module_sets), model_params.get('base_dim', 1024) + + +def build_performer_module_gene_mapping( + symbol2ens_path: str, + gene_list_path: str, + expr_genes: List[str], +) -> Tuple[Dict[str, int], List[int]]: + """Build a gene map""" + gene2ens = pd.read_csv(symbol2ens_path, index_col=0) + symbol2ens = gene2ens["Ensembl gene ID"].to_dict() + + input_genes = [symbol2ens.get(sym, sym) for sym in expr_genes] + input_gene_num = len(input_genes) + input_mapping = {gid: i for i, gid in enumerate(input_genes)} + + with open(gene_list_path, "r") as f: + performer_genes = [line.rstrip("\n") for line in f.readlines()] + performer_genes_pad = performer_genes + ["PAD"] + + performer_index_positions = [ + input_mapping.get(gid, input_gene_num) for gid in performer_genes_pad + ] + intersect = np.intersect1d(input_genes, performer_genes) + print_sys(f"Target and PerformerModule genes intersect: {len(intersect)}") + return input_mapping, performer_index_positions + + +def build_cell_embeddings_with_performer_module( + gexpr_feature: pd.DataFrame, + symbol2ens_path: str, + gene_list_path: str, + gene_mapping_path: str, + gene2vec_path: str, + checkpoint_path: str, + hyper_params_path: str, + module_data_dir: Optional[str], + module_data_path: Optional[str], + device: torch.device, + pool_method: str = "mean", + shuffle_module_genes: bool = False, + seed: int = 0, +) -> Tuple[Dict[str, np.ndarray], int]: + """ + Use PerformerModule to construct embedding for each cell line + + Args: + pool_method: pooling method, 'mean', 'max', 'cls', or 'concat' + - 'mean': average all modules (default) + - 'max': take the maximum value for all modules + - 'cls': only use CLS token (if available) + - 'concat': concatenate all modules (the dimension will be very large) + use_gene2vec: automatically restored by pre-trained PKL, no longer covered downstream + """ + model, num_modules, encoder_embed_dim = load_performer_module_model( + checkpoint_path=checkpoint_path, + hyper_params_path=hyper_params_path, + module_data_dir=module_data_dir, + module_data_path=module_data_path, + gene_list_path=gene_list_path, + gene_mapping_path=gene_mapping_path, + gene2vec_path=gene2vec_path, + device=device, + shuffle_module_genes=shuffle_module_genes, + seed=seed, + ) + + _, performer_index_positions = build_performer_module_gene_mapping( + symbol2ens_path=symbol2ens_path, + gene_list_path=gene_list_path, + expr_genes=list(gexpr_feature.columns), + ) + + cell_emb_dict: Dict[str, np.ndarray] = {} + first_emb_dim: int = -1 + + with torch.no_grad(): + for cell_line_id, row in gexpr_feature.iterrows(): + x = torch.tensor(row.values[None, :], dtype=torch.float32, device=device) + x_performer = reindex_tensor_universal( + x, performer_index_positions, dim=1, filler="0", device=device + ) + + # Extract features through encoder (without masking, mask_ratio=0.0) + out = model(x_performer, mask_ratio=0.0, return_encodings=True) + + # Extract module encodings + if 'module_encodings' in out: + module_encodings = out['module_encodings'] # (1, num_modules, encoder_embed_dim) + + # Processed according to pooling method + if pool_method == "cls": + if model.use_cls: + # Use CLS token (in latent, but module_encodings does not contain CLS) + # Need to get the complete latent from forward_encoder + enc_out = model.forward_encoder(x_performer, mask_ratio=0.0) + latent = enc_out[0] + if model.use_cls: + cell_emb = latent[:, 0, :] # (1, encoder_embed_dim) + else: + print_sys("Warning: pool_method='cls' but model has no CLS token, using 'mean' instead") + cell_emb = module_encodings.mean(dim=1) + else: + print_sys("Warning: pool_method='cls' but model has no CLS token, using 'mean' instead") + cell_emb = module_encodings.mean(dim=1) + elif pool_method == "mean": + cell_emb = module_encodings.mean(dim=1) # (1, encoder_embed_dim) + elif pool_method == "max": + cell_emb = module_encodings.max(dim=1)[0] # (1, encoder_embed_dim) + elif pool_method == "concat": + cell_emb = module_encodings.reshape(1, -1) # (1, num_modules * encoder_embed_dim) + else: + raise ValueError(f"Unknown pool_method: {pool_method}") + else: + raise ValueError(f"Expected 'module_encodings' in output, got: {list(out.keys())}") + + cell_emb_np = cell_emb.squeeze(0).cpu().numpy() + cell_emb_dict[cell_line_id] = cell_emb_np + if first_emb_dim < 0: + first_emb_dim = cell_emb_np.shape[0] + + print_sys( + f"Built PerformerModule embeddings for {len(cell_emb_dict)} cell lines, " + f"dim={first_emb_dim}, pool_method={pool_method}" + ) + return cell_emb_dict, first_emb_dim + + +# ===================== Data processing ===================== + +TCGA_LABEL_SET = [ + "ALL", "BLCA", "BRCA", "CESC", "DLBC", "LIHC", "LUAD", "ESCA", "GBM", + "HNSC", "KIRC", "LAML", "LCML", "LGG", "LUSC", "MESO", "MM", "NB", + "OV", "PAAD", "SCLC", "SKCM", "STAD", "THCA", "COAD/READ", +] + + +def normalize_adj(adj: np.ndarray) -> np.ndarray: + """Normalized adjacency matrix""" + adj = adj + np.eye(adj.shape[0]) + d = sp.diags(np.power(np.array(adj.sum(1)), -0.5).flatten(), 0).toarray() + a_norm = adj.dot(d).transpose().dot(d) + return a_norm + + +def calculate_graph_feat( + feat_mat: np.ndarray, + adj_list: List[List[int]], + max_atoms: int, +) -> Tuple[np.ndarray, np.ndarray]: + """Compute drug graph features""" + assert feat_mat.shape[0] == len(adj_list) + feat = np.zeros((max_atoms, feat_mat.shape[-1]), dtype=np.float32) + adj_mat = np.zeros((max_atoms, max_atoms), dtype=np.float32) + + feat[: feat_mat.shape[0], :] = feat_mat + for i, nodes in enumerate(adj_list): + for each in nodes: + adj_mat[i, int(each)] = 1 + + assert np.allclose(adj_mat, adj_mat.T) + adj_ = adj_mat[: len(adj_list), : len(adj_list)] + norm_adj_ = normalize_adj(adj_) + adj_mat[: len(adj_list), : len(adj_list)] = norm_adj_ + return feat, adj_mat + + +def metadata_generate( + drug_info_file: str, + cell_line_info_file: str, + drug_feature_dir: str, + gene_expression_file: str, +) -> Tuple[dict, pd.DataFrame, list]: + """Generate metadata""" + # drug_id --> pubchem_id + reader = pd.read_csv(drug_info_file) + drugid2pubchemid = {# Build drug_id → pubchem_id mapping + str(row.iloc[0]): str(row.iloc[5])# Mapping from column 0 of drug information CSV to column 5 (pubchem_id) + for _, row in reader.iterrows() + if str(row.iloc[5]).isdigit() + } + + # cellline --> cancer type # Read from the TSV file, the first column is cellline_id, the last column is the cancer type + cellline2cancertype: Dict[str, str] = {} + with open(cell_line_info_file, "r") as f: + for line in f.readlines()[1:]: + cellline_id = line.split("\t")[1] + tcga_label = line.strip().split("\t")[-1] + cellline2cancertype[cellline_id] = tcga_label + + gexpr_feature = pd.read_csv(# Gene expression matrix for cell lines + gene_expression_file, sep=",", header=0, index_col=0 + ) + + # load drug graph features + drug_feature: Dict[str, list] = {} + drug_pubchem_id_set = [] + for fname in os.listdir(drug_feature_dir): + if not fname.endswith(".hkl"): + continue + pubchem_id = fname.split(".")[0] + feat_mat, adj_list, degree_list = hkl.load( + os.path.join(drug_feature_dir, fname) + ) + drug_feature[pubchem_id] = [feat_mat, adj_list, degree_list]# [Feature matrix, adjacency list, degree list] + drug_pubchem_id_set.append(pubchem_id) + + # IC50 + # Read IC50 response data from GDSC database + # Filter out drug-cell line pairs with corresponding pubchem_id and gene expression data + cancer_response_exp_file = os.path.join( + os.path.dirname(gene_expression_file), "GDSC_IC50.csv" + ) + experiment_data = pd.read_csv( + cancer_response_exp_file, sep=",", header=0, index_col=0 + ) + drug_match_list = [ + item + for item in experiment_data.index + if item.split(":")[1] in drugid2pubchemid.keys() + ] + experiment_data_filtered = experiment_data.loc[drug_match_list] + + data_idx: list = []# Each entry contains: cell line ID, drug pubchem ID, IC50 value, cancer type. + for each_drug in experiment_data_filtered.index: + for each_cellline in experiment_data_filtered.columns: + pubchem_id = drugid2pubchemid[each_drug.split(":")[-1]] + if ( + str(pubchem_id) in drug_pubchem_id_set + and each_cellline in gexpr_feature.index + ): + if ( + not np.isnan( + experiment_data_filtered.loc[each_drug, each_cellline] + ) + and each_cellline in cellline2cancertype.keys() + ): + ln_ic50 = float( + experiment_data_filtered.loc[each_drug, each_cellline] + ) + data_idx.append( + ( + each_cellline, + pubchem_id, + ln_ic50, + cellline2cancertype[each_cellline], + ) + ) + + nb_celllines = len(set([item[0] for item in data_idx])) + nb_drugs = len(set([item[1] for item in data_idx])) + print_sys( + f"{len(data_idx)} instances across {nb_celllines} cell lines and {nb_drugs} drugs." + ) + return drug_feature, gexpr_feature, data_idx # Drug molecular graph features, cell line gene expression features, effective (cell line, drug, IC50, cancer type) four-tuple list + + +def data_split(data_idx: list, ratio: float = 0.95): + """Return training and validation splits stratified by cancer type.""" + train_idx, validation_idx = [], [] + for each_type in TCGA_LABEL_SET: + data_subtype_idx = [item for item in data_idx if item[-1] == each_type] + if len(data_subtype_idx) == 0: + continue + k = max(1, int(ratio * len(data_subtype_idx))) + train_list = random.sample(data_subtype_idx, k) + validation_list = [item for item in data_subtype_idx if item not in train_list] + train_idx += train_list + validation_idx += validation_list + return train_idx, validation_idx + + +def feature_extract( + data_idx: list, + drug_feature: dict, + max_atoms: int, +) -> Tuple[list, np.ndarray, list, list]: + """Extract drug map features""" + nb_instance = len(data_idx) + + drug_data = [[] for _ in range(nb_instance)] + target = np.zeros(nb_instance, dtype=np.float32) + cancer_type_list = [] + + for idx in range(nb_instance): + cell_line_id, pubchem_id, ln_ic50, cancer_type = data_idx[idx] + + # Drug map features + feat_mat, adj_list, _ = drug_feature[str(pubchem_id)] + feat, adj = calculate_graph_feat(feat_mat, adj_list, max_atoms) + drug_data[idx] = [feat, adj] + + # target value + target[idx] = ln_ic50 + + cancer_type_list.append([cancer_type, cell_line_id, pubchem_id]) + + return drug_data, target, cancer_type_list, data_idx + + +# ===================== Dataset ===================== + +class DrugPerformerModuleDataset(Dataset): + """Drug + PerformerModule Dataset""" + + def __init__( + self, + drug_data: list, + targets: np.ndarray, + index_list: list, + cell_emb_dict: Dict[str, np.ndarray], + ): + super().__init__() + self.drug_data = drug_data + self.targets = targets.astype(np.float32) + self.index_list = index_list + self.cell_emb_dict = cell_emb_dict + + def __len__(self) -> int: + return int(self.targets.shape[0]) + + def __getitem__(self, idx: int): + # Drug map features + feat, adj = self.drug_data[idx] + drug_feat = torch.from_numpy(feat).float() + drug_adj = torch.from_numpy(adj).float() + + # PerformerModule cell embedding + cell_line_id = self.index_list[idx][0] + cell_emb_np = self.cell_emb_dict[cell_line_id] + cell_emb = torch.from_numpy(cell_emb_np).float() + + # target value + target = torch.tensor(self.targets[idx], dtype=torch.float32) + + return { + "drug_feat": drug_feat, + "drug_adj": drug_adj, + "cell_emb": cell_emb, + "target": target, + } + + +# ===================== Model Definition ===================== + +class GraphConv(nn.Module): + """graph convolution layer""" + + def __init__(self, in_features: int, out_features: int, bias: bool = True): + super().__init__() + self.in_features = in_features + self.out_features = out_features + self.weight = nn.Parameter(torch.Tensor(in_features, out_features)) + if bias: + self.bias = nn.Parameter(torch.Tensor(out_features)) + else: + self.bias = None + self.reset_parameters() + + def reset_parameters(self): + nn.init.xavier_uniform_(self.weight) + if self.bias is not None: + nn.init.zeros_(self.bias) + + def forward(self, x: torch.Tensor, adj: torch.Tensor) -> torch.Tensor: + xw = torch.matmul(x, self.weight) + if self.bias is not None: + xw = xw + self.bias + out = torch.bmm(adj.transpose(1, 2), xw) + return out + + +class DrugPerformerModuleModel(nn.Module): + """ + Drugs + PerformerModule Predictive Model + + Architecture: + - Drug branch: 3-layer GraphConv + residual connection + double pooling + - Cell branch: 2-layer MLP (processing PerformerModule module embeddings) + - Fusion layer: 4-layer MLP + attention mechanism + """ + + def __init__( + self, + drug_feat_dim: int, + cell_emb_dim: int, + hidden_dim: int = 256, + dropout: float = 0.1, + ): + super().__init__() + self.relu = nn.ReLU() + self.dropout = nn.Dropout(dropout) + + # Drug branch: enhanced 3-layer GraphConv + residual connection + self.graph_conv1 = GraphConv(drug_feat_dim, 256) + self.graph_conv2 = GraphConv(256, 256) + self.graph_conv3 = GraphConv(256, 128) + self.drug_bn1 = nn.BatchNorm1d(256) + self.drug_bn2 = nn.BatchNorm1d(256) + self.drug_bn3 = nn.BatchNorm1d(128) + + # Cell branch: 2-layer MLP (processing PerformerModule embeddings) + self.cell_fc1 = nn.Linear(cell_emb_dim, 256) + self.cell_bn1 = nn.BatchNorm1d(256) + self.cell_fc2 = nn.Linear(256, 128) + self.cell_bn2 = nn.BatchNorm1d(128) + + # Fusion dimension: 128 (drug) + 128 (cell) = 256 + fusion_dim = 256 + + # attention mechanism + self.attention = nn.Sequential( + nn.Linear(fusion_dim, fusion_dim // 2), + nn.ReLU(), + nn.Linear(fusion_dim // 2, fusion_dim), + nn.Sigmoid() + ) + + # Fusion layer: 4-layer deep MLP + self.fusion_fc1 = nn.Linear(fusion_dim, hidden_dim * 2) + self.fusion_bn1 = nn.BatchNorm1d(hidden_dim * 2) + self.fusion_fc2 = nn.Linear(hidden_dim * 2, hidden_dim) + self.fusion_bn2 = nn.BatchNorm1d(hidden_dim) + self.fusion_fc3 = nn.Linear(hidden_dim, hidden_dim // 2) + self.fusion_bn3 = nn.BatchNorm1d(hidden_dim // 2) + self.output_fc = nn.Linear(hidden_dim // 2, 1) + + print_sys(f"Model initialized: drug_feat_dim={drug_feat_dim}, " + f"cell_emb_dim={cell_emb_dim}, hidden_dim={hidden_dim}") + + def forward( + self, + drug_feat: torch.Tensor, + drug_adj: torch.Tensor, + cell_emb: torch.Tensor, + ) -> torch.Tensor: + # Drug branch (enhanced version) + # first floor + x_drug = self.graph_conv1(drug_feat, drug_adj) # (B, N, 256) + x_drug = x_drug.transpose(1, 2) # (B, 256, N) for BatchNorm + x_drug = self.drug_bn1(x_drug) + x_drug = x_drug.transpose(1, 2) # (B, N, 256) + x_drug = self.relu(x_drug) + x_drug = self.dropout(x_drug) + + # Second layer + residual connection + x_drug_res = x_drug + x_drug = self.graph_conv2(x_drug, drug_adj) # (B, N, 256) + x_drug = x_drug.transpose(1, 2) + x_drug = self.drug_bn2(x_drug) + x_drug = x_drug.transpose(1, 2) + x_drug = self.relu(x_drug + x_drug_res) # residual connection + x_drug = self.dropout(x_drug) + + # third floor + x_drug = self.graph_conv3(x_drug, drug_adj) # (B, N, 128) + x_drug = x_drug.transpose(1, 2) + x_drug = self.drug_bn3(x_drug) + x_drug = x_drug.transpose(1, 2) + x_drug = self.relu(x_drug) + + # Aggregation: max pooling + mean pooling + x_drug_max = x_drug.max(dim=1)[0] # (B, 128) + x_drug_mean = x_drug.mean(dim=1) # (B, 128) + x_drug = x_drug_max + x_drug_mean # Fusion + + # Cell branching (handling PerformerModule embeddings) + x_cell = self.cell_fc1(cell_emb) # (B, 256) + x_cell = self.cell_bn1(x_cell) + x_cell = self.relu(x_cell) + x_cell = self.dropout(x_cell) + + x_cell = self.cell_fc2(x_cell) # (B, 128) + x_cell = self.cell_bn2(x_cell) + x_cell = self.relu(x_cell) + + # Feature fusion + attention + h = torch.cat([x_drug, x_cell], dim=1) # (B, 256) + + # Apply attention mechanism + att_weights = self.attention(h) + h = h * att_weights # weighted features + + # Deep Fusion MLP + h = self.fusion_fc1(h) # (B, hidden_dim * 2) + h = self.fusion_bn1(h) + h = self.relu(h) + h = self.dropout(h) + + h = self.fusion_fc2(h) # (B, hidden_dim) + h = self.fusion_bn2(h) + h = self.relu(h) + h = self.dropout(h) + + h = self.fusion_fc3(h) # (B, hidden_dim // 2) + h = self.fusion_bn3(h) + h = self.relu(h) + h = self.dropout(h) + + # output layer + out = self.output_fc(h) # (B, 1) + return out + + +# ===================== Training and Evaluation ===================== + +def train_epoch(model, loader, optimizer, criterion, device): + """train for one epoch""" + model.train() + total_loss = 0.0 + total = 0 + for batch in loader: + drug_feat = batch["drug_feat"].to(device) + drug_adj = batch["drug_adj"].to(device) + cell_emb = batch["cell_emb"].to(device) + target = batch["target"].to(device) + + optimizer.zero_grad(set_to_none=True) + pred = model(drug_feat, drug_adj, cell_emb).squeeze(-1) + + loss = criterion(pred, target) + loss.backward() + + # gradient crop + torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0) + + optimizer.step() + + total_loss += float(loss.item()) * target.size(0) + total += target.size(0) + return total_loss / total if total > 0 else 0.0 + + +def eval_epoch(model, loader, device): + """Evaluate an epoch""" + model.eval() + preds = [] + trues = [] + with torch.no_grad(): + for batch in loader: + drug_feat = batch["drug_feat"].to(device) + drug_adj = batch["drug_adj"].to(device) + cell_emb = batch["cell_emb"].to(device) + target = batch["target"].to(device) + + pred = model(drug_feat, drug_adj, cell_emb).squeeze(-1) + + preds.append(pred.cpu().numpy()) + trues.append(target.cpu().numpy()) + + preds = np.concatenate(preds, axis=0) + trues = np.concatenate(trues, axis=0) + pcc = pearsonr(preds, trues)[0] + return preds, trues, pcc + + +# ===================== Main ===================== + +def main(): + parser = argparse.ArgumentParser( + description="Drug response prediction using Drug + PerformerModule" + ) + parser.add_argument("--gpu_id", type=str, default="0") + parser.add_argument("--data_path", type=str, default=str(DOWNSTREAM_DATA_DIR / "single_drug_response" / "data")) + parser.add_argument("--epochs", type=int, default=300) + parser.add_argument("--batch_size", type=int, default=32) + parser.add_argument("--max_atoms", type=int, default=100) + parser.add_argument("--hidden_dim", type=int, default=256) + parser.add_argument("--learning_rate", type=float, default=1e-3) + parser.add_argument("--dropout", type=float, default=0.1) + parser.add_argument("--patience", type=int, default=30) + + # PerformerModule related paths + parser.add_argument( + "--checkpoint_path", + type=str, + required=True, + help="Path to pre-trained PerformerModule checkpoint" + ) + parser.add_argument( + "--hyper_params_path", + type=str, + required=True, + help="Path to PerformerModule model hyper parameters pkl file" + ) + parser.add_argument( + "--module_data_dir", + type=str, + default=None, + help="Directory containing module JSON files" + ) + parser.add_argument( + "--module_data_path", + type=str, + default=None, + help="Path to module JSON file (single file)" + ) + parser.add_argument( + "--gene_list_path", + type=str, + default=str(ASSET_DIR / "selected_genes_27k.txt"), + help="Path to gene list file" + ) + parser.add_argument( + "--gene_mapping_path", + type=str, + default=str(ASSET_DIR / "gene2ens.csv"), + help="Path to gene symbol to Ensembl ID mapping" + ) + parser.add_argument( + "--gene2vec_path", + type=str, + default=str(ASSET_DIR / "selected_gene2vec_27k.npy"), + help="Path to gene2vec file" + ) + parser.add_argument( + "--symbol2ens_path", + type=str, + default=str(ASSET_DIR / "gene2ens.csv"), + help="Path to symbol to Ensembl mapping (for data processing)" + ) + parser.add_argument( + "--pool_method", + type=str, + default="mean", + choices=["mean", "max", "cls", "concat"], + help="Pooling method for PerformerModule module embeddings" + ) + parser.add_argument( + "--shuffle_module_genes", + action="store_true", + help="Optional downstream-only extra reshuffle on top of module_sets restored from PKL. " + "Do not pass this when reproducing the pretrained backbone as-is." + ) + parser.add_argument( + "--shuffle_seed", + type=int, + default=2021, + help="Random seed for the downstream extra reshuffle when --shuffle_module_genes is enabled." + ) + # Output path + parser.add_argument( + "--save_dir", + type=str, + default="./models_drug_performer_module", + help="Directory to save the model" + ) + + args = parser.parse_args() + + os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu_id + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + + print_sys("=" * 70) + print_sys("Drug Response Prediction: Drug + PerformerModule") + print_sys("=" * 70) + print_sys(f"Device: {device}") + print_sys(f"Epochs: {args.epochs}") + print_sys(f"Batch size: {args.batch_size}") + print_sys(f"Learning rate: {args.learning_rate}") + print_sys(f"Hidden dim: {args.hidden_dim}") + print_sys(f"Dropout: {args.dropout}") + print_sys(f"Pool method: {args.pool_method}") + print_sys(f"Early stopping patience: {args.patience}") + print_sys("=" * 70) + + # Create save directory + os.makedirs(args.save_dir, exist_ok=True) + + dpath = args.data_path + drug_info_file = f"{dpath}/GDSC/1.Drug_listMon Jun 24 09_00_55 2019.csv" + cell_line_info_file = f"{dpath}/CCLE/Cell_lines_annotations_20181226.txt" + drug_feature_dir = f"{dpath}/GDSC/drug_graph_feat" + gene_expression_file = ( + f"{dpath}/CCLE/genomic_expression_561celllines_697genes_demap_features.csv" + ) + + # fixed random seed + random.seed(0) + np.random.seed(0) + torch.manual_seed(0) + if torch.cuda.is_available(): + torch.cuda.manual_seed_all(0) + + # 1) Read metadata + print_sys("\n[Step 1/6] Loading metadata...") + drug_feature, gexpr_feature, data_idx = metadata_generate( + drug_info_file, + cell_line_info_file, + drug_feature_dir, + gene_expression_file, + ) + + # 2) Build PerformerModule embeddings + print_sys("\n[Step 2/6] Building PerformerModule embeddings...") + cell_emb_dict, cell_emb_dim = build_cell_embeddings_with_performer_module( # cell_emb_dict │ Dict[str, np.ndarray] │ {cell_line_id: embedding} mapping dictionary + gexpr_feature=gexpr_feature, + symbol2ens_path=args.symbol2ens_path, + gene_list_path=args.gene_list_path, + gene_mapping_path=args.gene_mapping_path, + gene2vec_path=args.gene2vec_path, + checkpoint_path=args.checkpoint_path, + hyper_params_path=args.hyper_params_path, + module_data_dir=args.module_data_dir, + module_data_path=args.module_data_path, + device=device, + pool_method=args.pool_method, + shuffle_module_genes=args.shuffle_module_genes, + seed=args.shuffle_seed, + ) + + # 3) Split training and validation data; validation is used for model selection + print_sys("\n[Step 3/6] Splitting dataset...") + train_idx, validation_idx = data_split(data_idx) + print_sys(f"Train samples: {len(train_idx)}, Validation samples: {len(validation_idx)}") + + # 4) Extract features + print_sys("\n[Step 4/6] Extracting features...") + X_drug_train, Y_train, train_cancer_info, train_index_list = feature_extract( + train_idx, drug_feature, args.max_atoms + ) + X_drug_validation, Y_validation, validation_cancer_info, validation_index_list = feature_extract( + validation_idx, drug_feature, args.max_atoms + ) + + # 5) Build a data set + print_sys("\n[Step 5/6] Creating datasets...") + train_ds = DrugPerformerModuleDataset( + X_drug_train, Y_train, train_index_list, cell_emb_dict + ) + validation_ds = DrugPerformerModuleDataset( + X_drug_validation, Y_validation, validation_index_list, cell_emb_dict + ) + + drug_dim = X_drug_train[0][0].shape[-1] + + num_workers = min(4, os.cpu_count() or 1) + train_loader = DataLoader( + train_ds, + batch_size=args.batch_size, + shuffle=True, + num_workers=num_workers, + pin_memory=torch.cuda.is_available(), + ) + validation_loader = DataLoader( + validation_ds, + batch_size=args.batch_size, + shuffle=False, + num_workers=num_workers, + pin_memory=torch.cuda.is_available(), + ) + + # 6) Build the model + print_sys("\n[Step 6/6] Building model...") + model = DrugPerformerModuleModel( + drug_feat_dim=drug_dim, + cell_emb_dim=cell_emb_dim, + hidden_dim=args.hidden_dim, + dropout=args.dropout, + ).to(device) + + # Statistical parameters + total_params = sum(p.numel() for p in model.parameters()) + trainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad) + print_sys(f"Total parameters: {total_params:,}") + print_sys(f"Trainable parameters: {trainable_params:,}") + + # Optimizers and schedulers + optimizer = torch.optim.AdamW( + model.parameters(), lr=args.learning_rate, weight_decay=1e-5 + ) + scheduler = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts( + optimizer, T_0=20, T_mult=2, eta_min=1e-6 + ) + criterion = torch.nn.HuberLoss(delta=1.0) + + # train + print_sys("\n" + "=" * 70) + print_sys("Starting training...") + print_sys("=" * 70) + + best_pcc = -1.0 + best_pred = None + best_true = None + best_epoch = 0 + patience_counter = 0 + + for epoch in range(1, args.epochs + 1): + train_loss = train_epoch(model, train_loader, optimizer, criterion, device) + preds, trues, pcc = eval_epoch(model, validation_loader, device) + + scheduler.step() + current_lr = optimizer.param_groups[0]["lr"] + + print_sys( + f"Epoch {epoch:03d}/{args.epochs} | " + f"train_loss={train_loss:.4f} | " + f"validation_pcc={pcc:.4f} | " + f"lr={current_lr:.6f}" + ) + + if pcc > best_pcc: + best_pcc = pcc + best_pred = preds + best_true = trues + best_epoch = epoch + patience_counter = 0 + + # Save the best model selected by validation PCC + model_save_path = os.path.join(args.save_dir, "best_model.pth") + torch.save( + { + "epoch": epoch, + "model_state_dict": model.state_dict(), + "optimizer_state_dict": optimizer.state_dict(), + "scheduler_state_dict": scheduler.state_dict(), + "best_pcc": best_pcc, + "train_loss": train_loss, + "args": vars(args), + }, + model_save_path, + ) + print_sys(f" → Best model saved! Validation PCC improved to {best_pcc:.4f}") + else: + patience_counter += 1 + if patience_counter >= args.patience: + print_sys( + f"\n[Early Stopping] No improvement for {args.patience} epochs." + ) + break + + # Save final result + print_sys("\n" + "=" * 70) + print_sys("Training completed!") + print_sys("=" * 70) + print_sys(f"Best epoch: {best_epoch}") + print_sys(f"Best validation PCC: {best_pcc:.4f}") + + if best_pred is not None: + result_path = os.path.join(args.save_dir, "predictions.npz") + np.savez(result_path, pred=best_pred, validation=best_true) + print_sys(f"Validation predictions saved to: {result_path}") + + print_sys(f"Best model saved to: {os.path.join(args.save_dir, 'best_model.pth')}") + print_sys("=" * 70) + + +if __name__ == "__main__": + main() + diff --git a/BioMod/gene_perturbation/README.md b/BioMod/gene_perturbation/README.md index dbc969d53b0e220b3807305ebd1e9c83e87109a1..225701b1a88517ca9217396837a1b489a7a823c7 100644 --- a/BioMod/gene_perturbation/README.md +++ b/BioMod/gene_perturbation/README.md @@ -1,267 +1,276 @@ -# Gene Perturbation Prediction with BioMod - -This document describes the BioMod gene perturbation workflow (GEARS), using -the `PerformerEncoderOnly_Module` encoder. The -[BioMod model weights and matching configuration](../../README.md#biomod-model-weights) -are shared with the other downstream tasks. - -## Overview - -This adaptation replaces the original GEARS gene perturbation model based on -`DualEncoderSCFM` with `PerformerEncoderOnly_Module`. The main adaptation points -are: - -1. **Model architecture adaptation**: replace the original dual-encoder model with a module-based encoder-only model. -2. **Data format adaptation**: handle module-based input data. -3. **Embedding extraction adaptation**: recover gene-level embeddings from module encodings. - -## Main Adaptations - -### 1. `SCFM_Module` (`model_module.py`) - -`SCFM_Module` wraps `PerformerEncoderOnly_Module` and provides these functions: - -- **Input handling**: reindex input gene expression values to the BioMod gene order. -- **Module encoding**: obtain module-level encodings from `PerformerEncoderOnly_Module`. -- **Gene embedding recovery**: recover gene-level embeddings from module encodings. - - For each gene, find all modules that contain it. - - Average the encodings of those modules to obtain the gene embedding. - - If a gene is not included in any module, use the global average as a fallback. -- **Output reindexing**: reindex gene embeddings back to the input gene order. - -**Key adaptation**: - -```python -# Recover gene-level embeddings from module encodings. -for gene_idx in range(len(self.scfm_index_positions) - 1): - if gene_idx in self.gene_to_modules: - module_indices = self.gene_to_modules[gene_idx] - module_embs = module_encodings[:, module_indices, :] # B, num_modules_for_gene, D - gene_emb = module_embs.mean(dim=1) # B, D - else: - gene_emb = module_encodings.mean(dim=1) # B, D (global average) -``` - -### 2. `GEARS_Model_Module` (`model_module.py`) - -`GEARS_Model_Module` inherits the original GEARS architecture. Main changes: - -- **Use `SCFM_Module`** instead of the original `SCFM_different_genes`. -- **Module data loading** supports loading module data from hyper-parameters or file paths. -- **Other components remain unchanged**, including the GNN layers and decoder. - -### 3. Training Script (`train_module.py`) - -The training script is based on `GEARS/train_dual_ddp_reduce_dec.py`. Main changes: - -- **Model class**: use `GEARS_Model_Module` instead of `GEARS_Model_Acc`. -- **New arguments**: - - `--module_data_path`: path to one module JSON file. - - `--module_data_dir`: directory containing module JSON files. - - `--gene_mapping_path`: CSV file mapping gene symbols to Ensembl IDs. - -## Usage - -### 1. Prepare Data - -Make sure the following files exist: - -- Perturbation data, such as Norman, Adamson, or Dixit. -- BioMod model weights and their matching configuration file. -- Module data in JSON format. -- Gene mapping file, such as `gene2ens.csv`. - -### 2. Run Training - -```bash -# DDP training on multiple GPUs. -python -m torch.distributed.launch --nproc_per_node=4 \ - train_module.py \ - --data_dir /path/to/data \ - --data_name norman \ - --scfm_hyper_params_path /path/to/hyper_params.pkl \ - --scfm_ckpt_path /path/to/checkpoint.pth \ - --module_data_dir /path/to/module_data \ - --gene_mapping_path /path/to/gene2ens.csv \ - --scfm_genes_list_path /path/to/selected_genes_27k.txt \ - --scfm_gene2vec_file /path/to/selected_gene2vec_27k.npy \ - --result_dir ./results_module \ - --run_name norman_module_experiment \ - --epochs 20 \ - --batch_size 32 \ - --lr 1e-3 -``` - -### 3. Arguments - -**Required arguments**: - -- `--scfm_hyper_params_path`: path to the BioMod model configuration file. -- `--scfm_ckpt_path`: path to the BioMod model weights. -- `--module_data_path` or `--module_data_dir`: module data path; provide one of them. - -**Recommended arguments**: - -- `--gene_mapping_path`: gene mapping file, especially when module data use gene symbols. -- `--scfm_genes_list_path`: BioMod gene list file. -- `--scfm_gene2vec_file`: path to the gene2vec file. - -**Optional arguments**: - -- `--use_module_level`: use module-level features. This is experimental and is not fully supported at the moment. - - Default: False, meaning gene-level features are used and recommended. - - See `METHODS_COMPARISON.md` for details. - -## Comparison with Original sclong - -### Main Differences - -1. **Model architecture**: - - **Original sclong**: uses `DualEncoderSCFM` and directly processes gene sequences. - - **Module version**: uses `PerformerEncoderOnly_Module` and processes module-based data. - -2. **Data representation**: - - **Original sclong**: gene-level embeddings come directly from encoder outputs. - - **Module version**: gene-level embeddings are recovered from module encodings by mean pooling. - -3. **Input format**: - - **Original sclong**: directly inputs gene expression vectors. - - **Module version**: keeps the same external input format but internally processes data with module embeddings. - -### Performance Comparison - -To compare model performance: - -1. **Run original sclong**: - - ```bash - cd GEARS - python train_dual_ddp_reduce_dec.py [arguments...] - ``` - -2. **Run the module version**: - - ```bash - cd performer_module/gene_perturbation - python train_module.py [arguments...] - ``` - -3. **Compare metrics**: - - - MSE (Mean Squared Error) - - Pearson correlation - - MSE and Pearson correlation for the top 20 DE genes - - Direction accuracy and related metrics - -Results are saved under each `result_dir`. Compare these metrics to evaluate the -module version. - -## Technical Details - -### Module-to-Gene Mapping - -In `SCFM_Module`, `gene_to_modules` records which modules contain each gene: - -```python -self.gene_to_modules = {} # {gene_idx: [module_idx1, module_idx2, ...]} -for module_idx, module_genes in enumerate(module_sets): - for gene_idx in module_genes: - if gene_idx < len(scfm_genes): - if gene_idx not in self.gene_to_modules: - self.gene_to_modules[gene_idx] = [] - self.gene_to_modules[gene_idx].append(module_idx) -``` - -### Gene Embedding Recovery Strategy - -For each gene: - -1. If the gene appears in some modules, average the encodings of those modules. -2. If the gene does not appear in any module, use the global average of all modules as a fallback. - -This strategy: - -- Makes full use of module information. -- Provides a reasonable embedding for genes absent from all modules. -- Keeps embedding dimensionality similar to the original model. - -## File Structure - -```text -BioMod/gene_perturbation/ # Perturbation response prediction: processing, training and inference -├── data_utils_local.py -├── gears_reduce_local.py -├── inference_local.py -├── model_module.py -├── pertdata_local.py -├── README.md -├── scfm_utils_local.py -├── train_module.py -└── utils.py -``` - -## Notes - -1. **Module data format**: make sure the gene indices or names in the module data match the BioMod gene list. -2. **Gene mapping**: provide `gene_mapping_path` if the module data use gene symbols. -3. **Memory usage**: the module version may require more memory because it maintains the module-to-gene mapping. -4. **Training stability**: use the same learning rate and training strategy as original sclong when possible. - -## Troubleshooting - -### Issue 1: Module Data Not Found - -- Check whether `--module_data_path` or `--module_data_dir` is correct. -- Make sure the module data file exists and has the expected format. - -### Issue 2: Gene Mapping Failed - -- Check whether `--gene_mapping_path` is correct. -- Make sure the gene mapping file is a CSV containing `Approved symbol` and `Ensembl gene ID` columns. - -### Issue 3: Model Loading Failed - -- Check whether the checkpoint path is correct. -- Make sure `hyper_params` contains the required module information. - -## Gene-Based vs Module-Based Methods - -This implementation supports two feature extraction modes controlled by -`--use_module_level`. - -### Method 1: Gene-Based (Default, Recommended) - -```bash -# Default mode; do not pass --use_module_level or set it to False. -python train_module.py [other arguments...] -``` - -**Characteristics**: - -- Recovers gene-level embeddings from module encodings. -- Fully compatible with the original GEARS architecture. -- Downstream tasks operate at gene level, including the GNN and decoder. -- Outputs gene-level predictions. - -### Method 2: Module-Based (Experimental, Not Fully Implemented) - -```bash -# Enable module-level mode. -python train_module.py --use_module_level [other arguments...] -``` - -**Characteristics**: - -- Directly uses module-level encodings. -- Preserves the full module-level information. -- **Note**: module-level downstream tasks are not fully implemented in the current version. -- Requires redesigning the downstream architecture, including the GNN and decoder. - -See `METHODS_COMPARISON.md` for the detailed comparison and rationale. - -## References - -- Original GEARS code: `GEARS/gears_reduce/` -- PerformerEncoderOnly_Module model: `performer_module/models_performer_module.py` -- Module data processing: `performer_module/module_dataset.py` -- Method comparison: `METHODS_COMPARISON.md` +# Gene Perturbation Prediction with BioMod + +This document describes the BioMod gene perturbation workflow (GEARS), using +the `PerformerEncoderOnly_Module` encoder. The +[BioMod model weights and matching configuration](../../README.md#biomod-model-weights) +are shared with the other downstream tasks. + +## Overview + +This adaptation replaces the original GEARS gene perturbation model based on +`DualEncoderSCFM` with `PerformerEncoderOnly_Module`. The main adaptation points +are: + +1. **Model architecture adaptation**: replace the original dual-encoder model with a module-based encoder-only model. +2. **Data format adaptation**: handle module-based input data. +3. **Embedding extraction adaptation**: recover gene-level embeddings from module encodings. + +## Main Adaptations + +### 1. `SCFM_Module` (`model_module.py`) + +`SCFM_Module` wraps `PerformerEncoderOnly_Module` and provides these functions: + +- **Input handling**: reindex input gene expression values to the BioMod gene order. +- **Module encoding**: obtain module-level encodings from `PerformerEncoderOnly_Module`. +- **Gene embedding recovery**: recover gene-level embeddings from module encodings. + - For each gene, find all modules that contain it. + - Average the encodings of those modules to obtain the gene embedding. + - If a gene is not included in any module, use the global average as a fallback. +- **Output reindexing**: reindex gene embeddings back to the input gene order. + +**Key adaptation**: + +```python +# Recover gene-level embeddings from module encodings. +for gene_idx in range(len(self.scfm_index_positions) - 1): + if gene_idx in self.gene_to_modules: + module_indices = self.gene_to_modules[gene_idx] + module_embs = module_encodings[:, module_indices, :] # B, num_modules_for_gene, D + gene_emb = module_embs.mean(dim=1) # B, D + else: + gene_emb = module_encodings.mean(dim=1) # B, D (global average) +``` + +### 2. `GEARS_Model_Module` (`model_module.py`) + +`GEARS_Model_Module` inherits the original GEARS architecture. Main changes: + +- **Use `SCFM_Module`** instead of the original `SCFM_different_genes`. +- **Module data loading** supports loading module data from hyper-parameters or file paths. +- **Other components remain unchanged**, including the GNN layers and decoder. + +### 3. Training Script (`train_module.py`) + +The training script is based on `GEARS/train_dual_ddp_reduce_dec.py`. Main changes: + +- **Model class**: use `GEARS_Model_Module` instead of `GEARS_Model_Acc`. +- **New arguments**: + - `--module_data_path`: path to one module JSON file. + - `--module_data_dir`: directory containing module JSON files. + - `--gene_mapping_path`: CSV file mapping gene symbols to Ensembl IDs. + +## Usage + +### 1. Prepare Data + +Make sure the following files exist: + +- Perturbation data, such as Norman, Adamson, or Dixit. +- BioMod model weights and their matching configuration file. +- Module data in JSON format. +- Gene mapping file, such as `gene2ens.csv`. + +### 2. Run Training + +Run from the repository root in a Linux CUDA/NCCL environment. Built-in asset defaults are derived from the script location; explicit relative paths resolve from the working directory. + +```bash +# DDP training on multiple GPUs. +python -m torch.distributed.run --standalone --nproc_per_node=4 \ + BioMod/gene_perturbation/train_module.py \ + --data_dir data/downstream/final_data \ + --data_name norman \ + --scfm_hyper_params_path weight/0810model/mae_tost_70M_75mask_ec2.pkl \ + --scfm_ckpt_path weight/0810model/mae_tost_70M_75mask_ec2_epoch0.pth \ + --module_data_dir model_dependencies/module_data_all \ + --gene_mapping_path model_dependencies/gene2ens.csv \ + --scfm_genes_list_path model_dependencies/selected_genes_27k.txt \ + --scfm_gene2vec_file model_dependencies/selected_gene2vec_27k.npy \ + --result_dir tutorial_runs/gene_perturbation \ + --run_name norman_module_experiment \ + --epochs 20 \ + --batch_size 32 \ + --lr 1e-3 +``` + +### 3. Arguments + +**Model arguments** (defaults point to the bundled BioMod assets): + +- `--scfm_hyper_params_path`: path to the BioMod model configuration file. +- `--scfm_ckpt_path`: path to the BioMod model weights. +- `--module_data_path` or `--module_data_dir`: required only when the model configuration does not already contain module definitions. + +**Recommended arguments**: + +- `--gene_mapping_path`: gene mapping file, especially when module data use gene symbols. +- `--scfm_genes_list_path`: BioMod gene list file. +- `--scfm_gene2vec_file`: path to the gene2vec file. + +**Optional arguments**: + +- `--use_module_level`: use module-level features. This is experimental and is not fully supported at the moment. + - Default: False, meaning gene-level features are used and recommended. + +## Comparison with Original sclong + +### Main Differences + +1. **Model architecture**: + - **Original sclong**: uses `DualEncoderSCFM` and directly processes gene sequences. + - **Module version**: uses `PerformerEncoderOnly_Module` and processes module-based data. + +2. **Data representation**: + - **Original sclong**: gene-level embeddings come directly from encoder outputs. + - **Module version**: gene-level embeddings are recovered from module encodings by mean pooling. + +3. **Input format**: + - **Original sclong**: directly inputs gene expression vectors. + - **Module version**: keeps the same external input format but internally processes data with module embeddings. + +### Performance Comparison + +To compare model performance: + +1. **Run original sclong**: + + ```bash + # Requires a separate checkout of the original scLong/GEARS code. + python /path/to/upstream/GEARS/train_dual_ddp_reduce_dec.py [arguments...] + ``` + +2. **Run the module version**: + + ```bash + # Run from the BioMod repository root. + python -m torch.distributed.run --standalone --nproc_per_node=1 BioMod/gene_perturbation/train_module.py [arguments...] + ``` + +3. **Compare metrics**: + + - MSE (Mean Squared Error) + - Pearson correlation + - MSE and Pearson correlation for the top 20 DE genes + - Direction accuracy and related metrics + +Results are saved under each `result_dir`. Compare these metrics to evaluate the +module version. + +## Technical Details + +### Module-to-Gene Mapping + +In `SCFM_Module`, `gene_to_modules` records which modules contain each gene: + +```python +self.gene_to_modules = {} # {gene_idx: [module_idx1, module_idx2, ...]} +for module_idx, module_genes in enumerate(module_sets): + for gene_idx in module_genes: + if gene_idx < len(scfm_genes): + if gene_idx not in self.gene_to_modules: + self.gene_to_modules[gene_idx] = [] + self.gene_to_modules[gene_idx].append(module_idx) +``` + +### Gene Embedding Recovery Strategy + +For each gene: + +1. If the gene appears in some modules, average the encodings of those modules. +2. If the gene does not appear in any module, use the global average of all modules as a fallback. + +This strategy: + +- Makes full use of module information. +- Provides a reasonable embedding for genes absent from all modules. +- Keeps embedding dimensionality similar to the original model. + +## File Structure + +```text +BioMod/gene_perturbation/ # Perturbation response prediction: processing, training and inference +├── data_utils_local.py +├── gears_reduce_local.py +├── inference_local.py +├── model_module.py +├── pertdata_local.py +├── README.md +├── scfm_utils_local.py +├── train_module.py +└── utils.py +``` + +## Notes + +1. **Module data format**: make sure the gene indices or names in the module data match the BioMod gene list. +2. **Gene mapping**: provide `gene_mapping_path` if the module data use gene symbols. +3. **Memory usage**: the module version may require more memory because it maintains the module-to-gene mapping. +4. **Training stability**: use the same learning rate and training strategy as original sclong when possible. + +## Troubleshooting + +### Issue 1: Module Data Not Found + +- Check whether `--module_data_path` or `--module_data_dir` is correct. +- Make sure the module data file exists and has the expected format. + +### Issue 2: Gene Mapping Failed + +- Check whether `--gene_mapping_path` is correct. +- Make sure the gene mapping file is a CSV containing `Approved symbol` and `Ensembl gene ID` columns. + +### Issue 3: Model Loading Failed + +- Check whether the checkpoint path is correct. +- Make sure `hyper_params` contains the required module information. + +## Gene-Based vs Module-Based Methods + +This implementation supports two feature extraction modes controlled by +`--use_module_level`. + +### Method 1: Gene-Based (Default, Recommended) + +```bash +# Default mode; do not pass --use_module_level or set it to False. +python -m torch.distributed.run --standalone --nproc_per_node=1 BioMod/gene_perturbation/train_module.py [other arguments...] +``` + +**Characteristics**: + +- Recovers gene-level embeddings from module encodings. +- Fully compatible with the original GEARS architecture. +- Downstream tasks operate at gene level, including the GNN and decoder. +- Outputs gene-level predictions. + +### Method 2: Module-Based (Experimental, Not Fully Implemented) + +```bash +# Enable module-level mode. +python -m torch.distributed.run --standalone --nproc_per_node=1 BioMod/gene_perturbation/train_module.py --use_module_level [other arguments...] +``` + +**Characteristics**: + +- Directly uses module-level encodings. +- Preserves the full module-level information. +- **Note**: module-level downstream tasks are not fully implemented in the current version. +- Requires redesigning the downstream architecture, including the GNN and decoder. + + +## References + +- Original GEARS code is an external reference and is not included in this source directory. +- PerformerEncoderOnly_Module model: `BioMod/models_performer_module.py` +- Module data processing: `BioMod/module_dataset.py` + +## Optional external annotation resources + +The bundled Norman data include `go.csv`. If that network must be regenerated, +provide `gene2go_path` to `get_go_auto` or set `BIOMOD_GENE2GO_PATH` to the +appropriate `gene2go_all.pkl` file. This annotation file is not bundled. + +The optional `get_high_umi_idx` helper requires `high_umi_path` or +`BIOMOD_HIGH_UMI_PATH` pointing to `genes_with_hi_mean.npy`; that resource is also +not bundled. Neither helper silently searches an old workstation path. diff --git a/BioMod/gene_perturbation/gears_reduce_local.py b/BioMod/gene_perturbation/gears_reduce_local.py index a53c66484d376570f9b7150bb9f62de02019aa22..9331c841cb1bb6492ed914727519273fadace819 100644 --- a/BioMod/gene_perturbation/gears_reduce_local.py +++ b/BioMod/gene_perturbation/gears_reduce_local.py @@ -459,7 +459,7 @@ def predict(self, pert_list): else: return results_pred - def GI_predict(self, combo, GI_genes_file='./genes_with_hi_mean.npy'): + def GI_predict(self, combo, GI_genes_file=None): """ Predict the genetic interaction effect of gene pairs @@ -472,8 +472,14 @@ def GI_predict(self, combo, GI_genes_file='./genes_with_hi_mean.npy'): [parameter] - combo: gene pair, such as ('MAPK1', 'KRAS') - - GI_genes_file: Gene list file used for GI analysis + - GI_genes_file: Caller-supplied gene list used for GI analysis; alternatively set BIOMOD_HIGH_UMI_PATH. """ + GI_genes_file = GI_genes_file or os.environ.get('BIOMOD_HIGH_UMI_PATH') + if not GI_genes_file or not os.path.isfile(GI_genes_file): + raise FileNotFoundError( + "Provide GI_genes_file or set BIOMOD_HIGH_UMI_PATH to the " + "genes_with_hi_mean.npy resource; this file is not bundled." + ) try: # If there is already a prediction, use it directly pred = {} diff --git a/BioMod/gene_perturbation/inference_local.py b/BioMod/gene_perturbation/inference_local.py index 32652e993655f585e1ab9d778530acdc6d9b27b8..2ea5fb0379e1d7a487ea6f2844430367906f6db8 100644 --- a/BioMod/gene_perturbation/inference_local.py +++ b/BioMod/gene_perturbation/inference_local.py @@ -1,1034 +1,1038 @@ -""" -================================================================================ -Model evaluation module (inference_local.py) -================================================================================ - -[Module function] -Provides evaluation and analysis functions for genetic perturbation models, including: - -1. evaluate(): main evaluation function, calculates MSE, Pearson correlation coefficient and other indicators -2. compute_metrics(): Calculate the complete dictionary of evaluation metrics -3. deeper_analysis(): In-depth analysis of prediction results -4. non_dropout_analysis(): analyze on non-dropout genes - -[Description of key indicators] -- MSE: mean square error of all genes -- MSE_DE: MSE of Top 20 differentially expressed genes (paying more attention to biological significance) -- Pearson: Pearson correlation coefficient of all genes -- Pearson_DE: Pearson correlation coefficient of Top 20 DE genes - -[Evaluation Notes] -- real_num_genes: only evaluate on real genes, excluding extended/completed genes -- de_idx: The differentially expressed gene index of each sample may be different -================================================================================ -""" - -import torch -import numpy as np -import scanpy as sc - -from sklearn.metrics import r2_score -from scipy.stats import pearsonr, spearmanr -from sklearn.metrics import mean_squared_error as mse -from sklearn.metrics import mean_absolute_error as mae - -def evaluate(loader, model, uncertainty, device, real_num_genes = None): - """ - main evaluation function - - [Function] - Runs the model on the given data loader, collecting predicted and true values. - - [Key steps] - 1. Traverse all batches of the data loader - 2. Run model inference (without calculating gradients) - 3. Collect predicted values, true values, and DE gene predicted values - - [Parameter description] - - real_num_genes: only evaluate the first N real genes to avoid expanding gene interference indicators - """ - - model.eval() - model.to(device) - pert_cat = [] - pred = [] - truth = [] - pred_de = [] - truth_de = [] - results = {} - logvar = [] - - for itr, batch in enumerate(loader): - - batch.to(device) - pert_cat.extend(batch.pert) - - with torch.no_grad(): - if uncertainty: - p, unc = model(batch) - logvar.extend(unc.cpu()) - else: - p = model(batch) - # Only evaluate in the real gene dimension to avoid the interference of completed/expanded genes on indicators. - p = p[:, :real_num_genes] - t = batch.y[:, :real_num_genes] - pred.extend(p.cpu()) - truth.extend(t.cpu()) - - # The DE indicator is only counted on the de_idx corresponding to the sample and is one of the core calibers of downstream reporting. - for itr, de_idx in enumerate(batch.de_idx): - pred_de.append(p[itr, de_idx]) - truth_de.append(t[itr, de_idx]) - - # all genes - results['pert_cat'] = np.array(pert_cat) - pred = torch.stack(pred) - truth = torch.stack(truth) - results['pred']= pred.detach().cpu().numpy() - results['truth']= truth.detach().cpu().numpy() - - pred_de = torch.stack(pred_de) - truth_de = torch.stack(truth_de) - results['pred_de']= pred_de.detach().cpu().numpy() - results['truth_de']= truth_de.detach().cpu().numpy() - - if uncertainty: - results['logvar'] = torch.stack(logvar).detach().cpu().numpy() - - return results - -#This evaluation is just as test. IT HAS INFORMATION LEAKAGE! -def evaluate_with_nonzero_mask(loader, model, uncertainty, device): - """ - Run model in inference mode using a given data loader - """ - - model.eval() - model.to(device) - pert_cat = [] - pred = [] - truth = [] - pred_de = [] - truth_de = [] - results = {} - logvar = [] - - for itr, batch in enumerate(loader): - - batch.to(device) - pert_cat.extend(batch.pert) - - with torch.no_grad(): - if uncertainty: - p, unc = model(batch) - logvar.extend(unc.cpu()) - else: - p = model(batch) - t = batch.y - p = p * (t != 0) - pred.extend(p.cpu()) - truth.extend(t.cpu()) - - # Differentially expressed genes - for itr, de_idx in enumerate(batch.de_idx): - pred_de.append(p[itr, de_idx]) - truth_de.append(t[itr, de_idx]) - - # all genes - results['pert_cat'] = np.array(pert_cat) - pred = torch.stack(pred) - truth = torch.stack(truth) - results['pred']= pred.detach().cpu().numpy() - results['truth']= truth.detach().cpu().numpy() - - pred_de = torch.stack(pred_de) - truth_de = torch.stack(truth_de) - results['pred_de']= pred_de.detach().cpu().numpy() - results['truth_de']= truth_de.detach().cpu().numpy() - - if uncertainty: - results['logvar'] = torch.stack(logvar).detach().cpu().numpy() - - return results - - -def non_zero_evaluate(loader, model, uncertainty, device, - delta_list, penalty = { - "false_neg": 1, - "false_pos": 1 }): - """ - Run model in inference mode using a given data loader - """ - - model.eval() - model.to(device) - pert_cat = [] - pred = [] - truth = [] - pred_de = [] - truth_de = [] - results = {} - logvar = [] - - for itr, batch in enumerate(loader): - - batch.to(device) - pert_cat.extend(batch.pert) - - with torch.no_grad(): - if uncertainty: - p, unc = model(batch) - logvar.extend(unc.cpu()) - else: - p = model(batch) - t = batch.y - pred.extend(p.cpu()) - truth.extend(t.cpu()) - - # Differentially expressed genes - for itr, de_idx in enumerate(batch.de_idx): - pred_de.append(p[itr, de_idx]) - truth_de.append(t[itr, de_idx]) - - # all genes - results['pert_cat'] = np.array(pert_cat) - pred = torch.stack(pred) - truth = torch.stack(truth) - results['pred']= pred.detach().cpu().numpy() - results['truth']= truth.detach().cpu().numpy() - - pred_de = torch.stack(pred_de) - truth_de = torch.stack(truth_de) - results['pred_de']= pred_de.detach().cpu().numpy() - results['truth_de']= truth_de.detach().cpu().numpy() - - if uncertainty: - results['logvar'] = torch.stack(logvar).detach().cpu().numpy() - - return results - - -def compute_metrics(results, gene_idx=None): - """ - Given results from a model run and the ground truth, compute metrics - - """ - metrics = {} - metrics_pert = {} - - metric2fct = { - 'mse': mse, - 'pearson': pearsonr - } - - for m in metric2fct.keys(): - metrics[m] = [] - metrics[m + '_de'] = [] - - for pert in np.unique(results['pert_cat']): - - metrics_pert[pert] = {} - p_idx = np.where(results['pert_cat'] == pert)[0] - - for m, fct in metric2fct.items(): - # First average the cells of the same perturbation, and then calculate the indicator of the perturbation. - if m == 'pearson': - val = fct(results['pred'][p_idx].mean(0), results['truth'][p_idx].mean(0))[0] - if np.isnan(val): - val = 0 - else: - val = fct(results['pred'][p_idx].mean(0), results['truth'][p_idx].mean(0)) - - metrics_pert[pert][m] = val - metrics[m].append(metrics_pert[pert][m]) - - - if pert != 'ctrl': - - for m, fct in metric2fct.items(): - if m == 'pearson': - val = fct(results['pred_de'][p_idx].mean(0), results['truth_de'][p_idx].mean(0))[0] - if np.isnan(val): - val = 0 - else: - val = fct(results['pred_de'][p_idx].mean(0), results['truth_de'][p_idx].mean(0)) - - metrics_pert[pert][m + '_de'] = val - metrics[m + '_de'].append(metrics_pert[pert][m + '_de']) - - else: - for m, fct in metric2fct.items(): - metrics_pert[pert][m + '_de'] = 0 - - for m in metric2fct.keys(): - # The result is a perturbation-level macro average (each perturbation is equally weighted). - metrics[m] = np.mean(metrics[m]) - metrics[m + '_de'] = np.mean(metrics[m + '_de']) - - return metrics, metrics_pert - - -def compute_synergy_loss(results, mean_control, high_umi_idx, subtype = 'POTENTIATION'): - pred_res = get_test_set_results_seen2(results, subtype) - all_perts = np.unique(results['pert_cat']) - linear_params = get_linear_params(pred_res, high_umi_idx, - mean_control,all_perts) - synergy_loss = np.sum([np.abs(linear_params[k]['pred']['mag'] - - linear_params[k]['truth']['mag']) for k in - linear_params]) - - mag = np.sum([np.abs(linear_params[k]['pred']['mag']) for k in - linear_params]) - return synergy_loss, mag - - -def non_zero_analysis(adata, test_res): - metric2fct = { - 'pearson': pearsonr, - 'mse': mse - } - - pert_metric = {} - - ## in silico modeling and upperbounding - pert2pert_full_id = dict(adata.obs[['condition', 'condition_name']].values) - geneid2name = dict(zip(adata.var.index.values, adata.var['gene_name'])) - geneid2idx = dict(zip(adata.var.index.values, range(len(adata.var.index.values)))) - - # calculate mean expression for each condition - unique_conditions = adata.obs.condition.unique() - conditions2index = {} - for i in unique_conditions: - conditions2index[i] = np.where(adata.obs.condition == i)[0] - - condition2mean_expression = {} - for i, j in conditions2index.items(): - condition2mean_expression[i] = np.mean(adata.X[j], axis = 0) - pert_list = np.array(list(condition2mean_expression.keys())) - mean_expression = np.array(list(condition2mean_expression.values())).reshape(len(adata.obs.condition.unique()), adata.X.toarray().shape[1]) - ctrl = mean_expression[np.where(pert_list == 'ctrl')[0]] - - gene_list = adata.var['gene_name'].values - - for pert in np.unique(test_res['pert_cat']): - pert_metric[pert] = {} - - pert_idx = np.where(test_res['pert_cat'] == pert)[0] - de_idx = [geneid2idx[i] for i in adata.uns['top_non_zero_de_20'][pert2pert_full_id[pert]]] - - direc_change = np.abs(np.sign(test_res['pred'][pert_idx].mean(0)[de_idx] - ctrl[0][de_idx]) - np.sign(test_res['truth'][pert_idx].mean(0)[de_idx] - ctrl[0][de_idx])) - frac_correct_direction = len(np.where(direc_change == 0)[0])/len(de_idx) - pert_metric[pert]['frac_correct_direction_top20_non_zero'] = frac_correct_direction - - frac_direction_opposite = len(np.where(direc_change == 2)[0])/len(de_idx) - pert_metric[pert]['frac_opposite_direction_top20_non_zero'] = frac_direction_opposite - - frac_direction_opposite = len(np.where(direc_change == 1)[0])/len(de_idx) - pert_metric[pert]['frac_0/1_direction_top20_non_zero'] = frac_direction_opposite - - mean = np.mean(test_res['truth'][pert_idx][:, de_idx], axis = 0) - std = np.std(test_res['truth'][pert_idx][:, de_idx], axis = 0) - min_ = np.min(test_res['truth'][pert_idx][:, de_idx], axis = 0) - max_ = np.max(test_res['truth'][pert_idx][:, de_idx], axis = 0) - q25 = np.quantile(test_res['truth'][pert_idx][:, de_idx], 0.25, axis = 0) - q75 = np.quantile(test_res['truth'][pert_idx][:, de_idx], 0.75, axis = 0) - q55 = np.quantile(test_res['truth'][pert_idx][:, de_idx], 0.55, axis = 0) - q45 = np.quantile(test_res['truth'][pert_idx][:, de_idx], 0.45, axis = 0) - q40 = np.quantile(test_res['truth'][pert_idx][:, de_idx], 0.4, axis = 0) - q60 = np.quantile(test_res['truth'][pert_idx][:, de_idx], 0.6, axis = 0) - - zero_des = np.intersect1d(np.where(min_ == 0)[0], np.where(max_ == 0)[0]) - nonzero_des = np.setdiff1d(list(range(20)), zero_des) - - if len(nonzero_des) == 0: - pass - # pert that all de genes are 0... - else: - pred_mean = np.mean(test_res['pred'][pert_idx][:, de_idx], axis = 0).reshape(-1,) - true_mean = np.mean(test_res['truth'][pert_idx][:, de_idx], axis = 0).reshape(-1,) - - in_range = (pred_mean[nonzero_des] >= min_[nonzero_des]) & (pred_mean[nonzero_des] <= max_[nonzero_des]) - frac_in_range = sum(in_range)/len(nonzero_des) - pert_metric[pert]['frac_in_range_non_zero'] = frac_in_range - - in_range_5 = (pred_mean[nonzero_des] >= q45[nonzero_des]) & (pred_mean[nonzero_des] <= q55[nonzero_des]) - frac_in_range_45_55 = sum(in_range_5)/len(nonzero_des) - pert_metric[pert]['frac_in_range_45_55_non_zero'] = frac_in_range_45_55 - - in_range_10 = (pred_mean[nonzero_des] >= q40[nonzero_des]) & (pred_mean[nonzero_des] <= q60[nonzero_des]) - frac_in_range_40_60 = sum(in_range_10)/len(nonzero_des) - pert_metric[pert]['frac_in_range_40_60_non_zero'] = frac_in_range_40_60 - - in_range_25 = (pred_mean[nonzero_des] >= q25[nonzero_des]) & (pred_mean[nonzero_des] <= q75[nonzero_des]) - frac_in_range_25_75 = sum(in_range_25)/len(nonzero_des) - pert_metric[pert]['frac_in_range_25_75_non_zero'] = frac_in_range_25_75 - - zero_idx = np.where(std > 0)[0] - sigma = (np.abs(pred_mean[zero_idx] - mean[zero_idx]))/(std[zero_idx]) - pert_metric[pert]['mean_sigma_non_zero'] = np.mean(sigma) - pert_metric[pert]['std_sigma_non_zero'] = np.std(sigma) - pert_metric[pert]['frac_sigma_below_1_non_zero'] = 1 - len(np.where(sigma > 1)[0])/len(zero_idx) - pert_metric[pert]['frac_sigma_below_2_non_zero'] = 1 - len(np.where(sigma > 2)[0])/len(zero_idx) - - p_idx = np.where(test_res['pert_cat'] == pert)[0] - for m, fct in metric2fct.items(): - if m != 'mse': - val = fct(test_res['pred'][p_idx].mean(0)[de_idx] - ctrl[0][de_idx], test_res['truth'][p_idx].mean(0)[de_idx]-ctrl[0][de_idx])[0] - if np.isnan(val): - val = 0 - pert_metric[pert][m + '_delta_top20_de_non_zero'] = val - - - val = fct(test_res['pred'][p_idx].mean(0)[de_idx], test_res['truth'][p_idx].mean(0)[de_idx])[0] - if np.isnan(val): - val = 0 - pert_metric[pert][m + '_top20_de_non_zero'] = val - else: - val = fct(test_res['pred'][p_idx].mean(0)[de_idx] - ctrl[0][de_idx], test_res['truth'][p_idx].mean(0)[de_idx]-ctrl[0][de_idx]) - pert_metric[pert][m + '_top20_de_non_zero'] = val - - return pert_metric - -def non_dropout_analysis(adata, test_res): - metric2fct = { - 'pearson': pearsonr, - 'mse': mse - } - - pert_metric = {} - - ## in silico modeling and upperbounding - pert2pert_full_id = dict(adata.obs[['condition', 'condition_name']].values) - geneid2name = dict(zip(adata.var.index.values, adata.var['gene_name'])) - geneid2idx = dict(zip(adata.var.index.values, range(len(adata.var.index.values)))) - - # calculate mean expression for each condition - unique_conditions = adata.obs.condition.unique() - conditions2index = {} - for i in unique_conditions: - conditions2index[i] = np.where(adata.obs.condition == i)[0] - - condition2mean_expression = {} - for i, j in conditions2index.items(): - condition2mean_expression[i] = np.mean(adata.X[j], axis = 0) - pert_list = np.array(list(condition2mean_expression.keys())) - mean_expression = np.array(list(condition2mean_expression.values())).reshape(len(adata.obs.condition.unique()), adata.X.toarray().shape[1]) - ctrl = mean_expression[np.where(pert_list == 'ctrl')[0]] - - gene_list = adata.var['gene_name'].values - - for pert in np.unique(test_res['pert_cat']): - pert_metric[pert] = {} - - pert_idx = np.where(test_res['pert_cat'] == pert)[0] - de_idx = [geneid2idx[i] for i in adata.uns['top_non_dropout_de_20'][pert2pert_full_id[pert]]] - non_zero_idx = adata.uns['non_zeros_gene_idx'][pert2pert_full_id[pert]] - non_dropout_gene_idx = adata.uns['non_dropout_gene_idx'][pert2pert_full_id[pert]] - - direc_change = np.abs(np.sign(test_res['pred'][pert_idx].mean(0)[de_idx] - ctrl[0][de_idx]) - np.sign(test_res['truth'][pert_idx].mean(0)[de_idx] - ctrl[0][de_idx])) - frac_correct_direction = len(np.where(direc_change == 0)[0])/len(de_idx) - pert_metric[pert]['frac_correct_direction_top20_non_dropout'] = frac_correct_direction - - frac_direction_opposite = len(np.where(direc_change == 2)[0])/len(de_idx) - pert_metric[pert]['frac_opposite_direction_top20_non_dropout'] = frac_direction_opposite - - frac_direction_opposite = len(np.where(direc_change == 1)[0])/len(de_idx) - pert_metric[pert]['frac_0/1_direction_top20_non_dropout'] = frac_direction_opposite - - direc_change = np.abs(np.sign(test_res['pred'][pert_idx].mean(0)[non_zero_idx] - ctrl[0][non_zero_idx]) - np.sign(test_res['truth'][pert_idx].mean(0)[non_zero_idx] - ctrl[0][non_zero_idx])) - frac_correct_direction = len(np.where(direc_change == 0)[0])/len(non_zero_idx) - pert_metric[pert]['frac_correct_direction_non_zero'] = frac_correct_direction - - frac_direction_opposite = len(np.where(direc_change == 2)[0])/len(non_zero_idx) - pert_metric[pert]['frac_opposite_direction_non_zero'] = frac_direction_opposite - - frac_direction_opposite = len(np.where(direc_change == 1)[0])/len(non_zero_idx) - pert_metric[pert]['frac_0/1_direction_non_zero'] = frac_direction_opposite - - direc_change = np.abs(np.sign(test_res['pred'][pert_idx].mean(0)[non_dropout_gene_idx] - ctrl[0][non_dropout_gene_idx]) - np.sign(test_res['truth'][pert_idx].mean(0)[non_dropout_gene_idx] - ctrl[0][non_dropout_gene_idx])) - frac_correct_direction = len(np.where(direc_change == 0)[0])/len(non_dropout_gene_idx) - pert_metric[pert]['frac_correct_direction_non_dropout'] = frac_correct_direction - - frac_direction_opposite = len(np.where(direc_change == 2)[0])/len(non_dropout_gene_idx) - pert_metric[pert]['frac_opposite_direction_non_dropout'] = frac_direction_opposite - - frac_direction_opposite = len(np.where(direc_change == 1)[0])/len(non_dropout_gene_idx) - pert_metric[pert]['frac_0/1_direction_non_dropout'] = frac_direction_opposite - - mean = np.mean(test_res['truth'][pert_idx][:, de_idx], axis = 0) - std = np.std(test_res['truth'][pert_idx][:, de_idx], axis = 0) - min_ = np.min(test_res['truth'][pert_idx][:, de_idx], axis = 0) - max_ = np.max(test_res['truth'][pert_idx][:, de_idx], axis = 0) - q25 = np.quantile(test_res['truth'][pert_idx][:, de_idx], 0.25, axis = 0) - q75 = np.quantile(test_res['truth'][pert_idx][:, de_idx], 0.75, axis = 0) - q55 = np.quantile(test_res['truth'][pert_idx][:, de_idx], 0.55, axis = 0) - q45 = np.quantile(test_res['truth'][pert_idx][:, de_idx], 0.45, axis = 0) - q40 = np.quantile(test_res['truth'][pert_idx][:, de_idx], 0.4, axis = 0) - q60 = np.quantile(test_res['truth'][pert_idx][:, de_idx], 0.6, axis = 0) - - zero_des = np.intersect1d(np.where(min_ == 0)[0], np.where(max_ == 0)[0]) - nonzero_des = np.setdiff1d(list(range(20)), zero_des) - - if len(nonzero_des) == 0: - pass - # pert that all de genes are 0... - else: - pred_mean = np.mean(test_res['pred'][pert_idx][:, de_idx], axis = 0).reshape(-1,) - true_mean = np.mean(test_res['truth'][pert_idx][:, de_idx], axis = 0).reshape(-1,) - - in_range = (pred_mean[nonzero_des] >= min_[nonzero_des]) & (pred_mean[nonzero_des] <= max_[nonzero_des]) - frac_in_range = sum(in_range)/len(nonzero_des) - pert_metric[pert]['frac_in_range_non_dropout'] = frac_in_range - - in_range_5 = (pred_mean[nonzero_des] >= q45[nonzero_des]) & (pred_mean[nonzero_des] <= q55[nonzero_des]) - frac_in_range_45_55 = sum(in_range_5)/len(nonzero_des) - pert_metric[pert]['frac_in_range_45_55_non_dropout'] = frac_in_range_45_55 - - in_range_10 = (pred_mean[nonzero_des] >= q40[nonzero_des]) & (pred_mean[nonzero_des] <= q60[nonzero_des]) - frac_in_range_40_60 = sum(in_range_10)/len(nonzero_des) - pert_metric[pert]['frac_in_range_40_60_non_dropout'] = frac_in_range_40_60 - - in_range_25 = (pred_mean[nonzero_des] >= q25[nonzero_des]) & (pred_mean[nonzero_des] <= q75[nonzero_des]) - frac_in_range_25_75 = sum(in_range_25)/len(nonzero_des) - pert_metric[pert]['frac_in_range_25_75_non_dropout'] = frac_in_range_25_75 - - zero_idx = np.where(std > 0)[0] - sigma = (np.abs(pred_mean[zero_idx] - mean[zero_idx]))/(std[zero_idx]) - pert_metric[pert]['mean_sigma_non_dropout'] = np.mean(sigma) - pert_metric[pert]['std_sigma_non_dropout'] = np.std(sigma) - pert_metric[pert]['frac_sigma_below_1_non_dropout'] = 1 - len(np.where(sigma > 1)[0])/len(zero_idx) - pert_metric[pert]['frac_sigma_below_2_non_dropout'] = 1 - len(np.where(sigma > 2)[0])/len(zero_idx) - - p_idx = np.where(test_res['pert_cat'] == pert)[0] - for m, fct in metric2fct.items(): - if m != 'mse': - val = fct(test_res['pred'][p_idx].mean(0)[de_idx] - ctrl[0][de_idx], test_res['truth'][p_idx].mean(0)[de_idx]-ctrl[0][de_idx])[0] - if np.isnan(val): - val = 0 - pert_metric[pert][m + '_delta_top20_de_non_dropout'] = val - - - val = fct(test_res['pred'][p_idx].mean(0)[de_idx], test_res['truth'][p_idx].mean(0)[de_idx])[0] - if np.isnan(val): - val = 0 - pert_metric[pert][m + '_top20_de_non_dropout'] = val - else: - val = fct(test_res['pred'][p_idx].mean(0)[de_idx] - ctrl[0][de_idx], test_res['truth'][p_idx].mean(0)[de_idx]-ctrl[0][de_idx]) - pert_metric[pert][m + '_top20_de_non_dropout'] = val - - return pert_metric - -def deeper_analysis(adata, test_res, de_column_prefix = 'rank_genes_groups_cov', most_variable_genes = None): - - metric2fct = { - 'pearson': pearsonr, - 'mse': mse - } - - pert_metric = {} - - ## in silico modeling and upperbounding - pert2pert_full_id = dict(adata.obs[['condition', 'condition_name']].values) - geneid2name = dict(zip(adata.var.index.values, adata.var['gene_name'])) - geneid2idx = dict(zip(adata.var.index.values, range(len(adata.var.index.values)))) - - # calculate mean expression for each condition - unique_conditions = adata.obs.condition.unique() - conditions2index = {} - for i in unique_conditions: - conditions2index[i] = np.where(adata.obs.condition == i)[0] - - condition2mean_expression = {} - for i, j in conditions2index.items(): - condition2mean_expression[i] = np.mean(adata.X[j], axis = 0) - pert_list = np.array(list(condition2mean_expression.keys())) - mean_expression = np.array(list(condition2mean_expression.values())).reshape(len(adata.obs.condition.unique()), adata.X.toarray().shape[1]) - ctrl = mean_expression[np.where(pert_list == 'ctrl')[0]] - - if most_variable_genes is None: - most_variable_genes = np.argsort(np.std(mean_expression, axis = 0))[-200:] - - gene_list = adata.var['gene_name'].values - - for pert in np.unique(test_res['pert_cat']): - pert_metric[pert] = {} - de_idx = [geneid2idx[i] for i in adata.uns['rank_genes_groups_cov_all'][pert2pert_full_id[pert]][:20]] - de_idx_200 = [geneid2idx[i] for i in adata.uns['rank_genes_groups_cov_all'][pert2pert_full_id[pert]][:200]] - de_idx_100 = [geneid2idx[i] for i in adata.uns['rank_genes_groups_cov_all'][pert2pert_full_id[pert]][:100]] - de_idx_50 = [geneid2idx[i] for i in adata.uns['rank_genes_groups_cov_all'][pert2pert_full_id[pert]][:50]] - - pert_idx = np.where(test_res['pert_cat'] == pert)[0] - pred_mean = np.mean(test_res['pred_de'][pert_idx], axis = 0).reshape(-1,) - true_mean = np.mean(test_res['truth_de'][pert_idx], axis = 0).reshape(-1,) - - direc_change = np.abs(np.sign(test_res['pred'][pert_idx].mean(0) - ctrl[0]) - np.sign(test_res['truth'][pert_idx].mean(0) - ctrl[0])) - frac_correct_direction = len(np.where(direc_change == 0)[0])/len(geneid2name) - pert_metric[pert]['frac_correct_direction_all'] = frac_correct_direction - - de_idx_map = {20: de_idx, - 50: de_idx_50, - 100: de_idx_100, - 200: de_idx_200 - } - - for val in [20, 50, 100, 200]: - - direc_change = np.abs(np.sign(test_res['pred'][pert_idx].mean(0)[de_idx_map[val]] - ctrl[0][de_idx_map[val]]) - np.sign(test_res['truth'][pert_idx].mean(0)[de_idx_map[val]] - ctrl[0][de_idx_map[val]])) - frac_correct_direction = len(np.where(direc_change == 0)[0])/val - pert_metric[pert]['frac_correct_direction_' + str(val)] = frac_correct_direction - - mean = np.mean(test_res['truth_de'][pert_idx], axis = 0) - std = np.std(test_res['truth_de'][pert_idx], axis = 0) - min_ = np.min(test_res['truth_de'][pert_idx], axis = 0) - max_ = np.max(test_res['truth_de'][pert_idx], axis = 0) - q25 = np.quantile(test_res['truth_de'][pert_idx], 0.25, axis = 0) - q75 = np.quantile(test_res['truth_de'][pert_idx], 0.75, axis = 0) - q55 = np.quantile(test_res['truth_de'][pert_idx], 0.55, axis = 0) - q45 = np.quantile(test_res['truth_de'][pert_idx], 0.45, axis = 0) - q40 = np.quantile(test_res['truth_de'][pert_idx], 0.4, axis = 0) - q60 = np.quantile(test_res['truth_de'][pert_idx], 0.6, axis = 0) - - zero_des = np.intersect1d(np.where(min_ == 0)[0], np.where(max_ == 0)[0]) - nonzero_des = np.setdiff1d(list(range(20)), zero_des) - if len(nonzero_des) == 0: - pass - # pert that all de genes are 0... - else: - - direc_change = np.abs(np.sign(pred_mean[nonzero_des] - ctrl[0][de_idx][nonzero_des]) - np.sign(true_mean[nonzero_des] - ctrl[0][de_idx][nonzero_des])) - frac_correct_direction = len(np.where(direc_change == 0)[0])/len(nonzero_des) - pert_metric[pert]['frac_correct_direction_20_nonzero'] = frac_correct_direction - - in_range = (pred_mean[nonzero_des] >= min_[nonzero_des]) & (pred_mean[nonzero_des] <= max_[nonzero_des]) - frac_in_range = sum(in_range)/len(nonzero_des) - pert_metric[pert]['frac_in_range'] = frac_in_range - - in_range_5 = (pred_mean[nonzero_des] >= q45[nonzero_des]) & (pred_mean[nonzero_des] <= q55[nonzero_des]) - frac_in_range_45_55 = sum(in_range_5)/len(nonzero_des) - pert_metric[pert]['frac_in_range_45_55'] = frac_in_range_45_55 - - in_range_10 = (pred_mean[nonzero_des] >= q40[nonzero_des]) & (pred_mean[nonzero_des] <= q60[nonzero_des]) - frac_in_range_40_60 = sum(in_range_10)/len(nonzero_des) - pert_metric[pert]['frac_in_range_40_60'] = frac_in_range_40_60 - - in_range_25 = (pred_mean[nonzero_des] >= q25[nonzero_des]) & (pred_mean[nonzero_des] <= q75[nonzero_des]) - frac_in_range_25_75 = sum(in_range_25)/len(nonzero_des) - pert_metric[pert]['frac_in_range_25_75'] = frac_in_range_25_75 - - zero_idx = np.where(std > 0)[0] - sigma = (np.abs(pred_mean[zero_idx] - mean[zero_idx]))/(std[zero_idx]) - pert_metric[pert]['mean_sigma'] = np.mean(sigma) - pert_metric[pert]['std_sigma'] = np.std(sigma) - pert_metric[pert]['frac_sigma_below_1'] = 1 - len(np.where(sigma > 1)[0])/len(zero_idx) - pert_metric[pert]['frac_sigma_below_2'] = 1 - len(np.where(sigma > 2)[0])/len(zero_idx) - - ## correlation on delta - p_idx = np.where(test_res['pert_cat'] == pert)[0] - - for m, fct in metric2fct.items(): - if m != 'mse': - val = fct(test_res['pred'][p_idx].mean(0)- ctrl[0], test_res['truth'][p_idx].mean(0)-ctrl[0])[0] - if np.isnan(val): - val = 0 - - pert_metric[pert][m + '_delta'] = val - - val = fct(test_res['pred'][p_idx].mean(0)[de_idx] - ctrl[0][de_idx], test_res['truth'][p_idx].mean(0)[de_idx]-ctrl[0][de_idx])[0] - if np.isnan(val): - val = 0 - - pert_metric[pert][m + '_delta_de'] = val - - ## up fold changes > 10, - pert_mean = np.mean(test_res['truth'][p_idx], axis = 0).reshape(-1,) - - fold_change = pert_mean/ctrl - fold_change[np.isnan(fold_change)] = 0 - fold_change[np.isinf(fold_change)] = 0 - ## this is to remove the ones that are super low and the fold change becomes unmeaningful - fold_change[0][np.where(pert_mean < 0.5)[0]] = 0 - - o = np.where(fold_change[0] > 0)[0] - - pred_fc = test_res['pred'][p_idx].mean(0)[o] - true_fc = test_res['truth'][p_idx].mean(0)[o] - ctrl_fc = ctrl[0][o] - - if len(o) > 0: - pert_metric[pert]['fold_change_gap_all'] = np.mean(np.abs(pred_fc/ctrl_fc - true_fc/ctrl_fc)) - - - o = np.intersect1d(np.where(fold_change[0] <0.333)[0], np.where(fold_change[0] > 0)[0]) - - pred_fc = test_res['pred'][p_idx].mean(0)[o] - true_fc = test_res['truth'][p_idx].mean(0)[o] - ctrl_fc = ctrl[0][o] - - if len(o) > 0: - pert_metric[pert]['fold_change_gap_downreg_0.33'] = np.mean(np.abs(pred_fc/ctrl_fc - true_fc/ctrl_fc)) - - - o = np.intersect1d(np.where(fold_change[0] <0.1)[0], np.where(fold_change[0] > 0)[0]) - - pred_fc = test_res['pred'][p_idx].mean(0)[o] - true_fc = test_res['truth'][p_idx].mean(0)[o] - ctrl_fc = ctrl[0][o] - - if len(o) > 0: - pert_metric[pert]['fold_change_gap_downreg_0.1'] = np.mean(np.abs(pred_fc/ctrl_fc - true_fc/ctrl_fc)) - - o = np.where(fold_change[0] > 3)[0] - - pred_fc = test_res['pred'][p_idx].mean(0)[o] - true_fc = test_res['truth'][p_idx].mean(0)[o] - ctrl_fc = ctrl[0][o] - - if len(o) > 0: - pert_metric[pert]['fold_change_gap_upreg_3'] = np.mean(np.abs(pred_fc/ctrl_fc - true_fc/ctrl_fc)) - - o = np.where(fold_change[0] > 10)[0] - - pred_fc = test_res['pred'][p_idx].mean(0)[o] - true_fc = test_res['truth'][p_idx].mean(0)[o] - ctrl_fc = ctrl[0][o] - - if len(o) > 0: - pert_metric[pert]['fold_change_gap_upreg_10'] = np.mean(np.abs(pred_fc/ctrl_fc - true_fc/ctrl_fc)) - - ## most variable genes - for m, fct in metric2fct.items(): - if m != 'mse': - val = fct(test_res['pred'][p_idx].mean(0)[most_variable_genes] - ctrl[0][most_variable_genes], test_res['truth'][p_idx].mean(0)[most_variable_genes]-ctrl[0][most_variable_genes])[0] - if np.isnan(val): - val = 0 - pert_metric[pert][m + '_delta_top200_hvg'] = val - - - val = fct(test_res['pred'][p_idx].mean(0)[most_variable_genes], test_res['truth'][p_idx].mean(0)[most_variable_genes])[0] - if np.isnan(val): - val = 0 - pert_metric[pert][m + '_top200_hvg'] = val - else: - val = fct(test_res['pred'][p_idx].mean(0)[most_variable_genes], test_res['truth'][p_idx].mean(0)[most_variable_genes]) - pert_metric[pert][m + '_top200_hvg'] = val - - - ## top 20/50/100/200 DEs - for m, fct in metric2fct.items(): - if m != 'mse': - val = fct(test_res['pred'][p_idx].mean(0)[de_idx] - ctrl[0][de_idx], test_res['truth'][p_idx].mean(0)[de_idx]-ctrl[0][de_idx])[0] - if np.isnan(val): - val = 0 - pert_metric[pert][m + '_delta_top20_de'] = val - - - val = fct(test_res['pred'][p_idx].mean(0)[de_idx], test_res['truth'][p_idx].mean(0)[de_idx])[0] - if np.isnan(val): - val = 0 - pert_metric[pert][m + '_top20_de'] = val - else: - val = fct(test_res['pred'][p_idx].mean(0)[de_idx] - ctrl[0][de_idx], test_res['truth'][p_idx].mean(0)[de_idx]-ctrl[0][de_idx]) - pert_metric[pert][m + '_top20_de'] = val - - - for m, fct in metric2fct.items(): - if m != 'mse': - val = fct(test_res['pred'][p_idx].mean(0)[de_idx_200] - ctrl[0][de_idx_200], test_res['truth'][p_idx].mean(0)[de_idx_200]-ctrl[0][de_idx_200])[0] - if np.isnan(val): - val = 0 - pert_metric[pert][m + '_delta_top200_de'] = val - - - val = fct(test_res['pred'][p_idx].mean(0)[de_idx_200], test_res['truth'][p_idx].mean(0)[de_idx_200])[0] - if np.isnan(val): - val = 0 - pert_metric[pert][m + '_top200_de'] = val - else: - val = fct(test_res['pred'][p_idx].mean(0)[de_idx_200] - ctrl[0][de_idx_200], test_res['truth'][p_idx].mean(0)[de_idx_200]-ctrl[0][de_idx_200]) - pert_metric[pert][m + '_top200_de'] = val - - for m, fct in metric2fct.items(): - if m != 'mse': - - val = fct(test_res['pred'][p_idx].mean(0)[de_idx_100] - ctrl[0][de_idx_100], test_res['truth'][p_idx].mean(0)[de_idx_100]-ctrl[0][de_idx_100])[0] - if np.isnan(val): - val = 0 - pert_metric[pert][m + '_delta_top100_de'] = val - - - val = fct(test_res['pred'][p_idx].mean(0)[de_idx_100], test_res['truth'][p_idx].mean(0)[de_idx_100])[0] - if np.isnan(val): - val = 0 - pert_metric[pert][m + '_top100_de'] = val - else: - val = fct(test_res['pred'][p_idx].mean(0)[de_idx_100] - ctrl[0][de_idx_100], test_res['truth'][p_idx].mean(0)[de_idx_100]-ctrl[0][de_idx_100]) - pert_metric[pert][m + '_top100_de'] = val - - for m, fct in metric2fct.items(): - if m != 'mse': - - val = fct(test_res['pred'][p_idx].mean(0)[de_idx_50] - ctrl[0][de_idx_50], test_res['truth'][p_idx].mean(0)[de_idx_50]-ctrl[0][de_idx_50])[0] - if np.isnan(val): - val = 0 - pert_metric[pert][m + '_delta_top50_de'] = val - - - val = fct(test_res['pred'][p_idx].mean(0)[de_idx_50], test_res['truth'][p_idx].mean(0)[de_idx_50])[0] - if np.isnan(val): - val = 0 - pert_metric[pert][m + '_top50_de'] = val - else: - val = fct(test_res['pred'][p_idx].mean(0)[de_idx_50] - ctrl[0][de_idx_50], test_res['truth'][p_idx].mean(0)[de_idx_50]-ctrl[0][de_idx_50]) - pert_metric[pert][m + '_top50_de'] = val - - - - return pert_metric - - -def GI_subgroup(pert_metric): - GI_type2Score = {} - test_pert_list = list(pert_metric.keys()) - for GI_type, gi_list in GIs.items(): - intersect = np.intersect1d(gi_list, test_pert_list) - if len(intersect) != 0: - GI_type2Score[GI_type] = {} - - for m in list(list(pert_metric.values())[0].keys()): - GI_type2Score[GI_type][m] = np.mean([pert_metric[i][m] for i in intersect if m in pert_metric[i]]) - - return GI_type2Score - -def node_specific_batch_out(models, batch): - # Returns output for all node specific models as a matrix of dimension batch_size x nodes - outs = [] - for idx in range(len(models)): - outs.append(models[idx](batch).detach().cpu().numpy()[:,idx]) - return np.vstack(outs).T - -# Run prediction over all batches -def batch_predict(loader, loaded_models, args): - # Prediction for node specific GNNs - preds = [] - print("Loader size: ", len(loader)) - for itr, batch in enumerate(loader): - print(itr) - batch = batch.to(args['device']) - preds.append(node_specific_batch_out(loaded_models, batch)) - - preds = np.vstack(preds) - return preds - -def get_high_umi_idx(gene_list): - # Genes used for linear model fitting - try: - high_umi = np.load('../genes_with_hi_mean.npy', allow_pickle=True) - except: - high_umi = np.load('./genes_with_hi_mean.npy', allow_pickle=True) - high_umi_idx = np.where([g in high_umi for g in gene_list])[0] - return high_umi_idx - -def get_mean_ctrl(adata): - return adata[adata.obs['condition'] == 'ctrl'].to_df().mean().reset_index( - drop=True) - -def get_single_name(g, all_perts): - name = g+'+ctrl' - if name in all_perts: - return name - else: - return 'ctrl+'+g - - -def get_test_set_results_seen2(res, sel_GI_type): - # Get relevant test set results - test_pert_cats = [p for p in np.unique(res['pert_cat']) if - p in GIs[sel_GI_type] or 'ctrl' in p] - pred_idx = np.where([t in test_pert_cats for t in res['pert_cat']]) - out = {} - for key in res: - out[key] = res[key][pred_idx] - return out - -## Synergy loss calculation functions -def get_all_vectors(all_res, mean_control, double, - single1, single2, high_umi_idx): - # Pred - pred_df = pd.DataFrame(all_res['pred']) - pred_df['condition'] = all_res['pert_cat'] - subset_df = pred_df[pred_df['condition'] == double].iloc[:, :-1] - delta_double_pred = subset_df.mean(0) - mean_control - single_df_1_pred = pred_df[pred_df['condition'] == single1].iloc[:, :-1] - single_df_2_pred = pred_df[pred_df['condition'] == single2].iloc[:, :-1] - - # True - truth_df = pd.DataFrame(all_res['truth']) - truth_df['condition'] = all_res['pert_cat'] - subset_df = truth_df[truth_df['condition'] == double].iloc[:, :-1] - delta_double_truth = subset_df.mean(0) - mean_control - single_df_1_truth = truth_df[truth_df['condition'] == single1].iloc[:, :-1] - single_df_2_truth = truth_df[truth_df['condition'] == single2].iloc[:, :-1] - - delta_single_truth_1 = single_df_1_truth.mean(0) - mean_control - delta_single_truth_2 = single_df_2_truth.mean(0) - mean_control - delta_single_pred_1 = single_df_1_pred.mean(0) - mean_control - delta_single_pred_2 = single_df_2_pred.mean(0) - mean_control - - return {'single_pred_1': delta_single_pred_1.values[high_umi_idx], - 'single_pred_2': delta_single_pred_2.values[high_umi_idx], - 'double_pred': delta_double_pred.values[high_umi_idx], - 'single_truth_1': delta_single_truth_1.values[high_umi_idx], - 'single_truth_2': delta_single_truth_2.values[high_umi_idx], - 'double_truth': delta_double_truth.values[high_umi_idx]} - - -def get_coeffs_synergy(singles_expr, double_expr): - results = {} - results['ts'] = TheilSenRegressor(fit_intercept=False, - max_subpopulation=1e5, - max_iter=1000, - random_state=1000) - X = singles_expr - y = double_expr - try: - results['ts'].fit(X, y.ravel()) - except: - print(X) - print(y) - results['c1'] = results['ts'].coef_[0] - results['c2'] = results['ts'].coef_[1] - results['mag'] = np.sqrt((results['c1'] ** 2 + results['c2'] ** 2)) - return results - - -def Fit(all_vectors, type_='pertnet'): - if type_ == 'pertnet': - singles_expr = np.array( - [all_vectors['single_pred_1'], all_vectors['single_pred_2']]).T - first_expr = np.array([all_vectors['single_pred_1']]).T - second_expr = np.array([all_vectors['single_pred_2']]).T - double_expr = np.array(all_vectors['double_pred']).T - - elif type_ == 'truth': - singles_expr = np.array( - [all_vectors['single_truth_1'], all_vectors['single_truth_2']]).T - first_expr = np.array([all_vectors['single_truth_1']]).T - second_expr = np.array([all_vectors['single_truth_2']]).T - double_expr = np.array(all_vectors['double_truth']).T - - return get_coeffs_synergy(singles_expr, double_expr) - - -def get_linear_params(pred_res, high_umi_idx, mean_control, all_perts): - results = {} - for c in set(pred_res['pert_cat']): - if 'ctrl' in c: - continue - double = c - single1 = get_single_name(double.split('+')[0], all_perts) - single2 = get_single_name(double.split('+')[1], all_perts) - all_vectors = get_all_vectors(pred_res, mean_control, double, - single1, single2, high_umi_idx) - - pertnet_results = Fit(all_vectors, type_='pertnet') - truth_results = Fit(all_vectors, type_='truth') - - results[c] = { - 'truth': truth_results, - 'pred': pertnet_results} - - return results - - -# Read in model for each gene - -GIs = { - 'NEOMORPHIC': ['CBL+TGFBR2', - 'KLF1+TGFBR2', - 'MAP2K6+SPI1', - 'SAMD1+TGFBR2', - 'TGFBR2+C19orf26', - 'TGFBR2+ETS2', - 'CBL+UBASH3A', - 'CEBPE+KLF1', - 'DUSP9+MAPK1', - 'FOSB+PTPN12', - 'PLK4+STIL', - 'PTPN12+OSR2', - 'ZC3HAV1+CEBPE'], - 'ADDITIVE': ['BPGM+SAMD1', - 'CEBPB+MAPK1', - 'CEBPB+OSR2', - 'DUSP9+PRTG', - 'FOSB+OSR2', - 'IRF1+SET', - 'MAP2K3+ELMSAN1', - 'MAP2K6+ELMSAN1', - 'POU3F2+FOXL2', - 'RHOXF2BB+SET', - 'SAMD1+PTPN12', - 'SAMD1+UBASH3B', - 'SAMD1+ZBTB1', - 'SGK1+TBX2', - 'TBX3+TBX2', - 'ZBTB10+SNAI1'], - 'EPISTASIS': ['AHR+KLF1', - 'MAPK1+TGFBR2', - 'TGFBR2+IGDCC3', - 'TGFBR2+PRTG', - 'UBASH3B+OSR2', - 'DUSP9+ETS2', - 'KLF1+CEBPA', - 'MAP2K6+IKZF3', - 'ZC3HAV1+CEBPA'], - 'REDUNDANT': ['CDKN1C+CDKN1A', - 'MAP2K3+MAP2K6', - 'CEBPB+CEBPA', - 'CEBPE+CEBPA', - 'CEBPE+SPI1', - 'ETS2+MAPK1', - 'FOSB+CEBPE', - 'FOXA3+FOXA1'], - 'POTENTIATION': ['CNN1+UBASH3A', - 'ETS2+MAP7D1', - 'FEV+CBFA2T3', - 'FEV+ISL2', - 'FEV+MAP7D1', - 'PTPN12+UBASH3A'], - 'SYNERGY_SIMILAR_PHENO':['CBL+CNN1', - 'CBL+PTPN12', - 'CBL+PTPN9', - 'CBL+UBASH3B', - 'FOXA3+FOXL2', - 'FOXA3+HOXB9', - 'FOXL2+HOXB9', - 'UBASH3B+CNN1', - 'UBASH3B+PTPN12', - 'UBASH3B+PTPN9', - 'UBASH3B+ZBTB25'], - 'SYNERGY_DISSIMILAR_PHENO': ['AHR+FEV', - 'DUSP9+SNAI1', - 'FOXA1+FOXF1', - 'FOXA1+FOXL2', - 'FOXA1+HOXB9', - 'FOXF1+FOXL2', - 'FOXF1+HOXB9', - 'FOXL2+MEIS1', - 'IGDCC3+ZBTB25', - 'POU3F2+CBFA2T3', - 'PTPN12+ZBTB25', - 'SNAI1+DLX2', - 'SNAI1+UBASH3B'], - 'SUPPRESSOR': ['CEBPB+PTPN12', - 'CEBPE+CNN1', - 'CEBPE+PTPN12', - 'CNN1+MAPK1', - 'ETS2+CNN1', - 'ETS2+IGDCC3', - 'ETS2+PRTG', - 'FOSB+UBASH3B', - 'IGDCC3+MAPK1', - 'LYL1+CEBPB', - 'MAPK1+PRTG', - 'PTPN12+SNAI1'] -} +""" +================================================================================ +Model evaluation module (inference_local.py) +================================================================================ + +[Module function] +Provides evaluation and analysis functions for genetic perturbation models, including: + +1. evaluate(): main evaluation function, calculates MSE, Pearson correlation coefficient and other indicators +2. compute_metrics(): Calculate the complete dictionary of evaluation metrics +3. deeper_analysis(): In-depth analysis of prediction results +4. non_dropout_analysis(): analyze on non-dropout genes + +[Description of key indicators] +- MSE: mean square error of all genes +- MSE_DE: MSE of Top 20 differentially expressed genes (paying more attention to biological significance) +- Pearson: Pearson correlation coefficient of all genes +- Pearson_DE: Pearson correlation coefficient of Top 20 DE genes + +[Evaluation Notes] +- real_num_genes: only evaluate on real genes, excluding extended/completed genes +- de_idx: The differentially expressed gene index of each sample may be different +================================================================================ +""" + +import os +import torch +import numpy as np +import scanpy as sc + +from sklearn.metrics import r2_score +from scipy.stats import pearsonr, spearmanr +from sklearn.metrics import mean_squared_error as mse +from sklearn.metrics import mean_absolute_error as mae + +def evaluate(loader, model, uncertainty, device, real_num_genes = None): + """ + main evaluation function + + [Function] + Runs the model on the given data loader, collecting predicted and true values. + + [Key steps] + 1. Traverse all batches of the data loader + 2. Run model inference (without calculating gradients) + 3. Collect predicted values, true values, and DE gene predicted values + + [Parameter description] + - real_num_genes: only evaluate the first N real genes to avoid expanding gene interference indicators + """ + + model.eval() + model.to(device) + pert_cat = [] + pred = [] + truth = [] + pred_de = [] + truth_de = [] + results = {} + logvar = [] + + for itr, batch in enumerate(loader): + + batch.to(device) + pert_cat.extend(batch.pert) + + with torch.no_grad(): + if uncertainty: + p, unc = model(batch) + logvar.extend(unc.cpu()) + else: + p = model(batch) + # Only evaluate in the real gene dimension to avoid the interference of completed/expanded genes on indicators. + p = p[:, :real_num_genes] + t = batch.y[:, :real_num_genes] + pred.extend(p.cpu()) + truth.extend(t.cpu()) + + # The DE indicator is only counted on the de_idx corresponding to the sample and is one of the core calibers of downstream reporting. + for itr, de_idx in enumerate(batch.de_idx): + pred_de.append(p[itr, de_idx]) + truth_de.append(t[itr, de_idx]) + + # all genes + results['pert_cat'] = np.array(pert_cat) + pred = torch.stack(pred) + truth = torch.stack(truth) + results['pred']= pred.detach().cpu().numpy() + results['truth']= truth.detach().cpu().numpy() + + pred_de = torch.stack(pred_de) + truth_de = torch.stack(truth_de) + results['pred_de']= pred_de.detach().cpu().numpy() + results['truth_de']= truth_de.detach().cpu().numpy() + + if uncertainty: + results['logvar'] = torch.stack(logvar).detach().cpu().numpy() + + return results + +#This evaluation is just as test. IT HAS INFORMATION LEAKAGE! +def evaluate_with_nonzero_mask(loader, model, uncertainty, device): + """ + Run model in inference mode using a given data loader + """ + + model.eval() + model.to(device) + pert_cat = [] + pred = [] + truth = [] + pred_de = [] + truth_de = [] + results = {} + logvar = [] + + for itr, batch in enumerate(loader): + + batch.to(device) + pert_cat.extend(batch.pert) + + with torch.no_grad(): + if uncertainty: + p, unc = model(batch) + logvar.extend(unc.cpu()) + else: + p = model(batch) + t = batch.y + p = p * (t != 0) + pred.extend(p.cpu()) + truth.extend(t.cpu()) + + # Differentially expressed genes + for itr, de_idx in enumerate(batch.de_idx): + pred_de.append(p[itr, de_idx]) + truth_de.append(t[itr, de_idx]) + + # all genes + results['pert_cat'] = np.array(pert_cat) + pred = torch.stack(pred) + truth = torch.stack(truth) + results['pred']= pred.detach().cpu().numpy() + results['truth']= truth.detach().cpu().numpy() + + pred_de = torch.stack(pred_de) + truth_de = torch.stack(truth_de) + results['pred_de']= pred_de.detach().cpu().numpy() + results['truth_de']= truth_de.detach().cpu().numpy() + + if uncertainty: + results['logvar'] = torch.stack(logvar).detach().cpu().numpy() + + return results + + +def non_zero_evaluate(loader, model, uncertainty, device, + delta_list, penalty = { + "false_neg": 1, + "false_pos": 1 }): + """ + Run model in inference mode using a given data loader + """ + + model.eval() + model.to(device) + pert_cat = [] + pred = [] + truth = [] + pred_de = [] + truth_de = [] + results = {} + logvar = [] + + for itr, batch in enumerate(loader): + + batch.to(device) + pert_cat.extend(batch.pert) + + with torch.no_grad(): + if uncertainty: + p, unc = model(batch) + logvar.extend(unc.cpu()) + else: + p = model(batch) + t = batch.y + pred.extend(p.cpu()) + truth.extend(t.cpu()) + + # Differentially expressed genes + for itr, de_idx in enumerate(batch.de_idx): + pred_de.append(p[itr, de_idx]) + truth_de.append(t[itr, de_idx]) + + # all genes + results['pert_cat'] = np.array(pert_cat) + pred = torch.stack(pred) + truth = torch.stack(truth) + results['pred']= pred.detach().cpu().numpy() + results['truth']= truth.detach().cpu().numpy() + + pred_de = torch.stack(pred_de) + truth_de = torch.stack(truth_de) + results['pred_de']= pred_de.detach().cpu().numpy() + results['truth_de']= truth_de.detach().cpu().numpy() + + if uncertainty: + results['logvar'] = torch.stack(logvar).detach().cpu().numpy() + + return results + + +def compute_metrics(results, gene_idx=None): + """ + Given results from a model run and the ground truth, compute metrics + + """ + metrics = {} + metrics_pert = {} + + metric2fct = { + 'mse': mse, + 'pearson': pearsonr + } + + for m in metric2fct.keys(): + metrics[m] = [] + metrics[m + '_de'] = [] + + for pert in np.unique(results['pert_cat']): + + metrics_pert[pert] = {} + p_idx = np.where(results['pert_cat'] == pert)[0] + + for m, fct in metric2fct.items(): + # First average the cells of the same perturbation, and then calculate the indicator of the perturbation. + if m == 'pearson': + val = fct(results['pred'][p_idx].mean(0), results['truth'][p_idx].mean(0))[0] + if np.isnan(val): + val = 0 + else: + val = fct(results['pred'][p_idx].mean(0), results['truth'][p_idx].mean(0)) + + metrics_pert[pert][m] = val + metrics[m].append(metrics_pert[pert][m]) + + + if pert != 'ctrl': + + for m, fct in metric2fct.items(): + if m == 'pearson': + val = fct(results['pred_de'][p_idx].mean(0), results['truth_de'][p_idx].mean(0))[0] + if np.isnan(val): + val = 0 + else: + val = fct(results['pred_de'][p_idx].mean(0), results['truth_de'][p_idx].mean(0)) + + metrics_pert[pert][m + '_de'] = val + metrics[m + '_de'].append(metrics_pert[pert][m + '_de']) + + else: + for m, fct in metric2fct.items(): + metrics_pert[pert][m + '_de'] = 0 + + for m in metric2fct.keys(): + # The result is a perturbation-level macro average (each perturbation is equally weighted). + metrics[m] = np.mean(metrics[m]) + metrics[m + '_de'] = np.mean(metrics[m + '_de']) + + return metrics, metrics_pert + + +def compute_synergy_loss(results, mean_control, high_umi_idx, subtype = 'POTENTIATION'): + pred_res = get_test_set_results_seen2(results, subtype) + all_perts = np.unique(results['pert_cat']) + linear_params = get_linear_params(pred_res, high_umi_idx, + mean_control,all_perts) + synergy_loss = np.sum([np.abs(linear_params[k]['pred']['mag'] + - linear_params[k]['truth']['mag']) for k in + linear_params]) + + mag = np.sum([np.abs(linear_params[k]['pred']['mag']) for k in + linear_params]) + return synergy_loss, mag + + +def non_zero_analysis(adata, test_res): + metric2fct = { + 'pearson': pearsonr, + 'mse': mse + } + + pert_metric = {} + + ## in silico modeling and upperbounding + pert2pert_full_id = dict(adata.obs[['condition', 'condition_name']].values) + geneid2name = dict(zip(adata.var.index.values, adata.var['gene_name'])) + geneid2idx = dict(zip(adata.var.index.values, range(len(adata.var.index.values)))) + + # calculate mean expression for each condition + unique_conditions = adata.obs.condition.unique() + conditions2index = {} + for i in unique_conditions: + conditions2index[i] = np.where(adata.obs.condition == i)[0] + + condition2mean_expression = {} + for i, j in conditions2index.items(): + condition2mean_expression[i] = np.mean(adata.X[j], axis = 0) + pert_list = np.array(list(condition2mean_expression.keys())) + mean_expression = np.array(list(condition2mean_expression.values())).reshape(len(adata.obs.condition.unique()), adata.X.toarray().shape[1]) + ctrl = mean_expression[np.where(pert_list == 'ctrl')[0]] + + gene_list = adata.var['gene_name'].values + + for pert in np.unique(test_res['pert_cat']): + pert_metric[pert] = {} + + pert_idx = np.where(test_res['pert_cat'] == pert)[0] + de_idx = [geneid2idx[i] for i in adata.uns['top_non_zero_de_20'][pert2pert_full_id[pert]]] + + direc_change = np.abs(np.sign(test_res['pred'][pert_idx].mean(0)[de_idx] - ctrl[0][de_idx]) - np.sign(test_res['truth'][pert_idx].mean(0)[de_idx] - ctrl[0][de_idx])) + frac_correct_direction = len(np.where(direc_change == 0)[0])/len(de_idx) + pert_metric[pert]['frac_correct_direction_top20_non_zero'] = frac_correct_direction + + frac_direction_opposite = len(np.where(direc_change == 2)[0])/len(de_idx) + pert_metric[pert]['frac_opposite_direction_top20_non_zero'] = frac_direction_opposite + + frac_direction_opposite = len(np.where(direc_change == 1)[0])/len(de_idx) + pert_metric[pert]['frac_0/1_direction_top20_non_zero'] = frac_direction_opposite + + mean = np.mean(test_res['truth'][pert_idx][:, de_idx], axis = 0) + std = np.std(test_res['truth'][pert_idx][:, de_idx], axis = 0) + min_ = np.min(test_res['truth'][pert_idx][:, de_idx], axis = 0) + max_ = np.max(test_res['truth'][pert_idx][:, de_idx], axis = 0) + q25 = np.quantile(test_res['truth'][pert_idx][:, de_idx], 0.25, axis = 0) + q75 = np.quantile(test_res['truth'][pert_idx][:, de_idx], 0.75, axis = 0) + q55 = np.quantile(test_res['truth'][pert_idx][:, de_idx], 0.55, axis = 0) + q45 = np.quantile(test_res['truth'][pert_idx][:, de_idx], 0.45, axis = 0) + q40 = np.quantile(test_res['truth'][pert_idx][:, de_idx], 0.4, axis = 0) + q60 = np.quantile(test_res['truth'][pert_idx][:, de_idx], 0.6, axis = 0) + + zero_des = np.intersect1d(np.where(min_ == 0)[0], np.where(max_ == 0)[0]) + nonzero_des = np.setdiff1d(list(range(20)), zero_des) + + if len(nonzero_des) == 0: + pass + # pert that all de genes are 0... + else: + pred_mean = np.mean(test_res['pred'][pert_idx][:, de_idx], axis = 0).reshape(-1,) + true_mean = np.mean(test_res['truth'][pert_idx][:, de_idx], axis = 0).reshape(-1,) + + in_range = (pred_mean[nonzero_des] >= min_[nonzero_des]) & (pred_mean[nonzero_des] <= max_[nonzero_des]) + frac_in_range = sum(in_range)/len(nonzero_des) + pert_metric[pert]['frac_in_range_non_zero'] = frac_in_range + + in_range_5 = (pred_mean[nonzero_des] >= q45[nonzero_des]) & (pred_mean[nonzero_des] <= q55[nonzero_des]) + frac_in_range_45_55 = sum(in_range_5)/len(nonzero_des) + pert_metric[pert]['frac_in_range_45_55_non_zero'] = frac_in_range_45_55 + + in_range_10 = (pred_mean[nonzero_des] >= q40[nonzero_des]) & (pred_mean[nonzero_des] <= q60[nonzero_des]) + frac_in_range_40_60 = sum(in_range_10)/len(nonzero_des) + pert_metric[pert]['frac_in_range_40_60_non_zero'] = frac_in_range_40_60 + + in_range_25 = (pred_mean[nonzero_des] >= q25[nonzero_des]) & (pred_mean[nonzero_des] <= q75[nonzero_des]) + frac_in_range_25_75 = sum(in_range_25)/len(nonzero_des) + pert_metric[pert]['frac_in_range_25_75_non_zero'] = frac_in_range_25_75 + + zero_idx = np.where(std > 0)[0] + sigma = (np.abs(pred_mean[zero_idx] - mean[zero_idx]))/(std[zero_idx]) + pert_metric[pert]['mean_sigma_non_zero'] = np.mean(sigma) + pert_metric[pert]['std_sigma_non_zero'] = np.std(sigma) + pert_metric[pert]['frac_sigma_below_1_non_zero'] = 1 - len(np.where(sigma > 1)[0])/len(zero_idx) + pert_metric[pert]['frac_sigma_below_2_non_zero'] = 1 - len(np.where(sigma > 2)[0])/len(zero_idx) + + p_idx = np.where(test_res['pert_cat'] == pert)[0] + for m, fct in metric2fct.items(): + if m != 'mse': + val = fct(test_res['pred'][p_idx].mean(0)[de_idx] - ctrl[0][de_idx], test_res['truth'][p_idx].mean(0)[de_idx]-ctrl[0][de_idx])[0] + if np.isnan(val): + val = 0 + pert_metric[pert][m + '_delta_top20_de_non_zero'] = val + + + val = fct(test_res['pred'][p_idx].mean(0)[de_idx], test_res['truth'][p_idx].mean(0)[de_idx])[0] + if np.isnan(val): + val = 0 + pert_metric[pert][m + '_top20_de_non_zero'] = val + else: + val = fct(test_res['pred'][p_idx].mean(0)[de_idx] - ctrl[0][de_idx], test_res['truth'][p_idx].mean(0)[de_idx]-ctrl[0][de_idx]) + pert_metric[pert][m + '_top20_de_non_zero'] = val + + return pert_metric + +def non_dropout_analysis(adata, test_res): + metric2fct = { + 'pearson': pearsonr, + 'mse': mse + } + + pert_metric = {} + + ## in silico modeling and upperbounding + pert2pert_full_id = dict(adata.obs[['condition', 'condition_name']].values) + geneid2name = dict(zip(adata.var.index.values, adata.var['gene_name'])) + geneid2idx = dict(zip(adata.var.index.values, range(len(adata.var.index.values)))) + + # calculate mean expression for each condition + unique_conditions = adata.obs.condition.unique() + conditions2index = {} + for i in unique_conditions: + conditions2index[i] = np.where(adata.obs.condition == i)[0] + + condition2mean_expression = {} + for i, j in conditions2index.items(): + condition2mean_expression[i] = np.mean(adata.X[j], axis = 0) + pert_list = np.array(list(condition2mean_expression.keys())) + mean_expression = np.array(list(condition2mean_expression.values())).reshape(len(adata.obs.condition.unique()), adata.X.toarray().shape[1]) + ctrl = mean_expression[np.where(pert_list == 'ctrl')[0]] + + gene_list = adata.var['gene_name'].values + + for pert in np.unique(test_res['pert_cat']): + pert_metric[pert] = {} + + pert_idx = np.where(test_res['pert_cat'] == pert)[0] + de_idx = [geneid2idx[i] for i in adata.uns['top_non_dropout_de_20'][pert2pert_full_id[pert]]] + non_zero_idx = adata.uns['non_zeros_gene_idx'][pert2pert_full_id[pert]] + non_dropout_gene_idx = adata.uns['non_dropout_gene_idx'][pert2pert_full_id[pert]] + + direc_change = np.abs(np.sign(test_res['pred'][pert_idx].mean(0)[de_idx] - ctrl[0][de_idx]) - np.sign(test_res['truth'][pert_idx].mean(0)[de_idx] - ctrl[0][de_idx])) + frac_correct_direction = len(np.where(direc_change == 0)[0])/len(de_idx) + pert_metric[pert]['frac_correct_direction_top20_non_dropout'] = frac_correct_direction + + frac_direction_opposite = len(np.where(direc_change == 2)[0])/len(de_idx) + pert_metric[pert]['frac_opposite_direction_top20_non_dropout'] = frac_direction_opposite + + frac_direction_opposite = len(np.where(direc_change == 1)[0])/len(de_idx) + pert_metric[pert]['frac_0/1_direction_top20_non_dropout'] = frac_direction_opposite + + direc_change = np.abs(np.sign(test_res['pred'][pert_idx].mean(0)[non_zero_idx] - ctrl[0][non_zero_idx]) - np.sign(test_res['truth'][pert_idx].mean(0)[non_zero_idx] - ctrl[0][non_zero_idx])) + frac_correct_direction = len(np.where(direc_change == 0)[0])/len(non_zero_idx) + pert_metric[pert]['frac_correct_direction_non_zero'] = frac_correct_direction + + frac_direction_opposite = len(np.where(direc_change == 2)[0])/len(non_zero_idx) + pert_metric[pert]['frac_opposite_direction_non_zero'] = frac_direction_opposite + + frac_direction_opposite = len(np.where(direc_change == 1)[0])/len(non_zero_idx) + pert_metric[pert]['frac_0/1_direction_non_zero'] = frac_direction_opposite + + direc_change = np.abs(np.sign(test_res['pred'][pert_idx].mean(0)[non_dropout_gene_idx] - ctrl[0][non_dropout_gene_idx]) - np.sign(test_res['truth'][pert_idx].mean(0)[non_dropout_gene_idx] - ctrl[0][non_dropout_gene_idx])) + frac_correct_direction = len(np.where(direc_change == 0)[0])/len(non_dropout_gene_idx) + pert_metric[pert]['frac_correct_direction_non_dropout'] = frac_correct_direction + + frac_direction_opposite = len(np.where(direc_change == 2)[0])/len(non_dropout_gene_idx) + pert_metric[pert]['frac_opposite_direction_non_dropout'] = frac_direction_opposite + + frac_direction_opposite = len(np.where(direc_change == 1)[0])/len(non_dropout_gene_idx) + pert_metric[pert]['frac_0/1_direction_non_dropout'] = frac_direction_opposite + + mean = np.mean(test_res['truth'][pert_idx][:, de_idx], axis = 0) + std = np.std(test_res['truth'][pert_idx][:, de_idx], axis = 0) + min_ = np.min(test_res['truth'][pert_idx][:, de_idx], axis = 0) + max_ = np.max(test_res['truth'][pert_idx][:, de_idx], axis = 0) + q25 = np.quantile(test_res['truth'][pert_idx][:, de_idx], 0.25, axis = 0) + q75 = np.quantile(test_res['truth'][pert_idx][:, de_idx], 0.75, axis = 0) + q55 = np.quantile(test_res['truth'][pert_idx][:, de_idx], 0.55, axis = 0) + q45 = np.quantile(test_res['truth'][pert_idx][:, de_idx], 0.45, axis = 0) + q40 = np.quantile(test_res['truth'][pert_idx][:, de_idx], 0.4, axis = 0) + q60 = np.quantile(test_res['truth'][pert_idx][:, de_idx], 0.6, axis = 0) + + zero_des = np.intersect1d(np.where(min_ == 0)[0], np.where(max_ == 0)[0]) + nonzero_des = np.setdiff1d(list(range(20)), zero_des) + + if len(nonzero_des) == 0: + pass + # pert that all de genes are 0... + else: + pred_mean = np.mean(test_res['pred'][pert_idx][:, de_idx], axis = 0).reshape(-1,) + true_mean = np.mean(test_res['truth'][pert_idx][:, de_idx], axis = 0).reshape(-1,) + + in_range = (pred_mean[nonzero_des] >= min_[nonzero_des]) & (pred_mean[nonzero_des] <= max_[nonzero_des]) + frac_in_range = sum(in_range)/len(nonzero_des) + pert_metric[pert]['frac_in_range_non_dropout'] = frac_in_range + + in_range_5 = (pred_mean[nonzero_des] >= q45[nonzero_des]) & (pred_mean[nonzero_des] <= q55[nonzero_des]) + frac_in_range_45_55 = sum(in_range_5)/len(nonzero_des) + pert_metric[pert]['frac_in_range_45_55_non_dropout'] = frac_in_range_45_55 + + in_range_10 = (pred_mean[nonzero_des] >= q40[nonzero_des]) & (pred_mean[nonzero_des] <= q60[nonzero_des]) + frac_in_range_40_60 = sum(in_range_10)/len(nonzero_des) + pert_metric[pert]['frac_in_range_40_60_non_dropout'] = frac_in_range_40_60 + + in_range_25 = (pred_mean[nonzero_des] >= q25[nonzero_des]) & (pred_mean[nonzero_des] <= q75[nonzero_des]) + frac_in_range_25_75 = sum(in_range_25)/len(nonzero_des) + pert_metric[pert]['frac_in_range_25_75_non_dropout'] = frac_in_range_25_75 + + zero_idx = np.where(std > 0)[0] + sigma = (np.abs(pred_mean[zero_idx] - mean[zero_idx]))/(std[zero_idx]) + pert_metric[pert]['mean_sigma_non_dropout'] = np.mean(sigma) + pert_metric[pert]['std_sigma_non_dropout'] = np.std(sigma) + pert_metric[pert]['frac_sigma_below_1_non_dropout'] = 1 - len(np.where(sigma > 1)[0])/len(zero_idx) + pert_metric[pert]['frac_sigma_below_2_non_dropout'] = 1 - len(np.where(sigma > 2)[0])/len(zero_idx) + + p_idx = np.where(test_res['pert_cat'] == pert)[0] + for m, fct in metric2fct.items(): + if m != 'mse': + val = fct(test_res['pred'][p_idx].mean(0)[de_idx] - ctrl[0][de_idx], test_res['truth'][p_idx].mean(0)[de_idx]-ctrl[0][de_idx])[0] + if np.isnan(val): + val = 0 + pert_metric[pert][m + '_delta_top20_de_non_dropout'] = val + + + val = fct(test_res['pred'][p_idx].mean(0)[de_idx], test_res['truth'][p_idx].mean(0)[de_idx])[0] + if np.isnan(val): + val = 0 + pert_metric[pert][m + '_top20_de_non_dropout'] = val + else: + val = fct(test_res['pred'][p_idx].mean(0)[de_idx] - ctrl[0][de_idx], test_res['truth'][p_idx].mean(0)[de_idx]-ctrl[0][de_idx]) + pert_metric[pert][m + '_top20_de_non_dropout'] = val + + return pert_metric + +def deeper_analysis(adata, test_res, de_column_prefix = 'rank_genes_groups_cov', most_variable_genes = None): + + metric2fct = { + 'pearson': pearsonr, + 'mse': mse + } + + pert_metric = {} + + ## in silico modeling and upperbounding + pert2pert_full_id = dict(adata.obs[['condition', 'condition_name']].values) + geneid2name = dict(zip(adata.var.index.values, adata.var['gene_name'])) + geneid2idx = dict(zip(adata.var.index.values, range(len(adata.var.index.values)))) + + # calculate mean expression for each condition + unique_conditions = adata.obs.condition.unique() + conditions2index = {} + for i in unique_conditions: + conditions2index[i] = np.where(adata.obs.condition == i)[0] + + condition2mean_expression = {} + for i, j in conditions2index.items(): + condition2mean_expression[i] = np.mean(adata.X[j], axis = 0) + pert_list = np.array(list(condition2mean_expression.keys())) + mean_expression = np.array(list(condition2mean_expression.values())).reshape(len(adata.obs.condition.unique()), adata.X.toarray().shape[1]) + ctrl = mean_expression[np.where(pert_list == 'ctrl')[0]] + + if most_variable_genes is None: + most_variable_genes = np.argsort(np.std(mean_expression, axis = 0))[-200:] + + gene_list = adata.var['gene_name'].values + + for pert in np.unique(test_res['pert_cat']): + pert_metric[pert] = {} + de_idx = [geneid2idx[i] for i in adata.uns['rank_genes_groups_cov_all'][pert2pert_full_id[pert]][:20]] + de_idx_200 = [geneid2idx[i] for i in adata.uns['rank_genes_groups_cov_all'][pert2pert_full_id[pert]][:200]] + de_idx_100 = [geneid2idx[i] for i in adata.uns['rank_genes_groups_cov_all'][pert2pert_full_id[pert]][:100]] + de_idx_50 = [geneid2idx[i] for i in adata.uns['rank_genes_groups_cov_all'][pert2pert_full_id[pert]][:50]] + + pert_idx = np.where(test_res['pert_cat'] == pert)[0] + pred_mean = np.mean(test_res['pred_de'][pert_idx], axis = 0).reshape(-1,) + true_mean = np.mean(test_res['truth_de'][pert_idx], axis = 0).reshape(-1,) + + direc_change = np.abs(np.sign(test_res['pred'][pert_idx].mean(0) - ctrl[0]) - np.sign(test_res['truth'][pert_idx].mean(0) - ctrl[0])) + frac_correct_direction = len(np.where(direc_change == 0)[0])/len(geneid2name) + pert_metric[pert]['frac_correct_direction_all'] = frac_correct_direction + + de_idx_map = {20: de_idx, + 50: de_idx_50, + 100: de_idx_100, + 200: de_idx_200 + } + + for val in [20, 50, 100, 200]: + + direc_change = np.abs(np.sign(test_res['pred'][pert_idx].mean(0)[de_idx_map[val]] - ctrl[0][de_idx_map[val]]) - np.sign(test_res['truth'][pert_idx].mean(0)[de_idx_map[val]] - ctrl[0][de_idx_map[val]])) + frac_correct_direction = len(np.where(direc_change == 0)[0])/val + pert_metric[pert]['frac_correct_direction_' + str(val)] = frac_correct_direction + + mean = np.mean(test_res['truth_de'][pert_idx], axis = 0) + std = np.std(test_res['truth_de'][pert_idx], axis = 0) + min_ = np.min(test_res['truth_de'][pert_idx], axis = 0) + max_ = np.max(test_res['truth_de'][pert_idx], axis = 0) + q25 = np.quantile(test_res['truth_de'][pert_idx], 0.25, axis = 0) + q75 = np.quantile(test_res['truth_de'][pert_idx], 0.75, axis = 0) + q55 = np.quantile(test_res['truth_de'][pert_idx], 0.55, axis = 0) + q45 = np.quantile(test_res['truth_de'][pert_idx], 0.45, axis = 0) + q40 = np.quantile(test_res['truth_de'][pert_idx], 0.4, axis = 0) + q60 = np.quantile(test_res['truth_de'][pert_idx], 0.6, axis = 0) + + zero_des = np.intersect1d(np.where(min_ == 0)[0], np.where(max_ == 0)[0]) + nonzero_des = np.setdiff1d(list(range(20)), zero_des) + if len(nonzero_des) == 0: + pass + # pert that all de genes are 0... + else: + + direc_change = np.abs(np.sign(pred_mean[nonzero_des] - ctrl[0][de_idx][nonzero_des]) - np.sign(true_mean[nonzero_des] - ctrl[0][de_idx][nonzero_des])) + frac_correct_direction = len(np.where(direc_change == 0)[0])/len(nonzero_des) + pert_metric[pert]['frac_correct_direction_20_nonzero'] = frac_correct_direction + + in_range = (pred_mean[nonzero_des] >= min_[nonzero_des]) & (pred_mean[nonzero_des] <= max_[nonzero_des]) + frac_in_range = sum(in_range)/len(nonzero_des) + pert_metric[pert]['frac_in_range'] = frac_in_range + + in_range_5 = (pred_mean[nonzero_des] >= q45[nonzero_des]) & (pred_mean[nonzero_des] <= q55[nonzero_des]) + frac_in_range_45_55 = sum(in_range_5)/len(nonzero_des) + pert_metric[pert]['frac_in_range_45_55'] = frac_in_range_45_55 + + in_range_10 = (pred_mean[nonzero_des] >= q40[nonzero_des]) & (pred_mean[nonzero_des] <= q60[nonzero_des]) + frac_in_range_40_60 = sum(in_range_10)/len(nonzero_des) + pert_metric[pert]['frac_in_range_40_60'] = frac_in_range_40_60 + + in_range_25 = (pred_mean[nonzero_des] >= q25[nonzero_des]) & (pred_mean[nonzero_des] <= q75[nonzero_des]) + frac_in_range_25_75 = sum(in_range_25)/len(nonzero_des) + pert_metric[pert]['frac_in_range_25_75'] = frac_in_range_25_75 + + zero_idx = np.where(std > 0)[0] + sigma = (np.abs(pred_mean[zero_idx] - mean[zero_idx]))/(std[zero_idx]) + pert_metric[pert]['mean_sigma'] = np.mean(sigma) + pert_metric[pert]['std_sigma'] = np.std(sigma) + pert_metric[pert]['frac_sigma_below_1'] = 1 - len(np.where(sigma > 1)[0])/len(zero_idx) + pert_metric[pert]['frac_sigma_below_2'] = 1 - len(np.where(sigma > 2)[0])/len(zero_idx) + + ## correlation on delta + p_idx = np.where(test_res['pert_cat'] == pert)[0] + + for m, fct in metric2fct.items(): + if m != 'mse': + val = fct(test_res['pred'][p_idx].mean(0)- ctrl[0], test_res['truth'][p_idx].mean(0)-ctrl[0])[0] + if np.isnan(val): + val = 0 + + pert_metric[pert][m + '_delta'] = val + + val = fct(test_res['pred'][p_idx].mean(0)[de_idx] - ctrl[0][de_idx], test_res['truth'][p_idx].mean(0)[de_idx]-ctrl[0][de_idx])[0] + if np.isnan(val): + val = 0 + + pert_metric[pert][m + '_delta_de'] = val + + ## up fold changes > 10, + pert_mean = np.mean(test_res['truth'][p_idx], axis = 0).reshape(-1,) + + fold_change = pert_mean/ctrl + fold_change[np.isnan(fold_change)] = 0 + fold_change[np.isinf(fold_change)] = 0 + ## this is to remove the ones that are super low and the fold change becomes unmeaningful + fold_change[0][np.where(pert_mean < 0.5)[0]] = 0 + + o = np.where(fold_change[0] > 0)[0] + + pred_fc = test_res['pred'][p_idx].mean(0)[o] + true_fc = test_res['truth'][p_idx].mean(0)[o] + ctrl_fc = ctrl[0][o] + + if len(o) > 0: + pert_metric[pert]['fold_change_gap_all'] = np.mean(np.abs(pred_fc/ctrl_fc - true_fc/ctrl_fc)) + + + o = np.intersect1d(np.where(fold_change[0] <0.333)[0], np.where(fold_change[0] > 0)[0]) + + pred_fc = test_res['pred'][p_idx].mean(0)[o] + true_fc = test_res['truth'][p_idx].mean(0)[o] + ctrl_fc = ctrl[0][o] + + if len(o) > 0: + pert_metric[pert]['fold_change_gap_downreg_0.33'] = np.mean(np.abs(pred_fc/ctrl_fc - true_fc/ctrl_fc)) + + + o = np.intersect1d(np.where(fold_change[0] <0.1)[0], np.where(fold_change[0] > 0)[0]) + + pred_fc = test_res['pred'][p_idx].mean(0)[o] + true_fc = test_res['truth'][p_idx].mean(0)[o] + ctrl_fc = ctrl[0][o] + + if len(o) > 0: + pert_metric[pert]['fold_change_gap_downreg_0.1'] = np.mean(np.abs(pred_fc/ctrl_fc - true_fc/ctrl_fc)) + + o = np.where(fold_change[0] > 3)[0] + + pred_fc = test_res['pred'][p_idx].mean(0)[o] + true_fc = test_res['truth'][p_idx].mean(0)[o] + ctrl_fc = ctrl[0][o] + + if len(o) > 0: + pert_metric[pert]['fold_change_gap_upreg_3'] = np.mean(np.abs(pred_fc/ctrl_fc - true_fc/ctrl_fc)) + + o = np.where(fold_change[0] > 10)[0] + + pred_fc = test_res['pred'][p_idx].mean(0)[o] + true_fc = test_res['truth'][p_idx].mean(0)[o] + ctrl_fc = ctrl[0][o] + + if len(o) > 0: + pert_metric[pert]['fold_change_gap_upreg_10'] = np.mean(np.abs(pred_fc/ctrl_fc - true_fc/ctrl_fc)) + + ## most variable genes + for m, fct in metric2fct.items(): + if m != 'mse': + val = fct(test_res['pred'][p_idx].mean(0)[most_variable_genes] - ctrl[0][most_variable_genes], test_res['truth'][p_idx].mean(0)[most_variable_genes]-ctrl[0][most_variable_genes])[0] + if np.isnan(val): + val = 0 + pert_metric[pert][m + '_delta_top200_hvg'] = val + + + val = fct(test_res['pred'][p_idx].mean(0)[most_variable_genes], test_res['truth'][p_idx].mean(0)[most_variable_genes])[0] + if np.isnan(val): + val = 0 + pert_metric[pert][m + '_top200_hvg'] = val + else: + val = fct(test_res['pred'][p_idx].mean(0)[most_variable_genes], test_res['truth'][p_idx].mean(0)[most_variable_genes]) + pert_metric[pert][m + '_top200_hvg'] = val + + + ## top 20/50/100/200 DEs + for m, fct in metric2fct.items(): + if m != 'mse': + val = fct(test_res['pred'][p_idx].mean(0)[de_idx] - ctrl[0][de_idx], test_res['truth'][p_idx].mean(0)[de_idx]-ctrl[0][de_idx])[0] + if np.isnan(val): + val = 0 + pert_metric[pert][m + '_delta_top20_de'] = val + + + val = fct(test_res['pred'][p_idx].mean(0)[de_idx], test_res['truth'][p_idx].mean(0)[de_idx])[0] + if np.isnan(val): + val = 0 + pert_metric[pert][m + '_top20_de'] = val + else: + val = fct(test_res['pred'][p_idx].mean(0)[de_idx] - ctrl[0][de_idx], test_res['truth'][p_idx].mean(0)[de_idx]-ctrl[0][de_idx]) + pert_metric[pert][m + '_top20_de'] = val + + + for m, fct in metric2fct.items(): + if m != 'mse': + val = fct(test_res['pred'][p_idx].mean(0)[de_idx_200] - ctrl[0][de_idx_200], test_res['truth'][p_idx].mean(0)[de_idx_200]-ctrl[0][de_idx_200])[0] + if np.isnan(val): + val = 0 + pert_metric[pert][m + '_delta_top200_de'] = val + + + val = fct(test_res['pred'][p_idx].mean(0)[de_idx_200], test_res['truth'][p_idx].mean(0)[de_idx_200])[0] + if np.isnan(val): + val = 0 + pert_metric[pert][m + '_top200_de'] = val + else: + val = fct(test_res['pred'][p_idx].mean(0)[de_idx_200] - ctrl[0][de_idx_200], test_res['truth'][p_idx].mean(0)[de_idx_200]-ctrl[0][de_idx_200]) + pert_metric[pert][m + '_top200_de'] = val + + for m, fct in metric2fct.items(): + if m != 'mse': + + val = fct(test_res['pred'][p_idx].mean(0)[de_idx_100] - ctrl[0][de_idx_100], test_res['truth'][p_idx].mean(0)[de_idx_100]-ctrl[0][de_idx_100])[0] + if np.isnan(val): + val = 0 + pert_metric[pert][m + '_delta_top100_de'] = val + + + val = fct(test_res['pred'][p_idx].mean(0)[de_idx_100], test_res['truth'][p_idx].mean(0)[de_idx_100])[0] + if np.isnan(val): + val = 0 + pert_metric[pert][m + '_top100_de'] = val + else: + val = fct(test_res['pred'][p_idx].mean(0)[de_idx_100] - ctrl[0][de_idx_100], test_res['truth'][p_idx].mean(0)[de_idx_100]-ctrl[0][de_idx_100]) + pert_metric[pert][m + '_top100_de'] = val + + for m, fct in metric2fct.items(): + if m != 'mse': + + val = fct(test_res['pred'][p_idx].mean(0)[de_idx_50] - ctrl[0][de_idx_50], test_res['truth'][p_idx].mean(0)[de_idx_50]-ctrl[0][de_idx_50])[0] + if np.isnan(val): + val = 0 + pert_metric[pert][m + '_delta_top50_de'] = val + + + val = fct(test_res['pred'][p_idx].mean(0)[de_idx_50], test_res['truth'][p_idx].mean(0)[de_idx_50])[0] + if np.isnan(val): + val = 0 + pert_metric[pert][m + '_top50_de'] = val + else: + val = fct(test_res['pred'][p_idx].mean(0)[de_idx_50] - ctrl[0][de_idx_50], test_res['truth'][p_idx].mean(0)[de_idx_50]-ctrl[0][de_idx_50]) + pert_metric[pert][m + '_top50_de'] = val + + + + return pert_metric + + +def GI_subgroup(pert_metric): + GI_type2Score = {} + test_pert_list = list(pert_metric.keys()) + for GI_type, gi_list in GIs.items(): + intersect = np.intersect1d(gi_list, test_pert_list) + if len(intersect) != 0: + GI_type2Score[GI_type] = {} + + for m in list(list(pert_metric.values())[0].keys()): + GI_type2Score[GI_type][m] = np.mean([pert_metric[i][m] for i in intersect if m in pert_metric[i]]) + + return GI_type2Score + +def node_specific_batch_out(models, batch): + # Returns output for all node specific models as a matrix of dimension batch_size x nodes + outs = [] + for idx in range(len(models)): + outs.append(models[idx](batch).detach().cpu().numpy()[:,idx]) + return np.vstack(outs).T + +# Run prediction over all batches +def batch_predict(loader, loaded_models, args): + # Prediction for node specific GNNs + preds = [] + print("Loader size: ", len(loader)) + for itr, batch in enumerate(loader): + print(itr) + batch = batch.to(args['device']) + preds.append(node_specific_batch_out(loaded_models, batch)) + + preds = np.vstack(preds) + return preds + +def get_high_umi_idx(gene_list, high_umi_path=None): + """Return indices for a caller-supplied high-expression gene list.""" + high_umi_path = high_umi_path or os.environ.get('BIOMOD_HIGH_UMI_PATH') + if not high_umi_path or not os.path.isfile(high_umi_path): + raise FileNotFoundError( + "Provide high_umi_path or set BIOMOD_HIGH_UMI_PATH to the " + "genes_with_hi_mean.npy resource; this file is not bundled." + ) + high_umi = np.load(high_umi_path, allow_pickle=True) + high_umi_idx = np.where([g in high_umi for g in gene_list])[0] + return high_umi_idx + +def get_mean_ctrl(adata): + return adata[adata.obs['condition'] == 'ctrl'].to_df().mean().reset_index( + drop=True) + +def get_single_name(g, all_perts): + name = g+'+ctrl' + if name in all_perts: + return name + else: + return 'ctrl+'+g + + +def get_test_set_results_seen2(res, sel_GI_type): + # Get relevant test set results + test_pert_cats = [p for p in np.unique(res['pert_cat']) if + p in GIs[sel_GI_type] or 'ctrl' in p] + pred_idx = np.where([t in test_pert_cats for t in res['pert_cat']]) + out = {} + for key in res: + out[key] = res[key][pred_idx] + return out + +## Synergy loss calculation functions +def get_all_vectors(all_res, mean_control, double, + single1, single2, high_umi_idx): + # Pred + pred_df = pd.DataFrame(all_res['pred']) + pred_df['condition'] = all_res['pert_cat'] + subset_df = pred_df[pred_df['condition'] == double].iloc[:, :-1] + delta_double_pred = subset_df.mean(0) - mean_control + single_df_1_pred = pred_df[pred_df['condition'] == single1].iloc[:, :-1] + single_df_2_pred = pred_df[pred_df['condition'] == single2].iloc[:, :-1] + + # True + truth_df = pd.DataFrame(all_res['truth']) + truth_df['condition'] = all_res['pert_cat'] + subset_df = truth_df[truth_df['condition'] == double].iloc[:, :-1] + delta_double_truth = subset_df.mean(0) - mean_control + single_df_1_truth = truth_df[truth_df['condition'] == single1].iloc[:, :-1] + single_df_2_truth = truth_df[truth_df['condition'] == single2].iloc[:, :-1] + + delta_single_truth_1 = single_df_1_truth.mean(0) - mean_control + delta_single_truth_2 = single_df_2_truth.mean(0) - mean_control + delta_single_pred_1 = single_df_1_pred.mean(0) - mean_control + delta_single_pred_2 = single_df_2_pred.mean(0) - mean_control + + return {'single_pred_1': delta_single_pred_1.values[high_umi_idx], + 'single_pred_2': delta_single_pred_2.values[high_umi_idx], + 'double_pred': delta_double_pred.values[high_umi_idx], + 'single_truth_1': delta_single_truth_1.values[high_umi_idx], + 'single_truth_2': delta_single_truth_2.values[high_umi_idx], + 'double_truth': delta_double_truth.values[high_umi_idx]} + + +def get_coeffs_synergy(singles_expr, double_expr): + results = {} + results['ts'] = TheilSenRegressor(fit_intercept=False, + max_subpopulation=1e5, + max_iter=1000, + random_state=1000) + X = singles_expr + y = double_expr + try: + results['ts'].fit(X, y.ravel()) + except: + print(X) + print(y) + results['c1'] = results['ts'].coef_[0] + results['c2'] = results['ts'].coef_[1] + results['mag'] = np.sqrt((results['c1'] ** 2 + results['c2'] ** 2)) + return results + + +def Fit(all_vectors, type_='pertnet'): + if type_ == 'pertnet': + singles_expr = np.array( + [all_vectors['single_pred_1'], all_vectors['single_pred_2']]).T + first_expr = np.array([all_vectors['single_pred_1']]).T + second_expr = np.array([all_vectors['single_pred_2']]).T + double_expr = np.array(all_vectors['double_pred']).T + + elif type_ == 'truth': + singles_expr = np.array( + [all_vectors['single_truth_1'], all_vectors['single_truth_2']]).T + first_expr = np.array([all_vectors['single_truth_1']]).T + second_expr = np.array([all_vectors['single_truth_2']]).T + double_expr = np.array(all_vectors['double_truth']).T + + return get_coeffs_synergy(singles_expr, double_expr) + + +def get_linear_params(pred_res, high_umi_idx, mean_control, all_perts): + results = {} + for c in set(pred_res['pert_cat']): + if 'ctrl' in c: + continue + double = c + single1 = get_single_name(double.split('+')[0], all_perts) + single2 = get_single_name(double.split('+')[1], all_perts) + all_vectors = get_all_vectors(pred_res, mean_control, double, + single1, single2, high_umi_idx) + + pertnet_results = Fit(all_vectors, type_='pertnet') + truth_results = Fit(all_vectors, type_='truth') + + results[c] = { + 'truth': truth_results, + 'pred': pertnet_results} + + return results + + +# Read in model for each gene + +GIs = { + 'NEOMORPHIC': ['CBL+TGFBR2', + 'KLF1+TGFBR2', + 'MAP2K6+SPI1', + 'SAMD1+TGFBR2', + 'TGFBR2+C19orf26', + 'TGFBR2+ETS2', + 'CBL+UBASH3A', + 'CEBPE+KLF1', + 'DUSP9+MAPK1', + 'FOSB+PTPN12', + 'PLK4+STIL', + 'PTPN12+OSR2', + 'ZC3HAV1+CEBPE'], + 'ADDITIVE': ['BPGM+SAMD1', + 'CEBPB+MAPK1', + 'CEBPB+OSR2', + 'DUSP9+PRTG', + 'FOSB+OSR2', + 'IRF1+SET', + 'MAP2K3+ELMSAN1', + 'MAP2K6+ELMSAN1', + 'POU3F2+FOXL2', + 'RHOXF2BB+SET', + 'SAMD1+PTPN12', + 'SAMD1+UBASH3B', + 'SAMD1+ZBTB1', + 'SGK1+TBX2', + 'TBX3+TBX2', + 'ZBTB10+SNAI1'], + 'EPISTASIS': ['AHR+KLF1', + 'MAPK1+TGFBR2', + 'TGFBR2+IGDCC3', + 'TGFBR2+PRTG', + 'UBASH3B+OSR2', + 'DUSP9+ETS2', + 'KLF1+CEBPA', + 'MAP2K6+IKZF3', + 'ZC3HAV1+CEBPA'], + 'REDUNDANT': ['CDKN1C+CDKN1A', + 'MAP2K3+MAP2K6', + 'CEBPB+CEBPA', + 'CEBPE+CEBPA', + 'CEBPE+SPI1', + 'ETS2+MAPK1', + 'FOSB+CEBPE', + 'FOXA3+FOXA1'], + 'POTENTIATION': ['CNN1+UBASH3A', + 'ETS2+MAP7D1', + 'FEV+CBFA2T3', + 'FEV+ISL2', + 'FEV+MAP7D1', + 'PTPN12+UBASH3A'], + 'SYNERGY_SIMILAR_PHENO':['CBL+CNN1', + 'CBL+PTPN12', + 'CBL+PTPN9', + 'CBL+UBASH3B', + 'FOXA3+FOXL2', + 'FOXA3+HOXB9', + 'FOXL2+HOXB9', + 'UBASH3B+CNN1', + 'UBASH3B+PTPN12', + 'UBASH3B+PTPN9', + 'UBASH3B+ZBTB25'], + 'SYNERGY_DISSIMILAR_PHENO': ['AHR+FEV', + 'DUSP9+SNAI1', + 'FOXA1+FOXF1', + 'FOXA1+FOXL2', + 'FOXA1+HOXB9', + 'FOXF1+FOXL2', + 'FOXF1+HOXB9', + 'FOXL2+MEIS1', + 'IGDCC3+ZBTB25', + 'POU3F2+CBFA2T3', + 'PTPN12+ZBTB25', + 'SNAI1+DLX2', + 'SNAI1+UBASH3B'], + 'SUPPRESSOR': ['CEBPB+PTPN12', + 'CEBPE+CNN1', + 'CEBPE+PTPN12', + 'CNN1+MAPK1', + 'ETS2+CNN1', + 'ETS2+IGDCC3', + 'ETS2+PRTG', + 'FOSB+UBASH3B', + 'IGDCC3+MAPK1', + 'LYL1+CEBPB', + 'MAPK1+PRTG', + 'PTPN12+SNAI1'] +} diff --git a/BioMod/gene_perturbation/model_module.py b/BioMod/gene_perturbation/model_module.py index 4fd4d50636db5be88a6a43cb523ffde9f51a8ca7..f40aaa1359db87da6b329c5f5acfaa93a73cb735 100644 --- a/BioMod/gene_perturbation/model_module.py +++ b/BioMod/gene_perturbation/model_module.py @@ -1,760 +1,761 @@ -""" -================================================================================ -Genetic perturbation task model definition (model_module.py) -================================================================================ - -[Model Architecture Overview] -This document implements a deep learning model for genetic perturbation prediction. The core components include: - -1. ModuleEmbedding: Convert gene-level expression values into module-level tokens - - Supports multiple integration methods (multiply/add/gated_residual) - - Use attention pooling to aggregate genetic information within the module - -2. PerformerEncoderOnly_Module: Encoder-only Transformer model - - Use Performer (Linear Complexity Attention) to process large numbers of module tokens - - Support Masked Autoencoder (MAE) pre-training paradigm - -3. SCFM_Module: wrapper for pre-trained models - - Process the mapping of input genes to pre-trained model genes - - Recover gene-level embeddings from module encoding - -4. GEARS_Model_Module: Genetic perturbation prediction model - - Integrate pre-trained encoder + GNN + decoder - - Predict expression changes after gene perturbation - -[Key technical points] -- Module-based gene representation: Group thousands of genes into hundreds of modules to reduce computational complexity -- Gene embedding recovery: Recover gene-level features from module encoding through average pooling -- GO graph neural network: using gene ontology knowledge to enhance perturbation representation - -================================================================================ -""" - -import torch -import torch.nn as nn -import torch.nn.functional as F -from torch.nn import Sequential, Linear, ReLU -import numpy as np -import pickle -from torch_geometric.nn import SGConv -from typing import List, Tuple, Optional -import sys -import os -from copy import deepcopy -import scipy - -# ============================================================ -# Import Performer related components (does not depend on gears_reduce) -# Performer is a linear complexity Transformer variant suitable for processing long sequences -# ============================================================ -try: - from BioMod.models_performer_module import ( - ModuleEmbedding, - PerformerEncoderOnly_Module, - sanitize_performer_module_config, - ) -except ModuleNotFoundError: - project_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - if project_root not in sys.path: - sys.path.insert(0, project_root) - from BioMod.models_performer_module import ( - ModuleEmbedding, - PerformerEncoderOnly_Module, - sanitize_performer_module_config, - ) - -# ============================================================ -# Original GEARS related classes -# ============================================================ - -class MLP(torch.nn.Module): - """MLP network""" - def __init__(self, sizes, batch_norm=True, last_layer_act="linear"): - super(MLP, self).__init__() - layers = [] - for s in range(len(sizes) - 1): - layers = layers + [ - torch.nn.Linear(sizes[s], sizes[s + 1]), - torch.nn.BatchNorm1d(sizes[s + 1]) - if batch_norm and s < len(sizes) - 1 else None, - torch.nn.ReLU() - ] - - layers = [l for l in layers if l is not None][:-1] - self.activation = last_layer_act - self.network = torch.nn.Sequential(*layers) - self.relu = torch.nn.ReLU() - - def forward(self, x): - return self.network(x) - - -def reindex_tensor_universal(tensor, index_positions, dim, filler='0', device='cpu'): - """ - Reindex tensor along specified dimensions - """ - index_tensor = torch.tensor(index_positions, device=device) - tensor_shape = list(tensor.shape) - tensor_shape[dim] = len(index_positions) - index_shape = [1] * len(tensor_shape) - index_shape[dim] = len(index_positions) - index_tensor = index_tensor.view(*index_shape).expand(*tensor_shape) - - if filler == '0': - padder_shape = deepcopy(tensor_shape) - padder_shape[dim] = 1 - padder = torch.zeros(padder_shape, dtype=tensor.dtype, device=device) - elif filler == 'mean': - padder = tensor.mean(dim=dim, keepdim=True) - else: - raise ValueError("filler should be 0 or mean!") - expanded_tensor = torch.cat((tensor, padder), dim=dim) - reindexed = torch.gather(expanded_tensor, dim, index_tensor) - return reindexed - - -class SCFM_Module(torch.nn.Module): - """ - ================================================================================ - SCFM_Module: wrapper for pre-trained single cell base models - - [Core functions] - Adapt the PerformerEncoderOnly_Module model to the genetic perturbation task to address the following key issues: - - 1. Gene mapping: The gene order of the input data may be different from that of the pre-trained model, and re-indexing is required. - 2. Module-to-gene recovery: Recovering gene-level embeddings from module encodings - 3. Dimension adjustment: adjust the encoder output to the dimensions required by downstream tasks - - [Key Design Decisions] - - use_module_level=False (recommended): restore gene level embedding from module encoding - Advantages: Compatible with existing GEARS architecture, no need to modify downstream tasks - - use_module_level=True (experimental): use module level encoding directly - Downstream architecture needs to be redesigned - - [Gene Embedding Recovery Strategy] - For each gene: - 1. Find all modules containing the gene - 2. Perform average pooling on the encoding of these modules - 3. If the gene is not in any module, use the global average as fallback - - [Performance Optimization] - - Precompute scatter index to avoid Python loops - - Vectorized aggregation using scatter_add - ================================================================================ - """ - def __init__(self, input_genes, scfm_genes, - scfm_hyper_params, scfm_ckpt, - module_sets=None, - requires_grad=False, - device='cpu', - output_dim=200, - use_module_level=False): - super(SCFM_Module, self).__init__() - self.device = device - self.use_module_level = use_module_level - - # ============================================================ - # [Gene Mapping] Establish index mapping between input genes and pre-trained model genes - # This is a critical step in dealing with inconsistencies in gene order across different datasets - # ============================================================ - input_mapping = {idx: i for i, idx in enumerate(input_genes)} - self.input_gene_num = len(input_genes) - - # ============================================================ - # [Key] fusion_method processing - # Read from pkl to ensure consistency with pre-training - # ============================================================ - fusion_method = scfm_hyper_params.get('fusion_method', 'multiply') - print(f"[SCFM_Module] Using fusion_method from hyper_params: {fusion_method}") - - # Convert new indices to a list of index positions from the original tensor - scfm_genes_pad = scfm_genes + ['PAD'] - self.scfm_seq_len = len(scfm_genes_pad) - scfm_mapping = {idx: i for i, idx in enumerate(scfm_genes_pad)} - - # Create bidirectional index mapping - self.scfm_index_positions = [input_mapping.get(idx, self.input_gene_num) for idx in scfm_genes_pad] - self.input_index_positions = [scfm_mapping.get(idx, self.scfm_seq_len) for idx in input_genes] - - # Load module data (if not provided in hyper_params) - if module_sets is None: - if 'module_sets' in scfm_hyper_params and scfm_hyper_params['module_sets'] is not None: - module_sets = scfm_hyper_params['module_sets'] - else: - raise ValueError("module_sets must be provided (via parameters or hyper_params)") - - self.module_sets = module_sets - self.num_modules = len(module_sets) - self.scfm_seq_len = len(scfm_genes) - - # ============================================================ - # [Key] Create a gene-to-module mapping - # Each gene may appear in multiple modules for subsequent average pooling. - # This is the core data structure of module representation. - # ============================================================ - self.gene_to_modules = {} # {gene_idx: [module_idx1, module_idx2, ...]} - for module_idx, module_genes in enumerate(module_sets): - for gene_idx in module_genes: - if gene_idx < len(scfm_genes): # Only consider genes in scfm_genes - if gene_idx not in self.gene_to_modules: - self.gene_to_modules[gene_idx] = [] - self.gene_to_modules[gene_idx].append(module_idx) - - # ============================================================ - # [Performance Optimization] Pre-built sparse index for efficient module->gene aggregation - # Use scatter_add to replace Python loop, greatly improving forward efficiency - # This is the standard optimization method in PyTorch for handling irregular aggregations - # ============================================================ - import torch.sparse - max_modules_per_gene = max(len(v) for v in self.gene_to_modules.values()) if self.gene_to_modules else 1 - - # Construct a sparse index matrix: shape (num_genes, max_modules_per_gene) - gene_to_module_matrix = [] - for gene_idx in range(self.scfm_seq_len): - if gene_idx in self.gene_to_modules: - modules = self.gene_to_modules[gene_idx] - padded = modules + [-1] * (max_modules_per_gene - len(modules)) - gene_to_module_matrix.append(padded) - else: - gene_to_module_matrix.append([-1] * max_modules_per_gene) - - # Registered as a buffer (will not be updated by the optimizer, but will be moved to the correct device with the model) - self.register_buffer('gene_to_module_matrix', - torch.tensor(gene_to_module_matrix, dtype=torch.long)) - gene_module_counts = torch.tensor([ - len(self.gene_to_modules.get(i, [])) for i in range(self.scfm_seq_len) - ], dtype=torch.long) - self.register_buffer('gene_module_counts', gene_module_counts) - - # ============================================================ - # [Performance Optimization] Precompute scatter_add index - # Module->gene aggregation for fully vectorized, avoiding Python loops - # ============================================================ - module_indices = [] - gene_indices = [] - for module_idx, module_genes in enumerate(module_sets): - for gene_idx in module_genes: - if gene_idx < self.scfm_seq_len: - module_indices.append(module_idx) - gene_indices.append(gene_idx) - - self.register_buffer('scatter_module_idx', torch.tensor(module_indices, dtype=torch.long)) - self.register_buffer('scatter_gene_idx', torch.tensor(gene_indices, dtype=torch.long)) - - # Create PerformerEncoderOnly_Module model - scfm_hyper_params['module_sets'] = module_sets - scfm_hyper_params['num_genes'] = len(scfm_genes) - scfm_hyper_params['max_seq_len'] = len(module_sets) - - self.scfm = PerformerEncoderOnly_Module(**scfm_hyper_params) - - # ============================================================ - # [Key] Load pre-trained weights - # Filter out incompatible parameters (GO related, module_heads, etc.) - # strict=False allows partial parameter mismatch (used for fine-tuning mode) - # ============================================================ - if scfm_ckpt != 'None': - ckpt = torch.load(scfm_ckpt, map_location=device) - model_state_dict = ckpt.get('model_state_dict', ckpt) - current_state_dict = self.scfm.state_dict() - filtered_state_dict = {} - for k, v in model_state_dict.items(): - # Filter out GO related parameters (may not be available in different data sets) - if 'go_conv' in k or 'G_go' in k.lower(): - continue - # Filter out module_heads (not needed for fine-tuning) - if 'module_heads' in k: - continue - # Filter out parameters that do not exist or have mismatched shapes in the current model - if k not in current_state_dict: - continue - if current_state_dict[k].shape != v.shape: - print( - f"[SCFM_Module] skip shape-mismatched key: {k}, " - f"ckpt={tuple(v.shape)} current={tuple(current_state_dict[k].shape)}" - ) - continue - filtered_state_dict[k] = v - # Use strict=False because module_heads is intentionally removed - missing, unexpected = self.scfm.load_state_dict(filtered_state_dict, strict=False) - for param in self.scfm.parameters(): - param.requires_grad = requires_grad - else: - for param in self.scfm.parameters(): - param.requires_grad = True - - # Output dimension adjustment (if needed) - self.output_dim = output_dim - if self.output_dim != 200: - try: - self.feedforward = nn.Linear(self.scfm.base_dim, self.output_dim) - except AttributeError: - self.feedforward = nn.Linear(scfm_hyper_params.get('base_dim', 200), self.output_dim) - - def forward(self, x): - """ - Forward propagation: from gene expression values to gene-level embeddings - - [Processing Process] - 1. Re-index the input gene expression into the gene order of the pre-trained model - 2. Get the module encoding through PerformerEncoderOnly_Module - 3. Restore gene-level embedding from module encoding (or return module encoding directly) - 4. Reindex gene embeddings back to input gene order - - [Input and output] - Input: x (B, input_gene_num) - gene expression value - Output: - - use_module_level=False: output_reindex (B, input_gene_num, D) - gene level embedding - - use_module_level=True: module_encodings (B, num_modules, D) - module level encoding - """ - x = x.view((-1, self.input_gene_num)) # B, M - - # ============================================================ - # [Step 1] Re-index: Map the input gene expression to the gene order of the pre-trained model - # Genes that are not present are filled with 0 (PAD position) - # ============================================================ - x_scfm = reindex_tensor_universal(x, self.scfm_index_positions, dim=1, filler='0', device=self.device) # B, N - - # ============================================================ - # [Step 2] Obtain the module code through the pre-trained model - # mask_ratio=0 means not to mask any modules (fine-tuning mode) - # Use mask_ratio>0 for MAE pre-training during pre-training. - # ============================================================ - out = self.scfm(x_scfm, mask_ratio=0.0, return_encodings=True) - - if 'encodings' in out: - module_encodings = out['encodings'] # Local legacy interface - elif 'module_encodings' in out: - module_encodings = out['module_encodings'] # Reuse pre-trained shared implementation interfaces - else: - raise ValueError(f"Expected one of ['encodings', 'module_encodings'] in output, got: {list(out.keys())}") - - # ============================================================ - # [Option] Directly return to module level encoding - # For experimental module-level downstream tasks - # ============================================================ - if self.use_module_level: - if self.output_dim != 200: - B, num_modules, D = module_encodings.shape - module_encodings_flat = module_encodings.reshape(B * num_modules, D) - module_encodings_flat = self.feedforward(module_encodings_flat) - module_encodings = module_encodings_flat.reshape(B, num_modules, self.output_dim) - return module_encodings # (B, num_modules, D) - - # ============================================================ - # [Step 3] Restore gene-level embedding from module encoding - # Core idea: The embedding of each gene = the average encoding of all modules containing the gene - # Vectorized aggregation using scatter_add to avoid Python loops - # ============================================================ - B, num_modules, D = module_encodings.shape - G = self.scfm_seq_len - - # Calculate global average (for genes without any modules) - global_mean = module_encodings.mean(dim=1) # (B, D) - - # Extract the corresponding module embeddings: (B, num_pairs, D) - module_embs = module_encodings[:, self.scatter_module_idx, :] # (B, num_pairs, D) - - # Aggregate to gene dimension using scatter_add - scfm_gene_embeddings = torch.zeros(B, G, D, device=module_encodings.device) - gene_idx_expanded = self.scatter_gene_idx.unsqueeze(0).unsqueeze(-1).expand(B, -1, D) - scfm_gene_embeddings.scatter_add_(1, gene_idx_expanded, module_embs) - - # ============================================================ - # [Key] Divide by the number of modules for each gene to get the average - # This is the core operation from module to gene aggregation - # clamp(min=1) avoids division by zero errors - # ============================================================ - gene_counts = self.gene_module_counts.float().view(1, G, 1).clamp(min=1) - scfm_gene_embeddings = scfm_gene_embeddings / gene_counts - - # For genes without any modules, use global average as fallback - no_module_mask = (self.gene_module_counts == 0).view(1, G, 1).float() - scfm_gene_embeddings = scfm_gene_embeddings * (1 - no_module_mask) + global_mean.unsqueeze(1) * no_module_mask - - # Add embedding of PAD positions (for handling non-existent genes) - pad_emb = torch.zeros(B, 1, D, device=module_encodings.device) - output_encodings = torch.cat([scfm_gene_embeddings, pad_emb], dim=1) # (B, G+1, D) - - # ============================================================ - # [Step 4] Reindex back to the input gene sequence - # Use mean as fallback to fill in non-existent genes - # ============================================================ - output_reindex = reindex_tensor_universal( - output_encodings, - self.input_index_positions, - dim=1, - filler='mean', - device=self.device - ) # B, M, D - - if self.output_dim != 200: - output_reindex = self.feedforward(output_reindex) - - return output_reindex - - -class GEARS_Model_Module(torch.nn.Module): - """ - ================================================================================ - GEARS_Model_Module: Genetic perturbation prediction model - - [Core functions] - Predicting transcriptomic changes in cells following genetic perturbation. Enter a perturbation combination (such as knockout gene A), - Output the expression value prediction of each gene after perturbation. - - [Model Architecture] - 1. Pretrained encoder (SCFM_Module): Encode gene expression into embedding vectors - 2. Perturbation embedding layer: generate perturbation features for perturbed genes - 3. GO graph neural network (GNN): using gene ontology knowledge to enhance perturbation representation - 4. Decoder: predict gene expression values after perturbation - - [Key Design Decisions] - - Use GO graph neural network: gene ontology contains gene functional relationships, which can guide model understanding - How perturbation effects spread between genes - - Gene specific decoder: Each gene has independent decoding parameters to capture gene specificity - - Cross-gene MLP: Modeling interdependencies between genes - - [Input and output] - Input: data (PyG Data object) - - x: (B*N, 2) feature matrix, [:, 0] is gene expression, [:, 1] is perturbation mark - - y: true expression value (used for training) - Output: out (B, N) - predicted gene expression value after perturbation - ================================================================================ - """ - def __init__(self, args): - super(GEARS_Model_Module, self).__init__() - self.args = args - - # Forecast record related settings - if args.get('record_pred', False): - self.pred_dir = f"./result_process/preds/{self.args['exp_name']}" - if not os.path.exists(self.pred_dir): - os.makedirs(self.pred_dir) - self.pred_batch_idx = 0 - - # ============================================================ - # [Basic parameters] - # hidden_size: embedding dimension, affecting model capacity - # num_genes: number of genes, determines the output dimension - # ============================================================ - self.num_genes = args['num_genes'] - hidden_size = args['hidden_size'] - self.uncertainty = args['uncertainty'] - self.num_layers = args['num_go_gnn_layers'] # GO GNN layers - self.indv_out_hidden_size = args['decoder_hidden_size'] - self.num_layers_gene_pos = args['num_gene_gnn_layers'] - self.pert_emb_lambda = 0.2 - - self.input_genes_list = args['input_genes_ens_ids'] - self.scfm_genes_list = args['scfm_genes_ens_ids'] - - # ============================================================ - # [Load pre-trained model configuration] - # hyper_params contains model architecture parameters, which must be consistent with pre-training - # ============================================================ - try: - with open(args["scfm_hyper_params_path"], 'rb') as handle: - self.scfm_hyper_params = pickle.load(handle) - except ValueError: - import pickle5 - with open(args["scfm_hyper_params_path"], 'rb') as handle: - self.scfm_hyper_params = pickle5.load(handle) - self.scfm_hyper_params.pop('module_embed_chunk_padded_gene_budget', None) - self.scfm_hyper_params = sanitize_performer_module_config( - self.scfm_hyper_params, - source=args["scfm_hyper_params_path"], - log_fn=print, - ) - - self.scfm_hyper_params['gene2vec_file'] = args['scfm_gene2vec_file'] - self.scfm_ckpt = args["scfm_ckpt"] - - # ============================================================ - # [Module data loading] - # module_sets defines how genes are grouped into modules - # Can be loaded from hyper_params or external files - # ============================================================ - module_sets = None - if 'module_sets' in self.scfm_hyper_params and self.scfm_hyper_params['module_sets'] is not None: - module_sets = self.scfm_hyper_params['module_sets'] - elif args.get('module_data_path') or args.get('module_data_dir'): - from BioMod.module_dataset import load_module_data - module_sets = load_module_data( - module_data_path=args.get('module_data_path'), - module_data_dir=args.get('module_data_dir'), - gene_list_path=args.get('scfm_genes_list_path'), - gene_mapping_path=args.get('gene_mapping_path', '../gene2ens.csv'), - verbose=True - ) - else: - raise ValueError("module_sets must be provided (via hyper_params or module_data_path/module_data_dir)") - - # ============================================================ - # [Ablation Experiment]shuffle_module_genes - # Disrupt the gene assignment within the module and verify the importance of the module structure (gene grouping) - # If performance decreases after scrambling, it means that the module structure contains useful biological information. - # ============================================================ - if args.get('shuffle_module_genes', False): - shuffle_seed = args.get('shuffle_seed', 2021) - print( - f"[GEARS_Model_Module] Applying downstream-only extra module reshuffle " - f"(seed={shuffle_seed}). Pretraining-time module_sets from PKL will be reshuffled again." - ) - all_gene_indices = [] - module_sizes = [] - for module in module_sets: - all_gene_indices.extend(module) - module_sizes.append(len(module)) - - import random - rng = random.Random(shuffle_seed) - rng.shuffle(all_gene_indices) - - shuffled_module_sets = [] - start_idx = 0 - for size in module_sizes: - shuffled_module_sets.append(all_gene_indices[start_idx:start_idx + size]) - start_idx += size - module_sets = shuffled_module_sets - print(f"[GEARS_Model_Module] Shuffled {len(all_gene_indices)} genes across {len(module_sets)} modules") - - # ============================================================ - # [Perturbation Embedding Layer] - # pert_w: Convert perturbation markers (0/1) to embedding vectors - # gene_emb: Basic embedding of genes (used when pre-trained models are not available) - # pert_emb: dedicated embedding of perturbed genes - # ============================================================ - self.pert_w = nn.Linear(1, hidden_size) - - # gene/global perturbation embedding dictionary lookup - self.gene_emb = nn.Embedding(self.num_genes, hidden_size, max_norm=True) - self.pert_emb = nn.Embedding(self.num_genes, hidden_size, max_norm=True) - - # transformation layer - self.emb_trans = nn.ReLU() - self.pert_base_trans = nn.ReLU() - self.transform = nn.ReLU() - self.pert_fuse = MLP([hidden_size, hidden_size, hidden_size], last_layer_act='ReLU') - - # ============================================================ - # [GO graph neural network] - # G_sim: edge index of GO similarity graph - # G_sim_weight: edge weight - # Using SGConv (Simplified Graph Convolution) for message passing - # GO diagram helps the model understand gene functional relationships, and perturbation effects propagate along the diagram. - # ============================================================ - self.G_sim = args['G_go'].to(args['device']) - self.G_sim_weight = args['G_go_weight'].to(args['device']) - - self.sim_layers = torch.nn.ModuleList() - for i in range(1, self.num_layers + 1): - self.sim_layers.append(SGConv(hidden_size, hidden_size, 1)) - - # ============================================================ - # [Decoder] - # recovery_w: shared MLP, projects embeddings into decoding space - # indv_w1/b1: Gene specific parameters, each gene has independent decoding weight - # cross_gene_state: cross-gene MLP, modeling inter-gene dependencies - # indv_w2/b2: final decoding layer - # ============================================================ - self.recovery_w = MLP([hidden_size, hidden_size*2, hidden_size], last_layer_act='linear') - - # gene specific decoder - self.indv_w1 = nn.Parameter(torch.rand(self.num_genes, hidden_size, 1)) - self.indv_b1 = nn.Parameter(torch.rand(self.num_genes, 1)) - self.act = nn.ReLU() - nn.init.xavier_normal_(self.indv_w1) - nn.init.xavier_normal_(self.indv_b1) - - # Cross gene MLP - self.cross_gene_state = MLP([self.num_genes, hidden_size, hidden_size]) - self.indv_w2 = nn.Parameter(torch.rand(1, self.num_genes, hidden_size+1)) - self.indv_b2 = nn.Parameter(torch.rand(1, self.num_genes)) - nn.init.xavier_normal_(self.indv_w2) - nn.init.xavier_normal_(self.indv_b2) - - # BatchNorm layers - self.bn_emb = nn.BatchNorm1d(hidden_size) - self.bn_pert_base = nn.BatchNorm1d(hidden_size) - self.bn_pert_base_trans = nn.BatchNorm1d(hidden_size) - - # uncertainty mode (optional) - if self.uncertainty: - self.uncertainty_w = MLP([hidden_size, hidden_size*2, hidden_size, 1], last_layer_act='linear') - - # ============================================================ - # [Core] Initialize pre-trained encoder (SCFM_Module) - # requires_grad: whether to fine-tune pre-training parameters - # - finetune_method='frozen': freeze pre-training parameters - # - Others: Full parameter fine-tuning - # ============================================================ - self.use_module_level = args.get('use_module_level', False) - self.singlecell_model = SCFM_Module( - self.input_genes_list, - self.scfm_genes_list, - self.scfm_hyper_params, - self.scfm_ckpt, - module_sets=module_sets, - requires_grad=args.get('finetune_method') != 'frozen', - device=args['device'], - output_dim=hidden_size, - use_module_level=self.use_module_level - ) - self.pretrained = True - - # ============================================================ - # [Experimental feature] module level features - # In the current implementation, module-level downstream tasks are not fully implemented. - # GNN and decoder need to be redesigned to handle module dimensions - # ============================================================ - if self.use_module_level: - self.num_modules = len(module_sets) - import warnings - warnings.warn( - f"Using module-level features. This requires compatible downstream architecture. " - f"Number of modules: {self.num_modules}, Number of genes: {self.num_genes}. " - f"Currently module-level downstream task is not fully implemented.", - UserWarning - ) - - def forward(self, data): - """ - Forward propagation: predicting expression values after gene perturbation - - [enter] - data: PyG Data object - - x: (B*N, 2) characteristic matrix - [:, 0]: gene expression value (control group) - [:, 1]: perturbation flag (1 indicates that the gene is perturbed) - - y: real expression value (used during training) - - pert: disturbance type identifier - - [Output] - out: (B, N) predicted gene expression value after perturbation - - [Processing Process] - 1. Obtain base gene embeddings (from pretrained encoder or random embeddings) - 2. Generate perturbation embedding (enhanced by GO-GNN) - 3. Merge base embedding and perturbation embedding - 4. Predict the expression value through the decoder - """ - if str(type(data)) == "": - x = data.reshape((-1, 2)) - else: - x, batch = data.x, data.batch - x_pert = x[:, 1].reshape((-1, self.num_genes)) # Disturbance mark - num_graphs = len(data.batch.unique()) - - # ============================================================ - # [Step 1] Obtain basic gene embedding - # If using a pretrained model, get embeddings from gene expression encoding - # Otherwise use randomly initialized gene embeddings - # ============================================================ - if self.pretrained: - exp = x[:, 0].reshape((-1, self.num_genes)) # B, N - gene expression value - emb = self.singlecell_model(exp) # B, N, D - by pre-trained encoder - emb = emb.reshape((num_graphs * self.num_genes, -1)) - else: - emb = self.gene_emb(torch.LongTensor(list(range(self.num_genes))).repeat(num_graphs, ).to(self.args['device'])) - emb = self.bn_emb(emb) - base_emb = self.emb_trans(emb) - - base_emb = base_emb.reshape(num_graphs, self.num_genes, -1) - - pert = x_pert.reshape(-1, 1) # B * N, 1 - - # ============================================================ - # [Step 2] Generate perturbation embedding (GO-GNN enhancement) - # Perturbation embeddings are propagated through GO similarity graphs to capture gene functional relationships - # This enables the model to exploit a priori knowledge of gene function - # ============================================================ - pert_embeddings = self.pert_emb(torch.LongTensor(list(range(self.num_genes))).to(self.args['device'])) # N, D - - # Propagating perturbation embeddings via GO-GNN - for idx, layer in enumerate(self.sim_layers): - pert_embeddings = layer(pert_embeddings, self.G_sim, self.G_sim_weight) - if idx < self.num_layers - 1: - pert_embeddings = pert_embeddings.relu() - - # After GNN: N, D - pert_embeddings = torch.stack(list((pert_embeddings,)) * num_graphs) # B, N, D - pert_embeddings = pert_embeddings * x_pert.reshape((num_graphs, self.num_genes, 1)) # Only perturbed genes are retained - pert_embeddings = pert_embeddings.sum(1) # B, D - aggregate all perturbed genes - pert_mask = (x_pert.sum(1) != 0).reshape((num_graphs, 1)) # B, 1 - pert_embeddings = self.pert_fuse(pert_embeddings) * pert_mask # B, D - pert_embeddings = pert_embeddings.reshape((num_graphs, 1, pert_embeddings.shape[1])) # B, 1, D - - # ============================================================ - # [Step 3] Merge base embedding and perturbation embedding - # Perturbation information is added to the embedding of each gene - # ============================================================ - base_emb = base_emb + pert_embeddings - - base_emb = base_emb.reshape(num_graphs * self.num_genes, -1) - - # Add perturbed position embedding - pert_emb = self.pert_w(pert) - combined = pert_emb + base_emb - combined = self.bn_pert_base_trans(combined) - base_emb = self.pert_base_trans(combined) - base_emb = self.bn_pert_base(base_emb) - - # ============================================================ - # [Step 4] Decoder: predict the expression value after perturbation - # Using gene-specific decoders + cross-gene MLP - # ============================================================ - base_emb = self.transform(base_emb) - out = self.recovery_w(base_emb) # B * N, D - out = out.reshape(num_graphs, self.num_genes, -1) # B, N, D - out = out.unsqueeze(-1) * self.indv_w1 # B, N, D, 1 - Gene specific transformation - w = torch.sum(out, axis=2) # B, N, 1 - out = w + self.indv_b1 # B, N, 1 - - # Cross gene MLP: Modeling inter-gene dependencies - cross_gene_embed = self.cross_gene_state(out.reshape(num_graphs, self.num_genes, -1).squeeze(2)) # B, D - cross_gene_embed = cross_gene_embed.repeat(1, self.num_genes) # B, N*D - - cross_gene_embed = cross_gene_embed.reshape([num_graphs, self.num_genes, -1]) # B, N, D - cross_gene_out = torch.cat([out, cross_gene_embed], 2) # B, N, D+1 - - cross_gene_out = cross_gene_out * self.indv_w2 # B, N, D+1 - cross_gene_out = torch.sum(cross_gene_out, axis=2) # B, N - out = cross_gene_out + self.indv_b2 # B, N - - # ============================================================ - # [Key] Residual connection: predicted value = change + original value - # This is the core design of GEARS. The model predicts expression changes rather than absolute values. - # ============================================================ - out = out.reshape(num_graphs * self.num_genes, -1) + x[:, 0].reshape(-1, 1) # B * N, 1 - - # ============================================================ - # [Prediction record] for subsequent analysis - # ============================================================ - if self.args.get('start_record', False): - y_flat = data.y.reshape(-1, 1) # B * N, 1 - pred = torch.cat((pert, out, y_flat), dim=1) # B * N, 3 - pred = pred.cpu().to_sparse() - - values = pred.values().numpy() - indices = pred.indices().numpy() - sparse_matrix = scipy.sparse.coo_matrix((values, indices), shape=pred.shape) - - scipy.sparse.save_npz(f'{self.pred_dir}/b{self.pred_batch_idx}_{num_graphs}.npz', sparse_matrix) - self.pred_batch_idx += 1 - - # ============================================================ - # [Key] reshape output is (B, N) format - # In this way, the loss function can correctly index the predicted value and the true value. - # ============================================================ - out = torch.split(torch.flatten(out), self.num_genes) - out = torch.stack(out) # (B, N) - - # uncertainty mode (optional) - if self.uncertainty: - logvar = self.uncertainty_w(base_emb) - logvar = torch.split(torch.flatten(logvar), self.num_genes) - return out, torch.stack(logvar) - - return out # (B, N) - +""" +================================================================================ +Genetic perturbation task model definition (model_module.py) +================================================================================ + +[Model Architecture Overview] +This document implements a deep learning model for genetic perturbation prediction. The core components include: + +1. ModuleEmbedding: Convert gene-level expression values into module-level tokens + - Supports multiple integration methods (multiply/add/gated_residual) + - Use attention pooling to aggregate genetic information within the module + +2. PerformerEncoderOnly_Module: Encoder-only Transformer model + - Use Performer (Linear Complexity Attention) to process large numbers of module tokens + - Support Masked Autoencoder (MAE) pre-training paradigm + +3. SCFM_Module: wrapper for pre-trained models + - Process the mapping of input genes to pre-trained model genes + - Recover gene-level embeddings from module encoding + +4. GEARS_Model_Module: Genetic perturbation prediction model + - Integrate pre-trained encoder + GNN + decoder + - Predict expression changes after gene perturbation + +[Key technical points] +- Module-based gene representation: Group thousands of genes into hundreds of modules to reduce computational complexity +- Gene embedding recovery: Recover gene-level features from module encoding through average pooling +- GO graph neural network: using gene ontology knowledge to enhance perturbation representation + +================================================================================ +""" + +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.nn import Sequential, Linear, ReLU +import numpy as np +import pickle +from torch_geometric.nn import SGConv +from typing import List, Tuple, Optional +import sys +from pathlib import Path +import os +from copy import deepcopy +import scipy + +# ============================================================ +# Import Performer related components (does not depend on gears_reduce) +# Performer is a linear complexity Transformer variant suitable for processing long sequences +# ============================================================ +try: + from BioMod.models_performer_module import ( + ModuleEmbedding, + PerformerEncoderOnly_Module, + sanitize_performer_module_config, + ) +except ModuleNotFoundError: + project_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + if project_root not in sys.path: + sys.path.insert(0, project_root) + from BioMod.models_performer_module import ( + ModuleEmbedding, + PerformerEncoderOnly_Module, + sanitize_performer_module_config, + ) + +# ============================================================ +# Original GEARS related classes +# ============================================================ + +class MLP(torch.nn.Module): + """MLP network""" + def __init__(self, sizes, batch_norm=True, last_layer_act="linear"): + super(MLP, self).__init__() + layers = [] + for s in range(len(sizes) - 1): + layers = layers + [ + torch.nn.Linear(sizes[s], sizes[s + 1]), + torch.nn.BatchNorm1d(sizes[s + 1]) + if batch_norm and s < len(sizes) - 1 else None, + torch.nn.ReLU() + ] + + layers = [l for l in layers if l is not None][:-1] + self.activation = last_layer_act + self.network = torch.nn.Sequential(*layers) + self.relu = torch.nn.ReLU() + + def forward(self, x): + return self.network(x) + + +def reindex_tensor_universal(tensor, index_positions, dim, filler='0', device='cpu'): + """ + Reindex tensor along specified dimensions + """ + index_tensor = torch.tensor(index_positions, device=device) + tensor_shape = list(tensor.shape) + tensor_shape[dim] = len(index_positions) + index_shape = [1] * len(tensor_shape) + index_shape[dim] = len(index_positions) + index_tensor = index_tensor.view(*index_shape).expand(*tensor_shape) + + if filler == '0': + padder_shape = deepcopy(tensor_shape) + padder_shape[dim] = 1 + padder = torch.zeros(padder_shape, dtype=tensor.dtype, device=device) + elif filler == 'mean': + padder = tensor.mean(dim=dim, keepdim=True) + else: + raise ValueError("filler should be 0 or mean!") + expanded_tensor = torch.cat((tensor, padder), dim=dim) + reindexed = torch.gather(expanded_tensor, dim, index_tensor) + return reindexed + + +class SCFM_Module(torch.nn.Module): + """ + ================================================================================ + SCFM_Module: wrapper for pre-trained single cell base models + + [Core functions] + Adapt the PerformerEncoderOnly_Module model to the genetic perturbation task to address the following key issues: + + 1. Gene mapping: The gene order of the input data may be different from that of the pre-trained model, and re-indexing is required. + 2. Module-to-gene recovery: Recovering gene-level embeddings from module encodings + 3. Dimension adjustment: adjust the encoder output to the dimensions required by downstream tasks + + [Key Design Decisions] + - use_module_level=False (recommended): restore gene level embedding from module encoding + Advantages: Compatible with existing GEARS architecture, no need to modify downstream tasks + - use_module_level=True (experimental): use module level encoding directly + Downstream architecture needs to be redesigned + + [Gene Embedding Recovery Strategy] + For each gene: + 1. Find all modules containing the gene + 2. Perform average pooling on the encoding of these modules + 3. If the gene is not in any module, use the global average as fallback + + [Performance Optimization] + - Precompute scatter index to avoid Python loops + - Vectorized aggregation using scatter_add + ================================================================================ + """ + def __init__(self, input_genes, scfm_genes, + scfm_hyper_params, scfm_ckpt, + module_sets=None, + requires_grad=False, + device='cpu', + output_dim=200, + use_module_level=False): + super(SCFM_Module, self).__init__() + self.device = device + self.use_module_level = use_module_level + + # ============================================================ + # [Gene Mapping] Establish index mapping between input genes and pre-trained model genes + # This is a critical step in dealing with inconsistencies in gene order across different datasets + # ============================================================ + input_mapping = {idx: i for i, idx in enumerate(input_genes)} + self.input_gene_num = len(input_genes) + + # ============================================================ + # [Key] fusion_method processing + # Read from pkl to ensure consistency with pre-training + # ============================================================ + fusion_method = scfm_hyper_params.get('fusion_method', 'multiply') + print(f"[SCFM_Module] Using fusion_method from hyper_params: {fusion_method}") + + # Convert new indices to a list of index positions from the original tensor + scfm_genes_pad = scfm_genes + ['PAD'] + self.scfm_seq_len = len(scfm_genes_pad) + scfm_mapping = {idx: i for i, idx in enumerate(scfm_genes_pad)} + + # Create bidirectional index mapping + self.scfm_index_positions = [input_mapping.get(idx, self.input_gene_num) for idx in scfm_genes_pad] + self.input_index_positions = [scfm_mapping.get(idx, self.scfm_seq_len) for idx in input_genes] + + # Load module data (if not provided in hyper_params) + if module_sets is None: + if 'module_sets' in scfm_hyper_params and scfm_hyper_params['module_sets'] is not None: + module_sets = scfm_hyper_params['module_sets'] + else: + raise ValueError("module_sets must be provided (via parameters or hyper_params)") + + self.module_sets = module_sets + self.num_modules = len(module_sets) + self.scfm_seq_len = len(scfm_genes) + + # ============================================================ + # [Key] Create a gene-to-module mapping + # Each gene may appear in multiple modules for subsequent average pooling. + # This is the core data structure of module representation. + # ============================================================ + self.gene_to_modules = {} # {gene_idx: [module_idx1, module_idx2, ...]} + for module_idx, module_genes in enumerate(module_sets): + for gene_idx in module_genes: + if gene_idx < len(scfm_genes): # Only consider genes in scfm_genes + if gene_idx not in self.gene_to_modules: + self.gene_to_modules[gene_idx] = [] + self.gene_to_modules[gene_idx].append(module_idx) + + # ============================================================ + # [Performance Optimization] Pre-built sparse index for efficient module->gene aggregation + # Use scatter_add to replace Python loop, greatly improving forward efficiency + # This is the standard optimization method in PyTorch for handling irregular aggregations + # ============================================================ + import torch.sparse + max_modules_per_gene = max(len(v) for v in self.gene_to_modules.values()) if self.gene_to_modules else 1 + + # Construct a sparse index matrix: shape (num_genes, max_modules_per_gene) + gene_to_module_matrix = [] + for gene_idx in range(self.scfm_seq_len): + if gene_idx in self.gene_to_modules: + modules = self.gene_to_modules[gene_idx] + padded = modules + [-1] * (max_modules_per_gene - len(modules)) + gene_to_module_matrix.append(padded) + else: + gene_to_module_matrix.append([-1] * max_modules_per_gene) + + # Registered as a buffer (will not be updated by the optimizer, but will be moved to the correct device with the model) + self.register_buffer('gene_to_module_matrix', + torch.tensor(gene_to_module_matrix, dtype=torch.long)) + gene_module_counts = torch.tensor([ + len(self.gene_to_modules.get(i, [])) for i in range(self.scfm_seq_len) + ], dtype=torch.long) + self.register_buffer('gene_module_counts', gene_module_counts) + + # ============================================================ + # [Performance Optimization] Precompute scatter_add index + # Module->gene aggregation for fully vectorized, avoiding Python loops + # ============================================================ + module_indices = [] + gene_indices = [] + for module_idx, module_genes in enumerate(module_sets): + for gene_idx in module_genes: + if gene_idx < self.scfm_seq_len: + module_indices.append(module_idx) + gene_indices.append(gene_idx) + + self.register_buffer('scatter_module_idx', torch.tensor(module_indices, dtype=torch.long)) + self.register_buffer('scatter_gene_idx', torch.tensor(gene_indices, dtype=torch.long)) + + # Create PerformerEncoderOnly_Module model + scfm_hyper_params['module_sets'] = module_sets + scfm_hyper_params['num_genes'] = len(scfm_genes) + scfm_hyper_params['max_seq_len'] = len(module_sets) + + self.scfm = PerformerEncoderOnly_Module(**scfm_hyper_params) + + # ============================================================ + # [Key] Load pre-trained weights + # Filter out incompatible parameters (GO related, module_heads, etc.) + # strict=False allows partial parameter mismatch (used for fine-tuning mode) + # ============================================================ + if scfm_ckpt != 'None': + ckpt = torch.load(scfm_ckpt, map_location=device) + model_state_dict = ckpt.get('model_state_dict', ckpt) + current_state_dict = self.scfm.state_dict() + filtered_state_dict = {} + for k, v in model_state_dict.items(): + # Filter out GO related parameters (may not be available in different data sets) + if 'go_conv' in k or 'G_go' in k.lower(): + continue + # Filter out module_heads (not needed for fine-tuning) + if 'module_heads' in k: + continue + # Filter out parameters that do not exist or have mismatched shapes in the current model + if k not in current_state_dict: + continue + if current_state_dict[k].shape != v.shape: + print( + f"[SCFM_Module] skip shape-mismatched key: {k}, " + f"ckpt={tuple(v.shape)} current={tuple(current_state_dict[k].shape)}" + ) + continue + filtered_state_dict[k] = v + # Use strict=False because module_heads is intentionally removed + missing, unexpected = self.scfm.load_state_dict(filtered_state_dict, strict=False) + for param in self.scfm.parameters(): + param.requires_grad = requires_grad + else: + for param in self.scfm.parameters(): + param.requires_grad = True + + # Output dimension adjustment (if needed) + self.output_dim = output_dim + if self.output_dim != 200: + try: + self.feedforward = nn.Linear(self.scfm.base_dim, self.output_dim) + except AttributeError: + self.feedforward = nn.Linear(scfm_hyper_params.get('base_dim', 200), self.output_dim) + + def forward(self, x): + """ + Forward propagation: from gene expression values to gene-level embeddings + + [Processing Process] + 1. Re-index the input gene expression into the gene order of the pre-trained model + 2. Get the module encoding through PerformerEncoderOnly_Module + 3. Restore gene-level embedding from module encoding (or return module encoding directly) + 4. Reindex gene embeddings back to input gene order + + [Input and output] + Input: x (B, input_gene_num) - gene expression value + Output: + - use_module_level=False: output_reindex (B, input_gene_num, D) - gene level embedding + - use_module_level=True: module_encodings (B, num_modules, D) - module level encoding + """ + x = x.view((-1, self.input_gene_num)) # B, M + + # ============================================================ + # [Step 1] Re-index: Map the input gene expression to the gene order of the pre-trained model + # Genes that are not present are filled with 0 (PAD position) + # ============================================================ + x_scfm = reindex_tensor_universal(x, self.scfm_index_positions, dim=1, filler='0', device=self.device) # B, N + + # ============================================================ + # [Step 2] Obtain the module code through the pre-trained model + # mask_ratio=0 means not to mask any modules (fine-tuning mode) + # Use mask_ratio>0 for MAE pre-training during pre-training. + # ============================================================ + out = self.scfm(x_scfm, mask_ratio=0.0, return_encodings=True) + + if 'encodings' in out: + module_encodings = out['encodings'] # Local legacy interface + elif 'module_encodings' in out: + module_encodings = out['module_encodings'] # Reuse pre-trained shared implementation interfaces + else: + raise ValueError(f"Expected one of ['encodings', 'module_encodings'] in output, got: {list(out.keys())}") + + # ============================================================ + # [Option] Directly return to module level encoding + # For experimental module-level downstream tasks + # ============================================================ + if self.use_module_level: + if self.output_dim != 200: + B, num_modules, D = module_encodings.shape + module_encodings_flat = module_encodings.reshape(B * num_modules, D) + module_encodings_flat = self.feedforward(module_encodings_flat) + module_encodings = module_encodings_flat.reshape(B, num_modules, self.output_dim) + return module_encodings # (B, num_modules, D) + + # ============================================================ + # [Step 3] Restore gene-level embedding from module encoding + # Core idea: The embedding of each gene = the average encoding of all modules containing the gene + # Vectorized aggregation using scatter_add to avoid Python loops + # ============================================================ + B, num_modules, D = module_encodings.shape + G = self.scfm_seq_len + + # Calculate global average (for genes without any modules) + global_mean = module_encodings.mean(dim=1) # (B, D) + + # Extract the corresponding module embeddings: (B, num_pairs, D) + module_embs = module_encodings[:, self.scatter_module_idx, :] # (B, num_pairs, D) + + # Aggregate to gene dimension using scatter_add + scfm_gene_embeddings = torch.zeros(B, G, D, device=module_encodings.device) + gene_idx_expanded = self.scatter_gene_idx.unsqueeze(0).unsqueeze(-1).expand(B, -1, D) + scfm_gene_embeddings.scatter_add_(1, gene_idx_expanded, module_embs) + + # ============================================================ + # [Key] Divide by the number of modules for each gene to get the average + # This is the core operation from module to gene aggregation + # clamp(min=1) avoids division by zero errors + # ============================================================ + gene_counts = self.gene_module_counts.float().view(1, G, 1).clamp(min=1) + scfm_gene_embeddings = scfm_gene_embeddings / gene_counts + + # For genes without any modules, use global average as fallback + no_module_mask = (self.gene_module_counts == 0).view(1, G, 1).float() + scfm_gene_embeddings = scfm_gene_embeddings * (1 - no_module_mask) + global_mean.unsqueeze(1) * no_module_mask + + # Add embedding of PAD positions (for handling non-existent genes) + pad_emb = torch.zeros(B, 1, D, device=module_encodings.device) + output_encodings = torch.cat([scfm_gene_embeddings, pad_emb], dim=1) # (B, G+1, D) + + # ============================================================ + # [Step 4] Reindex back to the input gene sequence + # Use mean as fallback to fill in non-existent genes + # ============================================================ + output_reindex = reindex_tensor_universal( + output_encodings, + self.input_index_positions, + dim=1, + filler='mean', + device=self.device + ) # B, M, D + + if self.output_dim != 200: + output_reindex = self.feedforward(output_reindex) + + return output_reindex + + +class GEARS_Model_Module(torch.nn.Module): + """ + ================================================================================ + GEARS_Model_Module: Genetic perturbation prediction model + + [Core functions] + Predicting transcriptomic changes in cells following genetic perturbation. Enter a perturbation combination (such as knockout gene A), + Output the expression value prediction of each gene after perturbation. + + [Model Architecture] + 1. Pretrained encoder (SCFM_Module): Encode gene expression into embedding vectors + 2. Perturbation embedding layer: generate perturbation features for perturbed genes + 3. GO graph neural network (GNN): using gene ontology knowledge to enhance perturbation representation + 4. Decoder: predict gene expression values after perturbation + + [Key Design Decisions] + - Use GO graph neural network: gene ontology contains gene functional relationships, which can guide model understanding + How perturbation effects spread between genes + - Gene specific decoder: Each gene has independent decoding parameters to capture gene specificity + - Cross-gene MLP: Modeling interdependencies between genes + + [Input and output] + Input: data (PyG Data object) + - x: (B*N, 2) feature matrix, [:, 0] is gene expression, [:, 1] is perturbation mark + - y: true expression value (used for training) + Output: out (B, N) - predicted gene expression value after perturbation + ================================================================================ + """ + def __init__(self, args): + super(GEARS_Model_Module, self).__init__() + self.args = args + + # Forecast record related settings + if args.get('record_pred', False): + self.pred_dir = f"./result_process/preds/{self.args['exp_name']}" + if not os.path.exists(self.pred_dir): + os.makedirs(self.pred_dir) + self.pred_batch_idx = 0 + + # ============================================================ + # [Basic parameters] + # hidden_size: embedding dimension, affecting model capacity + # num_genes: number of genes, determines the output dimension + # ============================================================ + self.num_genes = args['num_genes'] + hidden_size = args['hidden_size'] + self.uncertainty = args['uncertainty'] + self.num_layers = args['num_go_gnn_layers'] # GO GNN layers + self.indv_out_hidden_size = args['decoder_hidden_size'] + self.num_layers_gene_pos = args['num_gene_gnn_layers'] + self.pert_emb_lambda = 0.2 + + self.input_genes_list = args['input_genes_ens_ids'] + self.scfm_genes_list = args['scfm_genes_ens_ids'] + + # ============================================================ + # [Load pre-trained model configuration] + # hyper_params contains model architecture parameters, which must be consistent with pre-training + # ============================================================ + try: + with open(args["scfm_hyper_params_path"], 'rb') as handle: + self.scfm_hyper_params = pickle.load(handle) + except ValueError: + import pickle5 + with open(args["scfm_hyper_params_path"], 'rb') as handle: + self.scfm_hyper_params = pickle5.load(handle) + self.scfm_hyper_params.pop('module_embed_chunk_padded_gene_budget', None) + self.scfm_hyper_params = sanitize_performer_module_config( + self.scfm_hyper_params, + source=args["scfm_hyper_params_path"], + log_fn=print, + ) + + self.scfm_hyper_params['gene2vec_file'] = args['scfm_gene2vec_file'] + self.scfm_ckpt = args["scfm_ckpt"] + + # ============================================================ + # [Module data loading] + # module_sets defines how genes are grouped into modules + # Can be loaded from hyper_params or external files + # ============================================================ + module_sets = None + if 'module_sets' in self.scfm_hyper_params and self.scfm_hyper_params['module_sets'] is not None: + module_sets = self.scfm_hyper_params['module_sets'] + elif args.get('module_data_path') or args.get('module_data_dir'): + from BioMod.module_dataset import load_module_data + module_sets = load_module_data( + module_data_path=args.get('module_data_path'), + module_data_dir=args.get('module_data_dir'), + gene_list_path=args.get('scfm_genes_list_path'), + gene_mapping_path=args.get('gene_mapping_path', str(Path(__file__).resolve().parents[2] / 'model_dependencies' / 'gene2ens.csv')), + verbose=True + ) + else: + raise ValueError("module_sets must be provided (via hyper_params or module_data_path/module_data_dir)") + + # ============================================================ + # [Ablation Experiment]shuffle_module_genes + # Disrupt the gene assignment within the module and verify the importance of the module structure (gene grouping) + # If performance decreases after scrambling, it means that the module structure contains useful biological information. + # ============================================================ + if args.get('shuffle_module_genes', False): + shuffle_seed = args.get('shuffle_seed', 2021) + print( + f"[GEARS_Model_Module] Applying downstream-only extra module reshuffle " + f"(seed={shuffle_seed}). Pretraining-time module_sets from PKL will be reshuffled again." + ) + all_gene_indices = [] + module_sizes = [] + for module in module_sets: + all_gene_indices.extend(module) + module_sizes.append(len(module)) + + import random + rng = random.Random(shuffle_seed) + rng.shuffle(all_gene_indices) + + shuffled_module_sets = [] + start_idx = 0 + for size in module_sizes: + shuffled_module_sets.append(all_gene_indices[start_idx:start_idx + size]) + start_idx += size + module_sets = shuffled_module_sets + print(f"[GEARS_Model_Module] Shuffled {len(all_gene_indices)} genes across {len(module_sets)} modules") + + # ============================================================ + # [Perturbation Embedding Layer] + # pert_w: Convert perturbation markers (0/1) to embedding vectors + # gene_emb: Basic embedding of genes (used when pre-trained models are not available) + # pert_emb: dedicated embedding of perturbed genes + # ============================================================ + self.pert_w = nn.Linear(1, hidden_size) + + # gene/global perturbation embedding dictionary lookup + self.gene_emb = nn.Embedding(self.num_genes, hidden_size, max_norm=True) + self.pert_emb = nn.Embedding(self.num_genes, hidden_size, max_norm=True) + + # transformation layer + self.emb_trans = nn.ReLU() + self.pert_base_trans = nn.ReLU() + self.transform = nn.ReLU() + self.pert_fuse = MLP([hidden_size, hidden_size, hidden_size], last_layer_act='ReLU') + + # ============================================================ + # [GO graph neural network] + # G_sim: edge index of GO similarity graph + # G_sim_weight: edge weight + # Using SGConv (Simplified Graph Convolution) for message passing + # GO diagram helps the model understand gene functional relationships, and perturbation effects propagate along the diagram. + # ============================================================ + self.G_sim = args['G_go'].to(args['device']) + self.G_sim_weight = args['G_go_weight'].to(args['device']) + + self.sim_layers = torch.nn.ModuleList() + for i in range(1, self.num_layers + 1): + self.sim_layers.append(SGConv(hidden_size, hidden_size, 1)) + + # ============================================================ + # [Decoder] + # recovery_w: shared MLP, projects embeddings into decoding space + # indv_w1/b1: Gene specific parameters, each gene has independent decoding weight + # cross_gene_state: cross-gene MLP, modeling inter-gene dependencies + # indv_w2/b2: final decoding layer + # ============================================================ + self.recovery_w = MLP([hidden_size, hidden_size*2, hidden_size], last_layer_act='linear') + + # gene specific decoder + self.indv_w1 = nn.Parameter(torch.rand(self.num_genes, hidden_size, 1)) + self.indv_b1 = nn.Parameter(torch.rand(self.num_genes, 1)) + self.act = nn.ReLU() + nn.init.xavier_normal_(self.indv_w1) + nn.init.xavier_normal_(self.indv_b1) + + # Cross gene MLP + self.cross_gene_state = MLP([self.num_genes, hidden_size, hidden_size]) + self.indv_w2 = nn.Parameter(torch.rand(1, self.num_genes, hidden_size+1)) + self.indv_b2 = nn.Parameter(torch.rand(1, self.num_genes)) + nn.init.xavier_normal_(self.indv_w2) + nn.init.xavier_normal_(self.indv_b2) + + # BatchNorm layers + self.bn_emb = nn.BatchNorm1d(hidden_size) + self.bn_pert_base = nn.BatchNorm1d(hidden_size) + self.bn_pert_base_trans = nn.BatchNorm1d(hidden_size) + + # uncertainty mode (optional) + if self.uncertainty: + self.uncertainty_w = MLP([hidden_size, hidden_size*2, hidden_size, 1], last_layer_act='linear') + + # ============================================================ + # [Core] Initialize pre-trained encoder (SCFM_Module) + # requires_grad: whether to fine-tune pre-training parameters + # - finetune_method='frozen': freeze pre-training parameters + # - Others: Full parameter fine-tuning + # ============================================================ + self.use_module_level = args.get('use_module_level', False) + self.singlecell_model = SCFM_Module( + self.input_genes_list, + self.scfm_genes_list, + self.scfm_hyper_params, + self.scfm_ckpt, + module_sets=module_sets, + requires_grad=args.get('finetune_method') != 'frozen', + device=args['device'], + output_dim=hidden_size, + use_module_level=self.use_module_level + ) + self.pretrained = True + + # ============================================================ + # [Experimental feature] module level features + # In the current implementation, module-level downstream tasks are not fully implemented. + # GNN and decoder need to be redesigned to handle module dimensions + # ============================================================ + if self.use_module_level: + self.num_modules = len(module_sets) + import warnings + warnings.warn( + f"Using module-level features. This requires compatible downstream architecture. " + f"Number of modules: {self.num_modules}, Number of genes: {self.num_genes}. " + f"Currently module-level downstream task is not fully implemented.", + UserWarning + ) + + def forward(self, data): + """ + Forward propagation: predicting expression values after gene perturbation + + [enter] + data: PyG Data object + - x: (B*N, 2) characteristic matrix + [:, 0]: gene expression value (control group) + [:, 1]: perturbation flag (1 indicates that the gene is perturbed) + - y: real expression value (used during training) + - pert: disturbance type identifier + + [Output] + out: (B, N) predicted gene expression value after perturbation + + [Processing Process] + 1. Obtain base gene embeddings (from pretrained encoder or random embeddings) + 2. Generate perturbation embedding (enhanced by GO-GNN) + 3. Merge base embedding and perturbation embedding + 4. Predict the expression value through the decoder + """ + if str(type(data)) == "": + x = data.reshape((-1, 2)) + else: + x, batch = data.x, data.batch + x_pert = x[:, 1].reshape((-1, self.num_genes)) # Disturbance mark + num_graphs = len(data.batch.unique()) + + # ============================================================ + # [Step 1] Obtain basic gene embedding + # If using a pretrained model, get embeddings from gene expression encoding + # Otherwise use randomly initialized gene embeddings + # ============================================================ + if self.pretrained: + exp = x[:, 0].reshape((-1, self.num_genes)) # B, N - gene expression value + emb = self.singlecell_model(exp) # B, N, D - by pre-trained encoder + emb = emb.reshape((num_graphs * self.num_genes, -1)) + else: + emb = self.gene_emb(torch.LongTensor(list(range(self.num_genes))).repeat(num_graphs, ).to(self.args['device'])) + emb = self.bn_emb(emb) + base_emb = self.emb_trans(emb) + + base_emb = base_emb.reshape(num_graphs, self.num_genes, -1) + + pert = x_pert.reshape(-1, 1) # B * N, 1 + + # ============================================================ + # [Step 2] Generate perturbation embedding (GO-GNN enhancement) + # Perturbation embeddings are propagated through GO similarity graphs to capture gene functional relationships + # This enables the model to exploit a priori knowledge of gene function + # ============================================================ + pert_embeddings = self.pert_emb(torch.LongTensor(list(range(self.num_genes))).to(self.args['device'])) # N, D + + # Propagating perturbation embeddings via GO-GNN + for idx, layer in enumerate(self.sim_layers): + pert_embeddings = layer(pert_embeddings, self.G_sim, self.G_sim_weight) + if idx < self.num_layers - 1: + pert_embeddings = pert_embeddings.relu() + + # After GNN: N, D + pert_embeddings = torch.stack(list((pert_embeddings,)) * num_graphs) # B, N, D + pert_embeddings = pert_embeddings * x_pert.reshape((num_graphs, self.num_genes, 1)) # Only perturbed genes are retained + pert_embeddings = pert_embeddings.sum(1) # B, D - aggregate all perturbed genes + pert_mask = (x_pert.sum(1) != 0).reshape((num_graphs, 1)) # B, 1 + pert_embeddings = self.pert_fuse(pert_embeddings) * pert_mask # B, D + pert_embeddings = pert_embeddings.reshape((num_graphs, 1, pert_embeddings.shape[1])) # B, 1, D + + # ============================================================ + # [Step 3] Merge base embedding and perturbation embedding + # Perturbation information is added to the embedding of each gene + # ============================================================ + base_emb = base_emb + pert_embeddings + + base_emb = base_emb.reshape(num_graphs * self.num_genes, -1) + + # Add perturbed position embedding + pert_emb = self.pert_w(pert) + combined = pert_emb + base_emb + combined = self.bn_pert_base_trans(combined) + base_emb = self.pert_base_trans(combined) + base_emb = self.bn_pert_base(base_emb) + + # ============================================================ + # [Step 4] Decoder: predict the expression value after perturbation + # Using gene-specific decoders + cross-gene MLP + # ============================================================ + base_emb = self.transform(base_emb) + out = self.recovery_w(base_emb) # B * N, D + out = out.reshape(num_graphs, self.num_genes, -1) # B, N, D + out = out.unsqueeze(-1) * self.indv_w1 # B, N, D, 1 - Gene specific transformation + w = torch.sum(out, axis=2) # B, N, 1 + out = w + self.indv_b1 # B, N, 1 + + # Cross gene MLP: Modeling inter-gene dependencies + cross_gene_embed = self.cross_gene_state(out.reshape(num_graphs, self.num_genes, -1).squeeze(2)) # B, D + cross_gene_embed = cross_gene_embed.repeat(1, self.num_genes) # B, N*D + + cross_gene_embed = cross_gene_embed.reshape([num_graphs, self.num_genes, -1]) # B, N, D + cross_gene_out = torch.cat([out, cross_gene_embed], 2) # B, N, D+1 + + cross_gene_out = cross_gene_out * self.indv_w2 # B, N, D+1 + cross_gene_out = torch.sum(cross_gene_out, axis=2) # B, N + out = cross_gene_out + self.indv_b2 # B, N + + # ============================================================ + # [Key] Residual connection: predicted value = change + original value + # This is the core design of GEARS. The model predicts expression changes rather than absolute values. + # ============================================================ + out = out.reshape(num_graphs * self.num_genes, -1) + x[:, 0].reshape(-1, 1) # B * N, 1 + + # ============================================================ + # [Prediction record] for subsequent analysis + # ============================================================ + if self.args.get('start_record', False): + y_flat = data.y.reshape(-1, 1) # B * N, 1 + pred = torch.cat((pert, out, y_flat), dim=1) # B * N, 3 + pred = pred.cpu().to_sparse() + + values = pred.values().numpy() + indices = pred.indices().numpy() + sparse_matrix = scipy.sparse.coo_matrix((values, indices), shape=pred.shape) + + scipy.sparse.save_npz(f'{self.pred_dir}/b{self.pred_batch_idx}_{num_graphs}.npz', sparse_matrix) + self.pred_batch_idx += 1 + + # ============================================================ + # [Key] reshape output is (B, N) format + # In this way, the loss function can correctly index the predicted value and the true value. + # ============================================================ + out = torch.split(torch.flatten(out), self.num_genes) + out = torch.stack(out) # (B, N) + + # uncertainty mode (optional) + if self.uncertainty: + logvar = self.uncertainty_w(base_emb) + logvar = torch.split(torch.flatten(logvar), self.num_genes) + return out, torch.stack(logvar) + + return out # (B, N) + diff --git a/BioMod/gene_perturbation/pertdata_local.py b/BioMod/gene_perturbation/pertdata_local.py index 0aa82ac0bdd289db4040c3ef475c9a4945a12cc2..a79db50ece4225a41c421b35457ce0e078e39bdc 100644 --- a/BioMod/gene_perturbation/pertdata_local.py +++ b/BioMod/gene_perturbation/pertdata_local.py @@ -1,524 +1,524 @@ -""" -================================================================================ -Perturbation data processing module (pertdata_local.py) -================================================================================ - -[Module function] -Load and preprocess the genetic perturbation dataset, converting the raw data into a model-usable PyG graph format. - -[Supported data sets] -- norman: Norman et al. 2019, Large-scale CRISPR perturbation data -- adamson: Adamson et al. 2016, single gene perturbation data -- dixit: Dixit et al. 2016, single gene perturbation data -- replogle_k562_essential/replogle_rpe1_essential: Replogle et al. 2022 - -[Data format] -Each cell is represented as a graph data object: -- x: (num_genes, 2) feature matrix, [:, 0] is the expression value, [:, 1] is the perturbation mark -- y: target expression value -- de_idx: differentially expressed gene index -- pert: disturbance type identifier - -[Key steps] -1. Load the original h5ad file -2. Calculate differentially expressed genes -3. Divide training/validation/test sets -4. Create PyG graph data objects -================================================================================ -""" - -from torch_geometric.data import Data -import torch -import numpy as np -import pickle -from torch_geometric.data import DataLoader -import os -import scanpy as sc -import networkx as nx -from tqdm import tqdm -import scipy.sparse as sp - -import warnings -warnings.filterwarnings("ignore") -sc.settings.verbosity = 0 - -# ============================================================ -# Use local modules and avoid external gears_reduce imports -# ============================================================ -from utils import print_sys, zip_data_download_wrapper, filter_pert_cond_in_go -from data_utils_local import get_DE_genes, get_dropout_non_zero_genes, DataSplitter - -class PertData: - - def __init__(self, data_path): - """ - Initialize the perturbation data processing class - - Args: - data_path: data storage directory path - """ - self.data_path = data_path - self.gene_names = None - if not os.path.exists(self.data_path): - os.mkdir(self.data_path) - - def load(self, data_name = None, - data_path = None, exclude_pert = set()): - """ - Load perturbation dataset - - [Data source] - - Download public data sets such as norman/adamson/dixit from Harvard Dataverse - - You can also provide the local h5ad file path - - [Processing Process] - 1. Download or load data - 2. Filter disturbances that are not in the GO graph - 3. Create PyG graph data objects - """ - if data_name in ['norman', 'adamson', 'dixit', - 'replogle_k562_essential', - 'replogle_rpe1_essential']: - # Download public datasets from Harvard Dataverse - if data_name == 'norman': - url = 'https://dataverse.harvard.edu/api/access/datafile/6154020' - elif data_name == 'adamson': - url = 'https://dataverse.harvard.edu/api/access/datafile/6154417' - elif data_name == 'dixit': - url = 'https://dataverse.harvard.edu/api/access/datafile/6154416' - elif data_name == 'replogle_k562_essential': - url = 'https://dataverse.harvard.edu/api/access/datafile/7458695' - elif data_name == 'replogle_rpe1_essential': - url = 'https://dataverse.harvard.edu/api/access/datafile/7458694' - data_path = os.path.join(self.data_path, data_name) - zip_data_download_wrapper(url, data_path, self.data_path) - self.dataset_name = data_path.split('/')[-1] - self.dataset_path = data_path - adata_path = os.path.join(data_path, 'perturb_processed.h5ad') - self.adata = sc.read_h5ad(adata_path) - - elif os.path.exists(data_path): - # Load local data - adata_path = os.path.join(data_path, 'perturb_processed.h5ad') - self.adata = sc.read_h5ad(adata_path) - self.dataset_name = data_path.split('/')[-1] - self.dataset_path = data_path - else: - raise ValueError("data is either Norman/Adamson/Dixit or a path to an h5ad file") - - # Filter perturbed genes that are not in the GO graph - print_sys('These perturbations are not in the GO graph and their ' - 'perturbation can thus not be predicted') - print_sys(exclude_pert) - if len(exclude_pert) > 0: - filter_go = self.adata.obs[self.adata.obs.condition.apply( - lambda x: filter_pert_cond_in_go(x, exclude_pert))] - self.adata = self.adata[filter_go.index.values, :] - - # ============================================================ - # [Key] Create or load PyG graph data objects - # This is the core step of data preprocessing, converting each cell into graph format - # ============================================================ - pyg_path = os.path.join(data_path, 'data_pyg') - if not os.path.exists(pyg_path): - os.mkdir(pyg_path) - dataset_fname = os.path.join(pyg_path, 'cell_graphs.pkl') - - if os.path.isfile(dataset_fname): - # Already processed data can be loaded directly. - print_sys("Local copy of pyg dataset is detected. Loading...") - self.dataset_processed = pickle.load(open(dataset_fname, "rb")) - print_sys("Done!") - else: - # Data needs to be reprocessed - self.ctrl_adata = self.adata[self.adata.obs['condition'] == 'ctrl'] - self.gene_names = list(self.adata.var.gene_name) - conditions = list(self.adata.obs.condition.unique()) - pert_genes = [] - for condition in conditions: - if condition == 'ctrl': - continue - # Analyze perturbation combinations (format: gene1+gene2) - cond1 = condition.split('+')[0] - cond2 = condition.split('+')[1] - if cond1 != 'ctrl': - if cond1 not in pert_genes: - pert_genes.append(cond1) - if cond1 not in self.gene_names: - self.gene_names.append(cond1) - if cond2 != 'ctrl': - if cond2 not in pert_genes: - pert_genes.append(cond2) - if cond2 not in self.gene_names: - self.gene_names.append(cond2) - self.gene_names = np.array(self.gene_names) - self.real_gene_names = np.array(list(self.adata.var.gene_name)) - print_sys(f"len(self.gene_names): {len(self.gene_names)}") - print_sys(f"len(self.real_gene_names): {len(self.real_gene_names)}") - print_sys("Creating pyg object for each cell in the data...") - self.dataset_processed = self.create_dataset_file() - print_sys("Saving new dataset pyg object at " + dataset_fname) - pickle.dump(self.dataset_processed, open(dataset_fname, "wb")) - print_sys("Done!") - - def new_data_process(self, dataset_name, - adata = None): - - if 'condition' not in adata.obs.columns.values: - raise ValueError("Please specify condition") - if 'gene_name' not in adata.var.columns.values: - raise ValueError("Please specify gene name") - if 'cell_type' not in adata.obs.columns.values: - raise ValueError("Please specify cell type") - - dataset_name = dataset_name.lower() - self.dataset_name = dataset_name - save_data_folder = os.path.join(self.data_path, dataset_name) - - if not os.path.exists(save_data_folder): - os.mkdir(save_data_folder) - self.dataset_path = save_data_folder - self.adata = get_DE_genes(adata) - self.adata = get_dropout_non_zero_genes(self.adata) - self.adata.write_h5ad(os.path.join(save_data_folder, 'perturb_processed.h5ad')) - - self.ctrl_adata = self.adata[self.adata.obs['condition'] == 'ctrl'] - - self.gene_names = list(self.adata.var.gene_name) - conditions = list(self.adata.obs.condition.unique()) - pert_genes = [] - for condition in conditions: - if condition == 'ctrl': - continue - cond1 = condition.split('+')[0] - cond2 = condition.split('+')[1] - if cond1 != 'ctrl': - if cond1 not in pert_genes: - pert_genes.append(cond1) - if cond1 not in self.gene_names: - self.gene_names.append(cond1) - if cond2 != 'ctrl': - if cond2 not in pert_genes: - pert_genes.append(cond2) - if cond2 not in self.gene_names: - self.gene_names.append(cond2) - self.gene_names = np.array(self.gene_names) - self.real_gene_names = np.array(list(self.adata.var.gene_name)) - - - pyg_path = os.path.join(save_data_folder, 'data_pyg') - if not os.path.exists(pyg_path): - os.mkdir(pyg_path) - dataset_fname = os.path.join(pyg_path, 'cell_graphs.pkl') - print_sys("Creating pyg object for each cell in the data...") - self.dataset_processed = self.create_dataset_file() - print_sys("Saving new dataset pyg object at " + dataset_fname) - pickle.dump(self.dataset_processed, open(dataset_fname, "wb")) - print_sys("Done!") - - def prepare_split(self, split = 'simulation', - seed = 1, - train_gene_set_size = 0.75, - combo_seen2_train_frac = 0.75, - combo_single_split_test_set_fraction = 0.1, - test_perts = None, - only_test_set_perts = False, - test_pert_genes = None): - """ - Prepare data partitioning - - [Description of partition strategy] - - 'simulation': simulate the division of real application scenarios - The test set contains: unseen_single (unseen single gene perturbation), - combo_seen0/1/2 (combination perturbations with different familiarity levels) - This is the most commonly used division method to evaluate the generalization ability of the model. - - 'combo_seen0/1/2': divided according to the number of genes seen in the combination - - 'single': only evaluate single gene perturbations - - 'no_test': Do not divide the test set - - [Key parameters] - - train_gene_set_size: training set gene proportion (0.75 means 75% of genes are used for training) - - seed: random seed to ensure the experiment is reproducible - """ - available_splits = ['simulation', 'simulation_single', 'combo_seen0', 'combo_seen1', - 'combo_seen2', 'single', 'no_test', 'no_split'] - if split not in available_splits: - raise ValueError('currently, we only support ' + ','.join(available_splits)) - self.split = split - self.seed = seed - self.subgroup = None - self.train_gene_set_size = train_gene_set_size - - # Divide file path - split_folder = os.path.join(self.dataset_path, 'splits') - if not os.path.exists(split_folder): - os.mkdir(split_folder) - split_file = self.dataset_name + '_' + split + '_' + str(seed) + '_' + str(train_gene_set_size) + '.pkl' - split_path = os.path.join(split_folder, split_file) - - if test_perts: - split_path = split_path[:-4] + '_' + test_perts + '.pkl' - - if os.path.exists(split_path): - # Already divided, load directly - print_sys("Local copy of split is detected. Loading...") - set2conditions = pickle.load(open(split_path, "rb")) - if split == 'simulation': - subgroup_path = split_path[:-4] + '_subgroup.pkl' - subgroup = pickle.load(open(subgroup_path, "rb")) - self.subgroup = subgroup - else: - # Create new partition - print_sys("Creating new splits....") - if test_perts: - test_perts = test_perts.split('_') - - if split in ['simulation', 'simulation_single']: - DS = DataSplitter(self.adata, split_type=split) - - adata, subgroup = DS.split_data(train_gene_set_size = train_gene_set_size, - combo_seen2_train_frac = combo_seen2_train_frac, - seed=seed, - test_perts = test_perts, - only_test_set_perts = only_test_set_perts - ) - subgroup_path = split_path[:-4] + '_subgroup.pkl' - pickle.dump(subgroup, open(subgroup_path, "wb")) - self.subgroup = subgroup - - elif split[:5] == 'combo': - split_type = 'combo' - seen = int(split[-1]) - - if test_pert_genes: - test_pert_genes = test_pert_genes.split('_') - - DS = DataSplitter(self.adata, split_type=split_type, seen=int(seen)) - adata = DS.split_data(test_size=combo_single_split_test_set_fraction, - test_perts=test_perts, - test_pert_genes=test_pert_genes, - seed=seed) - - elif split == 'single': - DS = DataSplitter(self.adata, split_type=split) - adata = DS.split_data(test_size=combo_single_split_test_set_fraction, seed=seed) - - elif split == 'no_test': - DS = DataSplitter(self.adata, split_type=split) - adata = DS.split_data(test_size=combo_single_split_test_set_fraction, seed=seed) - - elif split == 'no_split': - adata = self.adata - adata.obs['split'] = 'test' - - # Save partition results - set2conditions = dict(adata.obs.groupby('split').agg({'condition': lambda x: x}).condition) - set2conditions = {i: j.unique().tolist() for i,j in set2conditions.items()} - pickle.dump(set2conditions, open(split_path, "wb")) - print_sys("Saving new splits at " + split_path) - - self.set2conditions = set2conditions - - if split == 'simulation': - print_sys('Simulation split test composition:') - for i,j in subgroup['test_subgroup'].items(): - print_sys(i + ':' + str(len(j))) - print_sys("Done!") - - def get_dataloader(self, batch_size, test_batch_size = None): - """ - Create data loader - - [Data loader] - Encapsulate PyG graph data objects as DataLoader, supporting batch processing and shuffling. - - [Key settings] - - Training set: shuffle=True, drop_last=True (make sure each batch size is consistent) - - Validation/test set: shuffle=False (maintain order for easy analysis) - """ - if test_batch_size is None: - test_batch_size = batch_size - if self.gene_names is None: - self.gene_names = list(self.adata.var.gene_name) - conditions = list(self.adata.obs.condition.unique()) - pert_genes = [] - for condition in conditions: - if condition == 'ctrl': - continue - cond1 = condition.split('+')[0] - cond2 = condition.split('+')[1] - if cond1 != 'ctrl': - if cond1 not in pert_genes: - pert_genes.append(cond1) - if cond1 not in self.gene_names: - self.gene_names.append(cond1) - if cond2 != 'ctrl': - if cond2 not in pert_genes: - pert_genes.append(cond2) - if cond2 not in self.gene_names: - self.gene_names.append(cond2) - self.gene_names = np.array(self.gene_names) - self.real_gene_names = np.array(list(self.adata.var.gene_name)) - - # Gene name to index mapping - self.node_map = {x: it for it, x in enumerate(self.gene_names)} - if (self.gene_names[: len(self.real_gene_names)] != self.real_gene_names).any(): - raise ValueError("self.real_gene_names must be in the start of self.gene_names") - - # Create cytogram data - cell_graphs = {} - if self.split == 'no_split': - i = 'test' - cell_graphs[i] = [] - for p in self.set2conditions[i]: - if p != 'ctrl': - cell_graphs[i].extend(self.dataset_processed[p]) - - print_sys("Creating dataloaders....") - test_loader = DataLoader(cell_graphs['test'], - batch_size=batch_size, shuffle=False) - - print_sys("Dataloaders created...") - return {'test_loader': test_loader} - else: - if self.split =='no_test': - splits = ['train','val'] - else: - splits = ['train','val','test'] - for i in splits: - cell_graphs[i] = [] - for p in self.set2conditions[i]: - cell_graphs[i].extend(self.dataset_processed[p]) - - print_sys("Creating dataloaders....") - - # Set up data loader - train_loader = DataLoader(cell_graphs['train'], - batch_size=batch_size, shuffle=True, drop_last = True) - val_loader = DataLoader(cell_graphs['val'], - batch_size=batch_size, shuffle=True) - - if self.split !='no_test': - test_loader = DataLoader(cell_graphs['test'], - batch_size=batch_size, shuffle=False) - self.dataloader = {'train_loader': train_loader, - 'val_loader': val_loader, - 'test_loader': test_loader} - - else: - self.dataloader = {'train_loader': train_loader, - 'val_loader': val_loader} - print_sys("Done!") - del self.dataset_processed # Clean memory - - - def create_dataset_file(self): - dl = {} - for p in tqdm(self.adata.obs['condition'].unique()): - cell_graph_dataset = self.create_cell_graph_dataset(self.adata, p, num_samples=1) - dl[p] = cell_graph_dataset - return dl - - def get_pert_idx(self, pert_category, adata_): - pert_idx = [np.where(p == self.gene_names)[0][0] - for p in pert_category.split('+') - if p != 'ctrl'] - return pert_idx - - # Set up feature matrix and output - def create_cell_graph(self, X, y, de_idx, pert, pert_idx=None): - """ - Create a graph data object of a single cell - - [enter] - - X: Control group gene expression value (expression before perturbation) - - y: gene expression value after perturbation (target value) - - de_idx: index of differentially expressed genes - - pert: disturbance type identifier - - pert_idx: index list of perturbed genes - - [Output] - PyG Data object: - - x: (num_genes, 2) feature matrix - [:, 0]: Control group expression value - [:, 1]: perturbation flag (1=the gene is perturbed) - - y: target expression value - - de_idx: differentially expressed gene index - - pert: perturbation type - """ - # Create perturbation feature vectors (mark which genes are perturbed) - pert_feats = np.zeros(len(X[0])) - if pert_idx is not None: - for p in pert_idx: - pert_feats[int(np.abs(p))] = 1 - pert_feats = np.expand_dims(pert_feats, 0) - # Spliced expression values and perturbation markers - feature_mat = torch.Tensor(np.concatenate([X, pert_feats])).T - - return Data(x=feature_mat, edge_index=None, edge_attr=None, - y=torch.Tensor(y), de_idx=de_idx, pert=pert) - - def create_cell_graph_dataset(self, split_adata, pert_category, - num_samples=1): - """ - Create a graph dataset of perturbation categories - - [Processing logic] - 1. For non-control group disturbances: - - Get perturbed gene index - - Expression before perturbation = randomly sample control cells - - Target = expression of cells after perturbation - 2. For the control group: - - Expression before perturbation = target = expression in control cells - - [num_samples parameter] - Each perturbed cell can be paired with multiple control group cells, - Increase the diversity of training data - """ - num_de_genes = 20 # Only focus on the top 20 differentially expressed genes for evaluation - adata_ = split_adata[split_adata.obs['condition'] == pert_category] - de_genes = adata_.uns['rank_genes_groups_cov_all'] - Xs = [] - ys = [] - - # Handling non-control group disturbances - if pert_category != 'ctrl': - # Get the index of the perturbed gene - pert_idx = self.get_pert_idx(pert_category, adata_) - - # Get differentially expressed gene index - pert_de_category = adata_.obs['condition_name'][0] - de_idx = np.where(adata_.var_names.isin( - np.array(de_genes[pert_de_category][:num_de_genes])))[0] - - # Expand feature matrix to include perturbed genes - padded_X = sp.lil_matrix((adata_.X.shape[0], len(self.gene_names)), dtype=adata_.X.dtype) - padded_X[:, :len(self.real_gene_names)] = adata_.X - - for cell_z in padded_X: - # Use control group samples as expression before perturbation - ctrl_samples = self.ctrl_adata[np.random.randint(0, - len(self.ctrl_adata), num_samples), :] - padded_ctrl_X = sp.lil_matrix((ctrl_samples.X.shape[0], len(self.gene_names)), dtype=ctrl_samples.X.dtype) - padded_ctrl_X[:, :len(self.real_gene_names)] = ctrl_samples.X - for c in padded_ctrl_X: - Xs.append(c) # control group expression - ys.append(cell_z) # expression after perturbation - - # Process control group - else: - pert_idx = None - de_idx = [-1] * num_de_genes - padded_X = sp.lil_matrix((adata_.X.shape[0], len(self.gene_names)), dtype=adata_.X.dtype) - padded_X[:, :len(self.real_gene_names)] = adata_.X - for cell_z in padded_X: - Xs.append(cell_z) - ys.append(cell_z) - - # Create a cell diagram data object - cell_graphs = [] - for X, y in zip(Xs, ys): - cell_graphs.append(self.create_cell_graph(X.toarray(), - y.toarray(), de_idx, pert_category, pert_idx)) - +""" +================================================================================ +Perturbation data processing module (pertdata_local.py) +================================================================================ + +[Module function] +Load and preprocess the genetic perturbation dataset, converting the raw data into a model-usable PyG graph format. + +[Supported data sets] +- norman: Norman et al. 2019, Large-scale CRISPR perturbation data +- adamson: Adamson et al. 2016, single gene perturbation data +- dixit: Dixit et al. 2016, single gene perturbation data +- replogle_k562_essential/replogle_rpe1_essential: Replogle et al. 2022 + +[Data format] +Each cell is represented as a graph data object: +- x: (num_genes, 2) feature matrix, [:, 0] is the expression value, [:, 1] is the perturbation mark +- y: target expression value +- de_idx: differentially expressed gene index +- pert: disturbance type identifier + +[Key steps] +1. Load the original h5ad file +2. Calculate differentially expressed genes +3. Divide training/validation/test sets +4. Create PyG graph data objects +================================================================================ +""" + +from torch_geometric.data import Data +import torch +import numpy as np +import pickle +from torch_geometric.data import DataLoader +import os +import scanpy as sc +import networkx as nx +from tqdm import tqdm +import scipy.sparse as sp + +import warnings +warnings.filterwarnings("ignore") +sc.settings.verbosity = 0 + +# ============================================================ +# Use local modules and avoid external gears_reduce imports +# ============================================================ +from utils import print_sys, zip_data_download_wrapper, filter_pert_cond_in_go +from data_utils_local import get_DE_genes, get_dropout_non_zero_genes, DataSplitter + +class PertData: + + def __init__(self, data_path): + """ + Initialize the perturbation data processing class + + Args: + data_path: data storage directory path + """ + self.data_path = data_path + self.gene_names = None + if not os.path.exists(self.data_path): + os.mkdir(self.data_path) + + def load(self, data_name = None, + data_path = None, exclude_pert = set()): + """ + Load perturbation dataset + + [Data source] + - Download public data sets such as norman/adamson/dixit from Harvard Dataverse + - You can also provide the local h5ad file path + + [Processing Process] + 1. Download or load data + 2. Filter disturbances that are not in the GO graph + 3. Create PyG graph data objects + """ + if data_name in ['norman', 'adamson', 'dixit', + 'replogle_k562_essential', + 'replogle_rpe1_essential']: + # Download public datasets from Harvard Dataverse + if data_name == 'norman': + url = 'https://dataverse.harvard.edu/api/access/datafile/6154020' + elif data_name == 'adamson': + url = 'https://dataverse.harvard.edu/api/access/datafile/6154417' + elif data_name == 'dixit': + url = 'https://dataverse.harvard.edu/api/access/datafile/6154416' + elif data_name == 'replogle_k562_essential': + url = 'https://dataverse.harvard.edu/api/access/datafile/7458695' + elif data_name == 'replogle_rpe1_essential': + url = 'https://dataverse.harvard.edu/api/access/datafile/7458694' + data_path = os.path.join(self.data_path, data_name) + zip_data_download_wrapper(url, data_path, self.data_path) + self.dataset_name = os.path.basename(os.path.normpath(data_path)) + self.dataset_path = data_path + adata_path = os.path.join(data_path, 'perturb_processed.h5ad') + self.adata = sc.read_h5ad(adata_path) + + elif os.path.exists(data_path): + # Load local data + adata_path = os.path.join(data_path, 'perturb_processed.h5ad') + self.adata = sc.read_h5ad(adata_path) + self.dataset_name = os.path.basename(os.path.normpath(data_path)) + self.dataset_path = data_path + else: + raise ValueError("data is either Norman/Adamson/Dixit or a path to an h5ad file") + + # Filter perturbed genes that are not in the GO graph + print_sys('These perturbations are not in the GO graph and their ' + 'perturbation can thus not be predicted') + print_sys(exclude_pert) + if len(exclude_pert) > 0: + filter_go = self.adata.obs[self.adata.obs.condition.apply( + lambda x: filter_pert_cond_in_go(x, exclude_pert))] + self.adata = self.adata[filter_go.index.values, :] + + # ============================================================ + # [Key] Create or load PyG graph data objects + # This is the core step of data preprocessing, converting each cell into graph format + # ============================================================ + pyg_path = os.path.join(data_path, 'data_pyg') + if not os.path.exists(pyg_path): + os.mkdir(pyg_path) + dataset_fname = os.path.join(pyg_path, 'cell_graphs.pkl') + + if os.path.isfile(dataset_fname): + # Already processed data can be loaded directly. + print_sys("Local copy of pyg dataset is detected. Loading...") + self.dataset_processed = pickle.load(open(dataset_fname, "rb")) + print_sys("Done!") + else: + # Data needs to be reprocessed + self.ctrl_adata = self.adata[self.adata.obs['condition'] == 'ctrl'] + self.gene_names = list(self.adata.var.gene_name) + conditions = list(self.adata.obs.condition.unique()) + pert_genes = [] + for condition in conditions: + if condition == 'ctrl': + continue + # Analyze perturbation combinations (format: gene1+gene2) + cond1 = condition.split('+')[0] + cond2 = condition.split('+')[1] + if cond1 != 'ctrl': + if cond1 not in pert_genes: + pert_genes.append(cond1) + if cond1 not in self.gene_names: + self.gene_names.append(cond1) + if cond2 != 'ctrl': + if cond2 not in pert_genes: + pert_genes.append(cond2) + if cond2 not in self.gene_names: + self.gene_names.append(cond2) + self.gene_names = np.array(self.gene_names) + self.real_gene_names = np.array(list(self.adata.var.gene_name)) + print_sys(f"len(self.gene_names): {len(self.gene_names)}") + print_sys(f"len(self.real_gene_names): {len(self.real_gene_names)}") + print_sys("Creating pyg object for each cell in the data...") + self.dataset_processed = self.create_dataset_file() + print_sys("Saving new dataset pyg object at " + dataset_fname) + pickle.dump(self.dataset_processed, open(dataset_fname, "wb")) + print_sys("Done!") + + def new_data_process(self, dataset_name, + adata = None): + + if 'condition' not in adata.obs.columns.values: + raise ValueError("Please specify condition") + if 'gene_name' not in adata.var.columns.values: + raise ValueError("Please specify gene name") + if 'cell_type' not in adata.obs.columns.values: + raise ValueError("Please specify cell type") + + dataset_name = dataset_name.lower() + self.dataset_name = dataset_name + save_data_folder = os.path.join(self.data_path, dataset_name) + + if not os.path.exists(save_data_folder): + os.mkdir(save_data_folder) + self.dataset_path = save_data_folder + self.adata = get_DE_genes(adata) + self.adata = get_dropout_non_zero_genes(self.adata) + self.adata.write_h5ad(os.path.join(save_data_folder, 'perturb_processed.h5ad')) + + self.ctrl_adata = self.adata[self.adata.obs['condition'] == 'ctrl'] + + self.gene_names = list(self.adata.var.gene_name) + conditions = list(self.adata.obs.condition.unique()) + pert_genes = [] + for condition in conditions: + if condition == 'ctrl': + continue + cond1 = condition.split('+')[0] + cond2 = condition.split('+')[1] + if cond1 != 'ctrl': + if cond1 not in pert_genes: + pert_genes.append(cond1) + if cond1 not in self.gene_names: + self.gene_names.append(cond1) + if cond2 != 'ctrl': + if cond2 not in pert_genes: + pert_genes.append(cond2) + if cond2 not in self.gene_names: + self.gene_names.append(cond2) + self.gene_names = np.array(self.gene_names) + self.real_gene_names = np.array(list(self.adata.var.gene_name)) + + + pyg_path = os.path.join(save_data_folder, 'data_pyg') + if not os.path.exists(pyg_path): + os.mkdir(pyg_path) + dataset_fname = os.path.join(pyg_path, 'cell_graphs.pkl') + print_sys("Creating pyg object for each cell in the data...") + self.dataset_processed = self.create_dataset_file() + print_sys("Saving new dataset pyg object at " + dataset_fname) + pickle.dump(self.dataset_processed, open(dataset_fname, "wb")) + print_sys("Done!") + + def prepare_split(self, split = 'simulation', + seed = 1, + train_gene_set_size = 0.75, + combo_seen2_train_frac = 0.75, + combo_single_split_test_set_fraction = 0.1, + test_perts = None, + only_test_set_perts = False, + test_pert_genes = None): + """ + Prepare data partitioning + + [Description of partition strategy] + - 'simulation': simulate the division of real application scenarios + The test set contains: unseen_single (unseen single gene perturbation), + combo_seen0/1/2 (combination perturbations with different familiarity levels) + This is the most commonly used division method to evaluate the generalization ability of the model. + - 'combo_seen0/1/2': divided according to the number of genes seen in the combination + - 'single': only evaluate single gene perturbations + - 'no_test': Do not divide the test set + + [Key parameters] + - train_gene_set_size: training set gene proportion (0.75 means 75% of genes are used for training) + - seed: random seed to ensure the experiment is reproducible + """ + available_splits = ['simulation', 'simulation_single', 'combo_seen0', 'combo_seen1', + 'combo_seen2', 'single', 'no_test', 'no_split'] + if split not in available_splits: + raise ValueError('currently, we only support ' + ','.join(available_splits)) + self.split = split + self.seed = seed + self.subgroup = None + self.train_gene_set_size = train_gene_set_size + + # Divide file path + split_folder = os.path.join(self.dataset_path, 'splits') + if not os.path.exists(split_folder): + os.mkdir(split_folder) + split_file = self.dataset_name + '_' + split + '_' + str(seed) + '_' + str(train_gene_set_size) + '.pkl' + split_path = os.path.join(split_folder, split_file) + + if test_perts: + split_path = split_path[:-4] + '_' + test_perts + '.pkl' + + if os.path.exists(split_path): + # Already divided, load directly + print_sys("Local copy of split is detected. Loading...") + set2conditions = pickle.load(open(split_path, "rb")) + if split == 'simulation': + subgroup_path = split_path[:-4] + '_subgroup.pkl' + subgroup = pickle.load(open(subgroup_path, "rb")) + self.subgroup = subgroup + else: + # Create new partition + print_sys("Creating new splits....") + if test_perts: + test_perts = test_perts.split('_') + + if split in ['simulation', 'simulation_single']: + DS = DataSplitter(self.adata, split_type=split) + + adata, subgroup = DS.split_data(train_gene_set_size = train_gene_set_size, + combo_seen2_train_frac = combo_seen2_train_frac, + seed=seed, + test_perts = test_perts, + only_test_set_perts = only_test_set_perts + ) + subgroup_path = split_path[:-4] + '_subgroup.pkl' + pickle.dump(subgroup, open(subgroup_path, "wb")) + self.subgroup = subgroup + + elif split[:5] == 'combo': + split_type = 'combo' + seen = int(split[-1]) + + if test_pert_genes: + test_pert_genes = test_pert_genes.split('_') + + DS = DataSplitter(self.adata, split_type=split_type, seen=int(seen)) + adata = DS.split_data(test_size=combo_single_split_test_set_fraction, + test_perts=test_perts, + test_pert_genes=test_pert_genes, + seed=seed) + + elif split == 'single': + DS = DataSplitter(self.adata, split_type=split) + adata = DS.split_data(test_size=combo_single_split_test_set_fraction, seed=seed) + + elif split == 'no_test': + DS = DataSplitter(self.adata, split_type=split) + adata = DS.split_data(test_size=combo_single_split_test_set_fraction, seed=seed) + + elif split == 'no_split': + adata = self.adata + adata.obs['split'] = 'test' + + # Save partition results + set2conditions = dict(adata.obs.groupby('split').agg({'condition': lambda x: x}).condition) + set2conditions = {i: j.unique().tolist() for i,j in set2conditions.items()} + pickle.dump(set2conditions, open(split_path, "wb")) + print_sys("Saving new splits at " + split_path) + + self.set2conditions = set2conditions + + if split == 'simulation': + print_sys('Simulation split test composition:') + for i,j in subgroup['test_subgroup'].items(): + print_sys(i + ':' + str(len(j))) + print_sys("Done!") + + def get_dataloader(self, batch_size, test_batch_size = None): + """ + Create data loader + + [Data loader] + Encapsulate PyG graph data objects as DataLoader, supporting batch processing and shuffling. + + [Key settings] + - Training set: shuffle=True, drop_last=True (make sure each batch size is consistent) + - Validation/test set: shuffle=False (maintain order for easy analysis) + """ + if test_batch_size is None: + test_batch_size = batch_size + if self.gene_names is None: + self.gene_names = list(self.adata.var.gene_name) + conditions = list(self.adata.obs.condition.unique()) + pert_genes = [] + for condition in conditions: + if condition == 'ctrl': + continue + cond1 = condition.split('+')[0] + cond2 = condition.split('+')[1] + if cond1 != 'ctrl': + if cond1 not in pert_genes: + pert_genes.append(cond1) + if cond1 not in self.gene_names: + self.gene_names.append(cond1) + if cond2 != 'ctrl': + if cond2 not in pert_genes: + pert_genes.append(cond2) + if cond2 not in self.gene_names: + self.gene_names.append(cond2) + self.gene_names = np.array(self.gene_names) + self.real_gene_names = np.array(list(self.adata.var.gene_name)) + + # Gene name to index mapping + self.node_map = {x: it for it, x in enumerate(self.gene_names)} + if (self.gene_names[: len(self.real_gene_names)] != self.real_gene_names).any(): + raise ValueError("self.real_gene_names must be in the start of self.gene_names") + + # Create cytogram data + cell_graphs = {} + if self.split == 'no_split': + i = 'test' + cell_graphs[i] = [] + for p in self.set2conditions[i]: + if p != 'ctrl': + cell_graphs[i].extend(self.dataset_processed[p]) + + print_sys("Creating dataloaders....") + test_loader = DataLoader(cell_graphs['test'], + batch_size=batch_size, shuffle=False) + + print_sys("Dataloaders created...") + return {'test_loader': test_loader} + else: + if self.split =='no_test': + splits = ['train','val'] + else: + splits = ['train','val','test'] + for i in splits: + cell_graphs[i] = [] + for p in self.set2conditions[i]: + cell_graphs[i].extend(self.dataset_processed[p]) + + print_sys("Creating dataloaders....") + + # Set up data loader + train_loader = DataLoader(cell_graphs['train'], + batch_size=batch_size, shuffle=True, drop_last = True) + val_loader = DataLoader(cell_graphs['val'], + batch_size=batch_size, shuffle=True) + + if self.split !='no_test': + test_loader = DataLoader(cell_graphs['test'], + batch_size=batch_size, shuffle=False) + self.dataloader = {'train_loader': train_loader, + 'val_loader': val_loader, + 'test_loader': test_loader} + + else: + self.dataloader = {'train_loader': train_loader, + 'val_loader': val_loader} + print_sys("Done!") + del self.dataset_processed # Clean memory + + + def create_dataset_file(self): + dl = {} + for p in tqdm(self.adata.obs['condition'].unique()): + cell_graph_dataset = self.create_cell_graph_dataset(self.adata, p, num_samples=1) + dl[p] = cell_graph_dataset + return dl + + def get_pert_idx(self, pert_category, adata_): + pert_idx = [np.where(p == self.gene_names)[0][0] + for p in pert_category.split('+') + if p != 'ctrl'] + return pert_idx + + # Set up feature matrix and output + def create_cell_graph(self, X, y, de_idx, pert, pert_idx=None): + """ + Create a graph data object of a single cell + + [enter] + - X: Control group gene expression value (expression before perturbation) + - y: gene expression value after perturbation (target value) + - de_idx: index of differentially expressed genes + - pert: disturbance type identifier + - pert_idx: index list of perturbed genes + + [Output] + PyG Data object: + - x: (num_genes, 2) feature matrix + [:, 0]: Control group expression value + [:, 1]: perturbation flag (1=the gene is perturbed) + - y: target expression value + - de_idx: differentially expressed gene index + - pert: perturbation type + """ + # Create perturbation feature vectors (mark which genes are perturbed) + pert_feats = np.zeros(len(X[0])) + if pert_idx is not None: + for p in pert_idx: + pert_feats[int(np.abs(p))] = 1 + pert_feats = np.expand_dims(pert_feats, 0) + # Spliced expression values and perturbation markers + feature_mat = torch.Tensor(np.concatenate([X, pert_feats])).T + + return Data(x=feature_mat, edge_index=None, edge_attr=None, + y=torch.Tensor(y), de_idx=de_idx, pert=pert) + + def create_cell_graph_dataset(self, split_adata, pert_category, + num_samples=1): + """ + Create a graph dataset of perturbation categories + + [Processing logic] + 1. For non-control group disturbances: + - Get perturbed gene index + - Expression before perturbation = randomly sample control cells + - Target = expression of cells after perturbation + 2. For the control group: + - Expression before perturbation = target = expression in control cells + + [num_samples parameter] + Each perturbed cell can be paired with multiple control group cells, + Increase the diversity of training data + """ + num_de_genes = 20 # Only focus on the top 20 differentially expressed genes for evaluation + adata_ = split_adata[split_adata.obs['condition'] == pert_category] + de_genes = adata_.uns['rank_genes_groups_cov_all'] + Xs = [] + ys = [] + + # Handling non-control group disturbances + if pert_category != 'ctrl': + # Get the index of the perturbed gene + pert_idx = self.get_pert_idx(pert_category, adata_) + + # Get differentially expressed gene index + pert_de_category = adata_.obs['condition_name'][0] + de_idx = np.where(adata_.var_names.isin( + np.array(de_genes[pert_de_category][:num_de_genes])))[0] + + # Expand feature matrix to include perturbed genes + padded_X = sp.lil_matrix((adata_.X.shape[0], len(self.gene_names)), dtype=adata_.X.dtype) + padded_X[:, :len(self.real_gene_names)] = adata_.X + + for cell_z in padded_X: + # Use control group samples as expression before perturbation + ctrl_samples = self.ctrl_adata[np.random.randint(0, + len(self.ctrl_adata), num_samples), :] + padded_ctrl_X = sp.lil_matrix((ctrl_samples.X.shape[0], len(self.gene_names)), dtype=ctrl_samples.X.dtype) + padded_ctrl_X[:, :len(self.real_gene_names)] = ctrl_samples.X + for c in padded_ctrl_X: + Xs.append(c) # control group expression + ys.append(cell_z) # expression after perturbation + + # Process control group + else: + pert_idx = None + de_idx = [-1] * num_de_genes + padded_X = sp.lil_matrix((adata_.X.shape[0], len(self.gene_names)), dtype=adata_.X.dtype) + padded_X[:, :len(self.real_gene_names)] = adata_.X + for cell_z in padded_X: + Xs.append(cell_z) + ys.append(cell_z) + + # Create a cell diagram data object + cell_graphs = [] + for X, y in zip(Xs, ys): + cell_graphs.append(self.create_cell_graph(X.toarray(), + y.toarray(), de_idx, pert_category, pert_idx)) + return cell_graphs \ No newline at end of file diff --git a/BioMod/gene_perturbation/train_module.py b/BioMod/gene_perturbation/train_module.py index 936c7445b9a3370fd5e2406d116e3f4c6d50b74a..06f6652fd2a27f3fe9e395d94030db93ae30aa4a 100644 --- a/BioMod/gene_perturbation/train_module.py +++ b/BioMod/gene_perturbation/train_module.py @@ -1,325 +1,333 @@ -""" -================================================================================ -Genetic perturbation task training script (train_module.py) -================================================================================ - -[Task Overview] -This script is used to train a model that predicts the effects of genetic perturbation. Gene perturbation refers to knockout or -Overexpress a gene and observe its effect on the cell transcriptome. The goal of the model is: given a combination of genetic perturbations, -Predict the gene expression profile of cells after perturbation. - -[Core process] -1. Load perturbation data sets (such as Norman, Adamson and other public data sets) -2. Initialize the pre-trained single-cell basic model (PerformerEncoderOnly_Module) -3. Fine-tune the model on perturbed data and learn to predict the effect of perturbation -4. Evaluate model performance (MSE, Pearson correlation coefficient and other indicators) - -[Key technical points] -- Use DDP (DistributedDataParallel) for multi-GPU distributed training -- Use module-based gene representation to group genes and then encode them -- Supports multiple gene expression and gene2vec fusion methods - -[Adaptation instructions] -This script is based on GEARS/train_dual_ddp_reduce_dec.py and adapted to the module-based model. -Use a completely separate native module, avoiding any external gears_reduce imports. -================================================================================ -""" - -import os -# [Key] Disable Numba JIT compilation to avoid compilation conflicts in multi-process environments -# Numba may cause problems during distributed training. Use pure Python to implement it after disabling it. -os.environ['NUMBA_DISABLE_JIT'] = '1' -import sys -import time -import argparse -import pandas as pd -import numpy as np -import scanpy as sc -from os.path import join as pjoin - -import torch -from torch.utils.data.distributed import DistributedSampler -from torch.nn.parallel import DistributedDataParallel as DDP -import torch.distributed as dist -import tempfile - -# ============================================================ -# [Key] Correctly set the order of sys.path -# Local modules must take precedence and avoid importing into external utils.py -# This is a common Python module import trap: if a module with the same name exists in the project root directory, -# can be imported incorrectly, causing problems that are difficult to debug -# ============================================================ -current_dir = os.path.dirname(os.path.abspath(__file__)) -project_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - -# First add the project root directory (for performer_pytorch_cont) -if project_root not in sys.path: - sys.path.append(project_root) - -# Finally add the current directory (make sure local modules take precedence) -# Use append instead of insert(0), but since the current directory is empty there will be no conflict -if current_dir not in sys.path: - sys.path.insert(0, current_dir) - -# ============================================================ -# [Key] Create an independent Numba cache directory for each process -# In distributed training, multiple processes sharing the same cache directory can cause file conflicts. -# This is a common problem with Numba in a DDP environment. An independent cache must be allocated for each rank. -# ============================================================ -if "RANK" in os.environ: # Distributed environment - cache_dir = f"/tmp/numba_cache_{os.environ['RANK']}" -else: - cache_dir = tempfile.mkdtemp(prefix="numba_cache_") -os.environ["NUMBA_CACHE_DIR"] = cache_dir - -# ============================================================ -# [Module Import] Use completely independent local modules -# These local modules are to avoid conflicts with modules of the same name in other parts of the project -# ============================================================ -from gears_reduce_local import GEARS -from utils import print_sys -from pertdata_local import PertData # Local version, does not rely on external gears_reduce - -# Import module model -from model_module import GEARS_Model_Module - -def main(parser): - """ - main training function - - [Training process] - 1. Initialize the distributed training environment (DDP) - 2. Loading and preprocessing perturbation data - 3. Prepare data partitioning (training/validation/testing) - 4. Initialize model and optimizer - 5. Training loop + validation + testing - """ - # Turn on anomaly detection to help debug gradient calculation problems - torch.autograd.set_detect_anomaly(True) - - args = parser.parse_args() - - # ============================================================ - # [Distributed training initialization] - # DDP uses NCCL backend for inter-GPU communication - # local_rank: the GPU number of the current process on this machine - # RANK: global process number (different machines have different ranks during multi-machine training) - # ============================================================ - local_rank = args.local_rank - rank = int(os.environ["RANK"]) - is_master = (rank == 0) # Rank 0 is the main process, responsible for printing logs and saving models - - dist.init_process_group(backend='nccl', init_method='env://') - print('DDP init!') - local_rank = int(os.environ['LOCAL_RANK']) - device = f'cuda:{local_rank}' - torch.cuda.set_device(device) - world_size = torch.distributed.get_world_size() # Total number of processes - - print("local_rank: ", local_rank) - - valid_every = args.valid_every # Verify every epoch - - # ============================================================ - # [Data loading] - # The PertData class is responsible for loading and processing perturbation data - # Supported data sets: norman, adamson, dixit and other public data sets - # ============================================================ - pert_data = PertData(args.data_dir) - # load dataset in paper: norman, adamson, dixit. - try: - if args.data_name in ['norman', 'adamson', 'dixit']: - pert_data.load(data_name=args.data_name) - else: - print('load data') - pert_data.load(data_path=pjoin(args.data_dir, args.data_name)) - except: - # If the data format is not standard, try loading from h5ad file - adata = sc.read_h5ad(pjoin(args.data_dir, args.data_name+'.h5ad')) - adata.uns['log1p'] = {} - adata.uns['log1p']['base'] = None - pert_data.new_data_process(dataset_name=args.data_name, adata=adata) - - # ============================================================ - # [Gene Mapping] - # Check the overlap between the genes of the input data and the genes of the pre-trained model - # An overlap ratio that is too low may result in poor model performance. - # ============================================================ - input_genes_ens_ids = pert_data.adata.var.index.tolist() - with open(args.scfm_genes_list_path, 'r') as f: - scfm_genes_ens_ids = [line.rstrip('\n') for line in f.readlines()] - if is_master: - print("target and scfm genes intersect: ", len(np.intersect1d(input_genes_ens_ids, scfm_genes_ens_ids))) - - # ============================================================ - # [Data division] - # The split parameter determines the data partitioning strategy: - # - 'simulation': simulates the division of real application scenarios, and the test set contains unseen gene combinations - # - 'combo_seen0/1/2': divided according to the number of genes seen in the combination - # train_gene_set_size: training set gene proportion - # ============================================================ - pert_data.prepare_split(split=args.split, seed=args.seed, train_gene_set_size=args.train_gene_set_size) - # get dataloader with batch size - pert_data.get_dataloader(batch_size=args.batch_size, test_batch_size=args.test_batch_size) - - # set up and train a model - if is_master: - print_sys("EXPERIMENT: " + args.run_name) - - # ============================================================ - # [Model initialization] - # The GEARS class encapsulates the complete process of training, verification, and testing - # ============================================================ - gears_model = GEARS(pert_data, - local_rank=local_rank, - is_master=is_master, - world_size=world_size, - weight_bias_track=args.wandb, - train_bs=args.batch_size, - test_bs=args.test_batch_size, - device=device, - exp_name=args.run_name) - - # ============================================================ - # [Model parameter configuration] - # Use GEARS_Model_Module instead of GEARS_Model_Acc - # Key parameter description: - # - hidden_size: hidden layer dimension, affecting model capacity - # - finetune_method: fine-tuning method ('frozen' freezes pre-training parameters, None means fine-tuning all parameters) - # - accumulation_steps: the number of gradient accumulation steps, used to simulate larger batch size - # ============================================================ - gears_model.model_initialize( - hidden_size=args.hidden_size, - model_type=args.model_type, - bin_set=args.bin_set, - load_path=args.singlecell_model_path, - finetune_method=args.finetune_method, - accumulation_steps=args.accumulation_steps, - mode=args.mode, - input_genes_ens_ids=input_genes_ens_ids, - scfm_genes_ens_ids=scfm_genes_ens_ids, - scfm_hyper_params_path=args.scfm_hyper_params_path, - scfm_ckpt_path=args.scfm_ckpt_path, - scfm_class=None, # Do not use scfm_class, use module model directly - key_enc=None, - scfm_gene2vec_file=args.scfm_gene2vec_file, - record_pred=args.record_pred, - model_class=GEARS_Model_Module, # Use module model class - # ============================================================ - # [Module related parameters] The following parameters affect how the model processes module-based data - # module_data_path/module_data_dir: module divides the data path - # gene_mapping_path: mapping file from gene symbol to Ensembl ID - # use_module_level: whether to use module level features directly (default False, reverts to gene level) - # fusion_method: Read from pkl to ensure consistency with pre-training - # shuffle_module_genes: [Ablation experiment] Disrupt the gene distribution within the module - # ============================================================ - module_data_path=args.module_data_path if hasattr(args, 'module_data_path') else None, - module_data_dir=args.module_data_dir if hasattr(args, 'module_data_dir') else None, - gene_mapping_path=args.gene_mapping_path if hasattr(args, 'gene_mapping_path') else '../gene2ens.csv', - scfm_genes_list_path=args.scfm_genes_list_path, - use_module_level=args.use_module_level if hasattr(args, 'use_module_level') else False, - shuffle_module_genes=args.shuffle_module_genes if hasattr(args, 'shuffle_module_genes') else False, - shuffle_seed=args.shuffle_seed if hasattr(args, 'shuffle_seed') else 2021, - ) - - gears_model.train( - epochs=args.epochs, - result_dir=args.result_dir, - lr=args.lr, - valid_every=valid_every - ) - - # save params - if is_master: - param_pd = pd.DataFrame(vars(args), index=['params']).T - param_pd.to_csv(f'{args.result_dir}/params.csv') - -if __name__ == '__main__': - parser = argparse.ArgumentParser(description='GEARS with PerformerEncoderOnly_Module') - - # ============================================================ - # [Distributed training parameters] - # local-rank: automatically set by torch.distributed.launch, no need to specify manually - # ============================================================ - parser.add_argument("--local-rank", type=int, default=-1, help='Local process rank.') - - # ============================================================ - # [Data related parameters] - # data_dir: data storage directory - # data_name: Data set name (norman/adamson/dixit is a public data set) - # split: data partitioning strategy, simulation is the most commonly used - # train_gene_set_size: The proportion of genes in the training set, which affects the evaluation of model generalization ability - # ============================================================ - parser.add_argument('--data_dir', type=str, default='../../GEARS/data') - parser.add_argument('--data_name', type=str, default='norman') - parser.add_argument('--split', type=str, default='simulation') - parser.add_argument('--result_dir', type=str, default='./results') - parser.add_argument('--seed', type=int, default=1) - - # ============================================================ - # [Training hyperparameters] - # epochs: number of training rounds, usually 15-20 rounds is enough - # batch_size: Batch size, adjusted according to GPU memory - # lr: learning rate, usually a smaller learning rate (1e-3 or less) is used during fine-tuning - # accumulation_steps: Number of gradient accumulation steps, used to simulate larger batch size - # ============================================================ - parser.add_argument('--epochs', type=int, default=20) - parser.add_argument('--batch_size', type=int, default=32) - parser.add_argument('--valid_every', type=int, default=1) - parser.add_argument('--test_batch_size', type=int, default=128) - parser.add_argument('--train_gene_set_size', type=float, default=0.75) - parser.add_argument('--hidden_size', type=int, default=64) - parser.add_argument('--device', type=str, default='cuda') - parser.add_argument('--model_type', type=str, default=None) - parser.add_argument('--bin_set', type=str, default=None) - parser.add_argument('--singlecell_model_path', type=str, default=None) - parser.add_argument('--finetune_method', type=str, default=None) - parser.add_argument('--mode', type=str, default='v1') - parser.add_argument('--accumulation_steps', type=int, default=1) - parser.add_argument('-wandb', action='store_true', help='Enable Weights & Biases logging') - parser.add_argument('-record_pred', action='store_true', help='Record predictions for analysis') - - # ============================================================ - # [Related parameters of pre-training model] - # scfm_genes_list_path: list of genes used by the pre-trained model - # scfm_hyper_params_path: Hyperparameter configuration file of the pre-trained model - # scfm_ckpt_path: checkpoint file of the pre-trained model - # scfm_gene2vec_file: gene2vec pre-trained embedding file - # ============================================================ - parser.add_argument('--scfm_genes_list_path', type=str, - default="../../selected_genes_27k.txt") - parser.add_argument('--scfm_hyper_params_path', type=str, - default="../ckpts_performer_module_test/performer_module_test.pkl") - parser.add_argument('--scfm_ckpt_path', type=str, - default="../ckpts_performer_module_test/performer_module_test_epoch6.pth") - parser.add_argument('--run_name', type=str, - default="default_run_module") - parser.add_argument('--scfm_gene2vec_file', type=str, - default='../../selected_gene2vec_27k.npy') - - # ============================================================ - # [Module related parameters] These parameters affect how the model handles module-based gene representation. - # module_data_path: single module JSON file path - # module_data_dir: module JSON file directory - # gene_mapping_path: mapping file from gene symbol to Ensembl ID - # use_module_level: whether to use module level features directly (experimental feature) - # fusion_method: Read from pkl to ensure consistency with pre-training - # shuffle_module_genes: [Ablation experiment] Disrupt the genes in the module to verify the importance of the module structure - # ============================================================ - parser.add_argument('--module_data_path', type=str, default=None, - help='Path to module JSON file (single file)') - parser.add_argument('--module_data_dir', type=str, default='../module_data', - help='Path to directory containing module JSON files') - parser.add_argument('--gene_mapping_path', type=str, default='../../gene2ens.csv', - help='Path to gene symbol to Ensembl ID mapping CSV') - parser.add_argument('--use_module_level', action='store_true', - help='Use module-level features directly instead of recovering to gene level. ' - 'Note: This requires compatible downstream architecture (not fully implemented).') - parser.add_argument('--shuffle_module_genes', action='store_true', - help='[Ablation experiment] Additionally shuffle module_sets after restoring them from the pretrained PKL. ' - 'Do not pass this parameter when reproducing the experiment normally.') - parser.add_argument('--shuffle_seed', type=int, default=2021, - help='Only takes effect when --shuffle_module_genes is turned on and is used for additional downstream shuffling.') - parser.add_argument('--lr', type=float, default=1e-3) - - main(parser) +""" +================================================================================ +Genetic perturbation task training script (train_module.py) +================================================================================ + +[Task Overview] +This script is used to train a model that predicts the effects of genetic perturbation. Gene perturbation refers to knockout or +Overexpress a gene and observe its effect on the cell transcriptome. The goal of the model is: given a combination of genetic perturbations, +Predict the gene expression profile of cells after perturbation. + +[Core process] +1. Load perturbation data sets (such as Norman, Adamson and other public data sets) +2. Initialize the pre-trained single-cell basic model (PerformerEncoderOnly_Module) +3. Fine-tune the model on perturbed data and learn to predict the effect of perturbation +4. Evaluate model performance (MSE, Pearson correlation coefficient and other indicators) + +[Key technical points] +- Use DDP (DistributedDataParallel) for multi-GPU distributed training +- Use module-based gene representation to group genes and then encode them +- Supports multiple gene expression and gene2vec fusion methods + +[Adaptation instructions] +This script is based on GEARS/train_dual_ddp_reduce_dec.py and adapted to the module-based model. +Use a completely separate native module, avoiding any external gears_reduce imports. +================================================================================ +""" + +import os +# [Key] Disable Numba JIT compilation to avoid compilation conflicts in multi-process environments +# Numba may cause problems during distributed training. Use pure Python to implement it after disabling it. +os.environ['NUMBA_DISABLE_JIT'] = '1' +import sys +import time +import argparse +import pandas as pd +import numpy as np +import scanpy as sc +from os.path import join as pjoin + +import torch +from torch.utils.data.distributed import DistributedSampler +from torch.nn.parallel import DistributedDataParallel as DDP +import torch.distributed as dist +import tempfile +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +ASSET_DIR = REPO_ROOT / "model_dependencies" +DOWNSTREAM_DATA_DIR = REPO_ROOT / "data" / "downstream" / "final_data" +WEIGHTS_DIR = REPO_ROOT / "weight" / "0810model" + +# ============================================================ +# [Key] Correctly set the order of sys.path +# Local modules must take precedence and avoid importing into external utils.py +# This is a common Python module import trap: if a module with the same name exists in the project root directory, +# can be imported incorrectly, causing problems that are difficult to debug +# ============================================================ +current_dir = os.path.dirname(os.path.abspath(__file__)) +project_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +# First add the project root directory (for performer_pytorch_cont) +if project_root not in sys.path: + sys.path.append(project_root) + +# Finally add the current directory (make sure local modules take precedence) +# Use append instead of insert(0), but since the current directory is empty there will be no conflict +if current_dir not in sys.path: + sys.path.insert(0, current_dir) + +# ============================================================ +# [Key] Create an independent Numba cache directory for each process +# In distributed training, multiple processes sharing the same cache directory can cause file conflicts. +# This is a common problem with Numba in a DDP environment. An independent cache must be allocated for each rank. +# ============================================================ +if "RANK" in os.environ: # Distributed environment + cache_dir = tempfile.mkdtemp(prefix=f"biomod_numba_rank{os.environ['RANK']}_") +else: + cache_dir = tempfile.mkdtemp(prefix="numba_cache_") +os.environ["NUMBA_CACHE_DIR"] = cache_dir + +# ============================================================ +# [Module Import] Use completely independent local modules +# These local modules are to avoid conflicts with modules of the same name in other parts of the project +# ============================================================ +from gears_reduce_local import GEARS +from utils import print_sys +from pertdata_local import PertData # Local version, does not rely on external gears_reduce + +# Import module model +from model_module import GEARS_Model_Module + +def main(parser): + """ + main training function + + [Training process] + 1. Initialize the distributed training environment (DDP) + 2. Loading and preprocessing perturbation data + 3. Prepare data partitioning (training/validation/testing) + 4. Initialize model and optimizer + 5. Training loop + validation + testing + """ + # Turn on anomaly detection to help debug gradient calculation problems + torch.autograd.set_detect_anomaly(True) + + args = parser.parse_args() + if args.module_data_path is None and args.module_data_dir is None: + args.module_data_dir = str(ASSET_DIR / 'module_data_all') + + # ============================================================ + # [Distributed training initialization] + # DDP uses NCCL backend for inter-GPU communication + # local_rank: the GPU number of the current process on this machine + # RANK: global process number (different machines have different ranks during multi-machine training) + # ============================================================ + local_rank = args.local_rank + rank = int(os.environ["RANK"]) + is_master = (rank == 0) # Rank 0 is the main process, responsible for printing logs and saving models + + dist.init_process_group(backend='nccl', init_method='env://') + print('DDP init!') + local_rank = int(os.environ['LOCAL_RANK']) + device = f'cuda:{local_rank}' + torch.cuda.set_device(device) + world_size = torch.distributed.get_world_size() # Total number of processes + + print("local_rank: ", local_rank) + + valid_every = args.valid_every # Verify every epoch + + # ============================================================ + # [Data loading] + # The PertData class is responsible for loading and processing perturbation data + # Supported data sets: norman, adamson, dixit and other public data sets + # ============================================================ + pert_data = PertData(args.data_dir) + # load dataset in paper: norman, adamson, dixit. + try: + if args.data_name in ['norman', 'adamson', 'dixit']: + pert_data.load(data_name=args.data_name) + else: + print('load data') + pert_data.load(data_path=pjoin(args.data_dir, args.data_name)) + except: + # If the data format is not standard, try loading from h5ad file + adata = sc.read_h5ad(pjoin(args.data_dir, args.data_name+'.h5ad')) + adata.uns['log1p'] = {} + adata.uns['log1p']['base'] = None + pert_data.new_data_process(dataset_name=args.data_name, adata=adata) + + # ============================================================ + # [Gene Mapping] + # Check the overlap between the genes of the input data and the genes of the pre-trained model + # An overlap ratio that is too low may result in poor model performance. + # ============================================================ + input_genes_ens_ids = pert_data.adata.var.index.tolist() + with open(args.scfm_genes_list_path, 'r') as f: + scfm_genes_ens_ids = [line.rstrip('\n') for line in f.readlines()] + if is_master: + print("target and scfm genes intersect: ", len(np.intersect1d(input_genes_ens_ids, scfm_genes_ens_ids))) + + # ============================================================ + # [Data division] + # The split parameter determines the data partitioning strategy: + # - 'simulation': simulates the division of real application scenarios, and the test set contains unseen gene combinations + # - 'combo_seen0/1/2': divided according to the number of genes seen in the combination + # train_gene_set_size: training set gene proportion + # ============================================================ + pert_data.prepare_split(split=args.split, seed=args.seed, train_gene_set_size=args.train_gene_set_size) + # get dataloader with batch size + pert_data.get_dataloader(batch_size=args.batch_size, test_batch_size=args.test_batch_size) + + # set up and train a model + if is_master: + print_sys("EXPERIMENT: " + args.run_name) + + # ============================================================ + # [Model initialization] + # The GEARS class encapsulates the complete process of training, verification, and testing + # ============================================================ + gears_model = GEARS(pert_data, + local_rank=local_rank, + is_master=is_master, + world_size=world_size, + weight_bias_track=args.wandb, + train_bs=args.batch_size, + test_bs=args.test_batch_size, + device=device, + exp_name=args.run_name) + + # ============================================================ + # [Model parameter configuration] + # Use GEARS_Model_Module instead of GEARS_Model_Acc + # Key parameter description: + # - hidden_size: hidden layer dimension, affecting model capacity + # - finetune_method: fine-tuning method ('frozen' freezes pre-training parameters, None means fine-tuning all parameters) + # - accumulation_steps: the number of gradient accumulation steps, used to simulate larger batch size + # ============================================================ + gears_model.model_initialize( + hidden_size=args.hidden_size, + model_type=args.model_type, + bin_set=args.bin_set, + load_path=args.singlecell_model_path, + finetune_method=args.finetune_method, + accumulation_steps=args.accumulation_steps, + mode=args.mode, + input_genes_ens_ids=input_genes_ens_ids, + scfm_genes_ens_ids=scfm_genes_ens_ids, + scfm_hyper_params_path=args.scfm_hyper_params_path, + scfm_ckpt_path=args.scfm_ckpt_path, + scfm_class=None, # Do not use scfm_class, use module model directly + key_enc=None, + scfm_gene2vec_file=args.scfm_gene2vec_file, + record_pred=args.record_pred, + model_class=GEARS_Model_Module, # Use module model class + # ============================================================ + # [Module related parameters] The following parameters affect how the model processes module-based data + # module_data_path/module_data_dir: module divides the data path + # gene_mapping_path: mapping file from gene symbol to Ensembl ID + # use_module_level: whether to use module level features directly (default False, reverts to gene level) + # fusion_method: Read from pkl to ensure consistency with pre-training + # shuffle_module_genes: [Ablation experiment] Disrupt the gene distribution within the module + # ============================================================ + module_data_path=args.module_data_path if hasattr(args, 'module_data_path') else None, + module_data_dir=args.module_data_dir if hasattr(args, 'module_data_dir') else None, + gene_mapping_path=args.gene_mapping_path if hasattr(args, 'gene_mapping_path') else str(ASSET_DIR / 'gene2ens.csv'), + scfm_genes_list_path=args.scfm_genes_list_path, + use_module_level=args.use_module_level if hasattr(args, 'use_module_level') else False, + shuffle_module_genes=args.shuffle_module_genes if hasattr(args, 'shuffle_module_genes') else False, + shuffle_seed=args.shuffle_seed if hasattr(args, 'shuffle_seed') else 2021, + ) + + gears_model.train( + epochs=args.epochs, + result_dir=args.result_dir, + lr=args.lr, + valid_every=valid_every + ) + + # save params + if is_master: + param_pd = pd.DataFrame(vars(args), index=['params']).T + param_pd.to_csv(f'{args.result_dir}/params.csv') + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='GEARS with PerformerEncoderOnly_Module') + + # ============================================================ + # [Distributed training parameters] + # local-rank: automatically set by torch.distributed.launch, no need to specify manually + # ============================================================ + parser.add_argument("--local-rank", type=int, default=-1, help='Local process rank.') + + # ============================================================ + # [Data related parameters] + # data_dir: data storage directory + # data_name: Data set name (norman/adamson/dixit is a public data set) + # split: data partitioning strategy, simulation is the most commonly used + # train_gene_set_size: The proportion of genes in the training set, which affects the evaluation of model generalization ability + # ============================================================ + parser.add_argument('--data_dir', type=str, default=str(DOWNSTREAM_DATA_DIR)) + parser.add_argument('--data_name', type=str, default='norman') + parser.add_argument('--split', type=str, default='simulation') + parser.add_argument('--result_dir', type=str, default='./results') + parser.add_argument('--seed', type=int, default=1) + + # ============================================================ + # [Training hyperparameters] + # epochs: number of training rounds, usually 15-20 rounds is enough + # batch_size: Batch size, adjusted according to GPU memory + # lr: learning rate, usually a smaller learning rate (1e-3 or less) is used during fine-tuning + # accumulation_steps: Number of gradient accumulation steps, used to simulate larger batch size + # ============================================================ + parser.add_argument('--epochs', type=int, default=20) + parser.add_argument('--batch_size', type=int, default=32) + parser.add_argument('--valid_every', type=int, default=1) + parser.add_argument('--test_batch_size', type=int, default=128) + parser.add_argument('--train_gene_set_size', type=float, default=0.75) + parser.add_argument('--hidden_size', type=int, default=64) + parser.add_argument('--device', type=str, default='cuda') + parser.add_argument('--model_type', type=str, default=None) + parser.add_argument('--bin_set', type=str, default=None) + parser.add_argument('--singlecell_model_path', type=str, default=None) + parser.add_argument('--finetune_method', type=str, default=None) + parser.add_argument('--mode', type=str, default='v1') + parser.add_argument('--accumulation_steps', type=int, default=1) + parser.add_argument('-wandb', action='store_true', help='Enable Weights & Biases logging') + parser.add_argument('-record_pred', action='store_true', help='Record predictions for analysis') + + # ============================================================ + # [Related parameters of pre-training model] + # scfm_genes_list_path: list of genes used by the pre-trained model + # scfm_hyper_params_path: Hyperparameter configuration file of the pre-trained model + # scfm_ckpt_path: checkpoint file of the pre-trained model + # scfm_gene2vec_file: gene2vec pre-trained embedding file + # ============================================================ + parser.add_argument('--scfm_genes_list_path', type=str, + default=str(ASSET_DIR / "selected_genes_27k.txt")) + parser.add_argument('--scfm_hyper_params_path', type=str, + default=str(WEIGHTS_DIR / "mae_tost_70M_75mask_ec2.pkl")) + parser.add_argument('--scfm_ckpt_path', type=str, + default=str(WEIGHTS_DIR / "mae_tost_70M_75mask_ec2_epoch0.pth")) + parser.add_argument('--run_name', type=str, + default="default_run_module") + parser.add_argument('--scfm_gene2vec_file', type=str, + default=str(ASSET_DIR / 'selected_gene2vec_27k.npy')) + + # ============================================================ + # [Module related parameters] These parameters affect how the model handles module-based gene representation. + # module_data_path: single module JSON file path + # module_data_dir: module JSON file directory + # gene_mapping_path: mapping file from gene symbol to Ensembl ID + # use_module_level: whether to use module level features directly (experimental feature) + # fusion_method: Read from pkl to ensure consistency with pre-training + # shuffle_module_genes: [Ablation experiment] Disrupt the genes in the module to verify the importance of the module structure + # ============================================================ + parser.add_argument('--module_data_path', type=str, default=None, + help='Path to module JSON file (single file)') + parser.add_argument('--module_data_dir', type=str, default=None, + help='Module JSON directory; defaults to bundled module_data_all only when no module file or directory is supplied') + parser.add_argument('--gene_mapping_path', type=str, default=str(ASSET_DIR / 'gene2ens.csv'), + help='Path to gene symbol to Ensembl ID mapping CSV') + parser.add_argument('--use_module_level', action='store_true', + help='Use module-level features directly instead of recovering to gene level. ' + 'Note: This requires compatible downstream architecture (not fully implemented).') + parser.add_argument('--shuffle_module_genes', action='store_true', + help='[Ablation experiment] Additionally shuffle module_sets after restoring them from the pretrained PKL. ' + 'Do not pass this parameter when reproducing the experiment normally.') + parser.add_argument('--shuffle_seed', type=int, default=2021, + help='Only takes effect when --shuffle_module_genes is turned on and is used for additional downstream shuffling.') + parser.add_argument('--lr', type=float, default=1e-3) + + main(parser) diff --git a/BioMod/gene_perturbation/utils.py b/BioMod/gene_perturbation/utils.py index 1444bb89a41d4ecb3c1d9d9460c5988ba5bc3640..100c24b216692dfea441e051a23a219cb1db52f3 100644 --- a/BioMod/gene_perturbation/utils.py +++ b/BioMod/gene_perturbation/utils.py @@ -1,489 +1,496 @@ -""" -================================================================================ -Utility function module (utils.py) -================================================================================ - -[Module function] -Provides various tool functions required for genetic perturbation tasks, including: - -1. Perturbation analysis function: analyze the gene name of single gene/combination perturbation -2. Gene relationship graph construction: co-expression graph and GO similarity graph -3. Loss function: training loss and uncertainty loss -4. Genetic interaction (GI) analysis: Calculate genetic interaction coefficients - -[Key concepts] -- Perturbation format: "gene1+gene2", such as "ctrl+MAPK1" or "MAPK1+KRAS" -- Co-expression map: constructed based on gene expression correlations in the data -- GO graph: Jaccard similarity construction based on Gene Ontology annotation - -[Key parameters that may affect the results] -- coexpress_threshold: Threshold for co-expression graph construction -- dict_filter: training loss is only calculated on non-zero genes -- direction_lambda: direction consistency loss weight -================================================================================ -""" - -import torch -import numpy as np -import pandas as pd -from sklearn.linear_model import TheilSenRegressor -import torch.nn as nn - -import networkx as nx -from tqdm import tqdm -import pickle -import sys, os -import requests -from torch_geometric.data import Data - -from zipfile import ZipFile - -import statsmodels.api as sm -from sklearn.linear_model import LinearRegression, TheilSenRegressor -# ============================================================ -# Lazy loading of dcor to avoid numba compatibility issues in DDP multi-process environments -# dcor will be imported when needed (in the get_coeffs function) -# ============================================================ -from sklearn.metrics import r2_score - -def parse_single_pert(i): - """ - Parsing single gene perturbation strings - - [Format] - - "ctrl+GENE": The perturbed gene is GENE - - "GENE+ctrl": The perturbed gene is GENE - - [return] - Perturbed gene name - """ - a = i.split('+')[0] - b = i.split('+')[1] - if a == 'ctrl': - pert = b - else: - pert = a - return pert - -def filter_pert_cond_in_go(condition, exclude_pert): - """ - Filter perturbations not in the GO graph - - [use] - Genes without annotations in the GO diagram cannot predict their perturbation effects. - These disturbing conditions need to be eliminated during data loading. - """ - if condition == 'ctrl': - return True - else: - if condition in exclude_pert: - return False - else: - return True - -def parse_combo_pert(i): - """ - Parsing combined perturbation strings - - [Format] - - "GENE1+GENE2": perturb two genes simultaneously - - [return] - (gene1, gene2) tuple - """ - return i.split('+')[0], i.split('+')[1] - -def combine_res(res_1, res_2): - """Merge two evaluation result dictionaries""" - res_out = {} - for key in res_1: - res_out[key] = np.concatenate([res_1[key], res_2[key]]) - return res_out - -def parse_any_pert(p): - """ - Parse arbitrary perturbation strings (single genes or combinations) - - [return] - Gene list: - - Single gene perturbation: [gene] - - Combinatorial perturbation: [gene1, gene2] - """ - if ('ctrl' in p) and (p != 'ctrl'): - return [parse_single_pert(p)] - elif 'ctrl' not in p: - out = parse_combo_pert(p) - return [out[0], out[1]] - -def np_pearson_cor(x, y): - """ - Calculate Pearson correlation coefficient matrix - - [use] - Used to construct co-expressed gene relationship diagrams - """ - xv = x - x.mean(axis=0) - yv = y - y.mean(axis=0) - xvss = (xv * xv).sum(axis=0) - yvss = (yv * yv).sum(axis=0) - result = np.matmul(xv.transpose(), yv) / np.sqrt(np.outer(xvss, yvss)) - return np.maximum(np.minimum(result, 1.0), -1.0) - - -def dataverse_download(url, save_path): - """dataverse download helper with progress bar - - Args: - url (str): the url of the dataset - path (str): the path to save the dataset - """ - - if os.path.exists(save_path): - print_sys('Found local copy...') - else: - print_sys("Downloading...") - response = requests.get(url, stream=True) - total_size_in_bytes= int(response.headers.get('content-length', 0)) - block_size = 1024 - progress_bar = tqdm(total=total_size_in_bytes, unit='iB', unit_scale=True) - with open(save_path, 'wb') as file: - for data in response.iter_content(block_size): - progress_bar.update(len(data)) - file.write(data) - progress_bar.close() - - -def zip_data_download_wrapper(url, save_path, data_path): - - if os.path.exists(save_path): - print_sys('Found local copy...') - else: - dataverse_download(url, save_path + '.zip') - print_sys('Extracting zip file...') - with ZipFile((save_path + '.zip'), 'r') as zip: - zip.extractall(path = data_path) - print_sys("Done!") - -def get_go_auto(gene_list, data_path, data_name): - go_path = os.path.join(data_path, data_name, 'go.csv') - - if os.path.exists(go_path): - return pd.read_csv(go_path) - else: - with open(os.path.join('/home/ding.bai/pert_new/GEARS/data_all/gene2go_all.pkl'), 'rb') as f: - gene2go = pickle.load(f) - - gene2go = {i: list(gene2go[i]) for i in gene_list if i in gene2go} - edge_list = [] - for g1 in tqdm(gene2go.keys()): - for g2 in gene2go.keys(): - edge_list.append((g1, g2, len(np.intersect1d(gene2go[g1], gene2go[g2]))/len(np.union1d(gene2go[g1], gene2go[g2])))) - - edge_list_filter = [i for i in edge_list if i[2] > 0] - further_filter = [i for i in edge_list if i[2] > 0.1] - df_edge_list = pd.DataFrame(further_filter).rename(columns = {0: 'gene1', 1: 'gene2', 2: 'score'}) - - df_edge_list = df_edge_list.rename(columns = {'gene1': 'source', 'gene2': 'target', 'score': 'importance'}) - - df_edge_list.to_csv(go_path, index = False) - return df_edge_list - -def get_go(df_gene2go): - df_gene2go['Entry name'] = df_gene2go['Entry name'].apply(lambda x: x.split('_')[0]) - df_gene2go = df_gene2go[df_gene2go['Gene ontology IDs'].notnull()] - df_gene2go = df_gene2go.rename(columns = {[i for i in df_gene2go.columns.values if 'yourlist' in i][0]: 'gene_id'}) - - geneid2go = dict(df_gene2go[['gene_id', 'Gene ontology IDs']].values) - - gene2go = {} - for i,j in geneid2go.items(): - j = [k.strip() for k in j.split(';')] - for k in i.split(','): - gene2go[ensembl2genename[k]] = j - - from tqdm import tqdm - edge_list = [] - for g1 in tqdm(gene2go.keys()): - for g2 in gene2go.keys(): - - edge_list.append((g1, g2, len(np.intersect1d(gene2go[g1], gene2go[g2]))/len(np.union1d(gene2go[g1], gene2go[g2])))) - - edge_list_filter = [i for i in edge_list if i[2] > 0] - further_filter = [i for i in edge_list if i[2] > 0.1] - df_edge_list = pd.DataFrame(further_filter).rename(columns = {0: 'gene1', 1: 'gene2', 2: 'score'}) - - df_edge_list = df_edge_list.rename(columns = {'gene1': 'source', 'gene2': 'target', 'score': 'importance'}) - - return df_edge_list - -class GeneSimNetwork(): - ''' - Represents a gene similarity network. Has the following attributes: - - self.edge_list - - self.gene_list - - self.G - - self.edge_index (Tensor, can be used to set G_go in model_initialize) - - self.edge_weight (Tensor, can be used to set G_go_weight in model_initialize) - ''' - def __init__(self, edge_list, gene_list, node_map): - # Keeps edges if both source and target exist in node_map - keep_edges = np.zeros(len(edge_list)).astype(bool) - for i, (source, target, importance) in edge_list.iterrows(): - keep_edges[i] = source in node_map and target in node_map - self.edge_list = edge_list[keep_edges] - - # Constructs NetworkX graph - self.G = nx.from_pandas_edgelist(self.edge_list, source='source', - target='target', edge_attr=['importance'], - create_using=nx.DiGraph()) - self.gene_list = gene_list - - # Add any isolated nodes (represented in the tensor with self edges) - for n in self.gene_list: - added_nodes = [] - if n not in self.G.nodes(): - self.G.add_node(n) - added_nodes.append(n) - if len(added_nodes) > 0: - print('The following nodes were not present in the edge list and have been added to the graph as self-edges:', added_nodes) - - # Convert graph edges to tensor - edge_index_ = [(node_map[e[0]], node_map[e[1]]) for e in self.G.edges] - self.edge_index = torch.tensor(edge_index_, dtype=torch.long).T - - # Convert edge weights to tensor - edge_attr = nx.get_edge_attributes(self.G, 'importance') - importance = np.array([edge_attr[e] for e in self.G.edges]) - self.edge_weight = torch.Tensor(importance) - - -def get_similarity_network(network_type, adata, threshold, k, gene_list, data_path, data_name, split, seed, train_gene_set_size, set2conditions): - - if network_type == 'co-express': - df_out = get_coexpression_network_from_train(adata, threshold, k, data_path, data_name, split, seed, train_gene_set_size, set2conditions) - elif network_type == 'go': - df_jaccard = get_go_auto(gene_list, data_path, data_name) - df_out = df_jaccard.groupby('target').apply(lambda x: x.nlargest(k + 1,['importance'])).reset_index(drop = True) - - return df_out - -def get_coexpression_network_from_train(adata, threshold, k, data_path, data_name, split, seed, train_gene_set_size, set2conditions): - - fname = os.path.join(os.path.join(data_path, data_name), split + '_' + str(seed) + '_' + str(train_gene_set_size) + '_' + str(threshold) + '_' + str(k) + '_co_expression_network.csv') - - - if os.path.exists(fname): - return pd.read_csv(fname) - else: - gene_list = [f for f in adata.var.gene_name.values] - idx2gene = dict(zip(range(len(gene_list)), gene_list)) - X = adata.X - train_perts = set2conditions['train'] - X_tr = X[np.isin(adata.obs.condition, [i for i in train_perts if 'ctrl' in i])] - gene_list = adata.var['gene_name'].values - - X_tr = X_tr.toarray() - out = np_pearson_cor(X_tr, X_tr) - out[np.isnan(out)] = 0 - out = np.abs(out) - - out_sort_idx = np.argsort(out)[:, -(k + 1):] - out_sort_val = np.sort(out)[:, -(k + 1):] - - df_g = [] - for i in range(out_sort_idx.shape[0]): - target = idx2gene[i] - for j in range(out_sort_idx.shape[1]): - df_g.append((idx2gene[out_sort_idx[i, j]], target, out_sort_val[i, j])) - - df_g = [i for i in df_g if i[2] > threshold] - - df_co_expression = pd.DataFrame(df_g).rename(columns = {0: 'source', 1: 'target', 2: 'importance'}) - df_co_expression.to_csv(fname, index = False) - return df_co_expression - - -def uncertainty_loss_fct(pred, logvar, y, perts, reg = 0.1, ctrl = None, direction_lambda = 1e-3, dict_filter = None): - """ - Uncertain version loss: - 1) The error term is weighted by logvar (heteroscedastic modeling) - 2) Optional direction penalty (relative to the up/down direction of control) - 3) `dict_filter` is only calculated on a non-zero gene subset, and the caliber is consistent with the evaluation - """ - gamma = 2 - perts = np.array(perts) - losses = torch.tensor(0.0, requires_grad=True).to(pred.device) - for p in set(perts): - if p!= 'ctrl': - # Training only on non-zero genes of this perturbation can reduce the impact of dropout noise. - retain_idx = dict_filter[p] - pred_p = pred[np.where(perts==p)[0]][:, retain_idx] - y_p = y[np.where(perts==p)[0]][:, retain_idx] - logvar_p = logvar[np.where(perts==p)[0]][:, retain_idx] - else: - pred_p = pred[np.where(perts==p)[0]] - y_p = y[np.where(perts==p)[0]] - logvar_p = logvar[np.where(perts==p)[0]] - - # uncertainty based loss - - losses += torch.sum((pred_p - y_p)**(2 + gamma) + reg * torch.exp(-logvar_p) * (pred_p - y_p)**(2 + gamma))/pred_p.shape[0]/pred_p.shape[1] - - # direction loss - if p!= 'ctrl': - # Directional consistency term: The constraint prediction is consistent with the true direction of change relative to control. - losses += torch.sum(direction_lambda * (torch.sign(y_p - ctrl[retain_idx]) - torch.sign(pred_p - ctrl[retain_idx]))**2)/pred_p.shape[0]/pred_p.shape[1] - else: - losses += torch.sum(direction_lambda * (torch.sign(y_p - ctrl) - torch.sign(pred_p - ctrl))**2)/pred_p.shape[0]/pred_p.shape[1] - - return losses/(len(set(perts))) - -def loss_fct(pred, y, perts, ctrl = None, direction_lambda = 1e-3, dict_filter = None): - """ - Main training loss (without uncertainty): - - Basic item: MSE - - Optional direction item: Penalize upward/downward direction errors - - `dict_filter` will change the training gene set and is an important switch that affects the results. - """ - gamma = 2 - mse_p = torch.nn.MSELoss() - perts = np.array(perts) - losses = torch.tensor(0.0).to(pred.device) - - for p in set(perts): - pert_idx = np.where(perts == p)[0] - - - # during training, we remove the all zero genes into calculation of loss. this gives a cleaner direction loss. empirically, the performance stays the same. - - # during training, we remove the all zero genes into calculation of loss. - # this gives a cleaner direction loss. empirically, the performance stays the same. - - if p!= 'ctrl': - # Generally more stable when consistent with the evaluated non-zero / non-dropout settings. - retain_idx = dict_filter[p] - pred_p = pred[pert_idx][:, retain_idx] - y_p = y[pert_idx][:, retain_idx] - else: - pred_p = pred[pert_idx] - y_p = y[pert_idx] - - - losses += torch.sum((pred_p - y_p)**(2 + gamma))/pred_p.shape[0]/pred_p.shape[1] - - ## direction loss - if (p!= 'ctrl'): - losses += torch.sum(direction_lambda * (torch.sign(y_p - ctrl[retain_idx]) - torch.sign(pred_p - ctrl[retain_idx]))**2)/pred_p.shape[0]/pred_p.shape[1] - else: - losses += torch.sum(direction_lambda * (torch.sign(y_p - ctrl) - torch.sign(pred_p - ctrl))**2)/pred_p.shape[0]/pred_p.shape[1] - - return losses/(len(set(perts))) - - -def print_sys(s): - """system print - - Args: - s (str): the string to print - """ - print(s, flush = True, file = sys.stderr) - -def create_cell_graph_for_prediction(X, pert_idx, pert_gene): - - - # If perturbations will be represented as node features - pert_feats = np.zeros(len(X)) - for p in pert_idx: - pert_feats[int(np.abs(p))] = np.sign(p) - feature_mat = torch.Tensor(np.vstack([X, pert_feats])).T - - return Data(x=feature_mat, pert=pert_gene) - - -def create_cell_graph_dataset_for_prediction(pert_gene, ctrl_adata, gene_names, device, num_samples = 300): - Xs = [] - - # Get the indices (and signs) of applied perturbation - pert_idx = [np.where(p == np.array(gene_names))[0][0] for p in pert_gene] - - Xs = ctrl_adata[np.random.randint(0, len(ctrl_adata), num_samples), :].X.toarray() - # Create cell graphs - cell_graphs = [create_cell_graph_for_prediction(X, pert_idx, pert_gene).to(device) for X in Xs] - return cell_graphs - -## -##GI related utils -## - -def get_coeffs(singles_expr, first_expr, second_expr, double_expr): - # ============================================================ - # Lazy loading of dcor to avoid numba conflicts when importing into multiple processes - # Import dcor only when the function is called - # ============================================================ - try: - from dcor import distance_correlation - except Exception as e: - # Use alternative if dcor import fails (numba compatibility issue) - import warnings - warnings.warn(f"dcor import failed: {e}. Using fallback distance correlation implementation.") - # Use a simplified scipy/numpy based implementation as an alternative - def distance_correlation(x, y): - """Simplified version of distance correlation coefficient, does not rely on numba""" - from scipy.spatial.distance import pdist, squareform - x = np.asarray(x) - y = np.asarray(y) - a = squareform(pdist(x)) - b = squareform(pdist(y)) - A = a - a.mean(axis=0)[None, :] - a.mean(axis=1)[:, None] + a.mean() - B = b - b.mean(axis=0)[None, :] - b.mean(axis=1)[:, None] + b.mean() - dcov2 = (A * B).sum() / (A.size * A.size) - dvar_x = (A * A).sum() / (A.size * A.size) - dvar_y = (B * B).sum() / (B.size * B.size) - if dvar_x == 0 or dvar_y == 0: - return 0.0 - return np.sqrt(dcov2 / np.sqrt(dvar_x * dvar_y)) - - results = {} - results['ts'] = TheilSenRegressor(fit_intercept=False, - max_subpopulation=1e5, - max_iter=1000, - random_state=1000) - X = singles_expr - y = double_expr - results['ts'].fit(X, y.ravel()) - Zts = results['ts'].predict(X) - results['c1'] = results['ts'].coef_[0] - results['c2'] = results['ts'].coef_[1] - results['mag'] = np.sqrt((results['c1']**2 + results['c2']**2)) - - results['dcor'] = distance_correlation(singles_expr, double_expr) - results['dcor_singles'] = distance_correlation(first_expr, second_expr) - results['dcor_first'] = distance_correlation(first_expr, double_expr) - results['dcor_second'] = distance_correlation(second_expr, double_expr) - results['corr_fit'] = np.corrcoef(Zts.flatten(), double_expr.flatten())[0,1] - results['dominance'] = np.abs(np.log10(results['c1']/results['c2'])) - results['eq_contr'] = np.min([results['dcor_first'], results['dcor_second']])/\ - np.max([results['dcor_first'], results['dcor_second']]) - - return results - -def get_GI_params(preds, combo): - - singles_expr = np.array([preds[combo[0]], preds[combo[1]]]).T - first_expr = np.array(preds[combo[0]]).T - second_expr = np.array(preds[combo[1]]).T - double_expr = np.array(preds[combo[0]+'_'+combo[1]]).T - - return get_coeffs(singles_expr, first_expr, second_expr, double_expr) - -def get_GI_genes_idx(adata, GI_gene_file): - # Genes used for linear model fitting - GI_genes = np.load(GI_gene_file, allow_pickle=True) - GI_genes_idx = np.where([g in GI_genes for g in adata.var.gene_name.values])[0] - - return GI_genes_idx - -def get_mean_control(adata): - cols = adata.var.gene_name.values.astype('str') - mean_ctrl_exp = adata[adata.obs['condition'] == 'ctrl'].to_df().mean() - return mean_ctrl_exp +""" +================================================================================ +Utility function module (utils.py) +================================================================================ + +[Module function] +Provides various tool functions required for genetic perturbation tasks, including: + +1. Perturbation analysis function: analyze the gene name of single gene/combination perturbation +2. Gene relationship graph construction: co-expression graph and GO similarity graph +3. Loss function: training loss and uncertainty loss +4. Genetic interaction (GI) analysis: Calculate genetic interaction coefficients + +[Key concepts] +- Perturbation format: "gene1+gene2", such as "ctrl+MAPK1" or "MAPK1+KRAS" +- Co-expression map: constructed based on gene expression correlations in the data +- GO graph: Jaccard similarity construction based on Gene Ontology annotation + +[Key parameters that may affect the results] +- coexpress_threshold: Threshold for co-expression graph construction +- dict_filter: training loss is only calculated on non-zero genes +- direction_lambda: direction consistency loss weight +================================================================================ +""" + +import torch +import numpy as np +import pandas as pd +from sklearn.linear_model import TheilSenRegressor +import torch.nn as nn + +import networkx as nx +from tqdm import tqdm +import pickle +import sys, os +import requests +from torch_geometric.data import Data + +from zipfile import ZipFile + +import statsmodels.api as sm +from sklearn.linear_model import LinearRegression, TheilSenRegressor +# ============================================================ +# Lazy loading of dcor to avoid numba compatibility issues in DDP multi-process environments +# dcor will be imported when needed (in the get_coeffs function) +# ============================================================ +from sklearn.metrics import r2_score + +def parse_single_pert(i): + """ + Parsing single gene perturbation strings + + [Format] + - "ctrl+GENE": The perturbed gene is GENE + - "GENE+ctrl": The perturbed gene is GENE + + [return] + Perturbed gene name + """ + a = i.split('+')[0] + b = i.split('+')[1] + if a == 'ctrl': + pert = b + else: + pert = a + return pert + +def filter_pert_cond_in_go(condition, exclude_pert): + """ + Filter perturbations not in the GO graph + + [use] + Genes without annotations in the GO diagram cannot predict their perturbation effects. + These disturbing conditions need to be eliminated during data loading. + """ + if condition == 'ctrl': + return True + else: + if condition in exclude_pert: + return False + else: + return True + +def parse_combo_pert(i): + """ + Parsing combined perturbation strings + + [Format] + - "GENE1+GENE2": perturb two genes simultaneously + + [return] + (gene1, gene2) tuple + """ + return i.split('+')[0], i.split('+')[1] + +def combine_res(res_1, res_2): + """Merge two evaluation result dictionaries""" + res_out = {} + for key in res_1: + res_out[key] = np.concatenate([res_1[key], res_2[key]]) + return res_out + +def parse_any_pert(p): + """ + Parse arbitrary perturbation strings (single genes or combinations) + + [return] + Gene list: + - Single gene perturbation: [gene] + - Combinatorial perturbation: [gene1, gene2] + """ + if ('ctrl' in p) and (p != 'ctrl'): + return [parse_single_pert(p)] + elif 'ctrl' not in p: + out = parse_combo_pert(p) + return [out[0], out[1]] + +def np_pearson_cor(x, y): + """ + Calculate Pearson correlation coefficient matrix + + [use] + Used to construct co-expressed gene relationship diagrams + """ + xv = x - x.mean(axis=0) + yv = y - y.mean(axis=0) + xvss = (xv * xv).sum(axis=0) + yvss = (yv * yv).sum(axis=0) + result = np.matmul(xv.transpose(), yv) / np.sqrt(np.outer(xvss, yvss)) + return np.maximum(np.minimum(result, 1.0), -1.0) + + +def dataverse_download(url, save_path): + """dataverse download helper with progress bar + + Args: + url (str): the url of the dataset + path (str): the path to save the dataset + """ + + if os.path.exists(save_path): + print_sys('Found local copy...') + else: + print_sys("Downloading...") + response = requests.get(url, stream=True) + total_size_in_bytes= int(response.headers.get('content-length', 0)) + block_size = 1024 + progress_bar = tqdm(total=total_size_in_bytes, unit='iB', unit_scale=True) + with open(save_path, 'wb') as file: + for data in response.iter_content(block_size): + progress_bar.update(len(data)) + file.write(data) + progress_bar.close() + + +def zip_data_download_wrapper(url, save_path, data_path): + + if os.path.exists(save_path): + print_sys('Found local copy...') + else: + dataverse_download(url, save_path + '.zip') + print_sys('Extracting zip file...') + with ZipFile((save_path + '.zip'), 'r') as zip: + zip.extractall(path = data_path) + print_sys("Done!") + +def get_go_auto(gene_list, data_path, data_name, gene2go_path=None): + go_path = os.path.join(data_path, data_name, 'go.csv') + + if os.path.exists(go_path): + return pd.read_csv(go_path) + else: + # This optional annotation resource is not bundled with the project. + gene2go_path = gene2go_path or os.environ.get('BIOMOD_GENE2GO_PATH') + if not gene2go_path or not os.path.isfile(gene2go_path): + raise FileNotFoundError( + f"GO network not found: {go_path}. Provide gene2go_path or set " + "BIOMOD_GENE2GO_PATH to a gene2go_all.pkl annotation file to regenerate it." + ) + with open(gene2go_path, 'rb') as f: + gene2go = pickle.load(f) + + gene2go = {i: list(gene2go[i]) for i in gene_list if i in gene2go} + edge_list = [] + for g1 in tqdm(gene2go.keys()): + for g2 in gene2go.keys(): + edge_list.append((g1, g2, len(np.intersect1d(gene2go[g1], gene2go[g2]))/len(np.union1d(gene2go[g1], gene2go[g2])))) + + edge_list_filter = [i for i in edge_list if i[2] > 0] + further_filter = [i for i in edge_list if i[2] > 0.1] + df_edge_list = pd.DataFrame(further_filter).rename(columns = {0: 'gene1', 1: 'gene2', 2: 'score'}) + + df_edge_list = df_edge_list.rename(columns = {'gene1': 'source', 'gene2': 'target', 'score': 'importance'}) + + df_edge_list.to_csv(go_path, index = False) + return df_edge_list + +def get_go(df_gene2go): + df_gene2go['Entry name'] = df_gene2go['Entry name'].apply(lambda x: x.split('_')[0]) + df_gene2go = df_gene2go[df_gene2go['Gene ontology IDs'].notnull()] + df_gene2go = df_gene2go.rename(columns = {[i for i in df_gene2go.columns.values if 'yourlist' in i][0]: 'gene_id'}) + + geneid2go = dict(df_gene2go[['gene_id', 'Gene ontology IDs']].values) + + gene2go = {} + for i,j in geneid2go.items(): + j = [k.strip() for k in j.split(';')] + for k in i.split(','): + gene2go[ensembl2genename[k]] = j + + from tqdm import tqdm + edge_list = [] + for g1 in tqdm(gene2go.keys()): + for g2 in gene2go.keys(): + + edge_list.append((g1, g2, len(np.intersect1d(gene2go[g1], gene2go[g2]))/len(np.union1d(gene2go[g1], gene2go[g2])))) + + edge_list_filter = [i for i in edge_list if i[2] > 0] + further_filter = [i for i in edge_list if i[2] > 0.1] + df_edge_list = pd.DataFrame(further_filter).rename(columns = {0: 'gene1', 1: 'gene2', 2: 'score'}) + + df_edge_list = df_edge_list.rename(columns = {'gene1': 'source', 'gene2': 'target', 'score': 'importance'}) + + return df_edge_list + +class GeneSimNetwork(): + ''' + Represents a gene similarity network. Has the following attributes: + - self.edge_list + - self.gene_list + - self.G + - self.edge_index (Tensor, can be used to set G_go in model_initialize) + - self.edge_weight (Tensor, can be used to set G_go_weight in model_initialize) + ''' + def __init__(self, edge_list, gene_list, node_map): + # Keeps edges if both source and target exist in node_map + keep_edges = np.zeros(len(edge_list)).astype(bool) + for i, (source, target, importance) in edge_list.iterrows(): + keep_edges[i] = source in node_map and target in node_map + self.edge_list = edge_list[keep_edges] + + # Constructs NetworkX graph + self.G = nx.from_pandas_edgelist(self.edge_list, source='source', + target='target', edge_attr=['importance'], + create_using=nx.DiGraph()) + self.gene_list = gene_list + + # Add any isolated nodes (represented in the tensor with self edges) + for n in self.gene_list: + added_nodes = [] + if n not in self.G.nodes(): + self.G.add_node(n) + added_nodes.append(n) + if len(added_nodes) > 0: + print('The following nodes were not present in the edge list and have been added to the graph as self-edges:', added_nodes) + + # Convert graph edges to tensor + edge_index_ = [(node_map[e[0]], node_map[e[1]]) for e in self.G.edges] + self.edge_index = torch.tensor(edge_index_, dtype=torch.long).T + + # Convert edge weights to tensor + edge_attr = nx.get_edge_attributes(self.G, 'importance') + importance = np.array([edge_attr[e] for e in self.G.edges]) + self.edge_weight = torch.Tensor(importance) + + +def get_similarity_network(network_type, adata, threshold, k, gene_list, data_path, data_name, split, seed, train_gene_set_size, set2conditions): + + if network_type == 'co-express': + df_out = get_coexpression_network_from_train(adata, threshold, k, data_path, data_name, split, seed, train_gene_set_size, set2conditions) + elif network_type == 'go': + df_jaccard = get_go_auto(gene_list, data_path, data_name) + df_out = df_jaccard.groupby('target').apply(lambda x: x.nlargest(k + 1,['importance'])).reset_index(drop = True) + + return df_out + +def get_coexpression_network_from_train(adata, threshold, k, data_path, data_name, split, seed, train_gene_set_size, set2conditions): + + fname = os.path.join(os.path.join(data_path, data_name), split + '_' + str(seed) + '_' + str(train_gene_set_size) + '_' + str(threshold) + '_' + str(k) + '_co_expression_network.csv') + + + if os.path.exists(fname): + return pd.read_csv(fname) + else: + gene_list = [f for f in adata.var.gene_name.values] + idx2gene = dict(zip(range(len(gene_list)), gene_list)) + X = adata.X + train_perts = set2conditions['train'] + X_tr = X[np.isin(adata.obs.condition, [i for i in train_perts if 'ctrl' in i])] + gene_list = adata.var['gene_name'].values + + X_tr = X_tr.toarray() + out = np_pearson_cor(X_tr, X_tr) + out[np.isnan(out)] = 0 + out = np.abs(out) + + out_sort_idx = np.argsort(out)[:, -(k + 1):] + out_sort_val = np.sort(out)[:, -(k + 1):] + + df_g = [] + for i in range(out_sort_idx.shape[0]): + target = idx2gene[i] + for j in range(out_sort_idx.shape[1]): + df_g.append((idx2gene[out_sort_idx[i, j]], target, out_sort_val[i, j])) + + df_g = [i for i in df_g if i[2] > threshold] + + df_co_expression = pd.DataFrame(df_g).rename(columns = {0: 'source', 1: 'target', 2: 'importance'}) + df_co_expression.to_csv(fname, index = False) + return df_co_expression + + +def uncertainty_loss_fct(pred, logvar, y, perts, reg = 0.1, ctrl = None, direction_lambda = 1e-3, dict_filter = None): + """ + Uncertain version loss: + 1) The error term is weighted by logvar (heteroscedastic modeling) + 2) Optional direction penalty (relative to the up/down direction of control) + 3) `dict_filter` is only calculated on a non-zero gene subset, and the caliber is consistent with the evaluation + """ + gamma = 2 + perts = np.array(perts) + losses = torch.tensor(0.0, requires_grad=True).to(pred.device) + for p in set(perts): + if p!= 'ctrl': + # Training only on non-zero genes of this perturbation can reduce the impact of dropout noise. + retain_idx = dict_filter[p] + pred_p = pred[np.where(perts==p)[0]][:, retain_idx] + y_p = y[np.where(perts==p)[0]][:, retain_idx] + logvar_p = logvar[np.where(perts==p)[0]][:, retain_idx] + else: + pred_p = pred[np.where(perts==p)[0]] + y_p = y[np.where(perts==p)[0]] + logvar_p = logvar[np.where(perts==p)[0]] + + # uncertainty based loss + + losses += torch.sum((pred_p - y_p)**(2 + gamma) + reg * torch.exp(-logvar_p) * (pred_p - y_p)**(2 + gamma))/pred_p.shape[0]/pred_p.shape[1] + + # direction loss + if p!= 'ctrl': + # Directional consistency term: The constraint prediction is consistent with the true direction of change relative to control. + losses += torch.sum(direction_lambda * (torch.sign(y_p - ctrl[retain_idx]) - torch.sign(pred_p - ctrl[retain_idx]))**2)/pred_p.shape[0]/pred_p.shape[1] + else: + losses += torch.sum(direction_lambda * (torch.sign(y_p - ctrl) - torch.sign(pred_p - ctrl))**2)/pred_p.shape[0]/pred_p.shape[1] + + return losses/(len(set(perts))) + +def loss_fct(pred, y, perts, ctrl = None, direction_lambda = 1e-3, dict_filter = None): + """ + Main training loss (without uncertainty): + - Basic item: MSE + - Optional direction item: Penalize upward/downward direction errors + - `dict_filter` will change the training gene set and is an important switch that affects the results. + """ + gamma = 2 + mse_p = torch.nn.MSELoss() + perts = np.array(perts) + losses = torch.tensor(0.0).to(pred.device) + + for p in set(perts): + pert_idx = np.where(perts == p)[0] + + + # during training, we remove the all zero genes into calculation of loss. this gives a cleaner direction loss. empirically, the performance stays the same. + + # during training, we remove the all zero genes into calculation of loss. + # this gives a cleaner direction loss. empirically, the performance stays the same. + + if p!= 'ctrl': + # Generally more stable when consistent with the evaluated non-zero / non-dropout settings. + retain_idx = dict_filter[p] + pred_p = pred[pert_idx][:, retain_idx] + y_p = y[pert_idx][:, retain_idx] + else: + pred_p = pred[pert_idx] + y_p = y[pert_idx] + + + losses += torch.sum((pred_p - y_p)**(2 + gamma))/pred_p.shape[0]/pred_p.shape[1] + + ## direction loss + if (p!= 'ctrl'): + losses += torch.sum(direction_lambda * (torch.sign(y_p - ctrl[retain_idx]) - torch.sign(pred_p - ctrl[retain_idx]))**2)/pred_p.shape[0]/pred_p.shape[1] + else: + losses += torch.sum(direction_lambda * (torch.sign(y_p - ctrl) - torch.sign(pred_p - ctrl))**2)/pred_p.shape[0]/pred_p.shape[1] + + return losses/(len(set(perts))) + + +def print_sys(s): + """system print + + Args: + s (str): the string to print + """ + print(s, flush = True, file = sys.stderr) + +def create_cell_graph_for_prediction(X, pert_idx, pert_gene): + + + # If perturbations will be represented as node features + pert_feats = np.zeros(len(X)) + for p in pert_idx: + pert_feats[int(np.abs(p))] = np.sign(p) + feature_mat = torch.Tensor(np.vstack([X, pert_feats])).T + + return Data(x=feature_mat, pert=pert_gene) + + +def create_cell_graph_dataset_for_prediction(pert_gene, ctrl_adata, gene_names, device, num_samples = 300): + Xs = [] + + # Get the indices (and signs) of applied perturbation + pert_idx = [np.where(p == np.array(gene_names))[0][0] for p in pert_gene] + + Xs = ctrl_adata[np.random.randint(0, len(ctrl_adata), num_samples), :].X.toarray() + # Create cell graphs + cell_graphs = [create_cell_graph_for_prediction(X, pert_idx, pert_gene).to(device) for X in Xs] + return cell_graphs + +## +##GI related utils +## + +def get_coeffs(singles_expr, first_expr, second_expr, double_expr): + # ============================================================ + # Lazy loading of dcor to avoid numba conflicts when importing into multiple processes + # Import dcor only when the function is called + # ============================================================ + try: + from dcor import distance_correlation + except Exception as e: + # Use alternative if dcor import fails (numba compatibility issue) + import warnings + warnings.warn(f"dcor import failed: {e}. Using fallback distance correlation implementation.") + # Use a simplified scipy/numpy based implementation as an alternative + def distance_correlation(x, y): + """Simplified version of distance correlation coefficient, does not rely on numba""" + from scipy.spatial.distance import pdist, squareform + x = np.asarray(x) + y = np.asarray(y) + a = squareform(pdist(x)) + b = squareform(pdist(y)) + A = a - a.mean(axis=0)[None, :] - a.mean(axis=1)[:, None] + a.mean() + B = b - b.mean(axis=0)[None, :] - b.mean(axis=1)[:, None] + b.mean() + dcov2 = (A * B).sum() / (A.size * A.size) + dvar_x = (A * A).sum() / (A.size * A.size) + dvar_y = (B * B).sum() / (B.size * B.size) + if dvar_x == 0 or dvar_y == 0: + return 0.0 + return np.sqrt(dcov2 / np.sqrt(dvar_x * dvar_y)) + + results = {} + results['ts'] = TheilSenRegressor(fit_intercept=False, + max_subpopulation=1e5, + max_iter=1000, + random_state=1000) + X = singles_expr + y = double_expr + results['ts'].fit(X, y.ravel()) + Zts = results['ts'].predict(X) + results['c1'] = results['ts'].coef_[0] + results['c2'] = results['ts'].coef_[1] + results['mag'] = np.sqrt((results['c1']**2 + results['c2']**2)) + + results['dcor'] = distance_correlation(singles_expr, double_expr) + results['dcor_singles'] = distance_correlation(first_expr, second_expr) + results['dcor_first'] = distance_correlation(first_expr, double_expr) + results['dcor_second'] = distance_correlation(second_expr, double_expr) + results['corr_fit'] = np.corrcoef(Zts.flatten(), double_expr.flatten())[0,1] + results['dominance'] = np.abs(np.log10(results['c1']/results['c2'])) + results['eq_contr'] = np.min([results['dcor_first'], results['dcor_second']])/\ + np.max([results['dcor_first'], results['dcor_second']]) + + return results + +def get_GI_params(preds, combo): + + singles_expr = np.array([preds[combo[0]], preds[combo[1]]]).T + first_expr = np.array(preds[combo[0]]).T + second_expr = np.array(preds[combo[1]]).T + double_expr = np.array(preds[combo[0]+'_'+combo[1]]).T + + return get_coeffs(singles_expr, first_expr, second_expr, double_expr) + +def get_GI_genes_idx(adata, GI_gene_file): + # Genes used for linear model fitting + GI_genes = np.load(GI_gene_file, allow_pickle=True) + GI_genes_idx = np.where([g in GI_genes for g in adata.var.gene_name.values])[0] + + return GI_genes_idx + +def get_mean_control(adata): + cols = adata.var.gene_name.values.astype('str') + mean_ctrl_exp = adata[adata.obs['condition'] == 'ctrl'].to_df().mean() + return mean_ctrl_exp diff --git a/BioMod/interpretability/README.md b/BioMod/interpretability/README.md index 17a7fd6bb656c2f53d60519b64e92ccdcd2fe8c4..68267b82c51f48803b10df5c68388e7d10d8b9a6 100644 --- a/BioMod/interpretability/README.md +++ b/BioMod/interpretability/README.md @@ -1,413 +1,361 @@ -# Module Interpretability Experiments for BioMod - -This directory contains analyses of BioMod module representations, covering -latent geometry, pathway-text alignment, module importance, and attention. -These workflows examine structural, semantic, and functional properties of the -learned modules. Use the -[BioMod model weights and matching configuration](../../README.md#biomod-model-weights) -when loading the pretrained backbone. - -## Experiment Roadmap, Ordered by Priority - -| # | Script | Core question | Status | -| --- | --- | --- | --- | -| 1 | `module_geometry.py` | Does the module latent space self-organize by pathway family, | Completed | -| 2 | `module_text_alignment/module_text_alignment.py` | Do module embeddings align with natural-language pathway descriptions, | Implemented | -| 3 | `module_importance/module_importance.py` | Does task-driven module importance match known pathway-cell-type or pathway-disease associations, | Implemented | -| 4 | `module_vs_gene.py` | Is BioMod better than GenePT/scGPT for pathway-level interpretability, | TODO | -| 6 | `attention/module_crosstalk.py` | Do module interactions match Reactome/KEGG pathway relationships, | Implemented | - -Experiments 1 and 2 are the core differentiators because gene-level baselines -cannot structurally reproduce them. Experiment 3 provides task-level functional -validation. Experiment 4 provides direct comparison to baselines. Additional -experiments, such as module crosstalk vs Reactome, module dynamics along -developmental trajectories, and retraining-based module-granularity ablation, can -be pursued after the core experiments are complete. - -## Experiment 1: Intrinsic Geometry of Module Embeddings (`module_geometry.py`) - -**Hypothesis:** the BioMod encoder is trained with a masked-module -reconstruction task, but it should still encode semantically related modules, -such as modules from the same pathway family or MSigDB hallmark biological -process, into nearby regions of latent space, even though family labels are -never seen during training. - -**Workflow:** - -1. Load the pretrained backbone and run inference with `mask_ratio=0.0` and `return_encodings=True` to obtain module encodings shaped `[B, num_modules, base_dim]`. -2. Average over the cell population to obtain `[num_modules, base_dim]`, using streaming accumulation rather than storing the full table in memory. -3. Parse family assignments from the keys in `merged_modules.json`: `Hallmarks` (50), `Reactome` (491), `Hotspot` (73), and `Processed` (46). Hallmark modules are also annotated with one of eight MSigDB biological-process categories: immune, metabolic, signaling, proliferation, development, dna_damage, pathway, and cellular_component. -4. Quantify family separation: - - silhouette coefficient, using both cosine and Euclidean metrics. - - kNN purity at `k=10`, using both cosine and Euclidean metrics. - - ratio of mean inter-family to intra-family distance. -5. Compare to a permutation null distribution by shuffling family labels `K=200` times, recomputing metrics, and reporting the observed value, null mean, null standard deviation, and right-tail empirical p-value. -6. Visualize with UMAP and PCA in 2D, colored by family or hallmark biological process. - -**Differentiation from gene-level baselines:** gene-level foundation models embed -single genes rather than pathway groups. To reproduce this experiment, they would -first have to manually aggregate gene embeddings by pathway, which compresses -gene-level signals and introduces a hand-designed step. BioMod learns this -aggregation end to end during pretraining. - -## File Structure - -```text -BioMod/interpretability/ # Module representations and biological interpretability -├── attention/ # Module crosstalk and within-module gene attention -│ ├── __init__.py -│ ├── module_crosstalk.py -│ ├── module_gene_attention.py -│ ├── README.md -│ └── README_gene_attention.md -├── module_importance/ # Module importance, ablation studies and marker-gene validation -│ ├── __init__.py -│ ├── cross_class_similarity.py -│ ├── family_ablation.py -│ ├── family_importance_analysis.py -│ ├── marker_gene_verification.py -│ ├── masking_ablation.py -│ ├── module_importance.py -│ ├── module_importance_publication.py -│ ├── pub_utils.py -│ ├── random_baseline.py -│ ├── README.md -│ ├── RESULTS.md -│ ├── tcell_reactome_analysis.py -│ └── TODO.md -├── module_text_alignment/ # Semantic alignment of module representations and pathway text -│ ├── __init__.py -│ ├── DESIGN.md -│ ├── fetch_reactome_descriptions.py -│ ├── MANUSCRIPT_SECTION.md -│ ├── module_text_alignment.py -│ ├── module_text_alignment_extended.py -│ ├── module_text_alignment_publication.py -│ └── reactome_descriptions.json -├── __init__.py -├── module_geometry.py -├── README.md -└── utils.py -``` - -## Command-Line Usage - -Run scripts from `BioMod/interpretability/` within the project root; relative paths below are -based on that directory. - -```bash -# General h5ad input: cell annotation, zero-shot, or any annotated dataset. -python module_geometry.py \ - --performer_checkpoint ../mae_tost_48M_cross_attn_only_pure_memory/mae_tost_48M_cross_attn_only_pure_memory_epoch12.pth \ - --config_pkl ../mae_tost_48M_cross_attn_only_pure_memory/mae_tost_48M_cross_attn_only_pure_memory.pkl \ - --module_data_dir ../module_data_all \ - --gene_list_path ../../selected_genes_27k.txt \ - --gene_mapping_path ../../gene2ens.csv \ - --gene2vec_data_path ../../selected_gene2vec_27k.npy \ - --gene2ens_file ../../gene2ens.csv \ - --data_path ../../zero-shot-batch/pancreas_scib.h5ad \ - --data_mode h5ad \ - --gene_name_column auto \ - --batch_size 640 \ - --output_dir ./results/geometry/pancreas_scib -``` - -For pretraining data, where each cell is an `.npz` file already in the 27,874 -model-gene space: - -```bash -python module_geometry.py \ - --performer_checkpoint ../mae_tost_48M_cross_attn_only_pure_memory/mae_tost_48M_cross_attn_only_pure_memory_epoch12.pth \ - --config_pkl ../mae_tost_48M_cross_attn_only_pure_memory/mae_tost_48M_cross_attn_only_pure_memory.pkl \ - --module_data_dir ../module_data_all \ - --gene_list_path ../../selected_genes_27k.txt \ - --gene_mapping_path ../../gene2ens.csv \ - --gene2vec_data_path ../../selected_gene2vec_27k.npy \ - --data_path ../../../../share/cell_files \ - --data_mode pretrain_npz \ - --cell_num 48024242 \ - --max_cells 50000 \ - --batch_size 512 \ - --output_dir ./results/geometry/pretrain_subset -``` - -### Multi-GPU DDP Acceleration - -`torchrun` multi-GPU inference is supported. When `WORLD_SIZE > 1`, each GPU -processes a subset of cells, embeddings are aggregated with `all_reduce`, and -rank 0 computes metrics and figures. Single-GPU and CPU behavior is unchanged. - -```bash -torchrun --nproc_per_node=2 module_geometry.py \ - --performer_checkpoint ../mae_tost_48M_cross_attn_only_pure_memory/mae_tost_48M_cross_attn_only_pure_memory_epoch2.pth \ - --config_pkl ../mae_tost_48M_cross_attn_only_pure_memory/mae_tost_48M_cross_attn_only_pure_memory.pkl \ - --module_data_dir ../module_data_all \ - --gene_list_path ../../selected_genes_27k.txt \ - --gene_mapping_path ../../gene2ens.csv \ - --gene2vec_data_path ../../selected_gene2vec_27k.npy \ - --data_path ../../../share/cell_files \ - --data_mode pretrain_npz \ - --cell_num 49738024 \ - --max_cells 100000 \ - --batch_size 16 \ - --output_dir ./results/geometry/pretrain_ddp -``` - -### Common Arguments - -| Argument | Default | Description | -| --- | --- | --- | -| `--max_cells` | None | Number of cells to subsample; the mean is stable from a few thousand cells onward. | -| `--cell_num` | None | Total cell count in `pretrain_npz` mode, used for sequential names `cell0.npz` to `cellN.npz`. | -| `--knn_k` | 10 | `k` for kNN purity. | -| `--null_permutations` | 200 | Number of label permutations for the empirical null distribution. | -| `--include_other_in_metrics` | Off | By default, modules from the `Other` family are excluded. | -| `--shuffle_module_genes` | Off | Randomly shuffle gene-to-module assignments as a negative control or ablation. Uses the same flag as pretraining and downstream finetuning. | -| `--umap_n_neighbors` | 15 | UMAP graph-construction parameter. | -| `--umap_min_dist` | 0.3 | UMAP layout parameter. | - -## Output Files - -Files are written to `/`: - -| File | Content | -| --- | --- | -| `module_embeddings.npy` | Cell-averaged module embeddings with shape `[num_modules, base_dim]` | -| `module_metadata.csv` | Metadata for each module: `module_name`, `family`, and `hallmark_process` | -| `metrics.json` | Observed metrics, null mean/std, and right-tail p-values for family-level and hallmark-process-level analyses | -| `pca_family.png` | 2D PCA colored by family | -| `umap_family.png` | 2D UMAP colored by family; requires `umap-learn` | -| `pca_hallmark_process.png` | PCA for Hallmark modules only, colored by MSigDB biological process | -| `umap_hallmark_process.png` | UMAP for Hallmark modules only, colored by MSigDB biological process | -| `run.log` | Full log with formatted core metrics | - -## Recommended Negative Controls - -Two control experiments make Experiment 1 more rigorous: - -1. **Random module assignment**: rerun with `--shuffle_module_genes`. This flag preserves module sizes but randomly assigns genes to modules, matching the same ablation flag used for pretraining and finetuning. If geometry is pathway-driven rather than a generic property of the encoder, family separation should drop close to the null distribution. - -2. **Untrained backbone**: set `--performer_checkpoint` to the `*_init.pth` checkpoint generated during pretraining with `--save_init_ckpt_and_exit`. Family separation should be close to the null distribution. The gap between trained and untrained models quantifies the contribution of pretraining. - -Together, these controls support the claim that the family structure of the module -latent space is produced by pathway-based module design plus pretraining, rather -than by module design alone or by a generic encoder architecture. - -### Control 1: Random Module Assignment (`shuffle_module_genes`) - -Add `--shuffle_module_genes` to the original command and write to a control output -directory. - -For h5ad data, using `pancreas_scib` as an example: - -```bash -python module_geometry.py \ - --performer_checkpoint ../mae_tost_48M_cross_attn_only_pure_memory/mae_tost_48M_cross_attn_only_pure_memory_epoch12.pth \ - --config_pkl ../mae_tost_48M_cross_attn_only_pure_memory/mae_tost_48M_cross_attn_only_pure_memory.pkl \ - --module_data_dir ../module_data_all \ - --gene_list_path ../../selected_genes_27k.txt \ - --gene_mapping_path ../../gene2ens.csv \ - --gene2vec_data_path ../../selected_gene2vec_27k.npy \ - --gene2ens_file ../../gene2ens.csv \ - --data_path ../../zero-shot-batch/pancreas_scib.h5ad \ - --data_mode h5ad \ - --gene_name_column auto \ - --batch_size 640 \ - --shuffle_module_genes \ - --output_dir ./results/geometry/pancreas_scib_shuffle -``` - -For pretraining npz data: - -```bash -python module_geometry.py \ - --performer_checkpoint ../mae_tost_48M_cross_attn_only_pure_memory/mae_tost_48M_cross_attn_only_pure_memory_epoch12.pth \ - --config_pkl ../mae_tost_48M_cross_attn_only_pure_memory/mae_tost_48M_cross_attn_only_pure_memory.pkl \ - --module_data_dir ../module_data_all \ - --gene_list_path ../../selected_genes_27k.txt \ - --gene_mapping_path ../../gene2ens.csv \ - --gene2vec_data_path ../../selected_gene2vec_27k.npy \ - --gene2ens_file ../../gene2ens.csv \ - --data_path ../../../../share/cell_files \ - --data_mode pretrain_npz \ - --cell_num 48024242 \ - --max_cells 50000 \ - --batch_size 640 \ - --shuffle_module_genes \ - --output_dir ./results/geometry/pretrain_subset_shuffle -``` - -How `--shuffle_module_genes` works: inside `load_backbone()`, module sizes are -kept unchanged, all genes are collected into one list, the list is randomly -shuffled, and genes are reassigned to modules. The seed is controlled by -`--shuffle_seed`, defaulting to 2021. Model weights are unchanged; only the -gene-to-module mapping is shuffled. - -**Expected result:** kNN purity should drop to about 0.576, the null level, and -the inter-/intra-family distance ratio should drop to about 1.0. `metrics.json` -records `"shuffle_module_genes": true`. - -### Control 2: Untrained Backbone with Random Initialization - -First generate a randomly initialized checkpoint, then run `module_geometry.py` -with it. - -**Step 1: Generate `_init.pth`** - -Use the pretraining script with `--save_init_ckpt_and_exit`. The command -initializes the model, saves a random-weight checkpoint and the matching config -PKL, then exits immediately without entering training. - -```bash -cd /mnt/c/Users/250010166/Desktop/single_cell/scLong/performer_module - -torchrun --nproc_per_node=1 performer_module.py \ - --model_name mae_tost_48M_cross_attn_only_pure_memory \ - --ckpt_dir ./mae_tost_48M_cross_attn_only_pure_memory \ - --gene_list_path ../selected_genes_27k.txt \ - --gene2vec_data_path ../selected_gene2vec_27k.npy \ - --module_data_dir module_data_all \ - --gene_mapping_path ../gene2ens.csv \ - --data_path ../../../share/cell_files \ - --cell_num 48024242 \ - --enc_depth 32 \ - --enc_heads 16 \ - --base_dim 1024 \ - --pretrain_arch mae \ - --decoder_type cross_attn \ - --save_init_ckpt_and_exit -``` - -The following files are generated under -`mae_tost_48M_cross_attn_only_pure_memory/`: - -- `mae_tost_48M_cross_attn_only_pure_memory_init.pth`, randomly initialized weights. -- `mae_tost_48M_cross_attn_only_pure_memory_init.pkl`, model config. - -**Step 2: Run `module_geometry.py` with `_init.pth`** - -For h5ad data: - -```bash -cd /mnt/c/Users/250010166/Desktop/single_cell/scLong/performer_module/interpretability - -python module_geometry.py \ - --performer_checkpoint ../mae_tost_48M_cross_attn_only_pure_memory_init/mae_tost_48M_cross_attn_only_pure_memory_init.pth \ - --config_pkl ../mae_tost_48M_cross_attn_only_pure_memory_init/mae_tost_48M_cross_attn_only_pure_memory_init.pkl \ - --module_data_dir ../module_data_all \ - --gene_list_path ../../selected_genes_27k.txt \ - --gene_mapping_path ../../gene2ens.csv \ - --gene2vec_data_path ../../selected_gene2vec_27k.npy \ - --gene2ens_file ../../gene2ens.csv \ - --data_path ../../zero-shot-batch/pancreas_scib.h5ad \ - --data_mode h5ad \ - --gene_name_column auto \ - --batch_size 640 \ - --output_dir ./results/geometry/pancreas_scib_init -``` - -For pretraining npz data: - -```bash -python module_geometry.py \ - --performer_checkpoint ../mae_tost_48M_cross_attn_only_pure_memory_init/mae_tost_48M_cross_attn_only_pure_memory_init.pth \ - --config_pkl ../mae_tost_48M_cross_attn_only_pure_memory_init/mae_tost_48M_cross_attn_only_pure_memory_init.pkl \ - --module_data_dir ../module_data_all \ - --gene_list_path ../../selected_genes_27k.txt \ - --gene_mapping_path ../../gene2ens.csv \ - --gene2vec_data_path ../../selected_gene2vec_27k.npy \ - --gene2ens_file ../../gene2ens.csv \ - --data_path ../../../../share/cell_files \ - --data_mode pretrain_npz \ - --cell_num 48024242 \ - --max_cells 50000 \ - --batch_size 640 \ - --output_dir ./results/geometry/pretrain_subset_init -``` - -**Expected result:** all family-level metrics should be close to the null -distribution: kNN purity about 0.576, distance ratio about 1.0, and p-value -above 0.05. This indicates that a randomly initialized encoder cannot separate -pathway families. - -### Result Comparison Matrix - -After running all controls, construct the following table for the paper: - -| Condition | kNN purity, cosine | Inter-/intra-family ratio, cosine | Conclusion | -| --- | --- | --- | --- | -| **Normal**, trained + pathway module | about 0.72, p<0.005 | 3.8-6.2, p<0.005 | Significant family structure | -| **Shuffle**, trained + random module | about 0.576 expected | about 1.0 expected | Structure is driven by pathway design | -| **Init**, untrained + pathway module | about 0.576 expected | about 1.0 expected | Structure is learned during pretraining | - -The three experiments jointly show that the family structure of the module latent -space is generated by both pathway-based module design and pretraining; both are -necessary. - -## Experiment 2: Module Embedding to Pathway Text Alignment (`module_text_alignment/`) - -**Hypothesis:** the structure of the BioMod module embedding space significantly -aligns with natural-language pathway semantics. Semantically similar pathways -are also encoded nearby by the model. Gene-level models cannot reproduce this -experiment because individual gene names do not carry phrase-level semantics. - -**Method:** - -1. Parse 541 semantically named modules from `merged_modules.json`: 50 Hallmark and 491 Reactome. -2. Encode pathway names with S-PubMedBert-MS-MARCO, a sentence-level fine-tuned PubMedBERT model. -3. Load `module_embeddings.npy` from Experiment 1, containing cell-averaged module embeddings. -4. Measure alignment: - - **RSA / Mantel test**, the main metric: Spearman correlation between paired distance matrices from the two spaces, with a permutation test. - - **CKA**: global representation-structure alignment. - - **LOO Ridge retrieval**: cross-modal retrieval recall@k. - -**Data requirement:** only `module_embeddings.npy` from Experiment 1 is needed; -raw expression data are not required. - -### Commands - -```bash -cd performer_module/interpretability/module_text_alignment - -# Download the sentence encoder. -wget https://hf-mirror.com/hfd/hfd.sh && chmod a+x hfd.sh && export HF_ENDPOINT=https://hf-mirror.com -./hfd.sh pritamdeka/S-PubMedBert-MS-MARCO --local-dir ./S-PubMedBert-MS-MARCO - -# Main experiment: RSA + CKA + LOO retrieval. -python module_text_alignment.py \ - --module_embedding_path ../results/geometry/pretrain_subset/module_embeddings.npy \ - --module_json ../../module_data_all/merged_modules.json \ - --sentence_model ./S-PubMedBert-MS-MARCO \ - --n_permutations 1000 \ - --output_dir ../results/text_alignment/pretrain_subset_v2 - -# Extended analysis: broad Reactome categories and Hallmark deep dive. -python module_text_alignment_extended.py \ - --module_embedding_path ../results/geometry/pretrain_subset/module_embeddings.npy \ - --module_json ../../module_data_all/merged_modules.json \ - --sentence_model ./S-PubMedBert-MS-MARCO \ - --descriptions_file ./reactome_descriptions.json \ - --n_permutations 1000 \ - --output_dir ../results/text_alignment/pretrain_subset_v2_extended -``` - -### Output Files - -| File | Content | -| --- | --- | -| `alignment_metrics.json` | RSA, CKA, retrieval recall@k, and p-values | -| `rsa_scatter.png` | Paired-distance scatter plot, module space vs text space | -| `rsa_null_distribution.png` | Mantel test: observed value vs null distribution | -| `retrieval_ranks.png` | Histogram of retrieval ranks | -| `retrieval_examples.csv` | Top-5 retrieval results for representative pathways | -| `extended_metrics.json` | Extended-analysis results: broad-category RSA and Hallmark deep dive | -| `reactome_category_heatmaps.png` | Distance-matrix comparison at the broad Reactome category level | -| `hallmarks_deep_dive.png` | Hallmark per-module alignment scores and similarity matrix | -| `hallmarks_per_module_alignment.csv` | Per-module alignment ranking for 50 Hallmarks | - -## Dependencies - -`torch`, `numpy`, `pandas`, `scikit-learn`, `matplotlib`, `anndata`, and `scipy`. -UMAP plots additionally require `umap-learn`; if it is not installed, the script -warns and skips only UMAP plots. Experiment 2 also requires -`sentence-transformers`, installable with `pip install sentence-transformers`. +# Module Interpretability Experiments for BioMod + +This directory contains analyses of BioMod module representations, covering +latent geometry, pathway-text alignment, module importance, and attention. +These workflows examine structural, semantic, and functional properties of the +learned modules. Use the +[BioMod model weights and matching configuration](../../README.md#biomod-model-weights) +when loading the pretrained backbone. + +## Experiment Roadmap, Ordered by Priority + +| # | Script | Core question | Status | +| --- | --- | --- | --- | +| 1 | `module_geometry.py` | Does the module latent space self-organize by pathway family, | Completed | +| 2 | `module_text_alignment/module_text_alignment.py` | Do module embeddings align with natural-language pathway descriptions, | Implemented | +| 3 | `module_importance/module_importance.py` | Does task-driven module importance match known pathway-cell-type or pathway-disease associations, | Implemented | +| 4 | `module_vs_gene.py` | Is BioMod better than GenePT/scGPT for pathway-level interpretability, | TODO | +| 6 | `attention/module_crosstalk.py` | Do module interactions match Reactome/KEGG pathway relationships, | Implemented | + +Experiments 1 and 2 are the core differentiators because gene-level baselines +cannot structurally reproduce them. Experiment 3 provides task-level functional +validation. Experiment 4 provides direct comparison to baselines. Additional +experiments, such as module crosstalk vs Reactome, module dynamics along +developmental trajectories, and retraining-based module-granularity ablation, can +be pursued after the core experiments are complete. + +## Experiment 1: Intrinsic Geometry of Module Embeddings (`module_geometry.py`) + +**Hypothesis:** the BioMod encoder is trained with a masked-module +reconstruction task, but it should still encode semantically related modules, +such as modules from the same pathway family or MSigDB hallmark biological +process, into nearby regions of latent space, even though family labels are +never seen during training. + +**Workflow:** + +1. Load the pretrained backbone and run inference with `mask_ratio=0.0` and `return_encodings=True` to obtain module encodings shaped `[B, num_modules, base_dim]`. +2. Average over the cell population to obtain `[num_modules, base_dim]`, using streaming accumulation rather than storing the full table in memory. +3. Parse family assignments from the keys in `merged_modules.json`: `Hallmarks` (50), `Reactome` (491), `Hotspot` (73), and `Processed` (46). Hallmark modules are also annotated with one of eight MSigDB biological-process categories: immune, metabolic, signaling, proliferation, development, dna_damage, pathway, and cellular_component. +4. Quantify family separation: + - silhouette coefficient, using both cosine and Euclidean metrics. + - kNN purity at `k=10`, using both cosine and Euclidean metrics. + - ratio of mean inter-family to intra-family distance. +5. Compare to a permutation null distribution by shuffling family labels `K=200` times, recomputing metrics, and reporting the observed value, null mean, null standard deviation, and right-tail empirical p-value. +6. Visualize with UMAP and PCA in 2D, colored by family or hallmark biological process. + +**Differentiation from gene-level baselines:** gene-level foundation models embed +single genes rather than pathway groups. To reproduce this experiment, they would +first have to manually aggregate gene embeddings by pathway, which compresses +gene-level signals and introduces a hand-designed step. BioMod learns this +aggregation end to end during pretraining. + +## File Structure + +```text +BioMod/interpretability/ # Module representations and biological interpretability +├── attention/ # Module crosstalk and within-module gene attention +│ ├── __init__.py +│ ├── module_crosstalk.py +│ ├── module_gene_attention.py +│ ├── README.md +│ └── README_gene_attention.md +├── module_importance/ # Module importance, ablation studies and marker-gene validation +│ ├── __init__.py +│ ├── cross_class_similarity.py +│ ├── family_ablation.py +│ ├── family_importance_analysis.py +│ ├── marker_gene_verification.py +│ ├── masking_ablation.py +│ ├── module_importance.py +│ ├── module_importance_publication.py +│ ├── pub_utils.py +│ ├── random_baseline.py +│ ├── README.md +│ ├── RESULTS.md +│ ├── tcell_reactome_analysis.py +│ └── TODO.md +├── module_text_alignment/ # Semantic alignment of module representations and pathway text +│ ├── __init__.py +│ ├── DESIGN.md +│ ├── fetch_reactome_descriptions.py +│ ├── MANUSCRIPT_SECTION.md +│ ├── module_text_alignment.py +│ ├── module_text_alignment_extended.py +│ ├── module_text_alignment_publication.py +│ └── reactome_descriptions.json +├── __init__.py +├── module_geometry.py +├── README.md +└── utils.py +``` + +## Command-Line Usage + +Run all commands below from the BioMod repository root. Bundled default assets +are located relative to the script files, so those defaults do not depend on the +working directory. Explicit relative CLI paths still resolve from your current +working directory. + +The h5ad examples use the bundled pancreas dataset. Pretraining `.npz` cell files +are not included: replace `/absolute/path/to/pretraining_cell_npz` with your own +prepared directory. Generated embeddings and analysis results are also not +bundled; run the corresponding upstream analysis before using those paths. + +```bash +# General h5ad input: cell annotation, zero-shot, or any annotated dataset. +python BioMod/interpretability/module_geometry.py \ + --performer_checkpoint weight/0810model/mae_tost_70M_75mask_ec2_epoch0.pth \ + --config_pkl weight/0810model/mae_tost_70M_75mask_ec2.pkl \ + --module_data_dir model_dependencies/module_data_all \ + --gene_list_path model_dependencies/selected_genes_27k.txt \ + --gene_mapping_path model_dependencies/gene2ens.csv \ + --gene2vec_data_path model_dependencies/selected_gene2vec_27k.npy \ + --gene2ens_file model_dependencies/gene2ens.csv \ + --data_path data/downstream/final_data/human_pancreas_norm_complexBatch_with_gene_name.h5ad \ + --data_mode h5ad \ + --gene_name_column auto \ + --batch_size 640 \ + --output_dir outputs/interpretability/geometry/pancreas_scib +``` + +For pretraining data, where each cell is an `.npz` file already in the 27,874 +model-gene space: + +```bash +python BioMod/interpretability/module_geometry.py \ + --performer_checkpoint weight/0810model/mae_tost_70M_75mask_ec2_epoch0.pth \ + --config_pkl weight/0810model/mae_tost_70M_75mask_ec2.pkl \ + --module_data_dir model_dependencies/module_data_all \ + --gene_list_path model_dependencies/selected_genes_27k.txt \ + --gene_mapping_path model_dependencies/gene2ens.csv \ + --gene2vec_data_path model_dependencies/selected_gene2vec_27k.npy \ + --data_path /absolute/path/to/pretraining_cell_npz \ + --data_mode pretrain_npz \ + --max_cells 50000 \ + --batch_size 512 \ + --output_dir outputs/interpretability/geometry/pretrain_subset +``` + +### Multi-GPU DDP Acceleration + +`torchrun` multi-GPU inference is supported. When `WORLD_SIZE > 1`, each GPU +processes a subset of cells, embeddings are aggregated with `all_reduce`, and +rank 0 computes metrics and figures. Single-GPU and CPU behavior is unchanged. + +```bash +torchrun --nproc_per_node=2 BioMod/interpretability/module_geometry.py \ + --performer_checkpoint weight/0810model/mae_tost_70M_75mask_ec2_epoch0.pth \ + --config_pkl weight/0810model/mae_tost_70M_75mask_ec2.pkl \ + --module_data_dir model_dependencies/module_data_all \ + --gene_list_path model_dependencies/selected_genes_27k.txt \ + --gene_mapping_path model_dependencies/gene2ens.csv \ + --gene2vec_data_path model_dependencies/selected_gene2vec_27k.npy \ + --data_path /absolute/path/to/pretraining_cell_npz \ + --data_mode pretrain_npz \ + --max_cells 100000 \ + --batch_size 16 \ + --output_dir outputs/interpretability/geometry/pretrain_ddp +``` + +### Common Arguments + +| Argument | Default | Description | +| --- | --- | --- | +| `--max_cells` | None | Number of cells to subsample; the mean is stable from a few thousand cells onward. | +| `--cell_num` | None | Total cell count in `pretrain_npz` mode, used for sequential names `cell0.npz` to `cellN.npz`. | +| `--knn_k` | 10 | `k` for kNN purity. | +| `--null_permutations` | 200 | Number of label permutations for the empirical null distribution. | +| `--include_other_in_metrics` | Off | By default, modules from the `Other` family are excluded. | +| `--shuffle_module_genes` | Off | Randomly shuffle gene-to-module assignments as a negative control or ablation. Uses the same flag as pretraining and downstream finetuning. | +| `--umap_n_neighbors` | 15 | UMAP graph-construction parameter. | +| `--umap_min_dist` | 0.3 | UMAP layout parameter. | + +## Output Files + +Files are written to `/`: + +| File | Content | +| --- | --- | +| `module_embeddings.npy` | Cell-averaged module embeddings with shape `[num_modules, base_dim]` | +| `module_metadata.csv` | Metadata for each module: `module_name`, `family`, and `hallmark_process` | +| `metrics.json` | Observed metrics, null mean/std, and right-tail p-values for family-level and hallmark-process-level analyses | +| `pca_family.png` | 2D PCA colored by family | +| `umap_family.png` | 2D UMAP colored by family; requires `umap-learn` | +| `pca_hallmark_process.png` | PCA for Hallmark modules only, colored by MSigDB biological process | +| `umap_hallmark_process.png` | UMAP for Hallmark modules only, colored by MSigDB biological process | +| `run.log` | Full log with formatted core metrics | + +## Recommended Negative Controls + +Two control experiments make Experiment 1 more rigorous: + +1. **Random module assignment**: rerun with `--shuffle_module_genes`. This flag preserves module sizes but randomly assigns genes to modules, matching the same ablation flag used for pretraining and finetuning. If geometry is pathway-driven rather than a generic property of the encoder, family separation should drop close to the null distribution. + +2. **Untrained backbone**: set `--performer_checkpoint` to the `*_init.pth` checkpoint generated during pretraining with `--save_init_ckpt_and_exit`. Family separation should be close to the null distribution. The gap between trained and untrained models quantifies the contribution of pretraining. + +Together, these controls support the claim that the family structure of the module +latent space is produced by pathway-based module design plus pretraining, rather +than by module design alone or by a generic encoder architecture. + +### Control 1: Random Module Assignment (`shuffle_module_genes`) + +Add `--shuffle_module_genes` to the original command and write to a control output +directory. + +For h5ad data, using `pancreas_scib` as an example: + +```bash +python BioMod/interpretability/module_geometry.py \ + --performer_checkpoint weight/0810model/mae_tost_70M_75mask_ec2_epoch0.pth \ + --config_pkl weight/0810model/mae_tost_70M_75mask_ec2.pkl \ + --module_data_dir model_dependencies/module_data_all \ + --gene_list_path model_dependencies/selected_genes_27k.txt \ + --gene_mapping_path model_dependencies/gene2ens.csv \ + --gene2vec_data_path model_dependencies/selected_gene2vec_27k.npy \ + --gene2ens_file model_dependencies/gene2ens.csv \ + --data_path data/downstream/final_data/human_pancreas_norm_complexBatch_with_gene_name.h5ad \ + --data_mode h5ad \ + --gene_name_column auto \ + --batch_size 640 \ + --shuffle_module_genes \ + --output_dir outputs/interpretability/geometry/pancreas_scib_shuffle +``` + +For pretraining npz data: + +```bash +python BioMod/interpretability/module_geometry.py \ + --performer_checkpoint weight/0810model/mae_tost_70M_75mask_ec2_epoch0.pth \ + --config_pkl weight/0810model/mae_tost_70M_75mask_ec2.pkl \ + --module_data_dir model_dependencies/module_data_all \ + --gene_list_path model_dependencies/selected_genes_27k.txt \ + --gene_mapping_path model_dependencies/gene2ens.csv \ + --gene2vec_data_path model_dependencies/selected_gene2vec_27k.npy \ + --gene2ens_file model_dependencies/gene2ens.csv \ + --data_path /absolute/path/to/pretraining_cell_npz \ + --data_mode pretrain_npz \ + --max_cells 50000 \ + --batch_size 640 \ + --shuffle_module_genes \ + --output_dir outputs/interpretability/geometry/pretrain_subset_shuffle +``` + +How `--shuffle_module_genes` works: inside `load_backbone()`, module sizes are +kept unchanged, all genes are collected into one list, the list is randomly +shuffled, and genes are reassigned to modules. The seed is controlled by +`--shuffle_seed`, defaulting to 2021. Model weights are unchanged; only the +gene-to-module mapping is shuffled. + +**Expected result:** kNN purity should drop to about 0.576, the null level, and +the inter-/intra-family distance ratio should drop to about 1.0. `metrics.json` +records `"shuffle_module_genes": true`. + +### Control 2: Untrained Backbone with Random Initialization + +First generate a randomly initialized checkpoint, then run `module_geometry.py` +with it. + +A matching randomly initialized checkpoint and configuration are not bundled. +Create them separately with the pretraining script's `--save_init_ckpt_and_exit` +option and architecture settings matching the BioMod checkpoint being compared. +Keep the generated control files in a separate output directory. Then pass their +paths explicitly; do not use the pretrained BioMod weights for this control. + +```bash +python BioMod/interpretability/module_geometry.py \ + --performer_checkpoint /absolute/path/to/random_init.pth \ + --config_pkl /absolute/path/to/random_init.pkl \ + --data_path data/downstream/final_data/human_pancreas_norm_complexBatch_with_gene_name.h5ad \ + --data_mode h5ad \ + --gene_name_column auto \ + --batch_size 8 \ + --output_dir outputs/interpretability/geometry/pancreas_init +``` + +**Expected result:** all family-level metrics should be close to the null +distribution: kNN purity about 0.576, distance ratio about 1.0, and p-value +above 0.05. This indicates that a randomly initialized encoder cannot separate +pathway families. + +### Result Comparison Matrix + +After running all controls, construct the following table for the paper: + +| Condition | kNN purity, cosine | Inter-/intra-family ratio, cosine | Conclusion | +| --- | --- | --- | --- | +| **Normal**, trained + pathway module | about 0.72, p<0.005 | 3.8-6.2, p<0.005 | Significant family structure | +| **Shuffle**, trained + random module | about 0.576 expected | about 1.0 expected | Structure is driven by pathway design | +| **Init**, untrained + pathway module | about 0.576 expected | about 1.0 expected | Structure is learned during pretraining | + +The three experiments jointly show that the family structure of the module latent +space is generated by both pathway-based module design and pretraining; both are +necessary. + +## Experiment 2: Module Embedding to Pathway Text Alignment (`module_text_alignment/`) + +**Hypothesis:** the structure of the BioMod module embedding space significantly +aligns with natural-language pathway semantics. Semantically similar pathways +are also encoded nearby by the model. Gene-level models cannot reproduce this +experiment because individual gene names do not carry phrase-level semantics. + +**Method:** + +1. Parse 541 semantically named modules from `merged_modules.json`: 50 Hallmark and 491 Reactome. +2. Encode pathway names with S-PubMedBert-MS-MARCO, a sentence-level fine-tuned PubMedBERT model. +3. Load `module_embeddings.npy` from Experiment 1, containing cell-averaged module embeddings. +4. Measure alignment: + - **RSA / Mantel test**, the main metric: Spearman correlation between paired distance matrices from the two spaces, with a permutation test. + - **CKA**: global representation-structure alignment. + - **LOO Ridge retrieval**: cross-modal retrieval recall@k. + +**Data requirement:** only `module_embeddings.npy` from Experiment 1 is needed; +raw expression data are not required. + +### Commands + +```bash +# Run from the BioMod repository root. + +# Download the sentence encoder. +# The sentence-transformers loader downloads this model on first use. +# Network access or an existing local model cache is required. + +# Main experiment: RSA + CKA + LOO retrieval. +python BioMod/interpretability/module_text_alignment/module_text_alignment.py \ + --module_embedding_path outputs/interpretability/geometry/pretrain_subset/module_embeddings.npy \ + --module_json model_dependencies/module_data_all/merged_modules.json \ + --sentence_model pritamdeka/S-PubMedBert-MS-MARCO \ + --n_permutations 1000 \ + --output_dir outputs/interpretability/text_alignment/pretrain_subset_v2 + +# Extended analysis: broad Reactome categories and Hallmark deep dive. +python BioMod/interpretability/module_text_alignment/module_text_alignment_extended.py \ + --module_embedding_path outputs/interpretability/geometry/pretrain_subset/module_embeddings.npy \ + --module_json model_dependencies/module_data_all/merged_modules.json \ + --sentence_model pritamdeka/S-PubMedBert-MS-MARCO \ + --descriptions_file BioMod/interpretability/module_text_alignment/reactome_descriptions.json \ + --n_permutations 1000 \ + --output_dir outputs/interpretability/text_alignment/pretrain_subset_v2_extended +``` + +### Output Files + +| File | Content | +| --- | --- | +| `alignment_metrics.json` | RSA, CKA, retrieval recall@k, and p-values | +| `rsa_scatter.png` | Paired-distance scatter plot, module space vs text space | +| `rsa_null_distribution.png` | Mantel test: observed value vs null distribution | +| `retrieval_ranks.png` | Histogram of retrieval ranks | +| `retrieval_examples.csv` | Top-5 retrieval results for representative pathways | +| `extended_metrics.json` | Extended-analysis results: broad-category RSA and Hallmark deep dive | +| `reactome_category_heatmaps.png` | Distance-matrix comparison at the broad Reactome category level | +| `hallmarks_deep_dive.png` | Hallmark per-module alignment scores and similarity matrix | +| `hallmarks_per_module_alignment.csv` | Per-module alignment ranking for 50 Hallmarks | + +## Dependencies + +`torch`, `numpy`, `pandas`, `scikit-learn`, `matplotlib`, `anndata`, and `scipy`. +UMAP plots additionally require `umap-learn`; if it is not installed, the script +warns and skips only UMAP plots. Experiment 2 also requires +`sentence-transformers`, installable with `pip install sentence-transformers`. diff --git a/BioMod/interpretability/attention/README.md b/BioMod/interpretability/attention/README.md index bf2c9e39a5cfa8a634edb188998babcaa8849abc..0b17f528e282c771059c1e6633eda391e1ffb669 100644 --- a/BioMod/interpretability/attention/README.md +++ b/BioMod/interpretability/attention/README.md @@ -1,108 +1,111 @@ -# Experiment 6: Module Crosstalk Graph vs Pathway Interaction Database - -This experiment tests whether module-module interactions learned by the BioMod -encoder are consistent with known pathway-pathway relationships. - -## Methods - -| Method | Principle | Compute cost | -| --- | --- | --- | -| A, cosine | Cosine similarity between cell-averaged module embeddings | Low | -| C, jacobian | Cell-averaged norm of `d encoder_out[j] / d module_emb[i]` | High | - -Ground truth: Jaccard index and overlap coefficient of shared genes between -modules. - -## Evaluation Metrics - -- Spearman correlation between the model interaction matrix and database overlap matrix. -- AUROC, using database edges as positives and model scores as predictions. -- Precision@k, the fraction of top-k model interactions that exist in the database. -- p-value from a permutation null distribution. - -## Run - -```bash -cd /path/to/performer_module/interpretability/attention - -# Full run, methods A + C. -python module_crosstalk.py \ - --performer_checkpoint ../../mae_tost_48M_cross_attn_only_pure_memory/mae_tost_48M_cross_attn_only_pure_memory_epoch12.pth \ - --config_pkl ../../mae_tost_48M_cross_attn_only_pure_memory/mae_tost_48M_cross_attn_only_pure_memory.pkl \ - --module_data_dir ../../module_data_all \ - --gene_list_path ../../../selected_genes_27k.txt \ - --gene_mapping_path ../../../gene2ens.csv \ - --gene2vec_data_path ../../../selected_gene2vec_27k.npy \ - --gene2ens_file ../../../gene2ens.csv \ - --data_path ../../../zero-shot-batch/pancreas_scib.h5ad \ - --data_mode h5ad \ - --batch_size 640 \ - --jacobian_cells 200 \ - --output_dir ../results/crosstalk/pancreas_max \ - --pool_mode max - -# Method A only, for quick validation. -python module_crosstalk.py \ - --performer_checkpoint ../../mae_tost_48M_cross_attn_only_pure_memory/mae_tost_48M_cross_attn_only_pure_memory_epoch12.pth \ - --config_pkl ../../mae_tost_48M_cross_attn_only_pure_memory/mae_tost_48M_cross_attn_only_pure_memory.pkl \ - --module_data_dir ../../module_data_all \ - --gene_list_path ../../../selected_genes_27k.txt \ - --gene_mapping_path ../../../gene2ens.csv \ - --gene2vec_data_path ../../../selected_gene2vec_27k.npy \ - --gene2ens_file ../../../gene2ens.csv \ - --data_path ../../../zero-shot-batch/pancreas_scib.h5ad \ - --data_mode h5ad \ - --batch_size 64 \ - --skip_jacobian \ - --output_dir ../results/crosstalk/pancreas_cosine_only - -# Pretraining npz data. -python module_crosstalk.py \ - --performer_checkpoint ../../mae_tost_48M_cross_attn_only_pure_memory/mae_tost_48M_cross_attn_only_pure_memory_epoch12.pth \ - --config_pkl ../../mae_tost_48M_cross_attn_only_pure_memory/mae_tost_48M_cross_attn_only_pure_memory.pkl \ - --module_data_dir ../../module_data_all \ - --gene_list_path ../../../selected_genes_27k.txt \ - --gene_mapping_path ../../../gene2ens.csv \ - --gene2vec_data_path ../../../selected_gene2vec_27k.npy \ - --gene2ens_file ../../../gene2ens.csv \ - --data_path ../../../../share/cell_files \ - --data_mode pretrain_npz \ - --cell_num 49738024 \ - --max_cells 5000 \ - --batch_size 16 \ - --jacobian_cells 100 \ - --output_dir ../results/crosstalk/pretrain -``` - -## Key Arguments - -| Argument | Default | Description | -| --- | --- | --- | -| `--jacobian_cells` | 200 | Number of cells sampled for Jacobian computation; more is stabler but slower | -| `--skip_jacobian` | off | Run method A only | -| `--max_cells` | 5000 | Maximum cell count used by method A | -| `--null_permutations` | 200 | Number of permutations | -| `--top_k_fractions` | 0.01,0.05,0.1 | Fractions used for Precision@k | - -## Outputs - -```text -output_dir/ -|-- cosine_interaction.npy # (660, 660) method A interaction matrix -|-- jacobian_interaction.npy # (660, 660) method C interaction matrix -|-- mean_module_embeddings.npy # (660, 1024) cell-averaged module embeddings -|-- gt_jaccard.npy # (660, 660) gene Jaccard overlap -|-- gt_overlap_coef.npy # (660, 660) overlap coefficient -|-- metrics.json # All evaluation metrics and p-values -|-- heatmap_cosine.png # Method A heatmap, sorted by family -|-- heatmap_jacobian.png # Method C heatmap -|-- heatmap_gt_jaccard.png # Ground-truth heatmap -|-- scatter_cosine_vs_jaccard.png -`-- scatter_jacobian_vs_jaccard.png -``` - -## Notes - -- Jacobian computation requires 660 backward passes per cell. `--jacobian_cells 200` takes about 10-30 minutes on one A100. -- Method A has almost no extra overhead, so use `--skip_jacobian` for quick validation first. -- The current model uses TSSA attention rather than standard softmax attention, so attention weights cannot be read directly. Jacobian is used as a substitute. +# Experiment 6: Module Crosstalk Graph vs Pathway Interaction Database + +This experiment tests whether module-module interactions learned by the BioMod +encoder are consistent with known pathway-pathway relationships. + +## Methods + +| Method | Principle | Compute cost | +| --- | --- | --- | +| A, cosine | Cosine similarity between cell-averaged module embeddings | Low | +| C, jacobian | Cell-averaged norm of `d encoder_out[j] / d module_emb[i]` | High | + +Ground truth: Jaccard index and overlap coefficient of shared genes between +modules. + +## Evaluation Metrics + +- Spearman correlation between the model interaction matrix and database overlap matrix. +- AUROC, using database edges as positives and model scores as predictions. +- Precision@k, the fraction of top-k model interactions that exist in the database. +- p-value from a permutation null distribution. + +## Run + +Run from the BioMod repository root. The h5ad examples use the bundled pancreas +dataset and BioMod model weights. The optional pretraining `.npz` dataset is not +included; replace `/absolute/path/to/pretraining_cell_npz` with your own directory. + +```bash +# Run from the BioMod repository root. + +# Full run, methods A + C. +python BioMod/interpretability/attention/module_crosstalk.py \ + --performer_checkpoint weight/0810model/mae_tost_70M_75mask_ec2_epoch0.pth \ + --config_pkl weight/0810model/mae_tost_70M_75mask_ec2.pkl \ + --module_data_dir model_dependencies/module_data_all \ + --gene_list_path model_dependencies/selected_genes_27k.txt \ + --gene_mapping_path model_dependencies/gene2ens.csv \ + --gene2vec_data_path model_dependencies/selected_gene2vec_27k.npy \ + --gene2ens_file model_dependencies/gene2ens.csv \ + --data_path data/downstream/final_data/human_pancreas_norm_complexBatch_with_gene_name.h5ad \ + --data_mode h5ad \ + --batch_size 640 \ + --jacobian_cells 200 \ + --output_dir outputs/interpretability/crosstalk/pancreas_max \ + --pool_mode max + +# Method A only, for quick validation. +python BioMod/interpretability/attention/module_crosstalk.py \ + --performer_checkpoint weight/0810model/mae_tost_70M_75mask_ec2_epoch0.pth \ + --config_pkl weight/0810model/mae_tost_70M_75mask_ec2.pkl \ + --module_data_dir model_dependencies/module_data_all \ + --gene_list_path model_dependencies/selected_genes_27k.txt \ + --gene_mapping_path model_dependencies/gene2ens.csv \ + --gene2vec_data_path model_dependencies/selected_gene2vec_27k.npy \ + --gene2ens_file model_dependencies/gene2ens.csv \ + --data_path data/downstream/final_data/human_pancreas_norm_complexBatch_with_gene_name.h5ad \ + --data_mode h5ad \ + --batch_size 64 \ + --skip_jacobian \ + --output_dir outputs/interpretability/crosstalk/pancreas_cosine_only + +# Pretraining npz data. +python BioMod/interpretability/attention/module_crosstalk.py \ + --performer_checkpoint weight/0810model/mae_tost_70M_75mask_ec2_epoch0.pth \ + --config_pkl weight/0810model/mae_tost_70M_75mask_ec2.pkl \ + --module_data_dir model_dependencies/module_data_all \ + --gene_list_path model_dependencies/selected_genes_27k.txt \ + --gene_mapping_path model_dependencies/gene2ens.csv \ + --gene2vec_data_path model_dependencies/selected_gene2vec_27k.npy \ + --gene2ens_file model_dependencies/gene2ens.csv \ + --data_path /absolute/path/to/pretraining_cell_npz \ + --data_mode pretrain_npz \ + --max_cells 5000 \ + --batch_size 16 \ + --jacobian_cells 100 \ + --output_dir outputs/interpretability/crosstalk/pretrain +``` + +## Key Arguments + +| Argument | Default | Description | +| --- | --- | --- | +| `--jacobian_cells` | 200 | Number of cells sampled for Jacobian computation; more is stabler but slower | +| `--skip_jacobian` | off | Run method A only | +| `--max_cells` | 5000 | Maximum cell count used by method A | +| `--null_permutations` | 200 | Number of permutations | +| `--top_k_fractions` | 0.01,0.05,0.1 | Fractions used for Precision@k | + +## Outputs + +```text +output_dir/ +|-- cosine_interaction.npy # (660, 660) method A interaction matrix +|-- jacobian_interaction.npy # (660, 660) method C interaction matrix +|-- mean_module_embeddings.npy # (660, 1024) cell-averaged module embeddings +|-- gt_jaccard.npy # (660, 660) gene Jaccard overlap +|-- gt_overlap_coef.npy # (660, 660) overlap coefficient +|-- metrics.json # All evaluation metrics and p-values +|-- heatmap_cosine.png # Method A heatmap, sorted by family +|-- heatmap_jacobian.png # Method C heatmap +|-- heatmap_gt_jaccard.png # Ground-truth heatmap +|-- scatter_cosine_vs_jaccard.png +`-- scatter_jacobian_vs_jaccard.png +``` + +## Notes + +- Jacobian computation requires 660 backward passes per cell. `--jacobian_cells 200` takes about 10-30 minutes on one A100. +- Method A has almost no extra overhead, so use `--skip_jacobian` for quick validation first. +- The current model uses TSSA attention rather than standard softmax attention, so attention weights cannot be read directly. Jacobian is used as a substitute. diff --git a/BioMod/interpretability/attention/README_gene_attention.md b/BioMod/interpretability/attention/README_gene_attention.md index 0317ee92e6d82a00583a6b49810a9a7c0fc5d21e..64b5e65a1d9b829534e9a112066adaac1193a294 100644 --- a/BioMod/interpretability/attention/README_gene_attention.md +++ b/BioMod/interpretability/attention/README_gene_attention.md @@ -1,114 +1,112 @@ -# Gene Attention Interpretability: ModuleEmbedding - -Analyze gene contributions within modules using the content-based attention -pooling in ModuleEmbedding. - -## Analyses - -1. Global gene-importance ranking, averaged attention over all cells. -2. Cell-type specificity with Mann-Whitney U test and fold change. -3. Marker gene validation with precision@k and recall@k. -4. Pathway enrichment analysis with GO BP, requiring `gseapy`. -5. Comparison with DEG results using Spearman correlation. - -## Dependencies - -```bash -pip install torch numpy pandas scipy matplotlib seaborn anndata -# Optional -pip install gseapy # Pathway enrichment analysis -``` - -## Run Commands - -### Basic Usage on the Pancreas Dataset with Built-In Markers - -```bash -cd ../data/250010166/scLong/performer_module/interpretability/attention - -python module_gene_attention.py \ - --performer_checkpoint ../../mae_tost_48M_cross_attn_only_pure_memory/mae_tost_48M_cross_attn_only_pure_memory_epoch12.pth \ - --config_pkl ../../mae_tost_48M_cross_attn_only_pure_memory/mae_tost_48M_cross_attn_only_pure_memory.pkl \ - --module_data_dir ../../module_data_all \ - --gene_list_path ../../../selected_genes_27k.txt \ - --gene_mapping_path ../../../gene2ens.csv \ - --gene2vec_data_path ../../../selected_gene2vec_27k.npy \ - --gene2ens_file ../../../gene2ens.csv \ - --data_path ../../../zero-shot-batch/pancreas_scib.h5ad \ - --celltype_column celltype \ - --batch_size 640 \ - --top_k_genes 20 \ - --output_dir ../results/gene_attention/pancreas -``` - -### With DEG Comparison - -```bash -python module_gene_attention.py \ - --performer_checkpoint ../../mae_tost_48M_cross_attn_only_pure_memory/mae_tost_48M_cross_attn_only_pure_memory_epoch12.pth \ - --config_pkl ../../mae_tost_48M_cross_attn_only_pure_memory/mae_tost_48M_cross_attn_only_pure_memory.pkl \ - --module_data_dir ../../module_data_all \ - --gene_list_path ../../../selected_genes_27k.txt \ - --gene_mapping_path ../../../gene2ens.csv \ - --gene2vec_data_path ../../../selected_gene2vec_27k.npy \ - --gene2ens_file ../../../gene2ens.csv \ - --data_path ../../../zero-shot-batch/pancreas_scib.h5ad \ - --celltype_column celltype \ - --batch_size 64 \ - --deg_csv /path/to/deg_results.csv \ - --output_dir ../results/gene_attention/pancreas_with_deg -``` - -DEG CSV columns must include `gene`, `cell_type`, `log2fc`, and `pval`. - -### Custom Marker Database - -```bash -python module_gene_attention.py \ - ... \ - --marker_db /path/to/markers.csv \ - --output_dir ../results/gene_attention/custom_markers -``` - -Marker CSV columns must include `cell_type` and `gene_symbol`. - -### Limit Cell Count for Faster Testing - -```bash -python module_gene_attention.py \ - ... \ - --max_cells 5000 \ - --output_dir ../results/gene_attention/pancreas_5k -``` - -## Key Arguments - -| Argument | Default | Description | -| --- | --- | --- | -| `--batch_size` | 64 | Inference batch size | -| `--top_k_genes` | 20 | Number of top genes shown for each cell type in visualizations | -| `--max_cells` | None | Number of cells to subsample | -| `--marker_db` | None | Custom marker CSV; built-in pancreas markers are used when omitted | -| `--deg_csv` | None | DEG result CSV; DEG correlation analysis is skipped when omitted | -| `--celltype_column` | celltype | Cell-type column name in h5ad | -| `--shuffle_module_genes` | off | Randomly shuffle gene-to-module assignment as a negative control | - -## Output Files - -```text -output_dir/ -|-- global_gene_importance.csv # Global gene importance across all modules and genes -|-- global_top_genes.png # Top-N gene bar chart -|-- celltype__attention.csv # Gene attention for each cell type -|-- celltype_attention_heatmap.png # Cell type x gene heatmap -|-- gene_specificity.csv # Gene specificity with fold change and p-value -|-- specificity_volcano.png # Volcano plot -|-- marker_validation.json # Marker validation results -|-- marker_recall.png # Recall@50/100 bar chart -|-- deg_correlation.json # DEG correlation, if --deg_csv is provided -|-- deg_correlation.png # Spearman rho bar chart -|-- enrichment/ # Pathway enrichment, if gseapy is installed -| |-- _enrichment.csv -| `-- enrichment_summary.json -`-- summary.json # Run summary -``` +# Gene Attention Interpretability: ModuleEmbedding + +Analyze gene contributions within modules using the content-based attention +pooling in ModuleEmbedding. + +## Analyses + +1. Global gene-importance ranking, averaged attention over all cells. +2. Cell-type specificity with Mann-Whitney U test and fold change. +3. Marker gene validation with precision@k and recall@k. +4. Pathway enrichment analysis with GO BP, requiring `gseapy`. +5. Comparison with DEG results using Spearman correlation. + +## Dependencies + +```bash +pip install torch numpy pandas scipy matplotlib seaborn anndata +# Optional +pip install gseapy # Pathway enrichment analysis +``` + +## Run Commands + +Run from the BioMod repository root. The examples use bundled BioMod model +weights and pancreas data. Custom DEG and marker CSV files are optional external +inputs; replace their example paths with your own files before using those flags. +Set `CELL_TYPE_COLUMN` below to the actual cell-type field in your h5ad metadata; +the column name has not been assumed for the bundled pancreas file. + +### Basic Usage on the Pancreas Dataset with Built-In Markers + +```bash +# Run from the BioMod repository root. + +python BioMod/interpretability/attention/module_gene_attention.py \ + --performer_checkpoint weight/0810model/mae_tost_70M_75mask_ec2_epoch0.pth \ + --config_pkl weight/0810model/mae_tost_70M_75mask_ec2.pkl \ + --module_data_dir model_dependencies/module_data_all \ + --gene_list_path model_dependencies/selected_genes_27k.txt \ + --gene_mapping_path model_dependencies/gene2ens.csv \ + --gene2vec_data_path model_dependencies/selected_gene2vec_27k.npy \ + --gene2ens_file model_dependencies/gene2ens.csv \ + --data_path data/downstream/final_data/human_pancreas_norm_complexBatch_with_gene_name.h5ad \ + --celltype_column CELL_TYPE_COLUMN \ + --batch_size 640 \ + --top_k_genes 20 \ + --output_dir outputs/interpretability/gene_attention/pancreas +``` + +### With DEG Comparison + +```bash +python BioMod/interpretability/attention/module_gene_attention.py \ + --performer_checkpoint weight/0810model/mae_tost_70M_75mask_ec2_epoch0.pth \ + --config_pkl weight/0810model/mae_tost_70M_75mask_ec2.pkl \ + --module_data_dir model_dependencies/module_data_all \ + --gene_list_path model_dependencies/selected_genes_27k.txt \ + --gene_mapping_path model_dependencies/gene2ens.csv \ + --gene2vec_data_path model_dependencies/selected_gene2vec_27k.npy \ + --gene2ens_file model_dependencies/gene2ens.csv \ + --data_path data/downstream/final_data/human_pancreas_norm_complexBatch_with_gene_name.h5ad \ + --celltype_column CELL_TYPE_COLUMN \ + --batch_size 64 \ + --deg_csv /path/to/deg_results.csv \ + --output_dir outputs/interpretability/gene_attention/pancreas_with_deg +``` + +DEG CSV columns must include `gene`, `cell_type`, `log2fc`, and `pval`. + +### Custom Marker Database + +Add `--marker_db /path/to/markers.csv` to the complete basic command above and +change `--output_dir` to `outputs/interpretability/gene_attention/custom_markers`. + +Marker CSV columns must include `cell_type` and `gene_symbol`. + +### Limit Cell Count for Faster Testing + +Add `--max_cells 5000` to the complete basic command above and change +`--output_dir` to `outputs/interpretability/gene_attention/pancreas_5k`. + +## Key Arguments + +| Argument | Default | Description | +| --- | --- | --- | +| `--batch_size` | 64 | Inference batch size | +| `--top_k_genes` | 20 | Number of top genes shown for each cell type in visualizations | +| `--max_cells` | None | Number of cells to subsample | +| `--marker_db` | None | Custom marker CSV; built-in pancreas markers are used when omitted | +| `--deg_csv` | None | DEG result CSV; DEG correlation analysis is skipped when omitted | +| `--celltype_column` | celltype | Cell-type column name in h5ad | +| `--shuffle_module_genes` | off | Randomly shuffle gene-to-module assignment as a negative control | + +## Output Files + +```text +output_dir/ +|-- global_gene_importance.csv # Global gene importance across all modules and genes +|-- global_top_genes.png # Top-N gene bar chart +|-- celltype__attention.csv # Gene attention for each cell type +|-- celltype_attention_heatmap.png # Cell type x gene heatmap +|-- gene_specificity.csv # Gene specificity with fold change and p-value +|-- specificity_volcano.png # Volcano plot +|-- marker_validation.json # Marker validation results +|-- marker_recall.png # Recall@50/100 bar chart +|-- deg_correlation.json # DEG correlation, if --deg_csv is provided +|-- deg_correlation.png # Spearman rho bar chart +|-- enrichment/ # Pathway enrichment, if gseapy is installed +| |-- _enrichment.csv +| `-- enrichment_summary.json +`-- summary.json # Run summary +``` diff --git a/BioMod/interpretability/attention/module_crosstalk.py b/BioMod/interpretability/attention/module_crosstalk.py index e7f01de480738b0147f286219264680b35f5650d..c83530710dacf57bb926befbea0859d26a890f9f 100644 --- a/BioMod/interpretability/attention/module_crosstalk.py +++ b/BioMod/interpretability/attention/module_crosstalk.py @@ -1,605 +1,606 @@ -"""Experiment 6 — Module crosstalk graph vs Reactome/KEGG pathway interactions. - -Two complementary methods to quantify module-to-module interaction strength: - - Method A (cosine similarity): Compute cosine similarity between cell-averaged - module encoder outputs. Fast, no gradient needed. - - Method C (Jacobian): Compute ||d encoder_out[j] / d module_input[i]|| averaged - over cells. Measures causal influence of module i on module j's representation. - Requires gradient; more expensive but more principled. - -Both interaction matrices are compared against known pathway-pathway relationships -from Reactome (shared genes / hierarchical overlap) and optionally KEGG. - -Metrics: - - Spearman correlation between model interaction matrix and DB overlap matrix - - Precision@k: among top-k model interactions, fraction present in DB - - AUROC: treating DB edges as positives, model scores as predictions - - Permutation null p-values for all metrics - -Usage: - python module_crosstalk.py \ - --performer_checkpoint ../mae_tost_48M_cross_attn_only_pure_memory/mae_tost_48M_cross_attn_only_pure_memory_epoch12.pth \ - --config_pkl ../mae_tost_48M_cross_attn_only_pure_memory/mae_tost_48M_cross_attn_only_pure_memory.pkl \ - --module_data_dir ../module_data_all \ - --gene_list_path ../../selected_genes_27k.txt \ - --gene_mapping_path ../../gene2ens.csv \ - --gene2vec_data_path ../../selected_gene2vec_27k.npy \ - --data_path ../../zero-shot-batch/pancreas_scib.h5ad \ - --data_mode h5ad \ - --batch_size 16 \ - --jacobian_cells 200 \ - --output_dir ./results/crosstalk/pancreas -""" -from __future__ import annotations - -import argparse -import json -import logging -import sys -from pathlib import Path -from typing import Dict, List, Optional, Tuple - -import numpy as np -import torch -from torch.utils.data import DataLoader, Subset - -import matplotlib -matplotlib.use("Agg") -import matplotlib.pyplot as plt - -from scipy.stats import spearmanr -from sklearn.metrics import roc_auc_score - -HERE = Path(__file__).resolve() -PERFORMER_MODULE = HERE.parents[2] -REPO_ROOT = HERE.parents[3] -for p in (str(PERFORMER_MODULE), str(REPO_ROOT)): - if p not in sys.path: - sys.path.insert(0, p) - -from cell_annotation.performer_module_finetune import ( - DataPreprocessor, - load_backbone, - seed_all, -) - -from interpretability.utils import ( - MatrixRowDataset, - NpzCellDataset, - build_module_metadata, - ensure_dir, - load_module_names, -) - -logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") -logger = logging.getLogger(__name__) - -# --------------------------------------------------------------------------- -# Argparse -# --------------------------------------------------------------------------- - -def parse_args() -> argparse.Namespace: - p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - - p.add_argument("--performer_checkpoint", type=str, required=True) - p.add_argument("--config_pkl", type=str, required=True) - p.add_argument("--module_data_dir", type=str, default="../module_data_all") - p.add_argument("--module_json", type=str, default=None) - p.add_argument("--gene_list_path", type=str, default="../../selected_genes_27k.txt") - p.add_argument("--gene_mapping_path", type=str, default="../../gene2ens.csv") - p.add_argument("--gene2vec_data_path", type=str, default="../../selected_gene2vec_27k.npy") - p.add_argument("--gene2ens_file", type=str, default="../../gene2ens.csv") - p.add_argument("--gene2ens_column", type=str, default="Ensembl gene ID") - p.add_argument("--shuffle_module_genes", action="store_true") - p.add_argument("--shuffle_seed", type=int, default=2021) - - p.add_argument("--data_path", type=str, required=True) - p.add_argument("--data_mode", type=str, default="h5ad", - choices=["h5ad", "pretrain_h5ad", "pretrain_npz"]) - p.add_argument("--cell_num", type=int, default=None) - p.add_argument("--gene_name_column", type=str, default="auto") - p.add_argument("--dataset_name", type=str, default="crosstalk") - p.add_argument("--max_cells", type=int, default=5000, - help="Max cells for method A (cosine). Method C uses --jacobian_cells.") - - p.add_argument("--batch_size", type=int, default=16) - p.add_argument("--num_workers", type=int, default=0) - p.add_argument("--device", type=str, default="cuda") - p.add_argument("--seed", type=int, default=2021) - p.add_argument("--pool_mode", type=str, default="mean", - choices=["mean", "max"], - help="Pooling over cells: 'mean' or 'max' (element-wise max).") - - p.add_argument("--jacobian_cells", type=int, default=200, - help="Number of cells for Jacobian computation (expensive).") - p.add_argument("--jacobian_batch", type=int, default=4, - help="Cells per Jacobian forward-backward pass.") - p.add_argument("--skip_jacobian", action="store_true", - help="Only run method A (cosine), skip method C (Jacobian).") - - p.add_argument("--null_permutations", type=int, default=200) - p.add_argument("--top_k_fractions", type=str, default="0.01,0.05,0.1", - help="Comma-separated fractions for precision@k.") - - p.add_argument("--output_dir", type=str, required=True) - return p.parse_args() - - -# --------------------------------------------------------------------------- -# Data loading (reuses patterns from module_geometry.py) -# --------------------------------------------------------------------------- - -def build_dataset(args, preprocessor): - """Build dataset and optional reindex_state.""" - if args.data_mode == "h5ad": - import anndata as ad - adata, reindex_state = preprocessor.preprocess_adata( - adata_path=args.data_path, - min_genes=0, min_cells=0, - max_cells=args.max_cells, - dataset_name=args.dataset_name, - gene_name_column=args.gene_name_column, - ) - dataset = MatrixRowDataset(adata.X) - n_cells = adata.n_obs - elif args.data_mode == "pretrain_h5ad": - import anndata as ad - adata = ad.read_h5ad(args.data_path) - if args.max_cells and adata.n_obs > args.max_cells: - rng = np.random.default_rng(args.seed) - chosen = np.sort(rng.choice(adata.n_obs, size=args.max_cells, replace=False)) - adata = adata[chosen].copy() - dataset = MatrixRowDataset(adata.X) - n_cells = adata.n_obs - reindex_state = None - elif args.data_mode == "pretrain_npz": - dataset = NpzCellDataset( - data_dir=args.data_path, - cell_num=args.cell_num, - max_cells=args.max_cells, - seed=args.seed, - ) - n_cells = len(dataset) - reindex_state = None - else: - raise ValueError(f"Unsupported data_mode: {args.data_mode}") - return dataset, reindex_state, n_cells - - -# --------------------------------------------------------------------------- -# Method A: Cosine similarity of cell-averaged module embeddings -# --------------------------------------------------------------------------- - -@torch.no_grad() -def compute_cosine_interaction(backbone, loader, preprocessor, reindex_state, - device, num_modules, base_dim, expected_genes, - pool_mode="mean"): - """Compute pooled module embeddings, return cosine similarity matrix.""" - backbone.eval() - if pool_mode == "mean": - agg_emb = torch.zeros(num_modules, base_dim, device=device, dtype=torch.float64) - else: - agg_emb = torch.full((num_modules, base_dim), -float("inf"), device=device, dtype=torch.float64) - n_cells = 0 - - for batch_idx, x in enumerate(loader): - x = x.to(device) - if reindex_state is not None: - x = preprocessor.reindex_tensor_to_model_space(x, reindex_state=reindex_state) - out = backbone(x, mask_ratio=0.0, return_encodings=True) - enc = out["module_encodings"].to(torch.float64) # (B, num_modules, D) - if pool_mode == "mean": - agg_emb += enc.sum(dim=0) - else: - batch_max = enc.max(dim=0).values # (num_modules, D) - agg_emb = torch.maximum(agg_emb, batch_max) - n_cells += enc.shape[0] - if (batch_idx + 1) % 50 == 0: - logger.info(" [cosine/%s] batch %d, %d cells", pool_mode, batch_idx + 1, n_cells) - - if pool_mode == "mean": - pooled_emb = (agg_emb / max(n_cells, 1)).float().cpu().numpy() - else: - pooled_emb = agg_emb.float().cpu().numpy() - - norms = np.linalg.norm(pooled_emb, axis=1, keepdims=True) - norms = np.clip(norms, 1e-8, None) - normed = pooled_emb / norms - cos_sim = normed @ normed.T - np.fill_diagonal(cos_sim, 0.0) - logger.info("Method A (%s) done: %d cells, interaction matrix shape %s", pool_mode, n_cells, cos_sim.shape) - return cos_sim, pooled_emb - - -# --------------------------------------------------------------------------- -# Method C: Jacobian-based module interaction -# --------------------------------------------------------------------------- - -def compute_jacobian_interaction(backbone, loader, preprocessor, reindex_state, - device, num_modules, base_dim, expected_genes, - max_cells, batch_size): - """Compute average ||d encoder_out[j] / d module_emb[i]|| interaction matrix.""" - backbone.eval() - for param in backbone.parameters(): - param.requires_grad_(False) - - interaction = torch.zeros(num_modules, num_modules, device=device, dtype=torch.float64) - n_cells = 0 - - for x in loader: - if n_cells >= max_cells: - break - x = x.to(device) - if reindex_state is not None: - x = preprocessor.reindex_tensor_to_model_space(x, reindex_state=reindex_state) - - actual_batch = min(x.shape[0], max_cells - n_cells) - x = x[:actual_batch] - - for cell_idx in range(x.shape[0]): - cell = x[cell_idx:cell_idx+1] - J = _compute_single_cell_jacobian(backbone, cell, num_modules, base_dim, device) - interaction += J - n_cells += 1 - if n_cells % 20 == 0: - logger.info(" [jacobian] %d / %d cells", n_cells, max_cells) - if n_cells >= max_cells: - break - - interaction /= max(n_cells, 1) - result = interaction.cpu().numpy() - np.fill_diagonal(result, 0.0) - logger.info("Method C done: %d cells, interaction matrix shape %s", n_cells, result.shape) - return result - - -def _compute_single_cell_jacobian(backbone, cell, num_modules, base_dim, device): - """For one cell, compute (num_modules, num_modules) Jacobian norm matrix. - - J[i, j] = ||d encoder_out[j] / d module_emb[i]||_F - - Strategy: run module_embed to get intermediate, detach, enable grad, - then replicate the encoder forward path and backprop from each output module. - """ - J = torch.zeros(num_modules, num_modules, device=device, dtype=torch.float32) - - with torch.no_grad(): - x_raw, _ = backbone.module_embed(cell) # (1, num_modules, D) - - x_modules = x_raw.detach().clone().requires_grad_(True) - - with torch.enable_grad(): - if backbone.mask_style == "replace": - x = backbone._add_module_position_embedding(x_modules) - else: - x = backbone._add_module_position_embedding(x_modules) - - if backbone.use_cls and backbone.cls_token is not None: - cls_token = backbone.cls_token + backbone.pos_emb[:, :1, :] - cls_tokens = cls_token.expand(x.shape[0], -1, -1) - x = torch.cat((cls_tokens, x), dim=1) - - x_emb = backbone.dropout(x) - layer_pos_emb = backbone.layer_pos_emb(x_emb) - enc_out = backbone.encoder(x_emb, pos_emb=layer_pos_emb) - enc_out = backbone.encoder_norm(enc_out) # (1, L, D) - - if backbone.use_cls and backbone.cls_token is not None: - enc_out = enc_out[:, 1:, :] # skip CLS - - for j in range(num_modules): - if x_modules.grad is not None: - x_modules.grad.zero_() - scalar = enc_out[0, j].pow(2).sum() - scalar.backward(retain_graph=(j < num_modules - 1)) - grad = x_modules.grad[0] # (num_modules, D) - J[:, j] = grad.norm(dim=1) - - return J.to(torch.float64) - - -# --------------------------------------------------------------------------- -# Ground truth: pathway overlap from module gene sets -# --------------------------------------------------------------------------- - -def compute_pathway_overlap(module_names: List[str], module_gene_dict: dict, - metric: str = "jaccard") -> np.ndarray: - """Compute pairwise pathway overlap matrix from shared genes. - - metric: 'jaccard' (|A∩B| / |A∪B|) or 'overlap_coef' (|A∩B| / min(|A|,|B|)) - """ - n = len(module_names) - gene_sets = [] - for name in module_names: - genes = module_gene_dict.get(name, []) - gene_sets.append(set(genes)) - - overlap = np.zeros((n, n), dtype=np.float32) - for i in range(n): - for j in range(i + 1, n): - inter = len(gene_sets[i] & gene_sets[j]) - if inter == 0: - continue - if metric == "jaccard": - union = len(gene_sets[i] | gene_sets[j]) - score = inter / union if union > 0 else 0.0 - else: - min_size = min(len(gene_sets[i]), len(gene_sets[j])) - score = inter / min_size if min_size > 0 else 0.0 - overlap[i, j] = score - overlap[j, i] = score - return overlap - - -# --------------------------------------------------------------------------- -# Evaluation metrics -# --------------------------------------------------------------------------- - -def evaluate_interaction(model_matrix: np.ndarray, gt_matrix: np.ndarray, - top_k_fracs: List[float], n_permutations: int = 200): - """Compare model interaction matrix against ground truth overlap. - - Returns dict with spearman_r, auroc, precision@k, and permutation p-values. - """ - n = model_matrix.shape[0] - triu_idx = np.triu_indices(n, k=1) - model_flat = model_matrix[triu_idx] - gt_flat = gt_matrix[triu_idx] - - has_positive = gt_flat.max() > 0 - has_variation = model_flat.std() > 1e-10 - - rho, _ = spearmanr(model_flat, gt_flat) if has_variation else (0.0, 1.0) - - gt_binary = (gt_flat > 0).astype(np.float32) - n_pos = gt_binary.sum() - if n_pos > 0 and n_pos < len(gt_binary) and has_variation: - auroc = roc_auc_score(gt_binary, model_flat) - else: - auroc = 0.5 - - precisions = {} - n_pairs = len(model_flat) - for frac in top_k_fracs: - k = max(1, int(n_pairs * frac)) - top_k_idx = np.argsort(model_flat)[-k:] - precisions[f"precision@{frac}"] = float(gt_binary[top_k_idx].mean()) - - null_rhos = [] - null_aurocs = [] - rng = np.random.default_rng(42) - for _ in range(n_permutations): - perm = rng.permutation(n) - perm_matrix = model_matrix[perm][:, perm] - perm_flat = perm_matrix[triu_idx] - r, _ = spearmanr(perm_flat, gt_flat) if has_variation else (0.0, 1.0) - null_rhos.append(r) - if n_pos > 0 and n_pos < len(gt_binary): - null_aurocs.append(roc_auc_score(gt_binary, perm_flat)) - else: - null_aurocs.append(0.5) - - p_rho = (np.sum(np.array(null_rhos) >= rho) + 1) / (n_permutations + 1) - p_auroc = (np.sum(np.array(null_aurocs) >= auroc) + 1) / (n_permutations + 1) - - return { - "spearman_r": float(rho), - "spearman_p_perm": float(p_rho), - "auroc": float(auroc), - "auroc_p_perm": float(p_auroc), - "null_rho_mean": float(np.mean(null_rhos)), - "null_rho_std": float(np.std(null_rhos)), - **precisions, - } - - -# --------------------------------------------------------------------------- -# Visualization -# --------------------------------------------------------------------------- - -def plot_interaction_matrix(matrix: np.ndarray, title: str, path: Path, - module_families: Optional[List[str]] = None): - """Plot heatmap of interaction matrix, optionally sorted by family.""" - if module_families is not None: - order = np.argsort(module_families) - matrix = matrix[order][:, order] - sorted_families = [module_families[i] for i in order] - else: - sorted_families = None - - fig, ax = plt.subplots(figsize=(10, 9)) - vmax = np.percentile(np.abs(matrix), 99) - im = ax.imshow(matrix, cmap="RdBu_r", vmin=-vmax, vmax=vmax, aspect="equal") - ax.set_title(title, fontsize=12) - ax.set_xlabel("Module j") - ax.set_ylabel("Module i") - plt.colorbar(im, ax=ax, fraction=0.046) - - if sorted_families is not None: - unique_fams = [] - boundaries = [0] - prev = sorted_families[0] - for idx, f in enumerate(sorted_families): - if f != prev: - unique_fams.append(prev) - boundaries.append(idx) - prev = f - unique_fams.append(prev) - boundaries.append(len(sorted_families)) - for b in boundaries[1:-1]: - ax.axhline(b - 0.5, color="black", linewidth=0.3, alpha=0.5) - ax.axvline(b - 0.5, color="black", linewidth=0.3, alpha=0.5) - - plt.tight_layout() - fig.savefig(path, dpi=150, bbox_inches="tight") - plt.close(fig) - - -def plot_scatter(model_flat, gt_flat, title, path): - """Scatter plot of model interaction vs ground truth overlap.""" - fig, ax = plt.subplots(figsize=(6, 5)) - subsample = min(len(model_flat), 50000) - if subsample < len(model_flat): - rng = np.random.default_rng(0) - idx = rng.choice(len(model_flat), size=subsample, replace=False) - model_flat = model_flat[idx] - gt_flat = gt_flat[idx] - ax.scatter(gt_flat, model_flat, alpha=0.1, s=2, rasterized=True) - ax.set_xlabel("Ground truth overlap (Jaccard)") - ax.set_ylabel("Model interaction score") - ax.set_title(title) - plt.tight_layout() - fig.savefig(path, dpi=150, bbox_inches="tight") - plt.close(fig) - - -# --------------------------------------------------------------------------- -# Main -# --------------------------------------------------------------------------- - -def main(): - args = parse_args() - seed_all(args.seed) - device = torch.device(args.device if torch.cuda.is_available() else "cpu") - output_dir = Path(args.output_dir) - ensure_dir(output_dir) - - # Resolve all paths to absolute (relative to CWD) so that load_backbone's - # internal resolve_path (which resolves relative to cell_annotation/) is bypassed. - for attr in ("performer_checkpoint", "config_pkl", "module_data_dir", - "gene_list_path", "gene_mapping_path", "gene2vec_data_path", - "gene2ens_file", "data_path"): - val = getattr(args, attr, None) - if val is not None and not Path(val).is_absolute(): - setattr(args, attr, str(Path(val).resolve())) - - logger.info("Loading backbone...") - backbone, module_data, model_config = load_backbone(args, device) - backbone.eval() - - num_modules = len(module_data) - base_dim = int(model_config["base_dim"]) - expected_genes = int(model_config.get("num_genes", 27874)) - logger.info("Model: %d modules, dim=%d, genes=%d", num_modules, base_dim, expected_genes) - - module_json_path = args.module_json or str(Path(args.module_data_dir) / "merged_modules.json") - module_names = load_module_names(module_json_path) - with open(module_json_path, "r", encoding="utf-8") as f: - module_gene_dict = json.load(f) - - preprocessor = None - if args.data_mode == "h5ad": - preprocessor = DataPreprocessor( - gene2ens_file=args.gene2ens_file, - gene2ens_column=args.gene2ens_column, - model_genes_file=args.gene_list_path, - ) - preprocessor.load_gene_mapping() - preprocessor.load_model_genes() - - dataset, reindex_state, n_cells = build_dataset(args, preprocessor) - logger.info("Dataset: %d cells", n_cells) - - # --- Method A: Cosine --- - logger.info("=" * 60) - logger.info("Method A: Cosine similarity of module embeddings") - logger.info("=" * 60) - loader_a = DataLoader(dataset, batch_size=args.batch_size, shuffle=False, - num_workers=args.num_workers, pin_memory=True) - cos_matrix, mean_emb = compute_cosine_interaction( - backbone, loader_a, preprocessor, reindex_state, - device, num_modules, base_dim, expected_genes, - pool_mode=args.pool_mode, - ) - - # --- Method C: Jacobian --- - jac_matrix = None - if not args.skip_jacobian: - logger.info("=" * 60) - logger.info("Method C: Jacobian-based module interaction") - logger.info("=" * 60) - jac_n = min(args.jacobian_cells, n_cells) - if jac_n < n_cells: - rng = np.random.default_rng(args.seed) - jac_indices = np.sort(rng.choice(n_cells, size=jac_n, replace=False)) - jac_dataset = Subset(dataset, jac_indices.tolist()) - else: - jac_dataset = dataset - loader_c = DataLoader(jac_dataset, batch_size=args.jacobian_batch, shuffle=False, - num_workers=0, pin_memory=True) - jac_matrix = compute_jacobian_interaction( - backbone, loader_c, preprocessor, reindex_state, - device, num_modules, base_dim, expected_genes, - max_cells=jac_n, batch_size=args.jacobian_batch, - ) - - # --- Ground truth overlap --- - logger.info("Computing pathway gene overlap (Jaccard)...") - gt_jaccard = compute_pathway_overlap(module_names, module_gene_dict, metric="jaccard") - gt_overlap = compute_pathway_overlap(module_names, module_gene_dict, metric="overlap_coef") - - # --- Evaluate --- - top_k_fracs = [float(f) for f in args.top_k_fractions.split(",")] - results = {} - - logger.info("Evaluating Method A vs Jaccard overlap...") - results["cosine_vs_jaccard"] = evaluate_interaction( - cos_matrix, gt_jaccard, top_k_fracs, args.null_permutations) - results["cosine_vs_overlap_coef"] = evaluate_interaction( - cos_matrix, gt_overlap, top_k_fracs, args.null_permutations) - - if jac_matrix is not None: - logger.info("Evaluating Method C vs Jaccard overlap...") - jac_sym = (jac_matrix + jac_matrix.T) / 2 - results["jacobian_vs_jaccard"] = evaluate_interaction( - jac_sym, gt_jaccard, top_k_fracs, args.null_permutations) - results["jacobian_vs_overlap_coef"] = evaluate_interaction( - jac_sym, gt_overlap, top_k_fracs, args.null_permutations) - - # --- Save --- - np.save(output_dir / "cosine_interaction.npy", cos_matrix) - np.save(output_dir / "mean_module_embeddings.npy", mean_emb) - np.save(output_dir / "gt_jaccard.npy", gt_jaccard) - np.save(output_dir / "gt_overlap_coef.npy", gt_overlap) - if jac_matrix is not None: - np.save(output_dir / "jacobian_interaction.npy", jac_matrix) - - with open(output_dir / "metrics.json", "w", encoding="utf-8") as f: - json.dump(results, f, indent=2, ensure_ascii=False) - - # --- Plots --- - meta = build_module_metadata(module_names) - families = [row["family"] for row in meta] - - plot_interaction_matrix(cos_matrix, "Method A: Cosine Similarity", - output_dir / "heatmap_cosine.png", families) - if jac_matrix is not None: - plot_interaction_matrix(jac_sym, "Method C: Jacobian Interaction", - output_dir / "heatmap_jacobian.png", families) - plot_interaction_matrix(gt_jaccard, "Ground Truth: Jaccard Gene Overlap", - output_dir / "heatmap_gt_jaccard.png", families) - - triu_idx = np.triu_indices(num_modules, k=1) - plot_scatter(cos_matrix[triu_idx], gt_jaccard[triu_idx], - "Cosine vs Jaccard", output_dir / "scatter_cosine_vs_jaccard.png") - if jac_matrix is not None: - plot_scatter(jac_sym[triu_idx], gt_jaccard[triu_idx], - "Jacobian vs Jaccard", output_dir / "scatter_jacobian_vs_jaccard.png") - - # --- Print summary --- - logger.info("=" * 60) - logger.info("Results Summary") - logger.info("=" * 60) - for method_key, metrics in results.items(): - logger.info(" %s:", method_key) - logger.info(" Spearman r = %.4f (p_perm = %.4f)", metrics["spearman_r"], metrics["spearman_p_perm"]) - logger.info(" AUROC = %.4f (p_perm = %.4f)", metrics["auroc"], metrics["auroc_p_perm"]) - for k in top_k_fracs: - logger.info(" Precision@%.2f = %.4f", k, metrics[f"precision@{k}"]) - logger.info("Output saved to: %s", output_dir) - - -if __name__ == "__main__": - main() +"""Experiment 6 — Module crosstalk graph vs Reactome/KEGG pathway interactions. + +Two complementary methods to quantify module-to-module interaction strength: + + Method A (cosine similarity): Compute cosine similarity between cell-averaged + module encoder outputs. Fast, no gradient needed. + + Method C (Jacobian): Compute ||d encoder_out[j] / d module_input[i]|| averaged + over cells. Measures causal influence of module i on module j's representation. + Requires gradient; more expensive but more principled. + +Both interaction matrices are compared against known pathway-pathway relationships +from Reactome (shared genes / hierarchical overlap) and optionally KEGG. + +Metrics: + - Spearman correlation between model interaction matrix and DB overlap matrix + - Precision@k: among top-k model interactions, fraction present in DB + - AUROC: treating DB edges as positives, model scores as predictions + - Permutation null p-values for all metrics + +Usage (run from the BioMod repository root): + python BioMod/interpretability/attention/module_crosstalk.py \ + --performer_checkpoint weight/0810model/mae_tost_70M_75mask_ec2_epoch0.pth \ + --config_pkl weight/0810model/mae_tost_70M_75mask_ec2.pkl \ + --module_data_dir model_dependencies/module_data_all \ + --gene_list_path model_dependencies/selected_genes_27k.txt \ + --gene_mapping_path model_dependencies/gene2ens.csv \ + --gene2vec_data_path model_dependencies/selected_gene2vec_27k.npy \ + --data_path data/downstream/final_data/human_pancreas_norm_complexBatch_with_gene_name.h5ad \ + --data_mode h5ad \ + --batch_size 16 \ + --jacobian_cells 200 \ + --output_dir outputs/interpretability/crosstalk/pancreas +""" +from __future__ import annotations + +import argparse +import json +import logging +import sys +from pathlib import Path +from typing import Dict, List, Optional, Tuple + +import numpy as np +import torch +from torch.utils.data import DataLoader, Subset + +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt + +from scipy.stats import spearmanr +from sklearn.metrics import roc_auc_score + +HERE = Path(__file__).resolve() +PERFORMER_MODULE = HERE.parents[2] +REPO_ROOT = HERE.parents[3] +ASSET_ROOT = REPO_ROOT / "model_dependencies" +for p in (str(PERFORMER_MODULE), str(REPO_ROOT)): + if p not in sys.path: + sys.path.insert(0, p) + +from cell_annotation.performer_module_finetune import ( + DataPreprocessor, + load_backbone, + seed_all, +) + +from interpretability.utils import ( + MatrixRowDataset, + NpzCellDataset, + build_module_metadata, + ensure_dir, + load_module_names, +) + +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") +logger = logging.getLogger(__name__) + +# --------------------------------------------------------------------------- +# Argparse +# --------------------------------------------------------------------------- + +def parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + + p.add_argument("--performer_checkpoint", type=str, required=True) + p.add_argument("--config_pkl", type=str, required=True) + p.add_argument("--module_data_dir", type=str, default=str(ASSET_ROOT / "module_data_all")) + p.add_argument("--module_json", type=str, default=None) + p.add_argument("--gene_list_path", type=str, default=str(ASSET_ROOT / "selected_genes_27k.txt")) + p.add_argument("--gene_mapping_path", type=str, default=str(ASSET_ROOT / "gene2ens.csv")) + p.add_argument("--gene2vec_data_path", type=str, default=str(ASSET_ROOT / "selected_gene2vec_27k.npy")) + p.add_argument("--gene2ens_file", type=str, default=str(ASSET_ROOT / "gene2ens.csv")) + p.add_argument("--gene2ens_column", type=str, default="Ensembl gene ID") + p.add_argument("--shuffle_module_genes", action="store_true") + p.add_argument("--shuffle_seed", type=int, default=2021) + + p.add_argument("--data_path", type=str, required=True) + p.add_argument("--data_mode", type=str, default="h5ad", + choices=["h5ad", "pretrain_h5ad", "pretrain_npz"]) + p.add_argument("--cell_num", type=int, default=None) + p.add_argument("--gene_name_column", type=str, default="auto") + p.add_argument("--dataset_name", type=str, default="crosstalk") + p.add_argument("--max_cells", type=int, default=5000, + help="Max cells for method A (cosine). Method C uses --jacobian_cells.") + + p.add_argument("--batch_size", type=int, default=16) + p.add_argument("--num_workers", type=int, default=0) + p.add_argument("--device", type=str, default="cuda") + p.add_argument("--seed", type=int, default=2021) + p.add_argument("--pool_mode", type=str, default="mean", + choices=["mean", "max"], + help="Pooling over cells: 'mean' or 'max' (element-wise max).") + + p.add_argument("--jacobian_cells", type=int, default=200, + help="Number of cells for Jacobian computation (expensive).") + p.add_argument("--jacobian_batch", type=int, default=4, + help="Cells per Jacobian forward-backward pass.") + p.add_argument("--skip_jacobian", action="store_true", + help="Only run method A (cosine), skip method C (Jacobian).") + + p.add_argument("--null_permutations", type=int, default=200) + p.add_argument("--top_k_fractions", type=str, default="0.01,0.05,0.1", + help="Comma-separated fractions for precision@k.") + + p.add_argument("--output_dir", type=str, required=True) + return p.parse_args() + + +# --------------------------------------------------------------------------- +# Data loading (reuses patterns from module_geometry.py) +# --------------------------------------------------------------------------- + +def build_dataset(args, preprocessor): + """Build dataset and optional reindex_state.""" + if args.data_mode == "h5ad": + import anndata as ad + adata, reindex_state = preprocessor.preprocess_adata( + adata_path=args.data_path, + min_genes=0, min_cells=0, + max_cells=args.max_cells, + dataset_name=args.dataset_name, + gene_name_column=args.gene_name_column, + ) + dataset = MatrixRowDataset(adata.X) + n_cells = adata.n_obs + elif args.data_mode == "pretrain_h5ad": + import anndata as ad + adata = ad.read_h5ad(args.data_path) + if args.max_cells and adata.n_obs > args.max_cells: + rng = np.random.default_rng(args.seed) + chosen = np.sort(rng.choice(adata.n_obs, size=args.max_cells, replace=False)) + adata = adata[chosen].copy() + dataset = MatrixRowDataset(adata.X) + n_cells = adata.n_obs + reindex_state = None + elif args.data_mode == "pretrain_npz": + dataset = NpzCellDataset( + data_dir=args.data_path, + cell_num=args.cell_num, + max_cells=args.max_cells, + seed=args.seed, + ) + n_cells = len(dataset) + reindex_state = None + else: + raise ValueError(f"Unsupported data_mode: {args.data_mode}") + return dataset, reindex_state, n_cells + + +# --------------------------------------------------------------------------- +# Method A: Cosine similarity of cell-averaged module embeddings +# --------------------------------------------------------------------------- + +@torch.no_grad() +def compute_cosine_interaction(backbone, loader, preprocessor, reindex_state, + device, num_modules, base_dim, expected_genes, + pool_mode="mean"): + """Compute pooled module embeddings, return cosine similarity matrix.""" + backbone.eval() + if pool_mode == "mean": + agg_emb = torch.zeros(num_modules, base_dim, device=device, dtype=torch.float64) + else: + agg_emb = torch.full((num_modules, base_dim), -float("inf"), device=device, dtype=torch.float64) + n_cells = 0 + + for batch_idx, x in enumerate(loader): + x = x.to(device) + if reindex_state is not None: + x = preprocessor.reindex_tensor_to_model_space(x, reindex_state=reindex_state) + out = backbone(x, mask_ratio=0.0, return_encodings=True) + enc = out["module_encodings"].to(torch.float64) # (B, num_modules, D) + if pool_mode == "mean": + agg_emb += enc.sum(dim=0) + else: + batch_max = enc.max(dim=0).values # (num_modules, D) + agg_emb = torch.maximum(agg_emb, batch_max) + n_cells += enc.shape[0] + if (batch_idx + 1) % 50 == 0: + logger.info(" [cosine/%s] batch %d, %d cells", pool_mode, batch_idx + 1, n_cells) + + if pool_mode == "mean": + pooled_emb = (agg_emb / max(n_cells, 1)).float().cpu().numpy() + else: + pooled_emb = agg_emb.float().cpu().numpy() + + norms = np.linalg.norm(pooled_emb, axis=1, keepdims=True) + norms = np.clip(norms, 1e-8, None) + normed = pooled_emb / norms + cos_sim = normed @ normed.T + np.fill_diagonal(cos_sim, 0.0) + logger.info("Method A (%s) done: %d cells, interaction matrix shape %s", pool_mode, n_cells, cos_sim.shape) + return cos_sim, pooled_emb + + +# --------------------------------------------------------------------------- +# Method C: Jacobian-based module interaction +# --------------------------------------------------------------------------- + +def compute_jacobian_interaction(backbone, loader, preprocessor, reindex_state, + device, num_modules, base_dim, expected_genes, + max_cells, batch_size): + """Compute average ||d encoder_out[j] / d module_emb[i]|| interaction matrix.""" + backbone.eval() + for param in backbone.parameters(): + param.requires_grad_(False) + + interaction = torch.zeros(num_modules, num_modules, device=device, dtype=torch.float64) + n_cells = 0 + + for x in loader: + if n_cells >= max_cells: + break + x = x.to(device) + if reindex_state is not None: + x = preprocessor.reindex_tensor_to_model_space(x, reindex_state=reindex_state) + + actual_batch = min(x.shape[0], max_cells - n_cells) + x = x[:actual_batch] + + for cell_idx in range(x.shape[0]): + cell = x[cell_idx:cell_idx+1] + J = _compute_single_cell_jacobian(backbone, cell, num_modules, base_dim, device) + interaction += J + n_cells += 1 + if n_cells % 20 == 0: + logger.info(" [jacobian] %d / %d cells", n_cells, max_cells) + if n_cells >= max_cells: + break + + interaction /= max(n_cells, 1) + result = interaction.cpu().numpy() + np.fill_diagonal(result, 0.0) + logger.info("Method C done: %d cells, interaction matrix shape %s", n_cells, result.shape) + return result + + +def _compute_single_cell_jacobian(backbone, cell, num_modules, base_dim, device): + """For one cell, compute (num_modules, num_modules) Jacobian norm matrix. + + J[i, j] = ||d encoder_out[j] / d module_emb[i]||_F + + Strategy: run module_embed to get intermediate, detach, enable grad, + then replicate the encoder forward path and backprop from each output module. + """ + J = torch.zeros(num_modules, num_modules, device=device, dtype=torch.float32) + + with torch.no_grad(): + x_raw, _ = backbone.module_embed(cell) # (1, num_modules, D) + + x_modules = x_raw.detach().clone().requires_grad_(True) + + with torch.enable_grad(): + if backbone.mask_style == "replace": + x = backbone._add_module_position_embedding(x_modules) + else: + x = backbone._add_module_position_embedding(x_modules) + + if backbone.use_cls and backbone.cls_token is not None: + cls_token = backbone.cls_token + backbone.pos_emb[:, :1, :] + cls_tokens = cls_token.expand(x.shape[0], -1, -1) + x = torch.cat((cls_tokens, x), dim=1) + + x_emb = backbone.dropout(x) + layer_pos_emb = backbone.layer_pos_emb(x_emb) + enc_out = backbone.encoder(x_emb, pos_emb=layer_pos_emb) + enc_out = backbone.encoder_norm(enc_out) # (1, L, D) + + if backbone.use_cls and backbone.cls_token is not None: + enc_out = enc_out[:, 1:, :] # skip CLS + + for j in range(num_modules): + if x_modules.grad is not None: + x_modules.grad.zero_() + scalar = enc_out[0, j].pow(2).sum() + scalar.backward(retain_graph=(j < num_modules - 1)) + grad = x_modules.grad[0] # (num_modules, D) + J[:, j] = grad.norm(dim=1) + + return J.to(torch.float64) + + +# --------------------------------------------------------------------------- +# Ground truth: pathway overlap from module gene sets +# --------------------------------------------------------------------------- + +def compute_pathway_overlap(module_names: List[str], module_gene_dict: dict, + metric: str = "jaccard") -> np.ndarray: + """Compute pairwise pathway overlap matrix from shared genes. + + metric: 'jaccard' (|A∩B| / |A∪B|) or 'overlap_coef' (|A∩B| / min(|A|,|B|)) + """ + n = len(module_names) + gene_sets = [] + for name in module_names: + genes = module_gene_dict.get(name, []) + gene_sets.append(set(genes)) + + overlap = np.zeros((n, n), dtype=np.float32) + for i in range(n): + for j in range(i + 1, n): + inter = len(gene_sets[i] & gene_sets[j]) + if inter == 0: + continue + if metric == "jaccard": + union = len(gene_sets[i] | gene_sets[j]) + score = inter / union if union > 0 else 0.0 + else: + min_size = min(len(gene_sets[i]), len(gene_sets[j])) + score = inter / min_size if min_size > 0 else 0.0 + overlap[i, j] = score + overlap[j, i] = score + return overlap + + +# --------------------------------------------------------------------------- +# Evaluation metrics +# --------------------------------------------------------------------------- + +def evaluate_interaction(model_matrix: np.ndarray, gt_matrix: np.ndarray, + top_k_fracs: List[float], n_permutations: int = 200): + """Compare model interaction matrix against ground truth overlap. + + Returns dict with spearman_r, auroc, precision@k, and permutation p-values. + """ + n = model_matrix.shape[0] + triu_idx = np.triu_indices(n, k=1) + model_flat = model_matrix[triu_idx] + gt_flat = gt_matrix[triu_idx] + + has_positive = gt_flat.max() > 0 + has_variation = model_flat.std() > 1e-10 + + rho, _ = spearmanr(model_flat, gt_flat) if has_variation else (0.0, 1.0) + + gt_binary = (gt_flat > 0).astype(np.float32) + n_pos = gt_binary.sum() + if n_pos > 0 and n_pos < len(gt_binary) and has_variation: + auroc = roc_auc_score(gt_binary, model_flat) + else: + auroc = 0.5 + + precisions = {} + n_pairs = len(model_flat) + for frac in top_k_fracs: + k = max(1, int(n_pairs * frac)) + top_k_idx = np.argsort(model_flat)[-k:] + precisions[f"precision@{frac}"] = float(gt_binary[top_k_idx].mean()) + + null_rhos = [] + null_aurocs = [] + rng = np.random.default_rng(42) + for _ in range(n_permutations): + perm = rng.permutation(n) + perm_matrix = model_matrix[perm][:, perm] + perm_flat = perm_matrix[triu_idx] + r, _ = spearmanr(perm_flat, gt_flat) if has_variation else (0.0, 1.0) + null_rhos.append(r) + if n_pos > 0 and n_pos < len(gt_binary): + null_aurocs.append(roc_auc_score(gt_binary, perm_flat)) + else: + null_aurocs.append(0.5) + + p_rho = (np.sum(np.array(null_rhos) >= rho) + 1) / (n_permutations + 1) + p_auroc = (np.sum(np.array(null_aurocs) >= auroc) + 1) / (n_permutations + 1) + + return { + "spearman_r": float(rho), + "spearman_p_perm": float(p_rho), + "auroc": float(auroc), + "auroc_p_perm": float(p_auroc), + "null_rho_mean": float(np.mean(null_rhos)), + "null_rho_std": float(np.std(null_rhos)), + **precisions, + } + + +# --------------------------------------------------------------------------- +# Visualization +# --------------------------------------------------------------------------- + +def plot_interaction_matrix(matrix: np.ndarray, title: str, path: Path, + module_families: Optional[List[str]] = None): + """Plot heatmap of interaction matrix, optionally sorted by family.""" + if module_families is not None: + order = np.argsort(module_families) + matrix = matrix[order][:, order] + sorted_families = [module_families[i] for i in order] + else: + sorted_families = None + + fig, ax = plt.subplots(figsize=(10, 9)) + vmax = np.percentile(np.abs(matrix), 99) + im = ax.imshow(matrix, cmap="RdBu_r", vmin=-vmax, vmax=vmax, aspect="equal") + ax.set_title(title, fontsize=12) + ax.set_xlabel("Module j") + ax.set_ylabel("Module i") + plt.colorbar(im, ax=ax, fraction=0.046) + + if sorted_families is not None: + unique_fams = [] + boundaries = [0] + prev = sorted_families[0] + for idx, f in enumerate(sorted_families): + if f != prev: + unique_fams.append(prev) + boundaries.append(idx) + prev = f + unique_fams.append(prev) + boundaries.append(len(sorted_families)) + for b in boundaries[1:-1]: + ax.axhline(b - 0.5, color="black", linewidth=0.3, alpha=0.5) + ax.axvline(b - 0.5, color="black", linewidth=0.3, alpha=0.5) + + plt.tight_layout() + fig.savefig(path, dpi=150, bbox_inches="tight") + plt.close(fig) + + +def plot_scatter(model_flat, gt_flat, title, path): + """Scatter plot of model interaction vs ground truth overlap.""" + fig, ax = plt.subplots(figsize=(6, 5)) + subsample = min(len(model_flat), 50000) + if subsample < len(model_flat): + rng = np.random.default_rng(0) + idx = rng.choice(len(model_flat), size=subsample, replace=False) + model_flat = model_flat[idx] + gt_flat = gt_flat[idx] + ax.scatter(gt_flat, model_flat, alpha=0.1, s=2, rasterized=True) + ax.set_xlabel("Ground truth overlap (Jaccard)") + ax.set_ylabel("Model interaction score") + ax.set_title(title) + plt.tight_layout() + fig.savefig(path, dpi=150, bbox_inches="tight") + plt.close(fig) + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +def main(): + args = parse_args() + seed_all(args.seed) + device = torch.device(args.device if torch.cuda.is_available() else "cpu") + output_dir = Path(args.output_dir) + ensure_dir(output_dir) + + # Resolve all paths to absolute (relative to CWD) so that load_backbone's + # internal resolve_path (which resolves relative to cell_annotation/) is bypassed. + for attr in ("performer_checkpoint", "config_pkl", "module_data_dir", + "gene_list_path", "gene_mapping_path", "gene2vec_data_path", + "gene2ens_file", "data_path"): + val = getattr(args, attr, None) + if val is not None and not Path(val).is_absolute(): + setattr(args, attr, str(Path(val).resolve())) + + logger.info("Loading backbone...") + backbone, module_data, model_config = load_backbone(args, device) + backbone.eval() + + num_modules = len(module_data) + base_dim = int(model_config["base_dim"]) + expected_genes = int(model_config.get("num_genes", 27874)) + logger.info("Model: %d modules, dim=%d, genes=%d", num_modules, base_dim, expected_genes) + + module_json_path = args.module_json or str(Path(args.module_data_dir) / "merged_modules.json") + module_names = load_module_names(module_json_path) + with open(module_json_path, "r", encoding="utf-8") as f: + module_gene_dict = json.load(f) + + preprocessor = None + if args.data_mode == "h5ad": + preprocessor = DataPreprocessor( + gene2ens_file=args.gene2ens_file, + gene2ens_column=args.gene2ens_column, + model_genes_file=args.gene_list_path, + ) + preprocessor.load_gene_mapping() + preprocessor.load_model_genes() + + dataset, reindex_state, n_cells = build_dataset(args, preprocessor) + logger.info("Dataset: %d cells", n_cells) + + # --- Method A: Cosine --- + logger.info("=" * 60) + logger.info("Method A: Cosine similarity of module embeddings") + logger.info("=" * 60) + loader_a = DataLoader(dataset, batch_size=args.batch_size, shuffle=False, + num_workers=args.num_workers, pin_memory=True) + cos_matrix, mean_emb = compute_cosine_interaction( + backbone, loader_a, preprocessor, reindex_state, + device, num_modules, base_dim, expected_genes, + pool_mode=args.pool_mode, + ) + + # --- Method C: Jacobian --- + jac_matrix = None + if not args.skip_jacobian: + logger.info("=" * 60) + logger.info("Method C: Jacobian-based module interaction") + logger.info("=" * 60) + jac_n = min(args.jacobian_cells, n_cells) + if jac_n < n_cells: + rng = np.random.default_rng(args.seed) + jac_indices = np.sort(rng.choice(n_cells, size=jac_n, replace=False)) + jac_dataset = Subset(dataset, jac_indices.tolist()) + else: + jac_dataset = dataset + loader_c = DataLoader(jac_dataset, batch_size=args.jacobian_batch, shuffle=False, + num_workers=0, pin_memory=True) + jac_matrix = compute_jacobian_interaction( + backbone, loader_c, preprocessor, reindex_state, + device, num_modules, base_dim, expected_genes, + max_cells=jac_n, batch_size=args.jacobian_batch, + ) + + # --- Ground truth overlap --- + logger.info("Computing pathway gene overlap (Jaccard)...") + gt_jaccard = compute_pathway_overlap(module_names, module_gene_dict, metric="jaccard") + gt_overlap = compute_pathway_overlap(module_names, module_gene_dict, metric="overlap_coef") + + # --- Evaluate --- + top_k_fracs = [float(f) for f in args.top_k_fractions.split(",")] + results = {} + + logger.info("Evaluating Method A vs Jaccard overlap...") + results["cosine_vs_jaccard"] = evaluate_interaction( + cos_matrix, gt_jaccard, top_k_fracs, args.null_permutations) + results["cosine_vs_overlap_coef"] = evaluate_interaction( + cos_matrix, gt_overlap, top_k_fracs, args.null_permutations) + + if jac_matrix is not None: + logger.info("Evaluating Method C vs Jaccard overlap...") + jac_sym = (jac_matrix + jac_matrix.T) / 2 + results["jacobian_vs_jaccard"] = evaluate_interaction( + jac_sym, gt_jaccard, top_k_fracs, args.null_permutations) + results["jacobian_vs_overlap_coef"] = evaluate_interaction( + jac_sym, gt_overlap, top_k_fracs, args.null_permutations) + + # --- Save --- + np.save(output_dir / "cosine_interaction.npy", cos_matrix) + np.save(output_dir / "mean_module_embeddings.npy", mean_emb) + np.save(output_dir / "gt_jaccard.npy", gt_jaccard) + np.save(output_dir / "gt_overlap_coef.npy", gt_overlap) + if jac_matrix is not None: + np.save(output_dir / "jacobian_interaction.npy", jac_matrix) + + with open(output_dir / "metrics.json", "w", encoding="utf-8") as f: + json.dump(results, f, indent=2, ensure_ascii=False) + + # --- Plots --- + meta = build_module_metadata(module_names) + families = [row["family"] for row in meta] + + plot_interaction_matrix(cos_matrix, "Method A: Cosine Similarity", + output_dir / "heatmap_cosine.png", families) + if jac_matrix is not None: + plot_interaction_matrix(jac_sym, "Method C: Jacobian Interaction", + output_dir / "heatmap_jacobian.png", families) + plot_interaction_matrix(gt_jaccard, "Ground Truth: Jaccard Gene Overlap", + output_dir / "heatmap_gt_jaccard.png", families) + + triu_idx = np.triu_indices(num_modules, k=1) + plot_scatter(cos_matrix[triu_idx], gt_jaccard[triu_idx], + "Cosine vs Jaccard", output_dir / "scatter_cosine_vs_jaccard.png") + if jac_matrix is not None: + plot_scatter(jac_sym[triu_idx], gt_jaccard[triu_idx], + "Jacobian vs Jaccard", output_dir / "scatter_jacobian_vs_jaccard.png") + + # --- Print summary --- + logger.info("=" * 60) + logger.info("Results Summary") + logger.info("=" * 60) + for method_key, metrics in results.items(): + logger.info(" %s:", method_key) + logger.info(" Spearman r = %.4f (p_perm = %.4f)", metrics["spearman_r"], metrics["spearman_p_perm"]) + logger.info(" AUROC = %.4f (p_perm = %.4f)", metrics["auroc"], metrics["auroc_p_perm"]) + for k in top_k_fracs: + logger.info(" Precision@%.2f = %.4f", k, metrics[f"precision@{k}"]) + logger.info("Output saved to: %s", output_dir) + + +if __name__ == "__main__": + main() diff --git a/BioMod/interpretability/attention/module_gene_attention.py b/BioMod/interpretability/attention/module_gene_attention.py index c94966ce083776804c9de16a47e9ba228d41f52a..2f0cc58b7e5aba7ff4cacd688a8e0e45351848ce 100644 --- a/BioMod/interpretability/attention/module_gene_attention.py +++ b/BioMod/interpretability/attention/module_gene_attention.py @@ -1,1090 +1,1093 @@ -"""ModuleEmbedding Attention-based Gene Interpretability Analysis (Streaming). - -Interpretability analysis using single-head content-based attention pooling in ModuleEmbedding. -Streaming computing mode: memory usage O(n_celltypes × n_genes). - -Analysis content: - 1. Global gene importance ranking - 2. Cell type specificity (z-score, Cohen's d, Welch's t) - 3. Marker gene verification - 4. Attention-Expression correlation - 5. Random baseline comparison (KL divergence) - 6. Attention concentration (Gini coefficient) - 7. Pathway Enrichment Analysis - 8. DEG comparison - -Usage: - python module_gene_attention.py \ - --performer_checkpoint ../../mae_tost_48M_cross_attn_only_pure_memory/mae_tost_48M_cross_attn_only_pure_memory_epoch12.pth \ - --config_pkl ../../mae_tost_48M_cross_attn_only_pure_memory/mae_tost_48M_cross_attn_only_pure_memory.pkl \ - --module_data_dir ../../module_data_all \ - --gene_list_path ../../../selected_genes_27k.txt \ - --gene_mapping_path ../../../gene2ens.csv \ - --gene2vec_data_path ../../../selected_gene2vec_27k.npy \ - --gene2ens_file ../../../gene2ens.csv \ - --data_path ../../../zero-shot-batch/pancreas_scib.h5ad \ - --celltype_column celltype \ - --batch_size 64 \ - --output_dir ../results/gene_attention/pancreas -""" -from __future__ import annotations - -import argparse -import json -import logging -import sys -from pathlib import Path -from typing import Dict, List, Optional, Tuple - -import numpy as np -import pandas as pd -import torch -from torch.utils.data import DataLoader - -import matplotlib -matplotlib.use("Agg") -import matplotlib.pyplot as plt -import seaborn as sns - -from scipy.stats import spearmanr, t as t_dist - -HERE = Path(__file__).resolve() -PERFORMER_MODULE = HERE.parents[2] -REPO_ROOT = HERE.parents[3] -for p in (str(PERFORMER_MODULE), str(REPO_ROOT)): - if p not in sys.path: - sys.path.insert(0, p) - -# Publication utilities -MODULE_IMPORTANCE_DIR = PERFORMER_MODULE / "interpretability" / "module_importance" -if str(MODULE_IMPORTANCE_DIR) not in sys.path: - sys.path.insert(0, str(MODULE_IMPORTANCE_DIR)) - -from pub_utils import setup_publication_style, savefig as pub_savefig - -from cell_annotation.performer_module_finetune import ( - DataPreprocessor, - load_backbone, - seed_all, -) -from interpretability.utils import ( - MatrixRowDataset, - build_module_metadata, - ensure_dir, - load_module_names, -) - -logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") -logger = logging.getLogger(__name__) - - -# --------------------------------------------------------------------------- -# Argparse -# --------------------------------------------------------------------------- - -def parse_args() -> argparse.Namespace: - p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - - p.add_argument("--performer_checkpoint", type=str, required=True) - p.add_argument("--config_pkl", type=str, required=True) - p.add_argument("--module_data_dir", type=str, default="../../module_data_all") - p.add_argument("--module_json", type=str, default=None) - p.add_argument("--gene_list_path", type=str, default="../../../selected_genes_27k.txt") - p.add_argument("--gene_mapping_path", type=str, default="../../../gene2ens.csv") - p.add_argument("--gene2vec_data_path", type=str, default="../../../selected_gene2vec_27k.npy") - p.add_argument("--gene2ens_file", type=str, default="../../../gene2ens.csv") - p.add_argument("--gene2ens_column", type=str, default="Ensembl gene ID") - p.add_argument("--shuffle_module_genes", action="store_true") - p.add_argument("--shuffle_seed", type=int, default=2021) - - p.add_argument("--data_path", type=str, required=True) - p.add_argument("--celltype_column", type=str, default="celltype") - p.add_argument("--gene_name_column", type=str, default="auto") - p.add_argument("--dataset_name", type=str, default="gene_attention") - p.add_argument("--max_cells", type=int, default=None) - - p.add_argument("--batch_size", type=int, default=64) - p.add_argument("--num_workers", type=int, default=0) - p.add_argument("--device", type=str, default="cuda") - p.add_argument("--seed", type=int, default=2021) - - p.add_argument("--top_k_genes", type=int, default=20) - p.add_argument("--marker_db", type=str, default=None) - p.add_argument("--deg_csv", type=str, default=None) - p.add_argument("--min_cells", type=int, default=30) - - p.add_argument("--output_dir", type=str, required=True) - return p.parse_args() - - -# --------------------------------------------------------------------------- -# Built-in pancreas marker genes (from PanglaoDB + CellMarker) -# --------------------------------------------------------------------------- - -PANCREAS_MARKERS: Dict[str, List[str]] = { - "alpha": ["GCG", "ARX", "IRX2", "TTR", "LOXL4", "GC", "CRYBA2", "DPP4", "IRX1", "PCSK2"], - "beta": ["INS", "IAPP", "MAFA", "NKX6-1", "PDX1", "HADH", "SLC30A8", "NPTX2", "DLK1", "PCSK1"], - "delta": ["SST", "HHEX", "RBP4", "LEPR", "PCSK1", "SEC11C"], - "gamma": ["PPY", "SERTM1", "CARTPT", "MEIS2", "ETV1", "ABCC9"], - "ductal": ["KRT19", "KRT7", "SPP1", "CFTR", "MUC1", "AMBP", "ANXA4", "BICC1", "SOX9"], - "acinar": ["PRSS1", "CELA3A", "CELA2A", "CPA1", "CPA2", "PNLIP", "CEL", "CLPS", "REG1A", "CTRB1"], - "activated_stellate": ["COL1A1", "COL1A2", "COL3A1", "ACTA2", "TAGLN", "RGS5", "PDGFRB"], - "endothelial": ["PECAM1", "VWF", "CDH5", "KDR", "ENG", "PLVAP", "ESAM"], - "macrophage": ["CD68", "CD163", "CSF1R", "MARCO", "MSR1", "C1QA", "C1QB"], - "epsilon": ["GHRL", "PCSK1", "ARX"], -} - -PANCREAS_COLORS: Dict[str, str] = { - "alpha": "#E41A1C", "beta": "#377EB8", "delta": "#4DAF4A", - "gamma": "#984EA3", "ductal": "#FF7F00", "acinar": "#A65628", - "activated_stellate": "#F781BF", "endothelial": "#66C2A5", - "macrophage": "#FC8D62", "epsilon": "#8DA0CB", "t_cell": "#E78AC3", - "schwann": "#B3B3B3", "mast": "#FDBF6F", "quiescent_stellate": "#CAB2D6", -} - - -# --------------------------------------------------------------------------- -# Streaming attention extraction + statistics accumulation -# --------------------------------------------------------------------------- - -@torch.no_grad() -def extract_attention_streaming( - backbone, loader, preprocessor, reindex_state, device, - cell_labels: np.ndarray, num_modules: int, module_sizes: List[int], -) -> Tuple: - """Extract attention and accumulate statistics in streaming fashion. - - Returns: - global_mean, global_std, ct_mean, ct_std, ct_count, n_cells, - expr_mean, attn_expr_prod_sum - """ - backbone.eval() - module_embed = backbone.module_embed - - global_sum = {i: np.zeros(module_sizes[i], dtype=np.float64) for i in range(num_modules)} - global_sum_sq = {i: np.zeros(module_sizes[i], dtype=np.float64) for i in range(num_modules)} - expr_sum = {i: np.zeros(module_sizes[i], dtype=np.float64) for i in range(num_modules)} - attn_expr_prod_sum = {i: np.zeros(module_sizes[i], dtype=np.float64) for i in range(num_modules)} - - celltypes = np.unique(cell_labels) - ct_sum = {ct: {i: np.zeros(module_sizes[i], dtype=np.float64) for i in range(num_modules)} for ct in celltypes} - ct_sum_sq = {ct: {i: np.zeros(module_sizes[i], dtype=np.float64) for i in range(num_modules)} for ct in celltypes} - ct_count = {ct: 0 for ct in celltypes} - - n_cells = 0 - cell_idx = 0 - - for batch_idx, x in enumerate(loader): - x = x.to(device) - if reindex_state is not None: - x = preprocessor.reindex_tensor_to_model_space(x, reindex_state=reindex_state) - - batch_size = x.shape[0] - batch_labels = cell_labels[cell_idx:cell_idx + batch_size] - cell_idx += batch_size - - for module_i, gene_idx in enumerate(module_embed.module_sets): - gene_idx_dev = gene_idx.to(device) - expr_slice = torch.index_select(x, dim=1, index=gene_idx_dev) - - gene_emb = module_embed.gene_embed(gene_idx_dev) - gene_emb = gene_emb.unsqueeze(0).expand(batch_size, -1, -1) - gene_tok = module_embed._fuse_expr_and_gene_emb(expr_slice, gene_emb) - gene_tok = module_embed.proj(gene_tok) - - attn = module_embed.attn_score(gene_tok).squeeze(-1) - attn = torch.softmax(attn, dim=1) - attn_np = attn.cpu().numpy().astype(np.float64) - - # Expression for correlation - expr_np = expr_slice.cpu().numpy().astype(np.float64) - expr_sum[module_i] += expr_np.sum(axis=0) - attn_expr_prod_sum[module_i] += (attn_np * expr_np).sum(axis=0) - - # Global accumulation - global_sum[module_i] += attn_np.sum(axis=0) - global_sum_sq[module_i] += (attn_np ** 2).sum(axis=0) - - # Per-celltype accumulation - for ct in celltypes: - mask = batch_labels == ct - if mask.any(): - ct_attn = attn_np[mask] - ct_sum[ct][module_i] += ct_attn.sum(axis=0) - ct_sum_sq[ct][module_i] += (ct_attn ** 2).sum(axis=0) - - for ct in celltypes: - ct_count[ct] += int((batch_labels == ct).sum()) - - n_cells += batch_size - if (batch_idx + 1) % 50 == 0: - logger.info(" [attention] batch %d, %d cells processed", batch_idx + 1, n_cells) - - # Compute means and stds - global_mean = {} - global_std = {} - for i in range(num_modules): - global_mean[i] = (global_sum[i] / n_cells).astype(np.float32) - variance = global_sum_sq[i] / n_cells - global_mean[i].astype(np.float64) ** 2 - global_std[i] = np.sqrt(np.maximum(variance, 0)).astype(np.float32) - - ct_mean = {} - ct_std = {} - for ct in celltypes: - ct_mean[ct] = {} - ct_std[ct] = {} - for i in range(num_modules): - if ct_count[ct] > 0: - ct_mean[ct][i] = (ct_sum[ct][i] / ct_count[ct]).astype(np.float32) - else: - ct_mean[ct][i] = np.zeros(module_sizes[i], dtype=np.float32) - if ct_count[ct] > 1: - var = ct_sum_sq[ct][i] / ct_count[ct] - ct_mean[ct][i].astype(np.float64) ** 2 - ct_std[ct][i] = np.sqrt(np.maximum(var, 0)).astype(np.float32) - else: - ct_std[ct][i] = np.zeros(module_sizes[i], dtype=np.float32) - - expr_mean = {i: (expr_sum[i] / n_cells).astype(np.float32) for i in range(num_modules)} - - logger.info("Extracted attention for %d cells across %d modules (streaming)", n_cells, num_modules) - return global_mean, global_std, ct_mean, ct_std, ct_count, n_cells, expr_mean, attn_expr_prod_sum - - -# --------------------------------------------------------------------------- -# Analysis functions -# --------------------------------------------------------------------------- - -def build_global_importance_df( - global_mean: Dict[int, np.ndarray], - global_std: Dict[int, np.ndarray], - module_gene_names: Dict[int, List[str]], - num_modules: int, -) -> pd.DataFrame: - """Build global gene importance DataFrame from streaming means.""" - rows = [] - for module_i in range(num_modules): - mean_attn = global_mean[module_i] - std_attn = global_std[module_i] - genes = module_gene_names[module_i] - rank_order = np.argsort(-mean_attn) - for g_idx, gene in enumerate(genes): - rows.append({ - "module_idx": module_i, - "gene": gene, - "mean_attention": float(mean_attn[g_idx]), - "std_attention": float(std_attn[g_idx]), - "rank_in_module": int(np.where(rank_order == g_idx)[0][0]) + 1, - "module_size": len(genes), - }) - df = pd.DataFrame(rows) - df["normalized_rank"] = df["rank_in_module"] / df["module_size"] - return df - - -def build_celltype_dfs( - ct_mean: Dict[str, Dict[int, np.ndarray]], - ct_count: Dict[str, int], - module_gene_names: Dict[int, List[str]], - num_modules: int, -) -> Dict[str, pd.DataFrame]: - """Build per-celltype DataFrames from streaming means.""" - ct_dfs = {} - for ct, module_means in ct_mean.items(): - rows = [] - for module_i in range(num_modules): - mean_attn = module_means[module_i] - genes = module_gene_names[module_i] - for g_idx, gene in enumerate(genes): - rows.append({ - "module_idx": module_i, - "gene": gene, - "mean_attention": float(mean_attn[g_idx]), - "n_cells": ct_count[ct], - }) - ct_dfs[ct] = pd.DataFrame(rows) - return ct_dfs - - -def compute_specificity_from_means( - global_mean: Dict[int, np.ndarray], - ct_mean: Dict[str, Dict[int, np.ndarray]], - module_gene_names: Dict[int, List[str]], - num_modules: int, -) -> pd.DataFrame: - """Fold-change specificity (legacy): max_ct(mean_attn_ct) / global_mean_attn.""" - celltypes = list(ct_mean.keys()) - rows = [] - for module_i in range(num_modules): - genes = module_gene_names[module_i] - g_mean = global_mean[module_i] - for g_idx, gene in enumerate(genes): - ct_scores = {ct: float(ct_mean[ct][module_i][g_idx]) for ct in celltypes} - top_ct = max(ct_scores, key=ct_scores.get) - top_score = ct_scores[top_ct] - fold_change = top_score / max(float(g_mean[g_idx]), 1e-10) - rows.append({ - "module_idx": module_i, "gene": gene, "top_celltype": top_ct, - "top_ct_mean_attn": top_score, "global_mean_attn": float(g_mean[g_idx]), - "fold_change": fold_change, - }) - return pd.DataFrame(rows) - - -def compute_specificity_zscore( - global_mean: Dict[int, np.ndarray], - global_std: Dict[int, np.ndarray], - ct_mean: Dict[str, Dict[int, np.ndarray]], - ct_count: Dict[str, int], - module_gene_names: Dict[int, List[str]], - num_modules: int, - min_cells: int = 30, -) -> pd.DataFrame: - """Z-score specificity: (ct_mean - global_mean) / global_std.""" - valid_cts = [ct for ct, n in ct_count.items() if n >= min_cells] - rows = [] - for module_i in range(num_modules): - genes = module_gene_names[module_i] - g_mean = global_mean[module_i] - g_std = global_std[module_i] - for g_idx, gene in enumerate(genes): - std_val = max(float(g_std[g_idx]), 1e-8) - ct_zscores = {} - for ct in valid_cts: - z = (float(ct_mean[ct][module_i][g_idx]) - float(g_mean[g_idx])) / std_val - ct_zscores[ct] = z - top_ct = max(ct_zscores, key=ct_zscores.get) - rows.append({ - "module_idx": module_i, "gene": gene, "top_celltype": top_ct, - "z_score": ct_zscores[top_ct], - "global_mean_attn": float(g_mean[g_idx]), - "ct_mean_attn": float(ct_mean[top_ct][module_i][g_idx]), - }) - return pd.DataFrame(rows) - - -def compute_cohens_d( - global_mean: Dict[int, np.ndarray], - ct_mean: Dict[str, Dict[int, np.ndarray]], - ct_std: Dict[str, Dict[int, np.ndarray]], - ct_count: Dict[str, int], - module_gene_names: Dict[int, List[str]], - num_modules: int, - n_cells: int, - min_cells: int = 30, -) -> pd.DataFrame: - """Cohen's d effect size for cell-type specificity.""" - valid_cts = [ct for ct, n in ct_count.items() if n >= min_cells] - rows = [] - for module_i in range(num_modules): - genes = module_gene_names[module_i] - for g_idx, gene in enumerate(genes): - best_d = -np.inf - best_ct = valid_cts[0] - for ct in valid_cts: - n_ct = ct_count[ct] - n_other = n_cells - n_ct - ct_m = float(ct_mean[ct][module_i][g_idx]) - ct_s = float(ct_std[ct][module_i][g_idx]) - other_m = (float(global_mean[module_i][g_idx]) * n_cells - ct_m * n_ct) / max(n_other, 1) - pooled = max(ct_s, 1e-10) - d = (ct_m - other_m) / pooled - if d > best_d: - best_d = d - best_ct = ct - rows.append({"module_idx": module_i, "gene": gene, "top_celltype": best_ct, "cohens_d": best_d}) - return pd.DataFrame(rows) - - -def compute_welch_ttest( - global_mean: Dict[int, np.ndarray], - global_std: Dict[int, np.ndarray], - ct_mean: Dict[str, Dict[int, np.ndarray]], - ct_std: Dict[str, Dict[int, np.ndarray]], - ct_count: Dict[str, int], - module_gene_names: Dict[int, List[str]], - num_modules: int, - n_cells: int, - min_cells: int = 30, -) -> pd.DataFrame: - """Welch's t-test from streaming statistics with BH FDR correction.""" - valid_cts = [ct for ct, n in ct_count.items() if n >= min_cells] - rows = [] - for module_i in range(num_modules): - genes = module_gene_names[module_i] - for g_idx, gene in enumerate(genes): - best_t = -np.inf - best_ct = valid_cts[0] - best_pval = 1.0 - for ct in valid_cts: - n_ct = ct_count[ct] - ct_m = float(ct_mean[ct][module_i][g_idx]) - ct_v = float(ct_std[ct][module_i][g_idx]) ** 2 - g_m = float(global_mean[module_i][g_idx]) - g_v = float(global_std[module_i][g_idx]) ** 2 - se = np.sqrt(ct_v / max(n_ct, 1) + g_v / max(n_cells, 1)) - if se < 1e-12: - continue - t_stat = (ct_m - g_m) / se - num = (ct_v / n_ct + g_v / n_cells) ** 2 - denom = (ct_v / n_ct) ** 2 / max(n_ct - 1, 1) + (g_v / n_cells) ** 2 / max(n_cells - 1, 1) - df = num / max(denom, 1e-12) - pval = float(t_dist.sf(abs(t_stat), df) * 2) - if t_stat > best_t: - best_t = t_stat - best_ct = ct - best_pval = pval - rows.append({"module_idx": module_i, "gene": gene, "top_celltype": best_ct, - "t_stat": best_t, "pval": best_pval}) - - df = pd.DataFrame(rows) - # BH FDR correction - pvals = df["pval"].values - n = len(pvals) - sorted_idx = np.argsort(pvals) - sorted_pvals = pvals[sorted_idx] - fdr = np.minimum(1.0, sorted_pvals * n / (np.arange(n) + 1)) - for i in range(n - 2, -1, -1): - fdr[i] = min(fdr[i], fdr[i + 1]) - fdr_all = np.empty(n) - fdr_all[sorted_idx] = fdr - df["fdr"] = fdr_all - return df - - -def compute_attention_expression_corr( - global_mean: Dict[int, np.ndarray], - expr_mean: Dict[int, np.ndarray], - module_gene_names: Dict[int, List[str]], - num_modules: int, -) -> Dict[str, float]: - """Correlation between mean attention and mean expression across genes.""" - all_attn = [] - all_expr = [] - for module_i in range(num_modules): - all_attn.extend(global_mean[module_i].tolist()) - all_expr.extend(expr_mean[module_i].tolist()) - all_attn = np.array(all_attn) - all_expr = np.array(all_expr) - mask = (all_attn > 0) & (all_expr > 0) - if mask.sum() > 10: - pearson_r = float(np.corrcoef(all_attn[mask], all_expr[mask])[0, 1]) - sp_r, sp_p = spearmanr(all_attn[mask], all_expr[mask]) - else: - pearson_r, sp_r, sp_p = 0.0, 0.0, 1.0 - return {"pearson_r": pearson_r, "spearman_r": float(sp_r), - "spearman_pval": float(sp_p), "n_genes": int(mask.sum())} - - -def compute_random_baseline( - global_mean: Dict[int, np.ndarray], - num_modules: int, -) -> Dict[str, float]: - """Compare learned attention to uniform baseline.""" - kl_divs = [] - learned_entropies = [] - for module_i in range(num_modules): - attn = global_mean[module_i].astype(np.float64) - attn = attn / max(attn.sum(), 1e-10) - g_size = len(attn) - uniform = np.ones(g_size) / g_size - mask = attn > 1e-12 - kl = float(np.sum(attn[mask] * np.log(attn[mask] / uniform[mask]))) - kl_divs.append(kl) - h_learned = -float(np.sum(attn[mask] * np.log(attn[mask]))) - h_max = np.log(g_size) - learned_entropies.append(h_learned / max(h_max, 1e-10)) - return { - "mean_kl_divergence": float(np.mean(kl_divs)), - "std_kl_divergence": float(np.std(kl_divs)), - "mean_normalized_entropy": float(np.mean(learned_entropies)), - "entropy_reduction": float(1.0 - np.mean(learned_entropies)), - "n_modules": num_modules, - } - - -def compute_attention_concentration( - global_mean: Dict[int, np.ndarray], - ct_mean: Dict[str, Dict[int, np.ndarray]], - ct_count: Dict[str, int], - num_modules: int, - min_cells: int = 30, -) -> pd.DataFrame: - """Gini coefficient per module measuring attention selectivity.""" - def gini(arr): - arr = np.sort(arr.flatten()) - n = len(arr) - if n == 0 or arr.sum() == 0: - return 0.0 - index = np.arange(1, n + 1) - return float((2 * np.sum(index * arr) - (n + 1) * np.sum(arr)) / (n * np.sum(arr))) - - valid_cts = [ct for ct, n in ct_count.items() if n >= min_cells] - rows = [] - for module_i in range(num_modules): - g_attn = global_mean[module_i] - g_gini = gini(g_attn) - ct_ginis = [gini(ct_mean[ct][module_i]) for ct in valid_cts] - rows.append({ - "module_idx": module_i, "global_gini": g_gini, - "mean_ct_gini": float(np.mean(ct_ginis)) if ct_ginis else 0.0, - "module_size": len(g_attn), - }) - return pd.DataFrame(rows) - - -def compute_marker_rank_within_celltype( - ct_mean: Dict[str, Dict[int, np.ndarray]], - ct_count: Dict[str, int], - module_gene_names: Dict[int, List[str]], - num_modules: int, - marker_dict: Dict[str, List[str]], - min_cells: int = 30, -) -> pd.DataFrame: - """Rank marker genes within their expected cell type by attention.""" - rows = [] - for ct, markers in marker_dict.items(): - if ct not in ct_count or ct_count[ct] < min_cells: - continue - gene_attn = {} - for module_i in range(num_modules): - genes = module_gene_names[module_i] - attn = ct_mean[ct][module_i] - for g_idx, gene in enumerate(genes): - if gene not in gene_attn or float(attn[g_idx]) > gene_attn[gene]: - gene_attn[gene] = float(attn[g_idx]) - sorted_genes = sorted(gene_attn.items(), key=lambda x: -x[1]) - gene_rank = {g: rank + 1 for rank, (g, _) in enumerate(sorted_genes)} - n_total = len(gene_rank) - for marker in markers: - if marker in gene_rank: - rank = gene_rank[marker] - rows.append({ - "celltype": ct, "marker_gene": marker, "rank": rank, - "percentile": rank / n_total * 100, "n_total_genes": n_total, - "attention": gene_attn[marker], - }) - return pd.DataFrame(rows) - - -# --------------------------------------------------------------------------- -# Marker gene validation -# --------------------------------------------------------------------------- - -def validate_markers( - specificity_df: pd.DataFrame, - marker_dict: Dict[str, List[str]], - sort_col: str = "fold_change", -) -> Dict[str, object]: - """Validate whether high-attention genes match known markers.""" - results = {} - all_genes_in_model = set(specificity_df["gene"].unique()) - for ct, markers in marker_dict.items(): - markers_in_model = [g for g in markers if g in all_genes_in_model] - if not markers_in_model: - continue - ct_specific = specificity_df[specificity_df["top_celltype"] == ct].copy() - ct_specific = ct_specific.sort_values(sort_col, ascending=False).reset_index(drop=True) - marker_set = set(markers_in_model) - n_total = len(ct_specific) - if n_total == 0: - continue - top_50_genes = set(ct_specific.head(50)["gene"]) - top_100_genes = set(ct_specific.head(100)["gene"]) - hits_50 = len(marker_set & top_50_genes) - hits_100 = len(marker_set & top_100_genes) - marker_rows = ct_specific[ct_specific["gene"].isin(marker_set)] - if len(marker_rows) > 0: - marker_ranks = (marker_rows.index + 1).tolist() - mean_rank = np.mean(marker_ranks) - median_rank = np.median(marker_ranks) - else: - mean_rank = median_rank = n_total - results[ct] = { - "n_markers_in_model": len(markers_in_model), - "markers_in_model": markers_in_model, - "n_genes_assigned_to_ct": n_total, - "hits_in_top50": hits_50, "hits_in_top100": hits_100, - "precision_at_50": hits_50 / min(50, n_total), - "precision_at_100": hits_100 / min(100, n_total), - "marker_mean_rank": float(mean_rank), - "marker_median_rank": float(median_rank), - "recall_at_50": hits_50 / len(markers_in_model) if markers_in_model else 0, - "recall_at_100": hits_100 / len(markers_in_model) if markers_in_model else 0, - } - return results - - -def validate_deg_correlation( - ct_dfs: Dict[str, pd.DataFrame], - deg_df: pd.DataFrame, -) -> Dict[str, Dict[str, float]]: - """Compare attention ranking with DEG ranking per cell type.""" - results = {} - for ct, attn_df in ct_dfs.items(): - ct_deg = deg_df[deg_df["cell_type"] == ct].copy() - if len(ct_deg) == 0: - continue - merged = attn_df.merge(ct_deg[["gene", "log2fc"]], on="gene", how="inner") - if len(merged) < 10: - continue - rho, pval = spearmanr(merged["mean_attention"], merged["log2fc"].abs()) - results[ct] = {"spearman_r": float(rho), "pval": float(pval), "n_genes": len(merged)} - return results - - -# --------------------------------------------------------------------------- -# Publication-quality visualization -# --------------------------------------------------------------------------- - -def plot_global_importance_pub(global_df, specificity_zscore_df, top_k, output_dir): - """Fig1: Top-K genes with std error bars, colored by top cell type.""" - setup_publication_style() - gene_stats = global_df.groupby("gene").agg( - mean_attn=("mean_attention", "mean"), std_attn=("std_attention", "mean"), - ).nlargest(top_k, "mean_attn") - gene_ct = specificity_zscore_df.drop_duplicates("gene").set_index("gene")["top_celltype"] - fig, ax = plt.subplots(figsize=(4.5, 5)) - genes = gene_stats.index.tolist() - y_pos = np.arange(len(genes)) - colors = [PANCREAS_COLORS.get(gene_ct.get(g, ""), "#999999") for g in genes] - ax.barh(y_pos, gene_stats["mean_attn"], xerr=gene_stats["std_attn"], - color=colors, edgecolor="none", height=0.7, capsize=2, error_kw={"linewidth": 0.8}) - ax.set_yticks(y_pos) - ax.set_yticklabels(genes) - ax.set_xlabel("Mean attention weight") - ax.set_title("Top genes by global attention") - ax.invert_yaxis() - pub_savefig(fig, output_dir / "fig1_global_importance") - logger.info("Saved fig1_global_importance (PDF+PNG)") - - -def plot_celltype_heatmap_pub(ct_dfs, ct_count, top_k, output_dir, min_cells=30): - """Fig2: Clustered heatmap of gene attention (z-score normalized).""" - setup_publication_style() - valid_cts = {ct for ct, n in ct_count.items() if n >= min_cells} - all_genes_attn = {} - for ct, df in ct_dfs.items(): - if ct not in valid_cts: - continue - all_genes_attn[ct] = df.groupby("gene")["mean_attention"].mean() - combined = pd.DataFrame(all_genes_attn) - gene_var = combined.var(axis=1).nlargest(top_k).index - heatmap_data = combined.loc[gene_var] - row_mean = heatmap_data.mean(axis=1) - row_std = heatmap_data.std(axis=1).replace(0, 1) - heatmap_z = heatmap_data.sub(row_mean, axis=0).div(row_std, axis=0) - g = sns.clustermap(heatmap_z, cmap="RdBu_r", center=0, figsize=(6, 7), - xticklabels=True, yticklabels=True, - dendrogram_ratio=(0.1, 0.15), cbar_kws={"label": "Z-score"}, - linewidths=0.3) - g.ax_heatmap.set_xlabel("") - g.ax_heatmap.set_ylabel("") - g.fig.suptitle(f"Top {top_k} variable genes (z-scored attention)", y=1.02, fontsize=11) - pub_savefig(g.fig, output_dir / "fig2_celltype_heatmap") - logger.info("Saved fig2_celltype_heatmap (PDF+PNG)") - - -def plot_specificity_volcano(welch_df, zscore_df, top_k, output_dir): - """Fig3: Volcano plot (z-score vs -log10 FDR).""" - setup_publication_style() - merged = zscore_df[["gene", "top_celltype", "z_score"]].drop_duplicates("gene") - welch_best = welch_df.loc[welch_df.groupby("gene")["t_stat"].idxmax()][["gene", "fdr"]] - merged = merged.merge(welch_best, on="gene", how="left") - merged["neg_log10_fdr"] = -np.log10(merged["fdr"].clip(lower=1e-300)) - fig, ax = plt.subplots(figsize=(5.5, 5)) - for ct in merged["top_celltype"].unique(): - subset = merged[merged["top_celltype"] == ct] - color = PANCREAS_COLORS.get(ct, "#999999") - ax.scatter(subset["z_score"], subset["neg_log10_fdr"], - c=color, s=8, alpha=0.5, label=ct, rasterized=True) - top_genes = merged.nlargest(top_k, "z_score") - for _, row in top_genes.iterrows(): - ax.annotate(row["gene"], (row["z_score"], row["neg_log10_fdr"]), - fontsize=6, ha="left", va="bottom") - ax.axhline(-np.log10(0.05), color="gray", linestyle="--", linewidth=0.6) - ax.axvline(2, color="gray", linestyle="--", linewidth=0.6) - ax.set_xlabel("Z-score (cell-type specificity)") - ax.set_ylabel(r"$-\log_{10}$(FDR)") - ax.set_title("Gene specificity volcano") - ax.legend(bbox_to_anchor=(1.02, 1), loc="upper left", fontsize=7, frameon=False, markerscale=2) - pub_savefig(fig, output_dir / "fig3_specificity_volcano") - logger.info("Saved fig3_specificity_volcano (PDF+PNG)") - - -def plot_marker_dotplot(marker_rank_df, output_dir): - """Fig4: Dot plot of marker gene rank percentile per cell type.""" - setup_publication_style() - if marker_rank_df.empty: - return - fig, ax = plt.subplots(figsize=(6, 4.5)) - celltypes = marker_rank_df["celltype"].unique() - ct_positions = {ct: i for i, ct in enumerate(celltypes)} - for _, row in marker_rank_df.iterrows(): - y = ct_positions[row["celltype"]] - size = max(10, 200 * (1 - row["percentile"] / 100)) - color = PANCREAS_COLORS.get(row["celltype"], "#999999") - ax.scatter(row["percentile"], y, s=size, c=color, alpha=0.7, - edgecolors="white", linewidth=0.3) - if row["percentile"] < 5: - ax.annotate(row["marker_gene"], (row["percentile"], y), - fontsize=6, ha="left", va="center", xytext=(3, 0), - textcoords="offset points") - ax.set_yticks(range(len(celltypes))) - ax.set_yticklabels(celltypes) - ax.set_xlabel("Rank percentile (lower = higher attention)") - ax.set_title("Marker gene rank within expected cell type") - ax.axvline(5, color="gray", linestyle="--", linewidth=0.6, alpha=0.5, label="Top 5%") - ax.legend(fontsize=7, frameon=False) - ax.set_xlim(-2, 105) - pub_savefig(fig, output_dir / "fig4_marker_dotplot") - logger.info("Saved fig4_marker_dotplot (PDF+PNG)") - - -def plot_marker_recall_pub(marker_results, ct_count, output_dir, min_cells=30): - """Fig5: Marker recall bar chart with random expectation line.""" - setup_publication_style() - if not marker_results: - return - cts = [ct for ct in marker_results.keys() if ct_count.get(ct, 0) >= min_cells] - if not cts: - return - recall_50 = [marker_results[ct]["recall_at_50"] for ct in cts] - recall_100 = [marker_results[ct]["recall_at_100"] for ct in cts] - x = np.arange(len(cts)) - width = 0.35 - fig, ax = plt.subplots(figsize=(max(5, len(cts) * 0.7), 4)) - colors_50 = [PANCREAS_COLORS.get(ct, "#999999") for ct in cts] - ax.bar(x - width / 2, recall_50, width, color=colors_50, alpha=0.9, label="Recall@50") - ax.bar(x + width / 2, recall_100, width, color=colors_50, alpha=0.5, label="Recall@100") - # Random expectation - for i, ct in enumerate(cts): - n_genes = marker_results[ct]["n_genes_assigned_to_ct"] - if n_genes > 0: - random_50 = min(50, n_genes) / n_genes - ax.hlines(random_50, i - 0.4, i + 0.4, colors="gray", linestyles=":", linewidth=0.8) - ax.set_xticks(x) - ax.set_xticklabels(cts, rotation=45, ha="right") - ax.set_ylabel("Recall") - ax.set_title("Marker gene recall by cell type") - ax.legend(fontsize=7, frameon=False) - pub_savefig(fig, output_dir / "fig5_marker_recall") - logger.info("Saved fig5_marker_recall (PDF+PNG)") - - -def plot_attention_vs_expression(global_mean, expr_mean, module_gene_names, num_modules, output_dir): - """Fig6: Scatter of mean attention vs mean expression.""" - setup_publication_style() - all_attn, all_expr = [], [] - for module_i in range(num_modules): - all_attn.extend(global_mean[module_i].tolist()) - all_expr.extend(expr_mean[module_i].tolist()) - all_attn = np.array(all_attn) - all_expr = np.array(all_expr) - mask = (all_attn > 0) & (all_expr > 0) - fig, ax = plt.subplots(figsize=(4.5, 4.5)) - ax.scatter(all_expr[mask], all_attn[mask], s=3, alpha=0.3, c="#377EB8", rasterized=True) - r = np.corrcoef(all_expr[mask], all_attn[mask])[0, 1] - ax.set_xlabel("Mean expression") - ax.set_ylabel("Mean attention weight") - ax.set_title("Attention vs expression") - ax.text(0.05, 0.95, f"Pearson r = {r:.3f}", transform=ax.transAxes, - fontsize=9, va="top", ha="left") - pub_savefig(fig, output_dir / "fig6_attn_vs_expression") - logger.info("Saved fig6_attn_vs_expression (PDF+PNG)") - - -def plot_concentration_pub(concentration_df, baseline_stats, output_dir): - """Fig7: Gini coefficient distribution (learned vs uniform).""" - setup_publication_style() - fig, axes = plt.subplots(1, 2, figsize=(8, 3.5)) - # Panel a: Gini histogram - ax = axes[0] - ax.hist(concentration_df["global_gini"], bins=30, alpha=0.7, color="#377EB8", label="Global") - ax.hist(concentration_df["mean_ct_gini"], bins=30, alpha=0.7, color="#E41A1C", label="Per cell-type") - ax.axvline(0, color="gray", linestyle="--", linewidth=0.6) - ax.set_xlabel("Gini coefficient") - ax.set_ylabel("Number of modules") - ax.set_title("Attention concentration") - ax.legend(fontsize=7, frameon=False) - # Panel b: Entropy comparison - ax = axes[1] - labels = ["Learned", "Uniform"] - values = [baseline_stats["mean_normalized_entropy"], 1.0] - bars = ax.bar(labels, values, color=["#377EB8", "#CCCCCC"], width=0.5) - ax.set_ylabel("Normalized entropy") - ax.set_title(f"Entropy reduction: {baseline_stats['entropy_reduction']:.1%}") - ax.set_ylim(0, 1.1) - plt.tight_layout() - pub_savefig(fig, output_dir / "fig7_concentration") - logger.info("Saved fig7_concentration (PDF+PNG)") - - -def plot_baseline_kl(baseline_stats, output_dir): - """Fig8: KL divergence summary.""" - setup_publication_style() - fig, ax = plt.subplots(figsize=(3.5, 3.5)) - ax.bar(["KL(learned||uniform)"], [baseline_stats["mean_kl_divergence"]], - yerr=[baseline_stats["std_kl_divergence"]], color="#377EB8", - capsize=5, width=0.4) - ax.set_ylabel("KL divergence (nats)") - ax.set_title("Deviation from uniform attention") - pub_savefig(fig, output_dir / "fig8_baseline_kl") - logger.info("Saved fig8_baseline_kl (PDF+PNG)") - - -def plot_deg_correlation_pub(deg_results, output_dir): - """Fig9: DEG correlation with significance.""" - setup_publication_style() - if not deg_results: - return - cts = sorted(deg_results.keys(), key=lambda c: deg_results[c]["spearman_r"], reverse=True) - rhos = [deg_results[ct]["spearman_r"] for ct in cts] - pvals = [deg_results[ct]["pval"] for ct in cts] - fig, ax = plt.subplots(figsize=(max(5, len(cts) * 0.6), 4)) - colors = ["#377EB8" if r > 0 else "#E41A1C" for r in rhos] - bars = ax.bar(range(len(cts)), rhos, color=colors, width=0.6) - for i, (bar, p) in enumerate(zip(bars, pvals)): - if p < 0.001: - ax.text(i, bar.get_height() + 0.02, "***", ha="center", fontsize=7) - elif p < 0.01: - ax.text(i, bar.get_height() + 0.02, "**", ha="center", fontsize=7) - elif p < 0.05: - ax.text(i, bar.get_height() + 0.02, "*", ha="center", fontsize=7) - ax.set_xticks(range(len(cts))) - ax.set_xticklabels(cts, rotation=45, ha="right") - ax.set_ylabel("Spearman r") - ax.set_title("Attention vs DEG correlation") - ax.axhline(0, color="gray", linewidth=0.5) - pub_savefig(fig, output_dir / "fig9_deg_correlation") - logger.info("Saved fig9_deg_correlation (PDF+PNG)") - - -# --------------------------------------------------------------------------- -# Enrichment analysis (optional) -# --------------------------------------------------------------------------- - -def run_enrichment_analysis( - global_df: pd.DataFrame, top_k: int, output_dir: Path, organism: str = "human", -) -> Optional[pd.DataFrame]: - """Run GO enrichment on top-K attention genes using gseapy.""" - try: - import gseapy as gp - except ImportError: - logger.warning("gseapy not installed, skipping enrichment. Install: pip install gseapy") - return None - top_genes = global_df.groupby("gene")["mean_attention"].mean().nlargest(top_k * 5).index.tolist() - logger.info("Running GO enrichment on top %d genes...", len(top_genes)) - try: - enr = gp.enrichr( - gene_list=top_genes, - gene_sets=["GO_Biological_Process_2021", "KEGG_2021_Human"], - organism=organism, - outdir=str(output_dir / "enrichment"), - no_plot=True, - ) - results_df = enr.results - results_df.to_csv(output_dir / "enrichment_results.csv", index=False) - logger.info("Enrichment results saved (%d terms)", len(results_df)) - return results_df - except Exception as e: - logger.warning("Enrichment analysis failed: %s", e) - return None - - -# --------------------------------------------------------------------------- -# Main -# --------------------------------------------------------------------------- - -def main(): - args = parse_args() - seed_all(args.seed) - - _script_dir = Path(__file__).resolve().parent - for attr in ("performer_checkpoint", "config_pkl", "module_data_dir", "gene_list_path", - "gene_mapping_path", "gene2vec_data_path", "gene2ens_file", "data_path"): - val = getattr(args, attr, None) - if val is not None and not Path(val).is_absolute(): - setattr(args, attr, str((_script_dir / val).resolve())) - - output_dir = ensure_dir(Path(args.output_dir)) - device = torch.device(args.device if torch.cuda.is_available() else "cpu") - logger.info("Device: %s", device) - - # --- Load model --- - logger.info("Loading backbone from %s", args.performer_checkpoint) - backbone, module_data, model_config = load_backbone(args, device) - backbone.eval() - - # --- Build module gene name mapping --- - model_genes_file = args.gene_list_path - with open(model_genes_file) as f: - model_genes = [line.strip() for line in f if line.strip()] - - gene2ens_df = pd.read_csv(args.gene2ens_file) - ens_to_symbol = dict(zip(gene2ens_df["Ensembl gene ID"], gene2ens_df.iloc[:, 0])) - - module_sets = backbone.module_embed.module_sets - num_modules = len(module_sets) - module_sizes = [len(ps) for ps in module_sets] - - module_gene_names: Dict[int, List[str]] = {} - for module_i, gene_idx in enumerate(module_sets): - names = [] - for idx in gene_idx.cpu().numpy(): - ens_id = model_genes[idx] - symbol = ens_to_symbol.get(ens_id, ens_id) - names.append(symbol) - module_gene_names[module_i] = names - - logger.info("Modules: %d, total genes: %d", num_modules, sum(module_sizes)) - - # --- Load and preprocess data --- - logger.info("Loading data from %s", args.data_path) - preprocessor = DataPreprocessor( - gene2ens_file=args.gene2ens_file, - gene2ens_column=args.gene2ens_column, - model_genes_file=model_genes_file, - ) - preprocessor.load_gene_mapping() - preprocessor.load_model_genes() - - gene_name_col = args.gene_name_column if args.gene_name_column != "auto" else None - adata, reindex_state = preprocessor.preprocess_adata( - adata_path=args.data_path, - max_cells=args.max_cells, - dataset_name=args.dataset_name, - gene_name_column=gene_name_col, - ) - cell_labels = adata.obs[args.celltype_column].values.astype(str) - logger.info("Data: %d cells, %d cell types", len(adata), len(np.unique(cell_labels))) - - # --- DataLoader --- - if hasattr(adata.X, "toarray"): - X = adata.X.toarray().astype(np.float32) - else: - X = np.array(adata.X, dtype=np.float32) - - dataset = MatrixRowDataset(X) - loader = DataLoader(dataset, batch_size=args.batch_size, shuffle=False, - num_workers=args.num_workers, pin_memory=True) - - # --- Extract attention (streaming) --- - logger.info("Extracting module attention (streaming mode)...") - (global_mean, global_std, ct_mean, ct_std, ct_count, n_cells, - expr_mean, attn_expr_prod_sum) = extract_attention_streaming( - backbone, loader, preprocessor, reindex_state, device, - cell_labels, num_modules, module_sizes, - ) - - # --- Build DataFrames --- - logger.info("Building analysis DataFrames...") - global_df = build_global_importance_df(global_mean, global_std, module_gene_names, num_modules) - ct_dfs = build_celltype_dfs(ct_mean, ct_count, module_gene_names, num_modules) - specificity_df = compute_specificity_from_means(global_mean, ct_mean, module_gene_names, num_modules) - - # --- New metrics --- - logger.info("Computing z-score specificity...") - zscore_df = compute_specificity_zscore( - global_mean, global_std, ct_mean, ct_count, module_gene_names, num_modules, args.min_cells) - - logger.info("Computing Cohen's d...") - cohens_d_df = compute_cohens_d( - global_mean, ct_mean, ct_std, ct_count, module_gene_names, num_modules, n_cells, args.min_cells) - - logger.info("Computing Welch's t-test...") - welch_df = compute_welch_ttest( - global_mean, global_std, ct_mean, ct_std, ct_count, module_gene_names, - num_modules, n_cells, args.min_cells) - - logger.info("Computing attention-expression correlation...") - attn_expr_corr = compute_attention_expression_corr( - global_mean, expr_mean, module_gene_names, num_modules) - - logger.info("Computing random baseline comparison...") - baseline_stats = compute_random_baseline(global_mean, num_modules) - - logger.info("Computing attention concentration...") - concentration_df = compute_attention_concentration( - global_mean, ct_mean, ct_count, num_modules, args.min_cells) - - # --- Marker validation --- - marker_dict = PANCREAS_MARKERS - if args.marker_db: - with open(args.marker_db) as f: - marker_dict = json.load(f) - - logger.info("Computing marker ranks within cell types...") - marker_rank_df = compute_marker_rank_within_celltype( - ct_mean, ct_count, module_gene_names, num_modules, marker_dict, args.min_cells) - - marker_results_zscore = validate_markers(zscore_df, marker_dict, sort_col="z_score") - marker_results_fc = validate_markers(specificity_df, marker_dict, sort_col="fold_change") - - # --- Save CSVs --- - logger.info("Saving CSV results...") - global_df.to_csv(output_dir / "global_gene_importance.csv", index=False) - specificity_df.to_csv(output_dir / "gene_specificity.csv", index=False) - zscore_df.to_csv(output_dir / "gene_specificity_zscore.csv", index=False) - cohens_d_df.to_csv(output_dir / "gene_specificity_cohens_d.csv", index=False) - welch_df.to_csv(output_dir / "gene_specificity_welch.csv", index=False) - concentration_df.to_csv(output_dir / "attention_concentration.csv", index=False) - if not marker_rank_df.empty: - marker_rank_df.to_csv(output_dir / "marker_ranks_within_celltype.csv", index=False) - for ct, df in ct_dfs.items(): - safe_name = ct.replace("/", "_").replace(" ", "_") - df.to_csv(output_dir / f"celltype_{safe_name}_attention.csv", index=False) - - with open(output_dir / "marker_validation_zscore.json", "w") as f: - json.dump(marker_results_zscore, f, indent=2, ensure_ascii=False) - with open(output_dir / "marker_validation.json", "w") as f: - json.dump(marker_results_fc, f, indent=2, ensure_ascii=False) - - # --- DEG correlation --- - deg_results = None - if args.deg_csv: - deg_df = pd.read_csv(args.deg_csv) - deg_results = validate_deg_correlation(ct_dfs, deg_df) - with open(output_dir / "deg_correlation.json", "w") as f: - json.dump(deg_results, f, indent=2) - - # --- Publication figures --- - logger.info("Generating publication figures...") - plot_global_importance_pub(global_df, zscore_df, args.top_k_genes, output_dir) - plot_celltype_heatmap_pub(ct_dfs, ct_count, args.top_k_genes, output_dir, args.min_cells) - plot_specificity_volcano(welch_df, zscore_df, args.top_k_genes, output_dir) - plot_marker_dotplot(marker_rank_df, output_dir) - plot_marker_recall_pub(marker_results_zscore, ct_count, output_dir, args.min_cells) - plot_attention_vs_expression(global_mean, expr_mean, module_gene_names, num_modules, output_dir) - plot_concentration_pub(concentration_df, baseline_stats, output_dir) - plot_baseline_kl(baseline_stats, output_dir) - if deg_results: - plot_deg_correlation_pub(deg_results, output_dir) - - # --- Enrichment --- - run_enrichment_analysis(global_df, args.top_k_genes, output_dir) - - # --- Summary --- - filtered_cts = [ct for ct, n in ct_count.items() if n < args.min_cells] - summary = { - "n_cells": n_cells, - "n_modules": num_modules, - "n_celltypes": len(ct_count), - "celltypes": {ct: int(n) for ct, n in ct_count.items()}, - "min_cells_filter": args.min_cells, - "filtered_celltypes": filtered_cts, - "top_10_global_genes": global_df.groupby("gene")["mean_attention"].mean() - .nlargest(10).to_dict(), - "attn_expr_correlation": attn_expr_corr, - "baseline_comparison": baseline_stats, - "mean_gini_global": float(concentration_df["global_gini"].mean()), - "mean_gini_celltype": float(concentration_df["mean_ct_gini"].mean()), - } - with open(output_dir / "summary.json", "w") as f: - json.dump(summary, f, indent=2, ensure_ascii=False) - - logger.info("Done. Results saved to %s", output_dir) - - -if __name__ == "__main__": - main() - +"""ModuleEmbedding Attention-based Gene Interpretability Analysis (Streaming). + +Interpretability analysis using single-head content-based attention pooling in ModuleEmbedding. +Streaming computing mode: memory usage O(n_celltypes × n_genes). + +Analysis content: + 1. Global gene importance ranking + 2. Cell type specificity (z-score, Cohen's d, Welch's t) + 3. Marker gene verification + 4. Attention-Expression correlation + 5. Random baseline comparison (KL divergence) + 6. Attention concentration (Gini coefficient) + 7. Pathway Enrichment Analysis + 8. DEG comparison + +Usage (run from the BioMod repository root): + python BioMod/interpretability/attention/module_gene_attention.py \ + --performer_checkpoint weight/0810model/mae_tost_70M_75mask_ec2_epoch0.pth \ + --config_pkl weight/0810model/mae_tost_70M_75mask_ec2.pkl \ + --module_data_dir model_dependencies/module_data_all \ + --gene_list_path model_dependencies/selected_genes_27k.txt \ + --gene_mapping_path model_dependencies/gene2ens.csv \ + --gene2vec_data_path model_dependencies/selected_gene2vec_27k.npy \ + --gene2ens_file model_dependencies/gene2ens.csv \ + --data_path data/downstream/final_data/human_pancreas_norm_complexBatch_with_gene_name.h5ad \ + --celltype_column CELL_TYPE_COLUMN \ + --batch_size 64 \ + --output_dir outputs/interpretability/gene_attention/pancreas + +Replace CELL_TYPE_COLUMN with the cell-type field in the input h5ad metadata. +""" +from __future__ import annotations + +import argparse +import json +import logging +import sys +from pathlib import Path +from typing import Dict, List, Optional, Tuple + +import numpy as np +import pandas as pd +import torch +from torch.utils.data import DataLoader + +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt +import seaborn as sns + +from scipy.stats import spearmanr, t as t_dist + +HERE = Path(__file__).resolve() +PERFORMER_MODULE = HERE.parents[2] +REPO_ROOT = HERE.parents[3] +ASSET_ROOT = REPO_ROOT / "model_dependencies" +for p in (str(PERFORMER_MODULE), str(REPO_ROOT)): + if p not in sys.path: + sys.path.insert(0, p) + +# Publication utilities +MODULE_IMPORTANCE_DIR = PERFORMER_MODULE / "interpretability" / "module_importance" +if str(MODULE_IMPORTANCE_DIR) not in sys.path: + sys.path.insert(0, str(MODULE_IMPORTANCE_DIR)) + +from pub_utils import setup_publication_style, savefig as pub_savefig + +from cell_annotation.performer_module_finetune import ( + DataPreprocessor, + load_backbone, + seed_all, +) +from interpretability.utils import ( + MatrixRowDataset, + build_module_metadata, + ensure_dir, + load_module_names, +) + +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") +logger = logging.getLogger(__name__) + + +# --------------------------------------------------------------------------- +# Argparse +# --------------------------------------------------------------------------- + +def parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + + p.add_argument("--performer_checkpoint", type=str, required=True) + p.add_argument("--config_pkl", type=str, required=True) + p.add_argument("--module_data_dir", type=str, default=str(ASSET_ROOT / "module_data_all")) + p.add_argument("--module_json", type=str, default=None) + p.add_argument("--gene_list_path", type=str, default=str(ASSET_ROOT / "selected_genes_27k.txt")) + p.add_argument("--gene_mapping_path", type=str, default=str(ASSET_ROOT / "gene2ens.csv")) + p.add_argument("--gene2vec_data_path", type=str, default=str(ASSET_ROOT / "selected_gene2vec_27k.npy")) + p.add_argument("--gene2ens_file", type=str, default=str(ASSET_ROOT / "gene2ens.csv")) + p.add_argument("--gene2ens_column", type=str, default="Ensembl gene ID") + p.add_argument("--shuffle_module_genes", action="store_true") + p.add_argument("--shuffle_seed", type=int, default=2021) + + p.add_argument("--data_path", type=str, required=True) + p.add_argument("--celltype_column", type=str, default="celltype") + p.add_argument("--gene_name_column", type=str, default="auto") + p.add_argument("--dataset_name", type=str, default="gene_attention") + p.add_argument("--max_cells", type=int, default=None) + + p.add_argument("--batch_size", type=int, default=64) + p.add_argument("--num_workers", type=int, default=0) + p.add_argument("--device", type=str, default="cuda") + p.add_argument("--seed", type=int, default=2021) + + p.add_argument("--top_k_genes", type=int, default=20) + p.add_argument("--marker_db", type=str, default=None) + p.add_argument("--deg_csv", type=str, default=None) + p.add_argument("--min_cells", type=int, default=30) + + p.add_argument("--output_dir", type=str, required=True) + return p.parse_args() + + +# --------------------------------------------------------------------------- +# Built-in pancreas marker genes (from PanglaoDB + CellMarker) +# --------------------------------------------------------------------------- + +PANCREAS_MARKERS: Dict[str, List[str]] = { + "alpha": ["GCG", "ARX", "IRX2", "TTR", "LOXL4", "GC", "CRYBA2", "DPP4", "IRX1", "PCSK2"], + "beta": ["INS", "IAPP", "MAFA", "NKX6-1", "PDX1", "HADH", "SLC30A8", "NPTX2", "DLK1", "PCSK1"], + "delta": ["SST", "HHEX", "RBP4", "LEPR", "PCSK1", "SEC11C"], + "gamma": ["PPY", "SERTM1", "CARTPT", "MEIS2", "ETV1", "ABCC9"], + "ductal": ["KRT19", "KRT7", "SPP1", "CFTR", "MUC1", "AMBP", "ANXA4", "BICC1", "SOX9"], + "acinar": ["PRSS1", "CELA3A", "CELA2A", "CPA1", "CPA2", "PNLIP", "CEL", "CLPS", "REG1A", "CTRB1"], + "activated_stellate": ["COL1A1", "COL1A2", "COL3A1", "ACTA2", "TAGLN", "RGS5", "PDGFRB"], + "endothelial": ["PECAM1", "VWF", "CDH5", "KDR", "ENG", "PLVAP", "ESAM"], + "macrophage": ["CD68", "CD163", "CSF1R", "MARCO", "MSR1", "C1QA", "C1QB"], + "epsilon": ["GHRL", "PCSK1", "ARX"], +} + +PANCREAS_COLORS: Dict[str, str] = { + "alpha": "#E41A1C", "beta": "#377EB8", "delta": "#4DAF4A", + "gamma": "#984EA3", "ductal": "#FF7F00", "acinar": "#A65628", + "activated_stellate": "#F781BF", "endothelial": "#66C2A5", + "macrophage": "#FC8D62", "epsilon": "#8DA0CB", "t_cell": "#E78AC3", + "schwann": "#B3B3B3", "mast": "#FDBF6F", "quiescent_stellate": "#CAB2D6", +} + + +# --------------------------------------------------------------------------- +# Streaming attention extraction + statistics accumulation +# --------------------------------------------------------------------------- + +@torch.no_grad() +def extract_attention_streaming( + backbone, loader, preprocessor, reindex_state, device, + cell_labels: np.ndarray, num_modules: int, module_sizes: List[int], +) -> Tuple: + """Extract attention and accumulate statistics in streaming fashion. + + Returns: + global_mean, global_std, ct_mean, ct_std, ct_count, n_cells, + expr_mean, attn_expr_prod_sum + """ + backbone.eval() + module_embed = backbone.module_embed + + global_sum = {i: np.zeros(module_sizes[i], dtype=np.float64) for i in range(num_modules)} + global_sum_sq = {i: np.zeros(module_sizes[i], dtype=np.float64) for i in range(num_modules)} + expr_sum = {i: np.zeros(module_sizes[i], dtype=np.float64) for i in range(num_modules)} + attn_expr_prod_sum = {i: np.zeros(module_sizes[i], dtype=np.float64) for i in range(num_modules)} + + celltypes = np.unique(cell_labels) + ct_sum = {ct: {i: np.zeros(module_sizes[i], dtype=np.float64) for i in range(num_modules)} for ct in celltypes} + ct_sum_sq = {ct: {i: np.zeros(module_sizes[i], dtype=np.float64) for i in range(num_modules)} for ct in celltypes} + ct_count = {ct: 0 for ct in celltypes} + + n_cells = 0 + cell_idx = 0 + + for batch_idx, x in enumerate(loader): + x = x.to(device) + if reindex_state is not None: + x = preprocessor.reindex_tensor_to_model_space(x, reindex_state=reindex_state) + + batch_size = x.shape[0] + batch_labels = cell_labels[cell_idx:cell_idx + batch_size] + cell_idx += batch_size + + for module_i, gene_idx in enumerate(module_embed.module_sets): + gene_idx_dev = gene_idx.to(device) + expr_slice = torch.index_select(x, dim=1, index=gene_idx_dev) + + gene_emb = module_embed.gene_embed(gene_idx_dev) + gene_emb = gene_emb.unsqueeze(0).expand(batch_size, -1, -1) + gene_tok = module_embed._fuse_expr_and_gene_emb(expr_slice, gene_emb) + gene_tok = module_embed.proj(gene_tok) + + attn = module_embed.attn_score(gene_tok).squeeze(-1) + attn = torch.softmax(attn, dim=1) + attn_np = attn.cpu().numpy().astype(np.float64) + + # Expression for correlation + expr_np = expr_slice.cpu().numpy().astype(np.float64) + expr_sum[module_i] += expr_np.sum(axis=0) + attn_expr_prod_sum[module_i] += (attn_np * expr_np).sum(axis=0) + + # Global accumulation + global_sum[module_i] += attn_np.sum(axis=0) + global_sum_sq[module_i] += (attn_np ** 2).sum(axis=0) + + # Per-celltype accumulation + for ct in celltypes: + mask = batch_labels == ct + if mask.any(): + ct_attn = attn_np[mask] + ct_sum[ct][module_i] += ct_attn.sum(axis=0) + ct_sum_sq[ct][module_i] += (ct_attn ** 2).sum(axis=0) + + for ct in celltypes: + ct_count[ct] += int((batch_labels == ct).sum()) + + n_cells += batch_size + if (batch_idx + 1) % 50 == 0: + logger.info(" [attention] batch %d, %d cells processed", batch_idx + 1, n_cells) + + # Compute means and stds + global_mean = {} + global_std = {} + for i in range(num_modules): + global_mean[i] = (global_sum[i] / n_cells).astype(np.float32) + variance = global_sum_sq[i] / n_cells - global_mean[i].astype(np.float64) ** 2 + global_std[i] = np.sqrt(np.maximum(variance, 0)).astype(np.float32) + + ct_mean = {} + ct_std = {} + for ct in celltypes: + ct_mean[ct] = {} + ct_std[ct] = {} + for i in range(num_modules): + if ct_count[ct] > 0: + ct_mean[ct][i] = (ct_sum[ct][i] / ct_count[ct]).astype(np.float32) + else: + ct_mean[ct][i] = np.zeros(module_sizes[i], dtype=np.float32) + if ct_count[ct] > 1: + var = ct_sum_sq[ct][i] / ct_count[ct] - ct_mean[ct][i].astype(np.float64) ** 2 + ct_std[ct][i] = np.sqrt(np.maximum(var, 0)).astype(np.float32) + else: + ct_std[ct][i] = np.zeros(module_sizes[i], dtype=np.float32) + + expr_mean = {i: (expr_sum[i] / n_cells).astype(np.float32) for i in range(num_modules)} + + logger.info("Extracted attention for %d cells across %d modules (streaming)", n_cells, num_modules) + return global_mean, global_std, ct_mean, ct_std, ct_count, n_cells, expr_mean, attn_expr_prod_sum + + +# --------------------------------------------------------------------------- +# Analysis functions +# --------------------------------------------------------------------------- + +def build_global_importance_df( + global_mean: Dict[int, np.ndarray], + global_std: Dict[int, np.ndarray], + module_gene_names: Dict[int, List[str]], + num_modules: int, +) -> pd.DataFrame: + """Build global gene importance DataFrame from streaming means.""" + rows = [] + for module_i in range(num_modules): + mean_attn = global_mean[module_i] + std_attn = global_std[module_i] + genes = module_gene_names[module_i] + rank_order = np.argsort(-mean_attn) + for g_idx, gene in enumerate(genes): + rows.append({ + "module_idx": module_i, + "gene": gene, + "mean_attention": float(mean_attn[g_idx]), + "std_attention": float(std_attn[g_idx]), + "rank_in_module": int(np.where(rank_order == g_idx)[0][0]) + 1, + "module_size": len(genes), + }) + df = pd.DataFrame(rows) + df["normalized_rank"] = df["rank_in_module"] / df["module_size"] + return df + + +def build_celltype_dfs( + ct_mean: Dict[str, Dict[int, np.ndarray]], + ct_count: Dict[str, int], + module_gene_names: Dict[int, List[str]], + num_modules: int, +) -> Dict[str, pd.DataFrame]: + """Build per-celltype DataFrames from streaming means.""" + ct_dfs = {} + for ct, module_means in ct_mean.items(): + rows = [] + for module_i in range(num_modules): + mean_attn = module_means[module_i] + genes = module_gene_names[module_i] + for g_idx, gene in enumerate(genes): + rows.append({ + "module_idx": module_i, + "gene": gene, + "mean_attention": float(mean_attn[g_idx]), + "n_cells": ct_count[ct], + }) + ct_dfs[ct] = pd.DataFrame(rows) + return ct_dfs + + +def compute_specificity_from_means( + global_mean: Dict[int, np.ndarray], + ct_mean: Dict[str, Dict[int, np.ndarray]], + module_gene_names: Dict[int, List[str]], + num_modules: int, +) -> pd.DataFrame: + """Fold-change specificity (legacy): max_ct(mean_attn_ct) / global_mean_attn.""" + celltypes = list(ct_mean.keys()) + rows = [] + for module_i in range(num_modules): + genes = module_gene_names[module_i] + g_mean = global_mean[module_i] + for g_idx, gene in enumerate(genes): + ct_scores = {ct: float(ct_mean[ct][module_i][g_idx]) for ct in celltypes} + top_ct = max(ct_scores, key=ct_scores.get) + top_score = ct_scores[top_ct] + fold_change = top_score / max(float(g_mean[g_idx]), 1e-10) + rows.append({ + "module_idx": module_i, "gene": gene, "top_celltype": top_ct, + "top_ct_mean_attn": top_score, "global_mean_attn": float(g_mean[g_idx]), + "fold_change": fold_change, + }) + return pd.DataFrame(rows) + + +def compute_specificity_zscore( + global_mean: Dict[int, np.ndarray], + global_std: Dict[int, np.ndarray], + ct_mean: Dict[str, Dict[int, np.ndarray]], + ct_count: Dict[str, int], + module_gene_names: Dict[int, List[str]], + num_modules: int, + min_cells: int = 30, +) -> pd.DataFrame: + """Z-score specificity: (ct_mean - global_mean) / global_std.""" + valid_cts = [ct for ct, n in ct_count.items() if n >= min_cells] + rows = [] + for module_i in range(num_modules): + genes = module_gene_names[module_i] + g_mean = global_mean[module_i] + g_std = global_std[module_i] + for g_idx, gene in enumerate(genes): + std_val = max(float(g_std[g_idx]), 1e-8) + ct_zscores = {} + for ct in valid_cts: + z = (float(ct_mean[ct][module_i][g_idx]) - float(g_mean[g_idx])) / std_val + ct_zscores[ct] = z + top_ct = max(ct_zscores, key=ct_zscores.get) + rows.append({ + "module_idx": module_i, "gene": gene, "top_celltype": top_ct, + "z_score": ct_zscores[top_ct], + "global_mean_attn": float(g_mean[g_idx]), + "ct_mean_attn": float(ct_mean[top_ct][module_i][g_idx]), + }) + return pd.DataFrame(rows) + + +def compute_cohens_d( + global_mean: Dict[int, np.ndarray], + ct_mean: Dict[str, Dict[int, np.ndarray]], + ct_std: Dict[str, Dict[int, np.ndarray]], + ct_count: Dict[str, int], + module_gene_names: Dict[int, List[str]], + num_modules: int, + n_cells: int, + min_cells: int = 30, +) -> pd.DataFrame: + """Cohen's d effect size for cell-type specificity.""" + valid_cts = [ct for ct, n in ct_count.items() if n >= min_cells] + rows = [] + for module_i in range(num_modules): + genes = module_gene_names[module_i] + for g_idx, gene in enumerate(genes): + best_d = -np.inf + best_ct = valid_cts[0] + for ct in valid_cts: + n_ct = ct_count[ct] + n_other = n_cells - n_ct + ct_m = float(ct_mean[ct][module_i][g_idx]) + ct_s = float(ct_std[ct][module_i][g_idx]) + other_m = (float(global_mean[module_i][g_idx]) * n_cells - ct_m * n_ct) / max(n_other, 1) + pooled = max(ct_s, 1e-10) + d = (ct_m - other_m) / pooled + if d > best_d: + best_d = d + best_ct = ct + rows.append({"module_idx": module_i, "gene": gene, "top_celltype": best_ct, "cohens_d": best_d}) + return pd.DataFrame(rows) + + +def compute_welch_ttest( + global_mean: Dict[int, np.ndarray], + global_std: Dict[int, np.ndarray], + ct_mean: Dict[str, Dict[int, np.ndarray]], + ct_std: Dict[str, Dict[int, np.ndarray]], + ct_count: Dict[str, int], + module_gene_names: Dict[int, List[str]], + num_modules: int, + n_cells: int, + min_cells: int = 30, +) -> pd.DataFrame: + """Welch's t-test from streaming statistics with BH FDR correction.""" + valid_cts = [ct for ct, n in ct_count.items() if n >= min_cells] + rows = [] + for module_i in range(num_modules): + genes = module_gene_names[module_i] + for g_idx, gene in enumerate(genes): + best_t = -np.inf + best_ct = valid_cts[0] + best_pval = 1.0 + for ct in valid_cts: + n_ct = ct_count[ct] + ct_m = float(ct_mean[ct][module_i][g_idx]) + ct_v = float(ct_std[ct][module_i][g_idx]) ** 2 + g_m = float(global_mean[module_i][g_idx]) + g_v = float(global_std[module_i][g_idx]) ** 2 + se = np.sqrt(ct_v / max(n_ct, 1) + g_v / max(n_cells, 1)) + if se < 1e-12: + continue + t_stat = (ct_m - g_m) / se + num = (ct_v / n_ct + g_v / n_cells) ** 2 + denom = (ct_v / n_ct) ** 2 / max(n_ct - 1, 1) + (g_v / n_cells) ** 2 / max(n_cells - 1, 1) + df = num / max(denom, 1e-12) + pval = float(t_dist.sf(abs(t_stat), df) * 2) + if t_stat > best_t: + best_t = t_stat + best_ct = ct + best_pval = pval + rows.append({"module_idx": module_i, "gene": gene, "top_celltype": best_ct, + "t_stat": best_t, "pval": best_pval}) + + df = pd.DataFrame(rows) + # BH FDR correction + pvals = df["pval"].values + n = len(pvals) + sorted_idx = np.argsort(pvals) + sorted_pvals = pvals[sorted_idx] + fdr = np.minimum(1.0, sorted_pvals * n / (np.arange(n) + 1)) + for i in range(n - 2, -1, -1): + fdr[i] = min(fdr[i], fdr[i + 1]) + fdr_all = np.empty(n) + fdr_all[sorted_idx] = fdr + df["fdr"] = fdr_all + return df + + +def compute_attention_expression_corr( + global_mean: Dict[int, np.ndarray], + expr_mean: Dict[int, np.ndarray], + module_gene_names: Dict[int, List[str]], + num_modules: int, +) -> Dict[str, float]: + """Correlation between mean attention and mean expression across genes.""" + all_attn = [] + all_expr = [] + for module_i in range(num_modules): + all_attn.extend(global_mean[module_i].tolist()) + all_expr.extend(expr_mean[module_i].tolist()) + all_attn = np.array(all_attn) + all_expr = np.array(all_expr) + mask = (all_attn > 0) & (all_expr > 0) + if mask.sum() > 10: + pearson_r = float(np.corrcoef(all_attn[mask], all_expr[mask])[0, 1]) + sp_r, sp_p = spearmanr(all_attn[mask], all_expr[mask]) + else: + pearson_r, sp_r, sp_p = 0.0, 0.0, 1.0 + return {"pearson_r": pearson_r, "spearman_r": float(sp_r), + "spearman_pval": float(sp_p), "n_genes": int(mask.sum())} + + +def compute_random_baseline( + global_mean: Dict[int, np.ndarray], + num_modules: int, +) -> Dict[str, float]: + """Compare learned attention to uniform baseline.""" + kl_divs = [] + learned_entropies = [] + for module_i in range(num_modules): + attn = global_mean[module_i].astype(np.float64) + attn = attn / max(attn.sum(), 1e-10) + g_size = len(attn) + uniform = np.ones(g_size) / g_size + mask = attn > 1e-12 + kl = float(np.sum(attn[mask] * np.log(attn[mask] / uniform[mask]))) + kl_divs.append(kl) + h_learned = -float(np.sum(attn[mask] * np.log(attn[mask]))) + h_max = np.log(g_size) + learned_entropies.append(h_learned / max(h_max, 1e-10)) + return { + "mean_kl_divergence": float(np.mean(kl_divs)), + "std_kl_divergence": float(np.std(kl_divs)), + "mean_normalized_entropy": float(np.mean(learned_entropies)), + "entropy_reduction": float(1.0 - np.mean(learned_entropies)), + "n_modules": num_modules, + } + + +def compute_attention_concentration( + global_mean: Dict[int, np.ndarray], + ct_mean: Dict[str, Dict[int, np.ndarray]], + ct_count: Dict[str, int], + num_modules: int, + min_cells: int = 30, +) -> pd.DataFrame: + """Gini coefficient per module measuring attention selectivity.""" + def gini(arr): + arr = np.sort(arr.flatten()) + n = len(arr) + if n == 0 or arr.sum() == 0: + return 0.0 + index = np.arange(1, n + 1) + return float((2 * np.sum(index * arr) - (n + 1) * np.sum(arr)) / (n * np.sum(arr))) + + valid_cts = [ct for ct, n in ct_count.items() if n >= min_cells] + rows = [] + for module_i in range(num_modules): + g_attn = global_mean[module_i] + g_gini = gini(g_attn) + ct_ginis = [gini(ct_mean[ct][module_i]) for ct in valid_cts] + rows.append({ + "module_idx": module_i, "global_gini": g_gini, + "mean_ct_gini": float(np.mean(ct_ginis)) if ct_ginis else 0.0, + "module_size": len(g_attn), + }) + return pd.DataFrame(rows) + + +def compute_marker_rank_within_celltype( + ct_mean: Dict[str, Dict[int, np.ndarray]], + ct_count: Dict[str, int], + module_gene_names: Dict[int, List[str]], + num_modules: int, + marker_dict: Dict[str, List[str]], + min_cells: int = 30, +) -> pd.DataFrame: + """Rank marker genes within their expected cell type by attention.""" + rows = [] + for ct, markers in marker_dict.items(): + if ct not in ct_count or ct_count[ct] < min_cells: + continue + gene_attn = {} + for module_i in range(num_modules): + genes = module_gene_names[module_i] + attn = ct_mean[ct][module_i] + for g_idx, gene in enumerate(genes): + if gene not in gene_attn or float(attn[g_idx]) > gene_attn[gene]: + gene_attn[gene] = float(attn[g_idx]) + sorted_genes = sorted(gene_attn.items(), key=lambda x: -x[1]) + gene_rank = {g: rank + 1 for rank, (g, _) in enumerate(sorted_genes)} + n_total = len(gene_rank) + for marker in markers: + if marker in gene_rank: + rank = gene_rank[marker] + rows.append({ + "celltype": ct, "marker_gene": marker, "rank": rank, + "percentile": rank / n_total * 100, "n_total_genes": n_total, + "attention": gene_attn[marker], + }) + return pd.DataFrame(rows) + + +# --------------------------------------------------------------------------- +# Marker gene validation +# --------------------------------------------------------------------------- + +def validate_markers( + specificity_df: pd.DataFrame, + marker_dict: Dict[str, List[str]], + sort_col: str = "fold_change", +) -> Dict[str, object]: + """Validate whether high-attention genes match known markers.""" + results = {} + all_genes_in_model = set(specificity_df["gene"].unique()) + for ct, markers in marker_dict.items(): + markers_in_model = [g for g in markers if g in all_genes_in_model] + if not markers_in_model: + continue + ct_specific = specificity_df[specificity_df["top_celltype"] == ct].copy() + ct_specific = ct_specific.sort_values(sort_col, ascending=False).reset_index(drop=True) + marker_set = set(markers_in_model) + n_total = len(ct_specific) + if n_total == 0: + continue + top_50_genes = set(ct_specific.head(50)["gene"]) + top_100_genes = set(ct_specific.head(100)["gene"]) + hits_50 = len(marker_set & top_50_genes) + hits_100 = len(marker_set & top_100_genes) + marker_rows = ct_specific[ct_specific["gene"].isin(marker_set)] + if len(marker_rows) > 0: + marker_ranks = (marker_rows.index + 1).tolist() + mean_rank = np.mean(marker_ranks) + median_rank = np.median(marker_ranks) + else: + mean_rank = median_rank = n_total + results[ct] = { + "n_markers_in_model": len(markers_in_model), + "markers_in_model": markers_in_model, + "n_genes_assigned_to_ct": n_total, + "hits_in_top50": hits_50, "hits_in_top100": hits_100, + "precision_at_50": hits_50 / min(50, n_total), + "precision_at_100": hits_100 / min(100, n_total), + "marker_mean_rank": float(mean_rank), + "marker_median_rank": float(median_rank), + "recall_at_50": hits_50 / len(markers_in_model) if markers_in_model else 0, + "recall_at_100": hits_100 / len(markers_in_model) if markers_in_model else 0, + } + return results + + +def validate_deg_correlation( + ct_dfs: Dict[str, pd.DataFrame], + deg_df: pd.DataFrame, +) -> Dict[str, Dict[str, float]]: + """Compare attention ranking with DEG ranking per cell type.""" + results = {} + for ct, attn_df in ct_dfs.items(): + ct_deg = deg_df[deg_df["cell_type"] == ct].copy() + if len(ct_deg) == 0: + continue + merged = attn_df.merge(ct_deg[["gene", "log2fc"]], on="gene", how="inner") + if len(merged) < 10: + continue + rho, pval = spearmanr(merged["mean_attention"], merged["log2fc"].abs()) + results[ct] = {"spearman_r": float(rho), "pval": float(pval), "n_genes": len(merged)} + return results + + +# --------------------------------------------------------------------------- +# Publication-quality visualization +# --------------------------------------------------------------------------- + +def plot_global_importance_pub(global_df, specificity_zscore_df, top_k, output_dir): + """Fig1: Top-K genes with std error bars, colored by top cell type.""" + setup_publication_style() + gene_stats = global_df.groupby("gene").agg( + mean_attn=("mean_attention", "mean"), std_attn=("std_attention", "mean"), + ).nlargest(top_k, "mean_attn") + gene_ct = specificity_zscore_df.drop_duplicates("gene").set_index("gene")["top_celltype"] + fig, ax = plt.subplots(figsize=(4.5, 5)) + genes = gene_stats.index.tolist() + y_pos = np.arange(len(genes)) + colors = [PANCREAS_COLORS.get(gene_ct.get(g, ""), "#999999") for g in genes] + ax.barh(y_pos, gene_stats["mean_attn"], xerr=gene_stats["std_attn"], + color=colors, edgecolor="none", height=0.7, capsize=2, error_kw={"linewidth": 0.8}) + ax.set_yticks(y_pos) + ax.set_yticklabels(genes) + ax.set_xlabel("Mean attention weight") + ax.set_title("Top genes by global attention") + ax.invert_yaxis() + pub_savefig(fig, output_dir / "fig1_global_importance") + logger.info("Saved fig1_global_importance (PDF+PNG)") + + +def plot_celltype_heatmap_pub(ct_dfs, ct_count, top_k, output_dir, min_cells=30): + """Fig2: Clustered heatmap of gene attention (z-score normalized).""" + setup_publication_style() + valid_cts = {ct for ct, n in ct_count.items() if n >= min_cells} + all_genes_attn = {} + for ct, df in ct_dfs.items(): + if ct not in valid_cts: + continue + all_genes_attn[ct] = df.groupby("gene")["mean_attention"].mean() + combined = pd.DataFrame(all_genes_attn) + gene_var = combined.var(axis=1).nlargest(top_k).index + heatmap_data = combined.loc[gene_var] + row_mean = heatmap_data.mean(axis=1) + row_std = heatmap_data.std(axis=1).replace(0, 1) + heatmap_z = heatmap_data.sub(row_mean, axis=0).div(row_std, axis=0) + g = sns.clustermap(heatmap_z, cmap="RdBu_r", center=0, figsize=(6, 7), + xticklabels=True, yticklabels=True, + dendrogram_ratio=(0.1, 0.15), cbar_kws={"label": "Z-score"}, + linewidths=0.3) + g.ax_heatmap.set_xlabel("") + g.ax_heatmap.set_ylabel("") + g.fig.suptitle(f"Top {top_k} variable genes (z-scored attention)", y=1.02, fontsize=11) + pub_savefig(g.fig, output_dir / "fig2_celltype_heatmap") + logger.info("Saved fig2_celltype_heatmap (PDF+PNG)") + + +def plot_specificity_volcano(welch_df, zscore_df, top_k, output_dir): + """Fig3: Volcano plot (z-score vs -log10 FDR).""" + setup_publication_style() + merged = zscore_df[["gene", "top_celltype", "z_score"]].drop_duplicates("gene") + welch_best = welch_df.loc[welch_df.groupby("gene")["t_stat"].idxmax()][["gene", "fdr"]] + merged = merged.merge(welch_best, on="gene", how="left") + merged["neg_log10_fdr"] = -np.log10(merged["fdr"].clip(lower=1e-300)) + fig, ax = plt.subplots(figsize=(5.5, 5)) + for ct in merged["top_celltype"].unique(): + subset = merged[merged["top_celltype"] == ct] + color = PANCREAS_COLORS.get(ct, "#999999") + ax.scatter(subset["z_score"], subset["neg_log10_fdr"], + c=color, s=8, alpha=0.5, label=ct, rasterized=True) + top_genes = merged.nlargest(top_k, "z_score") + for _, row in top_genes.iterrows(): + ax.annotate(row["gene"], (row["z_score"], row["neg_log10_fdr"]), + fontsize=6, ha="left", va="bottom") + ax.axhline(-np.log10(0.05), color="gray", linestyle="--", linewidth=0.6) + ax.axvline(2, color="gray", linestyle="--", linewidth=0.6) + ax.set_xlabel("Z-score (cell-type specificity)") + ax.set_ylabel(r"$-\log_{10}$(FDR)") + ax.set_title("Gene specificity volcano") + ax.legend(bbox_to_anchor=(1.02, 1), loc="upper left", fontsize=7, frameon=False, markerscale=2) + pub_savefig(fig, output_dir / "fig3_specificity_volcano") + logger.info("Saved fig3_specificity_volcano (PDF+PNG)") + + +def plot_marker_dotplot(marker_rank_df, output_dir): + """Fig4: Dot plot of marker gene rank percentile per cell type.""" + setup_publication_style() + if marker_rank_df.empty: + return + fig, ax = plt.subplots(figsize=(6, 4.5)) + celltypes = marker_rank_df["celltype"].unique() + ct_positions = {ct: i for i, ct in enumerate(celltypes)} + for _, row in marker_rank_df.iterrows(): + y = ct_positions[row["celltype"]] + size = max(10, 200 * (1 - row["percentile"] / 100)) + color = PANCREAS_COLORS.get(row["celltype"], "#999999") + ax.scatter(row["percentile"], y, s=size, c=color, alpha=0.7, + edgecolors="white", linewidth=0.3) + if row["percentile"] < 5: + ax.annotate(row["marker_gene"], (row["percentile"], y), + fontsize=6, ha="left", va="center", xytext=(3, 0), + textcoords="offset points") + ax.set_yticks(range(len(celltypes))) + ax.set_yticklabels(celltypes) + ax.set_xlabel("Rank percentile (lower = higher attention)") + ax.set_title("Marker gene rank within expected cell type") + ax.axvline(5, color="gray", linestyle="--", linewidth=0.6, alpha=0.5, label="Top 5%") + ax.legend(fontsize=7, frameon=False) + ax.set_xlim(-2, 105) + pub_savefig(fig, output_dir / "fig4_marker_dotplot") + logger.info("Saved fig4_marker_dotplot (PDF+PNG)") + + +def plot_marker_recall_pub(marker_results, ct_count, output_dir, min_cells=30): + """Fig5: Marker recall bar chart with random expectation line.""" + setup_publication_style() + if not marker_results: + return + cts = [ct for ct in marker_results.keys() if ct_count.get(ct, 0) >= min_cells] + if not cts: + return + recall_50 = [marker_results[ct]["recall_at_50"] for ct in cts] + recall_100 = [marker_results[ct]["recall_at_100"] for ct in cts] + x = np.arange(len(cts)) + width = 0.35 + fig, ax = plt.subplots(figsize=(max(5, len(cts) * 0.7), 4)) + colors_50 = [PANCREAS_COLORS.get(ct, "#999999") for ct in cts] + ax.bar(x - width / 2, recall_50, width, color=colors_50, alpha=0.9, label="Recall@50") + ax.bar(x + width / 2, recall_100, width, color=colors_50, alpha=0.5, label="Recall@100") + # Random expectation + for i, ct in enumerate(cts): + n_genes = marker_results[ct]["n_genes_assigned_to_ct"] + if n_genes > 0: + random_50 = min(50, n_genes) / n_genes + ax.hlines(random_50, i - 0.4, i + 0.4, colors="gray", linestyles=":", linewidth=0.8) + ax.set_xticks(x) + ax.set_xticklabels(cts, rotation=45, ha="right") + ax.set_ylabel("Recall") + ax.set_title("Marker gene recall by cell type") + ax.legend(fontsize=7, frameon=False) + pub_savefig(fig, output_dir / "fig5_marker_recall") + logger.info("Saved fig5_marker_recall (PDF+PNG)") + + +def plot_attention_vs_expression(global_mean, expr_mean, module_gene_names, num_modules, output_dir): + """Fig6: Scatter of mean attention vs mean expression.""" + setup_publication_style() + all_attn, all_expr = [], [] + for module_i in range(num_modules): + all_attn.extend(global_mean[module_i].tolist()) + all_expr.extend(expr_mean[module_i].tolist()) + all_attn = np.array(all_attn) + all_expr = np.array(all_expr) + mask = (all_attn > 0) & (all_expr > 0) + fig, ax = plt.subplots(figsize=(4.5, 4.5)) + ax.scatter(all_expr[mask], all_attn[mask], s=3, alpha=0.3, c="#377EB8", rasterized=True) + r = np.corrcoef(all_expr[mask], all_attn[mask])[0, 1] + ax.set_xlabel("Mean expression") + ax.set_ylabel("Mean attention weight") + ax.set_title("Attention vs expression") + ax.text(0.05, 0.95, f"Pearson r = {r:.3f}", transform=ax.transAxes, + fontsize=9, va="top", ha="left") + pub_savefig(fig, output_dir / "fig6_attn_vs_expression") + logger.info("Saved fig6_attn_vs_expression (PDF+PNG)") + + +def plot_concentration_pub(concentration_df, baseline_stats, output_dir): + """Fig7: Gini coefficient distribution (learned vs uniform).""" + setup_publication_style() + fig, axes = plt.subplots(1, 2, figsize=(8, 3.5)) + # Panel a: Gini histogram + ax = axes[0] + ax.hist(concentration_df["global_gini"], bins=30, alpha=0.7, color="#377EB8", label="Global") + ax.hist(concentration_df["mean_ct_gini"], bins=30, alpha=0.7, color="#E41A1C", label="Per cell-type") + ax.axvline(0, color="gray", linestyle="--", linewidth=0.6) + ax.set_xlabel("Gini coefficient") + ax.set_ylabel("Number of modules") + ax.set_title("Attention concentration") + ax.legend(fontsize=7, frameon=False) + # Panel b: Entropy comparison + ax = axes[1] + labels = ["Learned", "Uniform"] + values = [baseline_stats["mean_normalized_entropy"], 1.0] + bars = ax.bar(labels, values, color=["#377EB8", "#CCCCCC"], width=0.5) + ax.set_ylabel("Normalized entropy") + ax.set_title(f"Entropy reduction: {baseline_stats['entropy_reduction']:.1%}") + ax.set_ylim(0, 1.1) + plt.tight_layout() + pub_savefig(fig, output_dir / "fig7_concentration") + logger.info("Saved fig7_concentration (PDF+PNG)") + + +def plot_baseline_kl(baseline_stats, output_dir): + """Fig8: KL divergence summary.""" + setup_publication_style() + fig, ax = plt.subplots(figsize=(3.5, 3.5)) + ax.bar(["KL(learned||uniform)"], [baseline_stats["mean_kl_divergence"]], + yerr=[baseline_stats["std_kl_divergence"]], color="#377EB8", + capsize=5, width=0.4) + ax.set_ylabel("KL divergence (nats)") + ax.set_title("Deviation from uniform attention") + pub_savefig(fig, output_dir / "fig8_baseline_kl") + logger.info("Saved fig8_baseline_kl (PDF+PNG)") + + +def plot_deg_correlation_pub(deg_results, output_dir): + """Fig9: DEG correlation with significance.""" + setup_publication_style() + if not deg_results: + return + cts = sorted(deg_results.keys(), key=lambda c: deg_results[c]["spearman_r"], reverse=True) + rhos = [deg_results[ct]["spearman_r"] for ct in cts] + pvals = [deg_results[ct]["pval"] for ct in cts] + fig, ax = plt.subplots(figsize=(max(5, len(cts) * 0.6), 4)) + colors = ["#377EB8" if r > 0 else "#E41A1C" for r in rhos] + bars = ax.bar(range(len(cts)), rhos, color=colors, width=0.6) + for i, (bar, p) in enumerate(zip(bars, pvals)): + if p < 0.001: + ax.text(i, bar.get_height() + 0.02, "***", ha="center", fontsize=7) + elif p < 0.01: + ax.text(i, bar.get_height() + 0.02, "**", ha="center", fontsize=7) + elif p < 0.05: + ax.text(i, bar.get_height() + 0.02, "*", ha="center", fontsize=7) + ax.set_xticks(range(len(cts))) + ax.set_xticklabels(cts, rotation=45, ha="right") + ax.set_ylabel("Spearman r") + ax.set_title("Attention vs DEG correlation") + ax.axhline(0, color="gray", linewidth=0.5) + pub_savefig(fig, output_dir / "fig9_deg_correlation") + logger.info("Saved fig9_deg_correlation (PDF+PNG)") + + +# --------------------------------------------------------------------------- +# Enrichment analysis (optional) +# --------------------------------------------------------------------------- + +def run_enrichment_analysis( + global_df: pd.DataFrame, top_k: int, output_dir: Path, organism: str = "human", +) -> Optional[pd.DataFrame]: + """Run GO enrichment on top-K attention genes using gseapy.""" + try: + import gseapy as gp + except ImportError: + logger.warning("gseapy not installed, skipping enrichment. Install: pip install gseapy") + return None + top_genes = global_df.groupby("gene")["mean_attention"].mean().nlargest(top_k * 5).index.tolist() + logger.info("Running GO enrichment on top %d genes...", len(top_genes)) + try: + enr = gp.enrichr( + gene_list=top_genes, + gene_sets=["GO_Biological_Process_2021", "KEGG_2021_Human"], + organism=organism, + outdir=str(output_dir / "enrichment"), + no_plot=True, + ) + results_df = enr.results + results_df.to_csv(output_dir / "enrichment_results.csv", index=False) + logger.info("Enrichment results saved (%d terms)", len(results_df)) + return results_df + except Exception as e: + logger.warning("Enrichment analysis failed: %s", e) + return None + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +def main(): + args = parse_args() + seed_all(args.seed) + + _script_dir = Path(__file__).resolve().parent + for attr in ("performer_checkpoint", "config_pkl", "module_data_dir", "gene_list_path", + "gene_mapping_path", "gene2vec_data_path", "gene2ens_file", "data_path"): + val = getattr(args, attr, None) + if val is not None and not Path(val).is_absolute(): + setattr(args, attr, str((_script_dir / val).resolve())) + + output_dir = ensure_dir(Path(args.output_dir)) + device = torch.device(args.device if torch.cuda.is_available() else "cpu") + logger.info("Device: %s", device) + + # --- Load model --- + logger.info("Loading backbone from %s", args.performer_checkpoint) + backbone, module_data, model_config = load_backbone(args, device) + backbone.eval() + + # --- Build module gene name mapping --- + model_genes_file = args.gene_list_path + with open(model_genes_file) as f: + model_genes = [line.strip() for line in f if line.strip()] + + gene2ens_df = pd.read_csv(args.gene2ens_file) + ens_to_symbol = dict(zip(gene2ens_df["Ensembl gene ID"], gene2ens_df.iloc[:, 0])) + + module_sets = backbone.module_embed.module_sets + num_modules = len(module_sets) + module_sizes = [len(ps) for ps in module_sets] + + module_gene_names: Dict[int, List[str]] = {} + for module_i, gene_idx in enumerate(module_sets): + names = [] + for idx in gene_idx.cpu().numpy(): + ens_id = model_genes[idx] + symbol = ens_to_symbol.get(ens_id, ens_id) + names.append(symbol) + module_gene_names[module_i] = names + + logger.info("Modules: %d, total genes: %d", num_modules, sum(module_sizes)) + + # --- Load and preprocess data --- + logger.info("Loading data from %s", args.data_path) + preprocessor = DataPreprocessor( + gene2ens_file=args.gene2ens_file, + gene2ens_column=args.gene2ens_column, + model_genes_file=model_genes_file, + ) + preprocessor.load_gene_mapping() + preprocessor.load_model_genes() + + gene_name_col = args.gene_name_column if args.gene_name_column != "auto" else None + adata, reindex_state = preprocessor.preprocess_adata( + adata_path=args.data_path, + max_cells=args.max_cells, + dataset_name=args.dataset_name, + gene_name_column=gene_name_col, + ) + cell_labels = adata.obs[args.celltype_column].values.astype(str) + logger.info("Data: %d cells, %d cell types", len(adata), len(np.unique(cell_labels))) + + # --- DataLoader --- + if hasattr(adata.X, "toarray"): + X = adata.X.toarray().astype(np.float32) + else: + X = np.array(adata.X, dtype=np.float32) + + dataset = MatrixRowDataset(X) + loader = DataLoader(dataset, batch_size=args.batch_size, shuffle=False, + num_workers=args.num_workers, pin_memory=True) + + # --- Extract attention (streaming) --- + logger.info("Extracting module attention (streaming mode)...") + (global_mean, global_std, ct_mean, ct_std, ct_count, n_cells, + expr_mean, attn_expr_prod_sum) = extract_attention_streaming( + backbone, loader, preprocessor, reindex_state, device, + cell_labels, num_modules, module_sizes, + ) + + # --- Build DataFrames --- + logger.info("Building analysis DataFrames...") + global_df = build_global_importance_df(global_mean, global_std, module_gene_names, num_modules) + ct_dfs = build_celltype_dfs(ct_mean, ct_count, module_gene_names, num_modules) + specificity_df = compute_specificity_from_means(global_mean, ct_mean, module_gene_names, num_modules) + + # --- New metrics --- + logger.info("Computing z-score specificity...") + zscore_df = compute_specificity_zscore( + global_mean, global_std, ct_mean, ct_count, module_gene_names, num_modules, args.min_cells) + + logger.info("Computing Cohen's d...") + cohens_d_df = compute_cohens_d( + global_mean, ct_mean, ct_std, ct_count, module_gene_names, num_modules, n_cells, args.min_cells) + + logger.info("Computing Welch's t-test...") + welch_df = compute_welch_ttest( + global_mean, global_std, ct_mean, ct_std, ct_count, module_gene_names, + num_modules, n_cells, args.min_cells) + + logger.info("Computing attention-expression correlation...") + attn_expr_corr = compute_attention_expression_corr( + global_mean, expr_mean, module_gene_names, num_modules) + + logger.info("Computing random baseline comparison...") + baseline_stats = compute_random_baseline(global_mean, num_modules) + + logger.info("Computing attention concentration...") + concentration_df = compute_attention_concentration( + global_mean, ct_mean, ct_count, num_modules, args.min_cells) + + # --- Marker validation --- + marker_dict = PANCREAS_MARKERS + if args.marker_db: + with open(args.marker_db) as f: + marker_dict = json.load(f) + + logger.info("Computing marker ranks within cell types...") + marker_rank_df = compute_marker_rank_within_celltype( + ct_mean, ct_count, module_gene_names, num_modules, marker_dict, args.min_cells) + + marker_results_zscore = validate_markers(zscore_df, marker_dict, sort_col="z_score") + marker_results_fc = validate_markers(specificity_df, marker_dict, sort_col="fold_change") + + # --- Save CSVs --- + logger.info("Saving CSV results...") + global_df.to_csv(output_dir / "global_gene_importance.csv", index=False) + specificity_df.to_csv(output_dir / "gene_specificity.csv", index=False) + zscore_df.to_csv(output_dir / "gene_specificity_zscore.csv", index=False) + cohens_d_df.to_csv(output_dir / "gene_specificity_cohens_d.csv", index=False) + welch_df.to_csv(output_dir / "gene_specificity_welch.csv", index=False) + concentration_df.to_csv(output_dir / "attention_concentration.csv", index=False) + if not marker_rank_df.empty: + marker_rank_df.to_csv(output_dir / "marker_ranks_within_celltype.csv", index=False) + for ct, df in ct_dfs.items(): + safe_name = ct.replace("/", "_").replace(" ", "_") + df.to_csv(output_dir / f"celltype_{safe_name}_attention.csv", index=False) + + with open(output_dir / "marker_validation_zscore.json", "w") as f: + json.dump(marker_results_zscore, f, indent=2, ensure_ascii=False) + with open(output_dir / "marker_validation.json", "w") as f: + json.dump(marker_results_fc, f, indent=2, ensure_ascii=False) + + # --- DEG correlation --- + deg_results = None + if args.deg_csv: + deg_df = pd.read_csv(args.deg_csv) + deg_results = validate_deg_correlation(ct_dfs, deg_df) + with open(output_dir / "deg_correlation.json", "w") as f: + json.dump(deg_results, f, indent=2) + + # --- Publication figures --- + logger.info("Generating publication figures...") + plot_global_importance_pub(global_df, zscore_df, args.top_k_genes, output_dir) + plot_celltype_heatmap_pub(ct_dfs, ct_count, args.top_k_genes, output_dir, args.min_cells) + plot_specificity_volcano(welch_df, zscore_df, args.top_k_genes, output_dir) + plot_marker_dotplot(marker_rank_df, output_dir) + plot_marker_recall_pub(marker_results_zscore, ct_count, output_dir, args.min_cells) + plot_attention_vs_expression(global_mean, expr_mean, module_gene_names, num_modules, output_dir) + plot_concentration_pub(concentration_df, baseline_stats, output_dir) + plot_baseline_kl(baseline_stats, output_dir) + if deg_results: + plot_deg_correlation_pub(deg_results, output_dir) + + # --- Enrichment --- + run_enrichment_analysis(global_df, args.top_k_genes, output_dir) + + # --- Summary --- + filtered_cts = [ct for ct, n in ct_count.items() if n < args.min_cells] + summary = { + "n_cells": n_cells, + "n_modules": num_modules, + "n_celltypes": len(ct_count), + "celltypes": {ct: int(n) for ct, n in ct_count.items()}, + "min_cells_filter": args.min_cells, + "filtered_celltypes": filtered_cts, + "top_10_global_genes": global_df.groupby("gene")["mean_attention"].mean() + .nlargest(10).to_dict(), + "attn_expr_correlation": attn_expr_corr, + "baseline_comparison": baseline_stats, + "mean_gini_global": float(concentration_df["global_gini"].mean()), + "mean_gini_celltype": float(concentration_df["mean_ct_gini"].mean()), + } + with open(output_dir / "summary.json", "w") as f: + json.dump(summary, f, indent=2, ensure_ascii=False) + + logger.info("Done. Results saved to %s", output_dir) + + +if __name__ == "__main__": + main() + diff --git a/BioMod/interpretability/module_geometry.py b/BioMod/interpretability/module_geometry.py index aea2bf3e9e222826807660b93db27d6202500a04..6ce19e5f2f5007db686f04a73a8e4669c7601d88 100644 --- a/BioMod/interpretability/module_geometry.py +++ b/BioMod/interpretability/module_geometry.py @@ -1,654 +1,655 @@ -"""Experiment 1 — Intrinsic geometry of module embeddings. - -Question --------- -Does scLong's encoder organize its module latent space by pathway family -without any explicit family-level supervision, If yes, this is a structural -property gene-level baselines (GenePT / scGPT) cannot exhibit because they -do not have a module tier. - -Pipeline --------- -1. Load pretrained backbone (forward with mask_ratio=0). -2. Iterate over a population of cells, accumulate the per-module encoder output - in a streaming average -> [num_modules, base_dim]. -3. Parse each module's family from merged_modules.json keys - (Hallmarks / Reactome / Hotspot / Processed) and, where applicable, the - MSigDB hallmark process subcategory (immune / metabolic / signaling / ...). -4. Quantify family separation: - * silhouette score (cosine + euclidean) - * kNN purity at k=10 - * inter-family / intra-family mean-distance ratio - Compare each metric against a permutation null (shuffle family labels K - times) and report empirical p-values. -5. Visualise with UMAP and PCA, coloured by family / hallmark process. -6. Save embeddings, metadata, metrics.json, and figures. - -Data modes ----------- ---data_mode h5ad Generic h5ad. Goes through DataPreprocessor and - reindex_tensor_to_model_space. Works for - cell_annotation / zero_shot / any annotated h5ad. ---data_mode pretrain_npz Per-cell npz directory (cell0.npz, cell1.npz, ...), - already in the 27874-dim model gene space. Requires - --cell_num. Skips reindexing. ---data_mode pretrain_h5ad h5ad whose var.index is already the 27874-dim - model gene list (selected_genes_27k.txt). Skips - reindexing for speed. - -DDP ---- -Supports multi-GPU via torchrun. When WORLD_SIZE>1, each rank processes a -shard of cells; embeddings are all_reduced before metrics/plots. Single-GPU -and CPU work unchanged. - -CLI ---- -python -m performer_module.interpretability.module_geometry \\ - --performer_checkpoint \\ - --config_pkl \\ - --data_path \\ - --data_mode h5ad \\ - --output_dir ./interp/geometry/ \\ - --max_cells 5000 -""" -from __future__ import annotations - -import argparse -import json -import logging -import sys -from pathlib import Path -from typing import Dict, List, Optional, Tuple - -import numpy as np -import pandas as pd -import torch -import torch.distributed as dist -from torch.utils.data import DataLoader -from torch.utils.data.distributed import DistributedSampler - -import matplotlib -matplotlib.use("Agg") -import matplotlib.pyplot as plt - -from sklearn.decomposition import PCA -from sklearn.metrics import silhouette_score -from sklearn.metrics.pairwise import cosine_distances, euclidean_distances -from sklearn.neighbors import NearestNeighbors - -# Make sibling modules importable when launched as a script -HERE = Path(__file__).resolve() -PERFORMER_MODULE = HERE.parents[1] -REPO_ROOT = HERE.parents[2] -for p in (str(PERFORMER_MODULE), str(REPO_ROOT)): - if p not in sys.path: - sys.path.insert(0, p) - -from cell_annotation.performer_module_finetune import ( # noqa: E402 - DataPreprocessor, - get_rank, - get_world_size, - init_distributed_mode, - is_dist_initialized, - is_main_process, - load_backbone, - reduce_sum, - resolve_path as ca_resolve_path, - seed_all, -) - -from interpretability.utils import ( # noqa: E402 - MatrixRowDataset, - NpzCellDataset, - build_module_metadata, - ensure_dir, - load_module_names, -) - -logger = logging.getLogger(__name__) - - -# ---------- argparse ---------- - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - - # Pretrained backbone (mirrors cell_annotation.performer_module_finetune) - parser.add_argument("--performer_checkpoint", type=str, required=True) - parser.add_argument("--config_pkl", type=str, required=True) - parser.add_argument("--module_data_dir", type=str, default="../module_data_all") - parser.add_argument("--module_json", type=str, default=None, - help="Path to merged_modules.json. Defaults to /merged_modules.json.") - parser.add_argument("--gene_list_path", type=str, default="../../selected_genes_27k.txt") - parser.add_argument("--gene_mapping_path", type=str, default="../../gene2ens.csv") - parser.add_argument("--gene2vec_data_path", type=str, default="../../selected_gene2vec_27k.npy") - parser.add_argument("--gene2ens_file", type=str, default="../../gene2ens.csv") - parser.add_argument("--gene2ens_column", type=str, default="Ensembl gene ID") - parser.add_argument("--shuffle_module_genes", action="store_true", - help="Randomly reshuffle gene-to-module assignment (control / ablation).") - parser.add_argument("--shuffle_seed", type=int, default=2021) - - # Data - parser.add_argument("--data_path", type=str, required=True, - help="h5ad file path, or npz directory for pretrain_npz mode.") - parser.add_argument("--data_mode", type=str, default="h5ad", - choices=["h5ad", "pretrain_h5ad", "pretrain_npz"], - help="h5ad: generic, goes through DataPreprocessor reindex. " - "pretrain_h5ad: var.index is already the 27874 model gene list, skip reindex. " - "pretrain_npz: per-cell npz directory (cell0.npz, ...), skip reindex.") - parser.add_argument("--cell_num", type=int, default=None, - help="Total cell count for pretrain_npz sequential naming (cell0.npz..cellN.npz).") - parser.add_argument("--gene_name_column", type=str, default="auto", - help="Which adata.var column holds gene symbols. 'auto' / 'index' / column name. " - "Only used when data_mode=h5ad.") - parser.add_argument("--dataset_name", type=str, default="interpretability") - parser.add_argument("--min_genes", type=int, default=0) - parser.add_argument("--min_cells", type=int, default=0) - parser.add_argument("--max_cells", type=int, default=None, - help="Optional subsample for inference speed. None = use all cells.") - - # Inference - parser.add_argument("--device", type=str, default="cuda") - parser.add_argument("--batch_size", type=int, default=8) - parser.add_argument("--num_workers", type=int, default=0) - parser.add_argument("--seed", type=int, default=2021) - - # Geometry / metrics - parser.add_argument("--knn_k", type=int, default=10, - help="k for kNN-purity metric.") - parser.add_argument("--null_permutations", type=int, default=200, - help="Number of label-permutation samples for null distribution.") - parser.add_argument("--include_other_in_metrics", action="store_true", - help="If false (default), exclude modules whose family is 'Other' from metric computation.") - - # UMAP - parser.add_argument("--umap_n_neighbors", type=int, default=15) - parser.add_argument("--umap_min_dist", type=float, default=0.3) - - # Output - parser.add_argument("--output_dir", type=str, required=True) - parser.add_argument("--run_name", type=str, default="") - - return parser.parse_args() - - -# ---------- dataset construction ---------- - -def build_inference_loader( - args: argparse.Namespace, - preprocessor: Optional[DataPreprocessor], - use_ddp: bool, - rank: int, - world_size: int, -) -> Tuple[DataLoader, Optional[dict], int]: - """Build a DataLoader yielding raw [G_input] expression rows and a reindex_state. - - Returns: (loader, reindex_state_or_None, n_cells_total) - """ - if args.data_mode == "h5ad": - import anndata as ad - assert preprocessor is not None - adata, reindex_state = preprocessor.preprocess_adata( - adata_path=args.data_path, - min_genes=args.min_genes, - min_cells=args.min_cells, - max_cells=args.max_cells, - dataset_name=args.dataset_name, - gene_name_column=args.gene_name_column, - ) - dataset = MatrixRowDataset(adata.X) - n_cells = adata.n_obs - elif args.data_mode == "pretrain_h5ad": - import anndata as ad - logger.info("Loading pretrain-mode h5ad %s", args.data_path) - adata = ad.read_h5ad(args.data_path) - if args.max_cells is not None and adata.n_obs > args.max_cells: - rng = np.random.default_rng(args.seed) - chosen = np.sort(rng.choice(adata.n_obs, size=args.max_cells, replace=False)) - adata = adata[chosen].copy() - dataset = MatrixRowDataset(adata.X) - n_cells = adata.n_obs - reindex_state = None - elif args.data_mode == "pretrain_npz": - dataset = NpzCellDataset( - data_dir=args.data_path, - cell_num=args.cell_num, - max_cells=args.max_cells, - seed=args.seed, - ) - n_cells = len(dataset) - reindex_state = None - else: - raise ValueError(f"Unsupported data_mode: {args.data_mode}") - - if use_ddp: - sampler = DistributedSampler(dataset, num_replicas=world_size, rank=rank, - shuffle=False, drop_last=False) - else: - sampler = None - - loader = DataLoader( - dataset, - batch_size=args.batch_size, - shuffle=False, - sampler=sampler, - num_workers=args.num_workers, - pin_memory=False, - ) - logger.info("Built inference loader: %d cells total, batch_size=%d, ddp=%s", - n_cells, args.batch_size, use_ddp) - return loader, reindex_state, n_cells - - -# ---------- inference: cell-averaged module embeddings ---------- - -@torch.no_grad() -def extract_cell_averaged_module_embeddings( - backbone, - loader: DataLoader, - preprocessor: Optional[DataPreprocessor], - reindex_state: Optional[dict], - device: torch.device, - num_modules: int, - base_dim: int, - expected_input_genes: int, - use_ddp: bool, -) -> Tuple[np.ndarray, int]: - """Stream forward over the loader and accumulate the cell-mean module embedding. - - Under DDP each rank processes its shard; sum_emb and n_cells are all_reduced - before computing the mean. - - Returns: (mean_embedding [num_modules, base_dim] float32, n_cells_total) - """ - backbone.eval() - sum_emb = torch.zeros(num_modules, base_dim, device=device, dtype=torch.float64) - n_cells = 0 - - for batch_idx, x in enumerate(loader): - x = x.to(device, non_blocking=False) - if reindex_state is not None: - x_model = preprocessor.reindex_tensor_to_model_space(x, reindex_state=reindex_state) - else: - if x.shape[1] != expected_input_genes: - raise ValueError( - f"pretrain mode expects {expected_input_genes} genes per cell, got {x.shape[1]}. " - f"Use --data_mode h5ad (with reindex) for arbitrary gene spaces." - ) - x_model = x - - out = backbone(x_model, mask_ratio=0.0, return_encodings=True) - enc = out["module_encodings"] # (B, num_modules, D) - if enc.shape[1] != num_modules or enc.shape[2] != base_dim: - raise RuntimeError( - f"Encoder output shape mismatch: got {tuple(enc.shape)}, " - f"expected (*, {num_modules}, {base_dim}). " - "Module order between merged_modules.json and the loaded backbone may have drifted." - ) - sum_emb += enc.to(torch.float64).sum(dim=0) - n_cells += enc.shape[0] - if (batch_idx + 1) % 50 == 0: - logger.info(" forward batch %d (%d cells on this rank so far)", batch_idx + 1, n_cells) - - if use_ddp: - dist.all_reduce(sum_emb, op=dist.ReduceOp.SUM) - n_cells_tensor = torch.tensor(n_cells, device=device, dtype=torch.long) - dist.all_reduce(n_cells_tensor, op=dist.ReduceOp.SUM) - n_cells = int(n_cells_tensor.item()) - - mean_emb = (sum_emb / max(n_cells, 1)).cpu().numpy().astype(np.float32) - return mean_emb, n_cells - - -# ---------- structure metrics ---------- - -def _label_to_int(labels: List[str]) -> np.ndarray: - uniq = sorted(set(labels)) - mapping = {lab: i for i, lab in enumerate(uniq)} - return np.asarray([mapping[lab] for lab in labels], dtype=np.int64) - - -def _knn_purity(emb: np.ndarray, labels: np.ndarray, k: int, metric: str = "cosine") -> float: - n = emb.shape[0] - k_use = min(k, n - 1) - nn = NearestNeighbors(n_neighbors=k_use + 1, metric=metric) - nn.fit(emb) - _, idx = nn.kneighbors(emb) - # idx[:, 0] == self; drop it - neighbor_labels = labels[idx[:, 1:]] - own_labels = labels[:, None] - purity = (neighbor_labels == own_labels).mean() - return float(purity) - - -def _intra_inter_distance_ratio(distance_matrix: np.ndarray, labels: np.ndarray) -> Dict[str, float]: - """Mean intra-family vs inter-family pairwise distance and inter/intra ratio.""" - n = distance_matrix.shape[0] - iu = np.triu_indices(n, k=1) - pair_d = distance_matrix[iu] - same = (labels[iu[0]] == labels[iu[1]]) - intra = float(pair_d[same].mean()) if same.any() else float("nan") - inter = float(pair_d[~same].mean()) if (~same).any() else float("nan") - ratio = float(inter / intra) if intra > 0 else float("nan") - return {"intra_mean": intra, "inter_mean": inter, "inter_over_intra": ratio} - - -def compute_structure_metrics( - embedding: np.ndarray, - labels: np.ndarray, - knn_k: int, -) -> Dict[str, float]: - metrics: Dict[str, float] = {} - n_classes = len(np.unique(labels)) - if n_classes < 2 or len(labels) < 3: - return {"silhouette_cosine": float("nan"), - "silhouette_euclidean": float("nan"), - "knn_purity_cosine": float("nan"), - "knn_purity_euclidean": float("nan"), - "inter_over_intra_cosine": float("nan"), - "inter_over_intra_euclidean": float("nan")} - - metrics["silhouette_cosine"] = float(silhouette_score(embedding, labels, metric="cosine")) - metrics["silhouette_euclidean"] = float(silhouette_score(embedding, labels, metric="euclidean")) - metrics["knn_purity_cosine"] = _knn_purity(embedding, labels, knn_k, metric="cosine") - metrics["knn_purity_euclidean"] = _knn_purity(embedding, labels, knn_k, metric="euclidean") - - cos_d = cosine_distances(embedding) - euc_d = euclidean_distances(embedding) - cos_ratio = _intra_inter_distance_ratio(cos_d, labels) - euc_ratio = _intra_inter_distance_ratio(euc_d, labels) - metrics["intra_mean_cosine"] = cos_ratio["intra_mean"] - metrics["inter_mean_cosine"] = cos_ratio["inter_mean"] - metrics["inter_over_intra_cosine"] = cos_ratio["inter_over_intra"] - metrics["intra_mean_euclidean"] = euc_ratio["intra_mean"] - metrics["inter_mean_euclidean"] = euc_ratio["inter_mean"] - metrics["inter_over_intra_euclidean"] = euc_ratio["inter_over_intra"] - return metrics - - -def permutation_null( - embedding: np.ndarray, - labels: np.ndarray, - knn_k: int, - n_permutations: int, - seed: int, -) -> Dict[str, Dict[str, float]]: - """Shuffle labels K times, recompute metrics, return per-metric (mean, std, p_value). - - p-value: empirical right-tail Pr(null >= observed) under random labeling. - """ - observed = compute_structure_metrics(embedding, labels, knn_k) - rng = np.random.default_rng(seed) - null_records: Dict[str, List[float]] = {k: [] for k in observed} - - for _ in range(n_permutations): - permuted = rng.permutation(labels) - m = compute_structure_metrics(embedding, permuted, knn_k) - for k, v in m.items(): - null_records[k].append(v) - - summary: Dict[str, Dict[str, float]] = {} - for k, obs_v in observed.items(): - arr = np.asarray(null_records[k], dtype=float) - valid = arr[~np.isnan(arr)] - if valid.size == 0 or np.isnan(obs_v): - summary[k] = {"observed": obs_v, "null_mean": float("nan"), - "null_std": float("nan"), "p_right_tail": float("nan")} - continue - p = float((1 + np.sum(valid >= obs_v)) / (valid.size + 1)) - summary[k] = { - "observed": float(obs_v), - "null_mean": float(valid.mean()), - "null_std": float(valid.std()), - "p_right_tail": p, - } - return summary - - -# ---------- visualization ---------- - -def _color_map(labels: List[str]) -> Dict[str, tuple]: - uniq = sorted(set(labels)) - cmap = plt.get_cmap("tab20", max(len(uniq), 1)) - return {lab: cmap(i) for i, lab in enumerate(uniq)} - - -def _scatter(coords: np.ndarray, labels: List[str], title: str, out_path: Path) -> None: - fig, ax = plt.subplots(figsize=(7, 6)) - cmap = _color_map(labels) - for lab in sorted(set(labels)): - mask = np.asarray([lab == l for l in labels]) - ax.scatter(coords[mask, 0], coords[mask, 1], - color=cmap[lab], label=f"{lab} (n={mask.sum()})", - s=20, alpha=0.85, edgecolor="none") - ax.set_title(title) - ax.set_xlabel("dim 1") - ax.set_ylabel("dim 2") - ax.legend(loc="best", fontsize=8, frameon=True) - fig.tight_layout() - fig.savefig(out_path, dpi=160) - plt.close(fig) - - -def visualize( - embedding: np.ndarray, - metadata_df: pd.DataFrame, - output_dir: Path, - umap_n_neighbors: int, - umap_min_dist: float, -) -> None: - # PCA 2D - pca = PCA(n_components=2, random_state=0) - pca_coords = pca.fit_transform(embedding) - _scatter(pca_coords, metadata_df["family"].tolist(), - "Module embedding PCA — family", output_dir / "pca_family.png") - - has_hallmark = metadata_df["hallmark_process"].notna().any() - if has_hallmark: - # Project only hallmark modules to a separate PCA for readability - hm_mask = metadata_df["hallmark_process"].notna().to_numpy() - if hm_mask.sum() >= 3: - hm_pca = PCA(n_components=2, random_state=0).fit_transform(embedding[hm_mask]) - _scatter(hm_pca, metadata_df.loc[hm_mask, "hallmark_process"].tolist(), - "Hallmark modules PCA — process", - output_dir / "pca_hallmark_process.png") - - # UMAP (optional) - try: - import umap # type: ignore - reducer = umap.UMAP( - n_components=2, - n_neighbors=min(umap_n_neighbors, max(2, embedding.shape[0] - 1)), - min_dist=umap_min_dist, - metric="cosine", - random_state=0, - ) - umap_coords = reducer.fit_transform(embedding) - _scatter(umap_coords, metadata_df["family"].tolist(), - "Module embedding UMAP — family", output_dir / "umap_family.png") - - if has_hallmark and hm_mask.sum() >= 3: - hm_reducer = umap.UMAP( - n_components=2, - n_neighbors=min(umap_n_neighbors, max(2, hm_mask.sum() - 1)), - min_dist=umap_min_dist, - metric="cosine", - random_state=0, - ) - hm_umap = hm_reducer.fit_transform(embedding[hm_mask]) - _scatter(hm_umap, metadata_df.loc[hm_mask, "hallmark_process"].tolist(), - "Hallmark modules UMAP — process", - output_dir / "umap_hallmark_process.png") - except ImportError: - logger.warning("umap-learn not installed; skipping UMAP plots. `pip install umap-learn` to enable.") - - -# ---------- driver ---------- - -def setup_logging(output_dir: Path) -> None: - log_path = output_dir / "run.log" - logging.basicConfig( - level=logging.INFO, - format="%(asctime)s | %(levelname)s | %(name)s | %(message)s", - handlers=[logging.StreamHandler(sys.stdout), logging.FileHandler(log_path, mode="w")], - force=True, - ) - - -def resolve_module_json(args: argparse.Namespace) -> str: - """Resolve relative paths against performer_module/cell_annotation/ to stay - consistent with how `load_backbone` reads `module_data_dir`. - """ - if args.module_json is not None: - return str(ca_resolve_path(args.module_json)) - return str(ca_resolve_path(args.module_data_dir) / "merged_modules.json") - - -def main() -> None: - args = parse_args() - - # DDP init (auto-detects torchrun env; falls back to single-process) - device, use_ddp, rank, world_size, local_rank = init_distributed_mode(args.device) - seed_all(args.seed + rank) - - if is_main_process(): - output_dir = ensure_dir(args.output_dir) - setup_logging(output_dir) - else: - output_dir = Path(args.output_dir) - logging.basicConfig(level=logging.WARNING) - if use_ddp: - dist.barrier() - - logger.info("Args: %s", vars(args)) - logger.info("DDP: use_ddp=%s, rank=%d/%d, device=%s", use_ddp, rank, world_size, device) - - # 1. Load backbone - backbone, module_data, model_config = load_backbone(args, device) - backbone.eval() - base_dim = int(model_config["base_dim"]) - num_modules = len(module_data) - logger.info("Loaded backbone: num_modules=%d, base_dim=%d", num_modules, base_dim) - - # 2. Load module names (must align with module_data order) - module_json_path = resolve_module_json(args) - module_names = load_module_names(module_json_path) - if len(module_names) != num_modules: - raise RuntimeError( - f"Module name count ({len(module_names)} from {module_json_path}) does not match " - f"backbone num_modules ({num_modules}). The merged_modules.json used to build the " - "model and the one given here must be identical, with no modules dropped during " - "gene-symbol resolution." - ) - metadata = build_module_metadata(module_names) - metadata_df = pd.DataFrame(metadata) - if is_main_process(): - logger.info("Family distribution:\n%s", metadata_df["family"].value_counts().to_string()) - - # 3. Build dataloader - if args.data_mode == "h5ad": - preprocessor = DataPreprocessor( - gene2ens_file=args.gene2ens_file, - gene2ens_column=args.gene2ens_column, - model_genes_file=args.gene_list_path, - ) - preprocessor.load_gene_mapping() - preprocessor.load_model_genes() - expected_input_genes = -1 - else: - preprocessor = None - expected_input_genes = int(model_config["num_genes"]) - - loader, reindex_state, n_cells = build_inference_loader( - args, preprocessor, use_ddp, rank, world_size, - ) - if n_cells == 0: - raise RuntimeError("No cells survived preprocessing.") - - # 4. Inference - logger.info("Extracting cell-averaged module embeddings over %d cells...", n_cells) - mean_emb, processed = extract_cell_averaged_module_embeddings( - backbone, loader, preprocessor, reindex_state, device, - num_modules=num_modules, base_dim=base_dim, - expected_input_genes=expected_input_genes, - use_ddp=use_ddp, - ) - logger.info("Embedding ready: shape=%s, processed_cells=%d", mean_emb.shape, processed) - - # --- Everything below is rank-0 only --- - if is_main_process(): - # 5. Save raw outputs - np.save(output_dir / "module_embeddings.npy", mean_emb) - metadata_df.to_csv(output_dir / "module_metadata.csv", index=False) - - # 6. Metrics — family level - if args.include_other_in_metrics: - keep_mask = np.ones(len(metadata_df), dtype=bool) - else: - keep_mask = (metadata_df["family"] != "Other").to_numpy() - family_labels = metadata_df.loc[keep_mask, "family"].tolist() - family_emb = mean_emb[keep_mask] - - logger.info("Computing structure metrics — family level (n=%d)", keep_mask.sum()) - family_summary = permutation_null( - family_emb, _label_to_int(family_labels), - knn_k=args.knn_k, - n_permutations=args.null_permutations, - seed=args.seed, - ) - - # 6b. Metrics — hallmark process level - hm_mask = metadata_df["hallmark_process"].notna().to_numpy() - hallmark_summary: Optional[Dict[str, Dict[str, float]]] = None - if hm_mask.sum() >= 3 and metadata_df.loc[hm_mask, "hallmark_process"].nunique() >= 2: - logger.info("Computing structure metrics — hallmark process level (n=%d)", hm_mask.sum()) - hallmark_summary = permutation_null( - mean_emb[hm_mask], - _label_to_int(metadata_df.loc[hm_mask, "hallmark_process"].tolist()), - knn_k=args.knn_k, - n_permutations=args.null_permutations, - seed=args.seed, - ) - - metrics_payload = { - "n_cells_processed": int(processed), - "num_modules": int(num_modules), - "base_dim": int(base_dim), - "knn_k": int(args.knn_k), - "null_permutations": int(args.null_permutations), - "family_metrics": family_summary, - "hallmark_process_metrics": hallmark_summary, - "family_counts": metadata_df["family"].value_counts().to_dict(), - "hallmark_process_counts": metadata_df["hallmark_process"].dropna().value_counts().to_dict(), - "shuffle_module_genes": bool(args.shuffle_module_genes), - } - with open(output_dir / "metrics.json", "w", encoding="utf-8") as fh: - json.dump(metrics_payload, fh, indent=2) - - logger.info("=== Family-level structure (vs random-shuffled label null) ===") - for k, v in family_summary.items(): - logger.info(" %-30s observed=%.4f null=%.4f±%.4f p=%.4f", - k, v["observed"], v["null_mean"], v["null_std"], v["p_right_tail"]) - if hallmark_summary is not None: - logger.info("=== Hallmark-process-level structure (Hallmarks subset) ===") - for k, v in hallmark_summary.items(): - logger.info(" %-30s observed=%.4f null=%.4f±%.4f p=%.4f", - k, v["observed"], v["null_mean"], v["null_std"], v["p_right_tail"]) - - # 7. Visualisation - logger.info("Plotting UMAP / PCA...") - visualize(mean_emb, metadata_df, output_dir, - umap_n_neighbors=args.umap_n_neighbors, - umap_min_dist=args.umap_min_dist) - - logger.info("Done. Outputs saved to %s", output_dir) - - # Teardown - if use_ddp: - dist.barrier() - dist.destroy_process_group() - - -if __name__ == "__main__": - main() +"""Experiment 1 — Intrinsic geometry of module embeddings. + +Question +-------- +Does scLong's encoder organize its module latent space by pathway family +without any explicit family-level supervision, If yes, this is a structural +property gene-level baselines (GenePT / scGPT) cannot exhibit because they +do not have a module tier. + +Pipeline +-------- +1. Load pretrained backbone (forward with mask_ratio=0). +2. Iterate over a population of cells, accumulate the per-module encoder output + in a streaming average -> [num_modules, base_dim]. +3. Parse each module's family from merged_modules.json keys + (Hallmarks / Reactome / Hotspot / Processed) and, where applicable, the + MSigDB hallmark process subcategory (immune / metabolic / signaling / ...). +4. Quantify family separation: + * silhouette score (cosine + euclidean) + * kNN purity at k=10 + * inter-family / intra-family mean-distance ratio + Compare each metric against a permutation null (shuffle family labels K + times) and report empirical p-values. +5. Visualise with UMAP and PCA, coloured by family / hallmark process. +6. Save embeddings, metadata, metrics.json, and figures. + +Data modes +---------- +--data_mode h5ad Generic h5ad. Goes through DataPreprocessor and + reindex_tensor_to_model_space. Works for + cell_annotation / zero_shot / any annotated h5ad. +--data_mode pretrain_npz Per-cell npz directory (cell0.npz, cell1.npz, ...), + already in the 27874-dim model gene space. Requires + --cell_num. Skips reindexing. +--data_mode pretrain_h5ad h5ad whose var.index is already the 27874-dim + model gene list (selected_genes_27k.txt). Skips + reindexing for speed. + +DDP +--- +Supports multi-GPU via torchrun. When WORLD_SIZE>1, each rank processes a +shard of cells; embeddings are all_reduced before metrics/plots. Single-GPU +and CPU work unchanged. + +CLI +--- +python -m performer_module.interpretability.module_geometry \\ + --performer_checkpoint \\ + --config_pkl \\ + --data_path \\ + --data_mode h5ad \\ + --output_dir ./interp/geometry/ \\ + --max_cells 5000 +""" +from __future__ import annotations + +import argparse +import json +import logging +import sys +from pathlib import Path +from typing import Dict, List, Optional, Tuple + +import numpy as np +import pandas as pd +import torch +import torch.distributed as dist +from torch.utils.data import DataLoader +from torch.utils.data.distributed import DistributedSampler + +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt + +from sklearn.decomposition import PCA +from sklearn.metrics import silhouette_score +from sklearn.metrics.pairwise import cosine_distances, euclidean_distances +from sklearn.neighbors import NearestNeighbors + +# Make sibling modules importable when launched as a script +HERE = Path(__file__).resolve() +PERFORMER_MODULE = HERE.parents[1] +REPO_ROOT = HERE.parents[2] +ASSET_ROOT = REPO_ROOT / "model_dependencies" +for p in (str(PERFORMER_MODULE), str(REPO_ROOT)): + if p not in sys.path: + sys.path.insert(0, p) + +from cell_annotation.performer_module_finetune import ( # noqa: E402 + DataPreprocessor, + get_rank, + get_world_size, + init_distributed_mode, + is_dist_initialized, + is_main_process, + load_backbone, + reduce_sum, + resolve_path as ca_resolve_path, + seed_all, +) + +from interpretability.utils import ( # noqa: E402 + MatrixRowDataset, + NpzCellDataset, + build_module_metadata, + ensure_dir, + load_module_names, +) + +logger = logging.getLogger(__name__) + + +# ---------- argparse ---------- + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + + # Pretrained backbone (mirrors cell_annotation.performer_module_finetune) + parser.add_argument("--performer_checkpoint", type=str, required=True) + parser.add_argument("--config_pkl", type=str, required=True) + parser.add_argument("--module_data_dir", type=str, default=str(ASSET_ROOT / "module_data_all")) + parser.add_argument("--module_json", type=str, default=None, + help="Path to merged_modules.json. Defaults to /merged_modules.json.") + parser.add_argument("--gene_list_path", type=str, default=str(ASSET_ROOT / "selected_genes_27k.txt")) + parser.add_argument("--gene_mapping_path", type=str, default=str(ASSET_ROOT / "gene2ens.csv")) + parser.add_argument("--gene2vec_data_path", type=str, default=str(ASSET_ROOT / "selected_gene2vec_27k.npy")) + parser.add_argument("--gene2ens_file", type=str, default=str(ASSET_ROOT / "gene2ens.csv")) + parser.add_argument("--gene2ens_column", type=str, default="Ensembl gene ID") + parser.add_argument("--shuffle_module_genes", action="store_true", + help="Randomly reshuffle gene-to-module assignment (control / ablation).") + parser.add_argument("--shuffle_seed", type=int, default=2021) + + # Data + parser.add_argument("--data_path", type=str, required=True, + help="h5ad file path, or npz directory for pretrain_npz mode.") + parser.add_argument("--data_mode", type=str, default="h5ad", + choices=["h5ad", "pretrain_h5ad", "pretrain_npz"], + help="h5ad: generic, goes through DataPreprocessor reindex. " + "pretrain_h5ad: var.index is already the 27874 model gene list, skip reindex. " + "pretrain_npz: per-cell npz directory (cell0.npz, ...), skip reindex.") + parser.add_argument("--cell_num", type=int, default=None, + help="Total cell count for pretrain_npz sequential naming (cell0.npz..cellN.npz).") + parser.add_argument("--gene_name_column", type=str, default="auto", + help="Which adata.var column holds gene symbols. 'auto' / 'index' / column name. " + "Only used when data_mode=h5ad.") + parser.add_argument("--dataset_name", type=str, default="interpretability") + parser.add_argument("--min_genes", type=int, default=0) + parser.add_argument("--min_cells", type=int, default=0) + parser.add_argument("--max_cells", type=int, default=None, + help="Optional subsample for inference speed. None = use all cells.") + + # Inference + parser.add_argument("--device", type=str, default="cuda") + parser.add_argument("--batch_size", type=int, default=8) + parser.add_argument("--num_workers", type=int, default=0) + parser.add_argument("--seed", type=int, default=2021) + + # Geometry / metrics + parser.add_argument("--knn_k", type=int, default=10, + help="k for kNN-purity metric.") + parser.add_argument("--null_permutations", type=int, default=200, + help="Number of label-permutation samples for null distribution.") + parser.add_argument("--include_other_in_metrics", action="store_true", + help="If false (default), exclude modules whose family is 'Other' from metric computation.") + + # UMAP + parser.add_argument("--umap_n_neighbors", type=int, default=15) + parser.add_argument("--umap_min_dist", type=float, default=0.3) + + # Output + parser.add_argument("--output_dir", type=str, required=True) + parser.add_argument("--run_name", type=str, default="") + + return parser.parse_args() + + +# ---------- dataset construction ---------- + +def build_inference_loader( + args: argparse.Namespace, + preprocessor: Optional[DataPreprocessor], + use_ddp: bool, + rank: int, + world_size: int, +) -> Tuple[DataLoader, Optional[dict], int]: + """Build a DataLoader yielding raw [G_input] expression rows and a reindex_state. + + Returns: (loader, reindex_state_or_None, n_cells_total) + """ + if args.data_mode == "h5ad": + import anndata as ad + assert preprocessor is not None + adata, reindex_state = preprocessor.preprocess_adata( + adata_path=args.data_path, + min_genes=args.min_genes, + min_cells=args.min_cells, + max_cells=args.max_cells, + dataset_name=args.dataset_name, + gene_name_column=args.gene_name_column, + ) + dataset = MatrixRowDataset(adata.X) + n_cells = adata.n_obs + elif args.data_mode == "pretrain_h5ad": + import anndata as ad + logger.info("Loading pretrain-mode h5ad %s", args.data_path) + adata = ad.read_h5ad(args.data_path) + if args.max_cells is not None and adata.n_obs > args.max_cells: + rng = np.random.default_rng(args.seed) + chosen = np.sort(rng.choice(adata.n_obs, size=args.max_cells, replace=False)) + adata = adata[chosen].copy() + dataset = MatrixRowDataset(adata.X) + n_cells = adata.n_obs + reindex_state = None + elif args.data_mode == "pretrain_npz": + dataset = NpzCellDataset( + data_dir=args.data_path, + cell_num=args.cell_num, + max_cells=args.max_cells, + seed=args.seed, + ) + n_cells = len(dataset) + reindex_state = None + else: + raise ValueError(f"Unsupported data_mode: {args.data_mode}") + + if use_ddp: + sampler = DistributedSampler(dataset, num_replicas=world_size, rank=rank, + shuffle=False, drop_last=False) + else: + sampler = None + + loader = DataLoader( + dataset, + batch_size=args.batch_size, + shuffle=False, + sampler=sampler, + num_workers=args.num_workers, + pin_memory=False, + ) + logger.info("Built inference loader: %d cells total, batch_size=%d, ddp=%s", + n_cells, args.batch_size, use_ddp) + return loader, reindex_state, n_cells + + +# ---------- inference: cell-averaged module embeddings ---------- + +@torch.no_grad() +def extract_cell_averaged_module_embeddings( + backbone, + loader: DataLoader, + preprocessor: Optional[DataPreprocessor], + reindex_state: Optional[dict], + device: torch.device, + num_modules: int, + base_dim: int, + expected_input_genes: int, + use_ddp: bool, +) -> Tuple[np.ndarray, int]: + """Stream forward over the loader and accumulate the cell-mean module embedding. + + Under DDP each rank processes its shard; sum_emb and n_cells are all_reduced + before computing the mean. + + Returns: (mean_embedding [num_modules, base_dim] float32, n_cells_total) + """ + backbone.eval() + sum_emb = torch.zeros(num_modules, base_dim, device=device, dtype=torch.float64) + n_cells = 0 + + for batch_idx, x in enumerate(loader): + x = x.to(device, non_blocking=False) + if reindex_state is not None: + x_model = preprocessor.reindex_tensor_to_model_space(x, reindex_state=reindex_state) + else: + if x.shape[1] != expected_input_genes: + raise ValueError( + f"pretrain mode expects {expected_input_genes} genes per cell, got {x.shape[1]}. " + f"Use --data_mode h5ad (with reindex) for arbitrary gene spaces." + ) + x_model = x + + out = backbone(x_model, mask_ratio=0.0, return_encodings=True) + enc = out["module_encodings"] # (B, num_modules, D) + if enc.shape[1] != num_modules or enc.shape[2] != base_dim: + raise RuntimeError( + f"Encoder output shape mismatch: got {tuple(enc.shape)}, " + f"expected (*, {num_modules}, {base_dim}). " + "Module order between merged_modules.json and the loaded backbone may have drifted." + ) + sum_emb += enc.to(torch.float64).sum(dim=0) + n_cells += enc.shape[0] + if (batch_idx + 1) % 50 == 0: + logger.info(" forward batch %d (%d cells on this rank so far)", batch_idx + 1, n_cells) + + if use_ddp: + dist.all_reduce(sum_emb, op=dist.ReduceOp.SUM) + n_cells_tensor = torch.tensor(n_cells, device=device, dtype=torch.long) + dist.all_reduce(n_cells_tensor, op=dist.ReduceOp.SUM) + n_cells = int(n_cells_tensor.item()) + + mean_emb = (sum_emb / max(n_cells, 1)).cpu().numpy().astype(np.float32) + return mean_emb, n_cells + + +# ---------- structure metrics ---------- + +def _label_to_int(labels: List[str]) -> np.ndarray: + uniq = sorted(set(labels)) + mapping = {lab: i for i, lab in enumerate(uniq)} + return np.asarray([mapping[lab] for lab in labels], dtype=np.int64) + + +def _knn_purity(emb: np.ndarray, labels: np.ndarray, k: int, metric: str = "cosine") -> float: + n = emb.shape[0] + k_use = min(k, n - 1) + nn = NearestNeighbors(n_neighbors=k_use + 1, metric=metric) + nn.fit(emb) + _, idx = nn.kneighbors(emb) + # idx[:, 0] == self; drop it + neighbor_labels = labels[idx[:, 1:]] + own_labels = labels[:, None] + purity = (neighbor_labels == own_labels).mean() + return float(purity) + + +def _intra_inter_distance_ratio(distance_matrix: np.ndarray, labels: np.ndarray) -> Dict[str, float]: + """Mean intra-family vs inter-family pairwise distance and inter/intra ratio.""" + n = distance_matrix.shape[0] + iu = np.triu_indices(n, k=1) + pair_d = distance_matrix[iu] + same = (labels[iu[0]] == labels[iu[1]]) + intra = float(pair_d[same].mean()) if same.any() else float("nan") + inter = float(pair_d[~same].mean()) if (~same).any() else float("nan") + ratio = float(inter / intra) if intra > 0 else float("nan") + return {"intra_mean": intra, "inter_mean": inter, "inter_over_intra": ratio} + + +def compute_structure_metrics( + embedding: np.ndarray, + labels: np.ndarray, + knn_k: int, +) -> Dict[str, float]: + metrics: Dict[str, float] = {} + n_classes = len(np.unique(labels)) + if n_classes < 2 or len(labels) < 3: + return {"silhouette_cosine": float("nan"), + "silhouette_euclidean": float("nan"), + "knn_purity_cosine": float("nan"), + "knn_purity_euclidean": float("nan"), + "inter_over_intra_cosine": float("nan"), + "inter_over_intra_euclidean": float("nan")} + + metrics["silhouette_cosine"] = float(silhouette_score(embedding, labels, metric="cosine")) + metrics["silhouette_euclidean"] = float(silhouette_score(embedding, labels, metric="euclidean")) + metrics["knn_purity_cosine"] = _knn_purity(embedding, labels, knn_k, metric="cosine") + metrics["knn_purity_euclidean"] = _knn_purity(embedding, labels, knn_k, metric="euclidean") + + cos_d = cosine_distances(embedding) + euc_d = euclidean_distances(embedding) + cos_ratio = _intra_inter_distance_ratio(cos_d, labels) + euc_ratio = _intra_inter_distance_ratio(euc_d, labels) + metrics["intra_mean_cosine"] = cos_ratio["intra_mean"] + metrics["inter_mean_cosine"] = cos_ratio["inter_mean"] + metrics["inter_over_intra_cosine"] = cos_ratio["inter_over_intra"] + metrics["intra_mean_euclidean"] = euc_ratio["intra_mean"] + metrics["inter_mean_euclidean"] = euc_ratio["inter_mean"] + metrics["inter_over_intra_euclidean"] = euc_ratio["inter_over_intra"] + return metrics + + +def permutation_null( + embedding: np.ndarray, + labels: np.ndarray, + knn_k: int, + n_permutations: int, + seed: int, +) -> Dict[str, Dict[str, float]]: + """Shuffle labels K times, recompute metrics, return per-metric (mean, std, p_value). + + p-value: empirical right-tail Pr(null >= observed) under random labeling. + """ + observed = compute_structure_metrics(embedding, labels, knn_k) + rng = np.random.default_rng(seed) + null_records: Dict[str, List[float]] = {k: [] for k in observed} + + for _ in range(n_permutations): + permuted = rng.permutation(labels) + m = compute_structure_metrics(embedding, permuted, knn_k) + for k, v in m.items(): + null_records[k].append(v) + + summary: Dict[str, Dict[str, float]] = {} + for k, obs_v in observed.items(): + arr = np.asarray(null_records[k], dtype=float) + valid = arr[~np.isnan(arr)] + if valid.size == 0 or np.isnan(obs_v): + summary[k] = {"observed": obs_v, "null_mean": float("nan"), + "null_std": float("nan"), "p_right_tail": float("nan")} + continue + p = float((1 + np.sum(valid >= obs_v)) / (valid.size + 1)) + summary[k] = { + "observed": float(obs_v), + "null_mean": float(valid.mean()), + "null_std": float(valid.std()), + "p_right_tail": p, + } + return summary + + +# ---------- visualization ---------- + +def _color_map(labels: List[str]) -> Dict[str, tuple]: + uniq = sorted(set(labels)) + cmap = plt.get_cmap("tab20", max(len(uniq), 1)) + return {lab: cmap(i) for i, lab in enumerate(uniq)} + + +def _scatter(coords: np.ndarray, labels: List[str], title: str, out_path: Path) -> None: + fig, ax = plt.subplots(figsize=(7, 6)) + cmap = _color_map(labels) + for lab in sorted(set(labels)): + mask = np.asarray([lab == l for l in labels]) + ax.scatter(coords[mask, 0], coords[mask, 1], + color=cmap[lab], label=f"{lab} (n={mask.sum()})", + s=20, alpha=0.85, edgecolor="none") + ax.set_title(title) + ax.set_xlabel("dim 1") + ax.set_ylabel("dim 2") + ax.legend(loc="best", fontsize=8, frameon=True) + fig.tight_layout() + fig.savefig(out_path, dpi=160) + plt.close(fig) + + +def visualize( + embedding: np.ndarray, + metadata_df: pd.DataFrame, + output_dir: Path, + umap_n_neighbors: int, + umap_min_dist: float, +) -> None: + # PCA 2D + pca = PCA(n_components=2, random_state=0) + pca_coords = pca.fit_transform(embedding) + _scatter(pca_coords, metadata_df["family"].tolist(), + "Module embedding PCA — family", output_dir / "pca_family.png") + + has_hallmark = metadata_df["hallmark_process"].notna().any() + if has_hallmark: + # Project only hallmark modules to a separate PCA for readability + hm_mask = metadata_df["hallmark_process"].notna().to_numpy() + if hm_mask.sum() >= 3: + hm_pca = PCA(n_components=2, random_state=0).fit_transform(embedding[hm_mask]) + _scatter(hm_pca, metadata_df.loc[hm_mask, "hallmark_process"].tolist(), + "Hallmark modules PCA — process", + output_dir / "pca_hallmark_process.png") + + # UMAP (optional) + try: + import umap # type: ignore + reducer = umap.UMAP( + n_components=2, + n_neighbors=min(umap_n_neighbors, max(2, embedding.shape[0] - 1)), + min_dist=umap_min_dist, + metric="cosine", + random_state=0, + ) + umap_coords = reducer.fit_transform(embedding) + _scatter(umap_coords, metadata_df["family"].tolist(), + "Module embedding UMAP — family", output_dir / "umap_family.png") + + if has_hallmark and hm_mask.sum() >= 3: + hm_reducer = umap.UMAP( + n_components=2, + n_neighbors=min(umap_n_neighbors, max(2, hm_mask.sum() - 1)), + min_dist=umap_min_dist, + metric="cosine", + random_state=0, + ) + hm_umap = hm_reducer.fit_transform(embedding[hm_mask]) + _scatter(hm_umap, metadata_df.loc[hm_mask, "hallmark_process"].tolist(), + "Hallmark modules UMAP — process", + output_dir / "umap_hallmark_process.png") + except ImportError: + logger.warning("umap-learn not installed; skipping UMAP plots. `pip install umap-learn` to enable.") + + +# ---------- driver ---------- + +def setup_logging(output_dir: Path) -> None: + log_path = output_dir / "run.log" + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s | %(levelname)s | %(name)s | %(message)s", + handlers=[logging.StreamHandler(sys.stdout), logging.FileHandler(log_path, mode="w")], + force=True, + ) + + +def resolve_module_json(args: argparse.Namespace) -> str: + """Resolve relative paths against performer_module/cell_annotation/ to stay + consistent with how `load_backbone` reads `module_data_dir`. + """ + if args.module_json is not None: + return str(ca_resolve_path(args.module_json)) + return str(ca_resolve_path(args.module_data_dir) / "merged_modules.json") + + +def main() -> None: + args = parse_args() + + # DDP init (auto-detects torchrun env; falls back to single-process) + device, use_ddp, rank, world_size, local_rank = init_distributed_mode(args.device) + seed_all(args.seed + rank) + + if is_main_process(): + output_dir = ensure_dir(args.output_dir) + setup_logging(output_dir) + else: + output_dir = Path(args.output_dir) + logging.basicConfig(level=logging.WARNING) + if use_ddp: + dist.barrier() + + logger.info("Args: %s", vars(args)) + logger.info("DDP: use_ddp=%s, rank=%d/%d, device=%s", use_ddp, rank, world_size, device) + + # 1. Load backbone + backbone, module_data, model_config = load_backbone(args, device) + backbone.eval() + base_dim = int(model_config["base_dim"]) + num_modules = len(module_data) + logger.info("Loaded backbone: num_modules=%d, base_dim=%d", num_modules, base_dim) + + # 2. Load module names (must align with module_data order) + module_json_path = resolve_module_json(args) + module_names = load_module_names(module_json_path) + if len(module_names) != num_modules: + raise RuntimeError( + f"Module name count ({len(module_names)} from {module_json_path}) does not match " + f"backbone num_modules ({num_modules}). The merged_modules.json used to build the " + "model and the one given here must be identical, with no modules dropped during " + "gene-symbol resolution." + ) + metadata = build_module_metadata(module_names) + metadata_df = pd.DataFrame(metadata) + if is_main_process(): + logger.info("Family distribution:\n%s", metadata_df["family"].value_counts().to_string()) + + # 3. Build dataloader + if args.data_mode == "h5ad": + preprocessor = DataPreprocessor( + gene2ens_file=args.gene2ens_file, + gene2ens_column=args.gene2ens_column, + model_genes_file=args.gene_list_path, + ) + preprocessor.load_gene_mapping() + preprocessor.load_model_genes() + expected_input_genes = -1 + else: + preprocessor = None + expected_input_genes = int(model_config["num_genes"]) + + loader, reindex_state, n_cells = build_inference_loader( + args, preprocessor, use_ddp, rank, world_size, + ) + if n_cells == 0: + raise RuntimeError("No cells survived preprocessing.") + + # 4. Inference + logger.info("Extracting cell-averaged module embeddings over %d cells...", n_cells) + mean_emb, processed = extract_cell_averaged_module_embeddings( + backbone, loader, preprocessor, reindex_state, device, + num_modules=num_modules, base_dim=base_dim, + expected_input_genes=expected_input_genes, + use_ddp=use_ddp, + ) + logger.info("Embedding ready: shape=%s, processed_cells=%d", mean_emb.shape, processed) + + # --- Everything below is rank-0 only --- + if is_main_process(): + # 5. Save raw outputs + np.save(output_dir / "module_embeddings.npy", mean_emb) + metadata_df.to_csv(output_dir / "module_metadata.csv", index=False) + + # 6. Metrics — family level + if args.include_other_in_metrics: + keep_mask = np.ones(len(metadata_df), dtype=bool) + else: + keep_mask = (metadata_df["family"] != "Other").to_numpy() + family_labels = metadata_df.loc[keep_mask, "family"].tolist() + family_emb = mean_emb[keep_mask] + + logger.info("Computing structure metrics — family level (n=%d)", keep_mask.sum()) + family_summary = permutation_null( + family_emb, _label_to_int(family_labels), + knn_k=args.knn_k, + n_permutations=args.null_permutations, + seed=args.seed, + ) + + # 6b. Metrics — hallmark process level + hm_mask = metadata_df["hallmark_process"].notna().to_numpy() + hallmark_summary: Optional[Dict[str, Dict[str, float]]] = None + if hm_mask.sum() >= 3 and metadata_df.loc[hm_mask, "hallmark_process"].nunique() >= 2: + logger.info("Computing structure metrics — hallmark process level (n=%d)", hm_mask.sum()) + hallmark_summary = permutation_null( + mean_emb[hm_mask], + _label_to_int(metadata_df.loc[hm_mask, "hallmark_process"].tolist()), + knn_k=args.knn_k, + n_permutations=args.null_permutations, + seed=args.seed, + ) + + metrics_payload = { + "n_cells_processed": int(processed), + "num_modules": int(num_modules), + "base_dim": int(base_dim), + "knn_k": int(args.knn_k), + "null_permutations": int(args.null_permutations), + "family_metrics": family_summary, + "hallmark_process_metrics": hallmark_summary, + "family_counts": metadata_df["family"].value_counts().to_dict(), + "hallmark_process_counts": metadata_df["hallmark_process"].dropna().value_counts().to_dict(), + "shuffle_module_genes": bool(args.shuffle_module_genes), + } + with open(output_dir / "metrics.json", "w", encoding="utf-8") as fh: + json.dump(metrics_payload, fh, indent=2) + + logger.info("=== Family-level structure (vs random-shuffled label null) ===") + for k, v in family_summary.items(): + logger.info(" %-30s observed=%.4f null=%.4f±%.4f p=%.4f", + k, v["observed"], v["null_mean"], v["null_std"], v["p_right_tail"]) + if hallmark_summary is not None: + logger.info("=== Hallmark-process-level structure (Hallmarks subset) ===") + for k, v in hallmark_summary.items(): + logger.info(" %-30s observed=%.4f null=%.4f±%.4f p=%.4f", + k, v["observed"], v["null_mean"], v["null_std"], v["p_right_tail"]) + + # 7. Visualisation + logger.info("Plotting UMAP / PCA...") + visualize(mean_emb, metadata_df, output_dir, + umap_n_neighbors=args.umap_n_neighbors, + umap_min_dist=args.umap_min_dist) + + logger.info("Done. Outputs saved to %s", output_dir) + + # Teardown + if use_ddp: + dist.barrier() + dist.destroy_process_group() + + +if __name__ == "__main__": + main() diff --git a/BioMod/interpretability/module_importance/README.md b/BioMod/interpretability/module_importance/README.md index 2aac60044d1b8e3d095e7964f7165ca13291eebe..e6a85aa260a1ce4ed5fbdaa6e9f167c3b0466a1e 100644 --- a/BioMod/interpretability/module_importance/README.md +++ b/BioMod/interpretability/module_importance/README.md @@ -1,311 +1,319 @@ -# Experiment 3: Module Importance and Enrichment Analysis - -This experiment tests whether modules that a finetuned model considers -important are consistent with known cell-type marker genes. - -## Method - -1. Load a finetuned cell annotation model: backbone, attention-pooling head, and classifier. -2. Mask one module at a time on the test set and measure the drop in the correct-class logit (`DeltaLogit`). -3. Aggregate by cell type to obtain an importance matrix with shape `(num_classes x 660)`. -4. For the top-k modules of each cell type, run hypergeometric enrichment on their genes. - -## Masking Modes - -| Mode | Principle | Speed | -| --- | --- | --- | -| `post_encoder` | Zero out the module token after encoder output | Fast, because the encoder runs once | -| `pre_encoder` | Zero out the genes corresponding to the module before encoder input | Slow, because it requires 660 x cells forward passes | - -`pre_encoder` is more accurate because it captures interactions between modules, -but it is much slower. Use `--importance_cells` for subsampling control. - -## Run - -```bash -cd ../data/250010166/scLong/performer_module/interpretability/module_importance -torchrun --nproc_per_node=4 -python module_importance.py \ - --finetune_checkpoint mae_tost_48M_cross_attn_only_pure_memory/attn_pool_full_n1_lrh0.0001_lrb0.0001_seed2021/best_model.pt \ - --config_pkl ../../mae_tost_48M_cross_attn_only_pure_memory/mae_tost_48M_cross_attn_only_pure_memory.pkl \ - --module_data_dir ../../module_data_all \ - --gene_list_path ../../../selected_genes_27k.txt \ - --gene_mapping_path ../../../gene2ens.csv \ - --gene2vec_data_path ../../../selected_gene2vec_27k.npy \ - --gene2ens_file ../../../gene2ens.csv \ - --data_path ../../../cell_annotation/Zheng68K.h5ad \ - --label_column celltype \ - --mask_mode post_encoder \ - --batch_size 128 \ - --top_k 10 20 \ - --output_dir ../results/importance/zheng68k_post -``` - -Pre-encoder mode, more accurate but slower: - -```bash -python module_importance.py \ - --finetune_checkpoint mae_tost_48M_cross_attn_only_pure_memory/attn_pool_full_n1_lrh0.0001_lrb0.0001_seed2021/best_model.pt \ - --config_pkl ../../mae_tost_48M_cross_attn_only_pure_memory/mae_tost_48M_cross_attn_only_pure_memory.pkl \ - --module_data_dir ../../module_data_all \ - --gene_list_path ../../../selected_genes_27k.txt \ - --gene_mapping_path ../../../gene2ens.csv \ - --gene2vec_data_path ../../../selected_gene2vec_27k.npy \ - --gene2ens_file ../../../gene2ens.csv \ - --data_path ../../../cell_annotation/Zheng68K.h5ad \ - --label_column celltype \ - --mask_mode pre_encoder \ - --importance_cells 500 \ - --batch_size 256 \ - --top_k 10 20 \ - --output_dir ../results/importance/zheng68k_pre -``` - -Pre-encoder mode on all cells with DDP multi-GPU acceleration: - -```bash -torchrun --nproc_per_node=4 module_importance.py \ - --finetune_checkpoint mae_tost_48M_cross_attn_only_pure_memory/attn_pool_full_n1_lrh0.0001_lrb0.0001_seed2021/best_model.pt \ - --config_pkl ../../mae_tost_48M_cross_attn_only_pure_memory/mae_tost_48M_cross_attn_only_pure_memory.pkl \ - --module_data_dir ../../module_data_all \ - --gene_list_path ../../../selected_genes_27k.txt \ - --gene_mapping_path ../../../gene2ens.csv \ - --gene2vec_data_path ../../../selected_gene2vec_27k.npy \ - --gene2ens_file ../../../gene2ens.csv \ - --data_path ../../../cell_annotation/Zheng68K.h5ad \ - --label_column celltype \ - --mask_mode pre_encoder \ - --batch_size 512 \ - --top_k 10 20 \ - --output_dir ../results/importance/zheng68k_pre_full -``` - -## Arguments - -| Argument | Default | Description | -| --- | --- | --- | -| `--mask_mode` | post_encoder | Masking strategy | -| `--importance_cells` | 0 | Maximum number of cells for `pre_encoder`; 0 means all cells | -| `--top_k` | 10 20 | Top-k modules used for enrichment analysis | -| `--batch_size` | 64 | Inference batch size | -| `--test_ratio` | 0.2 | Test-set fraction; should match finetuning | -| `--seed` | 2021 | Random seed; should match finetuning | -| `--marker_db_dir` | None | Marker database cache directory | - -## Outputs - -```text -output_dir/ -|-- importance_matrix.npy # (11, 660) DeltaLogit matrix -|-- importance_matrix.csv # Same matrix with labels -|-- top_modules_per_celltype.json # Top modules and genes for each cell type -|-- enrichment_results.csv # Hypergeometric enrichment results -|-- heatmap_importance.png # Heatmap -|-- top_modules_bar.png # Bar plot of top modules per cell type -|-- metrics.json # Summary metrics -`-- marker_cache/ # PanglaoDB cache - `-- PanglaoDB_markers.tsv -``` - -## Expected Conclusion - -If the model learns meaningful module importance: - -- Top modules for immune cell types should be enriched for immune-related pathways. -- Enrichment p-values should be significant after Bonferroni correction, below 0.05. -- Marker genes for the corresponding cell types in PanglaoDB should appear in top modules. - ---- - -## Extended Experiments - -### 3-A: Masking Ablation Curves - -Progressively mask top, bottom, or random modules and measure the accuracy drop. - -```bash -python masking_ablation.py \ - --finetune_checkpoint mae_tost_48M_cross_attn_only_pure_memory/attn_pool_full_n1_lrh0.0001_lrb0.0001_seed2021/best_model.pt \ - --config_pkl ../../mae_tost_48M_cross_attn_only_pure_memory/mae_tost_48M_cross_attn_only_pure_memory.pkl \ - --module_data_dir ../../module_data_all \ - --gene_list_path ../../../selected_genes_27k.txt \ - --gene_mapping_path ../../../gene2ens.csv \ - --gene2vec_data_path ../../../selected_gene2vec_27k.npy \ - --gene2ens_file ../../../gene2ens.csv \ - --data_path ../../../cell_annotation/Zheng68K.h5ad \ - --label_column celltype \ - --importance_matrix ../results/importance/zheng68k_pre_13690/importance_matrix.npy \ - --k_values 1 2 5 10 20 50 100 200 \ - --n_random_trials 10 \ - --batch_size 360 \ - --save_deltas \ - --output_dir ../results/importance/ablation/ -``` - -### 3-B: Permutation Tests - -```bash -# Mode A: shuffle modules, GPU and slow. -python random_baseline.py \ - --mode shuffle_modules \ - --finetune_checkpoint mae_tost_48M_cross_attn_only_pure_memory/attn_pool_full_n1_lrh0.0001_lrb0.0001_seed2021/best_model.pt \ - --config_pkl ../../mae_tost_48M_cross_attn_only_pure_memory/mae_tost_48M_cross_attn_only_pure_memory.pkl \ - --module_data_dir ../../module_data_all \ - --gene_list_path ../../../selected_genes_27k.txt \ - --gene_mapping_path ../../../gene2ens.csv \ - --gene2vec_data_path ../../../selected_gene2vec_27k.npy \ - --gene2ens_file ../../../gene2ens.csv \ - --data_path ../../../cell_annotation/Zheng68K.h5ad \ - --label_column celltype \ - --importance_matrix ../results/importance/zheng68k_pre_13690/importance_matrix.npy\ - --n_permutations 10 \ - --output_dir ../results/importance/permutation/ - -# Mode B: shuffle labels, CPU and fast, requires per_cell_deltas from ablation. -python random_baseline.py \ - --mode shuffle_labels \ - --importance_matrix ../results/importance/zheng68k_pre_13690/importance_matrix.npy \ - --per_cell_deltas ../results/importance/ablation/per_cell_deltas.npy \ - --per_cell_labels ../results/importance/ablation/per_cell_labels.npy \ - --n_permutations 1000 \ - --output_dir ../results/importance/permutation/ -``` - -### 3-C: Marker Gene Verification, CPU Only - -```bash -python marker_gene_verification.py \ - --importance_matrix ../results/importance/zheng68k_pre_13690/importance_matrix.npy \ - --importance_csv \ - --module_data_dir ../../module_data_all \ - --output_dir ../results/importance/marker_verification/ -``` - -### Publication Figures - -```bash -python module_importance_publication.py \ - --results_dir ../results/importance/zheng68k_pre_13690/ \ - --ablation_dir ../results/importance/ablation \ - --permutation_dir ../results/importance/permutation/ \ - --module_data_dir ../../module_data_all \ - --output_dir ../results/importance/publication_figures/ -``` - -### 3-E: Family-Level Analysis, CPU Only - -Analyze importance distributions, contribution fractions, and enrichment tests by -module source family: Hallmarks, Reactome, Hotspot, and WGCNA. - -```bash -python family_importance_analysis.py \ - --results_dir ../results/importance/zheng68k_pre \ - --output_dir ../results/importance/family_analysis \ - --top_k 10 20 50 \ - --enrichment_k 50 -``` - -Outputs: - -- `fig1_family_importance_distribution.png/pdf`: family-wise importance distribution for each cell type, violin + box. -- `fig2_family_contribution_topk.png/pdf`: family fraction among top-K important modules, stacked bar. -- `fig3_family_enrichment_heatmap.png/pdf`: Fisher exact-test enrichment-ratio heatmap. -- `fig4_family_mean_importance.png/pdf`: family mean-importance comparison, grouped bar. -- `kruskal_wallis_family_test.csv`: Kruskal-Wallis test results. -- `pairwise_mannwhitney_families.csv`: pairwise Mann-Whitney U tests between families. -- `family_enrichment_fisher.csv`: detailed Fisher exact-test results with Bonferroni correction. -- `family_contribution_topk.csv`: numerical family contribution data. -- `family_mean_importance.csv`: family mean-importance matrix. - -### 3-F: Family-Level Ablation, GPU - -Ablate whole families by removing or keeping one module family and measuring -classification-accuracy changes. - -```bash -python family_ablation.py \ - --finetune_checkpoint mae_tost_48M_cross_attn_only_pure_memory/attn_pool_full_n1_lrh0.0001_lrb0.0001_seed2021/best_model.pt \ - --config_pkl ../../mae_tost_48M_cross_attn_only_pure_memory/mae_tost_48M_cross_attn_only_pure_memory.pkl \ - --module_data_dir ../../module_data_all \ - --gene_list_path ../../../selected_genes_27k.txt \ - --gene_mapping_path ../../../gene2ens.csv \ - --gene2vec_data_path ../../../selected_gene2vec_27k.npy \ - --gene2ens_file ../../../gene2ens.csv \ - --data_path ../../../cell_annotation/Zheng68K.h5ad \ - --label_column celltype \ - --batch_size 256 \ - --output_dir ../results/importance/family_analysis_gpu -``` - -Outputs: - -- `family_ablation_results.json`: overall and per-class accuracy under all conditions. -- `fig_family_ablation.png/pdf`: two-panel overall plot for remove-one vs keep-one. -- `fig_family_ablation_per_class.png/pdf`: two-panel heatmap of per-cell-type ablation results. - -### 3-G: Cross-Class Similarity, CPU Only - -Analyze correlations and similarity between important modules across the 11 cell -classes. - -```bash -python cross_class_similarity.py \ - --results_dir ../results/importance/zheng68k_pre \ - --output_dir ../results/importance/publication_figures \ - --top_k 50 -``` - -### 3-H: 1B Module Gating + 2B Family Embedding Finetuning Experiment - -Based on module-importance findings, most modules have near-zero importance and -different cell types depend on different families. During finetuning, add two -lightweight modules in a LoRA-style setup: pretrained weights are frozen and new -parameters are trained from zero initialization. - -- **1B, Sparse Module Gate**: `LayerNorm -> Linear(E,1) -> Sigmoid`, learning a 0-1 gate scalar for each module. -- **2B, Family Embedding**: `nn.Parameter(4, E)`, one learnable vector per family, added to `pos_emb`. - -```bash -# Baseline, no new modules. -python performer_module_finetune.py \ - --config_pkl ../../mae_tost_48M_cross_attn_only_pure_memory/mae_tost_48M_cross_attn_only_pure_memory.pkl \ - --performer_checkpoint ../../mae_tost_48M_cross_attn_only_pure_memory/mae_tost_48M_cross_attn_only_pure_memory_epoch6.pth \ - --module_data_dir ../../module_data_all \ - --gene_list_path ../../../selected_genes_27k.txt \ - --gene_mapping_path ../../../gene2ens.csv \ - --gene2vec_data_path ../../../selected_gene2vec_27k.npy \ - --gene2ens_file ../../../gene2ens.csv \ - --data_path ../../../cell_annotation/Zheng68K.h5ad \ - --label_column celltype \ - --head_type attn_pool \ - --finetune_mode last_n --unfreeze_last_n 1 \ - --lr_head 0.0001 --lr_backbone 0.0001 \ - --batch_size 256 --epochs 300 \ - --run_name baseline_zheng68k \ - --output_dir ../results/family_exp -``` - -The same command can be extended with: - -- `--use_family_emb` to enable family-type embedding, adding `4 x 200 = 800` parameters. -- `--use_module_gate` to enable sparse module gating, adding LayerNorm + Linear, 601 parameters. -- Both flags can be used together. - -The new module parameters use `lr_head`, a higher learning rate, and are -independent of frozen backbone parameters. - -## File Descriptions - -| File | Description | -| --- | --- | -| `module_importance.py` | Core importance computation with DDP support | -| `masking_ablation.py` | Ablation-curve experiment, GPU | -| `random_baseline.py` | Permutation tests, GPU/CPU | -| `marker_gene_verification.py` | Marker verification, CPU | -| `module_importance_publication.py` | Publication figures | -| `family_importance_analysis.py` | Family-level importance analysis, CPU | -| `family_ablation.py` | Family-level ablation, GPU | -| `cross_class_similarity.py` | Cross-class similarity analysis, CPU | -| `pub_utils.py` | Shared utilities | -| `RESULTS.md` | Result documentation with figure placeholders | -| `TODO.md` | Progress tracker | +# Experiment 3: Module Importance and Enrichment Analysis + +This experiment tests whether modules that a finetuned model considers +important are consistent with known cell-type marker genes. + +## Method + +1. Load a finetuned cell annotation model: backbone, attention-pooling head, and classifier. +2. Mask one module at a time on the test set and measure the drop in the correct-class logit (`DeltaLogit`). +3. Aggregate by cell type to obtain an importance matrix with shape `(num_classes x 660)`. +4. For the top-k modules of each cell type, run hypergeometric enrichment on their genes. + +## Masking Modes + +| Mode | Principle | Speed | +| --- | --- | --- | +| `post_encoder` | Zero out the module token after encoder output | Fast, because the encoder runs once | +| `pre_encoder` | Zero out the genes corresponding to the module before encoder input | Slow, because it requires 660 x cells forward passes | + +`pre_encoder` is more accurate because it captures interactions between modules, +but it is much slower. Use `--importance_cells` for subsampling control. + +## Run + +Run these commands from the BioMod repository root. First train a matching +cell-annotation model; the example checkpoint path is the output of +`BioMod/toturials/cell_annotation.ipynb`, not a bundled pretrained weight file. +Use the matching BioMod configuration, classifier architecture, preprocessing, +label mapping, and test split for that trained model. Substitute your own +`--finetune_checkpoint` if its output directory differs. Importance matrices, +ablation outputs, and permutation results below must be generated by the +preceding steps; they are not bundled inputs. + +```bash +# Run from the BioMod repository root. +python BioMod/interpretability/module_importance/module_importance.py \ + --finetune_checkpoint tutorial_runs/cell_annotation_zheng68k_biomod/best_model.pt \ + --config_pkl weight/0810model/mae_tost_70M_75mask_ec2.pkl \ + --module_data_dir model_dependencies/module_data_all \ + --gene_list_path model_dependencies/selected_genes_27k.txt \ + --gene_mapping_path model_dependencies/gene2ens.csv \ + --gene2vec_data_path model_dependencies/selected_gene2vec_27k.npy \ + --gene2ens_file model_dependencies/gene2ens.csv \ + --data_path data/downstream/final_data/Zheng68K.h5ad \ + --label_column celltype \ + --mask_mode post_encoder \ + --batch_size 128 \ + --top_k 10 20 \ + --output_dir outputs/interpretability/importance/zheng68k_post +``` + +Pre-encoder mode, more accurate but slower: + +```bash +python BioMod/interpretability/module_importance/module_importance.py \ + --finetune_checkpoint tutorial_runs/cell_annotation_zheng68k_biomod/best_model.pt \ + --config_pkl weight/0810model/mae_tost_70M_75mask_ec2.pkl \ + --module_data_dir model_dependencies/module_data_all \ + --gene_list_path model_dependencies/selected_genes_27k.txt \ + --gene_mapping_path model_dependencies/gene2ens.csv \ + --gene2vec_data_path model_dependencies/selected_gene2vec_27k.npy \ + --gene2ens_file model_dependencies/gene2ens.csv \ + --data_path data/downstream/final_data/Zheng68K.h5ad \ + --label_column celltype \ + --mask_mode pre_encoder \ + --importance_cells 500 \ + --batch_size 256 \ + --top_k 10 20 \ + --output_dir outputs/interpretability/importance/zheng68k_pre +``` + +Pre-encoder mode on all cells with DDP multi-GPU acceleration: + +```bash +torchrun --nproc_per_node=4 BioMod/interpretability/module_importance/module_importance.py \ + --finetune_checkpoint tutorial_runs/cell_annotation_zheng68k_biomod/best_model.pt \ + --config_pkl weight/0810model/mae_tost_70M_75mask_ec2.pkl \ + --module_data_dir model_dependencies/module_data_all \ + --gene_list_path model_dependencies/selected_genes_27k.txt \ + --gene_mapping_path model_dependencies/gene2ens.csv \ + --gene2vec_data_path model_dependencies/selected_gene2vec_27k.npy \ + --gene2ens_file model_dependencies/gene2ens.csv \ + --data_path data/downstream/final_data/Zheng68K.h5ad \ + --label_column celltype \ + --mask_mode pre_encoder \ + --batch_size 512 \ + --top_k 10 20 \ + --output_dir outputs/interpretability/importance/zheng68k_pre_full +``` + +## Arguments + +| Argument | Default | Description | +| --- | --- | --- | +| `--mask_mode` | post_encoder | Masking strategy | +| `--importance_cells` | 0 | Maximum number of cells for `pre_encoder`; 0 means all cells | +| `--top_k` | 10 20 | Top-k modules used for enrichment analysis | +| `--batch_size` | 64 | Inference batch size | +| `--test_ratio` | 0.2 | Test-set fraction; should match finetuning | +| `--seed` | 2021 | Random seed; should match finetuning | +| `--marker_db_dir` | None | Marker database cache directory | + +## Outputs + +```text +output_dir/ +|-- importance_matrix.npy # (11, 660) DeltaLogit matrix +|-- importance_matrix.csv # Same matrix with labels +|-- top_modules_per_celltype.json # Top modules and genes for each cell type +|-- enrichment_results.csv # Hypergeometric enrichment results +|-- heatmap_importance.png # Heatmap +|-- top_modules_bar.png # Bar plot of top modules per cell type +|-- metrics.json # Summary metrics +`-- marker_cache/ # PanglaoDB cache + `-- PanglaoDB_markers.tsv +``` + +## Expected Conclusion + +If the model learns meaningful module importance: + +- Top modules for immune cell types should be enriched for immune-related pathways. +- Enrichment p-values should be significant after Bonferroni correction, below 0.05. +- Marker genes for the corresponding cell types in PanglaoDB should appear in top modules. + +--- + +## Extended Experiments + +### 3-A: Masking Ablation Curves + +Progressively mask top, bottom, or random modules and measure the accuracy drop. + +```bash +python BioMod/interpretability/module_importance/masking_ablation.py \ + --finetune_checkpoint tutorial_runs/cell_annotation_zheng68k_biomod/best_model.pt \ + --config_pkl weight/0810model/mae_tost_70M_75mask_ec2.pkl \ + --module_data_dir model_dependencies/module_data_all \ + --gene_list_path model_dependencies/selected_genes_27k.txt \ + --gene_mapping_path model_dependencies/gene2ens.csv \ + --gene2vec_data_path model_dependencies/selected_gene2vec_27k.npy \ + --gene2ens_file model_dependencies/gene2ens.csv \ + --data_path data/downstream/final_data/Zheng68K.h5ad \ + --label_column celltype \ + --importance_matrix outputs/interpretability/importance/zheng68k_pre_13690/importance_matrix.npy \ + --k_values 1 2 5 10 20 50 100 200 \ + --n_random_trials 10 \ + --batch_size 360 \ + --save_deltas \ + --output_dir outputs/interpretability/importance/ablation/ +``` + +### 3-B: Permutation Tests + +```bash +# Mode A: shuffle modules, GPU and slow. +python BioMod/interpretability/module_importance/random_baseline.py \ + --mode shuffle_modules \ + --finetune_checkpoint tutorial_runs/cell_annotation_zheng68k_biomod/best_model.pt \ + --config_pkl weight/0810model/mae_tost_70M_75mask_ec2.pkl \ + --module_data_dir model_dependencies/module_data_all \ + --gene_list_path model_dependencies/selected_genes_27k.txt \ + --gene_mapping_path model_dependencies/gene2ens.csv \ + --gene2vec_data_path model_dependencies/selected_gene2vec_27k.npy \ + --gene2ens_file model_dependencies/gene2ens.csv \ + --data_path data/downstream/final_data/Zheng68K.h5ad \ + --label_column celltype \ + --importance_matrix outputs/interpretability/importance/zheng68k_pre_13690/importance_matrix.npy\ + --n_permutations 10 \ + --output_dir outputs/interpretability/importance/permutation/ + +# Mode B: shuffle labels, CPU and fast, requires per_cell_deltas from ablation. +python BioMod/interpretability/module_importance/random_baseline.py \ + --mode shuffle_labels \ + --importance_matrix outputs/interpretability/importance/zheng68k_pre_13690/importance_matrix.npy \ + --per_cell_deltas outputs/interpretability/importance/ablation/per_cell_deltas.npy \ + --per_cell_labels outputs/interpretability/importance/ablation/per_cell_labels.npy \ + --n_permutations 1000 \ + --output_dir outputs/interpretability/importance/permutation/ +``` + +### 3-C: Marker Gene Verification, CPU Only + +```bash +python BioMod/interpretability/module_importance/marker_gene_verification.py \ + --importance_matrix outputs/interpretability/importance/zheng68k_pre_13690/importance_matrix.npy \ + --importance_csv \ + --module_data_dir model_dependencies/module_data_all \ + --output_dir outputs/interpretability/importance/marker_verification/ +``` + +### Publication Figures + +```bash +python BioMod/interpretability/module_importance/module_importance_publication.py \ + --results_dir outputs/interpretability/importance/zheng68k_pre_13690/ \ + --ablation_dir outputs/interpretability/importance/ablation \ + --permutation_dir outputs/interpretability/importance/permutation/ \ + --module_data_dir model_dependencies/module_data_all \ + --output_dir outputs/interpretability/importance/publication_figures/ +``` + +### 3-E: Family-Level Analysis, CPU Only + +Analyze importance distributions, contribution fractions, and enrichment tests by +module source family: Hallmarks, Reactome, Hotspot, and WGCNA. + +```bash +python BioMod/interpretability/module_importance/family_importance_analysis.py \ + --results_dir outputs/interpretability/importance/zheng68k_pre \ + --output_dir outputs/interpretability/importance/family_analysis \ + --top_k 10 20 50 \ + --enrichment_k 50 +``` + +Outputs: + +- `fig1_family_importance_distribution.png/pdf`: family-wise importance distribution for each cell type, violin + box. +- `fig2_family_contribution_topk.png/pdf`: family fraction among top-K important modules, stacked bar. +- `fig3_family_enrichment_heatmap.png/pdf`: Fisher exact-test enrichment-ratio heatmap. +- `fig4_family_mean_importance.png/pdf`: family mean-importance comparison, grouped bar. +- `kruskal_wallis_family_test.csv`: Kruskal-Wallis test results. +- `pairwise_mannwhitney_families.csv`: pairwise Mann-Whitney U tests between families. +- `family_enrichment_fisher.csv`: detailed Fisher exact-test results with Bonferroni correction. +- `family_contribution_topk.csv`: numerical family contribution data. +- `family_mean_importance.csv`: family mean-importance matrix. + +### 3-F: Family-Level Ablation, GPU + +Ablate whole families by removing or keeping one module family and measuring +classification-accuracy changes. + +```bash +python BioMod/interpretability/module_importance/family_ablation.py \ + --finetune_checkpoint tutorial_runs/cell_annotation_zheng68k_biomod/best_model.pt \ + --config_pkl weight/0810model/mae_tost_70M_75mask_ec2.pkl \ + --module_data_dir model_dependencies/module_data_all \ + --gene_list_path model_dependencies/selected_genes_27k.txt \ + --gene_mapping_path model_dependencies/gene2ens.csv \ + --gene2vec_data_path model_dependencies/selected_gene2vec_27k.npy \ + --gene2ens_file model_dependencies/gene2ens.csv \ + --data_path data/downstream/final_data/Zheng68K.h5ad \ + --label_column celltype \ + --batch_size 256 \ + --output_dir outputs/interpretability/importance/family_analysis_gpu +``` + +Outputs: + +- `family_ablation_results.json`: overall and per-class accuracy under all conditions. +- `fig_family_ablation.png/pdf`: two-panel overall plot for remove-one vs keep-one. +- `fig_family_ablation_per_class.png/pdf`: two-panel heatmap of per-cell-type ablation results. + +### 3-G: Cross-Class Similarity, CPU Only + +Analyze correlations and similarity between important modules across the 11 cell +classes. + +```bash +python BioMod/interpretability/module_importance/cross_class_similarity.py \ + --results_dir outputs/interpretability/importance/zheng68k_pre \ + --output_dir outputs/interpretability/importance/publication_figures \ + --top_k 50 +``` + +### 3-H: 1B Module Gating + 2B Family Embedding Finetuning Experiment + +Based on module-importance findings, most modules have near-zero importance and +different cell types depend on different families. During finetuning, add two +lightweight modules in a LoRA-style setup: pretrained weights are frozen and new +parameters are trained from zero initialization. + +- **1B, Sparse Module Gate**: `LayerNorm -> Linear(E,1) -> Sigmoid`, learning a 0-1 gate scalar for each module. +- **2B, Family Embedding**: `nn.Parameter(4, E)`, one learnable vector per family, added to `pos_emb`. + +```bash +# Baseline, no new modules. +python BioMod/cell_annotation/performer_module_finetune.py \ + --config_pkl weight/0810model/mae_tost_70M_75mask_ec2.pkl \ + --performer_checkpoint weight/0810model/mae_tost_70M_75mask_ec2_epoch0.pth \ + --module_data_dir model_dependencies/module_data_all \ + --gene_list_path model_dependencies/selected_genes_27k.txt \ + --gene_mapping_path model_dependencies/gene2ens.csv \ + --gene2vec_data_path model_dependencies/selected_gene2vec_27k.npy \ + --gene2ens_file model_dependencies/gene2ens.csv \ + --data_path data/downstream/final_data/Zheng68K.h5ad \ + --label_column celltype \ + --head_type attn_pool \ + --finetune_mode last_n --unfreeze_last_n 1 \ + --lr_head 0.0001 --lr_backbone 0.0001 \ + --batch_size 256 --epochs 300 \ + --run_name baseline_zheng68k \ + --output_dir outputs/interpretability/family_exp +``` + +The same command can be extended with: + +- `--use_family_emb` to enable family-type embedding, adding `4 x 200 = 800` parameters. +- `--use_module_gate` to enable sparse module gating, adding LayerNorm + Linear, 601 parameters. +- Both flags can be used together. + +The new module parameters use `lr_head`, a higher learning rate, and are +independent of frozen backbone parameters. + +## File Descriptions + +| File | Description | +| --- | --- | +| `module_importance.py` | Core importance computation with DDP support | +| `masking_ablation.py` | Ablation-curve experiment, GPU | +| `random_baseline.py` | Permutation tests, GPU/CPU | +| `marker_gene_verification.py` | Marker verification, CPU | +| `module_importance_publication.py` | Publication figures | +| `family_importance_analysis.py` | Family-level importance analysis, CPU | +| `family_ablation.py` | Family-level ablation, GPU | +| `cross_class_similarity.py` | Cross-class similarity analysis, CPU | +| `pub_utils.py` | Shared utilities | +| `RESULTS.md` | Result documentation with figure placeholders | +| `TODO.md` | Progress tracker | diff --git a/BioMod/interpretability/module_importance/RESULTS.md b/BioMod/interpretability/module_importance/RESULTS.md index c0d7c9496491e9c573b71820127b8d7b9228756b..7b21a71f60c89892e0f3784884f483914aa9f97b 100644 --- a/BioMod/interpretability/module_importance/RESULTS.md +++ b/BioMod/interpretability/module_importance/RESULTS.md @@ -1,221 +1,226 @@ -# Module Importance: Result Documentation - -> For the manuscript subsection "Model Interpretability: Module Importance Analysis". - -## Dataset - -- **Zheng68K**: 11 cell types, 68,450 cells, with 13,690 test cells. -- Model: PerformerModuleCellClassifier with attention-pooling head and 48M backbone. -- Baseline test accuracy: **83.1%**. - -## Data Sources - -All analyses are based on the following core files, generated by `module_importance.py` -with `pre_encoder` over all 13,690 test cells: - -| File | Path | Description | -| --- | --- | --- | -| `importance_matrix.npy` | `results/importance/zheng68k_pre_13690/importance_matrix.npy` | `(11 x 660)` DeltaLogit matrix | -| `importance_matrix.csv` | `results/importance/zheng68k_pre_13690/importance_matrix.csv` | Same matrix with row and column labels | -| `enrichment_results.csv` | `results/importance/zheng68k_pre_13690/enrichment_results.csv` | Enrichment-test results | -| `top_modules_per_celltype.json` | `results/importance/zheng68k_pre_13690/top_modules_per_celltype.json` | Top-module details | -| `merged_modules.json` | `module_data_all/merged_modules.json` | Gene composition of 660 modules | - -Marker verification uses `importance_matrix.npy`, `importance_matrix.csv`, and -`merged_modules.json`. - -Publication figures use all files above plus ablation and permutation results. - ---- - -## Experiment 3 Core: Module Importance Matrix - -### Method - -For each test cell, mask one module at a time using `pre_encoder`, which zeros out -the corresponding gene inputs, and measure the drop in the correct-class logit -as the importance score. Aggregate by cell type to obtain an `(11 x 660)` -importance matrix. - -### Results - -- Masking mode: **pre_encoder** over all 13,690 cells. -- 660 modules: 491 Reactome, 50 Hallmarks, 73 Hotspot, and 46 Processed. - -**Representative top modules per cell type:** - -| Cell Type | #1 Module | #2 Module | #3 Module | -| --- | --- | --- | --- | -| CD14+ Monocyte | Neutrophil Degranulation | TLR4 Cascade | MHC-II Antigen Presentation | -| CD19+ B | MHC-II | IFN-gamma Signaling | CD28 Co-stimulation | -| CD56+ NK | SM_22 (cytotoxic) | module3_2 | Allograft Rejection | -| CD8+ Cytotoxic T | module3_2 | SM_22 | Allograft Rejection | -| Dendritic | MHC-II | Signaling by Interleukins | IFN-gamma | - -### Figure - -![Importance Heatmap (clustered)](publication_figures/fig4_importance_heatmap_clustered.png) - ---- - -## Experiment 3-A: Masking Ablation Curves - -### Method - -Progressively mask top-k, bottom-k, or random-k modules, with -`k = 1, 2, 5, 10, 20, 50, 100, 200`, and measure classification accuracy. This -uses `post_encoder` masking so the encoder runs only once. - -### Expected Result - -- Masking top modules should rapidly decrease accuracy. -- Masking bottom modules should have little effect. -- Random masking should fall between the two. - -### Results, Completed - -- Baseline accuracy: **83.0%** on 13,690 test cells. -- Post-encoder masking with k = [1, 2, 5, 10, 20, 50, 100, 200]. - -| Masked k | Top Acc | Bottom Acc | Random Acc | -| --- | --- | --- | --- | -| 1 | 83.0% | 83.0% | 83.0% | -| 2 | 82.6% | 83.0% | 83.0% | -| 5 | 82.0% | 83.0% | 83.0% | -| 10 | 77.1% | 83.0% | 83.0% | -| 20 | 73.7% | 83.1% | 82.7% | -| 50 | 69.5% | 83.2% | 82.4% | -| 100 | 65.1% | 82.7% | 81.7% | -| 200 | 49.7% | 81.9% | 78.6% | - -**Interpretation:** masking the top 200 modules, 30% of all modules, reduces -accuracy from 83% to 50%, a 33 percentage-point drop. Masking the bottom 200 -modules has almost no effect, with only a 1.1 point drop. Random masking falls -between the two. This shows that the importance ranking captures modules that -are truly important for classification. - -![Ablation Curves](publication_figures/fig1_ablation_curves.png) - -![Per-class Ablation](publication_figures/fig_s1_ablation_per_class.png) - ---- - -## Experiment 3-B: Permutation Tests, Random Baseline - -### Method - -**Mode A, Module-Shuffle:** shuffle module token order to break the module-gene -correspondence, then recompute importance. Repeat 10 times. - -**Mode B, Label-Shuffle:** shuffle cell type labels and re-aggregate per-cell -deltas. Repeat 1000 times. - -### Expected Result - -- The true importance maximum should far exceed the null distribution. -- Empirical p-value should be below 0.001. - -### Results, Completed - -**Mode A, Module-Shuffle** (10 permutations, GPU): - -| Metric | Value | -| --- | --- | -| Global p-value | 0.091, limited by n=10; minimum possible is 1/11 | -| Effect size | **1474.5** | -| Real global max | 2.046 | -| Null global max | 0.452 +/- 0.001 | - -**Mode B, Label-Shuffle** (1000 permutations, CPU): - -| Metric | Value | -| --- | --- | -| Global p-value | **< 0.001** | -| Per-class p-values | All < 0.001 (11/11) | -| Effect size | **48.1** | -| Real global max | 2.046 | -| Null global max | 0.145 +/- 0.039 | - -**Interpretation:** - -- **Module-Shuffle:** after shuffling the module-gene correspondence, the maximum importance drops from 2.046 to about 0.452. The effect size is extremely large, showing that importance depends on the correct module-gene mapping rather than random noise. The p-value is 0.091 because only 10 permutations were run, making the minimum possible p-value about 1/11; in practice, 0 of 10 null runs exceeded the observed value. -- **Label-Shuffle:** after shuffling cell type labels, the maximum importance drops from 2.046 to about 0.145. All 11 cell types have p < 0.001, showing that importance is cell-type-specific rather than a generic pattern shared by all cells. - -![Permutation Null](publication_figures/fig2_permutation_null.png) - ---- - -## Experiment 3-C: Canonical Marker Gene Verification - -### Method - -Check the importance ranks of modules containing known cell-type canonical -markers such as CD14, CD19, CD8A, and NKG7. Test whether markers are enriched in -top modules. - -### Results, Completed - -| Metric | Value | -| --- | --- | -| Total markers tested | 59 | -| Markers not found | 2 | -| **Median rank percentile** | **0.9%** | -| Markers in top 5% | **47/59 (80%)** | -| Markers in top 10% | 47/59 (80%) | -| Markers in top 20% | 50/59 (85%) | -| Binomial p-value, top 10% | **< 1e-50** | -| Wilcoxon p-value, <50% | **2.5e-11** | - -**Per cell type:** - -| Cell Type | Median Percentile | Best Marker | Best Rank | -| --- | --- | --- | --- | -| CD14+ Monocyte | 0.0% | LYZ | #1 | -| CD19+ B | 1.3% | MS4A1 | #7 | -| CD34+ | 18.5% | THY1 | #7 | -| CD4+ T Helper2 | 1.7% | CD4 | #1 | -| CD4+/CD25 T Reg | 0.9% | CD4 | #1 | -| CD4+/CD45RA+/CD25- Naive T | 10.5% | CD4 | #1 | -| CD4+/CD45RO+ Memory | 0.0% | CD4 | #1 | -| CD56+ NK | 0.0% | NKG7 | #1 | -| CD8+ Cytotoxic T | 0.0% | CD8A | #1 | -| CD8+/CD45RA+ Naive Cytotoxic | 20.3% | CD8A | #1 | -| Dendritic | 2.1% | ITGAX | #1 | - -**Interpretation:** 80% of canonical markers are in the top 5% importance -modules for the corresponding cell type. In 9 of 11 cell types, the best marker -is in the rank #1 module. This strongly shows that the model learns biologically -meaningful module importance. - -![Marker Ranks](marker_verification/fig_marker_ranks.png) - -![Marker Summary Bar](marker_verification/fig_marker_summary_bar.png) - ---- - -## Experiment 3-D: PanglaoDB Enrichment Analysis - -### Method - -For genes in each cell type's top-k modules, run a hypergeometric enrichment test -against marker gene sets for 178 PanglaoDB cell types, with Bonferroni -correction. - -### Results - -| Metric | Value | -| --- | --- | -| Significant enrichment hits, p_adj < 0.05 | **1011 / 3916** | -| Hit rate | **25.8%** | - -![Enrichment Summary](publication_figures/fig5_enrichment_summary.png) - ---- - -## Manuscript Writing Points - -1. **Core conclusion:** the model learns biologically meaningful gene organization through the module-based architecture. Top-importance modules strongly match known cell-type markers, with 80% of markers in the top 5%, and masking these modules causes a large classification-performance drop, from 83% to 50%. -2. **Methodological contribution:** pre-encoder masking gives a more accurate importance estimate than post-encoder masking because it accounts for interactions between modules. DDP support makes all-cell computation feasible. -3. **Statistical rigor:** label-shuffle p < 0.001 with effect size 48.1 shows that importance is cell-type-specific; module-shuffle effect size 1474 shows that importance depends on the correct module-gene correspondence. Top-k masking rapidly reduces accuracy while bottom-k masking has almost no effect, and the gap widens as k increases. -4. **Biological interpretation:** monocytes use Neutrophil Degranulation and TLR4, representing innate immune pathways; B cells use MHC-II, IFN-gamma, and CD28, representing antigen presentation and T-B cooperation; NK/CD8 T share cytotoxic modules; dendritic cells use MHC-II and interleukins, matching professional antigen-presenting cells. -5. **Key numbers for the main text:** 83.0% baseline accuracy; 80% canonical markers in top 5% importance modules; 9 of 11 cell types have the best marker in the rank #1 module; masking top-200 modules drops accuracy by 33 percentage points; label-shuffle permutation p < 0.001 with effect size 48.1; PanglaoDB enrichment hit rate 25.8% (1011/3916 significant). +# Module Importance: Result Documentation + +> For the manuscript subsection "Model Interpretability: Module Importance Analysis". + +## Dataset + +- **Zheng68K**: 11 cell types, 68,450 cells, with 13,690 test cells. +- Model: PerformerModuleCellClassifier with attention-pooling head and 48M backbone. +- Baseline test accuracy: **83.1%**. + +## Data Sources + +These are historical experiment results. The analysis output files below are not +included in the repository; recreate them using the commands in `README.md` or +point the analysis scripts to the original run directory. The shared module JSON +is bundled under `model_dependencies/module_data_all/` at the repository root. + +All analyses are based on the following core files, generated by `module_importance.py` +with `pre_encoder` over all 13,690 test cells: + +| File | Path | Description | +| --- | --- | --- | +| `importance_matrix.npy` | `outputs/interpretability/importance/zheng68k_pre_13690/importance_matrix.npy` | `(11 x 660)` DeltaLogit matrix | +| `importance_matrix.csv` | `outputs/interpretability/importance/zheng68k_pre_13690/importance_matrix.csv` | Same matrix with row and column labels | +| `enrichment_results.csv` | `outputs/interpretability/importance/zheng68k_pre_13690/enrichment_results.csv` | Enrichment-test results | +| `top_modules_per_celltype.json` | `outputs/interpretability/importance/zheng68k_pre_13690/top_modules_per_celltype.json` | Top-module details | +| `merged_modules.json` | `model_dependencies/module_data_all/merged_modules.json` | Gene composition of 660 modules | + +Marker verification uses `importance_matrix.npy`, `importance_matrix.csv`, and +`merged_modules.json`. + +Publication figures use all files above plus ablation and permutation results. + +--- + +## Experiment 3 Core: Module Importance Matrix + +### Method + +For each test cell, mask one module at a time using `pre_encoder`, which zeros out +the corresponding gene inputs, and measure the drop in the correct-class logit +as the importance score. Aggregate by cell type to obtain an `(11 x 660)` +importance matrix. + +### Results + +- Masking mode: **pre_encoder** over all 13,690 cells. +- 660 modules: 491 Reactome, 50 Hallmarks, 73 Hotspot, and 46 Processed. + +**Representative top modules per cell type:** + +| Cell Type | #1 Module | #2 Module | #3 Module | +| --- | --- | --- | --- | +| CD14+ Monocyte | Neutrophil Degranulation | TLR4 Cascade | MHC-II Antigen Presentation | +| CD19+ B | MHC-II | IFN-gamma Signaling | CD28 Co-stimulation | +| CD56+ NK | SM_22 (cytotoxic) | module3_2 | Allograft Rejection | +| CD8+ Cytotoxic T | module3_2 | SM_22 | Allograft Rejection | +| Dendritic | MHC-II | Signaling by Interleukins | IFN-gamma | + +### Figure + +Expected generated figure: `publication_figures/fig4_importance_heatmap_clustered.png` (Importance Heatmap (clustered); not bundled). + +--- + +## Experiment 3-A: Masking Ablation Curves + +### Method + +Progressively mask top-k, bottom-k, or random-k modules, with +`k = 1, 2, 5, 10, 20, 50, 100, 200`, and measure classification accuracy. This +uses `post_encoder` masking so the encoder runs only once. + +### Expected Result + +- Masking top modules should rapidly decrease accuracy. +- Masking bottom modules should have little effect. +- Random masking should fall between the two. + +### Results, Completed + +- Baseline accuracy: **83.0%** on 13,690 test cells. +- Post-encoder masking with k = [1, 2, 5, 10, 20, 50, 100, 200]. + +| Masked k | Top Acc | Bottom Acc | Random Acc | +| --- | --- | --- | --- | +| 1 | 83.0% | 83.0% | 83.0% | +| 2 | 82.6% | 83.0% | 83.0% | +| 5 | 82.0% | 83.0% | 83.0% | +| 10 | 77.1% | 83.0% | 83.0% | +| 20 | 73.7% | 83.1% | 82.7% | +| 50 | 69.5% | 83.2% | 82.4% | +| 100 | 65.1% | 82.7% | 81.7% | +| 200 | 49.7% | 81.9% | 78.6% | + +**Interpretation:** masking the top 200 modules, 30% of all modules, reduces +accuracy from 83% to 50%, a 33 percentage-point drop. Masking the bottom 200 +modules has almost no effect, with only a 1.1 point drop. Random masking falls +between the two. This shows that the importance ranking captures modules that +are truly important for classification. + +Expected generated figure: `publication_figures/fig1_ablation_curves.png` (Ablation Curves; not bundled). + +Expected generated figure: `publication_figures/fig_s1_ablation_per_class.png` (Per-class Ablation; not bundled). + +--- + +## Experiment 3-B: Permutation Tests, Random Baseline + +### Method + +**Mode A, Module-Shuffle:** shuffle module token order to break the module-gene +correspondence, then recompute importance. Repeat 10 times. + +**Mode B, Label-Shuffle:** shuffle cell type labels and re-aggregate per-cell +deltas. Repeat 1000 times. + +### Expected Result + +- The true importance maximum should far exceed the null distribution. +- Empirical p-value should be below 0.001. + +### Results, Completed + +**Mode A, Module-Shuffle** (10 permutations, GPU): + +| Metric | Value | +| --- | --- | +| Global p-value | 0.091, limited by n=10; minimum possible is 1/11 | +| Effect size | **1474.5** | +| Real global max | 2.046 | +| Null global max | 0.452 +/- 0.001 | + +**Mode B, Label-Shuffle** (1000 permutations, CPU): + +| Metric | Value | +| --- | --- | +| Global p-value | **< 0.001** | +| Per-class p-values | All < 0.001 (11/11) | +| Effect size | **48.1** | +| Real global max | 2.046 | +| Null global max | 0.145 +/- 0.039 | + +**Interpretation:** + +- **Module-Shuffle:** after shuffling the module-gene correspondence, the maximum importance drops from 2.046 to about 0.452. The effect size is extremely large, showing that importance depends on the correct module-gene mapping rather than random noise. The p-value is 0.091 because only 10 permutations were run, making the minimum possible p-value about 1/11; in practice, 0 of 10 null runs exceeded the observed value. +- **Label-Shuffle:** after shuffling cell type labels, the maximum importance drops from 2.046 to about 0.145. All 11 cell types have p < 0.001, showing that importance is cell-type-specific rather than a generic pattern shared by all cells. + +Expected generated figure: `publication_figures/fig2_permutation_null.png` (Permutation Null; not bundled). + +--- + +## Experiment 3-C: Canonical Marker Gene Verification + +### Method + +Check the importance ranks of modules containing known cell-type canonical +markers such as CD14, CD19, CD8A, and NKG7. Test whether markers are enriched in +top modules. + +### Results, Completed + +| Metric | Value | +| --- | --- | +| Total markers tested | 59 | +| Markers not found | 2 | +| **Median rank percentile** | **0.9%** | +| Markers in top 5% | **47/59 (80%)** | +| Markers in top 10% | 47/59 (80%) | +| Markers in top 20% | 50/59 (85%) | +| Binomial p-value, top 10% | **< 1e-50** | +| Wilcoxon p-value, <50% | **2.5e-11** | + +**Per cell type:** + +| Cell Type | Median Percentile | Best Marker | Best Rank | +| --- | --- | --- | --- | +| CD14+ Monocyte | 0.0% | LYZ | #1 | +| CD19+ B | 1.3% | MS4A1 | #7 | +| CD34+ | 18.5% | THY1 | #7 | +| CD4+ T Helper2 | 1.7% | CD4 | #1 | +| CD4+/CD25 T Reg | 0.9% | CD4 | #1 | +| CD4+/CD45RA+/CD25- Naive T | 10.5% | CD4 | #1 | +| CD4+/CD45RO+ Memory | 0.0% | CD4 | #1 | +| CD56+ NK | 0.0% | NKG7 | #1 | +| CD8+ Cytotoxic T | 0.0% | CD8A | #1 | +| CD8+/CD45RA+ Naive Cytotoxic | 20.3% | CD8A | #1 | +| Dendritic | 2.1% | ITGAX | #1 | + +**Interpretation:** 80% of canonical markers are in the top 5% importance +modules for the corresponding cell type. In 9 of 11 cell types, the best marker +is in the rank #1 module. This strongly shows that the model learns biologically +meaningful module importance. + +Expected generated figure: `marker_verification/fig_marker_ranks.png` (Marker Ranks; not bundled). + +Expected generated figure: `marker_verification/fig_marker_summary_bar.png` (Marker Summary Bar; not bundled). + +--- + +## Experiment 3-D: PanglaoDB Enrichment Analysis + +### Method + +For genes in each cell type's top-k modules, run a hypergeometric enrichment test +against marker gene sets for 178 PanglaoDB cell types, with Bonferroni +correction. + +### Results + +| Metric | Value | +| --- | --- | +| Significant enrichment hits, p_adj < 0.05 | **1011 / 3916** | +| Hit rate | **25.8%** | + +Expected generated figure: `publication_figures/fig5_enrichment_summary.png` (Enrichment Summary; not bundled). + +--- + +## Manuscript Writing Points + +1. **Core conclusion:** the model learns biologically meaningful gene organization through the module-based architecture. Top-importance modules strongly match known cell-type markers, with 80% of markers in the top 5%, and masking these modules causes a large classification-performance drop, from 83% to 50%. +2. **Methodological contribution:** pre-encoder masking gives a more accurate importance estimate than post-encoder masking because it accounts for interactions between modules. DDP support makes all-cell computation feasible. +3. **Statistical rigor:** label-shuffle p < 0.001 with effect size 48.1 shows that importance is cell-type-specific; module-shuffle effect size 1474 shows that importance depends on the correct module-gene correspondence. Top-k masking rapidly reduces accuracy while bottom-k masking has almost no effect, and the gap widens as k increases. +4. **Biological interpretation:** monocytes use Neutrophil Degranulation and TLR4, representing innate immune pathways; B cells use MHC-II, IFN-gamma, and CD28, representing antigen presentation and T-B cooperation; NK/CD8 T share cytotoxic modules; dendritic cells use MHC-II and interleukins, matching professional antigen-presenting cells. +5. **Key numbers for the main text:** 83.0% baseline accuracy; 80% canonical markers in top 5% importance modules; 9 of 11 cell types have the best marker in the rank #1 module; masking top-200 modules drops accuracy by 33 percentage points; label-shuffle permutation p < 0.001 with effect size 48.1; PanglaoDB enrichment hit rate 25.8% (1011/3916 significant). diff --git a/BioMod/interpretability/module_text_alignment/DESIGN.md b/BioMod/interpretability/module_text_alignment/DESIGN.md index fe51ff653ae365b190f79cdc9715ad13ecc6f17b..b55785f0f41844cd695be3b5afea6ab0af477d7d 100644 --- a/BioMod/interpretability/module_text_alignment/DESIGN.md +++ b/BioMod/interpretability/module_text_alignment/DESIGN.md @@ -1,225 +1,235 @@ -# Experiment 2: Module Embedding to Pathway Text Alignment - -## Overview - -This experiment tests whether the structure of the BioMod module embedding space -significantly aligns with the semantic space of natural-language pathway -descriptions. Gene-level models cannot reproduce this experiment because single -gene names do not carry phrase-level semantics. - -## Scripts - -| Script | Function | -| --- | --- | -| `module_text_alignment.py` | Main script: RSA/Mantel test, CKA, and leave-one-out Ridge retrieval | -| `module_text_alignment_extended.py` | Extended analysis: Reactome broad-category RSA, Reactome description-based RSA, and Hallmark deep dive | -| `module_text_alignment_publication.py` | Publication-level analysis: six main figures, bootstrap CIs, multi-model comparison, cross-dataset validation, and deep analysis | -| `fetch_reactome_descriptions.py` | Fetch `summaryText` for 491 pathways from the Reactome API | - -## Commands - -### 1. Download the Sentence Encoder - -```bash -cd ../data/250010166/scLong/performer_module/interpretability/module_text_alignment -conda activate sclong - -wget https://hf-mirror.com/hfd/hfd.sh && chmod a+x hfd.sh && export HF_ENDPOINT=https://hf-mirror.com -./hfd.sh pritamdeka/S-PubMedBert-MS-MARCO --local-dir ./S-PubMedBert-MS-MARCO - -pip install sentence-transformers -i https://pypi.tuna.tsinghua.edu.cn/simple -``` - -### 2. Main Experiment: RSA + CKA + LOO Retrieval - -```bash -# pretrain_subset is recommended because averaging over 50k cells is more stable. -python module_text_alignment.py \ - --module_embedding_path ../results/geometry/pretrain_subset/module_embeddings.npy \ - --module_json ../../module_data_all/merged_modules.json \ - --sentence_model ./S-PubMedBert-MS-MARCO \ - --n_permutations 1000 \ - --output_dir ../results/text_alignment/pretrain_subset_v2 - -# pancreas_scib validates cross-dataset consistency. -python module_text_alignment.py \ - --module_embedding_path ../results/geometry/pancreas_scib/module_embeddings.npy \ - --module_json ../../module_data_all/merged_modules.json \ - --sentence_model ./S-PubMedBert-MS-MARCO \ - --n_permutations 1000 \ - --output_dir ../results/text_alignment/pancreas_scib_v2 - -# Quick mode: skip LOO retrieval and run only RSA + CKA. -python module_text_alignment.py \ - --module_embedding_path ../results/geometry/pretrain_subset/module_embeddings.npy \ - --sentence_model ./S-PubMedBert-MS-MARCO \ - --skip_loo \ - --output_dir ../results/text_alignment/pretrain_subset_quick -``` - -### 3. Fetch Official Reactome Descriptions (Completed, 486/491) - -```bash -# Re-fetch if needed. Requires network access to reactome.org and takes about 2 minutes. -python fetch_reactome_descriptions.py \ - --module_json ../../module_data_all/merged_modules.json \ - --output reactome_descriptions.json --delay 0.05 -``` - -Fetched descriptions are stored in `reactome_descriptions.json`. - -### 4. Extended Analysis: Reactome Categories, Description-Based RSA, Hallmark Deep Dive - -```bash -python module_text_alignment_extended.py \ - --module_embedding_path ../results/geometry/pretrain_subset/module_embeddings.npy \ - --module_json ../../module_data_all/merged_modules.json \ - --sentence_model ./S-PubMedBert-MS-MARCO \ - --descriptions_file ./reactome_descriptions.json \ - --n_permutations 1000 \ - --output_dir ../results/text_alignment/pretrain_subset_v2_extended -``` - -### 5. Publication-Level Analysis - -```bash -# Basic run: one dataset and one model. -python module_text_alignment_publication.py \ - --module_embedding_path ../results/geometry/pretrain_subset/module_embeddings.npy \ - --module_json ../../module_data_all/merged_modules.json \ - --sentence_model ./S-PubMedBert-MS-MARCO \ - --descriptions_file ./reactome_descriptions.json \ - --n_permutations 1000 \ - --n_bootstrap 2000 \ - --output_dir ../results/text_alignment/publication - -# Full run: cross-dataset validation and multi-model comparison. -python module_text_alignment_publication.py \ - --module_embedding_path ../results/geometry/pretrain_subset/module_embeddings.npy \ - --module_embedding_path2 ../results/geometry/pancreas_scib/module_embeddings.npy \ - --module_json ../../module_data_all/merged_modules.json \ - --sentence_model ./S-PubMedBert-MS-MARCO \ - --sentence_models_compare "sentence-transformers/all-mpnet-base-v2,sentence-transformers/all-MiniLM-L6-v2" \ - --descriptions_file ./reactome_descriptions.json \ - --n_permutations 1000 \ - --n_bootstrap 2000 \ - --output_dir ../results/text_alignment/publication_full -``` - -**Arguments:** - -| Argument | Description | -| --- | --- | -| `--module_embedding_path` | `module_embeddings.npy` for the main dataset, from Experiment 1 | -| `--module_embedding_path2` | Optional second dataset for cross-dataset validation | -| `--sentence_models_compare` | Comma-separated model names for comparison, downloaded automatically from HuggingFace | -| `--n_permutations` | Mantel-test permutations; 1000 is standard and 10000 gives more precise p-values | -| `--n_bootstrap` | Bootstrap resampling iterations; 2000 is standard for 95% CIs | -| `--device` | Encoding device, `cpu` or `cuda`; default is CPU | - -## Dependencies - -```bash -pip install sentence-transformers -i https://pypi.tuna.tsinghua.edu.cn/simple -``` - -## Output Files - -### Main Experiment Outputs (`module_text_alignment.py`) - -| File | Content | -| --- | --- | -| `pathway_descriptions.json` | Text descriptions for 541 modules | -| `text_embeddings.npy` | Sentence-encoder embeddings with shape `[541, 768]` | -| `module_embeddings_subset.npy` | Corresponding model module embeddings with shape `[541, 1024]` | -| `alignment_metrics.json` | RSA, CKA, retrieval recall@k, and p-values | -| `rsa_scatter.png` | Paired-distance scatter plot: module space vs text space | -| `rsa_null_distribution.png` | Mantel test: observed value vs null distribution | -| `retrieval_ranks.png` | Histogram of retrieval ranks | -| `retrieval_examples.csv` | Top-5 retrieval results for representative pathways | -| `run.log` | Full log | - -### Extended Analysis Outputs (`module_text_alignment_extended.py`) - -| File | Content | -| --- | --- | -| `extended_metrics.json` | Summary metrics for the three analyses | -| `reactome_category_heatmaps.png` | Broad Reactome category distance-matrix comparison, module vs text | -| `hallmarks_deep_dive.png` | Hallmark similarity matrices and per-module alignment-score bar chart | -| `hallmarks_per_module_alignment.csv` | Per-module alignment ranking for 50 Hallmarks | -| `run_extended.log` | Extended-analysis log | - -### Publication-Level Outputs (`module_text_alignment_publication.py`) - -| File | Content | -| --- | --- | -| `fig1_rsa_overview.{pdf,png}` | 2D hexbin density plus Mantel null distribution with bootstrap CI | -| `fig2_hallmarks_heatmaps.{pdf,png}` | Three 50 x 50 similarity matrices: module, text, and difference; hierarchical clustering order and adaptive color scale | -| `fig3_per_hallmark_alignment.{pdf,png}` | Per-Hallmark alignment-score lollipop plot, colored by MSigDB process category | -| `fig4_category_alignment.{pdf,png}` | Box + strip plot grouped by biological process category with Kruskal-Wallis test | -| `fig5_granularity_effect.{pdf,png}` | Bar plot of RSA across granularity levels: fine-grained, category, superclass, Hallmark | -| `fig6_model_comparison.{pdf,png}` | Optional RSA comparison across models and datasets | -| `fig_alignment_vs_size.{pdf,png}` | Alignment score vs module gene count scatter plot | -| `publication_metrics.json` | Summary of all metrics, including bootstrap 95% CIs | -| `hallmarks_alignment_table.csv` | Alignment scores and process categories for 50 Hallmarks | -| `negative_analysis.csv` | Neighbor comparison for the worst- and best-aligned modules | -| `run_publication.log` | Full log | - -## Results (v2, pretrain_subset) - -### Main Experiment - -| Metric | All (541) | Hallmarks (50) | Reactome (491) | -| --- | --- | --- | --- | -| RSA Spearman r | 0.050 | **0.160** | -0.007 | -| Mantel p-value | 0.012 | 0.030 | 0.586 | -| z-score | 2.2 sigma | 2.1 sigma | -0.3 sigma | -| CKA | 0.043 (p=0.000) | - | - | -| Retrieval median rank | 232 (random=270) | - | - | - -### Interpretation - -- **Hallmark alignment is significant (r=0.16, p=0.03):** among 50 broad biological processes, distance structure in model space significantly correlates with semantic distance in text space. -- **Reactome does not align (r=-0.007, p=0.59):** among 491 fine-grained pathways there is no alignment, likely because fine-grained pathway gene-composition similarity, encoded by the model, and pathway-name language similarity, encoded by text, live on different axes. -- **Global CKA is significant (p=0.000):** the overall representation structure contains an alignment signal. - -### Improvement History - -**v1, BioLinkBERT + 1-to-1 retrieval:** - -- BioLinkBERT was used as a raw BERT model with mean pooling rather than a sentence-level fine-tuned encoder. -- The main metric was exact 1-to-1 retrieval, which is too strict for a 541-dimensional space. -- CKA=0.047 was statistically significant but weak; retrieval was close to random. - -**v2, S-PubMedBert-MS-MARCO + RSA/Mantel test:** - -1. Switched to a sentence encoder: PubMedBERT fine-tuned on MS-MARCO sentence similarity. -2. Added RSA/Mantel test as the main metric, testing structural rather than exact retrieval alignment. -3. Per-family RSA revealed that the alignment signal is concentrated in Hallmarks. - -### Extended Analysis Directions - -1. **Reactome broad-category aggregation:** aggregate the 491 pathways into about 16 top-level categories and retest RSA; broad-category alignment is expected to be significant. -2. **Reactome descriptions:** use API-fetched `summaryText` as richer text input and test whether alignment improves. -3. **Hallmark deep dive:** compute per-module alignment scores, kNN overlap, and the best/worst aligned Hallmarks. - -## Design Decisions - -### Why Use RSA Instead of 1-to-1 Retrieval, - -1-to-1 retrieval asks whether the paired text can be found exactly from a module -embedding, requiring a very strong cross-modal mapping. RSA asks whether modules -that are close in model space are also close in text space, which is a more -natural definition of structural alignment. - -### Why Does Reactome Not Align, - -- Reactome pathways overlap heavily because parent and child pathways share many genes. -- The model encodes gene-composition similarity, while text names reflect hierarchy and naming similarity; the two do not perfectly match. -- The granularity is too fine. For example, text distance between `Signaling by FGFR1` and `Signaling by FGFR2` does not necessarily reflect biological distance. - -### Why Exclude Hotspot/Processed, - -- Hotspot modules (73) are data-driven co-expression modules with only numeric IDs and no pathway semantics. -- Processed modules (46) are pipeline-derived gene groups such as `SM_2` and have no pathway names. -- Without text pairs, they cannot participate in this alignment experiment. +# Experiment 2: Module Embedding to Pathway Text Alignment + +## Overview + +This experiment tests whether the structure of the BioMod module embedding space +significantly aligns with the semantic space of natural-language pathway +descriptions. Gene-level models cannot reproduce this experiment because single +gene names do not carry phrase-level semantics. + +## Scripts + +| Script | Function | +| --- | --- | +| `module_text_alignment.py` | Main script: RSA/Mantel test, CKA, and leave-one-out Ridge retrieval | +| `module_text_alignment_extended.py` | Extended analysis: Reactome broad-category RSA, Reactome description-based RSA, and Hallmark deep dive | +| `module_text_alignment_publication.py` | Publication-level analysis: six main figures, bootstrap CIs, multi-model comparison, cross-dataset validation, and deep analysis | +| `fetch_reactome_descriptions.py` | Fetch `summaryText` for 491 pathways from the Reactome API | + +## Commands + +Run from the BioMod repository root. Generate each `module_embeddings.npy` with +`BioMod/interpretability/module_geometry.py` before using it here. The historical +`pretrain_subset` output requires a separately supplied pretraining dataset; +the `pancreas_scib` example uses the bundled pancreas h5ad file. Existing result +tables below describe historical runs; their raw output directories are not +included in this repository. + +### 1. Download the Sentence Encoder + +```bash +# Run from the BioMod repository root. +# Activate the environment configured for this project. + +# The sentence-transformers loader downloads this model on first use. +# Network access or an existing local model cache is required. + +pip install sentence-transformers -i https://pypi.tuna.tsinghua.edu.cn/simple +``` + +### 2. Main Experiment: RSA + CKA + LOO Retrieval + +```bash +# pretrain_subset is recommended because averaging over 50k cells is more stable. +python BioMod/interpretability/module_text_alignment/module_text_alignment.py \ + --module_embedding_path outputs/interpretability/geometry/pretrain_subset/module_embeddings.npy \ + --module_json model_dependencies/module_data_all/merged_modules.json \ + --sentence_model pritamdeka/S-PubMedBert-MS-MARCO \ + --n_permutations 1000 \ + --output_dir outputs/interpretability/text_alignment/pretrain_subset_v2 + +# pancreas_scib validates cross-dataset consistency. +python BioMod/interpretability/module_text_alignment/module_text_alignment.py \ + --module_embedding_path outputs/interpretability/geometry/pancreas_scib/module_embeddings.npy \ + --module_json model_dependencies/module_data_all/merged_modules.json \ + --sentence_model pritamdeka/S-PubMedBert-MS-MARCO \ + --n_permutations 1000 \ + --output_dir outputs/interpretability/text_alignment/pancreas_scib_v2 + +# Quick mode: skip LOO retrieval and run only RSA + CKA. +python BioMod/interpretability/module_text_alignment/module_text_alignment.py \ + --module_embedding_path outputs/interpretability/geometry/pretrain_subset/module_embeddings.npy \ + --sentence_model pritamdeka/S-PubMedBert-MS-MARCO \ + --skip_loo \ + --output_dir outputs/interpretability/text_alignment/pretrain_subset_quick +``` + +### 3. Fetch Official Reactome Descriptions (Completed, 486/491) + +```bash +# Re-fetch if needed. Requires network access to reactome.org and takes about 2 minutes. +python BioMod/interpretability/module_text_alignment/fetch_reactome_descriptions.py \ + --module_json model_dependencies/module_data_all/merged_modules.json \ + --output reactome_descriptions.json --delay 0.05 +``` + +The command writes refreshed descriptions to `reactome_descriptions.json` in the +repository root. The analysis examples below use the bundled +`BioMod/interpretability/module_text_alignment/reactome_descriptions.json` instead; +set `--descriptions_file reactome_descriptions.json` to use the refreshed output. + +### 4. Extended Analysis: Reactome Categories, Description-Based RSA, Hallmark Deep Dive + +```bash +python BioMod/interpretability/module_text_alignment/module_text_alignment_extended.py \ + --module_embedding_path outputs/interpretability/geometry/pretrain_subset/module_embeddings.npy \ + --module_json model_dependencies/module_data_all/merged_modules.json \ + --sentence_model pritamdeka/S-PubMedBert-MS-MARCO \ + --descriptions_file BioMod/interpretability/module_text_alignment/reactome_descriptions.json \ + --n_permutations 1000 \ + --output_dir outputs/interpretability/text_alignment/pretrain_subset_v2_extended +``` + +### 5. Publication-Level Analysis + +```bash +# Basic run: one dataset and one model. +python BioMod/interpretability/module_text_alignment/module_text_alignment_publication.py \ + --module_embedding_path outputs/interpretability/geometry/pretrain_subset/module_embeddings.npy \ + --module_json model_dependencies/module_data_all/merged_modules.json \ + --sentence_model pritamdeka/S-PubMedBert-MS-MARCO \ + --descriptions_file BioMod/interpretability/module_text_alignment/reactome_descriptions.json \ + --n_permutations 1000 \ + --n_bootstrap 2000 \ + --output_dir outputs/interpretability/text_alignment/publication + +# Full run: cross-dataset validation and multi-model comparison. +python BioMod/interpretability/module_text_alignment/module_text_alignment_publication.py \ + --module_embedding_path outputs/interpretability/geometry/pretrain_subset/module_embeddings.npy \ + --module_embedding_path2 outputs/interpretability/geometry/pancreas_scib/module_embeddings.npy \ + --module_json model_dependencies/module_data_all/merged_modules.json \ + --sentence_model pritamdeka/S-PubMedBert-MS-MARCO \ + --sentence_models_compare "sentence-transformers/all-mpnet-base-v2,sentence-transformers/all-MiniLM-L6-v2" \ + --descriptions_file BioMod/interpretability/module_text_alignment/reactome_descriptions.json \ + --n_permutations 1000 \ + --n_bootstrap 2000 \ + --output_dir outputs/interpretability/text_alignment/publication_full +``` + +**Arguments:** + +| Argument | Description | +| --- | --- | +| `--module_embedding_path` | `module_embeddings.npy` for the main dataset, from Experiment 1 | +| `--module_embedding_path2` | Optional second dataset for cross-dataset validation | +| `--sentence_models_compare` | Comma-separated model names for comparison, downloaded automatically from HuggingFace | +| `--n_permutations` | Mantel-test permutations; 1000 is standard and 10000 gives more precise p-values | +| `--n_bootstrap` | Bootstrap resampling iterations; 2000 is standard for 95% CIs | +| `--device` | Encoding device, `cpu` or `cuda`; default is CPU | + +## Dependencies + +```bash +pip install sentence-transformers -i https://pypi.tuna.tsinghua.edu.cn/simple +``` + +## Output Files + +### Main Experiment Outputs (`module_text_alignment.py`) + +| File | Content | +| --- | --- | +| `pathway_descriptions.json` | Text descriptions for 541 modules | +| `text_embeddings.npy` | Sentence-encoder embeddings with shape `[541, 768]` | +| `module_embeddings_subset.npy` | Corresponding model module embeddings with shape `[541, 1024]` | +| `alignment_metrics.json` | RSA, CKA, retrieval recall@k, and p-values | +| `rsa_scatter.png` | Paired-distance scatter plot: module space vs text space | +| `rsa_null_distribution.png` | Mantel test: observed value vs null distribution | +| `retrieval_ranks.png` | Histogram of retrieval ranks | +| `retrieval_examples.csv` | Top-5 retrieval results for representative pathways | +| `run.log` | Full log | + +### Extended Analysis Outputs (`module_text_alignment_extended.py`) + +| File | Content | +| --- | --- | +| `extended_metrics.json` | Summary metrics for the three analyses | +| `reactome_category_heatmaps.png` | Broad Reactome category distance-matrix comparison, module vs text | +| `hallmarks_deep_dive.png` | Hallmark similarity matrices and per-module alignment-score bar chart | +| `hallmarks_per_module_alignment.csv` | Per-module alignment ranking for 50 Hallmarks | +| `run_extended.log` | Extended-analysis log | + +### Publication-Level Outputs (`module_text_alignment_publication.py`) + +| File | Content | +| --- | --- | +| `fig1_rsa_overview.{pdf,png}` | 2D hexbin density plus Mantel null distribution with bootstrap CI | +| `fig2_hallmarks_heatmaps.{pdf,png}` | Three 50 x 50 similarity matrices: module, text, and difference; hierarchical clustering order and adaptive color scale | +| `fig3_per_hallmark_alignment.{pdf,png}` | Per-Hallmark alignment-score lollipop plot, colored by MSigDB process category | +| `fig4_category_alignment.{pdf,png}` | Box + strip plot grouped by biological process category with Kruskal-Wallis test | +| `fig5_granularity_effect.{pdf,png}` | Bar plot of RSA across granularity levels: fine-grained, category, superclass, Hallmark | +| `fig6_model_comparison.{pdf,png}` | Optional RSA comparison across models and datasets | +| `fig_alignment_vs_size.{pdf,png}` | Alignment score vs module gene count scatter plot | +| `publication_metrics.json` | Summary of all metrics, including bootstrap 95% CIs | +| `hallmarks_alignment_table.csv` | Alignment scores and process categories for 50 Hallmarks | +| `negative_analysis.csv` | Neighbor comparison for the worst- and best-aligned modules | +| `run_publication.log` | Full log | + +## Results (v2, pretrain_subset) + +### Main Experiment + +| Metric | All (541) | Hallmarks (50) | Reactome (491) | +| --- | --- | --- | --- | +| RSA Spearman r | 0.050 | **0.160** | -0.007 | +| Mantel p-value | 0.012 | 0.030 | 0.586 | +| z-score | 2.2 sigma | 2.1 sigma | -0.3 sigma | +| CKA | 0.043 (p=0.000) | - | - | +| Retrieval median rank | 232 (random=270) | - | - | + +### Interpretation + +- **Hallmark alignment is significant (r=0.16, p=0.03):** among 50 broad biological processes, distance structure in model space significantly correlates with semantic distance in text space. +- **Reactome does not align (r=-0.007, p=0.59):** among 491 fine-grained pathways there is no alignment, likely because fine-grained pathway gene-composition similarity, encoded by the model, and pathway-name language similarity, encoded by text, live on different axes. +- **Global CKA is significant (p=0.000):** the overall representation structure contains an alignment signal. + +### Improvement History + +**v1, BioLinkBERT + 1-to-1 retrieval:** + +- BioLinkBERT was used as a raw BERT model with mean pooling rather than a sentence-level fine-tuned encoder. +- The main metric was exact 1-to-1 retrieval, which is too strict for a 541-dimensional space. +- CKA=0.047 was statistically significant but weak; retrieval was close to random. + +**v2, S-PubMedBert-MS-MARCO + RSA/Mantel test:** + +1. Switched to a sentence encoder: PubMedBERT fine-tuned on MS-MARCO sentence similarity. +2. Added RSA/Mantel test as the main metric, testing structural rather than exact retrieval alignment. +3. Per-family RSA revealed that the alignment signal is concentrated in Hallmarks. + +### Extended Analysis Directions + +1. **Reactome broad-category aggregation:** aggregate the 491 pathways into about 16 top-level categories and retest RSA; broad-category alignment is expected to be significant. +2. **Reactome descriptions:** use API-fetched `summaryText` as richer text input and test whether alignment improves. +3. **Hallmark deep dive:** compute per-module alignment scores, kNN overlap, and the best/worst aligned Hallmarks. + +## Design Decisions + +### Why Use RSA Instead of 1-to-1 Retrieval, + +1-to-1 retrieval asks whether the paired text can be found exactly from a module +embedding, requiring a very strong cross-modal mapping. RSA asks whether modules +that are close in model space are also close in text space, which is a more +natural definition of structural alignment. + +### Why Does Reactome Not Align, + +- Reactome pathways overlap heavily because parent and child pathways share many genes. +- The model encodes gene-composition similarity, while text names reflect hierarchy and naming similarity; the two do not perfectly match. +- The granularity is too fine. For example, text distance between `Signaling by FGFR1` and `Signaling by FGFR2` does not necessarily reflect biological distance. + +### Why Exclude Hotspot/Processed, + +- Hotspot modules (73) are data-driven co-expression modules with only numeric IDs and no pathway semantics. +- Processed modules (46) are pipeline-derived gene groups such as `SM_2` and have no pathway names. +- Without text pairs, they cannot participate in this alignment experiment. diff --git a/BioMod/interpretability/module_text_alignment/fetch_reactome_descriptions.py b/BioMod/interpretability/module_text_alignment/fetch_reactome_descriptions.py index a30274ed39c16c0c996e61fae7d39320732c1977..f0af9684cfe173a83a9b56c27dd75141cb2924bc 100644 --- a/BioMod/interpretability/module_text_alignment/fetch_reactome_descriptions.py +++ b/BioMod/interpretability/module_text_alignment/fetch_reactome_descriptions.py @@ -1,97 +1,97 @@ -"""Fetch Reactome pathway descriptions via the Content Service REST API. - -Produces a JSON file mapping module_name -> summaryText that can be passed -to module_text_alignment_extended.py via --descriptions_file. - -Usage: - python fetch_reactome_descriptions.py \ - --module_json ../../module_data_all/merged_modules.json \ - --output reactome_descriptions.json - -Requires internet access to https://reactome.org/ContentService/ -""" -import argparse -import json -import sys -import time -from pathlib import Path - -try: - import requests -except ImportError: - print("ERROR: 'requests' package required. Install: pip install requests") - sys.exit(1) - -HERE = Path(__file__).resolve().parent -INTERP_DIR = HERE.parent -PERFORMER_MODULE = INTERP_DIR.parent -for p in (str(INTERP_DIR), str(PERFORMER_MODULE)): - if p not in sys.path: - sys.path.insert(0, p) - -from interpretability.utils import load_module_names, parse_module_family - -REACTOME_API = "https://reactome.org/ContentService/data/query" - - -def fetch_summary(rhs_id: str, retries: int = 3) -> str: - """Fetch summaryText for a Reactome pathway.""" - url = f"{REACTOME_API}/{rhs_id}" - for attempt in range(retries): - try: - resp = requests.get(url, timeout=10) - if resp.status_code == 200: - data = resp.json() - return data.get("summation", [{}])[0].get("text", "") - elif resp.status_code == 404: - return "" - elif resp.status_code == 429: - time.sleep(2 ** attempt) - else: - return "" - except (requests.RequestException, json.JSONDecodeError): - time.sleep(1) - return "" - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--module_json", type=str, required=True) - parser.add_argument("--output", type=str, default="reactome_descriptions.json") - parser.add_argument("--delay", type=float, default=0.1, - help="Delay between API calls (seconds)") - args = parser.parse_args() - - module_names = load_module_names(args.module_json) - reactome_modules = [n for n in module_names if parse_module_family(n) == "Reactome"] - print(f"Reactome modules to fetch: {len(reactome_modules)}") - - descriptions = {} - for i, pname in enumerate(reactome_modules): - raw = pname.split("__", 1)[1] - parts = raw.rsplit(" R-HSA-", 1) - if len(parts) != 2: - continue - rhs_id = "R-HSA-" + parts[1] - - summary = fetch_summary(rhs_id) - if summary: - # Strip HTML tags - import re - summary = re.sub(r"<[^>]+>", "", summary).strip() - descriptions[pname] = summary - - if (i + 1) % 50 == 0: - print(f" Fetched {i+1}/{len(reactome_modules)}, " - f"got descriptions for {len(descriptions)}") - time.sleep(args.delay) - - print(f"\nDone. Got descriptions for {len(descriptions)}/{len(reactome_modules)} pathways.") - - with open(args.output, "w", encoding="utf-8") as f: - json.dump(descriptions, f, indent=2, ensure_ascii=False) - print(f"Saved: {args.output}") - - -if __name__ == "__main__": - main() +"""Fetch Reactome pathway descriptions via the Content Service REST API. + +Produces a JSON file mapping module_name -> summaryText that can be passed +to module_text_alignment_extended.py via --descriptions_file. + +Usage (run from the BioMod repository root): + python BioMod/interpretability/module_text_alignment/fetch_reactome_descriptions.py \ + --module_json model_dependencies/module_data_all/merged_modules.json \ + --output reactome_descriptions.json + +Requires internet access to https://reactome.org/ContentService/ +""" +import argparse +import json +import sys +import time +from pathlib import Path + +try: + import requests +except ImportError: + print("ERROR: 'requests' package required. Install: pip install requests") + sys.exit(1) + +HERE = Path(__file__).resolve().parent +INTERP_DIR = HERE.parent +PERFORMER_MODULE = INTERP_DIR.parent +for p in (str(INTERP_DIR), str(PERFORMER_MODULE)): + if p not in sys.path: + sys.path.insert(0, p) + +from interpretability.utils import load_module_names, parse_module_family + +REACTOME_API = "https://reactome.org/ContentService/data/query" + + +def fetch_summary(rhs_id: str, retries: int = 3) -> str: + """Fetch summaryText for a Reactome pathway.""" + url = f"{REACTOME_API}/{rhs_id}" + for attempt in range(retries): + try: + resp = requests.get(url, timeout=10) + if resp.status_code == 200: + data = resp.json() + return data.get("summation", [{}])[0].get("text", "") + elif resp.status_code == 404: + return "" + elif resp.status_code == 429: + time.sleep(2 ** attempt) + else: + return "" + except (requests.RequestException, json.JSONDecodeError): + time.sleep(1) + return "" + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--module_json", type=str, required=True) + parser.add_argument("--output", type=str, default="reactome_descriptions.json") + parser.add_argument("--delay", type=float, default=0.1, + help="Delay between API calls (seconds)") + args = parser.parse_args() + + module_names = load_module_names(args.module_json) + reactome_modules = [n for n in module_names if parse_module_family(n) == "Reactome"] + print(f"Reactome modules to fetch: {len(reactome_modules)}") + + descriptions = {} + for i, pname in enumerate(reactome_modules): + raw = pname.split("__", 1)[1] + parts = raw.rsplit(" R-HSA-", 1) + if len(parts) != 2: + continue + rhs_id = "R-HSA-" + parts[1] + + summary = fetch_summary(rhs_id) + if summary: + # Strip HTML tags + import re + summary = re.sub(r"<[^>]+>", "", summary).strip() + descriptions[pname] = summary + + if (i + 1) % 50 == 0: + print(f" Fetched {i+1}/{len(reactome_modules)}, " + f"got descriptions for {len(descriptions)}") + time.sleep(args.delay) + + print(f"\nDone. Got descriptions for {len(descriptions)}/{len(reactome_modules)} pathways.") + + with open(args.output, "w", encoding="utf-8") as f: + json.dump(descriptions, f, indent=2, ensure_ascii=False) + print(f"Saved: {args.output}") + + +if __name__ == "__main__": + main() diff --git a/BioMod/interpretability/module_text_alignment/module_text_alignment.py b/BioMod/interpretability/module_text_alignment/module_text_alignment.py index bed9bc8ba79543c0d3222725e184ac45a0ba011a..f17d369217f4480b22e8f42cee72aec8eae24f25 100644 --- a/BioMod/interpretability/module_text_alignment/module_text_alignment.py +++ b/BioMod/interpretability/module_text_alignment/module_text_alignment.py @@ -1,665 +1,665 @@ -"""Experiment 2 — Module embedding <-> pathway text representation alignment. - -Question --------- -Do scLong's module embeddings align with the natural-language semantics of -their corresponding pathway names/descriptions, If yes, this demonstrates -that the model has learned biologically meaningful representations at the -module level — something gene-level baselines cannot replicate because -individual gene names lack phrase-level semantics. - -Pipeline --------- -1. Parse pathway names from merged_modules.json keys for Hallmarks (50) and - Reactome (491) modules. Hotspot/Processed are excluded (no semantic names). -2. Encode pathway names with a biomedical sentence encoder. -3. Load cell-averaged module embeddings from experiment 1 output. -4. Measure alignment: - - RSA (Representational Similarity Analysis) with Mantel test - - CKA (Centered Kernel Alignment) with permutation test - - Ridge regression linear mapping (LOO-CV) for retrieval -5. Visualize: RSA scatter, similarity distribution, retrieval examples. - -Data requirement ----------------- -Only needs module_embeddings.npy from experiment 1 (any dataset). No raw -expression data needed — this experiment operates entirely in embedding space. - -Sentence encoder ----------------- -Default: pritamdeka/S-PubMedBert-MS-MARCO (a PubMedBERT fine-tuned for -sentence similarity via MS-MARCO). Unlike raw BioLinkBERT (which is just a -masked LM without sentence-level training), this model produces embeddings -optimized for semantic similarity between sentences. - -Alternative: sentence-transformers/all-mpnet-base-v2 (general-domain, strong). -""" -from __future__ import annotations - -import argparse -import json -import logging -import sys -from pathlib import Path -from typing import Dict, List, Optional, Tuple - -import numpy as np -import pandas as pd -from scipy.spatial.distance import cosine as cosine_dist -from scipy.spatial.distance import pdist, squareform -from scipy.stats import spearmanr, pearsonr -from sklearn.linear_model import RidgeCV -from sklearn.metrics.pairwise import cosine_similarity -from sklearn.preprocessing import StandardScaler - -import matplotlib -matplotlib.use("Agg") -import matplotlib.pyplot as plt - -HERE = Path(__file__).resolve().parent -INTERP_DIR = HERE.parent -PERFORMER_MODULE = INTERP_DIR.parent -REPO_ROOT = PERFORMER_MODULE.parent -for p in (str(INTERP_DIR), str(PERFORMER_MODULE), str(REPO_ROOT)): - if p not in sys.path: - sys.path.insert(0, p) - -from interpretability.utils import ( - MODULE_FAMILY_MAP, - load_module_names, - parse_module_family, -) - -logger = logging.getLogger(__name__) - - -# ============================================================ -# 1. Pathway name parsing -# ============================================================ - -def parse_pathway_text(module_name: str) -> Optional[str]: - """Extract a human-readable pathway name from a module key.""" - family = parse_module_family(module_name) - if family == "Hallmarks": - raw = module_name.split("__", 1)[1] - raw = raw.replace("HALLMARK_", "", 1) - return raw.replace("_", " ").title() - elif family == "Reactome": - raw = module_name.split("__", 1)[1] - parts = raw.rsplit(" R-HSA-", 1) - return parts[0].strip() - return None - - -def build_text_descriptions( - module_names: List[str], - descriptions_file: Optional[str] = None, -) -> Tuple[List[int], List[str], List[str]]: - """Build text descriptions for modules that have semantic names. - - Returns: (indices, names, texts) — only for Hallmarks + Reactome modules. - """ - ext_descriptions = {} - if descriptions_file and Path(descriptions_file).exists(): - with open(descriptions_file, "r", encoding="utf-8") as f: - ext_descriptions = json.load(f) - - indices = [] - names = [] - texts = [] - for i, pname in enumerate(module_names): - text = parse_pathway_text(pname) - if text is None: - continue - indices.append(i) - names.append(pname) - if pname in ext_descriptions: - texts.append(f"{text}: {ext_descriptions[pname]}") - else: - texts.append(text) - return indices, names, texts - - -# ============================================================ -# 2. Text encoding -# ============================================================ - -def encode_texts( - texts: List[str], - model_name: str = "pritamdeka/S-PubMedBert-MS-MARCO", - batch_size: int = 32, - device: str = "cpu", -) -> np.ndarray: - """Encode texts using a sentence-transformers model.""" - try: - from sentence_transformers import SentenceTransformer - except ImportError: - raise ImportError( - "sentence-transformers required. Install: pip install sentence-transformers" - ) - - logger.info("Loading sentence encoder: %s", model_name) - model = SentenceTransformer(model_name, device=device) - logger.info("Encoding %d texts...", len(texts)) - embeddings = model.encode( - texts, batch_size=batch_size, show_progress_bar=True, - normalize_embeddings=True, - ) - return np.asarray(embeddings, dtype=np.float32) - - -# ============================================================ -# 3. RSA (Representational Similarity Analysis) -# ============================================================ - -def compute_rsa( - module_emb: np.ndarray, - text_emb: np.ndarray, - metric: str = "cosine", -) -> Tuple[float, float, float, float]: - """Representational Similarity Analysis. - - Computes pairwise distance matrices in both spaces, then correlates them. - This tests: "are modules that are close in model space also close in text space, " - - Returns: (spearman_r, spearman_p, pearson_r, pearson_p) - """ - dist_module = pdist(module_emb, metric=metric) - dist_text = pdist(text_emb, metric=metric) - - sp_r, sp_p = spearmanr(dist_module, dist_text) - pe_r, pe_p = pearsonr(dist_module, dist_text) - - return float(sp_r), float(sp_p), float(pe_r), float(pe_p) - - -def mantel_test( - module_emb: np.ndarray, - text_emb: np.ndarray, - metric: str = "cosine", - n_perm: int = 1000, - seed: int = 42, -) -> Dict[str, float]: - """Mantel test: permutation-based significance for RSA. - - Shuffles rows of one matrix and recomputes the distance-matrix correlation. - More robust than parametric p-values for distance correlations. - """ - dist_module = pdist(module_emb, metric=metric) - dist_text = pdist(text_emb, metric=metric) - - observed_r, _ = spearmanr(dist_module, dist_text) - - rng = np.random.default_rng(seed) - null_rs = np.zeros(n_perm) - n = module_emb.shape[0] - for k in range(n_perm): - perm = rng.permutation(n) - dist_text_perm = pdist(text_emb[perm], metric=metric) - null_rs[k], _ = spearmanr(dist_module, dist_text_perm) - - p_value = float(np.mean(null_rs >= observed_r)) - return { - "rsa_spearman_observed": float(observed_r), - "rsa_null_mean": float(null_rs.mean()), - "rsa_null_std": float(null_rs.std()), - "rsa_p_value": p_value, - "rsa_effect_size_z": float((observed_r - null_rs.mean()) / max(null_rs.std(), 1e-10)), - } - - -def compute_rsa_by_family( - module_emb: np.ndarray, - text_emb: np.ndarray, - names: List[str], - metric: str = "cosine", - n_perm: int = 500, - seed: int = 42, -) -> Dict[str, Dict[str, float]]: - """RSA computed separately for each module family (Hallmarks vs Reactome).""" - results = {} - families = [parse_module_family(n) for n in names] - for fam in ["Hallmarks", "Reactome"]: - mask = np.array([f == fam for f in families]) - if mask.sum() < 10: - continue - sub_module = module_emb[mask] - sub_text = text_emb[mask] - mantel = mantel_test(sub_module, sub_text, metric=metric, - n_perm=n_perm, seed=seed) - results[fam] = mantel - return results - - -# ============================================================ -# 4. CKA -# ============================================================ - -def compute_cka(X: np.ndarray, Y: np.ndarray) -> float: - """Centered Kernel Alignment (linear kernel).""" - n = X.shape[0] - H = np.eye(n) - np.ones((n, n)) / n - Kx = H @ (X @ X.T) @ H - Ky = H @ (Y @ Y.T) @ H - hsic = np.sum(Kx * Ky) - norm = np.sqrt(np.sum(Kx * Kx) * np.sum(Ky * Ky)) - if norm < 1e-10: - return 0.0 - return float(hsic / norm) - - -def permutation_test_cka( - module_emb: np.ndarray, - text_emb: np.ndarray, - n_perm: int = 1000, - seed: int = 42, -) -> Dict[str, float]: - """CKA with permutation null distribution.""" - observed = compute_cka(module_emb, text_emb) - rng = np.random.default_rng(seed) - null_values = np.zeros(n_perm) - for k in range(n_perm): - perm = rng.permutation(text_emb.shape[0]) - null_values[k] = compute_cka(module_emb, text_emb[perm]) - p_value = float(np.mean(null_values >= observed)) - return { - "cka_observed": observed, - "cka_null_mean": float(null_values.mean()), - "cka_null_std": float(null_values.std()), - "cka_p_value": p_value, - } - - -# ============================================================ -# 5. Retrieval (Ridge LOO) -# ============================================================ - -def compute_retrieval_metrics( - module_emb: np.ndarray, - text_emb: np.ndarray, - ks: List[int] = [1, 5, 10, 20], -) -> Tuple[Dict[str, float], np.ndarray]: - """Cross-modal retrieval using Ridge LOO projection.""" - n = module_emb.shape[0] - scaler_x = StandardScaler() - scaler_y = StandardScaler() - X_scaled = scaler_x.fit_transform(module_emb) - Y_scaled = scaler_y.fit_transform(text_emb) - - ranks = np.zeros(n, dtype=np.int64) - for i in range(n): - train_mask = np.ones(n, dtype=bool) - train_mask[i] = False - ridge = RidgeCV(alphas=np.logspace(-2, 4, 7)) - ridge.fit(X_scaled[train_mask], Y_scaled[train_mask]) - y_pred = ridge.predict(X_scaled[i:i+1]) - sims = cosine_similarity(y_pred, Y_scaled)[0] - sorted_idx = np.argsort(-sims) - ranks[i] = int(np.where(sorted_idx == i)[0][0]) - - metrics = {} - for k in ks: - metrics[f"recall@{k}"] = float(np.mean(ranks < k)) - metrics["mrr"] = float(np.mean(1.0 / (ranks + 1))) - metrics["median_rank"] = int(np.median(ranks)) - metrics["mean_rank"] = float(np.mean(ranks)) - return metrics, ranks - - -# ============================================================ -# 6. Visualization -# ============================================================ - -def plot_rsa_scatter( - module_emb: np.ndarray, - text_emb: np.ndarray, - rsa_r: float, - rsa_p: float, - output_path: str, - metric: str = "cosine", - max_pairs: int = 5000, -): - """Scatter plot of pairwise distances: module space vs text space.""" - dist_module = pdist(module_emb, metric=metric) - dist_text = pdist(text_emb, metric=metric) - - if len(dist_module) > max_pairs: - rng = np.random.default_rng(0) - idx = rng.choice(len(dist_module), size=max_pairs, replace=False) - dist_module = dist_module[idx] - dist_text = dist_text[idx] - - fig, ax = plt.subplots(figsize=(7, 6)) - ax.scatter(dist_text, dist_module, alpha=0.1, s=3, color="steelblue") - ax.set_xlabel("Text pairwise distance (cosine)") - ax.set_ylabel("Module embedding pairwise distance (cosine)") - ax.set_title(f"RSA: Spearman r = {rsa_r:.4f} (p = {rsa_p:.4f})") - - z = np.polyfit(dist_text, dist_module, 1) - p_line = np.poly1d(z) - x_range = np.linspace(dist_text.min(), dist_text.max(), 100) - ax.plot(x_range, p_line(x_range), "r-", linewidth=2, alpha=0.8) - - plt.tight_layout() - plt.savefig(output_path, dpi=150, bbox_inches="tight") - plt.close() - logger.info("Saved: %s", output_path) - - -def plot_retrieval_rank_distribution( - ranks: np.ndarray, - n_total: int, - output_path: str, -): - """Histogram of retrieval ranks.""" - fig, ax = plt.subplots(figsize=(8, 5)) - ax.hist(ranks, bins=50, alpha=0.7, color="darkorange", edgecolor="white") - ax.axvline(n_total / 2, color="gray", linestyle="--", linewidth=1.5, - label=f"Random baseline = {n_total//2}") - ax.axvline(np.median(ranks), color="blue", linestyle="-", linewidth=1.5, - label=f"Median = {int(np.median(ranks))}") - ax.set_xlabel("Retrieval Rank (0 = perfect)") - ax.set_ylabel("Count") - ax.set_title("Cross-Modal Retrieval: Rank Distribution (LOO Ridge)") - ax.legend() - plt.tight_layout() - plt.savefig(output_path, dpi=150, bbox_inches="tight") - plt.close() - logger.info("Saved: %s", output_path) - - -def plot_rsa_null_distribution( - observed: float, - null_values_mean: float, - null_values_std: float, - p_value: float, - output_path: str, -): - """Plot observed RSA vs null distribution.""" - fig, ax = plt.subplots(figsize=(7, 5)) - x = np.linspace(null_values_mean - 4*null_values_std, - max(observed + null_values_std, null_values_mean + 4*null_values_std), 200) - from scipy.stats import norm as norm_dist - y = norm_dist.pdf(x, null_values_mean, null_values_std) - ax.fill_between(x, y, alpha=0.3, color="gray", label="Null distribution") - ax.plot(x, y, color="gray", linewidth=1.5) - ax.axvline(observed, color="red", linewidth=2.5, linestyle="-", - label=f"Observed = {observed:.4f}") - ax.set_xlabel("Spearman r (distance matrix correlation)") - ax.set_ylabel("Density") - ax.set_title(f"Mantel Test: p = {p_value:.4f}") - ax.legend() - plt.tight_layout() - plt.savefig(output_path, dpi=150, bbox_inches="tight") - plt.close() - logger.info("Saved: %s", output_path) - - -def save_retrieval_examples( - module_emb: np.ndarray, - text_emb: np.ndarray, - texts: List[str], - names: List[str], - output_path: str, - top_k: int = 5, - query_indices: Optional[List[int]] = None, -): - """Save retrieval examples for selected modules.""" - n = module_emb.shape[0] - scaler_x = StandardScaler() - scaler_y = StandardScaler() - X = scaler_x.fit_transform(module_emb) - Y = scaler_y.fit_transform(text_emb) - - ridge = RidgeCV(alphas=np.logspace(-2, 4, 7)) - ridge.fit(X, Y) - - if query_indices is None: - rng = np.random.default_rng(0) - query_indices = sorted(rng.choice(n, size=min(20, n), replace=False)) - - rows = [] - for qi in query_indices: - y_pred = ridge.predict(X[qi:qi+1]) - sims = cosine_similarity(y_pred, Y)[0] - top_idx = np.argsort(-sims)[:top_k] - for rank, ti in enumerate(top_idx): - rows.append({ - "query_module": names[qi], - "query_text": texts[qi], - "retrieved_rank": rank + 1, - "retrieved_text": texts[ti], - "similarity": float(sims[ti]), - "is_self": ti == qi, - }) - - df = pd.DataFrame(rows) - df.to_csv(output_path, index=False) - logger.info("Saved retrieval examples: %s", output_path) - - -# ============================================================ -# 7. Main -# ============================================================ - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter, - ) - parser.add_argument("--module_embedding_path", type=str, required=True, - help="Path to module_embeddings.npy from experiment 1.") - parser.add_argument("--module_json", type=str, - default=str(PERFORMER_MODULE / "module_data_all" / "merged_modules.json"), - help="Path to merged_modules.json.") - parser.add_argument("--descriptions_file", type=str, default=None, - help="Optional JSON with extended pathway descriptions.") - parser.add_argument("--sentence_model", type=str, - default="pritamdeka/S-PubMedBert-MS-MARCO", - help="HuggingFace sentence-transformers model.") - parser.add_argument("--device", type=str, default="cpu", - help="Device for sentence encoder.") - parser.add_argument("--n_permutations", type=int, default=1000) - parser.add_argument("--seed", type=int, default=42) - parser.add_argument("--output_dir", type=str, required=True) - parser.add_argument("--skip_loo", action="store_true", - help="Skip LOO retrieval (slow). Use RSA/CKA only.") - parser.add_argument("--rsa_metric", type=str, default="cosine", - choices=["cosine", "euclidean"], - help="Distance metric for RSA.") - return parser.parse_args() - - -def main(): - args = parse_args() - - output_dir = Path(args.output_dir) - output_dir.mkdir(parents=True, exist_ok=True) - - logging.basicConfig( - level=logging.INFO, - format="%(asctime)s [%(levelname)s] %(message)s", - handlers=[ - logging.StreamHandler(), - logging.FileHandler(output_dir / "run.log", mode="w"), - ], - ) - - # --- Load module names and embeddings --- - logger.info("Loading module names from %s", args.module_json) - module_names = load_module_names(args.module_json) - logger.info("Total modules: %d", len(module_names)) - - logger.info("Loading module embeddings from %s", args.module_embedding_path) - all_embeddings = np.load(args.module_embedding_path) - assert all_embeddings.shape[0] == len(module_names), ( - f"Embedding count {all_embeddings.shape[0]} != module count {len(module_names)}" - ) - - # --- Build text descriptions (Hallmarks + Reactome only) --- - indices, names, texts = build_text_descriptions( - module_names, args.descriptions_file - ) - logger.info("Modules with text descriptions: %d / %d", len(indices), len(module_names)) - - module_emb = all_embeddings[indices] - logger.info("Module embedding shape: %s", module_emb.shape) - - # --- Encode texts --- - text_emb = encode_texts( - texts, model_name=args.sentence_model, device=args.device - ) - logger.info("Text embedding shape: %s", text_emb.shape) - - # Save intermediate - np.save(output_dir / "text_embeddings.npy", text_emb) - np.save(output_dir / "module_embeddings_subset.npy", module_emb) - with open(output_dir / "pathway_descriptions.json", "w", encoding="utf-8") as f: - json.dump( - [{"index": idx, "module_name": name, "text": text} - for idx, name, text in zip(indices, names, texts)], - f, indent=2, ensure_ascii=False, - ) - - # --- RSA / Mantel test (PRIMARY METRIC) --- - logger.info("=" * 50) - logger.info("RSA / Mantel Test (primary alignment metric)") - logger.info("=" * 50) - - sp_r, sp_p, pe_r, pe_p = compute_rsa(module_emb, text_emb, metric=args.rsa_metric) - logger.info("RSA (all modules): Spearman r=%.4f (p=%.2e), Pearson r=%.4f (p=%.2e)", - sp_r, sp_p, pe_r, pe_p) - - mantel_results = mantel_test( - module_emb, text_emb, metric=args.rsa_metric, - n_perm=args.n_permutations, seed=args.seed, - ) - logger.info("Mantel test: observed=%.4f, null=%.4f +/- %.4f, p=%.4f, z=%.1f", - mantel_results["rsa_spearman_observed"], - mantel_results["rsa_null_mean"], - mantel_results["rsa_null_std"], - mantel_results["rsa_p_value"], - mantel_results["rsa_effect_size_z"]) - - # RSA by family - family_rsa = compute_rsa_by_family( - module_emb, text_emb, names, metric=args.rsa_metric, - n_perm=min(args.n_permutations, 500), seed=args.seed, - ) - for fam, res in family_rsa.items(): - logger.info("RSA [%s]: r=%.4f, null=%.4f +/- %.4f, p=%.4f, z=%.1f", - fam, res["rsa_spearman_observed"], res["rsa_null_mean"], - res["rsa_null_std"], res["rsa_p_value"], res["rsa_effect_size_z"]) - - # --- CKA --- - logger.info("Computing CKA with %d permutations...", args.n_permutations) - cka_results = permutation_test_cka( - module_emb, text_emb, n_perm=args.n_permutations, seed=args.seed - ) - logger.info("CKA: observed=%.4f, null=%.4f +/- %.4f, p=%.4f", - cka_results["cka_observed"], cka_results["cka_null_mean"], - cka_results["cka_null_std"], cka_results["cka_p_value"]) - - # --- LOO retrieval (optional) --- - retrieval_results = {} - ranks = None - if not args.skip_loo: - logger.info("Computing LOO retrieval metrics (n=%d)...", len(indices)) - retrieval_results, ranks = compute_retrieval_metrics(module_emb, text_emb) - logger.info("Retrieval: recall@1=%.3f, @5=%.3f, @10=%.3f, @20=%.3f, MRR=%.3f, median_rank=%d", - retrieval_results.get("recall@1", 0), - retrieval_results.get("recall@5", 0), - retrieval_results.get("recall@10", 0), - retrieval_results.get("recall@20", 0), - retrieval_results.get("mrr", 0), - retrieval_results.get("median_rank", 0)) - - # --- Aggregate metrics --- - metrics = { - "n_modules_with_text": len(indices), - "n_modules_total": len(module_names), - "module_emb_dim": int(module_emb.shape[1]), - "text_emb_dim": int(text_emb.shape[1]), - "sentence_model": args.sentence_model, - "rsa_metric": args.rsa_metric, - "n_permutations": args.n_permutations, - "rsa_spearman_r": sp_r, - "rsa_spearman_p": sp_p, - "rsa_pearson_r": pe_r, - "rsa_pearson_p": pe_p, - **mantel_results, - "rsa_by_family": family_rsa, - **cka_results, - **retrieval_results, - } - - with open(output_dir / "alignment_metrics.json", "w") as f: - json.dump(metrics, f, indent=2) - logger.info("Saved alignment_metrics.json") - - # --- Plots --- - plot_rsa_scatter( - module_emb, text_emb, rsa_r=sp_r, rsa_p=mantel_results["rsa_p_value"], - output_path=str(output_dir / "rsa_scatter.png"), - metric=args.rsa_metric, - ) - - plot_rsa_null_distribution( - observed=mantel_results["rsa_spearman_observed"], - null_values_mean=mantel_results["rsa_null_mean"], - null_values_std=mantel_results["rsa_null_std"], - p_value=mantel_results["rsa_p_value"], - output_path=str(output_dir / "rsa_null_distribution.png"), - ) - - if ranks is not None: - plot_retrieval_rank_distribution( - ranks, n_total=len(indices), - output_path=str(output_dir / "retrieval_ranks.png"), - ) - - # --- Retrieval examples --- - representative_keywords = [ - "interferon", "cell cycle", "oxidative", "apoptosis", - "immune", "signaling", "metabolism", "dna repair", - "nfkb", "mtor", "wnt", "hedgehog", "notch", - "interleukin", "toll", "mapk", - ] - query_indices = [] - for kw in representative_keywords: - for i, t in enumerate(texts): - if kw.lower() in t.lower() and i not in query_indices: - query_indices.append(i) - break - if len(query_indices) < 10: - rng = np.random.default_rng(args.seed) - extra = rng.choice(len(texts), size=15, replace=False) - query_indices = list(set(query_indices) | set(extra.tolist()))[:20] - - save_retrieval_examples( - module_emb, text_emb, texts, names, - output_path=str(output_dir / "retrieval_examples.csv"), - query_indices=sorted(query_indices), - ) - - # --- Summary --- - logger.info("=" * 60) - logger.info("EXPERIMENT 2 SUMMARY") - logger.info("=" * 60) - logger.info("Modules with text: %d / %d", len(indices), len(module_names)) - logger.info("RSA (Mantel): Spearman r=%.4f, p=%.4f, z=%.1f sigma", - mantel_results["rsa_spearman_observed"], - mantel_results["rsa_p_value"], - mantel_results["rsa_effect_size_z"]) - logger.info("CKA: %.4f (p=%.4f)", cka_results["cka_observed"], cka_results["cka_p_value"]) - if retrieval_results: - logger.info("Retrieval: recall@1=%.3f, @5=%.3f, @10=%.3f, MRR=%.3f, median_rank=%d", - retrieval_results["recall@1"], retrieval_results["recall@5"], - retrieval_results["recall@10"], retrieval_results["mrr"], - retrieval_results["median_rank"]) - logger.info("Output: %s", output_dir) - logger.info("=" * 60) - - -if __name__ == "__main__": - main() +"""Experiment 2 — Module embedding <-> pathway text representation alignment. + +Question +-------- +Do scLong's module embeddings align with the natural-language semantics of +their corresponding pathway names/descriptions, If yes, this demonstrates +that the model has learned biologically meaningful representations at the +module level — something gene-level baselines cannot replicate because +individual gene names lack phrase-level semantics. + +Pipeline +-------- +1. Parse pathway names from merged_modules.json keys for Hallmarks (50) and + Reactome (491) modules. Hotspot/Processed are excluded (no semantic names). +2. Encode pathway names with a biomedical sentence encoder. +3. Load cell-averaged module embeddings from experiment 1 output. +4. Measure alignment: + - RSA (Representational Similarity Analysis) with Mantel test + - CKA (Centered Kernel Alignment) with permutation test + - Ridge regression linear mapping (LOO-CV) for retrieval +5. Visualize: RSA scatter, similarity distribution, retrieval examples. + +Data requirement +---------------- +Only needs module_embeddings.npy from experiment 1 (any dataset). No raw +expression data needed — this experiment operates entirely in embedding space. + +Sentence encoder +---------------- +Default: pritamdeka/S-PubMedBert-MS-MARCO (a PubMedBERT fine-tuned for +sentence similarity via MS-MARCO). Unlike raw BioLinkBERT (which is just a +masked LM without sentence-level training), this model produces embeddings +optimized for semantic similarity between sentences. + +Alternative: sentence-transformers/all-mpnet-base-v2 (general-domain, strong). +""" +from __future__ import annotations + +import argparse +import json +import logging +import sys +from pathlib import Path +from typing import Dict, List, Optional, Tuple + +import numpy as np +import pandas as pd +from scipy.spatial.distance import cosine as cosine_dist +from scipy.spatial.distance import pdist, squareform +from scipy.stats import spearmanr, pearsonr +from sklearn.linear_model import RidgeCV +from sklearn.metrics.pairwise import cosine_similarity +from sklearn.preprocessing import StandardScaler + +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt + +HERE = Path(__file__).resolve().parent +INTERP_DIR = HERE.parent +PERFORMER_MODULE = INTERP_DIR.parent +REPO_ROOT = PERFORMER_MODULE.parent +for p in (str(INTERP_DIR), str(PERFORMER_MODULE), str(REPO_ROOT)): + if p not in sys.path: + sys.path.insert(0, p) + +from interpretability.utils import ( + MODULE_FAMILY_MAP, + load_module_names, + parse_module_family, +) + +logger = logging.getLogger(__name__) + + +# ============================================================ +# 1. Pathway name parsing +# ============================================================ + +def parse_pathway_text(module_name: str) -> Optional[str]: + """Extract a human-readable pathway name from a module key.""" + family = parse_module_family(module_name) + if family == "Hallmarks": + raw = module_name.split("__", 1)[1] + raw = raw.replace("HALLMARK_", "", 1) + return raw.replace("_", " ").title() + elif family == "Reactome": + raw = module_name.split("__", 1)[1] + parts = raw.rsplit(" R-HSA-", 1) + return parts[0].strip() + return None + + +def build_text_descriptions( + module_names: List[str], + descriptions_file: Optional[str] = None, +) -> Tuple[List[int], List[str], List[str]]: + """Build text descriptions for modules that have semantic names. + + Returns: (indices, names, texts) — only for Hallmarks + Reactome modules. + """ + ext_descriptions = {} + if descriptions_file and Path(descriptions_file).exists(): + with open(descriptions_file, "r", encoding="utf-8") as f: + ext_descriptions = json.load(f) + + indices = [] + names = [] + texts = [] + for i, pname in enumerate(module_names): + text = parse_pathway_text(pname) + if text is None: + continue + indices.append(i) + names.append(pname) + if pname in ext_descriptions: + texts.append(f"{text}: {ext_descriptions[pname]}") + else: + texts.append(text) + return indices, names, texts + + +# ============================================================ +# 2. Text encoding +# ============================================================ + +def encode_texts( + texts: List[str], + model_name: str = "pritamdeka/S-PubMedBert-MS-MARCO", + batch_size: int = 32, + device: str = "cpu", +) -> np.ndarray: + """Encode texts using a sentence-transformers model.""" + try: + from sentence_transformers import SentenceTransformer + except ImportError: + raise ImportError( + "sentence-transformers required. Install: pip install sentence-transformers" + ) + + logger.info("Loading sentence encoder: %s", model_name) + model = SentenceTransformer(model_name, device=device) + logger.info("Encoding %d texts...", len(texts)) + embeddings = model.encode( + texts, batch_size=batch_size, show_progress_bar=True, + normalize_embeddings=True, + ) + return np.asarray(embeddings, dtype=np.float32) + + +# ============================================================ +# 3. RSA (Representational Similarity Analysis) +# ============================================================ + +def compute_rsa( + module_emb: np.ndarray, + text_emb: np.ndarray, + metric: str = "cosine", +) -> Tuple[float, float, float, float]: + """Representational Similarity Analysis. + + Computes pairwise distance matrices in both spaces, then correlates them. + This tests: "are modules that are close in model space also close in text space, " + + Returns: (spearman_r, spearman_p, pearson_r, pearson_p) + """ + dist_module = pdist(module_emb, metric=metric) + dist_text = pdist(text_emb, metric=metric) + + sp_r, sp_p = spearmanr(dist_module, dist_text) + pe_r, pe_p = pearsonr(dist_module, dist_text) + + return float(sp_r), float(sp_p), float(pe_r), float(pe_p) + + +def mantel_test( + module_emb: np.ndarray, + text_emb: np.ndarray, + metric: str = "cosine", + n_perm: int = 1000, + seed: int = 42, +) -> Dict[str, float]: + """Mantel test: permutation-based significance for RSA. + + Shuffles rows of one matrix and recomputes the distance-matrix correlation. + More robust than parametric p-values for distance correlations. + """ + dist_module = pdist(module_emb, metric=metric) + dist_text = pdist(text_emb, metric=metric) + + observed_r, _ = spearmanr(dist_module, dist_text) + + rng = np.random.default_rng(seed) + null_rs = np.zeros(n_perm) + n = module_emb.shape[0] + for k in range(n_perm): + perm = rng.permutation(n) + dist_text_perm = pdist(text_emb[perm], metric=metric) + null_rs[k], _ = spearmanr(dist_module, dist_text_perm) + + p_value = float(np.mean(null_rs >= observed_r)) + return { + "rsa_spearman_observed": float(observed_r), + "rsa_null_mean": float(null_rs.mean()), + "rsa_null_std": float(null_rs.std()), + "rsa_p_value": p_value, + "rsa_effect_size_z": float((observed_r - null_rs.mean()) / max(null_rs.std(), 1e-10)), + } + + +def compute_rsa_by_family( + module_emb: np.ndarray, + text_emb: np.ndarray, + names: List[str], + metric: str = "cosine", + n_perm: int = 500, + seed: int = 42, +) -> Dict[str, Dict[str, float]]: + """RSA computed separately for each module family (Hallmarks vs Reactome).""" + results = {} + families = [parse_module_family(n) for n in names] + for fam in ["Hallmarks", "Reactome"]: + mask = np.array([f == fam for f in families]) + if mask.sum() < 10: + continue + sub_module = module_emb[mask] + sub_text = text_emb[mask] + mantel = mantel_test(sub_module, sub_text, metric=metric, + n_perm=n_perm, seed=seed) + results[fam] = mantel + return results + + +# ============================================================ +# 4. CKA +# ============================================================ + +def compute_cka(X: np.ndarray, Y: np.ndarray) -> float: + """Centered Kernel Alignment (linear kernel).""" + n = X.shape[0] + H = np.eye(n) - np.ones((n, n)) / n + Kx = H @ (X @ X.T) @ H + Ky = H @ (Y @ Y.T) @ H + hsic = np.sum(Kx * Ky) + norm = np.sqrt(np.sum(Kx * Kx) * np.sum(Ky * Ky)) + if norm < 1e-10: + return 0.0 + return float(hsic / norm) + + +def permutation_test_cka( + module_emb: np.ndarray, + text_emb: np.ndarray, + n_perm: int = 1000, + seed: int = 42, +) -> Dict[str, float]: + """CKA with permutation null distribution.""" + observed = compute_cka(module_emb, text_emb) + rng = np.random.default_rng(seed) + null_values = np.zeros(n_perm) + for k in range(n_perm): + perm = rng.permutation(text_emb.shape[0]) + null_values[k] = compute_cka(module_emb, text_emb[perm]) + p_value = float(np.mean(null_values >= observed)) + return { + "cka_observed": observed, + "cka_null_mean": float(null_values.mean()), + "cka_null_std": float(null_values.std()), + "cka_p_value": p_value, + } + + +# ============================================================ +# 5. Retrieval (Ridge LOO) +# ============================================================ + +def compute_retrieval_metrics( + module_emb: np.ndarray, + text_emb: np.ndarray, + ks: List[int] = [1, 5, 10, 20], +) -> Tuple[Dict[str, float], np.ndarray]: + """Cross-modal retrieval using Ridge LOO projection.""" + n = module_emb.shape[0] + scaler_x = StandardScaler() + scaler_y = StandardScaler() + X_scaled = scaler_x.fit_transform(module_emb) + Y_scaled = scaler_y.fit_transform(text_emb) + + ranks = np.zeros(n, dtype=np.int64) + for i in range(n): + train_mask = np.ones(n, dtype=bool) + train_mask[i] = False + ridge = RidgeCV(alphas=np.logspace(-2, 4, 7)) + ridge.fit(X_scaled[train_mask], Y_scaled[train_mask]) + y_pred = ridge.predict(X_scaled[i:i+1]) + sims = cosine_similarity(y_pred, Y_scaled)[0] + sorted_idx = np.argsort(-sims) + ranks[i] = int(np.where(sorted_idx == i)[0][0]) + + metrics = {} + for k in ks: + metrics[f"recall@{k}"] = float(np.mean(ranks < k)) + metrics["mrr"] = float(np.mean(1.0 / (ranks + 1))) + metrics["median_rank"] = int(np.median(ranks)) + metrics["mean_rank"] = float(np.mean(ranks)) + return metrics, ranks + + +# ============================================================ +# 6. Visualization +# ============================================================ + +def plot_rsa_scatter( + module_emb: np.ndarray, + text_emb: np.ndarray, + rsa_r: float, + rsa_p: float, + output_path: str, + metric: str = "cosine", + max_pairs: int = 5000, +): + """Scatter plot of pairwise distances: module space vs text space.""" + dist_module = pdist(module_emb, metric=metric) + dist_text = pdist(text_emb, metric=metric) + + if len(dist_module) > max_pairs: + rng = np.random.default_rng(0) + idx = rng.choice(len(dist_module), size=max_pairs, replace=False) + dist_module = dist_module[idx] + dist_text = dist_text[idx] + + fig, ax = plt.subplots(figsize=(7, 6)) + ax.scatter(dist_text, dist_module, alpha=0.1, s=3, color="steelblue") + ax.set_xlabel("Text pairwise distance (cosine)") + ax.set_ylabel("Module embedding pairwise distance (cosine)") + ax.set_title(f"RSA: Spearman r = {rsa_r:.4f} (p = {rsa_p:.4f})") + + z = np.polyfit(dist_text, dist_module, 1) + p_line = np.poly1d(z) + x_range = np.linspace(dist_text.min(), dist_text.max(), 100) + ax.plot(x_range, p_line(x_range), "r-", linewidth=2, alpha=0.8) + + plt.tight_layout() + plt.savefig(output_path, dpi=150, bbox_inches="tight") + plt.close() + logger.info("Saved: %s", output_path) + + +def plot_retrieval_rank_distribution( + ranks: np.ndarray, + n_total: int, + output_path: str, +): + """Histogram of retrieval ranks.""" + fig, ax = plt.subplots(figsize=(8, 5)) + ax.hist(ranks, bins=50, alpha=0.7, color="darkorange", edgecolor="white") + ax.axvline(n_total / 2, color="gray", linestyle="--", linewidth=1.5, + label=f"Random baseline = {n_total//2}") + ax.axvline(np.median(ranks), color="blue", linestyle="-", linewidth=1.5, + label=f"Median = {int(np.median(ranks))}") + ax.set_xlabel("Retrieval Rank (0 = perfect)") + ax.set_ylabel("Count") + ax.set_title("Cross-Modal Retrieval: Rank Distribution (LOO Ridge)") + ax.legend() + plt.tight_layout() + plt.savefig(output_path, dpi=150, bbox_inches="tight") + plt.close() + logger.info("Saved: %s", output_path) + + +def plot_rsa_null_distribution( + observed: float, + null_values_mean: float, + null_values_std: float, + p_value: float, + output_path: str, +): + """Plot observed RSA vs null distribution.""" + fig, ax = plt.subplots(figsize=(7, 5)) + x = np.linspace(null_values_mean - 4*null_values_std, + max(observed + null_values_std, null_values_mean + 4*null_values_std), 200) + from scipy.stats import norm as norm_dist + y = norm_dist.pdf(x, null_values_mean, null_values_std) + ax.fill_between(x, y, alpha=0.3, color="gray", label="Null distribution") + ax.plot(x, y, color="gray", linewidth=1.5) + ax.axvline(observed, color="red", linewidth=2.5, linestyle="-", + label=f"Observed = {observed:.4f}") + ax.set_xlabel("Spearman r (distance matrix correlation)") + ax.set_ylabel("Density") + ax.set_title(f"Mantel Test: p = {p_value:.4f}") + ax.legend() + plt.tight_layout() + plt.savefig(output_path, dpi=150, bbox_inches="tight") + plt.close() + logger.info("Saved: %s", output_path) + + +def save_retrieval_examples( + module_emb: np.ndarray, + text_emb: np.ndarray, + texts: List[str], + names: List[str], + output_path: str, + top_k: int = 5, + query_indices: Optional[List[int]] = None, +): + """Save retrieval examples for selected modules.""" + n = module_emb.shape[0] + scaler_x = StandardScaler() + scaler_y = StandardScaler() + X = scaler_x.fit_transform(module_emb) + Y = scaler_y.fit_transform(text_emb) + + ridge = RidgeCV(alphas=np.logspace(-2, 4, 7)) + ridge.fit(X, Y) + + if query_indices is None: + rng = np.random.default_rng(0) + query_indices = sorted(rng.choice(n, size=min(20, n), replace=False)) + + rows = [] + for qi in query_indices: + y_pred = ridge.predict(X[qi:qi+1]) + sims = cosine_similarity(y_pred, Y)[0] + top_idx = np.argsort(-sims)[:top_k] + for rank, ti in enumerate(top_idx): + rows.append({ + "query_module": names[qi], + "query_text": texts[qi], + "retrieved_rank": rank + 1, + "retrieved_text": texts[ti], + "similarity": float(sims[ti]), + "is_self": ti == qi, + }) + + df = pd.DataFrame(rows) + df.to_csv(output_path, index=False) + logger.info("Saved retrieval examples: %s", output_path) + + +# ============================================================ +# 7. Main +# ============================================================ + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument("--module_embedding_path", type=str, required=True, + help="Path to module_embeddings.npy from experiment 1.") + parser.add_argument("--module_json", type=str, + default=str(REPO_ROOT / "model_dependencies" / "module_data_all" / "merged_modules.json"), + help="Path to merged_modules.json.") + parser.add_argument("--descriptions_file", type=str, default=None, + help="Optional JSON with extended pathway descriptions.") + parser.add_argument("--sentence_model", type=str, + default="pritamdeka/S-PubMedBert-MS-MARCO", + help="HuggingFace sentence-transformers model.") + parser.add_argument("--device", type=str, default="cpu", + help="Device for sentence encoder.") + parser.add_argument("--n_permutations", type=int, default=1000) + parser.add_argument("--seed", type=int, default=42) + parser.add_argument("--output_dir", type=str, required=True) + parser.add_argument("--skip_loo", action="store_true", + help="Skip LOO retrieval (slow). Use RSA/CKA only.") + parser.add_argument("--rsa_metric", type=str, default="cosine", + choices=["cosine", "euclidean"], + help="Distance metric for RSA.") + return parser.parse_args() + + +def main(): + args = parse_args() + + output_dir = Path(args.output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(message)s", + handlers=[ + logging.StreamHandler(), + logging.FileHandler(output_dir / "run.log", mode="w"), + ], + ) + + # --- Load module names and embeddings --- + logger.info("Loading module names from %s", args.module_json) + module_names = load_module_names(args.module_json) + logger.info("Total modules: %d", len(module_names)) + + logger.info("Loading module embeddings from %s", args.module_embedding_path) + all_embeddings = np.load(args.module_embedding_path) + assert all_embeddings.shape[0] == len(module_names), ( + f"Embedding count {all_embeddings.shape[0]} != module count {len(module_names)}" + ) + + # --- Build text descriptions (Hallmarks + Reactome only) --- + indices, names, texts = build_text_descriptions( + module_names, args.descriptions_file + ) + logger.info("Modules with text descriptions: %d / %d", len(indices), len(module_names)) + + module_emb = all_embeddings[indices] + logger.info("Module embedding shape: %s", module_emb.shape) + + # --- Encode texts --- + text_emb = encode_texts( + texts, model_name=args.sentence_model, device=args.device + ) + logger.info("Text embedding shape: %s", text_emb.shape) + + # Save intermediate + np.save(output_dir / "text_embeddings.npy", text_emb) + np.save(output_dir / "module_embeddings_subset.npy", module_emb) + with open(output_dir / "pathway_descriptions.json", "w", encoding="utf-8") as f: + json.dump( + [{"index": idx, "module_name": name, "text": text} + for idx, name, text in zip(indices, names, texts)], + f, indent=2, ensure_ascii=False, + ) + + # --- RSA / Mantel test (PRIMARY METRIC) --- + logger.info("=" * 50) + logger.info("RSA / Mantel Test (primary alignment metric)") + logger.info("=" * 50) + + sp_r, sp_p, pe_r, pe_p = compute_rsa(module_emb, text_emb, metric=args.rsa_metric) + logger.info("RSA (all modules): Spearman r=%.4f (p=%.2e), Pearson r=%.4f (p=%.2e)", + sp_r, sp_p, pe_r, pe_p) + + mantel_results = mantel_test( + module_emb, text_emb, metric=args.rsa_metric, + n_perm=args.n_permutations, seed=args.seed, + ) + logger.info("Mantel test: observed=%.4f, null=%.4f +/- %.4f, p=%.4f, z=%.1f", + mantel_results["rsa_spearman_observed"], + mantel_results["rsa_null_mean"], + mantel_results["rsa_null_std"], + mantel_results["rsa_p_value"], + mantel_results["rsa_effect_size_z"]) + + # RSA by family + family_rsa = compute_rsa_by_family( + module_emb, text_emb, names, metric=args.rsa_metric, + n_perm=min(args.n_permutations, 500), seed=args.seed, + ) + for fam, res in family_rsa.items(): + logger.info("RSA [%s]: r=%.4f, null=%.4f +/- %.4f, p=%.4f, z=%.1f", + fam, res["rsa_spearman_observed"], res["rsa_null_mean"], + res["rsa_null_std"], res["rsa_p_value"], res["rsa_effect_size_z"]) + + # --- CKA --- + logger.info("Computing CKA with %d permutations...", args.n_permutations) + cka_results = permutation_test_cka( + module_emb, text_emb, n_perm=args.n_permutations, seed=args.seed + ) + logger.info("CKA: observed=%.4f, null=%.4f +/- %.4f, p=%.4f", + cka_results["cka_observed"], cka_results["cka_null_mean"], + cka_results["cka_null_std"], cka_results["cka_p_value"]) + + # --- LOO retrieval (optional) --- + retrieval_results = {} + ranks = None + if not args.skip_loo: + logger.info("Computing LOO retrieval metrics (n=%d)...", len(indices)) + retrieval_results, ranks = compute_retrieval_metrics(module_emb, text_emb) + logger.info("Retrieval: recall@1=%.3f, @5=%.3f, @10=%.3f, @20=%.3f, MRR=%.3f, median_rank=%d", + retrieval_results.get("recall@1", 0), + retrieval_results.get("recall@5", 0), + retrieval_results.get("recall@10", 0), + retrieval_results.get("recall@20", 0), + retrieval_results.get("mrr", 0), + retrieval_results.get("median_rank", 0)) + + # --- Aggregate metrics --- + metrics = { + "n_modules_with_text": len(indices), + "n_modules_total": len(module_names), + "module_emb_dim": int(module_emb.shape[1]), + "text_emb_dim": int(text_emb.shape[1]), + "sentence_model": args.sentence_model, + "rsa_metric": args.rsa_metric, + "n_permutations": args.n_permutations, + "rsa_spearman_r": sp_r, + "rsa_spearman_p": sp_p, + "rsa_pearson_r": pe_r, + "rsa_pearson_p": pe_p, + **mantel_results, + "rsa_by_family": family_rsa, + **cka_results, + **retrieval_results, + } + + with open(output_dir / "alignment_metrics.json", "w") as f: + json.dump(metrics, f, indent=2) + logger.info("Saved alignment_metrics.json") + + # --- Plots --- + plot_rsa_scatter( + module_emb, text_emb, rsa_r=sp_r, rsa_p=mantel_results["rsa_p_value"], + output_path=str(output_dir / "rsa_scatter.png"), + metric=args.rsa_metric, + ) + + plot_rsa_null_distribution( + observed=mantel_results["rsa_spearman_observed"], + null_values_mean=mantel_results["rsa_null_mean"], + null_values_std=mantel_results["rsa_null_std"], + p_value=mantel_results["rsa_p_value"], + output_path=str(output_dir / "rsa_null_distribution.png"), + ) + + if ranks is not None: + plot_retrieval_rank_distribution( + ranks, n_total=len(indices), + output_path=str(output_dir / "retrieval_ranks.png"), + ) + + # --- Retrieval examples --- + representative_keywords = [ + "interferon", "cell cycle", "oxidative", "apoptosis", + "immune", "signaling", "metabolism", "dna repair", + "nfkb", "mtor", "wnt", "hedgehog", "notch", + "interleukin", "toll", "mapk", + ] + query_indices = [] + for kw in representative_keywords: + for i, t in enumerate(texts): + if kw.lower() in t.lower() and i not in query_indices: + query_indices.append(i) + break + if len(query_indices) < 10: + rng = np.random.default_rng(args.seed) + extra = rng.choice(len(texts), size=15, replace=False) + query_indices = list(set(query_indices) | set(extra.tolist()))[:20] + + save_retrieval_examples( + module_emb, text_emb, texts, names, + output_path=str(output_dir / "retrieval_examples.csv"), + query_indices=sorted(query_indices), + ) + + # --- Summary --- + logger.info("=" * 60) + logger.info("EXPERIMENT 2 SUMMARY") + logger.info("=" * 60) + logger.info("Modules with text: %d / %d", len(indices), len(module_names)) + logger.info("RSA (Mantel): Spearman r=%.4f, p=%.4f, z=%.1f sigma", + mantel_results["rsa_spearman_observed"], + mantel_results["rsa_p_value"], + mantel_results["rsa_effect_size_z"]) + logger.info("CKA: %.4f (p=%.4f)", cka_results["cka_observed"], cka_results["cka_p_value"]) + if retrieval_results: + logger.info("Retrieval: recall@1=%.3f, @5=%.3f, @10=%.3f, MRR=%.3f, median_rank=%d", + retrieval_results["recall@1"], retrieval_results["recall@5"], + retrieval_results["recall@10"], retrieval_results["mrr"], + retrieval_results["median_rank"]) + logger.info("Output: %s", output_dir) + logger.info("=" * 60) + + +if __name__ == "__main__": + main() diff --git a/BioMod/interpretability/module_text_alignment/module_text_alignment_extended.py b/BioMod/interpretability/module_text_alignment/module_text_alignment_extended.py index 7b4b73050f56c41ecfca1fc37b0c878061558ccd..3ec84212bd4c5524e32f7e34cbb080f4c3abfc21 100644 --- a/BioMod/interpretability/module_text_alignment/module_text_alignment_extended.py +++ b/BioMod/interpretability/module_text_alignment/module_text_alignment_extended.py @@ -1,537 +1,537 @@ -"""Extended analysis for Experiment 2 — Module-Text Alignment. - -Three analyses: -1. Reactome category-level RSA (aggregate 491 pathways into ~16 top-level categories) -2. Reactome with official descriptions (via descriptions JSON) -3. Hallmarks deep dive (per-module alignment scores, heatmap) - -Usage: - python module_text_alignment_extended.py \ - --module_embedding_path ../results/geometry/pretrain_subset/module_embeddings.npy \ - --module_json ../../module_data_all/merged_modules.json \ - --sentence_model ./S-PubMedBert-MS-MARCO \ - --output_dir ../results/text_alignment/pretrain_subset_v2_extended -""" -from __future__ import annotations - -import argparse -import json -import logging -import sys -from collections import Counter, defaultdict -from pathlib import Path -from typing import Dict, List, Optional, Tuple - -import numpy as np -import pandas as pd -from scipy.spatial.distance import pdist, squareform -from scipy.stats import spearmanr -from sklearn.linear_model import RidgeCV -from sklearn.metrics.pairwise import cosine_similarity -from sklearn.preprocessing import StandardScaler - -import matplotlib -matplotlib.use("Agg") -import matplotlib.pyplot as plt - -HERE = Path(__file__).resolve().parent -INTERP_DIR = HERE.parent -PERFORMER_MODULE = INTERP_DIR.parent -REPO_ROOT = PERFORMER_MODULE.parent -for p in (str(INTERP_DIR), str(PERFORMER_MODULE), str(REPO_ROOT)): - if p not in sys.path: - sys.path.insert(0, p) - -from interpretability.utils import load_module_names, parse_module_family -from module_text_alignment.module_text_alignment import ( - build_text_descriptions, - compute_cka, - compute_rsa, - encode_texts, - mantel_test, - parse_pathway_text, -) - -logger = logging.getLogger(__name__) - - -# ============================================================ -# Reactome top-level category mapping -# ============================================================ - -REACTOME_CATEGORIES = { - "Cell Cycle": ["cell cycle", "mitotic", "mitosis", "meiosis", "g1/s", "g2/m", - "s phase", "m phase", "checkpoint", "cdc", "apc/c", "cyclin", - "condensation of", "centrosome", "kinetochore", "spindle", - "cohes", "pre-replicative", "separation of sister", "aurora"], - "DNA Repair": ["dna repair", "base excision", "nucleotide excision", - "mismatch repair", "homologous recombination", "non-homologous", - "double-strand", "single-strand", "fanconi", "dna damage"], - "DNA Replication": ["dna replication", "replication", "okazaki", "telomere"], - "Signal Transduction": ["signaling", "signal transduction", "mapk", "ras", - "raf", "mek", "erk", "pi3k", "akt", "mtor", "wnt", - "notch", "hedgehog", "tgf-beta", "bmp", "fgfr", - "egfr", "vegf", "pdgf", "igf", "insulin", "erbb", - "met receptor", "receptor tyrosine", "gpcr", - "g alpha", "rho gtpase", "rac1", "cdc42", - "ca2+ pathway", "nfkb", "nf-kb", "nmda", - "calcium", "pkc", "plc", "dag", "pip"], - "Immune System": ["immune", "innate", "adaptive", "interferon", "interleukin", - "cytokine", "complement", "toll", "antigen", "mhc", "tcr", - "bcr", "fcr", "neutrophil", "phagocyt", "inflamm", - "antimicrobial", "antiviral", "ifn-stimulated", - "c-type lectin", "nod-like", "rig-i"], - "Metabolism": ["metabolism", "metabolic", "glycolysis", "gluconeogenesis", - "tca cycle", "citric acid", "oxidative phosphorylation", - "fatty acid", "lipid", "cholesterol", "bile", "amino acid", - "nucleotide", "purine", "pyrimidine", "pentose", "urea", - "biological oxidation", "glycosylation", "biosynthesis", - "synthesis of", "respiratory electron"], - "Gene Expression": ["transcription", "translation", "mrna", "rrna", "trna", - "splicing", "polyadenylation", "ribosom", - "eukaryotic translation", "nonsense-mediated", - "deadenylation", "rna polymerase", "processing of capped"], - "Protein Homeostasis": ["ubiquitin", "proteasome", "sumo", "nedd", - "autophagy", "chaperone", "unfolded protein", - "protein folding", "hsp", "degradation of", - "deubiquitin"], - "Transport": ["transport", "vesicle", "endocyt", "exocyt", "golgi", - "er to golgi", "membrane trafficking", "clathrin", "copi", - "copii", "ion channel", "slc-mediated", "abc transporter", - "nuclear pore", "nuclear import", "nuclear export", "cargo"], - "Apoptosis": ["apoptosis", "apoptotic", "programmed cell death", "caspase", - "death receptor", "bcl-2"], - "Cell-Cell Communication": ["cell junction", "gap junction", "tight junction", - "adherens", "cell-cell", "extracellular matrix", - "integrin", "synapse", "neurotransmitter", - "collagen", "laminin", "elastic fibre", - "anchoring"], - "Chromatin & Epigenetics": ["chromatin", "histone", "methylation", - "acetylation", "deacetylation", "epigenetic", - "polycomb", "hox"], - "Hemostasis": ["hemostasis", "platelet", "coagulation", "fibrin", "thrombin"], - "Developmental Biology": ["developmental", "axon", "neural", "muscle", - "myogenesis", "keratinization", "differentiation", - "cardiac conduction"], - "Disease": ["disease", "hiv", "influenza", "sars", "hcmv", "herpes", - "hepatitis", "cancer", "oncogene", "tumor", "amyloid"], - "Cellular Responses": ["cellular response", "oxidative stress", "hypoxia", - "heat shock", "senescence", "ferroptosis", - "detoxification"], -} - - -def categorize_reactome(pathway_name: str) -> str: - name_lower = pathway_name.lower() - for cat, keywords in REACTOME_CATEGORIES.items(): - for kw in keywords: - if kw in name_lower: - return cat - return "Other" - - -# ============================================================ -# Analysis 1: Reactome category-level RSA -# ============================================================ - -def reactome_category_rsa( - module_emb: np.ndarray, - text_emb: np.ndarray, - names: List[str], - texts: List[str], - output_dir: Path, - sentence_model: str, - device: str, - n_perm: int = 1000, - seed: int = 42, -) -> Dict: - """Aggregate Reactome modules by top-level category, then do RSA.""" - logger.info("=" * 50) - logger.info("Analysis 1: Reactome category-level RSA") - logger.info("=" * 50) - - families = [parse_module_family(n) for n in names] - reactome_mask = np.array([f == "Reactome" for f in families]) - reactome_idx = np.where(reactome_mask)[0] - - reactome_names = [names[i] for i in reactome_idx] - reactome_texts = [texts[i] for i in reactome_idx] - reactome_module_emb = module_emb[reactome_idx] - - cats = [categorize_reactome(t) for t in reactome_texts] - cat_counts = Counter(cats) - logger.info("Reactome category distribution:") - for cat, count in sorted(cat_counts.items(), key=lambda x: -x[1]): - logger.info(" %s: %d", cat, count) - - # Aggregate: mean embedding per category - unique_cats = sorted(set(c for c in cats if c != "Other")) - n_cats = len(unique_cats) - logger.info("Categories (excl. Other): %d", n_cats) - - cat_module_emb = np.zeros((n_cats, module_emb.shape[1])) - for i, cat in enumerate(unique_cats): - mask = np.array([c == cat for c in cats]) - cat_module_emb[i] = reactome_module_emb[mask].mean(axis=0) - - # Encode category names as text - cat_text_emb = encode_texts(unique_cats, model_name=sentence_model, device=device) - - # RSA on category-level - sp_r, sp_p, pe_r, pe_p = compute_rsa(cat_module_emb, cat_text_emb, metric="cosine") - logger.info("Category-level RSA: Spearman r=%.4f (p=%.4f), Pearson r=%.4f", - sp_r, sp_p, pe_r) - - mantel = mantel_test(cat_module_emb, cat_text_emb, metric="cosine", - n_perm=n_perm, seed=seed) - logger.info("Mantel test: observed=%.4f, null=%.4f +/- %.4f, p=%.4f, z=%.1f", - mantel["rsa_spearman_observed"], mantel["rsa_null_mean"], - mantel["rsa_null_std"], mantel["rsa_p_value"], - mantel["rsa_effect_size_z"]) - - cka = compute_cka(cat_module_emb, cat_text_emb) - logger.info("Category-level CKA: %.4f", cka) - - # Visualize: distance heatmaps - fig, axes = plt.subplots(1, 2, figsize=(14, 6)) - - dist_module = squareform(pdist(cat_module_emb, metric="cosine")) - dist_text = squareform(pdist(cat_text_emb, metric="cosine")) - - im0 = axes[0].imshow(dist_module, cmap="viridis", aspect="auto") - axes[0].set_xticks(range(n_cats)) - axes[0].set_xticklabels(unique_cats, rotation=45, ha="right", fontsize=7) - axes[0].set_yticks(range(n_cats)) - axes[0].set_yticklabels(unique_cats, fontsize=7) - axes[0].set_title("Module Embedding Distance (cosine)") - plt.colorbar(im0, ax=axes[0], fraction=0.046) - - im1 = axes[1].imshow(dist_text, cmap="viridis", aspect="auto") - axes[1].set_xticks(range(n_cats)) - axes[1].set_xticklabels(unique_cats, rotation=45, ha="right", fontsize=7) - axes[1].set_yticks(range(n_cats)) - axes[1].set_yticklabels(unique_cats, fontsize=7) - axes[1].set_title("Text Embedding Distance (cosine)") - plt.colorbar(im1, ax=axes[1], fraction=0.046) - - plt.suptitle(f"Reactome Category-Level: RSA r={sp_r:.3f}, p={mantel['rsa_p_value']:.4f}") - plt.tight_layout() - plt.savefig(output_dir / "reactome_category_heatmaps.png", dpi=150, bbox_inches="tight") - plt.close() - - results = { - "n_categories": n_cats, - "category_counts": dict(cat_counts), - "rsa_spearman_r": sp_r, - "rsa_spearman_p": sp_p, - "rsa_pearson_r": pe_r, - "mantel": mantel, - "cka": cka, - "categories": unique_cats, - } - return results - - -# ============================================================ -# Analysis 2: Reactome with descriptions -# ============================================================ - -def reactome_with_descriptions( - module_emb: np.ndarray, - names: List[str], - texts: List[str], - descriptions_file: str, - output_dir: Path, - sentence_model: str, - device: str, - n_perm: int = 1000, - seed: int = 42, -) -> Optional[Dict]: - """Re-run RSA using extended descriptions for Reactome pathways.""" - if not descriptions_file or not Path(descriptions_file).exists(): - logger.info("No descriptions file provided, skipping Analysis 2.") - return None - - logger.info("=" * 50) - logger.info("Analysis 2: Reactome with official descriptions") - logger.info("=" * 50) - - with open(descriptions_file, "r", encoding="utf-8") as f: - descriptions = json.load(f) - - families = [parse_module_family(n) for n in names] - reactome_mask = np.array([f == "Reactome" for f in families]) - reactome_idx = np.where(reactome_mask)[0] - - # Build enriched texts - enriched_texts = [] - valid_idx = [] - for i in reactome_idx: - pname = names[i] - base_text = texts[i] - if pname in descriptions and descriptions[pname]: - enriched_texts.append(f"{base_text}: {descriptions[pname]}") - valid_idx.append(i) - else: - enriched_texts.append(base_text) - valid_idx.append(i) - - logger.info("Reactome modules with descriptions: %d / %d", - sum(1 for i in reactome_idx if names[i] in descriptions), - len(reactome_idx)) - - sub_module_emb = module_emb[valid_idx] - text_emb_enriched = encode_texts(enriched_texts, model_name=sentence_model, device=device) - - sp_r, sp_p, pe_r, pe_p = compute_rsa(sub_module_emb, text_emb_enriched, metric="cosine") - logger.info("RSA with descriptions: Spearman r=%.4f (p=%.2e)", sp_r, sp_p) - - mantel = mantel_test(sub_module_emb, text_emb_enriched, metric="cosine", - n_perm=n_perm, seed=seed) - logger.info("Mantel: observed=%.4f, null=%.4f +/- %.4f, p=%.4f, z=%.1f", - mantel["rsa_spearman_observed"], mantel["rsa_null_mean"], - mantel["rsa_null_std"], mantel["rsa_p_value"], - mantel["rsa_effect_size_z"]) - - return { - "rsa_spearman_r": sp_r, - "rsa_spearman_p": sp_p, - "mantel": mantel, - } - - -# ============================================================ -# Analysis 3: Hallmarks deep dive -# ============================================================ - -def hallmarks_deep_dive( - module_emb: np.ndarray, - text_emb: np.ndarray, - names: List[str], - texts: List[str], - output_dir: Path, - seed: int = 42, -) -> Dict: - """Per-module alignment analysis for the 50 Hallmarks.""" - logger.info("=" * 50) - logger.info("Analysis 3: Hallmarks deep dive") - logger.info("=" * 50) - - families = [parse_module_family(n) for n in names] - hallmark_mask = np.array([f == "Hallmarks" for f in families]) - hallmark_idx = np.where(hallmark_mask)[0] - - h_module = module_emb[hallmark_idx] - h_text = text_emb[hallmark_idx] - h_names = [names[i] for i in hallmark_idx] - h_texts = [texts[i] for i in hallmark_idx] - n_h = len(hallmark_idx) - - logger.info("Hallmarks: %d modules", n_h) - - # Per-module alignment: for each hallmark, compute cosine sim between - # its module embedding's nearest neighbors and its text embedding's nearest neighbors - # (neighborhood overlap) - module_sim_matrix = cosine_similarity(h_module) - text_sim_matrix = cosine_similarity(h_text) - - # Per-module: correlation between its similarity profile in module space vs text space - per_module_corr = np.zeros(n_h) - for i in range(n_h): - mask = np.ones(n_h, dtype=bool) - mask[i] = False - r, _ = spearmanr(module_sim_matrix[i, mask], text_sim_matrix[i, mask]) - per_module_corr[i] = r - - # Sort by alignment quality - sort_idx = np.argsort(-per_module_corr) - logger.info("\nTop-10 best aligned Hallmarks:") - for rank, i in enumerate(sort_idx[:10]): - logger.info(" %2d. %s (r=%.3f)", rank+1, h_texts[i], per_module_corr[i]) - - logger.info("\nBottom-10 worst aligned Hallmarks:") - for rank, i in enumerate(sort_idx[-10:]): - logger.info(" %2d. %s (r=%.3f)", rank+1, h_texts[i], per_module_corr[i]) - - # Save per-module scores - df = pd.DataFrame({ - "module_name": h_names, - "text": h_texts, - "alignment_r": per_module_corr, - }).sort_values("alignment_r", ascending=False) - df.to_csv(output_dir / "hallmarks_per_module_alignment.csv", index=False) - - # Heatmap: module similarity vs text similarity - fig, axes = plt.subplots(1, 3, figsize=(18, 6)) - - im0 = axes[0].imshow(module_sim_matrix[sort_idx][:, sort_idx], cmap="RdBu_r", - vmin=-0.5, vmax=0.5, aspect="auto") - axes[0].set_title("Module Embedding Similarity") - axes[0].set_xticks(range(n_h)) - axes[0].set_xticklabels([h_texts[i][:15] for i in sort_idx], rotation=90, fontsize=5) - axes[0].set_yticks(range(n_h)) - axes[0].set_yticklabels([h_texts[i][:15] for i in sort_idx], fontsize=5) - plt.colorbar(im0, ax=axes[0], fraction=0.046) - - im1 = axes[1].imshow(text_sim_matrix[sort_idx][:, sort_idx], cmap="RdBu_r", - vmin=-0.5, vmax=0.5, aspect="auto") - axes[1].set_title("Text Embedding Similarity") - axes[1].set_xticks(range(n_h)) - axes[1].set_xticklabels([h_texts[i][:15] for i in sort_idx], rotation=90, fontsize=5) - axes[1].set_yticks([]) - plt.colorbar(im1, ax=axes[1], fraction=0.046) - - # Bar chart of per-module alignment - colors = ["steelblue" if r > 0 else "salmon" for r in per_module_corr[sort_idx]] - axes[2].barh(range(n_h), per_module_corr[sort_idx], color=colors) - axes[2].set_yticks(range(n_h)) - axes[2].set_yticklabels([h_texts[i][:20] for i in sort_idx], fontsize=5) - axes[2].set_xlabel("Spearman r (neighborhood alignment)") - axes[2].set_title("Per-Hallmark Alignment Score") - axes[2].axvline(0, color="black", linewidth=0.5) - - plt.tight_layout() - plt.savefig(output_dir / "hallmarks_deep_dive.png", dpi=150, bbox_inches="tight") - plt.close() - logger.info("Saved hallmarks_deep_dive.png") - - # kNN overlap analysis - k = 5 - module_knn = np.argsort(-module_sim_matrix, axis=1)[:, 1:k+1] - text_knn = np.argsort(-text_sim_matrix, axis=1)[:, 1:k+1] - knn_overlap = np.zeros(n_h) - for i in range(n_h): - overlap = len(set(module_knn[i]) & set(text_knn[i])) - knn_overlap[i] = overlap / k - - logger.info("\nkNN overlap (k=%d): mean=%.3f, median=%.3f", - k, knn_overlap.mean(), np.median(knn_overlap)) - - # Permutation test for kNN overlap - rng = np.random.default_rng(seed) - n_perm = 1000 - null_overlaps = np.zeros(n_perm) - for p in range(n_perm): - perm = rng.permutation(n_h) - text_knn_perm = text_knn[perm] - overlaps_perm = np.zeros(n_h) - for i in range(n_h): - overlaps_perm[i] = len(set(module_knn[i]) & set(text_knn_perm[i])) / k - null_overlaps[p] = overlaps_perm.mean() - - knn_p = float(np.mean(null_overlaps >= knn_overlap.mean())) - logger.info("kNN overlap permutation test: observed=%.3f, null=%.3f +/- %.3f, p=%.4f", - knn_overlap.mean(), null_overlaps.mean(), null_overlaps.std(), knn_p) - - results = { - "n_hallmarks": n_h, - "per_module_alignment_mean": float(per_module_corr.mean()), - "per_module_alignment_median": float(np.median(per_module_corr)), - "per_module_alignment_std": float(per_module_corr.std()), - "n_positive_alignment": int((per_module_corr > 0).sum()), - "n_negative_alignment": int((per_module_corr < 0).sum()), - "knn_k": k, - "knn_overlap_mean": float(knn_overlap.mean()), - "knn_overlap_null_mean": float(null_overlaps.mean()), - "knn_overlap_p": knn_p, - } - return results - - -# ============================================================ -# Main -# ============================================================ - -def parse_args(): - parser = argparse.ArgumentParser(description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument("--module_embedding_path", type=str, required=True) - parser.add_argument("--module_json", type=str, - default=str(PERFORMER_MODULE / "module_data_all" / "merged_modules.json")) - parser.add_argument("--descriptions_file", type=str, default=None, - help="JSON with Reactome summaryText descriptions.") - parser.add_argument("--sentence_model", type=str, - default="pritamdeka/S-PubMedBert-MS-MARCO") - parser.add_argument("--device", type=str, default="cpu") - parser.add_argument("--n_permutations", type=int, default=1000) - parser.add_argument("--seed", type=int, default=42) - parser.add_argument("--output_dir", type=str, required=True) - return parser.parse_args() - - -def main(): - args = parse_args() - output_dir = Path(args.output_dir) - output_dir.mkdir(parents=True, exist_ok=True) - - logging.basicConfig( - level=logging.INFO, - format="%(asctime)s [%(levelname)s] %(message)s", - handlers=[ - logging.StreamHandler(), - logging.FileHandler(output_dir / "run_extended.log", mode="w"), - ], - ) - - # Load data - module_names = load_module_names(args.module_json) - all_embeddings = np.load(args.module_embedding_path) - indices, names, texts = build_text_descriptions(module_names) - module_emb = all_embeddings[indices] - - # Load or compute text embeddings - text_emb_path = output_dir.parent / "pretrain_subset_v2" / "text_embeddings.npy" - if text_emb_path.exists(): - logger.info("Loading cached text embeddings from %s", text_emb_path) - text_emb = np.load(text_emb_path) - else: - text_emb = encode_texts(texts, model_name=args.sentence_model, device=args.device) - - all_results = {} - - # Analysis 1: Reactome category-level - cat_results = reactome_category_rsa( - module_emb, text_emb, names, texts, output_dir, - sentence_model=args.sentence_model, device=args.device, - n_perm=args.n_permutations, seed=args.seed, - ) - all_results["reactome_category_rsa"] = cat_results - - # Analysis 2: Reactome with descriptions - desc_results = reactome_with_descriptions( - module_emb, names, texts, args.descriptions_file, output_dir, - sentence_model=args.sentence_model, device=args.device, - n_perm=args.n_permutations, seed=args.seed, - ) - if desc_results: - all_results["reactome_with_descriptions"] = desc_results - - # Analysis 3: Hallmarks deep dive - hallmark_results = hallmarks_deep_dive( - module_emb, text_emb, names, texts, output_dir, seed=args.seed, - ) - all_results["hallmarks_deep_dive"] = hallmark_results - - # Save all results - with open(output_dir / "extended_metrics.json", "w") as f: - json.dump(all_results, f, indent=2, default=str) - logger.info("Saved extended_metrics.json") - - # Summary - logger.info("=" * 60) - logger.info("EXTENDED ANALYSIS SUMMARY") - logger.info("=" * 60) - logger.info("1. Reactome category-level RSA: r=%.4f, p=%.4f", - cat_results["rsa_spearman_r"], cat_results["mantel"]["rsa_p_value"]) - logger.info("2. Hallmarks per-module alignment: mean r=%.3f, %d/%d positive", - hallmark_results["per_module_alignment_mean"], - hallmark_results["n_positive_alignment"], - hallmark_results["n_hallmarks"]) - logger.info("3. Hallmarks kNN overlap: %.3f (null=%.3f, p=%.4f)", - hallmark_results["knn_overlap_mean"], - hallmark_results["knn_overlap_null_mean"], - hallmark_results["knn_overlap_p"]) - - -if __name__ == "__main__": - main() +"""Extended analysis for Experiment 2 — Module-Text Alignment. + +Three analyses: +1. Reactome category-level RSA (aggregate 491 pathways into ~16 top-level categories) +2. Reactome with official descriptions (via descriptions JSON) +3. Hallmarks deep dive (per-module alignment scores, heatmap) + +Usage (run from the BioMod repository root): + python BioMod/interpretability/module_text_alignment/module_text_alignment_extended.py \ + --module_embedding_path outputs/interpretability/geometry/pretrain_subset/module_embeddings.npy \ + --module_json model_dependencies/module_data_all/merged_modules.json \ + --sentence_model pritamdeka/S-PubMedBert-MS-MARCO \ + --output_dir outputs/interpretability/text_alignment/pretrain_subset_v2_extended +""" +from __future__ import annotations + +import argparse +import json +import logging +import sys +from collections import Counter, defaultdict +from pathlib import Path +from typing import Dict, List, Optional, Tuple + +import numpy as np +import pandas as pd +from scipy.spatial.distance import pdist, squareform +from scipy.stats import spearmanr +from sklearn.linear_model import RidgeCV +from sklearn.metrics.pairwise import cosine_similarity +from sklearn.preprocessing import StandardScaler + +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt + +HERE = Path(__file__).resolve().parent +INTERP_DIR = HERE.parent +PERFORMER_MODULE = INTERP_DIR.parent +REPO_ROOT = PERFORMER_MODULE.parent +for p in (str(INTERP_DIR), str(PERFORMER_MODULE), str(REPO_ROOT)): + if p not in sys.path: + sys.path.insert(0, p) + +from interpretability.utils import load_module_names, parse_module_family +from module_text_alignment.module_text_alignment import ( + build_text_descriptions, + compute_cka, + compute_rsa, + encode_texts, + mantel_test, + parse_pathway_text, +) + +logger = logging.getLogger(__name__) + + +# ============================================================ +# Reactome top-level category mapping +# ============================================================ + +REACTOME_CATEGORIES = { + "Cell Cycle": ["cell cycle", "mitotic", "mitosis", "meiosis", "g1/s", "g2/m", + "s phase", "m phase", "checkpoint", "cdc", "apc/c", "cyclin", + "condensation of", "centrosome", "kinetochore", "spindle", + "cohes", "pre-replicative", "separation of sister", "aurora"], + "DNA Repair": ["dna repair", "base excision", "nucleotide excision", + "mismatch repair", "homologous recombination", "non-homologous", + "double-strand", "single-strand", "fanconi", "dna damage"], + "DNA Replication": ["dna replication", "replication", "okazaki", "telomere"], + "Signal Transduction": ["signaling", "signal transduction", "mapk", "ras", + "raf", "mek", "erk", "pi3k", "akt", "mtor", "wnt", + "notch", "hedgehog", "tgf-beta", "bmp", "fgfr", + "egfr", "vegf", "pdgf", "igf", "insulin", "erbb", + "met receptor", "receptor tyrosine", "gpcr", + "g alpha", "rho gtpase", "rac1", "cdc42", + "ca2+ pathway", "nfkb", "nf-kb", "nmda", + "calcium", "pkc", "plc", "dag", "pip"], + "Immune System": ["immune", "innate", "adaptive", "interferon", "interleukin", + "cytokine", "complement", "toll", "antigen", "mhc", "tcr", + "bcr", "fcr", "neutrophil", "phagocyt", "inflamm", + "antimicrobial", "antiviral", "ifn-stimulated", + "c-type lectin", "nod-like", "rig-i"], + "Metabolism": ["metabolism", "metabolic", "glycolysis", "gluconeogenesis", + "tca cycle", "citric acid", "oxidative phosphorylation", + "fatty acid", "lipid", "cholesterol", "bile", "amino acid", + "nucleotide", "purine", "pyrimidine", "pentose", "urea", + "biological oxidation", "glycosylation", "biosynthesis", + "synthesis of", "respiratory electron"], + "Gene Expression": ["transcription", "translation", "mrna", "rrna", "trna", + "splicing", "polyadenylation", "ribosom", + "eukaryotic translation", "nonsense-mediated", + "deadenylation", "rna polymerase", "processing of capped"], + "Protein Homeostasis": ["ubiquitin", "proteasome", "sumo", "nedd", + "autophagy", "chaperone", "unfolded protein", + "protein folding", "hsp", "degradation of", + "deubiquitin"], + "Transport": ["transport", "vesicle", "endocyt", "exocyt", "golgi", + "er to golgi", "membrane trafficking", "clathrin", "copi", + "copii", "ion channel", "slc-mediated", "abc transporter", + "nuclear pore", "nuclear import", "nuclear export", "cargo"], + "Apoptosis": ["apoptosis", "apoptotic", "programmed cell death", "caspase", + "death receptor", "bcl-2"], + "Cell-Cell Communication": ["cell junction", "gap junction", "tight junction", + "adherens", "cell-cell", "extracellular matrix", + "integrin", "synapse", "neurotransmitter", + "collagen", "laminin", "elastic fibre", + "anchoring"], + "Chromatin & Epigenetics": ["chromatin", "histone", "methylation", + "acetylation", "deacetylation", "epigenetic", + "polycomb", "hox"], + "Hemostasis": ["hemostasis", "platelet", "coagulation", "fibrin", "thrombin"], + "Developmental Biology": ["developmental", "axon", "neural", "muscle", + "myogenesis", "keratinization", "differentiation", + "cardiac conduction"], + "Disease": ["disease", "hiv", "influenza", "sars", "hcmv", "herpes", + "hepatitis", "cancer", "oncogene", "tumor", "amyloid"], + "Cellular Responses": ["cellular response", "oxidative stress", "hypoxia", + "heat shock", "senescence", "ferroptosis", + "detoxification"], +} + + +def categorize_reactome(pathway_name: str) -> str: + name_lower = pathway_name.lower() + for cat, keywords in REACTOME_CATEGORIES.items(): + for kw in keywords: + if kw in name_lower: + return cat + return "Other" + + +# ============================================================ +# Analysis 1: Reactome category-level RSA +# ============================================================ + +def reactome_category_rsa( + module_emb: np.ndarray, + text_emb: np.ndarray, + names: List[str], + texts: List[str], + output_dir: Path, + sentence_model: str, + device: str, + n_perm: int = 1000, + seed: int = 42, +) -> Dict: + """Aggregate Reactome modules by top-level category, then do RSA.""" + logger.info("=" * 50) + logger.info("Analysis 1: Reactome category-level RSA") + logger.info("=" * 50) + + families = [parse_module_family(n) for n in names] + reactome_mask = np.array([f == "Reactome" for f in families]) + reactome_idx = np.where(reactome_mask)[0] + + reactome_names = [names[i] for i in reactome_idx] + reactome_texts = [texts[i] for i in reactome_idx] + reactome_module_emb = module_emb[reactome_idx] + + cats = [categorize_reactome(t) for t in reactome_texts] + cat_counts = Counter(cats) + logger.info("Reactome category distribution:") + for cat, count in sorted(cat_counts.items(), key=lambda x: -x[1]): + logger.info(" %s: %d", cat, count) + + # Aggregate: mean embedding per category + unique_cats = sorted(set(c for c in cats if c != "Other")) + n_cats = len(unique_cats) + logger.info("Categories (excl. Other): %d", n_cats) + + cat_module_emb = np.zeros((n_cats, module_emb.shape[1])) + for i, cat in enumerate(unique_cats): + mask = np.array([c == cat for c in cats]) + cat_module_emb[i] = reactome_module_emb[mask].mean(axis=0) + + # Encode category names as text + cat_text_emb = encode_texts(unique_cats, model_name=sentence_model, device=device) + + # RSA on category-level + sp_r, sp_p, pe_r, pe_p = compute_rsa(cat_module_emb, cat_text_emb, metric="cosine") + logger.info("Category-level RSA: Spearman r=%.4f (p=%.4f), Pearson r=%.4f", + sp_r, sp_p, pe_r) + + mantel = mantel_test(cat_module_emb, cat_text_emb, metric="cosine", + n_perm=n_perm, seed=seed) + logger.info("Mantel test: observed=%.4f, null=%.4f +/- %.4f, p=%.4f, z=%.1f", + mantel["rsa_spearman_observed"], mantel["rsa_null_mean"], + mantel["rsa_null_std"], mantel["rsa_p_value"], + mantel["rsa_effect_size_z"]) + + cka = compute_cka(cat_module_emb, cat_text_emb) + logger.info("Category-level CKA: %.4f", cka) + + # Visualize: distance heatmaps + fig, axes = plt.subplots(1, 2, figsize=(14, 6)) + + dist_module = squareform(pdist(cat_module_emb, metric="cosine")) + dist_text = squareform(pdist(cat_text_emb, metric="cosine")) + + im0 = axes[0].imshow(dist_module, cmap="viridis", aspect="auto") + axes[0].set_xticks(range(n_cats)) + axes[0].set_xticklabels(unique_cats, rotation=45, ha="right", fontsize=7) + axes[0].set_yticks(range(n_cats)) + axes[0].set_yticklabels(unique_cats, fontsize=7) + axes[0].set_title("Module Embedding Distance (cosine)") + plt.colorbar(im0, ax=axes[0], fraction=0.046) + + im1 = axes[1].imshow(dist_text, cmap="viridis", aspect="auto") + axes[1].set_xticks(range(n_cats)) + axes[1].set_xticklabels(unique_cats, rotation=45, ha="right", fontsize=7) + axes[1].set_yticks(range(n_cats)) + axes[1].set_yticklabels(unique_cats, fontsize=7) + axes[1].set_title("Text Embedding Distance (cosine)") + plt.colorbar(im1, ax=axes[1], fraction=0.046) + + plt.suptitle(f"Reactome Category-Level: RSA r={sp_r:.3f}, p={mantel['rsa_p_value']:.4f}") + plt.tight_layout() + plt.savefig(output_dir / "reactome_category_heatmaps.png", dpi=150, bbox_inches="tight") + plt.close() + + results = { + "n_categories": n_cats, + "category_counts": dict(cat_counts), + "rsa_spearman_r": sp_r, + "rsa_spearman_p": sp_p, + "rsa_pearson_r": pe_r, + "mantel": mantel, + "cka": cka, + "categories": unique_cats, + } + return results + + +# ============================================================ +# Analysis 2: Reactome with descriptions +# ============================================================ + +def reactome_with_descriptions( + module_emb: np.ndarray, + names: List[str], + texts: List[str], + descriptions_file: str, + output_dir: Path, + sentence_model: str, + device: str, + n_perm: int = 1000, + seed: int = 42, +) -> Optional[Dict]: + """Re-run RSA using extended descriptions for Reactome pathways.""" + if not descriptions_file or not Path(descriptions_file).exists(): + logger.info("No descriptions file provided, skipping Analysis 2.") + return None + + logger.info("=" * 50) + logger.info("Analysis 2: Reactome with official descriptions") + logger.info("=" * 50) + + with open(descriptions_file, "r", encoding="utf-8") as f: + descriptions = json.load(f) + + families = [parse_module_family(n) for n in names] + reactome_mask = np.array([f == "Reactome" for f in families]) + reactome_idx = np.where(reactome_mask)[0] + + # Build enriched texts + enriched_texts = [] + valid_idx = [] + for i in reactome_idx: + pname = names[i] + base_text = texts[i] + if pname in descriptions and descriptions[pname]: + enriched_texts.append(f"{base_text}: {descriptions[pname]}") + valid_idx.append(i) + else: + enriched_texts.append(base_text) + valid_idx.append(i) + + logger.info("Reactome modules with descriptions: %d / %d", + sum(1 for i in reactome_idx if names[i] in descriptions), + len(reactome_idx)) + + sub_module_emb = module_emb[valid_idx] + text_emb_enriched = encode_texts(enriched_texts, model_name=sentence_model, device=device) + + sp_r, sp_p, pe_r, pe_p = compute_rsa(sub_module_emb, text_emb_enriched, metric="cosine") + logger.info("RSA with descriptions: Spearman r=%.4f (p=%.2e)", sp_r, sp_p) + + mantel = mantel_test(sub_module_emb, text_emb_enriched, metric="cosine", + n_perm=n_perm, seed=seed) + logger.info("Mantel: observed=%.4f, null=%.4f +/- %.4f, p=%.4f, z=%.1f", + mantel["rsa_spearman_observed"], mantel["rsa_null_mean"], + mantel["rsa_null_std"], mantel["rsa_p_value"], + mantel["rsa_effect_size_z"]) + + return { + "rsa_spearman_r": sp_r, + "rsa_spearman_p": sp_p, + "mantel": mantel, + } + + +# ============================================================ +# Analysis 3: Hallmarks deep dive +# ============================================================ + +def hallmarks_deep_dive( + module_emb: np.ndarray, + text_emb: np.ndarray, + names: List[str], + texts: List[str], + output_dir: Path, + seed: int = 42, +) -> Dict: + """Per-module alignment analysis for the 50 Hallmarks.""" + logger.info("=" * 50) + logger.info("Analysis 3: Hallmarks deep dive") + logger.info("=" * 50) + + families = [parse_module_family(n) for n in names] + hallmark_mask = np.array([f == "Hallmarks" for f in families]) + hallmark_idx = np.where(hallmark_mask)[0] + + h_module = module_emb[hallmark_idx] + h_text = text_emb[hallmark_idx] + h_names = [names[i] for i in hallmark_idx] + h_texts = [texts[i] for i in hallmark_idx] + n_h = len(hallmark_idx) + + logger.info("Hallmarks: %d modules", n_h) + + # Per-module alignment: for each hallmark, compute cosine sim between + # its module embedding's nearest neighbors and its text embedding's nearest neighbors + # (neighborhood overlap) + module_sim_matrix = cosine_similarity(h_module) + text_sim_matrix = cosine_similarity(h_text) + + # Per-module: correlation between its similarity profile in module space vs text space + per_module_corr = np.zeros(n_h) + for i in range(n_h): + mask = np.ones(n_h, dtype=bool) + mask[i] = False + r, _ = spearmanr(module_sim_matrix[i, mask], text_sim_matrix[i, mask]) + per_module_corr[i] = r + + # Sort by alignment quality + sort_idx = np.argsort(-per_module_corr) + logger.info("\nTop-10 best aligned Hallmarks:") + for rank, i in enumerate(sort_idx[:10]): + logger.info(" %2d. %s (r=%.3f)", rank+1, h_texts[i], per_module_corr[i]) + + logger.info("\nBottom-10 worst aligned Hallmarks:") + for rank, i in enumerate(sort_idx[-10:]): + logger.info(" %2d. %s (r=%.3f)", rank+1, h_texts[i], per_module_corr[i]) + + # Save per-module scores + df = pd.DataFrame({ + "module_name": h_names, + "text": h_texts, + "alignment_r": per_module_corr, + }).sort_values("alignment_r", ascending=False) + df.to_csv(output_dir / "hallmarks_per_module_alignment.csv", index=False) + + # Heatmap: module similarity vs text similarity + fig, axes = plt.subplots(1, 3, figsize=(18, 6)) + + im0 = axes[0].imshow(module_sim_matrix[sort_idx][:, sort_idx], cmap="RdBu_r", + vmin=-0.5, vmax=0.5, aspect="auto") + axes[0].set_title("Module Embedding Similarity") + axes[0].set_xticks(range(n_h)) + axes[0].set_xticklabels([h_texts[i][:15] for i in sort_idx], rotation=90, fontsize=5) + axes[0].set_yticks(range(n_h)) + axes[0].set_yticklabels([h_texts[i][:15] for i in sort_idx], fontsize=5) + plt.colorbar(im0, ax=axes[0], fraction=0.046) + + im1 = axes[1].imshow(text_sim_matrix[sort_idx][:, sort_idx], cmap="RdBu_r", + vmin=-0.5, vmax=0.5, aspect="auto") + axes[1].set_title("Text Embedding Similarity") + axes[1].set_xticks(range(n_h)) + axes[1].set_xticklabels([h_texts[i][:15] for i in sort_idx], rotation=90, fontsize=5) + axes[1].set_yticks([]) + plt.colorbar(im1, ax=axes[1], fraction=0.046) + + # Bar chart of per-module alignment + colors = ["steelblue" if r > 0 else "salmon" for r in per_module_corr[sort_idx]] + axes[2].barh(range(n_h), per_module_corr[sort_idx], color=colors) + axes[2].set_yticks(range(n_h)) + axes[2].set_yticklabels([h_texts[i][:20] for i in sort_idx], fontsize=5) + axes[2].set_xlabel("Spearman r (neighborhood alignment)") + axes[2].set_title("Per-Hallmark Alignment Score") + axes[2].axvline(0, color="black", linewidth=0.5) + + plt.tight_layout() + plt.savefig(output_dir / "hallmarks_deep_dive.png", dpi=150, bbox_inches="tight") + plt.close() + logger.info("Saved hallmarks_deep_dive.png") + + # kNN overlap analysis + k = 5 + module_knn = np.argsort(-module_sim_matrix, axis=1)[:, 1:k+1] + text_knn = np.argsort(-text_sim_matrix, axis=1)[:, 1:k+1] + knn_overlap = np.zeros(n_h) + for i in range(n_h): + overlap = len(set(module_knn[i]) & set(text_knn[i])) + knn_overlap[i] = overlap / k + + logger.info("\nkNN overlap (k=%d): mean=%.3f, median=%.3f", + k, knn_overlap.mean(), np.median(knn_overlap)) + + # Permutation test for kNN overlap + rng = np.random.default_rng(seed) + n_perm = 1000 + null_overlaps = np.zeros(n_perm) + for p in range(n_perm): + perm = rng.permutation(n_h) + text_knn_perm = text_knn[perm] + overlaps_perm = np.zeros(n_h) + for i in range(n_h): + overlaps_perm[i] = len(set(module_knn[i]) & set(text_knn_perm[i])) / k + null_overlaps[p] = overlaps_perm.mean() + + knn_p = float(np.mean(null_overlaps >= knn_overlap.mean())) + logger.info("kNN overlap permutation test: observed=%.3f, null=%.3f +/- %.3f, p=%.4f", + knn_overlap.mean(), null_overlaps.mean(), null_overlaps.std(), knn_p) + + results = { + "n_hallmarks": n_h, + "per_module_alignment_mean": float(per_module_corr.mean()), + "per_module_alignment_median": float(np.median(per_module_corr)), + "per_module_alignment_std": float(per_module_corr.std()), + "n_positive_alignment": int((per_module_corr > 0).sum()), + "n_negative_alignment": int((per_module_corr < 0).sum()), + "knn_k": k, + "knn_overlap_mean": float(knn_overlap.mean()), + "knn_overlap_null_mean": float(null_overlaps.mean()), + "knn_overlap_p": knn_p, + } + return results + + +# ============================================================ +# Main +# ============================================================ + +def parse_args(): + parser = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--module_embedding_path", type=str, required=True) + parser.add_argument("--module_json", type=str, + default=str(REPO_ROOT / "model_dependencies" / "module_data_all" / "merged_modules.json")) + parser.add_argument("--descriptions_file", type=str, default=None, + help="JSON with Reactome summaryText descriptions.") + parser.add_argument("--sentence_model", type=str, + default="pritamdeka/S-PubMedBert-MS-MARCO") + parser.add_argument("--device", type=str, default="cpu") + parser.add_argument("--n_permutations", type=int, default=1000) + parser.add_argument("--seed", type=int, default=42) + parser.add_argument("--output_dir", type=str, required=True) + return parser.parse_args() + + +def main(): + args = parse_args() + output_dir = Path(args.output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(message)s", + handlers=[ + logging.StreamHandler(), + logging.FileHandler(output_dir / "run_extended.log", mode="w"), + ], + ) + + # Load data + module_names = load_module_names(args.module_json) + all_embeddings = np.load(args.module_embedding_path) + indices, names, texts = build_text_descriptions(module_names) + module_emb = all_embeddings[indices] + + # Load or compute text embeddings + text_emb_path = output_dir.parent / "pretrain_subset_v2" / "text_embeddings.npy" + if text_emb_path.exists(): + logger.info("Loading cached text embeddings from %s", text_emb_path) + text_emb = np.load(text_emb_path) + else: + text_emb = encode_texts(texts, model_name=args.sentence_model, device=args.device) + + all_results = {} + + # Analysis 1: Reactome category-level + cat_results = reactome_category_rsa( + module_emb, text_emb, names, texts, output_dir, + sentence_model=args.sentence_model, device=args.device, + n_perm=args.n_permutations, seed=args.seed, + ) + all_results["reactome_category_rsa"] = cat_results + + # Analysis 2: Reactome with descriptions + desc_results = reactome_with_descriptions( + module_emb, names, texts, args.descriptions_file, output_dir, + sentence_model=args.sentence_model, device=args.device, + n_perm=args.n_permutations, seed=args.seed, + ) + if desc_results: + all_results["reactome_with_descriptions"] = desc_results + + # Analysis 3: Hallmarks deep dive + hallmark_results = hallmarks_deep_dive( + module_emb, text_emb, names, texts, output_dir, seed=args.seed, + ) + all_results["hallmarks_deep_dive"] = hallmark_results + + # Save all results + with open(output_dir / "extended_metrics.json", "w") as f: + json.dump(all_results, f, indent=2, default=str) + logger.info("Saved extended_metrics.json") + + # Summary + logger.info("=" * 60) + logger.info("EXTENDED ANALYSIS SUMMARY") + logger.info("=" * 60) + logger.info("1. Reactome category-level RSA: r=%.4f, p=%.4f", + cat_results["rsa_spearman_r"], cat_results["mantel"]["rsa_p_value"]) + logger.info("2. Hallmarks per-module alignment: mean r=%.3f, %d/%d positive", + hallmark_results["per_module_alignment_mean"], + hallmark_results["n_positive_alignment"], + hallmark_results["n_hallmarks"]) + logger.info("3. Hallmarks kNN overlap: %.3f (null=%.3f, p=%.4f)", + hallmark_results["knn_overlap_mean"], + hallmark_results["knn_overlap_null_mean"], + hallmark_results["knn_overlap_p"]) + + +if __name__ == "__main__": + main() diff --git a/BioMod/interpretability/module_text_alignment/module_text_alignment_publication.py b/BioMod/interpretability/module_text_alignment/module_text_alignment_publication.py index 1ddb27bc4d79add6777b533a28de62a51819c651..470064c2f4a744245810a0ef0bb539a48470ca39 100644 --- a/BioMod/interpretability/module_text_alignment/module_text_alignment_publication.py +++ b/BioMod/interpretability/module_text_alignment/module_text_alignment_publication.py @@ -1,1277 +1,1277 @@ -"""Publication-quality analysis for Experiment 2 — Module-Text Alignment. - -Produces 6 main figures + comprehensive metrics with bootstrap CIs for paper. -Supports multi-model comparison and cross-dataset validation. - -Usage: - python module_text_alignment_publication.py \ - --module_embedding_path ../results/geometry/pretrain_subset/module_embeddings.npy \ - --module_embedding_path2 ../results/geometry/pancreas_scib/module_embeddings.npy \ - --module_json ../../module_data_all/merged_modules.json \ - --sentence_model ./S-PubMedBert-MS-MARCO \ - --sentence_models_compare "sentence-transformers/all-mpnet-base-v2" \ - --descriptions_file ./reactome_descriptions.json \ - --n_permutations 1000 \ - --n_bootstrap 2000 \ - --output_dir ../results/text_alignment/publication -""" -from __future__ import annotations - -import argparse -import json -import logging -import sys -from collections import Counter, defaultdict -from pathlib import Path -from typing import Dict, List, Optional, Tuple - -import numpy as np -import pandas as pd -from scipy.cluster.hierarchy import dendrogram, fcluster, linkage -from scipy.spatial.distance import pdist, squareform -from scipy.stats import kruskal, spearmanr -from sklearn.metrics.pairwise import cosine_similarity - -import matplotlib -matplotlib.use("Agg") -import matplotlib.pyplot as plt -from matplotlib.patches import Patch as LegendHandle -from matplotlib.lines import Line2D - -HERE = Path(__file__).resolve().parent -INTERP_DIR = HERE.parent -PERFORMER_MODULE = INTERP_DIR.parent -REPO_ROOT = PERFORMER_MODULE.parent -for p in (str(HERE), str(INTERP_DIR), str(PERFORMER_MODULE), str(REPO_ROOT)): - if p not in sys.path: - sys.path.insert(0, p) - -from interpretability.utils import ( - HALLMARK_PROCESS, - load_module_names, - parse_hallmark_process, - parse_module_family, -) -from module_text_alignment.module_text_alignment import ( - build_text_descriptions, - compute_cka, - compute_rsa, - encode_texts, - mantel_test, - permutation_test_cka, -) -from module_text_alignment.module_text_alignment_extended import ( - REACTOME_CATEGORIES, - categorize_reactome, -) - -logger = logging.getLogger(__name__) - -# ============================================================ -# Color palette for MSigDB hallmark process categories -# ============================================================ - -PROCESS_COLORS = { - "immune": "#2196F3", - "metabolic": "#4CAF50", - "signaling": "#FF9800", - "proliferation": "#E91E63", - "development": "#9C27B0", - "dna_damage": "#F44336", - "pathway": "#607D8B", - "cellular_component": "#795548", -} - -PROCESS_LABELS = { - "immune": "Immune", - "metabolic": "Metabolic", - "signaling": "Signaling", - "proliferation": "Proliferation", - "development": "Development", - "dna_damage": "DNA Damage", - "pathway": "Pathway", - "cellular_component": "Cellular Component", -} - -# Reactome super-categories (merge related categories for granularity sweep) -REACTOME_SUPER_CATEGORIES = { - "Information Processing": ["Gene Expression", "DNA Replication", "DNA Repair", - "Chromatin & Epigenetics"], - "Signal & Communication": ["Signal Transduction", "Cell-Cell Communication"], - "Cell Fate": ["Cell Cycle", "Apoptosis", "Cellular Responses"], - "Metabolism & Transport": ["Metabolism", "Transport"], - "Immune & Defense": ["Immune System", "Hemostasis", "Disease"], - "Development": ["Developmental Biology", "Protein Homeostasis"], -} - - -# ============================================================ -# Utility functions -# ============================================================ - -def savefig(fig, path_stem: str, dpi: int = 300): - """Save figure as both PDF and PNG.""" - fig.savefig(f"{path_stem}.pdf", dpi=dpi, bbox_inches="tight") - fig.savefig(f"{path_stem}.png", dpi=dpi, bbox_inches="tight") - plt.close(fig) - logger.info("Saved: %s.{pdf,png}", path_stem) - - -def adaptive_vlim(matrix: np.ndarray, percentile: int = 98, symmetric: bool = True): - """Compute adaptive vmin/vmax from data percentiles.""" - vals = matrix[np.triu_indices_from(matrix, k=1)] - if symmetric: - vlim = np.percentile(np.abs(vals), percentile) - return -vlim, vlim - else: - return np.percentile(vals, [100 - percentile, percentile]) - - -def bootstrap_ci( - values: np.ndarray, - stat_fn=np.mean, - n_boot: int = 2000, - ci: float = 0.95, - seed: int = 42, -) -> Tuple[float, float, float]: - """Bootstrap confidence interval. Returns (estimate, ci_low, ci_high).""" - rng = np.random.default_rng(seed) - n = len(values) - boot_stats = np.zeros(n_boot) - for i in range(n_boot): - idx = rng.integers(0, n, size=n) - boot_stats[i] = stat_fn(values[idx]) - alpha = (1 - ci) / 2 - return float(stat_fn(values)), float(np.percentile(boot_stats, alpha * 100)), \ - float(np.percentile(boot_stats, (1 - alpha) * 100)) - - -def bootstrap_rsa( - module_emb: np.ndarray, - text_emb: np.ndarray, - n_boot: int = 2000, - seed: int = 42, -) -> Tuple[float, float, float]: - """Bootstrap CI for RSA Spearman r by resampling modules.""" - rng = np.random.default_rng(seed) - n = module_emb.shape[0] - boot_rs = np.zeros(n_boot) - for i in range(n_boot): - idx = rng.choice(n, size=n, replace=True) - dist_p = pdist(module_emb[idx], metric="cosine") - dist_t = pdist(text_emb[idx], metric="cosine") - boot_rs[i], _ = spearmanr(dist_p, dist_t) - observed, _ = spearmanr(pdist(module_emb, "cosine"), pdist(text_emb, "cosine")) - return float(observed), float(np.percentile(boot_rs, 2.5)), \ - float(np.percentile(boot_rs, 97.5)) - - -def get_module_sizes(module_json: str) -> Dict[str, int]: - """Get number of genes per module from merged_modules.json.""" - with open(module_json, "r", encoding="utf-8") as f: - modules = json.load(f) - return {name: len(genes) for name, genes in modules.items()} - - -# ============================================================ -# CLI -# ============================================================ - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter, - ) - parser.add_argument("--module_embedding_path", type=str, required=True, - help="Primary module_embeddings.npy (e.g. pretrain_subset)") - parser.add_argument("--module_embedding_path2", type=str, default=None, - help="Second dataset module_embeddings.npy for cross-dataset comparison") - parser.add_argument("--module_json", type=str, - default=str(PERFORMER_MODULE / "module_data_all" / "merged_modules.json")) - parser.add_argument("--descriptions_file", type=str, default=None) - parser.add_argument("--sentence_model", type=str, - default="pritamdeka/S-PubMedBert-MS-MARCO") - parser.add_argument("--sentence_models_compare", type=str, default=None, - help="Comma-separated additional models for comparison") - parser.add_argument("--device", type=str, default="cpu") - parser.add_argument("--n_permutations", type=int, default=1000) - parser.add_argument("--n_bootstrap", type=int, default=2000) - parser.add_argument("--seed", type=int, default=42) - parser.add_argument("--output_dir", type=str, required=True) - return parser.parse_args() - - -# ============================================================ -# Figure 1: RSA Overview (hexbin + null distribution) -# ============================================================ - -def fig1_rsa_overview( - module_emb: np.ndarray, - text_emb: np.ndarray, - mantel_results: Dict, - output_dir: Path, - metric: str = "cosine", - n_bootstrap: int = 2000, - seed: int = 42, -) -> Dict: - """RSA overview: hexbin density + Mantel null distribution.""" - dist_module = pdist(module_emb, metric=metric) - dist_text = pdist(text_emb, metric=metric) - - observed_r = mantel_results["rsa_spearman_observed"] - null_mean = mantel_results["rsa_null_mean"] - null_std = mantel_results["rsa_null_std"] - p_value = mantel_results["rsa_p_value"] - z_score = mantel_results["rsa_effect_size_z"] - - # Bootstrap CI for RSA - rsa_obs, rsa_ci_lo, rsa_ci_hi = bootstrap_rsa( - module_emb, text_emb, n_boot=n_bootstrap, seed=seed - ) - - fig, axes = plt.subplots(1, 2, figsize=(12, 5)) - - # Panel A: Hexbin density - ax = axes[0] - hb = ax.hexbin(dist_text, dist_module, gridsize=40, cmap="YlOrRd", - mincnt=1, linewidths=0.2) - cb = plt.colorbar(hb, ax=ax, fraction=0.046, pad=0.02) - cb.set_label("Count", fontsize=9) - - # Trend line - z = np.polyfit(dist_text, dist_module, 1) - p_line = np.poly1d(z) - x_range = np.linspace(dist_text.min(), dist_text.max(), 100) - ax.plot(x_range, p_line(x_range), "k-", linewidth=2, alpha=0.8) - - ax.set_xlabel("Text pairwise distance (cosine)", fontsize=10) - ax.set_ylabel("Module embedding pairwise distance (cosine)", fontsize=10) - ax.set_title(f"A. RSA: r = {observed_r:.3f} [{rsa_ci_lo:.3f}, {rsa_ci_hi:.3f}]", - fontsize=11, fontweight="bold") - ax.text(0.05, 0.95, f"n = {module_emb.shape[0]} modules\n" - f"pairs = {len(dist_module):,}", - transform=ax.transAxes, fontsize=8, va="top", - bbox=dict(boxstyle="round", facecolor="white", alpha=0.8)) - - # Panel B: Null distribution - ax = axes[1] - from scipy.stats import norm as norm_dist - x = np.linspace(null_mean - 4.5 * null_std, - max(observed_r + 2 * null_std, null_mean + 4.5 * null_std), 300) - y = norm_dist.pdf(x, null_mean, null_std) - ax.fill_between(x, y, alpha=0.25, color="gray") - ax.plot(x, y, color="gray", linewidth=1.5, label="Null distribution") - - # Shade p-value region - x_tail = x[x >= observed_r] - y_tail = norm_dist.pdf(x_tail, null_mean, null_std) - ax.fill_between(x_tail, y_tail, alpha=0.4, color="red", label=f"p = {p_value:.4f}") - - ax.axvline(observed_r, color="red", linewidth=2.5, linestyle="-", - label=f"Observed r = {observed_r:.4f}") - ax.axvline(null_mean, color="gray", linewidth=1, linestyle="--", alpha=0.7) - - ax.set_xlabel("Spearman r (distance matrix correlation)", fontsize=10) - ax.set_ylabel("Density", fontsize=10) - ax.set_title(f"B. Mantel Test: z = {z_score:.1f}σ, p = {p_value:.4f}", - fontsize=11, fontweight="bold") - ax.legend(fontsize=9, loc="upper left") - - plt.tight_layout() - savefig(fig, str(output_dir / "fig1_rsa_overview")) - - return { - "rsa_r": rsa_obs, - "rsa_ci_95": [rsa_ci_lo, rsa_ci_hi], - "mantel_p": p_value, - "mantel_z": z_score, - } - - -# ============================================================ -# Figure 2: Hallmarks Similarity Matrices with Dendrograms -# ============================================================ - -def _rank_transform_matrix(sim_matrix: np.ndarray) -> np.ndarray: - """Rank-transform off-diagonal elements to [0,1] for visualization.""" - from scipy.stats import rankdata - n = sim_matrix.shape[0] - result = np.zeros_like(sim_matrix) - triu_idx = np.triu_indices(n, k=1) - vals = sim_matrix[triu_idx] - ranks = rankdata(vals) / len(vals) - result[triu_idx] = ranks - result[triu_idx[1], triu_idx[0]] = ranks - np.fill_diagonal(result, 1.0) - return result - - -def fig2_hallmarks_heatmaps( - module_emb: np.ndarray, - text_emb: np.ndarray, - names: List[str], - texts: List[str], - output_dir: Path, -) -> Dict: - """Hallmarks 50x50 similarity matrices with rank-transform and clustering.""" - families = [parse_module_family(n) for n in names] - hallmark_mask = np.array([f == "Hallmarks" for f in families]) - hallmark_idx = np.where(hallmark_mask)[0] - - h_module = module_emb[hallmark_idx] - h_text = text_emb[hallmark_idx] - h_names = [names[i] for i in hallmark_idx] - h_texts = [texts[i] for i in hallmark_idx] - n_h = len(hallmark_idx) - - h_processes = [] - for n in h_names: - proc = parse_hallmark_process(n) - h_processes.append(proc if proc else "pathway") - - module_sim = cosine_similarity(h_module) - text_sim = cosine_similarity(h_text) - - linkage_mat = linkage(pdist(h_module, "cosine"), method="ward") - dendro = dendrogram(linkage_mat, no_plot=True) - order = dendro["leaves"] - - module_sim_ord = module_sim[order][:, order] - text_sim_ord = text_sim[order][:, order] - texts_ord = [h_texts[i] for i in order] - procs_ord = [h_processes[i] for i in order] - - # Rank-transform to spread narrow cosine similarity ranges - module_rank_ord = _rank_transform_matrix(module_sim_ord) - text_rank_ord = _rank_transform_matrix(text_sim_ord) - diff_rank = module_rank_ord - text_rank_ord - - # Adaptive limit for difference panel - d_vals = diff_rank[np.triu_indices(n_h, k=1)] - vlim_d = float(np.percentile(np.abs(d_vals), 95)) - - # Figure - fig = plt.figure(figsize=(20, 8.5)) - gs = fig.add_gridspec(2, 3, height_ratios=[0.08, 1], hspace=0.02, wspace=0.3) - - # Dendrogram - ax_dend = fig.add_subplot(gs[0, :]) - dendrogram(linkage_mat, ax=ax_dend, no_labels=True, - color_threshold=0, above_threshold_color="gray") - ax_dend.set_xticks([]) - ax_dend.set_yticks([]) - ax_dend.spines[:].set_visible(False) - - for i, proc in enumerate(procs_ord): - color = PROCESS_COLORS.get(proc, "#999999") - ax_dend.axvspan(i * 10 + 5 - 4, i * 10 + 5 + 4, ymin=-0.2, ymax=-0.02, - color=color, clip_on=False) - - # Panel A: Module similarity (rank-transformed) - ax0 = fig.add_subplot(gs[1, 0]) - im0 = ax0.imshow(module_rank_ord, cmap="YlOrRd", vmin=0, vmax=1, - aspect="equal", interpolation="nearest") - ax0.set_title("A. Module Embedding Similarity\n(rank-transformed)", - fontsize=10, fontweight="bold") - ax0.set_xticks(range(n_h)) - ax0.set_xticklabels(texts_ord, rotation=90, fontsize=5.5, ha="center") - ax0.set_yticks(range(n_h)) - ax0.set_yticklabels(texts_ord, fontsize=5.5) - cb0 = plt.colorbar(im0, ax=ax0, fraction=0.046, pad=0.02) - cb0.set_label("Rank percentile", fontsize=8) - - # Panel B: Text similarity (rank-transformed) - ax1 = fig.add_subplot(gs[1, 1]) - im1 = ax1.imshow(text_rank_ord, cmap="YlOrRd", vmin=0, vmax=1, - aspect="equal", interpolation="nearest") - ax1.set_title("B. Text Embedding Similarity\n(rank-transformed)", - fontsize=10, fontweight="bold") - ax1.set_xticks(range(n_h)) - ax1.set_xticklabels(texts_ord, rotation=90, fontsize=5.5, ha="center") - ax1.set_yticks([]) - cb1 = plt.colorbar(im1, ax=ax1, fraction=0.046, pad=0.02) - cb1.set_label("Rank percentile", fontsize=8) - - # Panel C: Difference - ax2 = fig.add_subplot(gs[1, 2]) - im2 = ax2.imshow(diff_rank, cmap="PiYG", vmin=-vlim_d, vmax=vlim_d, - aspect="equal", interpolation="nearest") - ax2.set_title("C. Difference (Module − Text)\n(rank space)", - fontsize=10, fontweight="bold") - ax2.set_xticks(range(n_h)) - ax2.set_xticklabels(texts_ord, rotation=90, fontsize=5.5, ha="center") - ax2.set_yticks([]) - cb2 = plt.colorbar(im2, ax=ax2, fraction=0.046, pad=0.02) - cb2.set_label("Rank difference", fontsize=8) - - legend_elements = [LegendHandle(facecolor=PROCESS_COLORS[k], label=PROCESS_LABELS[k]) - for k in PROCESS_COLORS] - fig.legend(handles=legend_elements, loc="lower center", ncol=4, - fontsize=9, frameon=True, bbox_to_anchor=(0.5, -0.01)) - - plt.tight_layout(rect=[0, 0.03, 1, 1]) - savefig(fig, str(output_dir / "fig2_hallmarks_heatmaps")) - - return { - "module_sim_range": [float(module_sim.min()), float(module_sim.max())], - "text_sim_range": [float(text_sim.min()), float(text_sim.max())], - "clustering_order": [int(i) for i in order], - } - - -# ============================================================ -# Figure 3: Per-Hallmark Alignment (lollipop chart) -# ============================================================ - -def fig3_per_hallmark_alignment( - module_emb: np.ndarray, - text_emb: np.ndarray, - names: List[str], - texts: List[str], - output_dir: Path, - n_bootstrap: int = 2000, - seed: int = 42, -) -> Dict: - """Lollipop chart of per-hallmark alignment scores, colored by process.""" - families = [parse_module_family(n) for n in names] - hallmark_mask = np.array([f == "Hallmarks" for f in families]) - hallmark_idx = np.where(hallmark_mask)[0] - - h_module = module_emb[hallmark_idx] - h_text = text_emb[hallmark_idx] - h_names = [names[i] for i in hallmark_idx] - h_texts = [texts[i] for i in hallmark_idx] - n_h = len(hallmark_idx) - - h_processes = [] - for n in h_names: - proc = parse_hallmark_process(n) - h_processes.append(proc if proc else "pathway") - - # Per-module alignment: Spearman r of similarity profiles - module_sim = cosine_similarity(h_module) - text_sim = cosine_similarity(h_text) - per_module_corr = np.zeros(n_h) - for i in range(n_h): - mask = np.ones(n_h, dtype=bool) - mask[i] = False - r, _ = spearmanr(module_sim[i, mask], text_sim[i, mask]) - per_module_corr[i] = r - - # Sort by alignment - sort_idx = np.argsort(per_module_corr) # ascending for bottom-to-top lollipop - - # Bootstrap CI for mean alignment - mean_val, ci_lo, ci_hi = bootstrap_ci(per_module_corr, np.mean, n_bootstrap, seed=seed) - n_positive = int((per_module_corr > 0).sum()) - - # Figure - fig, ax = plt.subplots(figsize=(8, 12)) - - y_pos = np.arange(n_h) - for i, idx in enumerate(sort_idx): - color = PROCESS_COLORS.get(h_processes[idx], "#999999") - ax.hlines(y=i, xmin=0, xmax=per_module_corr[idx], color=color, - linewidth=1.5, alpha=0.7) - ax.scatter(per_module_corr[idx], i, color=color, s=40, zorder=3) - - ax.axvline(0, color="black", linewidth=0.8, linestyle="-") - ax.axvline(mean_val, color="black", linewidth=1.5, linestyle="--", alpha=0.6) - - ax.set_yticks(y_pos) - ax.set_yticklabels([h_texts[i] for i in sort_idx], fontsize=7) - ax.set_xlabel("Alignment Score (Spearman r of similarity profiles)", fontsize=10) - ax.set_title("Per-Hallmark Alignment Score", fontsize=12, fontweight="bold") - - # Summary text - summary = (f"Mean r = {mean_val:.3f} [{ci_lo:.3f}, {ci_hi:.3f}]\n" - f"Positive: {n_positive}/{n_h} ({100*n_positive/n_h:.0f}%)") - ax.text(0.98, 0.02, summary, transform=ax.transAxes, fontsize=9, - ha="right", va="bottom", - bbox=dict(boxstyle="round", facecolor="lightyellow", alpha=0.9)) - - # Legend - legend_elements = [Line2D([0], [0], marker="o", color="w", markerfacecolor=c, - markersize=8, label=PROCESS_LABELS[k]) - for k, c in PROCESS_COLORS.items()] - ax.legend(handles=legend_elements, loc="upper left", fontsize=7, - frameon=True, ncol=2) - - ax.set_xlim(min(per_module_corr) - 0.05, max(per_module_corr) + 0.05) - plt.tight_layout() - savefig(fig, str(output_dir / "fig3_per_hallmark_alignment")) - - # Save CSV - df = pd.DataFrame({ - "hallmark": h_texts, - "module_name": h_names, - "process_category": h_processes, - "alignment_r": per_module_corr, - }).sort_values("alignment_r", ascending=False) - df.to_csv(output_dir / "hallmarks_alignment_table.csv", index=False) - - return { - "mean_alignment": mean_val, - "mean_alignment_ci_95": [ci_lo, ci_hi], - "n_positive": n_positive, - "n_total": n_h, - "per_module_scores": {h_texts[i]: float(per_module_corr[i]) for i in range(n_h)}, - } - - -# ============================================================ -# Figure 4: Alignment by Biological Process Category -# ============================================================ - -def fig4_category_alignment( - module_emb: np.ndarray, - text_emb: np.ndarray, - names: List[str], - texts: List[str], - output_dir: Path, -) -> Dict: - """Violin/box plot of alignment scores grouped by MSigDB process category.""" - families = [parse_module_family(n) for n in names] - hallmark_mask = np.array([f == "Hallmarks" for f in families]) - hallmark_idx = np.where(hallmark_mask)[0] - - h_module = module_emb[hallmark_idx] - h_text = text_emb[hallmark_idx] - h_names = [names[i] for i in hallmark_idx] - n_h = len(hallmark_idx) - - h_processes = [] - for n in h_names: - proc = parse_hallmark_process(n) - h_processes.append(proc if proc else "pathway") - - # Per-module alignment - module_sim = cosine_similarity(h_module) - text_sim = cosine_similarity(h_text) - per_module_corr = np.zeros(n_h) - for i in range(n_h): - mask = np.ones(n_h, dtype=bool) - mask[i] = False - r, _ = spearmanr(module_sim[i, mask], text_sim[i, mask]) - per_module_corr[i] = r - - # Group by category - cat_scores = defaultdict(list) - for i in range(n_h): - cat_scores[h_processes[i]].append(per_module_corr[i]) - - # Sort categories by median alignment - cats_sorted = sorted(cat_scores.keys(), - key=lambda c: np.median(cat_scores[c]), reverse=True) - - # Kruskal-Wallis test - groups = [np.array(cat_scores[c]) for c in cats_sorted if len(cat_scores[c]) >= 2] - if len(groups) >= 2: - kw_stat, kw_p = kruskal(*groups) - else: - kw_stat, kw_p = 0.0, 1.0 - - # Figure - fig, ax = plt.subplots(figsize=(10, 6)) - - positions = range(len(cats_sorted)) - data_list = [np.array(cat_scores[c]) for c in cats_sorted] - colors_list = [PROCESS_COLORS.get(c, "#999999") for c in cats_sorted] - - # Box + strip plot - bp = ax.boxplot(data_list, positions=positions, widths=0.5, - patch_artist=True, showfliers=False) - for module, color in zip(bp["boxes"], colors_list): - module.set_facecolor(color) - module.set_alpha(0.4) - for median in bp["medians"]: - median.set_color("black") - median.set_linewidth(1.5) - - # Overlay individual points - for i, (data, color) in enumerate(zip(data_list, colors_list)): - jitter = np.random.default_rng(42).uniform(-0.15, 0.15, size=len(data)) - ax.scatter(np.full(len(data), i) + jitter, data, color=color, - s=30, alpha=0.7, edgecolors="white", linewidths=0.5, zorder=3) - - ax.axhline(0, color="gray", linewidth=0.8, linestyle="--") - ax.set_xticks(positions) - ax.set_xticklabels([PROCESS_LABELS.get(c, c) for c in cats_sorted], - rotation=30, ha="right", fontsize=9) - ax.set_ylabel("Alignment Score (Spearman r)", fontsize=10) - ax.set_title(f"Alignment by Biological Process Category\n" - f"Kruskal-Wallis H = {kw_stat:.2f}, p = {kw_p:.4f}", - fontsize=11, fontweight="bold") - - # Annotate counts - for i, c in enumerate(cats_sorted): - n = len(cat_scores[c]) - ax.text(i, ax.get_ylim()[1] * 0.95, f"n={n}", ha="center", fontsize=8) - - plt.tight_layout() - savefig(fig, str(output_dir / "fig4_category_alignment")) - - return { - "kruskal_wallis_H": float(kw_stat), - "kruskal_wallis_p": float(kw_p), - "category_medians": {c: float(np.median(cat_scores[c])) for c in cats_sorted}, - "category_counts": {c: len(cat_scores[c]) for c in cats_sorted}, - } - - -# ============================================================ -# Figure 5: Granularity Effect (Reactome hierarchy) -# ============================================================ - -def fig5_granularity_effect( - module_emb: np.ndarray, - text_emb: np.ndarray, - names: List[str], - texts: List[str], - n_permutations: int = 1000, - n_bootstrap: int = 2000, - seed: int = 42, - output_dir: Path = None, -) -> Dict: - """RSA at multiple granularity levels: fine → category → super-category.""" - families = [parse_module_family(n) for n in names] - reactome_mask = np.array([f == "Reactome" for f in families]) - hallmark_mask = np.array([f == "Hallmarks" for f in families]) - reactome_idx = np.where(reactome_mask)[0] - hallmark_idx = np.where(hallmark_mask)[0] - - results = {} - - # Level 0: Hallmarks (50 modules, already shown to align) - h_module = module_emb[hallmark_idx] - h_text = text_emb[hallmark_idx] - r_hall, ci_lo_h, ci_hi_h = bootstrap_rsa(h_module, h_text, n_bootstrap, seed) - results["hallmarks_50"] = {"r": r_hall, "ci": [ci_lo_h, ci_hi_h], "n": len(hallmark_idx)} - - # Level 1: Reactome fine-grained (491 pathways) - r_module = module_emb[reactome_idx] - r_text = text_emb[reactome_idx] - r_fine, ci_lo_f, ci_hi_f = bootstrap_rsa(r_module, r_text, n_bootstrap, seed) - results["reactome_fine_491"] = {"r": r_fine, "ci": [ci_lo_f, ci_hi_f], "n": len(reactome_idx)} - - # Level 2: Reactome categories (~16) - r_names = [names[i] for i in reactome_idx] - r_texts = [texts[i] for i in reactome_idx] - cat_map = {} - for i, n in enumerate(r_names): - cat = categorize_reactome(n) - if cat not in cat_map: - cat_map[cat] = {"module_embs": [], "text_embs": []} - cat_map[cat]["module_embs"].append(r_module[i]) - cat_map[cat]["text_embs"].append(r_text[i]) - - cats_with_data = {k: v for k, v in cat_map.items() if len(v["module_embs"]) >= 3} - if len(cats_with_data) >= 4: - cat_module_means = np.array([np.mean(v["module_embs"], axis=0) - for v in cats_with_data.values()]) - cat_text_means = np.array([np.mean(v["text_embs"], axis=0) - for v in cats_with_data.values()]) - r_cat, ci_lo_c, ci_hi_c = bootstrap_rsa(cat_module_means, cat_text_means, - n_bootstrap, seed) - results["reactome_category"] = { - "r": r_cat, "ci": [ci_lo_c, ci_hi_c], "n": len(cats_with_data) - } - else: - results["reactome_category"] = {"r": 0.0, "ci": [0.0, 0.0], "n": 0} - - # Level 3: Super-categories (5-6) - super_map = {} - for super_name, sub_cats in REACTOME_SUPER_CATEGORIES.items(): - embs_p, embs_t = [], [] - for sub in sub_cats: - if sub in cats_with_data: - embs_p.extend(cats_with_data[sub]["module_embs"]) - embs_t.extend(cats_with_data[sub]["text_embs"]) - if len(embs_p) >= 3: - super_map[super_name] = { - "module_mean": np.mean(embs_p, axis=0), - "text_mean": np.mean(embs_t, axis=0), - } - - if len(super_map) >= 4: - sup_module = np.array([v["module_mean"] for v in super_map.values()]) - sup_text = np.array([v["text_mean"] for v in super_map.values()]) - r_sup, ci_lo_s, ci_hi_s = bootstrap_rsa(sup_module, sup_text, n_bootstrap, seed) - results["reactome_super_category"] = { - "r": r_sup, "ci": [ci_lo_s, ci_hi_s], "n": len(super_map) - } - else: - results["reactome_super_category"] = {"r": 0.0, "ci": [0.0, 0.0], "n": 0} - - # Figure - fig, ax = plt.subplots(figsize=(8, 5)) - - levels = ["Reactome\nFine (491)", "Reactome\nCategory (~16)", - "Reactome\nSuper-Cat (~6)", "Hallmarks\n(50)"] - keys = ["reactome_fine_491", "reactome_category", - "reactome_super_category", "hallmarks_50"] - rs = [results[k]["r"] for k in keys] - cis_lo = [results[k]["ci"][0] for k in keys] - cis_hi = [results[k]["ci"][1] for k in keys] - ns = [results[k]["n"] for k in keys] - - colors = ["#90CAF9", "#42A5F5", "#1565C0", "#E91E63"] - x = np.arange(len(levels)) - bars = ax.bar(x, rs, color=colors, edgecolor="black", linewidth=0.8, width=0.6) - - # Error bars (CI) - for i in range(len(levels)): - ax.plot([i, i], [cis_lo[i], cis_hi[i]], color="black", linewidth=2) - ax.plot([i - 0.1, i + 0.1], [cis_lo[i], cis_lo[i]], color="black", linewidth=1.5) - ax.plot([i - 0.1, i + 0.1], [cis_hi[i], cis_hi[i]], color="black", linewidth=1.5) - - ax.axhline(0, color="gray", linewidth=0.8, linestyle="--") - ax.set_xticks(x) - ax.set_xticklabels(levels, fontsize=9) - ax.set_ylabel("RSA Spearman r", fontsize=10) - ax.set_title("Alignment Improves at Coarser Granularity", - fontsize=12, fontweight="bold") - - # Annotate n - for i, n in enumerate(ns): - ax.text(i, max(rs[i], cis_hi[i]) + 0.01, f"n={n}", ha="center", fontsize=8) - - plt.tight_layout() - savefig(fig, str(output_dir / "fig5_granularity_effect")) - - return results - - -# ============================================================ -# Figure 6: Multi-Model / Cross-Dataset Comparison -# ============================================================ - -def fig6_model_comparison( - module_emb_primary: np.ndarray, - module_emb_secondary: Optional[np.ndarray], - text_emb_primary: np.ndarray, - names: List[str], - texts: List[str], - primary_model_name: str, - compare_model_names: List[str], - compare_text_embs: List[np.ndarray], - n_permutations: int = 1000, - n_bootstrap: int = 2000, - seed: int = 42, - output_dir: Path = None, -) -> Dict: - """Grouped bar chart comparing RSA across models and datasets.""" - families = [parse_module_family(n) for n in names] - hallmark_mask = np.array([f == "Hallmarks" for f in families]) - hallmark_idx = np.where(hallmark_mask)[0] - - h_module = module_emb_primary[hallmark_idx] - - # Compute RSA for ALL models including primary - all_models = [primary_model_name] + compare_model_names - all_text_embs = [text_emb_primary] + compare_text_embs - - model_results = {} - for model_name, text_emb_full in zip(all_models, all_text_embs): - h_text = text_emb_full[hallmark_idx] - r, ci_lo, ci_hi = bootstrap_rsa(h_module, h_text, n_bootstrap, seed) - mantel_res = mantel_test(h_module, h_text, n_perm=n_permutations) - p = mantel_res["rsa_p_value"] - model_results[model_name] = { - "dataset": "primary", - "r": r, "ci": [ci_lo, ci_hi], "p": p - } - - # Cross-dataset (secondary) - dataset_results = {} - if module_emb_secondary is not None: - h_module2 = module_emb_secondary[hallmark_idx] - for model_name, text_emb_full in zip(all_models, all_text_embs): - h_text = text_emb_full[hallmark_idx] - r, ci_lo, ci_hi = bootstrap_rsa(h_module2, h_text, n_bootstrap, seed) - mantel_res = mantel_test(h_module2, h_text, n_perm=n_permutations) - p = mantel_res["rsa_p_value"] - dataset_results[model_name] = { - "dataset": "secondary", - "r": r, "ci": [ci_lo, ci_hi], "p": p - } - - # Figure - has_secondary = len(dataset_results) > 0 - n_models = len(model_results) - - if n_models == 0: - logger.warning("No models to compare, skipping fig6") - return {} - - fig, ax = plt.subplots(figsize=(max(8, n_models * 2.5), 5)) - - model_names_short = [m.split("/")[-1][:25] for m in model_results.keys()] - x = np.arange(len(model_names_short)) - width = 0.35 if has_secondary else 0.5 - - # Primary dataset bars - rs_primary = [model_results[m]["r"] for m in model_results] - cis_primary = [model_results[m]["ci"] for m in model_results] - yerr_lo = [r - ci[0] for r, ci in zip(rs_primary, cis_primary)] - yerr_hi = [ci[1] - r for r, ci in zip(rs_primary, cis_primary)] - - # Color: highlight primary model differently - colors_primary = ["#1E88E5"] + ["#90CAF9"] * (n_models - 1) - - offset = -width / 2 if has_secondary else 0 - bars1 = ax.bar(x + offset, rs_primary, width, label="Dataset 1 (pretrain)", - color=colors_primary, edgecolor="black", linewidth=0.8) - ax.errorbar(x + offset, rs_primary, yerr=[yerr_lo, yerr_hi], - fmt="none", color="black", capsize=4) - - if has_secondary: - rs_sec = [dataset_results.get(m, {}).get("r", 0) for m in model_results] - cis_sec = [dataset_results.get(m, {}).get("ci", [0, 0]) for m in model_results] - yerr_lo2 = [r - ci[0] for r, ci in zip(rs_sec, cis_sec)] - yerr_hi2 = [ci[1] - r for r, ci in zip(rs_sec, cis_sec)] - colors_sec = ["#E53935"] + ["#EF9A9A"] * (n_models - 1) - bars2 = ax.bar(x + width / 2, rs_sec, width, label="Dataset 2 (pancreas)", - color=colors_sec, edgecolor="black", linewidth=0.8) - ax.errorbar(x + width / 2, rs_sec, yerr=[yerr_lo2, yerr_hi2], - fmt="none", color="black", capsize=4) - - ax.axhline(0, color="gray", linewidth=0.8, linestyle="--") - ax.set_xticks(x) - ax.set_xticklabels(model_names_short, rotation=15, ha="right", fontsize=9) - ax.set_ylabel("RSA Spearman r (Hallmarks)", fontsize=10) - ax.set_title("Alignment Across Sentence Encoders", - fontsize=12, fontweight="bold") - ax.legend(fontsize=9) - - # Annotate p-values - for i, m in enumerate(model_results): - p = model_results[m]["p"] - sig = "***" if p < 0.001 else "**" if p < 0.01 else "*" if p < 0.05 else "ns" - y_pos = max(rs_primary[i], cis_primary[i][1]) + 0.02 - ax.text(i + offset, y_pos, sig, ha="center", fontsize=9, fontweight="bold") - - plt.tight_layout() - savefig(fig, str(output_dir / "fig6_model_comparison")) - - return {"model_results": model_results, "dataset_results": dataset_results} - - -# ============================================================ -# Deep Analysis: Alignment vs Module Size -# ============================================================ - -def analysis_alignment_vs_module_size( - module_emb: np.ndarray, - text_emb: np.ndarray, - names: List[str], - texts: List[str], - module_json: str, - output_dir: Path, -) -> Dict: - """Scatter: n_genes vs alignment score. Tests if larger modules align better.""" - families = [parse_module_family(n) for n in names] - hallmark_mask = np.array([f == "Hallmarks" for f in families]) - hallmark_idx = np.where(hallmark_mask)[0] - - h_module = module_emb[hallmark_idx] - h_text = text_emb[hallmark_idx] - h_names = [names[i] for i in hallmark_idx] - h_texts = [texts[i] for i in hallmark_idx] - n_h = len(hallmark_idx) - - # Per-module alignment - module_sim = cosine_similarity(h_module) - text_sim = cosine_similarity(h_text) - per_module_corr = np.zeros(n_h) - for i in range(n_h): - mask = np.ones(n_h, dtype=bool) - mask[i] = False - r, _ = spearmanr(module_sim[i, mask], text_sim[i, mask]) - per_module_corr[i] = r - - # Module sizes - module_sizes = get_module_sizes(module_json) - h_sizes = np.array([module_sizes.get(n, 0) for n in h_names]) - - # Correlation - r_size, p_size = spearmanr(h_sizes, per_module_corr) - - # Figure - fig, ax = plt.subplots(figsize=(7, 5)) - h_processes = [parse_hallmark_process(n) or "pathway" for n in h_names] - colors = [PROCESS_COLORS.get(p, "#999999") for p in h_processes] - - ax.scatter(h_sizes, per_module_corr, c=colors, s=50, alpha=0.7, - edgecolors="white", linewidths=0.5) - - # Trend line - z = np.polyfit(h_sizes, per_module_corr, 1) - p_line = np.poly1d(z) - x_range = np.linspace(h_sizes.min(), h_sizes.max(), 100) - ax.plot(x_range, p_line(x_range), "k--", linewidth=1.5, alpha=0.6) - - ax.axhline(0, color="gray", linewidth=0.8, linestyle=":") - ax.set_xlabel("Number of Genes in Module", fontsize=10) - ax.set_ylabel("Alignment Score (Spearman r)", fontsize=10) - ax.set_title(f"Alignment vs Module Size\n" - f"Spearman ρ = {r_size:.3f}, p = {p_size:.4f}", - fontsize=11, fontweight="bold") - - # Label outliers - for i in range(n_h): - if abs(per_module_corr[i]) > 0.35 or h_sizes[i] > 250: - ax.annotate(h_texts[i][:15], (h_sizes[i], per_module_corr[i]), - fontsize=6, alpha=0.8, xytext=(5, 3), - textcoords="offset points") - - plt.tight_layout() - savefig(fig, str(output_dir / "fig_alignment_vs_size")) - - return {"spearman_r": float(r_size), "spearman_p": float(p_size)} - - -# ============================================================ -# Deep Analysis: Negative Result Deep-Dive -# ============================================================ - -def analysis_negative_results( - module_emb: np.ndarray, - text_emb: np.ndarray, - names: List[str], - texts: List[str], - output_dir: Path, - k: int = 5, - n_show: int = 5, -) -> Dict: - """For poorly-aligned modules, show neighbor divergence.""" - families = [parse_module_family(n) for n in names] - hallmark_mask = np.array([f == "Hallmarks" for f in families]) - hallmark_idx = np.where(hallmark_mask)[0] - - h_module = module_emb[hallmark_idx] - h_text = text_emb[hallmark_idx] - h_names = [names[i] for i in hallmark_idx] - h_texts = [texts[i] for i in hallmark_idx] - n_h = len(hallmark_idx) - - # Per-module alignment - module_sim = cosine_similarity(h_module) - text_sim = cosine_similarity(h_text) - per_module_corr = np.zeros(n_h) - for i in range(n_h): - mask = np.ones(n_h, dtype=bool) - mask[i] = False - r, _ = spearmanr(module_sim[i, mask], text_sim[i, mask]) - per_module_corr[i] = r - - # Bottom n_show and top n_show - bottom_idx = np.argsort(per_module_corr)[:n_show] - top_idx = np.argsort(per_module_corr)[-n_show:][::-1] - - rows = [] - for idx_set, label in [(bottom_idx, "bottom"), (top_idx, "top")]: - for i in idx_set: - # kNN in module space - module_nn = np.argsort(-module_sim[i]) - module_nn = [j for j in module_nn if j != i][:k] - # kNN in text space - text_nn = np.argsort(-text_sim[i]) - text_nn = [j for j in text_nn if j != i][:k] - overlap = set(module_nn) & set(text_nn) - - rows.append({ - "rank": label, - "hallmark": h_texts[i], - "alignment_r": float(per_module_corr[i]), - "module_neighbors": ", ".join([h_texts[j] for j in module_nn]), - "text_neighbors": ", ".join([h_texts[j] for j in text_nn]), - "knn_overlap": len(overlap), - }) - - df = pd.DataFrame(rows) - df.to_csv(output_dir / "negative_analysis.csv", index=False) - logger.info("Saved negative_analysis.csv (%d rows)", len(df)) - - return { - "bottom_5": [{ - "hallmark": r["hallmark"], - "r": r["alignment_r"], - "module_nn": r["module_neighbors"], - "text_nn": r["text_neighbors"], - } for _, r in df[df["rank"] == "bottom"].iterrows()], - "top_5": [{ - "hallmark": r["hallmark"], - "r": r["alignment_r"], - } for _, r in df[df["rank"] == "top"].iterrows()], - } - - -# ============================================================ -# Deep Analysis: kNN Overlap with Bootstrap CI -# ============================================================ - -def analysis_knn_overlap( - module_emb: np.ndarray, - text_emb: np.ndarray, - names: List[str], - n_permutations: int = 1000, - n_bootstrap: int = 2000, - k: int = 5, - seed: int = 42, - output_dir: Path = None, -) -> Dict: - """kNN overlap with bootstrap CI and per-category breakdown.""" - families = [parse_module_family(n) for n in names] - hallmark_mask = np.array([f == "Hallmarks" for f in families]) - hallmark_idx = np.where(hallmark_mask)[0] - - h_module = module_emb[hallmark_idx] - h_text = text_emb[hallmark_idx] - h_names = [names[i] for i in hallmark_idx] - n_h = len(hallmark_idx) - - # Compute kNN overlap - module_sim = cosine_similarity(h_module) - text_sim = cosine_similarity(h_text) - - overlaps = np.zeros(n_h) - for i in range(n_h): - module_nn = set(np.argsort(-module_sim[i])[1:k + 1]) - text_nn = set(np.argsort(-text_sim[i])[1:k + 1]) - overlaps[i] = len(module_nn & text_nn) / k - - # Bootstrap CI - mean_obs, ci_lo, ci_hi = bootstrap_ci(overlaps, np.mean, n_bootstrap, seed=seed) - - # Null distribution (permutation) - rng = np.random.default_rng(seed) - null_means = np.zeros(n_permutations) - for p in range(n_permutations): - perm = rng.permutation(n_h) - text_sim_perm = text_sim[perm][:, perm] - ov_perm = np.zeros(n_h) - for i in range(n_h): - module_nn = set(np.argsort(-module_sim[i])[1:k + 1]) - text_nn = set(np.argsort(-text_sim_perm[i])[1:k + 1]) - ov_perm[i] = len(module_nn & text_nn) / k - null_means[p] = ov_perm.mean() - - p_value = float((null_means >= mean_obs).sum() / n_permutations) - null_mean = float(null_means.mean()) - null_std = float(null_means.std()) - z_score = (mean_obs - null_mean) / null_std if null_std > 0 else 0.0 - - # Per-category breakdown - h_processes = [parse_hallmark_process(n) or "pathway" for n in h_names] - cat_overlaps = defaultdict(list) - for i in range(n_h): - cat_overlaps[h_processes[i]].append(overlaps[i]) - - return { - "knn_overlap_mean": mean_obs, - "knn_overlap_ci_95": [ci_lo, ci_hi], - "null_mean": null_mean, - "null_std": null_std, - "p_value": p_value, - "z_score": z_score, - "k": k, - "category_means": {c: float(np.mean(v)) for c, v in cat_overlaps.items()}, - } - - -# ============================================================ -# Main -# ============================================================ - -def main(): - args = parse_args() - output_dir = Path(args.output_dir) - output_dir.mkdir(parents=True, exist_ok=True) - - # Setup logging - log_file = output_dir / "run_publication.log" - logging.basicConfig( - level=logging.INFO, - format="%(asctime)s [%(levelname)s] %(message)s", - handlers=[ - logging.FileHandler(log_file, mode="w", encoding="utf-8"), - logging.StreamHandler(), - ], - ) - logger.info("Publication analysis starting") - logger.info("Args: %s", vars(args)) - - # ---- Load data ---- - module_names = load_module_names(args.module_json) - all_module_emb = np.load(args.module_embedding_path) - logger.info("Loaded module embeddings: %s from %s", - all_module_emb.shape, args.module_embedding_path) - - # Filter to semantic modules (Hallmarks + Reactome) - families = [parse_module_family(n) for n in module_names] - semantic_mask = np.array([f in ("Hallmarks", "Reactome") for f in families]) - semantic_idx = np.where(semantic_mask)[0] - module_emb = all_module_emb[semantic_idx] - names = [module_names[i] for i in semantic_idx] - logger.info("Semantic modules: %d (Hallmarks + Reactome)", len(names)) - - # ---- Build text descriptions and encode ---- - desc_file = getattr(args, "descriptions_file", None) - _, _, texts = build_text_descriptions(names, descriptions_file=desc_file) - logger.info("Encoding %d texts with primary model: %s", len(texts), args.sentence_model) - text_emb = encode_texts(texts, args.sentence_model, device=args.device) - logger.info("Text embeddings: %s", text_emb.shape) - - # ---- Core metrics (RSA + Mantel) ---- - logger.info("Computing RSA + Mantel test (n_perm=%d)...", args.n_permutations) - mantel_results = mantel_test(module_emb, text_emb, n_perm=args.n_permutations) - logger.info("RSA r=%.4f, p=%.4f, z=%.1f", - mantel_results["rsa_spearman_observed"], - mantel_results["rsa_p_value"], - mantel_results["rsa_effect_size_z"]) - - # ---- Figure 1: RSA Overview ---- - logger.info("Generating Figure 1: RSA Overview...") - metrics_fig1 = fig1_rsa_overview( - module_emb, text_emb, mantel_results, output_dir, - n_bootstrap=args.n_bootstrap, seed=args.seed, - ) - - # ---- Figure 2: Hallmarks Heatmaps ---- - logger.info("Generating Figure 2: Hallmarks Heatmaps...") - metrics_fig2 = fig2_hallmarks_heatmaps( - module_emb, text_emb, names, texts, output_dir, - ) - - # ---- Figure 3: Per-Hallmark Alignment ---- - logger.info("Generating Figure 3: Per-Hallmark Alignment...") - metrics_fig3 = fig3_per_hallmark_alignment( - module_emb, text_emb, names, texts, output_dir, - n_bootstrap=args.n_bootstrap, seed=args.seed, - ) - - # ---- Figure 4: Category Alignment ---- - logger.info("Generating Figure 4: Category Alignment...") - metrics_fig4 = fig4_category_alignment( - module_emb, text_emb, names, texts, output_dir, - ) - - # ---- Figure 5: Granularity Effect ---- - logger.info("Generating Figure 5: Granularity Effect...") - metrics_fig5 = fig5_granularity_effect( - module_emb, text_emb, names, texts, - n_permutations=args.n_permutations, - n_bootstrap=args.n_bootstrap, - seed=args.seed, - output_dir=output_dir, - ) - - # ---- Figure 6: Multi-Model Comparison (optional) ---- - metrics_fig6 = {} - compare_models = [] - compare_embs = [] - if args.sentence_models_compare: - compare_models = [m.strip() for m in args.sentence_models_compare.split(",") - if m.strip()] - logger.info("Loading %d comparison models...", len(compare_models)) - for model_name in compare_models: - logger.info(" Encoding with: %s", model_name) - try: - emb = encode_texts(texts, model_name, device=args.device) - compare_embs.append(emb) - except Exception as e: - logger.warning(" Failed to load %s: %s", model_name, e) - compare_models.remove(model_name) - - module_emb2 = None - if args.module_embedding_path2: - all_module_emb2 = np.load(args.module_embedding_path2) - module_emb2 = all_module_emb2[semantic_idx] - logger.info("Loaded secondary dataset: %s", all_module_emb2.shape) - - if compare_models: - logger.info("Generating Figure 6: Model Comparison...") - metrics_fig6 = fig6_model_comparison( - module_emb, module_emb2, text_emb, names, texts, - primary_model_name=args.sentence_model.split("/")[-1], - compare_model_names=compare_models, - compare_text_embs=compare_embs, - n_permutations=args.n_permutations, - n_bootstrap=args.n_bootstrap, - seed=args.seed, - output_dir=output_dir, - ) - - # ---- Deep Analysis ---- - logger.info("Running deep analysis: alignment vs module size...") - metrics_size = analysis_alignment_vs_module_size( - module_emb, text_emb, names, texts, args.module_json, output_dir, - ) - - logger.info("Running deep analysis: negative results...") - metrics_neg = analysis_negative_results( - module_emb, text_emb, names, texts, output_dir, - ) - - logger.info("Running deep analysis: kNN overlap with bootstrap...") - metrics_knn = analysis_knn_overlap( - module_emb, text_emb, names, - n_permutations=args.n_permutations, - n_bootstrap=args.n_bootstrap, - seed=args.seed, - output_dir=output_dir, - ) - logger.info("kNN overlap: %.3f [%.3f, %.3f], null=%.3f, p=%.4f", - metrics_knn["knn_overlap_mean"], - metrics_knn["knn_overlap_ci_95"][0], - metrics_knn["knn_overlap_ci_95"][1], - metrics_knn["null_mean"], - metrics_knn["p_value"]) - - # ---- Cross-dataset RSA (if secondary available) ---- - metrics_cross = {} - if module_emb2 is not None: - logger.info("Cross-dataset RSA (secondary dataset)...") - mantel2 = mantel_test(module_emb2, text_emb, n_perm=args.n_permutations) - r2, ci2_lo, ci2_hi = bootstrap_rsa( - module_emb2, text_emb, args.n_bootstrap, args.seed - ) - metrics_cross = { - "secondary_rsa_r": r2, - "secondary_rsa_ci_95": [ci2_lo, ci2_hi], - "secondary_mantel_p": mantel2["rsa_p_value"], - "secondary_mantel_z": mantel2["rsa_effect_size_z"], - } - logger.info("Secondary RSA: r=%.4f [%.4f, %.4f], p=%.4f", - r2, ci2_lo, ci2_hi, mantel2["rsa_p_value"]) - - # ---- Aggregate and save metrics ---- - all_metrics = { - "primary_dataset": args.module_embedding_path, - "secondary_dataset": args.module_embedding_path2, - "sentence_model": args.sentence_model, - "n_modules": len(names), - "n_permutations": args.n_permutations, - "n_bootstrap": args.n_bootstrap, - "fig1_rsa_overview": metrics_fig1, - "fig2_heatmaps": metrics_fig2, - "fig3_per_hallmark": metrics_fig3, - "fig4_category": metrics_fig4, - "fig5_granularity": metrics_fig5, - "fig6_model_comparison": metrics_fig6, - "alignment_vs_size": metrics_size, - "negative_analysis": metrics_neg, - "knn_overlap": metrics_knn, - "cross_dataset": metrics_cross, - } - - metrics_path = output_dir / "publication_metrics.json" - with open(metrics_path, "w", encoding="utf-8") as f: - json.dump(all_metrics, f, indent=2, ensure_ascii=False, default=str) - logger.info("Saved: %s", metrics_path) - logger.info("Publication analysis complete. Output: %s", output_dir) - - -if __name__ == "__main__": - main() +"""Publication-quality analysis for Experiment 2 — Module-Text Alignment. + +Produces 6 main figures + comprehensive metrics with bootstrap CIs for paper. +Supports multi-model comparison and cross-dataset validation. + +Usage (run from the BioMod repository root): + python BioMod/interpretability/module_text_alignment/module_text_alignment_publication.py \ + --module_embedding_path outputs/interpretability/geometry/pretrain_subset/module_embeddings.npy \ + --module_embedding_path2 outputs/interpretability/geometry/pancreas_scib/module_embeddings.npy \ + --module_json model_dependencies/module_data_all/merged_modules.json \ + --sentence_model pritamdeka/S-PubMedBert-MS-MARCO \ + --sentence_models_compare "sentence-transformers/all-mpnet-base-v2" \ + --descriptions_file BioMod/interpretability/module_text_alignment/reactome_descriptions.json \ + --n_permutations 1000 \ + --n_bootstrap 2000 \ + --output_dir outputs/interpretability/text_alignment/publication +""" +from __future__ import annotations + +import argparse +import json +import logging +import sys +from collections import Counter, defaultdict +from pathlib import Path +from typing import Dict, List, Optional, Tuple + +import numpy as np +import pandas as pd +from scipy.cluster.hierarchy import dendrogram, fcluster, linkage +from scipy.spatial.distance import pdist, squareform +from scipy.stats import kruskal, spearmanr +from sklearn.metrics.pairwise import cosine_similarity + +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt +from matplotlib.patches import Patch as LegendHandle +from matplotlib.lines import Line2D + +HERE = Path(__file__).resolve().parent +INTERP_DIR = HERE.parent +PERFORMER_MODULE = INTERP_DIR.parent +REPO_ROOT = PERFORMER_MODULE.parent +for p in (str(HERE), str(INTERP_DIR), str(PERFORMER_MODULE), str(REPO_ROOT)): + if p not in sys.path: + sys.path.insert(0, p) + +from interpretability.utils import ( + HALLMARK_PROCESS, + load_module_names, + parse_hallmark_process, + parse_module_family, +) +from module_text_alignment.module_text_alignment import ( + build_text_descriptions, + compute_cka, + compute_rsa, + encode_texts, + mantel_test, + permutation_test_cka, +) +from module_text_alignment.module_text_alignment_extended import ( + REACTOME_CATEGORIES, + categorize_reactome, +) + +logger = logging.getLogger(__name__) + +# ============================================================ +# Color palette for MSigDB hallmark process categories +# ============================================================ + +PROCESS_COLORS = { + "immune": "#2196F3", + "metabolic": "#4CAF50", + "signaling": "#FF9800", + "proliferation": "#E91E63", + "development": "#9C27B0", + "dna_damage": "#F44336", + "pathway": "#607D8B", + "cellular_component": "#795548", +} + +PROCESS_LABELS = { + "immune": "Immune", + "metabolic": "Metabolic", + "signaling": "Signaling", + "proliferation": "Proliferation", + "development": "Development", + "dna_damage": "DNA Damage", + "pathway": "Pathway", + "cellular_component": "Cellular Component", +} + +# Reactome super-categories (merge related categories for granularity sweep) +REACTOME_SUPER_CATEGORIES = { + "Information Processing": ["Gene Expression", "DNA Replication", "DNA Repair", + "Chromatin & Epigenetics"], + "Signal & Communication": ["Signal Transduction", "Cell-Cell Communication"], + "Cell Fate": ["Cell Cycle", "Apoptosis", "Cellular Responses"], + "Metabolism & Transport": ["Metabolism", "Transport"], + "Immune & Defense": ["Immune System", "Hemostasis", "Disease"], + "Development": ["Developmental Biology", "Protein Homeostasis"], +} + + +# ============================================================ +# Utility functions +# ============================================================ + +def savefig(fig, path_stem: str, dpi: int = 300): + """Save figure as both PDF and PNG.""" + fig.savefig(f"{path_stem}.pdf", dpi=dpi, bbox_inches="tight") + fig.savefig(f"{path_stem}.png", dpi=dpi, bbox_inches="tight") + plt.close(fig) + logger.info("Saved: %s.{pdf,png}", path_stem) + + +def adaptive_vlim(matrix: np.ndarray, percentile: int = 98, symmetric: bool = True): + """Compute adaptive vmin/vmax from data percentiles.""" + vals = matrix[np.triu_indices_from(matrix, k=1)] + if symmetric: + vlim = np.percentile(np.abs(vals), percentile) + return -vlim, vlim + else: + return np.percentile(vals, [100 - percentile, percentile]) + + +def bootstrap_ci( + values: np.ndarray, + stat_fn=np.mean, + n_boot: int = 2000, + ci: float = 0.95, + seed: int = 42, +) -> Tuple[float, float, float]: + """Bootstrap confidence interval. Returns (estimate, ci_low, ci_high).""" + rng = np.random.default_rng(seed) + n = len(values) + boot_stats = np.zeros(n_boot) + for i in range(n_boot): + idx = rng.integers(0, n, size=n) + boot_stats[i] = stat_fn(values[idx]) + alpha = (1 - ci) / 2 + return float(stat_fn(values)), float(np.percentile(boot_stats, alpha * 100)), \ + float(np.percentile(boot_stats, (1 - alpha) * 100)) + + +def bootstrap_rsa( + module_emb: np.ndarray, + text_emb: np.ndarray, + n_boot: int = 2000, + seed: int = 42, +) -> Tuple[float, float, float]: + """Bootstrap CI for RSA Spearman r by resampling modules.""" + rng = np.random.default_rng(seed) + n = module_emb.shape[0] + boot_rs = np.zeros(n_boot) + for i in range(n_boot): + idx = rng.choice(n, size=n, replace=True) + dist_p = pdist(module_emb[idx], metric="cosine") + dist_t = pdist(text_emb[idx], metric="cosine") + boot_rs[i], _ = spearmanr(dist_p, dist_t) + observed, _ = spearmanr(pdist(module_emb, "cosine"), pdist(text_emb, "cosine")) + return float(observed), float(np.percentile(boot_rs, 2.5)), \ + float(np.percentile(boot_rs, 97.5)) + + +def get_module_sizes(module_json: str) -> Dict[str, int]: + """Get number of genes per module from merged_modules.json.""" + with open(module_json, "r", encoding="utf-8") as f: + modules = json.load(f) + return {name: len(genes) for name, genes in modules.items()} + + +# ============================================================ +# CLI +# ============================================================ + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument("--module_embedding_path", type=str, required=True, + help="Primary module_embeddings.npy (e.g. pretrain_subset)") + parser.add_argument("--module_embedding_path2", type=str, default=None, + help="Second dataset module_embeddings.npy for cross-dataset comparison") + parser.add_argument("--module_json", type=str, + default=str(REPO_ROOT / "model_dependencies" / "module_data_all" / "merged_modules.json")) + parser.add_argument("--descriptions_file", type=str, default=None) + parser.add_argument("--sentence_model", type=str, + default="pritamdeka/S-PubMedBert-MS-MARCO") + parser.add_argument("--sentence_models_compare", type=str, default=None, + help="Comma-separated additional models for comparison") + parser.add_argument("--device", type=str, default="cpu") + parser.add_argument("--n_permutations", type=int, default=1000) + parser.add_argument("--n_bootstrap", type=int, default=2000) + parser.add_argument("--seed", type=int, default=42) + parser.add_argument("--output_dir", type=str, required=True) + return parser.parse_args() + + +# ============================================================ +# Figure 1: RSA Overview (hexbin + null distribution) +# ============================================================ + +def fig1_rsa_overview( + module_emb: np.ndarray, + text_emb: np.ndarray, + mantel_results: Dict, + output_dir: Path, + metric: str = "cosine", + n_bootstrap: int = 2000, + seed: int = 42, +) -> Dict: + """RSA overview: hexbin density + Mantel null distribution.""" + dist_module = pdist(module_emb, metric=metric) + dist_text = pdist(text_emb, metric=metric) + + observed_r = mantel_results["rsa_spearman_observed"] + null_mean = mantel_results["rsa_null_mean"] + null_std = mantel_results["rsa_null_std"] + p_value = mantel_results["rsa_p_value"] + z_score = mantel_results["rsa_effect_size_z"] + + # Bootstrap CI for RSA + rsa_obs, rsa_ci_lo, rsa_ci_hi = bootstrap_rsa( + module_emb, text_emb, n_boot=n_bootstrap, seed=seed + ) + + fig, axes = plt.subplots(1, 2, figsize=(12, 5)) + + # Panel A: Hexbin density + ax = axes[0] + hb = ax.hexbin(dist_text, dist_module, gridsize=40, cmap="YlOrRd", + mincnt=1, linewidths=0.2) + cb = plt.colorbar(hb, ax=ax, fraction=0.046, pad=0.02) + cb.set_label("Count", fontsize=9) + + # Trend line + z = np.polyfit(dist_text, dist_module, 1) + p_line = np.poly1d(z) + x_range = np.linspace(dist_text.min(), dist_text.max(), 100) + ax.plot(x_range, p_line(x_range), "k-", linewidth=2, alpha=0.8) + + ax.set_xlabel("Text pairwise distance (cosine)", fontsize=10) + ax.set_ylabel("Module embedding pairwise distance (cosine)", fontsize=10) + ax.set_title(f"A. RSA: r = {observed_r:.3f} [{rsa_ci_lo:.3f}, {rsa_ci_hi:.3f}]", + fontsize=11, fontweight="bold") + ax.text(0.05, 0.95, f"n = {module_emb.shape[0]} modules\n" + f"pairs = {len(dist_module):,}", + transform=ax.transAxes, fontsize=8, va="top", + bbox=dict(boxstyle="round", facecolor="white", alpha=0.8)) + + # Panel B: Null distribution + ax = axes[1] + from scipy.stats import norm as norm_dist + x = np.linspace(null_mean - 4.5 * null_std, + max(observed_r + 2 * null_std, null_mean + 4.5 * null_std), 300) + y = norm_dist.pdf(x, null_mean, null_std) + ax.fill_between(x, y, alpha=0.25, color="gray") + ax.plot(x, y, color="gray", linewidth=1.5, label="Null distribution") + + # Shade p-value region + x_tail = x[x >= observed_r] + y_tail = norm_dist.pdf(x_tail, null_mean, null_std) + ax.fill_between(x_tail, y_tail, alpha=0.4, color="red", label=f"p = {p_value:.4f}") + + ax.axvline(observed_r, color="red", linewidth=2.5, linestyle="-", + label=f"Observed r = {observed_r:.4f}") + ax.axvline(null_mean, color="gray", linewidth=1, linestyle="--", alpha=0.7) + + ax.set_xlabel("Spearman r (distance matrix correlation)", fontsize=10) + ax.set_ylabel("Density", fontsize=10) + ax.set_title(f"B. Mantel Test: z = {z_score:.1f}σ, p = {p_value:.4f}", + fontsize=11, fontweight="bold") + ax.legend(fontsize=9, loc="upper left") + + plt.tight_layout() + savefig(fig, str(output_dir / "fig1_rsa_overview")) + + return { + "rsa_r": rsa_obs, + "rsa_ci_95": [rsa_ci_lo, rsa_ci_hi], + "mantel_p": p_value, + "mantel_z": z_score, + } + + +# ============================================================ +# Figure 2: Hallmarks Similarity Matrices with Dendrograms +# ============================================================ + +def _rank_transform_matrix(sim_matrix: np.ndarray) -> np.ndarray: + """Rank-transform off-diagonal elements to [0,1] for visualization.""" + from scipy.stats import rankdata + n = sim_matrix.shape[0] + result = np.zeros_like(sim_matrix) + triu_idx = np.triu_indices(n, k=1) + vals = sim_matrix[triu_idx] + ranks = rankdata(vals) / len(vals) + result[triu_idx] = ranks + result[triu_idx[1], triu_idx[0]] = ranks + np.fill_diagonal(result, 1.0) + return result + + +def fig2_hallmarks_heatmaps( + module_emb: np.ndarray, + text_emb: np.ndarray, + names: List[str], + texts: List[str], + output_dir: Path, +) -> Dict: + """Hallmarks 50x50 similarity matrices with rank-transform and clustering.""" + families = [parse_module_family(n) for n in names] + hallmark_mask = np.array([f == "Hallmarks" for f in families]) + hallmark_idx = np.where(hallmark_mask)[0] + + h_module = module_emb[hallmark_idx] + h_text = text_emb[hallmark_idx] + h_names = [names[i] for i in hallmark_idx] + h_texts = [texts[i] for i in hallmark_idx] + n_h = len(hallmark_idx) + + h_processes = [] + for n in h_names: + proc = parse_hallmark_process(n) + h_processes.append(proc if proc else "pathway") + + module_sim = cosine_similarity(h_module) + text_sim = cosine_similarity(h_text) + + linkage_mat = linkage(pdist(h_module, "cosine"), method="ward") + dendro = dendrogram(linkage_mat, no_plot=True) + order = dendro["leaves"] + + module_sim_ord = module_sim[order][:, order] + text_sim_ord = text_sim[order][:, order] + texts_ord = [h_texts[i] for i in order] + procs_ord = [h_processes[i] for i in order] + + # Rank-transform to spread narrow cosine similarity ranges + module_rank_ord = _rank_transform_matrix(module_sim_ord) + text_rank_ord = _rank_transform_matrix(text_sim_ord) + diff_rank = module_rank_ord - text_rank_ord + + # Adaptive limit for difference panel + d_vals = diff_rank[np.triu_indices(n_h, k=1)] + vlim_d = float(np.percentile(np.abs(d_vals), 95)) + + # Figure + fig = plt.figure(figsize=(20, 8.5)) + gs = fig.add_gridspec(2, 3, height_ratios=[0.08, 1], hspace=0.02, wspace=0.3) + + # Dendrogram + ax_dend = fig.add_subplot(gs[0, :]) + dendrogram(linkage_mat, ax=ax_dend, no_labels=True, + color_threshold=0, above_threshold_color="gray") + ax_dend.set_xticks([]) + ax_dend.set_yticks([]) + ax_dend.spines[:].set_visible(False) + + for i, proc in enumerate(procs_ord): + color = PROCESS_COLORS.get(proc, "#999999") + ax_dend.axvspan(i * 10 + 5 - 4, i * 10 + 5 + 4, ymin=-0.2, ymax=-0.02, + color=color, clip_on=False) + + # Panel A: Module similarity (rank-transformed) + ax0 = fig.add_subplot(gs[1, 0]) + im0 = ax0.imshow(module_rank_ord, cmap="YlOrRd", vmin=0, vmax=1, + aspect="equal", interpolation="nearest") + ax0.set_title("A. Module Embedding Similarity\n(rank-transformed)", + fontsize=10, fontweight="bold") + ax0.set_xticks(range(n_h)) + ax0.set_xticklabels(texts_ord, rotation=90, fontsize=5.5, ha="center") + ax0.set_yticks(range(n_h)) + ax0.set_yticklabels(texts_ord, fontsize=5.5) + cb0 = plt.colorbar(im0, ax=ax0, fraction=0.046, pad=0.02) + cb0.set_label("Rank percentile", fontsize=8) + + # Panel B: Text similarity (rank-transformed) + ax1 = fig.add_subplot(gs[1, 1]) + im1 = ax1.imshow(text_rank_ord, cmap="YlOrRd", vmin=0, vmax=1, + aspect="equal", interpolation="nearest") + ax1.set_title("B. Text Embedding Similarity\n(rank-transformed)", + fontsize=10, fontweight="bold") + ax1.set_xticks(range(n_h)) + ax1.set_xticklabels(texts_ord, rotation=90, fontsize=5.5, ha="center") + ax1.set_yticks([]) + cb1 = plt.colorbar(im1, ax=ax1, fraction=0.046, pad=0.02) + cb1.set_label("Rank percentile", fontsize=8) + + # Panel C: Difference + ax2 = fig.add_subplot(gs[1, 2]) + im2 = ax2.imshow(diff_rank, cmap="PiYG", vmin=-vlim_d, vmax=vlim_d, + aspect="equal", interpolation="nearest") + ax2.set_title("C. Difference (Module − Text)\n(rank space)", + fontsize=10, fontweight="bold") + ax2.set_xticks(range(n_h)) + ax2.set_xticklabels(texts_ord, rotation=90, fontsize=5.5, ha="center") + ax2.set_yticks([]) + cb2 = plt.colorbar(im2, ax=ax2, fraction=0.046, pad=0.02) + cb2.set_label("Rank difference", fontsize=8) + + legend_elements = [LegendHandle(facecolor=PROCESS_COLORS[k], label=PROCESS_LABELS[k]) + for k in PROCESS_COLORS] + fig.legend(handles=legend_elements, loc="lower center", ncol=4, + fontsize=9, frameon=True, bbox_to_anchor=(0.5, -0.01)) + + plt.tight_layout(rect=[0, 0.03, 1, 1]) + savefig(fig, str(output_dir / "fig2_hallmarks_heatmaps")) + + return { + "module_sim_range": [float(module_sim.min()), float(module_sim.max())], + "text_sim_range": [float(text_sim.min()), float(text_sim.max())], + "clustering_order": [int(i) for i in order], + } + + +# ============================================================ +# Figure 3: Per-Hallmark Alignment (lollipop chart) +# ============================================================ + +def fig3_per_hallmark_alignment( + module_emb: np.ndarray, + text_emb: np.ndarray, + names: List[str], + texts: List[str], + output_dir: Path, + n_bootstrap: int = 2000, + seed: int = 42, +) -> Dict: + """Lollipop chart of per-hallmark alignment scores, colored by process.""" + families = [parse_module_family(n) for n in names] + hallmark_mask = np.array([f == "Hallmarks" for f in families]) + hallmark_idx = np.where(hallmark_mask)[0] + + h_module = module_emb[hallmark_idx] + h_text = text_emb[hallmark_idx] + h_names = [names[i] for i in hallmark_idx] + h_texts = [texts[i] for i in hallmark_idx] + n_h = len(hallmark_idx) + + h_processes = [] + for n in h_names: + proc = parse_hallmark_process(n) + h_processes.append(proc if proc else "pathway") + + # Per-module alignment: Spearman r of similarity profiles + module_sim = cosine_similarity(h_module) + text_sim = cosine_similarity(h_text) + per_module_corr = np.zeros(n_h) + for i in range(n_h): + mask = np.ones(n_h, dtype=bool) + mask[i] = False + r, _ = spearmanr(module_sim[i, mask], text_sim[i, mask]) + per_module_corr[i] = r + + # Sort by alignment + sort_idx = np.argsort(per_module_corr) # ascending for bottom-to-top lollipop + + # Bootstrap CI for mean alignment + mean_val, ci_lo, ci_hi = bootstrap_ci(per_module_corr, np.mean, n_bootstrap, seed=seed) + n_positive = int((per_module_corr > 0).sum()) + + # Figure + fig, ax = plt.subplots(figsize=(8, 12)) + + y_pos = np.arange(n_h) + for i, idx in enumerate(sort_idx): + color = PROCESS_COLORS.get(h_processes[idx], "#999999") + ax.hlines(y=i, xmin=0, xmax=per_module_corr[idx], color=color, + linewidth=1.5, alpha=0.7) + ax.scatter(per_module_corr[idx], i, color=color, s=40, zorder=3) + + ax.axvline(0, color="black", linewidth=0.8, linestyle="-") + ax.axvline(mean_val, color="black", linewidth=1.5, linestyle="--", alpha=0.6) + + ax.set_yticks(y_pos) + ax.set_yticklabels([h_texts[i] for i in sort_idx], fontsize=7) + ax.set_xlabel("Alignment Score (Spearman r of similarity profiles)", fontsize=10) + ax.set_title("Per-Hallmark Alignment Score", fontsize=12, fontweight="bold") + + # Summary text + summary = (f"Mean r = {mean_val:.3f} [{ci_lo:.3f}, {ci_hi:.3f}]\n" + f"Positive: {n_positive}/{n_h} ({100*n_positive/n_h:.0f}%)") + ax.text(0.98, 0.02, summary, transform=ax.transAxes, fontsize=9, + ha="right", va="bottom", + bbox=dict(boxstyle="round", facecolor="lightyellow", alpha=0.9)) + + # Legend + legend_elements = [Line2D([0], [0], marker="o", color="w", markerfacecolor=c, + markersize=8, label=PROCESS_LABELS[k]) + for k, c in PROCESS_COLORS.items()] + ax.legend(handles=legend_elements, loc="upper left", fontsize=7, + frameon=True, ncol=2) + + ax.set_xlim(min(per_module_corr) - 0.05, max(per_module_corr) + 0.05) + plt.tight_layout() + savefig(fig, str(output_dir / "fig3_per_hallmark_alignment")) + + # Save CSV + df = pd.DataFrame({ + "hallmark": h_texts, + "module_name": h_names, + "process_category": h_processes, + "alignment_r": per_module_corr, + }).sort_values("alignment_r", ascending=False) + df.to_csv(output_dir / "hallmarks_alignment_table.csv", index=False) + + return { + "mean_alignment": mean_val, + "mean_alignment_ci_95": [ci_lo, ci_hi], + "n_positive": n_positive, + "n_total": n_h, + "per_module_scores": {h_texts[i]: float(per_module_corr[i]) for i in range(n_h)}, + } + + +# ============================================================ +# Figure 4: Alignment by Biological Process Category +# ============================================================ + +def fig4_category_alignment( + module_emb: np.ndarray, + text_emb: np.ndarray, + names: List[str], + texts: List[str], + output_dir: Path, +) -> Dict: + """Violin/box plot of alignment scores grouped by MSigDB process category.""" + families = [parse_module_family(n) for n in names] + hallmark_mask = np.array([f == "Hallmarks" for f in families]) + hallmark_idx = np.where(hallmark_mask)[0] + + h_module = module_emb[hallmark_idx] + h_text = text_emb[hallmark_idx] + h_names = [names[i] for i in hallmark_idx] + n_h = len(hallmark_idx) + + h_processes = [] + for n in h_names: + proc = parse_hallmark_process(n) + h_processes.append(proc if proc else "pathway") + + # Per-module alignment + module_sim = cosine_similarity(h_module) + text_sim = cosine_similarity(h_text) + per_module_corr = np.zeros(n_h) + for i in range(n_h): + mask = np.ones(n_h, dtype=bool) + mask[i] = False + r, _ = spearmanr(module_sim[i, mask], text_sim[i, mask]) + per_module_corr[i] = r + + # Group by category + cat_scores = defaultdict(list) + for i in range(n_h): + cat_scores[h_processes[i]].append(per_module_corr[i]) + + # Sort categories by median alignment + cats_sorted = sorted(cat_scores.keys(), + key=lambda c: np.median(cat_scores[c]), reverse=True) + + # Kruskal-Wallis test + groups = [np.array(cat_scores[c]) for c in cats_sorted if len(cat_scores[c]) >= 2] + if len(groups) >= 2: + kw_stat, kw_p = kruskal(*groups) + else: + kw_stat, kw_p = 0.0, 1.0 + + # Figure + fig, ax = plt.subplots(figsize=(10, 6)) + + positions = range(len(cats_sorted)) + data_list = [np.array(cat_scores[c]) for c in cats_sorted] + colors_list = [PROCESS_COLORS.get(c, "#999999") for c in cats_sorted] + + # Box + strip plot + bp = ax.boxplot(data_list, positions=positions, widths=0.5, + patch_artist=True, showfliers=False) + for module, color in zip(bp["boxes"], colors_list): + module.set_facecolor(color) + module.set_alpha(0.4) + for median in bp["medians"]: + median.set_color("black") + median.set_linewidth(1.5) + + # Overlay individual points + for i, (data, color) in enumerate(zip(data_list, colors_list)): + jitter = np.random.default_rng(42).uniform(-0.15, 0.15, size=len(data)) + ax.scatter(np.full(len(data), i) + jitter, data, color=color, + s=30, alpha=0.7, edgecolors="white", linewidths=0.5, zorder=3) + + ax.axhline(0, color="gray", linewidth=0.8, linestyle="--") + ax.set_xticks(positions) + ax.set_xticklabels([PROCESS_LABELS.get(c, c) for c in cats_sorted], + rotation=30, ha="right", fontsize=9) + ax.set_ylabel("Alignment Score (Spearman r)", fontsize=10) + ax.set_title(f"Alignment by Biological Process Category\n" + f"Kruskal-Wallis H = {kw_stat:.2f}, p = {kw_p:.4f}", + fontsize=11, fontweight="bold") + + # Annotate counts + for i, c in enumerate(cats_sorted): + n = len(cat_scores[c]) + ax.text(i, ax.get_ylim()[1] * 0.95, f"n={n}", ha="center", fontsize=8) + + plt.tight_layout() + savefig(fig, str(output_dir / "fig4_category_alignment")) + + return { + "kruskal_wallis_H": float(kw_stat), + "kruskal_wallis_p": float(kw_p), + "category_medians": {c: float(np.median(cat_scores[c])) for c in cats_sorted}, + "category_counts": {c: len(cat_scores[c]) for c in cats_sorted}, + } + + +# ============================================================ +# Figure 5: Granularity Effect (Reactome hierarchy) +# ============================================================ + +def fig5_granularity_effect( + module_emb: np.ndarray, + text_emb: np.ndarray, + names: List[str], + texts: List[str], + n_permutations: int = 1000, + n_bootstrap: int = 2000, + seed: int = 42, + output_dir: Path = None, +) -> Dict: + """RSA at multiple granularity levels: fine → category → super-category.""" + families = [parse_module_family(n) for n in names] + reactome_mask = np.array([f == "Reactome" for f in families]) + hallmark_mask = np.array([f == "Hallmarks" for f in families]) + reactome_idx = np.where(reactome_mask)[0] + hallmark_idx = np.where(hallmark_mask)[0] + + results = {} + + # Level 0: Hallmarks (50 modules, already shown to align) + h_module = module_emb[hallmark_idx] + h_text = text_emb[hallmark_idx] + r_hall, ci_lo_h, ci_hi_h = bootstrap_rsa(h_module, h_text, n_bootstrap, seed) + results["hallmarks_50"] = {"r": r_hall, "ci": [ci_lo_h, ci_hi_h], "n": len(hallmark_idx)} + + # Level 1: Reactome fine-grained (491 pathways) + r_module = module_emb[reactome_idx] + r_text = text_emb[reactome_idx] + r_fine, ci_lo_f, ci_hi_f = bootstrap_rsa(r_module, r_text, n_bootstrap, seed) + results["reactome_fine_491"] = {"r": r_fine, "ci": [ci_lo_f, ci_hi_f], "n": len(reactome_idx)} + + # Level 2: Reactome categories (~16) + r_names = [names[i] for i in reactome_idx] + r_texts = [texts[i] for i in reactome_idx] + cat_map = {} + for i, n in enumerate(r_names): + cat = categorize_reactome(n) + if cat not in cat_map: + cat_map[cat] = {"module_embs": [], "text_embs": []} + cat_map[cat]["module_embs"].append(r_module[i]) + cat_map[cat]["text_embs"].append(r_text[i]) + + cats_with_data = {k: v for k, v in cat_map.items() if len(v["module_embs"]) >= 3} + if len(cats_with_data) >= 4: + cat_module_means = np.array([np.mean(v["module_embs"], axis=0) + for v in cats_with_data.values()]) + cat_text_means = np.array([np.mean(v["text_embs"], axis=0) + for v in cats_with_data.values()]) + r_cat, ci_lo_c, ci_hi_c = bootstrap_rsa(cat_module_means, cat_text_means, + n_bootstrap, seed) + results["reactome_category"] = { + "r": r_cat, "ci": [ci_lo_c, ci_hi_c], "n": len(cats_with_data) + } + else: + results["reactome_category"] = {"r": 0.0, "ci": [0.0, 0.0], "n": 0} + + # Level 3: Super-categories (5-6) + super_map = {} + for super_name, sub_cats in REACTOME_SUPER_CATEGORIES.items(): + embs_p, embs_t = [], [] + for sub in sub_cats: + if sub in cats_with_data: + embs_p.extend(cats_with_data[sub]["module_embs"]) + embs_t.extend(cats_with_data[sub]["text_embs"]) + if len(embs_p) >= 3: + super_map[super_name] = { + "module_mean": np.mean(embs_p, axis=0), + "text_mean": np.mean(embs_t, axis=0), + } + + if len(super_map) >= 4: + sup_module = np.array([v["module_mean"] for v in super_map.values()]) + sup_text = np.array([v["text_mean"] for v in super_map.values()]) + r_sup, ci_lo_s, ci_hi_s = bootstrap_rsa(sup_module, sup_text, n_bootstrap, seed) + results["reactome_super_category"] = { + "r": r_sup, "ci": [ci_lo_s, ci_hi_s], "n": len(super_map) + } + else: + results["reactome_super_category"] = {"r": 0.0, "ci": [0.0, 0.0], "n": 0} + + # Figure + fig, ax = plt.subplots(figsize=(8, 5)) + + levels = ["Reactome\nFine (491)", "Reactome\nCategory (~16)", + "Reactome\nSuper-Cat (~6)", "Hallmarks\n(50)"] + keys = ["reactome_fine_491", "reactome_category", + "reactome_super_category", "hallmarks_50"] + rs = [results[k]["r"] for k in keys] + cis_lo = [results[k]["ci"][0] for k in keys] + cis_hi = [results[k]["ci"][1] for k in keys] + ns = [results[k]["n"] for k in keys] + + colors = ["#90CAF9", "#42A5F5", "#1565C0", "#E91E63"] + x = np.arange(len(levels)) + bars = ax.bar(x, rs, color=colors, edgecolor="black", linewidth=0.8, width=0.6) + + # Error bars (CI) + for i in range(len(levels)): + ax.plot([i, i], [cis_lo[i], cis_hi[i]], color="black", linewidth=2) + ax.plot([i - 0.1, i + 0.1], [cis_lo[i], cis_lo[i]], color="black", linewidth=1.5) + ax.plot([i - 0.1, i + 0.1], [cis_hi[i], cis_hi[i]], color="black", linewidth=1.5) + + ax.axhline(0, color="gray", linewidth=0.8, linestyle="--") + ax.set_xticks(x) + ax.set_xticklabels(levels, fontsize=9) + ax.set_ylabel("RSA Spearman r", fontsize=10) + ax.set_title("Alignment Improves at Coarser Granularity", + fontsize=12, fontweight="bold") + + # Annotate n + for i, n in enumerate(ns): + ax.text(i, max(rs[i], cis_hi[i]) + 0.01, f"n={n}", ha="center", fontsize=8) + + plt.tight_layout() + savefig(fig, str(output_dir / "fig5_granularity_effect")) + + return results + + +# ============================================================ +# Figure 6: Multi-Model / Cross-Dataset Comparison +# ============================================================ + +def fig6_model_comparison( + module_emb_primary: np.ndarray, + module_emb_secondary: Optional[np.ndarray], + text_emb_primary: np.ndarray, + names: List[str], + texts: List[str], + primary_model_name: str, + compare_model_names: List[str], + compare_text_embs: List[np.ndarray], + n_permutations: int = 1000, + n_bootstrap: int = 2000, + seed: int = 42, + output_dir: Path = None, +) -> Dict: + """Grouped bar chart comparing RSA across models and datasets.""" + families = [parse_module_family(n) for n in names] + hallmark_mask = np.array([f == "Hallmarks" for f in families]) + hallmark_idx = np.where(hallmark_mask)[0] + + h_module = module_emb_primary[hallmark_idx] + + # Compute RSA for ALL models including primary + all_models = [primary_model_name] + compare_model_names + all_text_embs = [text_emb_primary] + compare_text_embs + + model_results = {} + for model_name, text_emb_full in zip(all_models, all_text_embs): + h_text = text_emb_full[hallmark_idx] + r, ci_lo, ci_hi = bootstrap_rsa(h_module, h_text, n_bootstrap, seed) + mantel_res = mantel_test(h_module, h_text, n_perm=n_permutations) + p = mantel_res["rsa_p_value"] + model_results[model_name] = { + "dataset": "primary", + "r": r, "ci": [ci_lo, ci_hi], "p": p + } + + # Cross-dataset (secondary) + dataset_results = {} + if module_emb_secondary is not None: + h_module2 = module_emb_secondary[hallmark_idx] + for model_name, text_emb_full in zip(all_models, all_text_embs): + h_text = text_emb_full[hallmark_idx] + r, ci_lo, ci_hi = bootstrap_rsa(h_module2, h_text, n_bootstrap, seed) + mantel_res = mantel_test(h_module2, h_text, n_perm=n_permutations) + p = mantel_res["rsa_p_value"] + dataset_results[model_name] = { + "dataset": "secondary", + "r": r, "ci": [ci_lo, ci_hi], "p": p + } + + # Figure + has_secondary = len(dataset_results) > 0 + n_models = len(model_results) + + if n_models == 0: + logger.warning("No models to compare, skipping fig6") + return {} + + fig, ax = plt.subplots(figsize=(max(8, n_models * 2.5), 5)) + + model_names_short = [m.split("/")[-1][:25] for m in model_results.keys()] + x = np.arange(len(model_names_short)) + width = 0.35 if has_secondary else 0.5 + + # Primary dataset bars + rs_primary = [model_results[m]["r"] for m in model_results] + cis_primary = [model_results[m]["ci"] for m in model_results] + yerr_lo = [r - ci[0] for r, ci in zip(rs_primary, cis_primary)] + yerr_hi = [ci[1] - r for r, ci in zip(rs_primary, cis_primary)] + + # Color: highlight primary model differently + colors_primary = ["#1E88E5"] + ["#90CAF9"] * (n_models - 1) + + offset = -width / 2 if has_secondary else 0 + bars1 = ax.bar(x + offset, rs_primary, width, label="Dataset 1 (pretrain)", + color=colors_primary, edgecolor="black", linewidth=0.8) + ax.errorbar(x + offset, rs_primary, yerr=[yerr_lo, yerr_hi], + fmt="none", color="black", capsize=4) + + if has_secondary: + rs_sec = [dataset_results.get(m, {}).get("r", 0) for m in model_results] + cis_sec = [dataset_results.get(m, {}).get("ci", [0, 0]) for m in model_results] + yerr_lo2 = [r - ci[0] for r, ci in zip(rs_sec, cis_sec)] + yerr_hi2 = [ci[1] - r for r, ci in zip(rs_sec, cis_sec)] + colors_sec = ["#E53935"] + ["#EF9A9A"] * (n_models - 1) + bars2 = ax.bar(x + width / 2, rs_sec, width, label="Dataset 2 (pancreas)", + color=colors_sec, edgecolor="black", linewidth=0.8) + ax.errorbar(x + width / 2, rs_sec, yerr=[yerr_lo2, yerr_hi2], + fmt="none", color="black", capsize=4) + + ax.axhline(0, color="gray", linewidth=0.8, linestyle="--") + ax.set_xticks(x) + ax.set_xticklabels(model_names_short, rotation=15, ha="right", fontsize=9) + ax.set_ylabel("RSA Spearman r (Hallmarks)", fontsize=10) + ax.set_title("Alignment Across Sentence Encoders", + fontsize=12, fontweight="bold") + ax.legend(fontsize=9) + + # Annotate p-values + for i, m in enumerate(model_results): + p = model_results[m]["p"] + sig = "***" if p < 0.001 else "**" if p < 0.01 else "*" if p < 0.05 else "ns" + y_pos = max(rs_primary[i], cis_primary[i][1]) + 0.02 + ax.text(i + offset, y_pos, sig, ha="center", fontsize=9, fontweight="bold") + + plt.tight_layout() + savefig(fig, str(output_dir / "fig6_model_comparison")) + + return {"model_results": model_results, "dataset_results": dataset_results} + + +# ============================================================ +# Deep Analysis: Alignment vs Module Size +# ============================================================ + +def analysis_alignment_vs_module_size( + module_emb: np.ndarray, + text_emb: np.ndarray, + names: List[str], + texts: List[str], + module_json: str, + output_dir: Path, +) -> Dict: + """Scatter: n_genes vs alignment score. Tests if larger modules align better.""" + families = [parse_module_family(n) for n in names] + hallmark_mask = np.array([f == "Hallmarks" for f in families]) + hallmark_idx = np.where(hallmark_mask)[0] + + h_module = module_emb[hallmark_idx] + h_text = text_emb[hallmark_idx] + h_names = [names[i] for i in hallmark_idx] + h_texts = [texts[i] for i in hallmark_idx] + n_h = len(hallmark_idx) + + # Per-module alignment + module_sim = cosine_similarity(h_module) + text_sim = cosine_similarity(h_text) + per_module_corr = np.zeros(n_h) + for i in range(n_h): + mask = np.ones(n_h, dtype=bool) + mask[i] = False + r, _ = spearmanr(module_sim[i, mask], text_sim[i, mask]) + per_module_corr[i] = r + + # Module sizes + module_sizes = get_module_sizes(module_json) + h_sizes = np.array([module_sizes.get(n, 0) for n in h_names]) + + # Correlation + r_size, p_size = spearmanr(h_sizes, per_module_corr) + + # Figure + fig, ax = plt.subplots(figsize=(7, 5)) + h_processes = [parse_hallmark_process(n) or "pathway" for n in h_names] + colors = [PROCESS_COLORS.get(p, "#999999") for p in h_processes] + + ax.scatter(h_sizes, per_module_corr, c=colors, s=50, alpha=0.7, + edgecolors="white", linewidths=0.5) + + # Trend line + z = np.polyfit(h_sizes, per_module_corr, 1) + p_line = np.poly1d(z) + x_range = np.linspace(h_sizes.min(), h_sizes.max(), 100) + ax.plot(x_range, p_line(x_range), "k--", linewidth=1.5, alpha=0.6) + + ax.axhline(0, color="gray", linewidth=0.8, linestyle=":") + ax.set_xlabel("Number of Genes in Module", fontsize=10) + ax.set_ylabel("Alignment Score (Spearman r)", fontsize=10) + ax.set_title(f"Alignment vs Module Size\n" + f"Spearman ρ = {r_size:.3f}, p = {p_size:.4f}", + fontsize=11, fontweight="bold") + + # Label outliers + for i in range(n_h): + if abs(per_module_corr[i]) > 0.35 or h_sizes[i] > 250: + ax.annotate(h_texts[i][:15], (h_sizes[i], per_module_corr[i]), + fontsize=6, alpha=0.8, xytext=(5, 3), + textcoords="offset points") + + plt.tight_layout() + savefig(fig, str(output_dir / "fig_alignment_vs_size")) + + return {"spearman_r": float(r_size), "spearman_p": float(p_size)} + + +# ============================================================ +# Deep Analysis: Negative Result Deep-Dive +# ============================================================ + +def analysis_negative_results( + module_emb: np.ndarray, + text_emb: np.ndarray, + names: List[str], + texts: List[str], + output_dir: Path, + k: int = 5, + n_show: int = 5, +) -> Dict: + """For poorly-aligned modules, show neighbor divergence.""" + families = [parse_module_family(n) for n in names] + hallmark_mask = np.array([f == "Hallmarks" for f in families]) + hallmark_idx = np.where(hallmark_mask)[0] + + h_module = module_emb[hallmark_idx] + h_text = text_emb[hallmark_idx] + h_names = [names[i] for i in hallmark_idx] + h_texts = [texts[i] for i in hallmark_idx] + n_h = len(hallmark_idx) + + # Per-module alignment + module_sim = cosine_similarity(h_module) + text_sim = cosine_similarity(h_text) + per_module_corr = np.zeros(n_h) + for i in range(n_h): + mask = np.ones(n_h, dtype=bool) + mask[i] = False + r, _ = spearmanr(module_sim[i, mask], text_sim[i, mask]) + per_module_corr[i] = r + + # Bottom n_show and top n_show + bottom_idx = np.argsort(per_module_corr)[:n_show] + top_idx = np.argsort(per_module_corr)[-n_show:][::-1] + + rows = [] + for idx_set, label in [(bottom_idx, "bottom"), (top_idx, "top")]: + for i in idx_set: + # kNN in module space + module_nn = np.argsort(-module_sim[i]) + module_nn = [j for j in module_nn if j != i][:k] + # kNN in text space + text_nn = np.argsort(-text_sim[i]) + text_nn = [j for j in text_nn if j != i][:k] + overlap = set(module_nn) & set(text_nn) + + rows.append({ + "rank": label, + "hallmark": h_texts[i], + "alignment_r": float(per_module_corr[i]), + "module_neighbors": ", ".join([h_texts[j] for j in module_nn]), + "text_neighbors": ", ".join([h_texts[j] for j in text_nn]), + "knn_overlap": len(overlap), + }) + + df = pd.DataFrame(rows) + df.to_csv(output_dir / "negative_analysis.csv", index=False) + logger.info("Saved negative_analysis.csv (%d rows)", len(df)) + + return { + "bottom_5": [{ + "hallmark": r["hallmark"], + "r": r["alignment_r"], + "module_nn": r["module_neighbors"], + "text_nn": r["text_neighbors"], + } for _, r in df[df["rank"] == "bottom"].iterrows()], + "top_5": [{ + "hallmark": r["hallmark"], + "r": r["alignment_r"], + } for _, r in df[df["rank"] == "top"].iterrows()], + } + + +# ============================================================ +# Deep Analysis: kNN Overlap with Bootstrap CI +# ============================================================ + +def analysis_knn_overlap( + module_emb: np.ndarray, + text_emb: np.ndarray, + names: List[str], + n_permutations: int = 1000, + n_bootstrap: int = 2000, + k: int = 5, + seed: int = 42, + output_dir: Path = None, +) -> Dict: + """kNN overlap with bootstrap CI and per-category breakdown.""" + families = [parse_module_family(n) for n in names] + hallmark_mask = np.array([f == "Hallmarks" for f in families]) + hallmark_idx = np.where(hallmark_mask)[0] + + h_module = module_emb[hallmark_idx] + h_text = text_emb[hallmark_idx] + h_names = [names[i] for i in hallmark_idx] + n_h = len(hallmark_idx) + + # Compute kNN overlap + module_sim = cosine_similarity(h_module) + text_sim = cosine_similarity(h_text) + + overlaps = np.zeros(n_h) + for i in range(n_h): + module_nn = set(np.argsort(-module_sim[i])[1:k + 1]) + text_nn = set(np.argsort(-text_sim[i])[1:k + 1]) + overlaps[i] = len(module_nn & text_nn) / k + + # Bootstrap CI + mean_obs, ci_lo, ci_hi = bootstrap_ci(overlaps, np.mean, n_bootstrap, seed=seed) + + # Null distribution (permutation) + rng = np.random.default_rng(seed) + null_means = np.zeros(n_permutations) + for p in range(n_permutations): + perm = rng.permutation(n_h) + text_sim_perm = text_sim[perm][:, perm] + ov_perm = np.zeros(n_h) + for i in range(n_h): + module_nn = set(np.argsort(-module_sim[i])[1:k + 1]) + text_nn = set(np.argsort(-text_sim_perm[i])[1:k + 1]) + ov_perm[i] = len(module_nn & text_nn) / k + null_means[p] = ov_perm.mean() + + p_value = float((null_means >= mean_obs).sum() / n_permutations) + null_mean = float(null_means.mean()) + null_std = float(null_means.std()) + z_score = (mean_obs - null_mean) / null_std if null_std > 0 else 0.0 + + # Per-category breakdown + h_processes = [parse_hallmark_process(n) or "pathway" for n in h_names] + cat_overlaps = defaultdict(list) + for i in range(n_h): + cat_overlaps[h_processes[i]].append(overlaps[i]) + + return { + "knn_overlap_mean": mean_obs, + "knn_overlap_ci_95": [ci_lo, ci_hi], + "null_mean": null_mean, + "null_std": null_std, + "p_value": p_value, + "z_score": z_score, + "k": k, + "category_means": {c: float(np.mean(v)) for c, v in cat_overlaps.items()}, + } + + +# ============================================================ +# Main +# ============================================================ + +def main(): + args = parse_args() + output_dir = Path(args.output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + + # Setup logging + log_file = output_dir / "run_publication.log" + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(message)s", + handlers=[ + logging.FileHandler(log_file, mode="w", encoding="utf-8"), + logging.StreamHandler(), + ], + ) + logger.info("Publication analysis starting") + logger.info("Args: %s", vars(args)) + + # ---- Load data ---- + module_names = load_module_names(args.module_json) + all_module_emb = np.load(args.module_embedding_path) + logger.info("Loaded module embeddings: %s from %s", + all_module_emb.shape, args.module_embedding_path) + + # Filter to semantic modules (Hallmarks + Reactome) + families = [parse_module_family(n) for n in module_names] + semantic_mask = np.array([f in ("Hallmarks", "Reactome") for f in families]) + semantic_idx = np.where(semantic_mask)[0] + module_emb = all_module_emb[semantic_idx] + names = [module_names[i] for i in semantic_idx] + logger.info("Semantic modules: %d (Hallmarks + Reactome)", len(names)) + + # ---- Build text descriptions and encode ---- + desc_file = getattr(args, "descriptions_file", None) + _, _, texts = build_text_descriptions(names, descriptions_file=desc_file) + logger.info("Encoding %d texts with primary model: %s", len(texts), args.sentence_model) + text_emb = encode_texts(texts, args.sentence_model, device=args.device) + logger.info("Text embeddings: %s", text_emb.shape) + + # ---- Core metrics (RSA + Mantel) ---- + logger.info("Computing RSA + Mantel test (n_perm=%d)...", args.n_permutations) + mantel_results = mantel_test(module_emb, text_emb, n_perm=args.n_permutations) + logger.info("RSA r=%.4f, p=%.4f, z=%.1f", + mantel_results["rsa_spearman_observed"], + mantel_results["rsa_p_value"], + mantel_results["rsa_effect_size_z"]) + + # ---- Figure 1: RSA Overview ---- + logger.info("Generating Figure 1: RSA Overview...") + metrics_fig1 = fig1_rsa_overview( + module_emb, text_emb, mantel_results, output_dir, + n_bootstrap=args.n_bootstrap, seed=args.seed, + ) + + # ---- Figure 2: Hallmarks Heatmaps ---- + logger.info("Generating Figure 2: Hallmarks Heatmaps...") + metrics_fig2 = fig2_hallmarks_heatmaps( + module_emb, text_emb, names, texts, output_dir, + ) + + # ---- Figure 3: Per-Hallmark Alignment ---- + logger.info("Generating Figure 3: Per-Hallmark Alignment...") + metrics_fig3 = fig3_per_hallmark_alignment( + module_emb, text_emb, names, texts, output_dir, + n_bootstrap=args.n_bootstrap, seed=args.seed, + ) + + # ---- Figure 4: Category Alignment ---- + logger.info("Generating Figure 4: Category Alignment...") + metrics_fig4 = fig4_category_alignment( + module_emb, text_emb, names, texts, output_dir, + ) + + # ---- Figure 5: Granularity Effect ---- + logger.info("Generating Figure 5: Granularity Effect...") + metrics_fig5 = fig5_granularity_effect( + module_emb, text_emb, names, texts, + n_permutations=args.n_permutations, + n_bootstrap=args.n_bootstrap, + seed=args.seed, + output_dir=output_dir, + ) + + # ---- Figure 6: Multi-Model Comparison (optional) ---- + metrics_fig6 = {} + compare_models = [] + compare_embs = [] + if args.sentence_models_compare: + compare_models = [m.strip() for m in args.sentence_models_compare.split(",") + if m.strip()] + logger.info("Loading %d comparison models...", len(compare_models)) + for model_name in compare_models: + logger.info(" Encoding with: %s", model_name) + try: + emb = encode_texts(texts, model_name, device=args.device) + compare_embs.append(emb) + except Exception as e: + logger.warning(" Failed to load %s: %s", model_name, e) + compare_models.remove(model_name) + + module_emb2 = None + if args.module_embedding_path2: + all_module_emb2 = np.load(args.module_embedding_path2) + module_emb2 = all_module_emb2[semantic_idx] + logger.info("Loaded secondary dataset: %s", all_module_emb2.shape) + + if compare_models: + logger.info("Generating Figure 6: Model Comparison...") + metrics_fig6 = fig6_model_comparison( + module_emb, module_emb2, text_emb, names, texts, + primary_model_name=args.sentence_model.split("/")[-1], + compare_model_names=compare_models, + compare_text_embs=compare_embs, + n_permutations=args.n_permutations, + n_bootstrap=args.n_bootstrap, + seed=args.seed, + output_dir=output_dir, + ) + + # ---- Deep Analysis ---- + logger.info("Running deep analysis: alignment vs module size...") + metrics_size = analysis_alignment_vs_module_size( + module_emb, text_emb, names, texts, args.module_json, output_dir, + ) + + logger.info("Running deep analysis: negative results...") + metrics_neg = analysis_negative_results( + module_emb, text_emb, names, texts, output_dir, + ) + + logger.info("Running deep analysis: kNN overlap with bootstrap...") + metrics_knn = analysis_knn_overlap( + module_emb, text_emb, names, + n_permutations=args.n_permutations, + n_bootstrap=args.n_bootstrap, + seed=args.seed, + output_dir=output_dir, + ) + logger.info("kNN overlap: %.3f [%.3f, %.3f], null=%.3f, p=%.4f", + metrics_knn["knn_overlap_mean"], + metrics_knn["knn_overlap_ci_95"][0], + metrics_knn["knn_overlap_ci_95"][1], + metrics_knn["null_mean"], + metrics_knn["p_value"]) + + # ---- Cross-dataset RSA (if secondary available) ---- + metrics_cross = {} + if module_emb2 is not None: + logger.info("Cross-dataset RSA (secondary dataset)...") + mantel2 = mantel_test(module_emb2, text_emb, n_perm=args.n_permutations) + r2, ci2_lo, ci2_hi = bootstrap_rsa( + module_emb2, text_emb, args.n_bootstrap, args.seed + ) + metrics_cross = { + "secondary_rsa_r": r2, + "secondary_rsa_ci_95": [ci2_lo, ci2_hi], + "secondary_mantel_p": mantel2["rsa_p_value"], + "secondary_mantel_z": mantel2["rsa_effect_size_z"], + } + logger.info("Secondary RSA: r=%.4f [%.4f, %.4f], p=%.4f", + r2, ci2_lo, ci2_hi, mantel2["rsa_p_value"]) + + # ---- Aggregate and save metrics ---- + all_metrics = { + "primary_dataset": args.module_embedding_path, + "secondary_dataset": args.module_embedding_path2, + "sentence_model": args.sentence_model, + "n_modules": len(names), + "n_permutations": args.n_permutations, + "n_bootstrap": args.n_bootstrap, + "fig1_rsa_overview": metrics_fig1, + "fig2_heatmaps": metrics_fig2, + "fig3_per_hallmark": metrics_fig3, + "fig4_category": metrics_fig4, + "fig5_granularity": metrics_fig5, + "fig6_model_comparison": metrics_fig6, + "alignment_vs_size": metrics_size, + "negative_analysis": metrics_neg, + "knn_overlap": metrics_knn, + "cross_dataset": metrics_cross, + } + + metrics_path = output_dir / "publication_metrics.json" + with open(metrics_path, "w", encoding="utf-8") as f: + json.dump(all_metrics, f, indent=2, ensure_ascii=False, default=str) + logger.info("Saved: %s", metrics_path) + logger.info("Publication analysis complete. Output: %s", output_dir) + + +if __name__ == "__main__": + main() diff --git a/BioMod/models_performer_module.py b/BioMod/models_performer_module.py index db30dea20feafe0de31c5060869b31d24ebfb788..7752932147bad6968924fc75001e89fb503d73ef 100644 --- a/BioMod/models_performer_module.py +++ b/BioMod/models_performer_module.py @@ -1,1716 +1,1657 @@ -""" -Encoder-only Performer模型,使用module数据输入 -参考MAE的masking方式,但使用Performer作为encoder -只保留encoder,去掉decoder -""" - -import sys -import os -parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -if parent_dir not in sys.path: - sys.path.insert(0, parent_dir) - -# The bundled Performer implementation lives under model_dependencies rather -# than at the repository root. Add that directory explicitly so every task -# entry point can import the model without requiring a caller-managed -# PYTHONPATH. -model_dependencies_dir = os.path.join(parent_dir, "model_dependencies") -if os.path.isdir(model_dependencies_dir) and model_dependencies_dir not in sys.path: - sys.path.insert(0, model_dependencies_dir) - -try: - from performer_pytorch_cont.ding_models import ( - Performer, - Always, - cast_tuple, - ) -except ImportError: - try: - from performer_pytorch_cont.performer_pytorch import ( - Performer, - Always, - cast_tuple, - ) - except ImportError: - from performer_pytorch.performer_pytorch import ( - Performer, - Always, - cast_tuple, - ) -import torch -from torch import nn -import torch.nn.functional as F -from typing import Callable, Dict, List, Optional, Tuple, Union -import numpy as np - - -def _migrate_legacy_module_name(value: str) -> str: - """Translate serialized pre-rename names without changing their semantics.""" - replacements = ( - ("PATHCH", "MODULE"), - ("Pathch", "Module"), - ("pathch", "module"), - ("PATCHES", "MODULES"), - ("Patches", "Modules"), - ("patches", "modules"), - ("PATCH", "MODULE"), - ("Patch", "Module"), - ("patch", "module"), - ) - for old, new in replacements: - value = value.replace(old, new) - return value - - -def migrate_legacy_module_config(config: Dict[str, object]) -> Dict[str, object]: - """Return a module-named copy of an older serialized configuration.""" - def translate(value): - if isinstance(value, dict): - migrated = {} - for key, item in value.items(): - new_key = _migrate_legacy_module_name(key) if isinstance(key, str) else key - if new_key in migrated: - raise ValueError( - f"Legacy configuration migration would duplicate key {new_key!r}" - ) - migrated[new_key] = translate(item) - return migrated - if isinstance(value, list): - return [translate(item) for item in value] - if isinstance(value, tuple): - return tuple(translate(item) for item in value) - if isinstance(value, str): - return _migrate_legacy_module_name(value) - return value - - return translate(dict(config)) - - -def migrate_legacy_module_state_dict(state_dict): - """Translate old state-dict key names before PyTorch performs strict loading.""" - try: - migrated = state_dict.__class__() - except TypeError: - migrated = {} - - for key, value in state_dict.items(): - new_key = _migrate_legacy_module_name(key) if isinstance(key, str) else key - if new_key in migrated: - # If both spellings are supplied, prefer the explicit new spelling. - if key == new_key: - migrated[new_key] = value - continue - migrated[new_key] = value - - if hasattr(state_dict, "_metadata"): - migrated._metadata = { - _migrate_legacy_module_name(key): value - for key, value in state_dict._metadata.items() - } - return migrated - - -def normalize_fusion_method(fusion_method: str) -> str: - if fusion_method == "expression_only_mlp": - return "add" - if fusion_method in {"concatenate", "concat_v2"}: - raise ValueError( - f"fusion_method={fusion_method!r} has been removed. " - "Use one of: 'multiply', 'add', 'gated_residual'." - ) - if fusion_method not in {"multiply", "add", "gated_residual"}: - raise ValueError( - f"Unsupported fusion_method: {fusion_method}. " - "Use one of: 'multiply', 'add', 'gated_residual'." - ) - return fusion_method - - -REMOVED_PRETRAIN_CONFIG_KEYS = ( - "preprocess_gene2vec", - "extra_gene_mode", - "extra_gene_num", - "extra_gene_rank_path", - "extra_gene_module_size", - "extra_gene_module_strategy", - "module_embed_chunk_padded_gene_budget", - "cross_attn_pure_memory", -) - - -def sanitize_performer_module_config( - config: Dict[str, object], - *, - source: str = "config", - log_fn: Optional[Callable[[str], None]] = None, -) -> Dict[str, object]: - cleaned = migrate_legacy_module_config(config) - legacy_mode = str(cleaned.get("extra_gene_mode", "none")).lower() - legacy_num = int(cleaned.get("extra_gene_num", 0) or 0) - if legacy_mode != "none" or legacy_num > 0: - raise ValueError( - f"{source} requests removed extra-gene support " - f"(extra_gene_mode={legacy_mode}, extra_gene_num={legacy_num}). " - "Please use a checkpoint trained without extra genes." - ) - - removed = [key for key in REMOVED_PRETRAIN_CONFIG_KEYS if key in cleaned] - for key in removed: - cleaned.pop(key, None) - - if removed and log_fn is not None: - log_fn(f"{source}: dropped deprecated config keys: {', '.join(sorted(removed))}") - - return cleaned - -# 仅复用 ModuleToGeneDecoder,ModuleEmbedding 在本文件中重新实现以支持三种融合方式 -# from sclong_module.models_module import ModuleToGeneDecoder - - -class CrossAttentionDecoderBlock(nn.Module): - """ - Decoder block with masked-query -> visible-memory cross-attention only. - """ - - def __init__( - self, - dim: int, - heads: int, - ff_mult: int = 4, - attn_dropout: float = 0.0, - ff_dropout: float = 0.0, - ) -> None: - super().__init__() - if heads <= 0: - raise ValueError(f"CrossAttentionDecoderBlock requires heads > 0, got {heads}") - if dim % heads != 0: - raise ValueError( - f"CrossAttentionDecoderBlock requires dim divisible by heads, got dim={dim}, heads={heads}" - ) - - self.query_norm = nn.LayerNorm(dim) - self.memory_norm = nn.LayerNorm(dim) - self.cross_attn = nn.MultiheadAttention( - embed_dim=dim, - num_heads=heads, - dropout=attn_dropout, - batch_first=True, - ) - hidden_dim = int(dim * ff_mult) - self.ff_norm = nn.LayerNorm(dim) - self.ff = nn.Sequential( - nn.Linear(dim, hidden_dim), - nn.GELU(), - nn.Dropout(ff_dropout), - nn.Linear(hidden_dim, dim), - nn.Dropout(ff_dropout), - ) - - def forward(self, query: torch.Tensor, memory: torch.Tensor) -> torch.Tensor: - if query.shape[1] == 0: - return query - - if memory.shape[1] > 0: - attn_out, _ = self.cross_attn( - self.query_norm(query), - self.memory_norm(memory), - self.memory_norm(memory), - need_weights=False, - ) - query = query + attn_out - - query = query + self.ff(self.ff_norm(query)) - return query - - -class ModuleEmbedding(nn.Module): - """ - Gene-level embedding + MLP + attention pooling -> module token. - 支持三种融合方式: - - "multiply": 表达值 × gene2vec(原始方式) - - "add": 表达值经过独立 MLP,再与 gene embedding 相加 - - "gated_residual": 表达值先映射到 gene 空间,再由 gate 控制残差注入强度 - - 支持 use_gene2vec 参数控制是否使用预训练的 gene2vec: - - True: 使用预训练的 gene2vec 权重初始化(默认行为) - - False: 使用随机初始化的 embedding(用于消融实验) - """ - - @staticmethod - def _build_adaptive_buckets( - modules_by_length: Dict[int, List[int]], - bucket_count: int, - ) -> Dict[int, List[int]]: - """Minimize padded gene positions for a fixed number of buckets.""" - length_groups = sorted(modules_by_length.items()) - group_count = len(length_groups) - bucket_count = min(bucket_count, group_count) - if bucket_count <= 0: - raise ValueError("MODULE_EMBED_BUCKET_COUNT must be positive") - - prefix_counts = [0] - for _, module_ids in length_groups: - prefix_counts.append(prefix_counts[-1] + len(module_ids)) - - # dp[b][j] is the least padded element count for the first j distinct - # lengths split into b contiguous buckets. Contiguous length ranges - # preserve a single static padded shape per bucket. - infinity = float("inf") - previous = [infinity] * (group_count + 1) - previous[0] = 0 - split_points: List[List[int]] = [ - [-1] * (group_count + 1) for _ in range(bucket_count + 1) - ] - for bucket_idx in range(1, bucket_count + 1): - current = [infinity] * (group_count + 1) - for end in range(bucket_idx, group_count + 1): - bucket_length = length_groups[end - 1][0] - best_cost = infinity - best_start = -1 - for start in range(bucket_idx - 1, end): - cost = previous[start] + bucket_length * ( - prefix_counts[end] - prefix_counts[start] - ) - if cost < best_cost: - best_cost = cost - best_start = start - current[end] = best_cost - split_points[bucket_idx][end] = best_start - previous = current - - ranges = [] - end = group_count - for bucket_idx in range(bucket_count, 0, -1): - start = split_points[bucket_idx][end] - ranges.append((start, end)) - end = start - ranges.reverse() - - buckets: Dict[int, List[int]] = {} - for start, end in ranges: - bucket_length = length_groups[end - 1][0] - buckets[bucket_length] = [ - module_idx - for _, module_ids in length_groups[start:end] - for module_idx in module_ids - ] - return buckets - - def __init__( - self, - module_sets: List[List[int]], - num_genes: int, - gene_embed_dim: int, - embed_dim: int, - gene2vec_weight: Optional[torch.Tensor] = None, - fusion_method: str = "multiply", # "multiply" / "add" / "gated_residual" - use_gene2vec: bool = True, # 是否使用预训练的 gene2vec(False 则随机初始化) - ): - super().__init__() - self.num_modules = len(module_sets) - self.module_index_names: List[str] = [] - modules_by_length: Dict[int, List[int]] = {} - for module_idx, module_genes in enumerate(module_sets): - buffer_name = f"_module_gene_idx_{module_idx}" - # Module indices are static metadata. Non-persistent buffers follow - # module.to(device) without changing checkpoint state_dict keys. - self.register_buffer( - buffer_name, - torch.tensor(module_genes, dtype=torch.long), - persistent=False, - ) - self.module_index_names.append(buffer_name) - modules_by_length.setdefault(len(module_genes), []).append(module_idx) - - # Group modules with the same gene count so their identical operations - # can be issued as a single batched operation. All buffers are - # non-persistent: checkpoint keys remain byte-for-byte compatible. - self.module_batch_mode = os.environ.get("MODULE_EMBED_BATCH_MODE", "adaptive").lower() - if self.module_batch_mode not in {"legacy", "exact", "bucket", "adaptive"}: - raise ValueError( - "MODULE_EMBED_BATCH_MODE must be one of: " - "'legacy', 'exact', 'bucket', 'adaptive'; " - f"got {self.module_batch_mode!r}" - ) - self.module_groups: List[Tuple[int, Tuple[int, ...], int, int]] = [] - grouped_gene_indices: List[int] = [] - grouped_module_order: List[int] = [] - group_offset = 0 - for module_length in sorted(modules_by_length): - module_ids = tuple(modules_by_length[module_length]) - group_gene_indices = [ - gene_idx - for module_idx in module_ids - for gene_idx in module_sets[module_idx] - ] - group_end = group_offset + len(group_gene_indices) - self.module_groups.append( - (module_length, module_ids, group_offset, group_end) - ) - grouped_gene_indices.extend(group_gene_indices) - grouped_module_order.extend(module_ids) - group_offset = group_end - - restore_order = [0] * self.num_modules - for grouped_pos, module_idx in enumerate(grouped_module_order): - restore_order[module_idx] = grouped_pos - self.register_buffer( - "_grouped_gene_idx", - torch.tensor(grouped_gene_indices, dtype=torch.long), - persistent=False, - ) - self.register_buffer( - "_module_group_restore_idx", - torch.tensor(restore_order, dtype=torch.long), - persistent=False, - ) - - if self.module_batch_mode == "adaptive": - bucket_count = int(os.environ.get("MODULE_EMBED_BUCKET_COUNT", "16")) - modules_by_bucket = self._build_adaptive_buckets( - modules_by_length, - bucket_count, - ) - else: - modules_by_bucket: Dict[int, List[int]] = {} - for module_idx, module_genes in enumerate(module_sets): - bucket_length = 1 << (len(module_genes) - 1).bit_length() - modules_by_bucket.setdefault(bucket_length, []).append(module_idx) - self.module_buckets: List[ - Tuple[int, Tuple[int, ...], Tuple[int, ...], int, int, str] - ] = [] - bucketed_gene_indices: List[int] = [] - bucketed_module_order: List[int] = [] - bucket_offset = 0 - for bucket_idx, bucket_length in enumerate(sorted(modules_by_bucket)): - module_ids = tuple(modules_by_bucket[bucket_length]) - module_lengths = tuple(len(module_sets[idx]) for idx in module_ids) - group_gene_indices: List[int] = [] - valid_mask = torch.zeros(len(module_ids), bucket_length, dtype=torch.bool) - for group_pos, module_idx in enumerate(module_ids): - genes = module_sets[module_idx] - pad_count = bucket_length - len(genes) - group_gene_indices.extend(genes) - group_gene_indices.extend([0] * pad_count) - valid_mask[group_pos, :len(genes)] = True - group_end = bucket_offset + len(group_gene_indices) - mask_name = f"_module_bucket_mask_{bucket_idx}" - self.register_buffer(mask_name, valid_mask, persistent=False) - self.module_buckets.append( - ( - bucket_length, - module_ids, - module_lengths, - bucket_offset, - group_end, - mask_name, - ) - ) - bucketed_gene_indices.extend(group_gene_indices) - bucketed_module_order.extend(module_ids) - bucket_offset = group_end - - bucket_restore_order = [0] * self.num_modules - for grouped_pos, module_idx in enumerate(bucketed_module_order): - bucket_restore_order[module_idx] = grouped_pos - self.register_buffer( - "_bucketed_gene_idx", - torch.tensor(bucketed_gene_indices, dtype=torch.long), - persistent=False, - ) - self.register_buffer( - "_module_bucket_restore_idx", - torch.tensor(bucket_restore_order, dtype=torch.long), - persistent=False, - ) - # Deduplicate shared pointwise transformations, preserving module-local - # membership, order, multiplicity and attention normalization. - # Static maps add no checkpoint keys or trainable parameters. - unique_genes = sorted({gene for module in module_sets for gene in module}) - unique_position = {gene: pos for pos, gene in enumerate(unique_genes)} - bucket_unique_positions = [ - unique_position.get(gene, 0) for gene in bucketed_gene_indices - ] - self.register_buffer( - "_unique_gene_idx", torch.tensor(unique_genes, dtype=torch.long), - persistent=False, - ) - self.register_buffer( - "_bucket_unique_idx", - torch.tensor(bucket_unique_positions, dtype=torch.long), - persistent=False, - ) - self.gene_dedup_enabled = os.environ.get("MODULE_GENE_DEDUP", "1") == "1" - # The unlimited candidate was slower at batches 16/64/256 on H100. - # Default to the measured beneficial batch=1 path; 0 enables an explicit - # unlimited research comparison without changing checkpoint structure. - self.gene_dedup_max_batch_size = int( - os.environ.get("MODULE_GENE_DEDUP_MAX_BATCH_SIZE", "1") - ) - if self.gene_dedup_max_batch_size < 0: - raise ValueError("MODULE_GENE_DEDUP_MAX_BATCH_SIZE must be nonnegative") - self.fusion_method = normalize_fusion_method(fusion_method) - self.use_gene2vec = use_gene2vec - self.gene_embed_dim = gene_embed_dim - self.embed_dim = embed_dim - - # 根据 use_gene2vec 决定是否使用预训练权重 - if use_gene2vec and gene2vec_weight is not None: - self.gene_embed = nn.Embedding.from_pretrained(gene2vec_weight, freeze=False) - if self.gene_embed.weight.shape[0] < num_genes: - pad = torch.zeros(num_genes - self.gene_embed.weight.shape[0], self.gene_embed.weight.shape[1]) - self.gene_embed.weight.data = torch.cat([self.gene_embed.weight.data, pad], dim=0) - else: - # 随机初始化 embedding(消融实验:不使用 gene2vec 先验知识) - self.gene_embed = nn.Embedding(num_genes, gene_embed_dim) - # 使用 Xavier 初始化,与其他层保持一致 - nn.init.xavier_uniform_(self.gene_embed.weight) - - # 融合层(对齐MAE结构) - # 所有方式的proj都是: Dg → E → E(两层MLP) - self.expr_mlp = None - self.gated_expr_mlp = None - self.gated_residual_gate = None - - if self.fusion_method == "add": - # add: 表达值经过独立MLP,再与gene2vec相加 - hidden_dim = max(50, gene_embed_dim // 2) - self.expr_mlp = nn.Sequential( - nn.Linear(1, hidden_dim), - nn.GELU(), - nn.Linear(hidden_dim, gene_embed_dim), - ) - elif self.fusion_method == "gated_residual": - # gated_residual: gene2vec提供基底语义,表达值通过门控残差注入。 - hidden_dim = max(64, gene_embed_dim // 2) - self.gated_expr_mlp = nn.Sequential( - nn.Linear(1, hidden_dim), - nn.GELU(), - nn.Linear(hidden_dim, gene_embed_dim), - nn.LayerNorm(gene_embed_dim), - ) - self.gated_residual_gate = nn.Sequential( - nn.Linear(gene_embed_dim * 2, gene_embed_dim), - nn.GELU(), - nn.Linear(gene_embed_dim, gene_embed_dim), - nn.Sigmoid(), - ) - # else: multiply方式不需要额外的MLP - - # 统一的proj层: Dg → E → E - self.proj = nn.Sequential( - nn.Linear(gene_embed_dim, embed_dim), - nn.GELU(), - nn.Linear(embed_dim, embed_dim), - ) - self.attn_score = nn.Linear(embed_dim, 1) - self.pool_affine_enabled = os.environ.get("MODULE_POOL_AFFINE", "1") == "1" - # Paired H100 screening found this path helps batches 16/64/256 but - # regresses batch=1 against the dedup fast path. Preserve that fast path. - self.pool_affine_min_batch_size = int(os.environ.get("MODULE_POOL_AFFINE_MIN_BATCH_SIZE", "2")) - if self.pool_affine_min_batch_size < 1: - raise ValueError("MODULE_POOL_AFFINE_MIN_BATCH_SIZE must be positive") - # Derived model weights only; no cell-dependent features are cached. - # These buffers add no parameters or checkpoint entries. - self.register_buffer("_pool_affine_score_weight", torch.empty(0), persistent=False) - self.register_buffer("_pool_affine_score_bias", torch.empty(0), persistent=False) - self._pool_affine_cache_signature = None - - @property - def module_sets(self) -> List[torch.Tensor]: - """Expose module indices for existing interpretability callers.""" - return [getattr(self, name) for name in self.module_index_names] - - def _pool_affine_signature(self): - tensors = (self.proj[2].weight, self.proj[2].bias, - self.attn_score.weight, self.attn_score.bias) - return tuple((id(t), t.data_ptr(), t._version, t.device, t.dtype) for t in tensors) - - @torch.no_grad() - def prepare_pool_affine(self): - """Compose final affine and score AFTER checkpoint/device/dtype placement. - - Call again after changing weights, dtype or device. Forward rejects a - stale cache rather than silently using outdated model-derived weights. - """ - weight = self.proj[2].weight - compose_dtype = torch.float64 if weight.dtype == torch.float64 else torch.float32 - with torch.autocast(device_type=weight.device.type, enabled=False): - score_weight = self.attn_score.weight.to(dtype=compose_dtype) - self._pool_affine_score_weight = ( - score_weight @ weight.to(dtype=compose_dtype) - ).to(dtype=weight.dtype) - self._pool_affine_score_bias = ( - score_weight @ self.proj[2].bias.to(dtype=compose_dtype) - + self.attn_score.bias.to(dtype=compose_dtype) - ).to(dtype=weight.dtype) - self._pool_affine_cache_signature = self._pool_affine_signature() - - def _forward_pool_affine( - self, - expr: torch.Tensor, - return_module_targets: bool = True, - ) -> Tuple[torch.Tensor, List[torch.Tensor]]: - """Pool first-layer activations before the shared final affine. - - In real arithmetic, score(W2*h+b2)=(score_weight*W2)*h+const, - and sum(alpha*(W2*h+b2))=W2*sum(alpha*h)+b2*sum(alpha). - BF16 reassociation is validated separately; it is not bitwise equivalent. - """ - if self._pool_affine_cache_signature != self._pool_affine_signature(): - raise RuntimeError("Pool-affine cache is stale; call prepare_pool_affine() after loading/placing weights") - batch_size = expr.shape[0] - flat_expr = torch.index_select(expr, 1, self._bucketed_gene_idx) - flat_gene_emb = self.gene_embed(self._bucketed_gene_idx) - pooled_hidden = [] - attention_sums = [] - module_targets: List[Optional[torch.Tensor]] = ( - [None] * self.num_modules if return_module_targets else [] - ) - for ( - bucket_length, module_ids, module_lengths, start, end, mask_name - ) in self.module_buckets: - group_size = len(module_ids) - expr_group = flat_expr[:, start:end].reshape(batch_size, group_size, bucket_length) - gene_emb = flat_gene_emb[start:end].reshape(group_size, bucket_length, self.gene_embed_dim) - gene_emb = gene_emb.unsqueeze(0).expand(batch_size, -1, -1, -1) - fused = self._fuse_expr_and_gene_emb(expr_group, gene_emb) - hidden = self.proj[1](self.proj[0](fused)) - del fused - scores = F.linear(hidden, self._pool_affine_score_weight, self._pool_affine_score_bias).squeeze(-1) - valid_mask = getattr(self, mask_name) - scores = scores.masked_fill(~valid_mask.unsqueeze(0), float("-inf")) - attn = torch.softmax(scores, dim=2) - pooled_hidden.append(self._weighted_pool(hidden, attn)) - # Quantized softmax weights need not sum exactly to one. Preserve - # their sum in wider accumulation for the final bias contribution. - sum_dtype = torch.float64 if hidden.dtype == torch.float64 else torch.float32 - attention_sums.append(attn.sum(dim=2, dtype=sum_dtype)) - del hidden, scores, attn - if return_module_targets: - for group_pos, (module_idx, module_length) in enumerate(zip(module_ids, module_lengths)): - module_targets[module_idx] = expr_group[:, group_pos, :module_length] - pooled = torch.cat(pooled_hidden, dim=1) - sums = torch.cat(attention_sums, dim=1) - # One final affine over the module tokens, not over every gene position. - linear = F.linear(pooled, self.proj[2].weight, None) - with torch.autocast(device_type=expr.device.type, enabled=False): - wider = linear.to(dtype=sums.dtype) + sums.unsqueeze(-1) * self.proj[2].bias.to(dtype=sums.dtype) - tokens = wider.to(dtype=linear.dtype) - tokens = torch.index_select(tokens, 1, self._module_bucket_restore_idx) - return tokens, module_targets # type: ignore[return-value] - - def _fuse_expr_and_gene_emb(self, expr_slice: torch.Tensor, gene_emb: torch.Tensor) -> torch.Tensor: - """ - 将表达值与gene embedding融合到gene-token空间。 - 支持任意前缀维度,只要求最后一维是gene维。 - expr_slice: (..., G) / gene_emb: (..., G, Dg) -> (..., G, Dg) - """ - if self.fusion_method == "add": - expr_emb = self.expr_mlp(expr_slice.unsqueeze(-1)) # (B, G, Dg) - return expr_emb + gene_emb - if self.fusion_method == "gated_residual": - expr_emb = self.gated_expr_mlp(expr_slice.unsqueeze(-1)) # (B, G, Dg) - gate = self.gated_residual_gate(torch.cat([expr_emb, gene_emb], dim=-1)) # (B, G, Dg) - return gene_emb + gate * expr_emb - # 默认 multiply:表达值 × gene2vec - return expr_slice.unsqueeze(-1) * gene_emb # (B, G, Dg) - - @staticmethod - def _weighted_pool(gene_tok: torch.Tensor, attn: torch.Tensor) -> torch.Tensor: - """Pool without materializing the full attention-weighted tensor.""" - implementation = os.environ.get("MODULE_POOL_IMPL", "bmm") - if implementation == "bmm": - length, width = gene_tok.shape[-2:] - pooled = torch.bmm( - attn.reshape(-1, 1, length), - gene_tok.reshape(-1, length, width), - ) - return pooled.reshape(*gene_tok.shape[:-2], width) - if implementation != "mulsum": - raise ValueError(f"Unsupported MODULE_POOL_IMPL={implementation}") - return torch.sum(gene_tok * attn.unsqueeze(-1), dim=-2) - - def _forward_legacy( - self, - expr: torch.Tensor, - return_module_targets: bool = True, - ) -> Tuple[torch.Tensor, List[torch.Tensor]]: - """ - 逐module稳定实现。 - expr: (batch, num_genes) - returns: - module_tokens: (batch, num_modules, embed_dim) - module_targets: list of length num_modules, each (batch, genes_in_module) - """ - batch_size, _ = expr.shape - module_tokens = [] - module_targets: List[torch.Tensor] = [] - for index_name in self.module_index_names: - gene_idx = getattr(self, index_name) - expr_slice = torch.index_select(expr, dim=1, index=gene_idx) - if return_module_targets: - module_targets.append(expr_slice) - - gene_emb = self.gene_embed(gene_idx) - gene_emb = gene_emb.unsqueeze(0).expand(batch_size, -1, -1) - gene_tok = self._fuse_expr_and_gene_emb(expr_slice, gene_emb) - gene_tok = self.proj(gene_tok) - - attn = self.attn_score(gene_tok).squeeze(-1) - attn = torch.softmax(attn, dim=1) - pooled = self._weighted_pool(gene_tok, attn) - module_tokens.append(pooled) - - return torch.stack(module_tokens, dim=1), module_targets - - def _forward_grouped_exact( - self, - expr: torch.Tensor, - return_module_targets: bool = True, - ) -> Tuple[torch.Tensor, List[torch.Tensor]]: - """Batch modules of identical length without padding or approximation.""" - batch_size, _ = expr.shape - flat_expr = torch.index_select(expr, dim=1, index=self._grouped_gene_idx) - flat_gene_emb = self.gene_embed(self._grouped_gene_idx) - grouped_tokens = [] - module_targets: List[Optional[torch.Tensor]] = ( - [None] * self.num_modules if return_module_targets else [] - ) - - for module_length, module_ids, start, end in self.module_groups: - group_size = len(module_ids) - expr_group = flat_expr[:, start:end].reshape( - batch_size, group_size, module_length - ) - gene_emb = flat_gene_emb[start:end].reshape( - group_size, module_length, self.gene_embed_dim - ) - gene_emb = gene_emb.unsqueeze(0).expand(batch_size, -1, -1, -1) - gene_tok = self._fuse_expr_and_gene_emb(expr_group, gene_emb) - gene_tok = self.proj(gene_tok) - - attn = self.attn_score(gene_tok).squeeze(-1) - attn = torch.softmax(attn, dim=2) - pooled = self._weighted_pool(gene_tok, attn) - grouped_tokens.append(pooled) - - if return_module_targets: - for group_pos, module_idx in enumerate(module_ids): - module_targets[module_idx] = expr_group[:, group_pos, :] - - tokens = torch.cat(grouped_tokens, dim=1) - tokens = torch.index_select( - tokens, dim=1, index=self._module_group_restore_idx - ) - # Every module is assigned exactly once when groups are constructed. - return tokens, module_targets # type: ignore[return-value] - - def _forward_bucketed( - self, - expr: torch.Tensor, - return_module_targets: bool = True, - ) -> Tuple[torch.Tensor, List[torch.Tensor]]: - """Batch modules into fixed padded lengths and mask padded positions.""" - batch_size, _ = expr.shape - flat_expr = torch.index_select(expr, dim=1, index=self._bucketed_gene_idx) - flat_gene_emb = self.gene_embed(self._bucketed_gene_idx) - bucket_tokens = [] - module_targets: List[Optional[torch.Tensor]] = ( - [None] * self.num_modules if return_module_targets else [] - ) - - for ( - bucket_length, - module_ids, - module_lengths, - start, - end, - mask_name, - ) in self.module_buckets: - group_size = len(module_ids) - expr_group = flat_expr[:, start:end].reshape( - batch_size, group_size, bucket_length - ) - gene_emb = flat_gene_emb[start:end].reshape( - group_size, bucket_length, self.gene_embed_dim - ) - gene_emb = gene_emb.unsqueeze(0).expand(batch_size, -1, -1, -1) - gene_tok = self._fuse_expr_and_gene_emb(expr_group, gene_emb) - gene_tok = self.proj(gene_tok) - - valid_mask = getattr(self, mask_name) - attn = self.attn_score(gene_tok).squeeze(-1) - attn = attn.masked_fill(~valid_mask.unsqueeze(0), float("-inf")) - attn = torch.softmax(attn, dim=2) - pooled = self._weighted_pool(gene_tok, attn) - bucket_tokens.append(pooled) - - if return_module_targets: - for group_pos, (module_idx, module_length) in enumerate( - zip(module_ids, module_lengths) - ): - module_targets[module_idx] = expr_group[:, group_pos, :module_length] - - tokens = torch.cat(bucket_tokens, dim=1) - tokens = torch.index_select( - tokens, dim=1, index=self._module_bucket_restore_idx - ) - return tokens, module_targets # type: ignore[return-value] - - def _forward_unique_genes( - self, - expr: torch.Tensor, - return_module_targets: bool = True, - ) -> Tuple[torch.Tensor, List[torch.Tensor]]: - """Reuse gene features within this inference forward; pool per module. - - Gather one bucket at a time rather than the full repeated-gene tensor. - """ - batch_size = expr.shape[0] - unique_expr = torch.index_select(expr, 1, self._unique_gene_idx) - unique_emb = self.gene_embed(self._unique_gene_idx) - unique_emb = unique_emb.unsqueeze(0).expand(batch_size, -1, -1) - fused = self._fuse_expr_and_gene_emb(unique_expr, unique_emb) - unique_tokens = self.proj(fused) - del fused, unique_emb - unique_scores = self.attn_score(unique_tokens).squeeze(-1) - bucket_tokens = [] - module_targets: List[Optional[torch.Tensor]] = ( - [None] * self.num_modules if return_module_targets else [] - ) - for ( - bucket_length, module_ids, module_lengths, start, end, mask_name - ) in self.module_buckets: - group_size = len(module_ids) - indices = self._bucket_unique_idx[start:end] - scores = torch.index_select(unique_scores, 1, indices).reshape( - batch_size, group_size, bucket_length - ) - valid_mask = getattr(self, mask_name) - scores = scores.masked_fill(~valid_mask.unsqueeze(0), float("-inf")) - attn = torch.softmax(scores, dim=2) - gene_tok = torch.index_select(unique_tokens, 1, indices).reshape( - batch_size, group_size, bucket_length, self.embed_dim - ) - bucket_tokens.append(self._weighted_pool(gene_tok, attn)) - del gene_tok, attn, scores - if return_module_targets: - expr_group = torch.index_select(unique_expr, 1, indices).reshape( - batch_size, group_size, bucket_length - ) - for group_pos, (module_idx, module_length) in enumerate( - zip(module_ids, module_lengths) - ): - module_targets[module_idx] = expr_group[:, group_pos, :module_length] - tokens = torch.cat(bucket_tokens, dim=1) - tokens = torch.index_select(tokens, 1, self._module_bucket_restore_idx) - return tokens, module_targets # type: ignore[return-value] - - def forward( - self, - expr: torch.Tensor, - return_module_targets: bool = True, - ) -> Tuple[torch.Tensor, List[torch.Tensor]]: - # Keep original training/gradient behavior and gradient reduction order. - if ( - self.pool_affine_enabled and not self.training and not torch.is_grad_enabled() - and expr.shape[0] >= self.pool_affine_min_batch_size - ): - return self._forward_pool_affine(expr, return_module_targets) - if ( - self.gene_dedup_enabled and not self.training and not torch.is_grad_enabled() - and (self.gene_dedup_max_batch_size == 0 - or expr.shape[0] <= self.gene_dedup_max_batch_size) - ): - return self._forward_unique_genes(expr, return_module_targets) - if self.module_batch_mode == "legacy": - return self._forward_legacy(expr, return_module_targets) - if self.module_batch_mode == "exact": - return self._forward_grouped_exact(expr, return_module_targets) - return self._forward_bucketed(expr, return_module_targets) - - -class PerformerEncoderOnly_Module(nn.Module): - """ - Encoder-only Performer模型,使用module数据输入 - 参考MAE的masking方式,但使用Performer作为encoder - - 支持 use_gene2vec 参数控制是否使用预训练的 gene2vec: - - True: 使用预训练的 gene2vec 权重初始化(默认行为) - - False: 使用随机初始化的 embedding(用于消融实验) - """ - def __init__( - self, - module_sets: List[List[int]], # module数据 - num_genes: int, # 基因总数 - max_seq_len, # max length of sequence (module数量) - base_dim = 200, #基础embedding维度,所有token映射到这个维度 - enc_depth = 24, # encoder深度(参考MAE) - enc_heads = 16, # encoder头数(参考MAE) - enc_dim_head = 64, # encoder每个头的维度 - mask_ratio: float = 0.75, # masking比例(参考MAE) - # ==================== Performer特有参数 ==================== - local_attn_heads = 0,# 使用局部注意力的头数 - local_window_size = 256,# 局部注意力的窗口大小 - causal = False, # 是否使用因果(自回归)注意力 - ff_mult = 4,# 前馈网络(FFN)的扩展倍数 - nb_features = None,# Performer随机特征的数量 - feature_redraw_interval = 1000,# 重绘随机特征的步数间隔 - reversible = False,# 是否使用可逆层(节省显存) - ff_chunks = 1,# FFN的分块数量(用于内存优化) - ff_glu = False, # FFN是否使用GLU(门控线性单元)激活 - # ==================== Dropout参数 ==================== - emb_dropout = 0., - ff_dropout = 0., - attn_dropout = 0., - # ==================== Performer高级参数 ==================== - generalized_attention = False, - kernel_fn = nn.ReLU(), - use_scalenorm = False, - use_rezero = False, - cross_attend = False, - no_projection = False, - # ==================== 位置编码参数 ==================== - g2v_position_emb = False, # 对于modules,使用可学习的positional embedding - auto_check_redraw = True,# 是否自动检查并重绘projection矩阵 - qkv_bias = False, # QKV投影是否使用bias - # ==================== Gene2vec相关参数 ==================== - gene2vec_file = 'data/gene2vec_16906.npy', - gene_embed_dim: int = 200, # module embedding中基因embedding的维度 - fusion_method: str = "multiply", # "multiply" / "add" / "gated_residual" - use_cls_token: bool = False, # 是否使用CLS token(MAE默认不使用) - cls_loss_weight: float = 0.0, # CLS辅助loss权重;>0时直接训练CLS预测静态module均值 - cls_head_hidden_dim: int = 0, # CLS head隐藏层维度;0表示使用base_dim - use_gene2vec: bool = True, # 是否使用预训练的 gene2vec(False 则随机初始化,用于消融实验) - loss_mode: str = "gene_mean", # "gene_mean"(当前) / "module_mean"(MAE风格) - recon_target_level: str = "gene", # "gene"(逐基因重建) / "module_mean_value"(每module输出1个均值) - pretrain_arch: str = "mae", # "mae" / "encoder_only" / "encdec" - mask_style: str = "drop", # "drop" / "replace" - mask_token_mode: str = "learned", # 目前仅支持learned - pos_embed_mode: str = "module_id", # 目前仅支持module_id(learned module-id embedding) - decoder_type: str = "simple", # "none" / "simple" / "performer" / "cross_attn" - decoder_gene_agg: bool = False, # True: 先做module->gene聚合,再基因级重建 - dec_embed_dim: int = 512, - dec_depth: int = 4, - dec_heads: int = 8, - dec_dim_head: int = 64, - dec_ff_mult: int = 4, - dec_ff_dropout: float = 0.0, - dec_attn_dropout: float = 0.0, - cross_attn_pure_memory: Optional[bool] = None, - # ==================== Fine-tuning enhancements ==================== - use_family_emb: bool = False, - module_family_ids: Optional[List[int]] = None, - use_module_gate: bool = False, - ): - super().__init__() - local_attn_heads = cast_tuple(local_attn_heads) - self.max_seq_len = max_seq_len - self.base_dim = base_dim - self.num_genes = num_genes - self.module_sets = module_sets - self.num_modules = len(module_sets) - self.fusion_method = normalize_fusion_method(fusion_method) - self.mask_ratio = mask_ratio - self.zero_mask_identity_fastpath = ( - os.environ.get("ZERO_MASK_IDENTITY_FASTPATH", "1") == "1" - ) - self.use_cls = use_cls_token - self.cls_loss_weight = float(cls_loss_weight) if use_cls_token else 0.0 - self.cls_head_hidden_dim = int(cls_head_hidden_dim) - self.use_gene2vec = use_gene2vec - self.loss_mode = loss_mode - self.recon_target_level = recon_target_level - self.pretrain_arch = str(pretrain_arch).lower() - self.mask_style = str(mask_style).lower() - self.mask_token_mode = str(mask_token_mode).lower() - self.pos_embed_mode = str(pos_embed_mode).lower() - self.decoder_type = decoder_type - self.decoder_gene_agg = decoder_gene_agg - self.dec_embed_dim = dec_embed_dim - # Legacy knob retained only for backward-compatible kwargs/config loading. - # Current pretraining always uses pure visible memory in the cross-attn decoder. - self.cross_attn_pure_memory = True - self.base_module_gene_counts = [len(p) for p in module_sets] - self.module_gene_counts = list(self.base_module_gene_counts) - self.total_module_count = len(self.module_gene_counts) - if self.loss_mode not in {"gene_mean", "module_mean"}: - raise ValueError(f"Unsupported loss_mode: {self.loss_mode}. Use 'gene_mean' or 'module_mean'.") - if self.cls_loss_weight < 0: - raise ValueError(f"cls_loss_weight must be >= 0, got {self.cls_loss_weight}") - if self.cls_head_hidden_dim < 0: - raise ValueError(f"cls_head_hidden_dim must be >= 0, got {self.cls_head_hidden_dim}") - if self.recon_target_level not in {"gene", "module_mean_value"}: - raise ValueError( - f"Unsupported recon_target_level: {self.recon_target_level}. " - "Use 'gene' or 'module_mean_value'." - ) - if self.pretrain_arch not in {"mae", "encoder_only", "encdec"}: - raise ValueError( - f"Unsupported pretrain_arch: {self.pretrain_arch}. " - "Use 'mae', 'encoder_only' or 'encdec'." - ) - if self.mask_style not in {"drop", "replace"}: - raise ValueError( - f"Unsupported mask_style: {self.mask_style}. Use 'drop' or 'replace'." - ) - if self.mask_token_mode not in {"learned"}: - raise ValueError( - f"Unsupported mask_token_mode: {self.mask_token_mode}. " - "Only 'learned' is currently supported." - ) - if self.pos_embed_mode not in {"module_id"}: - raise ValueError( - f"Unsupported pos_embed_mode: {self.pos_embed_mode}. " - "Only 'module_id' is currently supported." - ) - if self.decoder_type not in {"none", "simple", "performer", "cross_attn"}: - raise ValueError( - f"Unsupported decoder_type: {self.decoder_type}. " - "Use 'none', 'simple', 'performer' or 'cross_attn'." - ) - if self.pretrain_arch == "mae": - if self.mask_style != "drop": - raise ValueError( - "pretrain_arch='mae' is fixed to mask_style='drop'. " - "Use pretrain_arch='encoder_only' or 'encdec' for mask_style='replace'." - ) - if self.decoder_type == "none": - raise ValueError("pretrain_arch='mae' requires decoder_type != 'none'.") - elif self.pretrain_arch == "encoder_only": - if self.mask_style != "replace": - raise ValueError("pretrain_arch='encoder_only' requires mask_style='replace'.") - if self.decoder_type != "none": - raise ValueError("pretrain_arch='encoder_only' requires decoder_type='none'.") - else: - if self.mask_style != "replace": - raise ValueError("pretrain_arch='encdec' requires mask_style='replace'.") - if self.decoder_type not in {"performer", "cross_attn"}: - raise ValueError( - "pretrain_arch='encdec' requires decoder_type in {'performer', 'cross_attn'}." - ) - if self.decoder_gene_agg and self.recon_target_level != "gene": - raise ValueError("decoder_gene_agg=True only supports recon_target_level='gene'.") - if self.dec_embed_dim <= 0: - raise ValueError(f"dec_embed_dim must be positive, got {self.dec_embed_dim}") - - # 加载gene2vec权重用于module embedding(仅当 use_gene2vec=True 时使用) - gene2vec_weight = None - if use_gene2vec and gene2vec_file and os.path.exists(gene2vec_file): - gene2vec_weight = torch.from_numpy(np.load(gene2vec_file)).float() - if gene2vec_weight.shape[0] < num_genes: - pad = torch.zeros(num_genes - gene2vec_weight.shape[0], gene2vec_weight.shape[1]) - gene2vec_weight = torch.cat([gene2vec_weight, pad], dim=0) - gene_embed_dim = gene2vec_weight.shape[1] - elif not use_gene2vec: - # 不使用 gene2vec 时,打印提示信息 - print(f"[INFO] use_gene2vec=False: Using randomly initialized gene embeddings (dim={gene_embed_dim})") - - # Module Embedding层 - self.module_embed = ModuleEmbedding( - module_sets=module_sets, - num_genes=num_genes, - gene_embed_dim=gene_embed_dim, - embed_dim=base_dim, - gene2vec_weight=gene2vec_weight, - fusion_method=self.fusion_method, - use_gene2vec=use_gene2vec, - ) - - # Positional embedding for modules - # 对pathway module来说更准确地说是 module-id embedding,而不是几何位置。 - pos_tokens = self.total_module_count + (1 if use_cls_token else 0) - self.pos_emb = nn.Parameter(torch.zeros(1, pos_tokens, base_dim)) - self.layer_pos_emb = Always(None) - - # Family-type embedding (2B): adds family identity to module tokens - self.use_family_emb = use_family_emb - if use_family_emb and module_family_ids is not None: - self.register_buffer( - 'module_family_ids', - torch.tensor(module_family_ids, dtype=torch.long), - ) - self.family_emb = nn.Parameter(torch.zeros(4, base_dim)) - else: - self.module_family_ids = None - self.family_emb = None - - # Sparse module gate (1B): learns per-module scalar gate - self.use_module_gate = use_module_gate - if use_module_gate: - self.module_gate = nn.Sequential( - nn.LayerNorm(base_dim), - nn.Linear(base_dim, 1), - nn.Sigmoid(), - ) - else: - self.module_gate = None - - # CLS token (可选) - if use_cls_token: - self.cls_token = nn.Parameter(torch.zeros(1, 1, base_dim)) - else: - self.cls_token = None - if self.use_cls and self.cls_loss_weight > 0: - cls_hidden_dim = self.cls_head_hidden_dim if self.cls_head_hidden_dim > 0 else base_dim - self.cls_head = nn.Sequential( - nn.LayerNorm(base_dim), - nn.Linear(base_dim, cls_hidden_dim), - nn.GELU(), - nn.Linear(cls_hidden_dim, self.num_modules), - ) - else: - self.cls_head = None - - self.dropout = nn.Dropout(emb_dropout) - - # Encoder (使用Performer) - self.encoder = Performer( - base_dim, enc_depth, enc_heads, enc_dim_head, - local_attn_heads, local_window_size, causal, ff_mult, nb_features, - feature_redraw_interval, reversible, ff_chunks, generalized_attention, - kernel_fn, use_scalenorm, use_rezero, ff_glu, ff_dropout, attn_dropout, - cross_attend, no_projection, auto_check_redraw, qkv_bias - ) - self.encoder_norm = nn.LayerNorm(base_dim) - - # Encoder -> Decoder 维度投影 - # - performer/cross_attn: 允许使用更小的 dec_embed_dim - # - simple/none: 不需要投影,直接使用 base_dim - if self.decoder_type in {"performer", "cross_attn"}: - self.enc_to_dec = nn.Identity() if self.dec_embed_dim == base_dim else nn.Linear(base_dim, self.dec_embed_dim) - self.decoder_dim = self.dec_embed_dim - else: - # simple/none: 不使用 enc_to_dec 投影,直接用 base_dim - self.enc_to_dec = nn.Identity() - self.decoder_dim = base_dim - - # Encoder-side mask token(replace-mask路径使用) - if self.mask_style == "replace": - self.encoder_mask_token = nn.Parameter(torch.zeros(1, 1, base_dim)) - else: - self.encoder_mask_token = None - - # Decoder-side mask token(仅MAE drop-mask路径需要) - if self.pretrain_arch == "mae": - self.mask_token = nn.Parameter(torch.zeros(1, 1, self.decoder_dim)) - else: - self.mask_token = None - - # 可选 decoder:用于让masked token与visible token发生信息交互(更接近MAE) - if self.decoder_type == "performer": - self.decoder_pos_emb = nn.Parameter(torch.zeros(1, pos_tokens, self.dec_embed_dim)) - self.decoder_dropout = nn.Dropout(emb_dropout) - self.decoder = Performer( - self.dec_embed_dim, dec_depth, dec_heads, dec_dim_head, - local_attn_heads=cast_tuple(0), local_window_size=256, causal=False, ff_mult=dec_ff_mult, - nb_features=None, feature_redraw_interval=feature_redraw_interval, - reversible=False, ff_chunks=1, generalized_attention=False, - kernel_fn=nn.ReLU(), use_scalenorm=False, use_rezero=False, ff_glu=False, - ff_dropout=dec_ff_dropout, attn_dropout=dec_attn_dropout, - cross_attend=False, no_projection=False, auto_check_redraw=auto_check_redraw, - qkv_bias=qkv_bias - ) - self.decoder_norm = nn.LayerNorm(self.dec_embed_dim) - elif self.decoder_type == "cross_attn": - self.decoder_pos_emb = nn.Parameter(torch.zeros(1, pos_tokens, self.dec_embed_dim)) - self.decoder_dropout = nn.Dropout(emb_dropout) - self.decoder = nn.ModuleList( - [ - CrossAttentionDecoderBlock( - dim=self.dec_embed_dim, - heads=dec_heads, - ff_mult=dec_ff_mult, - attn_dropout=dec_attn_dropout, - ff_dropout=dec_ff_dropout, - ) - for _ in range(dec_depth) - ] - ) - self.decoder_norm = nn.LayerNorm(self.dec_embed_dim) - else: - self.decoder_pos_emb = None - self.decoder_dropout = None - self.decoder = None - self.decoder_norm = None - - # 预计算 module<->gene 聚合索引(decoder_gene_agg 模式使用) - if self.decoder_gene_agg: - pg_module_idx = [] - pg_gene_idx = [] - for module_idx, module_genes in enumerate(self.module_sets): - for gene_idx in module_genes: - if 0 <= gene_idx < self.num_genes: - pg_module_idx.append(module_idx) - pg_gene_idx.append(gene_idx) - self.register_buffer("pg_module_idx", torch.tensor(pg_module_idx, dtype=torch.long)) - self.register_buffer("pg_gene_idx", torch.tensor(pg_gene_idx, dtype=torch.long)) - pg_gene_counts = torch.zeros(self.num_genes, dtype=torch.long) - if len(pg_gene_idx) > 0: - pg_gene_counts.scatter_add_(0, self.pg_gene_idx, torch.ones_like(self.pg_gene_idx)) - self.register_buffer("pg_gene_counts", pg_gene_counts) - else: - self.pg_module_idx = None - self.pg_gene_idx = None - self.pg_gene_counts = None - - # 重建头: - # - 默认: 每个module一个线性头 - # - decoder_gene_agg: module->gene后,用基因特异性头做重建 - # - module_mean_value: 每个module只输出一个均值标量 - if self.recon_target_level == "gene": - if self.decoder_gene_agg: - self.module_heads = None - self.gene_head = None - # 基因特异性重建头: 每个基因一套 (D->1) 参数 - self.gene_w = nn.Parameter(torch.rand(self.num_genes, self.decoder_dim, 1)) - self.gene_b = nn.Parameter(torch.rand(self.num_genes)) - nn.init.xavier_normal_(self.gene_w) - nn.init.xavier_normal_(self.gene_b.unsqueeze(-1)) - else: - self.module_heads = nn.ModuleList( - [nn.Linear(self.decoder_dim, g) for g in self.module_gene_counts] - ) - self.gene_head = None - self.gene_w = None - self.gene_b = None - self.module_scalar_heads = None - else: - self.module_heads = None - self.gene_head = None - self.gene_w = None - self.gene_b = None - self.module_scalar_heads = nn.ModuleList( - [nn.Linear(self.decoder_dim, 1) for _ in self.module_gene_counts] - ) - - self.last_loss_breakdown = { - "recon_loss": 0.0, - "cls_loss": 0.0, - "total_loss": 0.0, - } - - self._init_weights() - - def _init_weights(self) -> None: - """初始化权重""" - nn.init.normal_(self.pos_emb, std=0.02) - if self.encoder_mask_token is not None: - nn.init.normal_(self.encoder_mask_token, std=0.02) - if self.mask_token is not None: - nn.init.normal_(self.mask_token, std=0.02) - if self.decoder_pos_emb is not None: - nn.init.normal_(self.decoder_pos_emb, std=0.02) - if self.cls_token is not None: - nn.init.normal_(self.cls_token, std=0.02) - if self.family_emb is not None: - nn.init.normal_(self.family_emb, std=0.02) - for m in self.modules(): - if isinstance(m, nn.Linear): - nn.init.xavier_uniform_(m.weight) - if m.bias is not None: - nn.init.constant_(m.bias, 0) - elif isinstance(m, nn.LayerNorm): - nn.init.constant_(m.bias, 0) - nn.init.constant_(m.weight, 1.0) - - def _sample_module_mask( - self, - batch_size: int, - seq_len: int, - device: torch.device, - mask_ratio: float, - ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - if mask_ratio <= 0: - ids_restore = torch.arange(seq_len, device=device, dtype=torch.long).unsqueeze(0).expand(batch_size, -1) - mask = torch.zeros(batch_size, seq_len, device=device) - return mask, ids_restore, ids_restore - - len_keep = int(seq_len * (1 - mask_ratio)) - noise = torch.rand(batch_size, seq_len, device=device) - ids_shuffle = torch.argsort(noise, dim=1) - ids_restore = torch.argsort(ids_shuffle, dim=1) - mask = torch.ones([batch_size, seq_len], device=device) - mask[:, :len_keep] = 0 - mask = torch.gather(mask, dim=1, index=ids_restore) - return mask, ids_shuffle, ids_restore - - def random_masking(self, x: torch.Tensor, mask_ratio: float) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - """ - Per-sample random masking (参考MAE的实现) - x: (B, L, D) - module tokens - Returns: - x_masked: (B, L_visible, D) - 可见的module tokens - mask: (B, L) - mask矩阵,1表示masked,0表示visible - ids_restore: (B, L) - 用于恢复原始顺序的索引 - """ - """ - ┌─────────────────────────────────────────────────────────────────────────┐ - │ Random Masking 过程 │ - ├─────────────────────────────────────────────────────────────────────────┤ - │ │ - │ 原始顺序 (module 0-9): │ - │ [P0, P1, P2, P3, P4, P5, P6, P7, P8, P9] │ - │ │ - │ Step 1: 生成随机噪声 │ - │ noise = [0.3, 0.9, 0.1, 0.7, 0.2, 0.8, 0.4, 0.6, 0.5, 0.0] │ - │ │ - │ Step 2: 按噪声排序(随机打乱) │ - │ ids_shuffle = [9, 2, 4, 0, 6, 8, 7, 3, 5, 1] │ - │ # 噪声值: 0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9 │ - │ # 对应module: P9, P2, P4, P0, P6, P8, P7, P3, P5, P1 │ - │ │ - │ Step 3: 保留前25%(可见),mask后75% │ - │ ids_keep = [9, 2, 4] # 可见module的索引 │ - │ 可见tokens = [P9, P2, P4] # 打乱顺序后的可见module │ - │ │ - │ Step 4: ids_restore 用于恢复原始顺序 │ - │ ids_restore = [3, 9, 1, 7, 2, 8, 4, 6, 5, 0] │ - │ # 含义: │ - │ # ids_restore[0] = 3 → 位置0的module,打乱后在位置3 │ - │ # ids_restore[1] = 9 → 位置1的module,打乱后在位置9(被mask了) │ - │ # ids_restore[2] = 1 → 位置2的module,打乱后在位置1 │ - │ │ - └─────────────────────────────────────────────────────────────────────────┘ - """ - b, l, d = x.shape - mask, ids_shuffle, ids_restore = self._sample_module_mask( - batch_size=b, - seq_len=l, - device=x.device, - mask_ratio=mask_ratio, - ) - if mask_ratio <= 0 and self.zero_mask_identity_fastpath: - return x, mask, ids_restore - len_keep = int(l * (1 - mask_ratio)) - ids_keep = ids_shuffle[:, :len_keep] - x_masked = torch.gather(x, dim=1, index=ids_keep.unsqueeze(-1).repeat(1, 1, d)) - return x_masked, mask, ids_restore - - def replace_masking( - self, - x: torch.Tensor, - mask_ratio: float - ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - """ - BERT/scLong风格:保留完整module序列长度,仅将被mask的module替换为learned mask token。 - """ - if self.encoder_mask_token is None: - raise RuntimeError("replace_masking requires encoder_mask_token to be initialized.") - - b, l, _ = x.shape - mask, _, _ = self._sample_module_mask( - batch_size=b, - seq_len=l, - device=x.device, - mask_ratio=mask_ratio, - ) - ids_restore = torch.arange(l, device=x.device).unsqueeze(0).expand(b, -1) - mask_tokens = self.encoder_mask_token.expand(b, l, -1) - x_masked = torch.where(mask.bool().unsqueeze(-1), mask_tokens, x) - return x_masked, mask, ids_restore - - def _add_module_position_embedding(self, x: torch.Tensor) -> torch.Tensor: - pos = self.pos_emb[:, 1:, :] if self.use_cls else self.pos_emb - if pos.shape[1] != x.shape[1]: - raise RuntimeError( - f"pos_emb length mismatch: pos={pos.shape[1]}, tokens={x.shape[1]}, " - f"modules={self.num_modules}" - ) - x = x + pos - if self.use_family_emb and self.family_emb is not None: - x = x + self.family_emb[self.module_family_ids] - return x - - def forward_encoder( - self, - cells: torch.Tensor, - mask_ratio: Optional[float] = None, - return_module_targets: bool = True, - ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, List[torch.Tensor]]: - """ - 编码器前向传播 - cells: (B, num_genes) - 基因表达值 - Returns: - latent: (B, L_encoded+cls, D) - encoder输出 - mask: (B, L) - mask矩阵 - ids_restore: (B, L) - 恢复顺序的索引 - module_targets: List[Tensor] - 每个module的基因表达目标值 - """ - mask_ratio = self.mask_ratio if mask_ratio is None else mask_ratio - - # Step 1: 将基因表达转换为module tokens - x, module_targets = self.module_embed( - cells, - return_module_targets=return_module_targets, - ) # (B, L, D) Module Embedding层 - - # Step 2/3: 根据mask风格决定是在encoder前丢弃token,还是替换为learned mask token - if self.mask_style == "replace": - # 先替换内容token,再统一加module-id embedding,保留完整序列长度 - x, mask, ids_restore = self.replace_masking(x, mask_ratio) - x = self._add_module_position_embedding(x) - else: - # MAE路径:先加module-id embedding,再drop掉masked module - x = self._add_module_position_embedding(x) - x, mask, ids_restore = self.random_masking(x, mask_ratio) - - # Sparse module gating (1B): suppress unimportant modules - if self.use_module_gate and self.module_gate is not None: - gate = self.module_gate(x) # (B, L_visible, 1) - x = x * gate - - # Step 4: 添加CLS token(如果使用) - if self.use_cls and self.cls_token is not None: - cls_token = self.cls_token + self.pos_emb[:, :1, :] - cls_tokens = cls_token.expand(x.shape[0], -1, -1) - x = torch.cat((cls_tokens, x), dim=1) - - # Step 5: Encoder处理 - x_emb = self.dropout(x) - layer_pos_emb = self.layer_pos_emb(x_emb)# None - x = self.encoder(x_emb, pos_emb=layer_pos_emb) #输入输出 (B, L_visible, D)->(B, L_visible, D) 每个 token 从"单独的 module 表示"变成"全局上下文感知的表示" - x = self.encoder_norm(x) #layernorm - - return x, mask, ids_restore, module_targets - - def _build_full_tokens(self, x: torch.Tensor, ids_restore: torch.Tensor) -> torch.Tensor: - """ - 将encoder输出补齐mask token并恢复到原始module顺序。 - 返回: (B, L(+cls), D) - """ - # 先映射到decoder维度,再补mask token并恢复顺序 - x = self.enc_to_dec(x) - if self.use_cls: - cls_token = x[:, :1, :] - module_tokens = x[:, 1:, :] - else: - cls_token = None - module_tokens = x - - num_mask_tokens = ids_restore.shape[1] - module_tokens.shape[1] - if num_mask_tokens > 0: - if self.mask_token is None: - raise RuntimeError( - "Decoder-side mask_token is missing, but _build_full_tokens needs to restore masked positions." - ) - mask_tokens = self.mask_token.repeat(x.shape[0], num_mask_tokens, 1) - x_ = torch.cat([module_tokens, mask_tokens], dim=1) - else: - x_ = module_tokens - - x_ = torch.gather(x_, dim=1, index=ids_restore.unsqueeze(-1).repeat(1, 1, x.shape[2])) - - if self.use_cls and cls_token is not None: - return torch.cat([cls_token, x_], dim=1) - return x_ - - def _aggregate_module_to_gene( - self, - module_feats: torch.Tensor, - ) -> torch.Tensor: - """ - module_feats: (B, L, D) -> gene_feats: (B, G, D) - 每个基因特征=包含该基因的所有module特征均值。 - """ - bsz, _, dim = module_feats.shape - gene_feats = torch.zeros(bsz, self.num_genes, dim, device=module_feats.device, dtype=module_feats.dtype) - gene_counts = torch.zeros(bsz, self.num_genes, device=module_feats.device, dtype=module_feats.dtype) - - # 静态pathway module贡献 - if self.pg_module_idx is not None and self.pg_module_idx.numel() > 0: - pair_module_feats = module_feats[:, self.pg_module_idx, :] # (B, num_pairs, D) - gene_idx_expand = self.pg_gene_idx.view(1, -1, 1).expand(bsz, -1, dim) - gene_feats.scatter_add_(1, gene_idx_expand, pair_module_feats) - base_gene_counts = self.pg_gene_counts.to(dtype=module_feats.dtype, device=module_feats.device) - gene_counts = gene_counts + base_gene_counts.view(1, self.num_genes) - - gene_counts = gene_counts.clamp(min=1.0).unsqueeze(-1) # (B, G, 1) - return gene_feats / gene_counts - - def _predict_from_module_features( - self, - module_feats: torch.Tensor, - ) -> Union[List[torch.Tensor], torch.Tensor]: - """module_feats: (B, L, D)""" - if self.recon_target_level == "gene" and self.decoder_gene_agg: - gene_feats = self._aggregate_module_to_gene(module_feats) # (B, G, D) - assert self.gene_w is not None and self.gene_b is not None - # 基因特异性线性头:每个基因各自投影 - preds = torch.einsum("bgd,gdk->bgk", gene_feats, self.gene_w).squeeze(-1) - return preds + self.gene_b.unsqueeze(0) # (B, G) - - preds: List[torch.Tensor] = [] - if self.recon_target_level == "gene": - assert self.module_heads is not None - for i, head in enumerate(self.module_heads): - preds.append(head(module_feats[:, i, :])) - else: - assert self.module_scalar_heads is not None - for i, head in enumerate(self.module_scalar_heads): - preds.append(head(module_feats[:, i, :])) - return preds - - def _build_cls_targets(self, module_targets: List[torch.Tensor]) -> torch.Tensor: - """ - 使用静态module的均值表达作为CLS监督目标。 - 这样CLS需要从可见module中汇聚全局信息,预测整细胞的module-level profile。 - """ - static_module_targets = module_targets[: self.num_modules] - if len(static_module_targets) != self.num_modules: - raise RuntimeError( - f"cls target module count mismatch: expected {self.num_modules}, got {len(static_module_targets)}" - ) - module_means = [target.mean(dim=1, keepdim=True) for target in static_module_targets] - return torch.cat(module_means, dim=1) - - def compute_cls_loss( - self, - latent: torch.Tensor, - module_targets: List[torch.Tensor], - ) -> Tuple[torch.Tensor, torch.Tensor]: - if not self.use_cls or self.cls_head is None: - zero = torch.tensor(0.0, device=latent.device, dtype=latent.dtype) - return zero, zero - cls_encoding = latent[:, 0, :] - cls_preds = self.cls_head(cls_encoding) - cls_targets = self._build_cls_targets(module_targets).to(dtype=cls_preds.dtype) - cls_loss = F.mse_loss(cls_preds, cls_targets) - return cls_loss, cls_encoding - - def forward_decoder_simple( - self, - x: torch.Tensor, - ids_restore: torch.Tensor, - ) -> Union[List[torch.Tensor], torch.Tensor]: - """ - 简化的decoder:在encoder输出后,添加mask tokens,然后unshuffle,最后通过module heads重建 - x: (B, L_visible+cls, D) - encoder输出 - ids_restore: (B, L) - 恢复顺序的索引,L是总module数 - Returns: - - 默认模式: List[Tensor],每个module一个预测 - - decoder_gene_agg模式: Tensor(B, G),基因级预测 - """ - x_full = self._build_full_tokens(x, ids_restore) - module_feats = x_full[:, 1:, :] if self.use_cls else x_full - return self._predict_from_module_features(module_feats) - - def forward_decoder_performer( - self, - x: torch.Tensor, - ids_restore: torch.Tensor, - ) -> Union[List[torch.Tensor], torch.Tensor]: - """ - MAE风格decoder:补齐mask tokens并恢复顺序后,再经Performer做token mixing再重建。 - """ - x_full = self._build_full_tokens(x, ids_restore) - pos = self.decoder_pos_emb[:, :x_full.shape[1], :] - x_full = self.decoder_dropout(x_full + pos) - x_full = self.decoder(x_full, pos_emb=self.layer_pos_emb(x_full)) - x_full = self.decoder_norm(x_full) - module_feats = x_full[:, 1:, :] if self.use_cls else x_full - return self._predict_from_module_features(module_feats) - - def forward_decoder_cross_attn( - self, - x: torch.Tensor, - mask: torch.Tensor, - ids_restore: torch.Tensor, - ) -> Union[List[torch.Tensor], torch.Tensor]: - """ - Cross-attention-only decoder: - masked queries attend to visible encoder tokens, without query-query self-attention. - """ - if self.decoder is None or self.decoder_pos_emb is None or self.decoder_dropout is None or self.decoder_norm is None: - raise RuntimeError("cross_attn decoder requested but decoder modules are not initialized.") - - x_full = self._build_full_tokens(x, ids_restore) - if self.use_cls: - cls_token = x_full[:, :1, :] - module_tokens = x_full[:, 1:, :] - module_pos = self.decoder_pos_emb[:, 1:x_full.shape[1], :] - cls_memory = cls_token - else: - cls_token = None - module_tokens = x_full - module_pos = self.decoder_pos_emb[:, :x_full.shape[1], :] - cls_memory = None - - query_tokens = self.decoder_dropout(module_tokens + module_pos) - module_feats = module_tokens.clone() - - mask_bool = mask.bool() - visible_mask = ~mask_bool - batch_size, _, dim = module_feats.shape - visible_per_sample = int(visible_mask[0].sum().item()) if visible_mask.shape[1] > 0 else 0 - masked_per_sample = int(mask_bool[0].sum().item()) if mask_bool.shape[1] > 0 else 0 - - if masked_per_sample > 0: - visible_memory = module_feats[visible_mask].view(batch_size, visible_per_sample, dim) - if cls_memory is not None: - visible_memory = torch.cat([cls_memory, visible_memory], dim=1) - - masked_queries = query_tokens[mask_bool].view(batch_size, masked_per_sample, dim) - for block in self.decoder: - masked_queries = block(masked_queries, visible_memory) - # AMP may run attention/norm in fp32 for stability. Cast back before - # scattering into the decoder-state tensor, which may still be bf16. - masked_queries = self.decoder_norm(masked_queries).to(dtype=module_feats.dtype) - module_feats[mask_bool] = masked_queries.reshape(-1, dim) - - return self._predict_from_module_features(module_feats) - - def forward_predictor_from_encoder( - self, - x: torch.Tensor, - ) -> Union[List[torch.Tensor], torch.Tensor]: - module_feats = x[:, 1:, :] if self.use_cls else x - return self._predict_from_module_features(module_feats) - - def compute_loss( - self, - preds: Union[List[torch.Tensor], torch.Tensor], - targets: List[torch.Tensor], - mask: torch.Tensor, - ) -> torch.Tensor: # module层计算loss ablation study - """ - 计算重建损失(只对masked modules计算) - preds: List[Tensor] - 每个module的预测,每个 (B, genes_in_module) - targets: List[Tensor] - 每个module的目标,每个 (B, genes_in_module) - mask: (B, L) - mask矩阵,1表示masked,0表示visible - """ - if self.recon_target_level == "gene" and self.decoder_gene_agg: - assert isinstance(preds, torch.Tensor) - # 关键:严格按原module mask计算loss。 - # 对每个masked module,取该module涉及的基因在基因级预测中的子向量来计算误差。 - total_loss = torch.tensor(0.0, device=mask.device, dtype=torch.float32) - total_count = torch.tensor(0.0, device=mask.device, dtype=torch.float32) - for i, t in enumerate(targets): - module_mask = mask[:, i].float() # (B,) - module_gene_idx = torch.as_tensor(self.module_sets[i], dtype=torch.long, device=preds.device) - p = preds.index_select(dim=1, index=module_gene_idx) # (B, G_i) - diff = (p - t) ** 2 - if self.loss_mode == "gene_mean": - total_loss = total_loss + (diff * module_mask.unsqueeze(-1)).sum() - total_count = total_count + module_mask.sum() * t.shape[1] - else: - module_mse = diff.mean(dim=1) # (B,) - total_loss = total_loss + (module_mse * module_mask).sum() - total_count = total_count + module_mask.sum() - return total_loss / total_count.clamp(min=1.0) - - assert isinstance(preds, list) - if len(preds) != len(targets): - raise RuntimeError(f"pred/target module count mismatch: {len(preds)} vs {len(targets)}") - total_loss = torch.tensor(0.0, device=mask.device, dtype=torch.float32) - total_count = torch.tensor(0.0, device=mask.device, dtype=torch.float32) - - for i, (p, t) in enumerate(zip(preds, targets)): - module_mask = mask[:, i].float() # (B,) - if self.recon_target_level == "module_mean_value": - # decoder直接输出module均值,target为该module内基因均值 - # p: (B,1), t_mean: (B,1) - t_mean = t.mean(dim=1, keepdim=True) - diff = (p - t_mean) ** 2 - total_loss = total_loss + (diff.squeeze(-1) * module_mask).sum() - total_count = total_count + module_mask.sum() - else: - diff = (p - t) ** 2 # (B, G_i) - if self.loss_mode == "gene_mean": - # 当前逻辑:masked 区域内每个基因等权 - total_loss = total_loss + (diff * module_mask.unsqueeze(-1)).sum() - total_count = total_count + module_mask.sum() * t.shape[1] - else: - # MAE风格:先在module内部取均值,再在masked module上做均值(每个module等权) - module_mse = diff.mean(dim=1) # (B,) - total_loss = total_loss + (module_mse * module_mask).sum() - total_count = total_count + module_mask.sum() - - if total_count.item() == 0: - return torch.tensor(0.0, device=mask.device, dtype=torch.float32) - return total_loss / total_count - - def forward( - self, cells: torch.Tensor, mask_ratio: Optional[float] = None, - return_encodings: bool = False, return_module_targets: bool = True, - ): - """ - 前向传播 - cells: (B, num_genes) - 基因表达值 - mask_ratio: Optional[float] - masking比例,如果为None则使用self.mask_ratio - return_encodings: bool - 如果为True,返回编码而不是loss(用于zero-shot任务) - Returns: - 如果return_encodings=True: - dict with keys: 'module_encodings', 'mask', 'ids_restore', 'module_targets' - 否则: - loss: Tensor - 重建损失 - preds: - - 默认模式: List[Tensor](每个module一个预测) - - decoder_gene_agg模式: Tensor(B, G)(基因级预测) - mask: Tensor - mask矩阵 - """ - latent, mask, ids_restore, module_targets = self.forward_encoder( - cells, - mask_ratio=mask_ratio, - # Reconstruction always needs targets; representation callers can - # opt out explicitly while preserving the original API default. - return_module_targets=(return_module_targets or not return_encodings), - ) - - if return_encodings: - # 返回编码用于zero-shot任务 - # drop-mask路径返回可见token编码;replace-mask路径返回全module编码。 - if self.use_cls: - seq_encodings = latent[:, 1:, :] - else: - seq_encodings = latent - - out = { - 'module_encodings': seq_encodings, - 'mask': mask, - 'ids_restore': ids_restore, - 'module_targets': module_targets - } - if self.use_cls: - out['cls_encoding'] = latent[:, 0, :] - return out - - if self.decoder_type == "performer": - preds = self.forward_decoder_performer(latent, ids_restore) - elif self.decoder_type == "cross_attn": - preds = self.forward_decoder_cross_attn(latent, mask, ids_restore) - elif self.decoder_type == "simple": - preds = self.forward_decoder_simple(latent, ids_restore) - elif self.decoder_type == "none": - preds = self.forward_predictor_from_encoder(latent) - else: - raise RuntimeError(f"Unsupported decoder_type at forward time: {self.decoder_type}") - recon_loss = self.compute_loss(preds, module_targets, mask) - cls_loss, _ = self.compute_cls_loss(latent, module_targets) - total_loss = recon_loss + self.cls_loss_weight * cls_loss - self.last_loss_breakdown = { - "recon_loss": float(recon_loss.detach().cpu()), - "cls_loss": float(cls_loss.detach().cpu()), - "total_loss": float(total_loss.detach().cpu()), - } - return total_loss, preds, mask - - def get_last_loss_breakdown(self) -> dict: - return dict(self.last_loss_breakdown) - - def load_state_dict(self, state_dict, strict: bool = True, assign: bool = False): - """Load current or pre-rename weights under the module-named interface.""" - return super().load_state_dict( - migrate_legacy_module_state_dict(state_dict), - strict=strict, - assign=assign, - ) - - def check_redraw_projections(self): - """检查并重绘projection矩阵""" - self.encoder.check_redraw_projections() - - def fix_projection_matrices_(self): - """固定projection矩阵""" - self.encoder.fix_projection_matrices_() +"""Module-based Performer encoder with MAE-style masking and configurable +reconstruction decoders or direct prediction heads.""" + +import sys +import os +parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +if parent_dir not in sys.path: + sys.path.insert(0, parent_dir) + +# The bundled Performer implementation lives under model_dependencies rather +# than at the repository root. Add that directory explicitly so every task +# entry point can import the model without requiring a caller-managed +# PYTHONPATH. +model_dependencies_dir = os.path.join(parent_dir, "model_dependencies") +if os.path.isdir(model_dependencies_dir) and model_dependencies_dir not in sys.path: + sys.path.insert(0, model_dependencies_dir) + +try: + from performer_pytorch_cont.ding_models import ( + Performer, + Always, + cast_tuple, + ) +except ImportError: + try: + from performer_pytorch_cont.performer_pytorch import ( + Performer, + Always, + cast_tuple, + ) + except ImportError: + from performer_pytorch.performer_pytorch import ( + Performer, + Always, + cast_tuple, + ) +import torch +from torch import nn +import torch.nn.functional as F +from typing import Callable, Dict, List, Optional, Tuple, Union +import numpy as np + + +def _migrate_legacy_module_name(value: str) -> str: + """Translate serialized pre-rename names without changing their semantics.""" + replacements = ( + ("PATHCH", "MODULE"), + ("Pathch", "Module"), + ("pathch", "module"), + ("PATCHES", "MODULES"), + ("Patches", "Modules"), + ("patches", "modules"), + ("PATCH", "MODULE"), + ("Patch", "Module"), + ("patch", "module"), + ) + for old, new in replacements: + value = value.replace(old, new) + return value + + +def migrate_legacy_module_config(config: Dict[str, object]) -> Dict[str, object]: + """Return a module-named copy of an older serialized configuration.""" + def translate(value): + if isinstance(value, dict): + migrated = {} + for key, item in value.items(): + new_key = _migrate_legacy_module_name(key) if isinstance(key, str) else key + if new_key in migrated: + raise ValueError( + f"Legacy configuration migration would duplicate key {new_key!r}" + ) + migrated[new_key] = translate(item) + return migrated + if isinstance(value, list): + return [translate(item) for item in value] + if isinstance(value, tuple): + return tuple(translate(item) for item in value) + if isinstance(value, str): + return _migrate_legacy_module_name(value) + return value + + return translate(dict(config)) + + +def migrate_legacy_module_state_dict(state_dict): + """Translate old state-dict key names before PyTorch performs strict loading.""" + try: + migrated = state_dict.__class__() + except TypeError: + migrated = {} + + for key, value in state_dict.items(): + new_key = _migrate_legacy_module_name(key) if isinstance(key, str) else key + if new_key in migrated: + # If both spellings are supplied, prefer the explicit new spelling. + if key == new_key: + migrated[new_key] = value + continue + migrated[new_key] = value + + if hasattr(state_dict, "_metadata"): + migrated._metadata = { + _migrate_legacy_module_name(key): value + for key, value in state_dict._metadata.items() + } + return migrated + + +def normalize_fusion_method(fusion_method: str) -> str: + if fusion_method == "expression_only_mlp": + return "add" + if fusion_method in {"concatenate", "concat_v2"}: + raise ValueError( + f"fusion_method={fusion_method!r} has been removed. " + "Use one of: 'multiply', 'add', 'gated_residual'." + ) + if fusion_method not in {"multiply", "add", "gated_residual"}: + raise ValueError( + f"Unsupported fusion_method: {fusion_method}. " + "Use one of: 'multiply', 'add', 'gated_residual'." + ) + return fusion_method + + +REMOVED_PRETRAIN_CONFIG_KEYS = ( + "preprocess_gene2vec", + "extra_gene_mode", + "extra_gene_num", + "extra_gene_rank_path", + "extra_gene_module_size", + "extra_gene_module_strategy", + "module_embed_chunk_padded_gene_budget", + "cross_attn_pure_memory", +) + + +def sanitize_performer_module_config( + config: Dict[str, object], + *, + source: str = "config", + log_fn: Optional[Callable[[str], None]] = None, +) -> Dict[str, object]: + cleaned = migrate_legacy_module_config(config) + legacy_mode = str(cleaned.get("extra_gene_mode", "none")).lower() + legacy_num = int(cleaned.get("extra_gene_num", 0) or 0) + if legacy_mode != "none" or legacy_num > 0: + raise ValueError( + f"{source} requests removed extra-gene support " + f"(extra_gene_mode={legacy_mode}, extra_gene_num={legacy_num}). " + "Please use a checkpoint trained without extra genes." + ) + + removed = [key for key in REMOVED_PRETRAIN_CONFIG_KEYS if key in cleaned] + for key in removed: + cleaned.pop(key, None) + + if removed and log_fn is not None: + log_fn(f"{source}: dropped deprecated config keys: {', '.join(sorted(removed))}") + + return cleaned + +# ModuleEmbedding is implemented here to support three expression-fusion methods. +# from sclong_module.models_module import ModuleToGeneDecoder + + +class CrossAttentionDecoderBlock(nn.Module): + """ + Decoder block with masked-query -> visible-memory cross-attention only. + """ + + def __init__( + self, + dim: int, + heads: int, + ff_mult: int = 4, + attn_dropout: float = 0.0, + ff_dropout: float = 0.0, + ) -> None: + super().__init__() + if heads <= 0: + raise ValueError(f"CrossAttentionDecoderBlock requires heads > 0, got {heads}") + if dim % heads != 0: + raise ValueError( + f"CrossAttentionDecoderBlock requires dim divisible by heads, got dim={dim}, heads={heads}" + ) + + self.query_norm = nn.LayerNorm(dim) + self.memory_norm = nn.LayerNorm(dim) + self.cross_attn = nn.MultiheadAttention( + embed_dim=dim, + num_heads=heads, + dropout=attn_dropout, + batch_first=True, + ) + hidden_dim = int(dim * ff_mult) + self.ff_norm = nn.LayerNorm(dim) + self.ff = nn.Sequential( + nn.Linear(dim, hidden_dim), + nn.GELU(), + nn.Dropout(ff_dropout), + nn.Linear(hidden_dim, dim), + nn.Dropout(ff_dropout), + ) + + def forward(self, query: torch.Tensor, memory: torch.Tensor) -> torch.Tensor: + if query.shape[1] == 0: + return query + + if memory.shape[1] > 0: + attn_out, _ = self.cross_attn( + self.query_norm(query), + self.memory_norm(memory), + self.memory_norm(memory), + need_weights=False, + ) + query = query + attn_out + + query = query + self.ff(self.ff_norm(query)) + return query + + +class ModuleEmbedding(nn.Module): + """Gene embedding, MLP, and attention pooling produce a module token. + + Expression fusion supports three methods: + - multiply: multiply each expression value by its gene embedding. + - add: transform expression with a separate MLP and add the gene embedding. + - gated_residual: inject transformed expression through a learned gate. + + use_gene2vec=True initializes from supplied pretrained embeddings; + False uses random initialization for ablation experiments.""" + + @staticmethod + def _build_adaptive_buckets( + modules_by_length: Dict[int, List[int]], + bucket_count: int, + ) -> Dict[int, List[int]]: + """Minimize padded gene positions for a fixed number of buckets.""" + length_groups = sorted(modules_by_length.items()) + group_count = len(length_groups) + bucket_count = min(bucket_count, group_count) + if bucket_count <= 0: + raise ValueError("MODULE_EMBED_BUCKET_COUNT must be positive") + + prefix_counts = [0] + for _, module_ids in length_groups: + prefix_counts.append(prefix_counts[-1] + len(module_ids)) + + # dp[b][j] is the least padded element count for the first j distinct + # lengths split into b contiguous buckets. Contiguous length ranges + # preserve a single static padded shape per bucket. + infinity = float("inf") + previous = [infinity] * (group_count + 1) + previous[0] = 0 + split_points: List[List[int]] = [ + [-1] * (group_count + 1) for _ in range(bucket_count + 1) + ] + for bucket_idx in range(1, bucket_count + 1): + current = [infinity] * (group_count + 1) + for end in range(bucket_idx, group_count + 1): + bucket_length = length_groups[end - 1][0] + best_cost = infinity + best_start = -1 + for start in range(bucket_idx - 1, end): + cost = previous[start] + bucket_length * ( + prefix_counts[end] - prefix_counts[start] + ) + if cost < best_cost: + best_cost = cost + best_start = start + current[end] = best_cost + split_points[bucket_idx][end] = best_start + previous = current + + ranges = [] + end = group_count + for bucket_idx in range(bucket_count, 0, -1): + start = split_points[bucket_idx][end] + ranges.append((start, end)) + end = start + ranges.reverse() + + buckets: Dict[int, List[int]] = {} + for start, end in ranges: + bucket_length = length_groups[end - 1][0] + buckets[bucket_length] = [ + module_idx + for _, module_ids in length_groups[start:end] + for module_idx in module_ids + ] + return buckets + + def __init__( + self, + module_sets: List[List[int]], + num_genes: int, + gene_embed_dim: int, + embed_dim: int, + gene2vec_weight: Optional[torch.Tensor] = None, + fusion_method: str = "multiply", # "multiply" / "add" / "gated_residual" + use_gene2vec: bool = True, # Use pretrained gene2vec; False selects random initialization. + ): + super().__init__() + self.num_modules = len(module_sets) + self.module_index_names: List[str] = [] + modules_by_length: Dict[int, List[int]] = {} + for module_idx, module_genes in enumerate(module_sets): + buffer_name = f"_module_gene_idx_{module_idx}" + # Module indices are static metadata. Non-persistent buffers follow + # module.to(device) without changing checkpoint state_dict keys. + self.register_buffer( + buffer_name, + torch.tensor(module_genes, dtype=torch.long), + persistent=False, + ) + self.module_index_names.append(buffer_name) + modules_by_length.setdefault(len(module_genes), []).append(module_idx) + + # Group modules with the same gene count so their identical operations + # can be issued as a single batched operation. All buffers are + # non-persistent: checkpoint keys remain byte-for-byte compatible. + self.module_batch_mode = os.environ.get("MODULE_EMBED_BATCH_MODE", "adaptive").lower() + if self.module_batch_mode not in {"legacy", "exact", "bucket", "adaptive"}: + raise ValueError( + "MODULE_EMBED_BATCH_MODE must be one of: " + "'legacy', 'exact', 'bucket', 'adaptive'; " + f"got {self.module_batch_mode!r}" + ) + self.module_groups: List[Tuple[int, Tuple[int, ...], int, int]] = [] + grouped_gene_indices: List[int] = [] + grouped_module_order: List[int] = [] + group_offset = 0 + for module_length in sorted(modules_by_length): + module_ids = tuple(modules_by_length[module_length]) + group_gene_indices = [ + gene_idx + for module_idx in module_ids + for gene_idx in module_sets[module_idx] + ] + group_end = group_offset + len(group_gene_indices) + self.module_groups.append( + (module_length, module_ids, group_offset, group_end) + ) + grouped_gene_indices.extend(group_gene_indices) + grouped_module_order.extend(module_ids) + group_offset = group_end + + restore_order = [0] * self.num_modules + for grouped_pos, module_idx in enumerate(grouped_module_order): + restore_order[module_idx] = grouped_pos + self.register_buffer( + "_grouped_gene_idx", + torch.tensor(grouped_gene_indices, dtype=torch.long), + persistent=False, + ) + self.register_buffer( + "_module_group_restore_idx", + torch.tensor(restore_order, dtype=torch.long), + persistent=False, + ) + + if self.module_batch_mode == "adaptive": + bucket_count = int(os.environ.get("MODULE_EMBED_BUCKET_COUNT", "16")) + modules_by_bucket = self._build_adaptive_buckets( + modules_by_length, + bucket_count, + ) + else: + modules_by_bucket: Dict[int, List[int]] = {} + for module_idx, module_genes in enumerate(module_sets): + bucket_length = 1 << (len(module_genes) - 1).bit_length() + modules_by_bucket.setdefault(bucket_length, []).append(module_idx) + self.module_buckets: List[ + Tuple[int, Tuple[int, ...], Tuple[int, ...], int, int, str] + ] = [] + bucketed_gene_indices: List[int] = [] + bucketed_module_order: List[int] = [] + bucket_offset = 0 + for bucket_idx, bucket_length in enumerate(sorted(modules_by_bucket)): + module_ids = tuple(modules_by_bucket[bucket_length]) + module_lengths = tuple(len(module_sets[idx]) for idx in module_ids) + group_gene_indices: List[int] = [] + valid_mask = torch.zeros(len(module_ids), bucket_length, dtype=torch.bool) + for group_pos, module_idx in enumerate(module_ids): + genes = module_sets[module_idx] + pad_count = bucket_length - len(genes) + group_gene_indices.extend(genes) + group_gene_indices.extend([0] * pad_count) + valid_mask[group_pos, :len(genes)] = True + group_end = bucket_offset + len(group_gene_indices) + mask_name = f"_module_bucket_mask_{bucket_idx}" + self.register_buffer(mask_name, valid_mask, persistent=False) + self.module_buckets.append( + ( + bucket_length, + module_ids, + module_lengths, + bucket_offset, + group_end, + mask_name, + ) + ) + bucketed_gene_indices.extend(group_gene_indices) + bucketed_module_order.extend(module_ids) + bucket_offset = group_end + + bucket_restore_order = [0] * self.num_modules + for grouped_pos, module_idx in enumerate(bucketed_module_order): + bucket_restore_order[module_idx] = grouped_pos + self.register_buffer( + "_bucketed_gene_idx", + torch.tensor(bucketed_gene_indices, dtype=torch.long), + persistent=False, + ) + self.register_buffer( + "_module_bucket_restore_idx", + torch.tensor(bucket_restore_order, dtype=torch.long), + persistent=False, + ) + # Deduplicate shared pointwise transformations, preserving module-local + # membership, order, multiplicity and attention normalization. + # Static maps add no checkpoint keys or trainable parameters. + unique_genes = sorted({gene for module in module_sets for gene in module}) + unique_position = {gene: pos for pos, gene in enumerate(unique_genes)} + bucket_unique_positions = [ + unique_position.get(gene, 0) for gene in bucketed_gene_indices + ] + self.register_buffer( + "_unique_gene_idx", torch.tensor(unique_genes, dtype=torch.long), + persistent=False, + ) + self.register_buffer( + "_bucket_unique_idx", + torch.tensor(bucket_unique_positions, dtype=torch.long), + persistent=False, + ) + self.gene_dedup_enabled = os.environ.get("MODULE_GENE_DEDUP", "1") == "1" + # The unlimited candidate was slower at batches 16/64/256 on H100. + # Default to the measured beneficial batch=1 path; 0 enables an explicit + # unlimited research comparison without changing checkpoint structure. + self.gene_dedup_max_batch_size = int( + os.environ.get("MODULE_GENE_DEDUP_MAX_BATCH_SIZE", "1") + ) + if self.gene_dedup_max_batch_size < 0: + raise ValueError("MODULE_GENE_DEDUP_MAX_BATCH_SIZE must be nonnegative") + self.fusion_method = normalize_fusion_method(fusion_method) + self.use_gene2vec = use_gene2vec + self.gene_embed_dim = gene_embed_dim + self.embed_dim = embed_dim + + # Select pretrained or random initialization according to use_gene2vec. + if use_gene2vec and gene2vec_weight is not None: + self.gene_embed = nn.Embedding.from_pretrained(gene2vec_weight, freeze=False) + if self.gene_embed.weight.shape[0] < num_genes: + pad = torch.zeros(num_genes - self.gene_embed.weight.shape[0], self.gene_embed.weight.shape[1]) + self.gene_embed.weight.data = torch.cat([self.gene_embed.weight.data, pad], dim=0) + else: + # Random initialization for ablations without the gene2vec prior. + self.gene_embed = nn.Embedding(num_genes, gene_embed_dim) + # Use Xavier initialization, consistent with the other layers. + nn.init.xavier_uniform_(self.gene_embed.weight) + + # Expression-fusion layers. + # All fusion methods use a two-layer projection: Dg -> E -> E. + self.expr_mlp = None + self.gated_expr_mlp = None + self.gated_residual_gate = None + + if self.fusion_method == "add": + # add: transform expression with a separate MLP, then add gene2vec. + hidden_dim = max(50, gene_embed_dim // 2) + self.expr_mlp = nn.Sequential( + nn.Linear(1, hidden_dim), + nn.GELU(), + nn.Linear(hidden_dim, gene_embed_dim), + ) + elif self.fusion_method == "gated_residual": + # gated_residual: inject expression as a gated residual into the gene2vec representation. + hidden_dim = max(64, gene_embed_dim // 2) + self.gated_expr_mlp = nn.Sequential( + nn.Linear(1, hidden_dim), + nn.GELU(), + nn.Linear(hidden_dim, gene_embed_dim), + nn.LayerNorm(gene_embed_dim), + ) + self.gated_residual_gate = nn.Sequential( + nn.Linear(gene_embed_dim * 2, gene_embed_dim), + nn.GELU(), + nn.Linear(gene_embed_dim, gene_embed_dim), + nn.Sigmoid(), + ) + # multiply does not require an additional expression MLP. + + # Shared projection: Dg -> E -> E. + self.proj = nn.Sequential( + nn.Linear(gene_embed_dim, embed_dim), + nn.GELU(), + nn.Linear(embed_dim, embed_dim), + ) + self.attn_score = nn.Linear(embed_dim, 1) + self.pool_affine_enabled = os.environ.get("MODULE_POOL_AFFINE", "1") == "1" + # Paired H100 screening found this path helps batches 16/64/256 but + # regresses batch=1 against the dedup fast path. Preserve that fast path. + self.pool_affine_min_batch_size = int(os.environ.get("MODULE_POOL_AFFINE_MIN_BATCH_SIZE", "2")) + if self.pool_affine_min_batch_size < 1: + raise ValueError("MODULE_POOL_AFFINE_MIN_BATCH_SIZE must be positive") + # Derived model weights only; no cell-dependent features are cached. + # These buffers add no parameters or checkpoint entries. + self.register_buffer("_pool_affine_score_weight", torch.empty(0), persistent=False) + self.register_buffer("_pool_affine_score_bias", torch.empty(0), persistent=False) + self._pool_affine_cache_signature = None + + @property + def module_sets(self) -> List[torch.Tensor]: + """Expose module indices for existing interpretability callers.""" + return [getattr(self, name) for name in self.module_index_names] + + def _pool_affine_signature(self): + tensors = (self.proj[2].weight, self.proj[2].bias, + self.attn_score.weight, self.attn_score.bias) + return tuple((id(t), t.data_ptr(), t._version, t.device, t.dtype) for t in tensors) + + @torch.no_grad() + def prepare_pool_affine(self): + """Compose final affine and score AFTER checkpoint/device/dtype placement. + + Call again after changing weights, dtype or device. Forward rejects a + stale cache rather than silently using outdated model-derived weights. + """ + weight = self.proj[2].weight + compose_dtype = torch.float64 if weight.dtype == torch.float64 else torch.float32 + with torch.autocast(device_type=weight.device.type, enabled=False): + score_weight = self.attn_score.weight.to(dtype=compose_dtype) + self._pool_affine_score_weight = ( + score_weight @ weight.to(dtype=compose_dtype) + ).to(dtype=weight.dtype) + self._pool_affine_score_bias = ( + score_weight @ self.proj[2].bias.to(dtype=compose_dtype) + + self.attn_score.bias.to(dtype=compose_dtype) + ).to(dtype=weight.dtype) + self._pool_affine_cache_signature = self._pool_affine_signature() + + def _forward_pool_affine( + self, + expr: torch.Tensor, + return_module_targets: bool = True, + ) -> Tuple[torch.Tensor, List[torch.Tensor]]: + """Pool first-layer activations before the shared final affine. + + In real arithmetic, score(W2*h+b2)=(score_weight*W2)*h+const, + and sum(alpha*(W2*h+b2))=W2*sum(alpha*h)+b2*sum(alpha). + BF16 reassociation is validated separately; it is not bitwise equivalent. + """ + if self._pool_affine_cache_signature != self._pool_affine_signature(): + raise RuntimeError("Pool-affine cache is stale; call prepare_pool_affine() after loading/placing weights") + batch_size = expr.shape[0] + flat_expr = torch.index_select(expr, 1, self._bucketed_gene_idx) + flat_gene_emb = self.gene_embed(self._bucketed_gene_idx) + pooled_hidden = [] + attention_sums = [] + module_targets: List[Optional[torch.Tensor]] = ( + [None] * self.num_modules if return_module_targets else [] + ) + for ( + bucket_length, module_ids, module_lengths, start, end, mask_name + ) in self.module_buckets: + group_size = len(module_ids) + expr_group = flat_expr[:, start:end].reshape(batch_size, group_size, bucket_length) + gene_emb = flat_gene_emb[start:end].reshape(group_size, bucket_length, self.gene_embed_dim) + gene_emb = gene_emb.unsqueeze(0).expand(batch_size, -1, -1, -1) + fused = self._fuse_expr_and_gene_emb(expr_group, gene_emb) + hidden = self.proj[1](self.proj[0](fused)) + del fused + scores = F.linear(hidden, self._pool_affine_score_weight, self._pool_affine_score_bias).squeeze(-1) + valid_mask = getattr(self, mask_name) + scores = scores.masked_fill(~valid_mask.unsqueeze(0), float("-inf")) + attn = torch.softmax(scores, dim=2) + pooled_hidden.append(self._weighted_pool(hidden, attn)) + # Quantized softmax weights need not sum exactly to one. Preserve + # their sum in wider accumulation for the final bias contribution. + sum_dtype = torch.float64 if hidden.dtype == torch.float64 else torch.float32 + attention_sums.append(attn.sum(dim=2, dtype=sum_dtype)) + del hidden, scores, attn + if return_module_targets: + for group_pos, (module_idx, module_length) in enumerate(zip(module_ids, module_lengths)): + module_targets[module_idx] = expr_group[:, group_pos, :module_length] + pooled = torch.cat(pooled_hidden, dim=1) + sums = torch.cat(attention_sums, dim=1) + # One final affine over the module tokens, not over every gene position. + linear = F.linear(pooled, self.proj[2].weight, None) + with torch.autocast(device_type=expr.device.type, enabled=False): + wider = linear.to(dtype=sums.dtype) + sums.unsqueeze(-1) * self.proj[2].bias.to(dtype=sums.dtype) + tokens = wider.to(dtype=linear.dtype) + tokens = torch.index_select(tokens, 1, self._module_bucket_restore_idx) + return tokens, module_targets # type: ignore[return-value] + + def _fuse_expr_and_gene_emb(self, expr_slice: torch.Tensor, gene_emb: torch.Tensor) -> torch.Tensor: + """Fuse expression and gene embeddings into gene-token representations. + + Leading dimensions may vary; the final expression axis indexes genes. + expr_slice: (..., G), gene_emb: (..., G, Dg) -> (..., G, Dg).""" + if self.fusion_method == "add": + expr_emb = self.expr_mlp(expr_slice.unsqueeze(-1)) # (B, G, Dg) + return expr_emb + gene_emb + if self.fusion_method == "gated_residual": + expr_emb = self.gated_expr_mlp(expr_slice.unsqueeze(-1)) # (B, G, Dg) + gate = self.gated_residual_gate(torch.cat([expr_emb, gene_emb], dim=-1)) # (B, G, Dg) + return gene_emb + gate * expr_emb + # Default multiply fusion: expression value times gene2vec. + return expr_slice.unsqueeze(-1) * gene_emb # (B, G, Dg) + + @staticmethod + def _weighted_pool(gene_tok: torch.Tensor, attn: torch.Tensor) -> torch.Tensor: + """Pool without materializing the full attention-weighted tensor.""" + implementation = os.environ.get("MODULE_POOL_IMPL", "bmm") + if implementation == "bmm": + length, width = gene_tok.shape[-2:] + pooled = torch.bmm( + attn.reshape(-1, 1, length), + gene_tok.reshape(-1, length, width), + ) + return pooled.reshape(*gene_tok.shape[:-2], width) + if implementation != "mulsum": + raise ValueError(f"Unsupported MODULE_POOL_IMPL={implementation}") + return torch.sum(gene_tok * attn.unsqueeze(-1), dim=-2) + + def _forward_legacy( + self, + expr: torch.Tensor, + return_module_targets: bool = True, + ) -> Tuple[torch.Tensor, List[torch.Tensor]]: + """Reference implementation that processes one module at a time. + + expr: (batch, num_genes). + Returns module_tokens (batch, num_modules, embed_dim) and a list of + targets, each shaped (batch, genes_in_module).""" + batch_size, _ = expr.shape + module_tokens = [] + module_targets: List[torch.Tensor] = [] + for index_name in self.module_index_names: + gene_idx = getattr(self, index_name) + expr_slice = torch.index_select(expr, dim=1, index=gene_idx) + if return_module_targets: + module_targets.append(expr_slice) + + gene_emb = self.gene_embed(gene_idx) + gene_emb = gene_emb.unsqueeze(0).expand(batch_size, -1, -1) + gene_tok = self._fuse_expr_and_gene_emb(expr_slice, gene_emb) + gene_tok = self.proj(gene_tok) + + attn = self.attn_score(gene_tok).squeeze(-1) + attn = torch.softmax(attn, dim=1) + pooled = self._weighted_pool(gene_tok, attn) + module_tokens.append(pooled) + + return torch.stack(module_tokens, dim=1), module_targets + + def _forward_grouped_exact( + self, + expr: torch.Tensor, + return_module_targets: bool = True, + ) -> Tuple[torch.Tensor, List[torch.Tensor]]: + """Batch modules of identical length without padding or approximation.""" + batch_size, _ = expr.shape + flat_expr = torch.index_select(expr, dim=1, index=self._grouped_gene_idx) + flat_gene_emb = self.gene_embed(self._grouped_gene_idx) + grouped_tokens = [] + module_targets: List[Optional[torch.Tensor]] = ( + [None] * self.num_modules if return_module_targets else [] + ) + + for module_length, module_ids, start, end in self.module_groups: + group_size = len(module_ids) + expr_group = flat_expr[:, start:end].reshape( + batch_size, group_size, module_length + ) + gene_emb = flat_gene_emb[start:end].reshape( + group_size, module_length, self.gene_embed_dim + ) + gene_emb = gene_emb.unsqueeze(0).expand(batch_size, -1, -1, -1) + gene_tok = self._fuse_expr_and_gene_emb(expr_group, gene_emb) + gene_tok = self.proj(gene_tok) + + attn = self.attn_score(gene_tok).squeeze(-1) + attn = torch.softmax(attn, dim=2) + pooled = self._weighted_pool(gene_tok, attn) + grouped_tokens.append(pooled) + + if return_module_targets: + for group_pos, module_idx in enumerate(module_ids): + module_targets[module_idx] = expr_group[:, group_pos, :] + + tokens = torch.cat(grouped_tokens, dim=1) + tokens = torch.index_select( + tokens, dim=1, index=self._module_group_restore_idx + ) + # Every module is assigned exactly once when groups are constructed. + return tokens, module_targets # type: ignore[return-value] + + def _forward_bucketed( + self, + expr: torch.Tensor, + return_module_targets: bool = True, + ) -> Tuple[torch.Tensor, List[torch.Tensor]]: + """Batch modules into fixed padded lengths and mask padded positions.""" + batch_size, _ = expr.shape + flat_expr = torch.index_select(expr, dim=1, index=self._bucketed_gene_idx) + flat_gene_emb = self.gene_embed(self._bucketed_gene_idx) + bucket_tokens = [] + module_targets: List[Optional[torch.Tensor]] = ( + [None] * self.num_modules if return_module_targets else [] + ) + + for ( + bucket_length, + module_ids, + module_lengths, + start, + end, + mask_name, + ) in self.module_buckets: + group_size = len(module_ids) + expr_group = flat_expr[:, start:end].reshape( + batch_size, group_size, bucket_length + ) + gene_emb = flat_gene_emb[start:end].reshape( + group_size, bucket_length, self.gene_embed_dim + ) + gene_emb = gene_emb.unsqueeze(0).expand(batch_size, -1, -1, -1) + gene_tok = self._fuse_expr_and_gene_emb(expr_group, gene_emb) + gene_tok = self.proj(gene_tok) + + valid_mask = getattr(self, mask_name) + attn = self.attn_score(gene_tok).squeeze(-1) + attn = attn.masked_fill(~valid_mask.unsqueeze(0), float("-inf")) + attn = torch.softmax(attn, dim=2) + pooled = self._weighted_pool(gene_tok, attn) + bucket_tokens.append(pooled) + + if return_module_targets: + for group_pos, (module_idx, module_length) in enumerate( + zip(module_ids, module_lengths) + ): + module_targets[module_idx] = expr_group[:, group_pos, :module_length] + + tokens = torch.cat(bucket_tokens, dim=1) + tokens = torch.index_select( + tokens, dim=1, index=self._module_bucket_restore_idx + ) + return tokens, module_targets # type: ignore[return-value] + + def _forward_unique_genes( + self, + expr: torch.Tensor, + return_module_targets: bool = True, + ) -> Tuple[torch.Tensor, List[torch.Tensor]]: + """Reuse gene features within this inference forward; pool per module. + + Gather one bucket at a time rather than the full repeated-gene tensor. + """ + batch_size = expr.shape[0] + unique_expr = torch.index_select(expr, 1, self._unique_gene_idx) + unique_emb = self.gene_embed(self._unique_gene_idx) + unique_emb = unique_emb.unsqueeze(0).expand(batch_size, -1, -1) + fused = self._fuse_expr_and_gene_emb(unique_expr, unique_emb) + unique_tokens = self.proj(fused) + del fused, unique_emb + unique_scores = self.attn_score(unique_tokens).squeeze(-1) + bucket_tokens = [] + module_targets: List[Optional[torch.Tensor]] = ( + [None] * self.num_modules if return_module_targets else [] + ) + for ( + bucket_length, module_ids, module_lengths, start, end, mask_name + ) in self.module_buckets: + group_size = len(module_ids) + indices = self._bucket_unique_idx[start:end] + scores = torch.index_select(unique_scores, 1, indices).reshape( + batch_size, group_size, bucket_length + ) + valid_mask = getattr(self, mask_name) + scores = scores.masked_fill(~valid_mask.unsqueeze(0), float("-inf")) + attn = torch.softmax(scores, dim=2) + gene_tok = torch.index_select(unique_tokens, 1, indices).reshape( + batch_size, group_size, bucket_length, self.embed_dim + ) + bucket_tokens.append(self._weighted_pool(gene_tok, attn)) + del gene_tok, attn, scores + if return_module_targets: + expr_group = torch.index_select(unique_expr, 1, indices).reshape( + batch_size, group_size, bucket_length + ) + for group_pos, (module_idx, module_length) in enumerate( + zip(module_ids, module_lengths) + ): + module_targets[module_idx] = expr_group[:, group_pos, :module_length] + tokens = torch.cat(bucket_tokens, dim=1) + tokens = torch.index_select(tokens, 1, self._module_bucket_restore_idx) + return tokens, module_targets # type: ignore[return-value] + + def forward( + self, + expr: torch.Tensor, + return_module_targets: bool = True, + ) -> Tuple[torch.Tensor, List[torch.Tensor]]: + # Keep original training/gradient behavior and gradient reduction order. + if ( + self.pool_affine_enabled and not self.training and not torch.is_grad_enabled() + and expr.shape[0] >= self.pool_affine_min_batch_size + ): + return self._forward_pool_affine(expr, return_module_targets) + if ( + self.gene_dedup_enabled and not self.training and not torch.is_grad_enabled() + and (self.gene_dedup_max_batch_size == 0 + or expr.shape[0] <= self.gene_dedup_max_batch_size) + ): + return self._forward_unique_genes(expr, return_module_targets) + if self.module_batch_mode == "legacy": + return self._forward_legacy(expr, return_module_targets) + if self.module_batch_mode == "exact": + return self._forward_grouped_exact(expr, return_module_targets) + return self._forward_bucketed(expr, return_module_targets) + + +class PerformerEncoderOnly_Module(nn.Module): + """Performer encoder for module inputs with configurable masking and decoding. + + use_gene2vec=True initializes from supplied pretrained gene embeddings; + False uses random gene embeddings for ablation experiments.""" + def __init__( + self, + module_sets: List[List[int]], # Module definitions. + num_genes: int, # Total number of genes. + max_seq_len, # Maximum sequence length (number of modules). + base_dim = 200, # Base embedding dimension shared by all tokens. + enc_depth = 24, # Encoder depth. + enc_heads = 16, # Number of encoder attention heads. + enc_dim_head = 64, # Dimension of each encoder attention head. + mask_ratio: float = 0.75, # Fraction of modules to mask. + # ==================== Performer parameters ==================== + local_attn_heads = 0,# Number of local-attention heads. + local_window_size = 256,# Local-attention window size. + causal = False, # Enable causal (autoregressive) attention. + ff_mult = 4,# Feed-forward expansion factor. + nb_features = None,# Number of Performer random features. + feature_redraw_interval = 1000,# Steps between random-feature redraws. + reversible = False,# Use reversible layers to reduce memory use. + ff_chunks = 1,# Number of feed-forward chunks. + ff_glu = False, # Use gated linear units in the feed-forward network. + # ==================== Dropout parameters ==================== + emb_dropout = 0., + ff_dropout = 0., + attn_dropout = 0., + # ==================== Advanced Performer parameters ==================== + generalized_attention = False, + kernel_fn = nn.ReLU(), + use_scalenorm = False, + use_rezero = False, + cross_attend = False, + no_projection = False, + # ==================== Position embedding parameters ==================== + g2v_position_emb = False, # Use learned module position embeddings. + auto_check_redraw = True,# Automatically check and redraw projection matrices. + qkv_bias = False, # Include bias in query, key, and value projections. + # ==================== Gene2vec parameters ==================== + gene2vec_file = os.path.join(model_dependencies_dir, 'selected_gene2vec_27k.npy'), + gene_embed_dim: int = 200, # Gene embedding dimension used by the module embedder. + fusion_method: str = "multiply", # "multiply" / "add" / "gated_residual" + use_cls_token: bool = False, # Enable the optional CLS token. + cls_loss_weight: float = 0.0, # CLS auxiliary loss weight; positive values train CLS to predict static module means. + cls_head_hidden_dim: int = 0, # CLS-head hidden dimension; zero selects base_dim. + use_gene2vec: bool = True, # Use pretrained gene2vec; False selects random initialization for ablations. + loss_mode: str = "gene_mean", # gene_mean weights masked gene occurrences equally; module_mean weights modules equally. + recon_target_level: str = "gene", # gene reconstructs each gene; module_mean_value predicts one mean per module. + pretrain_arch: str = "mae", # "mae" / "encoder_only" / "encdec" + mask_style: str = "drop", # "drop" / "replace" + mask_token_mode: str = "learned", # Only learned mask tokens are supported. + pos_embed_mode: str = "module_id", # Only learned module-ID embeddings are supported. + decoder_type: str = "simple", # "none" / "simple" / "performer" / "cross_attn" + decoder_gene_agg: bool = False, # Aggregate module features into gene features before reconstruction. + dec_embed_dim: int = 512, + dec_depth: int = 4, + dec_heads: int = 8, + dec_dim_head: int = 64, + dec_ff_mult: int = 4, + dec_ff_dropout: float = 0.0, + dec_attn_dropout: float = 0.0, + cross_attn_pure_memory: Optional[bool] = None, + # ==================== Fine-tuning enhancements ==================== + use_family_emb: bool = False, + module_family_ids: Optional[List[int]] = None, + use_module_gate: bool = False, + ): + super().__init__() + local_attn_heads = cast_tuple(local_attn_heads) + self.max_seq_len = max_seq_len + self.base_dim = base_dim + self.num_genes = num_genes + self.module_sets = module_sets + self.num_modules = len(module_sets) + self.fusion_method = normalize_fusion_method(fusion_method) + self.mask_ratio = mask_ratio + self.zero_mask_identity_fastpath = ( + os.environ.get("ZERO_MASK_IDENTITY_FASTPATH", "1") == "1" + ) + self.use_cls = use_cls_token + self.cls_loss_weight = float(cls_loss_weight) if use_cls_token else 0.0 + self.cls_head_hidden_dim = int(cls_head_hidden_dim) + self.use_gene2vec = use_gene2vec + self.loss_mode = loss_mode + self.recon_target_level = recon_target_level + self.pretrain_arch = str(pretrain_arch).lower() + self.mask_style = str(mask_style).lower() + self.mask_token_mode = str(mask_token_mode).lower() + self.pos_embed_mode = str(pos_embed_mode).lower() + self.decoder_type = decoder_type + self.decoder_gene_agg = decoder_gene_agg + self.dec_embed_dim = dec_embed_dim + # Legacy knob retained only for backward-compatible kwargs/config loading. + # Current pretraining always uses pure visible memory in the cross-attn decoder. + self.cross_attn_pure_memory = True + self.base_module_gene_counts = [len(p) for p in module_sets] + self.module_gene_counts = list(self.base_module_gene_counts) + self.total_module_count = len(self.module_gene_counts) + if self.loss_mode not in {"gene_mean", "module_mean"}: + raise ValueError(f"Unsupported loss_mode: {self.loss_mode}. Use 'gene_mean' or 'module_mean'.") + if self.cls_loss_weight < 0: + raise ValueError(f"cls_loss_weight must be >= 0, got {self.cls_loss_weight}") + if self.cls_head_hidden_dim < 0: + raise ValueError(f"cls_head_hidden_dim must be >= 0, got {self.cls_head_hidden_dim}") + if self.recon_target_level not in {"gene", "module_mean_value"}: + raise ValueError( + f"Unsupported recon_target_level: {self.recon_target_level}. " + "Use 'gene' or 'module_mean_value'." + ) + if self.pretrain_arch not in {"mae", "encoder_only", "encdec"}: + raise ValueError( + f"Unsupported pretrain_arch: {self.pretrain_arch}. " + "Use 'mae', 'encoder_only' or 'encdec'." + ) + if self.mask_style not in {"drop", "replace"}: + raise ValueError( + f"Unsupported mask_style: {self.mask_style}. Use 'drop' or 'replace'." + ) + if self.mask_token_mode not in {"learned"}: + raise ValueError( + f"Unsupported mask_token_mode: {self.mask_token_mode}. " + "Only 'learned' is currently supported." + ) + if self.pos_embed_mode not in {"module_id"}: + raise ValueError( + f"Unsupported pos_embed_mode: {self.pos_embed_mode}. " + "Only 'module_id' is currently supported." + ) + if self.decoder_type not in {"none", "simple", "performer", "cross_attn"}: + raise ValueError( + f"Unsupported decoder_type: {self.decoder_type}. " + "Use 'none', 'simple', 'performer' or 'cross_attn'." + ) + if self.pretrain_arch == "mae": + if self.mask_style != "drop": + raise ValueError( + "pretrain_arch='mae' is fixed to mask_style='drop'. " + "Use pretrain_arch='encoder_only' or 'encdec' for mask_style='replace'." + ) + if self.decoder_type == "none": + raise ValueError("pretrain_arch='mae' requires decoder_type != 'none'.") + elif self.pretrain_arch == "encoder_only": + if self.mask_style != "replace": + raise ValueError("pretrain_arch='encoder_only' requires mask_style='replace'.") + if self.decoder_type != "none": + raise ValueError("pretrain_arch='encoder_only' requires decoder_type='none'.") + else: + if self.mask_style != "replace": + raise ValueError("pretrain_arch='encdec' requires mask_style='replace'.") + if self.decoder_type not in {"performer", "cross_attn"}: + raise ValueError( + "pretrain_arch='encdec' requires decoder_type in {'performer', 'cross_attn'}." + ) + if self.decoder_gene_agg and self.recon_target_level != "gene": + raise ValueError("decoder_gene_agg=True only supports recon_target_level='gene'.") + if self.dec_embed_dim <= 0: + raise ValueError(f"dec_embed_dim must be positive, got {self.dec_embed_dim}") + + # Load gene2vec for module embedding when use_gene2vec is enabled. + gene2vec_weight = None + if use_gene2vec and gene2vec_file and os.path.exists(gene2vec_file): + gene2vec_weight = torch.from_numpy(np.load(gene2vec_file)).float() + if gene2vec_weight.shape[0] < num_genes: + pad = torch.zeros(num_genes - gene2vec_weight.shape[0], gene2vec_weight.shape[1]) + gene2vec_weight = torch.cat([gene2vec_weight, pad], dim=0) + gene_embed_dim = gene2vec_weight.shape[1] + elif not use_gene2vec: + # Report random gene initialization when gene2vec is disabled. + print(f"[INFO] use_gene2vec=False: Using randomly initialized gene embeddings (dim={gene_embed_dim})") + + # Module embedding layer. + self.module_embed = ModuleEmbedding( + module_sets=module_sets, + num_genes=num_genes, + gene_embed_dim=gene_embed_dim, + embed_dim=base_dim, + gene2vec_weight=gene2vec_weight, + fusion_method=self.fusion_method, + use_gene2vec=use_gene2vec, + ) + + # Positional embedding for modules + # These are pathway module-ID embeddings, rather than geometric positions. + pos_tokens = self.total_module_count + (1 if use_cls_token else 0) + self.pos_emb = nn.Parameter(torch.zeros(1, pos_tokens, base_dim)) + self.layer_pos_emb = Always(None) + + # Family-type embedding (2B): adds family identity to module tokens + self.use_family_emb = use_family_emb + if use_family_emb and module_family_ids is not None: + self.register_buffer( + 'module_family_ids', + torch.tensor(module_family_ids, dtype=torch.long), + ) + self.family_emb = nn.Parameter(torch.zeros(4, base_dim)) + else: + self.module_family_ids = None + self.family_emb = None + + # Sparse module gate (1B): learns per-module scalar gate + self.use_module_gate = use_module_gate + if use_module_gate: + self.module_gate = nn.Sequential( + nn.LayerNorm(base_dim), + nn.Linear(base_dim, 1), + nn.Sigmoid(), + ) + else: + self.module_gate = None + + # Optional CLS token. + if use_cls_token: + self.cls_token = nn.Parameter(torch.zeros(1, 1, base_dim)) + else: + self.cls_token = None + if self.use_cls and self.cls_loss_weight > 0: + cls_hidden_dim = self.cls_head_hidden_dim if self.cls_head_hidden_dim > 0 else base_dim + self.cls_head = nn.Sequential( + nn.LayerNorm(base_dim), + nn.Linear(base_dim, cls_hidden_dim), + nn.GELU(), + nn.Linear(cls_hidden_dim, self.num_modules), + ) + else: + self.cls_head = None + + self.dropout = nn.Dropout(emb_dropout) + + # Performer encoder. + self.encoder = Performer( + base_dim, enc_depth, enc_heads, enc_dim_head, + local_attn_heads, local_window_size, causal, ff_mult, nb_features, + feature_redraw_interval, reversible, ff_chunks, generalized_attention, + kernel_fn, use_scalenorm, use_rezero, ff_glu, ff_dropout, attn_dropout, + cross_attend, no_projection, auto_check_redraw, qkv_bias + ) + self.encoder_norm = nn.LayerNorm(base_dim) + + # Encoder-to-decoder dimension projection. + # performer/cross_attn can use a smaller dec_embed_dim. + # simple/none use base_dim directly without a projection. + if self.decoder_type in {"performer", "cross_attn"}: + self.enc_to_dec = nn.Identity() if self.dec_embed_dim == base_dim else nn.Linear(base_dim, self.dec_embed_dim) + self.decoder_dim = self.dec_embed_dim + else: + # simple/none bypass enc_to_dec and use base_dim. + self.enc_to_dec = nn.Identity() + self.decoder_dim = base_dim + + # Encoder mask token used by the replace-mask path. + if self.mask_style == "replace": + self.encoder_mask_token = nn.Parameter(torch.zeros(1, 1, base_dim)) + else: + self.encoder_mask_token = None + + # Decoder mask token used by the MAE drop-mask path. + if self.pretrain_arch == "mae": + self.mask_token = nn.Parameter(torch.zeros(1, 1, self.decoder_dim)) + else: + self.mask_token = None + + # Optional decoder lets masked tokens exchange information with visible tokens. + if self.decoder_type == "performer": + self.decoder_pos_emb = nn.Parameter(torch.zeros(1, pos_tokens, self.dec_embed_dim)) + self.decoder_dropout = nn.Dropout(emb_dropout) + self.decoder = Performer( + self.dec_embed_dim, dec_depth, dec_heads, dec_dim_head, + local_attn_heads=cast_tuple(0), local_window_size=256, causal=False, ff_mult=dec_ff_mult, + nb_features=None, feature_redraw_interval=feature_redraw_interval, + reversible=False, ff_chunks=1, generalized_attention=False, + kernel_fn=nn.ReLU(), use_scalenorm=False, use_rezero=False, ff_glu=False, + ff_dropout=dec_ff_dropout, attn_dropout=dec_attn_dropout, + cross_attend=False, no_projection=False, auto_check_redraw=auto_check_redraw, + qkv_bias=qkv_bias + ) + self.decoder_norm = nn.LayerNorm(self.dec_embed_dim) + elif self.decoder_type == "cross_attn": + self.decoder_pos_emb = nn.Parameter(torch.zeros(1, pos_tokens, self.dec_embed_dim)) + self.decoder_dropout = nn.Dropout(emb_dropout) + self.decoder = nn.ModuleList( + [ + CrossAttentionDecoderBlock( + dim=self.dec_embed_dim, + heads=dec_heads, + ff_mult=dec_ff_mult, + attn_dropout=dec_attn_dropout, + ff_dropout=dec_ff_dropout, + ) + for _ in range(dec_depth) + ] + ) + self.decoder_norm = nn.LayerNorm(self.dec_embed_dim) + else: + self.decoder_pos_emb = None + self.decoder_dropout = None + self.decoder = None + self.decoder_norm = None + + # Precompute module-to-gene aggregation indices for decoder_gene_agg. + if self.decoder_gene_agg: + pg_module_idx = [] + pg_gene_idx = [] + for module_idx, module_genes in enumerate(self.module_sets): + for gene_idx in module_genes: + if 0 <= gene_idx < self.num_genes: + pg_module_idx.append(module_idx) + pg_gene_idx.append(gene_idx) + self.register_buffer("pg_module_idx", torch.tensor(pg_module_idx, dtype=torch.long)) + self.register_buffer("pg_gene_idx", torch.tensor(pg_gene_idx, dtype=torch.long)) + pg_gene_counts = torch.zeros(self.num_genes, dtype=torch.long) + if len(pg_gene_idx) > 0: + pg_gene_counts.scatter_add_(0, self.pg_gene_idx, torch.ones_like(self.pg_gene_idx)) + self.register_buffer("pg_gene_counts", pg_gene_counts) + else: + self.pg_module_idx = None + self.pg_gene_idx = None + self.pg_gene_counts = None + + # Reconstruction heads: + # Default: one linear head per module. + # decoder_gene_agg: aggregate to genes, then use gene-specific heads. + # module_mean_value: output one mean value per module. + if self.recon_target_level == "gene": + if self.decoder_gene_agg: + self.module_heads = None + self.gene_head = None + # Gene-specific reconstruction heads: separate D -> 1 parameters for each gene. + self.gene_w = nn.Parameter(torch.rand(self.num_genes, self.decoder_dim, 1)) + self.gene_b = nn.Parameter(torch.rand(self.num_genes)) + nn.init.xavier_normal_(self.gene_w) + nn.init.xavier_normal_(self.gene_b.unsqueeze(-1)) + else: + self.module_heads = nn.ModuleList( + [nn.Linear(self.decoder_dim, g) for g in self.module_gene_counts] + ) + self.gene_head = None + self.gene_w = None + self.gene_b = None + self.module_scalar_heads = None + else: + self.module_heads = None + self.gene_head = None + self.gene_w = None + self.gene_b = None + self.module_scalar_heads = nn.ModuleList( + [nn.Linear(self.decoder_dim, 1) for _ in self.module_gene_counts] + ) + + self.last_loss_breakdown = { + "recon_loss": 0.0, + "cls_loss": 0.0, + "total_loss": 0.0, + } + + self._init_weights() + + def _init_weights(self) -> None: + """Initialize model parameters.""" + nn.init.normal_(self.pos_emb, std=0.02) + if self.encoder_mask_token is not None: + nn.init.normal_(self.encoder_mask_token, std=0.02) + if self.mask_token is not None: + nn.init.normal_(self.mask_token, std=0.02) + if self.decoder_pos_emb is not None: + nn.init.normal_(self.decoder_pos_emb, std=0.02) + if self.cls_token is not None: + nn.init.normal_(self.cls_token, std=0.02) + if self.family_emb is not None: + nn.init.normal_(self.family_emb, std=0.02) + for m in self.modules(): + if isinstance(m, nn.Linear): + nn.init.xavier_uniform_(m.weight) + if m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.LayerNorm): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + + def _sample_module_mask( + self, + batch_size: int, + seq_len: int, + device: torch.device, + mask_ratio: float, + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + if mask_ratio <= 0: + ids_restore = torch.arange(seq_len, device=device, dtype=torch.long).unsqueeze(0).expand(batch_size, -1) + mask = torch.zeros(batch_size, seq_len, device=device) + return mask, ids_restore, ids_restore + + len_keep = int(seq_len * (1 - mask_ratio)) + noise = torch.rand(batch_size, seq_len, device=device) + ids_shuffle = torch.argsort(noise, dim=1) + ids_restore = torch.argsort(ids_shuffle, dim=1) + mask = torch.ones([batch_size, seq_len], device=device) + mask[:, :len_keep] = 0 + mask = torch.gather(mask, dim=1, index=ids_restore) + return mask, ids_shuffle, ids_restore + + def random_masking(self, x: torch.Tensor, mask_ratio: float) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Apply per-sample random masking following the MAE convention. + + x: (B, L, D), the module tokens. + Returns visible tokens (B, L_visible, D), a mask (B, L) with 1 for + masked positions, and restoration indices (B, L).""" + """Example with ten modules and a mask ratio of 0.75: + noise = [0.3, 0.9, 0.1, 0.7, 0.2, 0.8, 0.4, 0.6, 0.5, 0.0] + ids_shuffle = [9, 2, 4, 0, 6, 8, 7, 3, 5, 1] + len_keep = int(10 * (1 - 0.75)) = 2, so ids_keep = [9, 2]. + ids_restore = [3, 9, 1, 7, 2, 8, 4, 6, 5, 0] is the inverse permutation. + For example, original module 0 occupies position 3 after shuffling.""" + b, l, d = x.shape + mask, ids_shuffle, ids_restore = self._sample_module_mask( + batch_size=b, + seq_len=l, + device=x.device, + mask_ratio=mask_ratio, + ) + if mask_ratio <= 0 and self.zero_mask_identity_fastpath: + return x, mask, ids_restore + len_keep = int(l * (1 - mask_ratio)) + ids_keep = ids_shuffle[:, :len_keep] + x_masked = torch.gather(x, dim=1, index=ids_keep.unsqueeze(-1).repeat(1, 1, d)) + return x_masked, mask, ids_restore + + def replace_masking( + self, + x: torch.Tensor, + mask_ratio: float + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Retain all module positions and replace masked contents with a learned token.""" + if self.encoder_mask_token is None: + raise RuntimeError("replace_masking requires encoder_mask_token to be initialized.") + + b, l, _ = x.shape + mask, _, _ = self._sample_module_mask( + batch_size=b, + seq_len=l, + device=x.device, + mask_ratio=mask_ratio, + ) + ids_restore = torch.arange(l, device=x.device).unsqueeze(0).expand(b, -1) + mask_tokens = self.encoder_mask_token.expand(b, l, -1) + x_masked = torch.where(mask.bool().unsqueeze(-1), mask_tokens, x) + return x_masked, mask, ids_restore + + def _add_module_position_embedding(self, x: torch.Tensor) -> torch.Tensor: + pos = self.pos_emb[:, 1:, :] if self.use_cls else self.pos_emb + if pos.shape[1] != x.shape[1]: + raise RuntimeError( + f"pos_emb length mismatch: pos={pos.shape[1]}, tokens={x.shape[1]}, " + f"modules={self.num_modules}" + ) + x = x + pos + if self.use_family_emb and self.family_emb is not None: + x = x + self.family_emb[self.module_family_ids] + return x + + def forward_encoder( + self, + cells: torch.Tensor, + mask_ratio: Optional[float] = None, + return_module_targets: bool = True, + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, List[torch.Tensor]]: + """Encode gene expression into contextual module representations. + + cells: (B, num_genes). + Returns latent (B, L_encoded + optional_cls, D), mask (B, L), + ids_restore (B, L), and gene-expression targets for each module.""" + mask_ratio = self.mask_ratio if mask_ratio is None else mask_ratio + + # Step 1: convert gene expression into module tokens. + x, module_targets = self.module_embed( + cells, + return_module_targets=return_module_targets, + ) # Module embeddings with shape (B, L, D). + + # Steps 2/3: drop masked tokens or replace their contents before encoding. + if self.mask_style == "replace": + # Replace token contents, then add module-ID embeddings while retaining all positions. + x, mask, ids_restore = self.replace_masking(x, mask_ratio) + x = self._add_module_position_embedding(x) + else: + # MAE path: add module-ID embeddings, then drop masked modules. + x = self._add_module_position_embedding(x) + x, mask, ids_restore = self.random_masking(x, mask_ratio) + + # Sparse module gating (1B): suppress unimportant modules + if self.use_module_gate and self.module_gate is not None: + gate = self.module_gate(x) # (B, L_visible, 1) + x = x * gate + + # Step 4: prepend the CLS token when enabled. + if self.use_cls and self.cls_token is not None: + cls_token = self.cls_token + self.pos_emb[:, :1, :] + cls_tokens = cls_token.expand(x.shape[0], -1, -1) + x = torch.cat((cls_tokens, x), dim=1) + + # Step 5: run the encoder. + x_emb = self.dropout(x) + layer_pos_emb = self.layer_pos_emb(x_emb)# None + x = self.encoder(x_emb, pos_emb=layer_pos_emb) # Shape remains (B, L_visible, D); each token now incorporates global context. + x = self.encoder_norm(x) #layernorm + + return x, mask, ids_restore, module_targets + + def _build_full_tokens(self, x: torch.Tensor, ids_restore: torch.Tensor) -> torch.Tensor: + """Insert decoder mask tokens and restore the original module order. + + Returns (B, L + optional_cls, D).""" + # Project to decoder width, insert mask tokens, and restore module order. + x = self.enc_to_dec(x) + if self.use_cls: + cls_token = x[:, :1, :] + module_tokens = x[:, 1:, :] + else: + cls_token = None + module_tokens = x + + num_mask_tokens = ids_restore.shape[1] - module_tokens.shape[1] + if num_mask_tokens > 0: + if self.mask_token is None: + raise RuntimeError( + "Decoder-side mask_token is missing, but _build_full_tokens needs to restore masked positions." + ) + mask_tokens = self.mask_token.repeat(x.shape[0], num_mask_tokens, 1) + x_ = torch.cat([module_tokens, mask_tokens], dim=1) + else: + x_ = module_tokens + + x_ = torch.gather(x_, dim=1, index=ids_restore.unsqueeze(-1).repeat(1, 1, x.shape[2])) + + if self.use_cls and cls_token is not None: + return torch.cat([cls_token, x_], dim=1) + return x_ + + def _aggregate_module_to_gene( + self, + module_feats: torch.Tensor, + ) -> torch.Tensor: + """Map module_feats (B, L, D) to gene_feats (B, G, D). + + Each gene receives the mean features of all modules containing it.""" + bsz, _, dim = module_feats.shape + gene_feats = torch.zeros(bsz, self.num_genes, dim, device=module_feats.device, dtype=module_feats.dtype) + gene_counts = torch.zeros(bsz, self.num_genes, device=module_feats.device, dtype=module_feats.dtype) + + # Contributions from the fixed pathway module definitions. + if self.pg_module_idx is not None and self.pg_module_idx.numel() > 0: + pair_module_feats = module_feats[:, self.pg_module_idx, :] # (B, num_pairs, D) + gene_idx_expand = self.pg_gene_idx.view(1, -1, 1).expand(bsz, -1, dim) + gene_feats.scatter_add_(1, gene_idx_expand, pair_module_feats) + base_gene_counts = self.pg_gene_counts.to(dtype=module_feats.dtype, device=module_feats.device) + gene_counts = gene_counts + base_gene_counts.view(1, self.num_genes) + + gene_counts = gene_counts.clamp(min=1.0).unsqueeze(-1) # (B, G, 1) + return gene_feats / gene_counts + + def _predict_from_module_features( + self, + module_feats: torch.Tensor, + ) -> Union[List[torch.Tensor], torch.Tensor]: + """module_feats: (B, L, D)""" + if self.recon_target_level == "gene" and self.decoder_gene_agg: + gene_feats = self._aggregate_module_to_gene(module_feats) # (B, G, D) + assert self.gene_w is not None and self.gene_b is not None + # Apply a separate linear projection to each gene. + preds = torch.einsum("bgd,gdk->bgk", gene_feats, self.gene_w).squeeze(-1) + return preds + self.gene_b.unsqueeze(0) # (B, G) + + preds: List[torch.Tensor] = [] + if self.recon_target_level == "gene": + assert self.module_heads is not None + for i, head in enumerate(self.module_heads): + preds.append(head(module_feats[:, i, :])) + else: + assert self.module_scalar_heads is not None + for i, head in enumerate(self.module_scalar_heads): + preds.append(head(module_feats[:, i, :])) + return preds + + def _build_cls_targets(self, module_targets: List[torch.Tensor]) -> torch.Tensor: + """Supervise CLS with mean expression values from the static module sets. + + CLS aggregates visible-module information to predict the cell's + module-level expression profile.""" + static_module_targets = module_targets[: self.num_modules] + if len(static_module_targets) != self.num_modules: + raise RuntimeError( + f"cls target module count mismatch: expected {self.num_modules}, got {len(static_module_targets)}" + ) + module_means = [target.mean(dim=1, keepdim=True) for target in static_module_targets] + return torch.cat(module_means, dim=1) + + def compute_cls_loss( + self, + latent: torch.Tensor, + module_targets: List[torch.Tensor], + ) -> Tuple[torch.Tensor, torch.Tensor]: + if not self.use_cls or self.cls_head is None: + zero = torch.tensor(0.0, device=latent.device, dtype=latent.dtype) + return zero, zero + cls_encoding = latent[:, 0, :] + cls_preds = self.cls_head(cls_encoding) + cls_targets = self._build_cls_targets(module_targets).to(dtype=cls_preds.dtype) + cls_loss = F.mse_loss(cls_preds, cls_targets) + return cls_loss, cls_encoding + + def forward_decoder_simple( + self, + x: torch.Tensor, + ids_restore: torch.Tensor, + ) -> Union[List[torch.Tensor], torch.Tensor]: + """Insert mask tokens, restore module order, and apply reconstruction heads. + + x: (B, L_visible + optional_cls, D). + ids_restore: (B, L), where L is the total module count. + Returns per-module predictions, or a (B, G) tensor for decoder_gene_agg.""" + x_full = self._build_full_tokens(x, ids_restore) + module_feats = x_full[:, 1:, :] if self.use_cls else x_full + return self._predict_from_module_features(module_feats) + + def forward_decoder_performer( + self, + x: torch.Tensor, + ids_restore: torch.Tensor, + ) -> Union[List[torch.Tensor], torch.Tensor]: + """Restore masked positions, mix tokens with a Performer decoder, and reconstruct.""" + x_full = self._build_full_tokens(x, ids_restore) + pos = self.decoder_pos_emb[:, :x_full.shape[1], :] + x_full = self.decoder_dropout(x_full + pos) + x_full = self.decoder(x_full, pos_emb=self.layer_pos_emb(x_full)) + x_full = self.decoder_norm(x_full) + module_feats = x_full[:, 1:, :] if self.use_cls else x_full + return self._predict_from_module_features(module_feats) + + def forward_decoder_cross_attn( + self, + x: torch.Tensor, + mask: torch.Tensor, + ids_restore: torch.Tensor, + ) -> Union[List[torch.Tensor], torch.Tensor]: + """ + Cross-attention-only decoder: + masked queries attend to visible encoder tokens, without query-query self-attention. + """ + if self.decoder is None or self.decoder_pos_emb is None or self.decoder_dropout is None or self.decoder_norm is None: + raise RuntimeError("cross_attn decoder requested but decoder modules are not initialized.") + + x_full = self._build_full_tokens(x, ids_restore) + if self.use_cls: + cls_token = x_full[:, :1, :] + module_tokens = x_full[:, 1:, :] + module_pos = self.decoder_pos_emb[:, 1:x_full.shape[1], :] + cls_memory = cls_token + else: + cls_token = None + module_tokens = x_full + module_pos = self.decoder_pos_emb[:, :x_full.shape[1], :] + cls_memory = None + + query_tokens = self.decoder_dropout(module_tokens + module_pos) + module_feats = module_tokens.clone() + + mask_bool = mask.bool() + visible_mask = ~mask_bool + batch_size, _, dim = module_feats.shape + visible_per_sample = int(visible_mask[0].sum().item()) if visible_mask.shape[1] > 0 else 0 + masked_per_sample = int(mask_bool[0].sum().item()) if mask_bool.shape[1] > 0 else 0 + + if masked_per_sample > 0: + visible_memory = module_feats[visible_mask].view(batch_size, visible_per_sample, dim) + if cls_memory is not None: + visible_memory = torch.cat([cls_memory, visible_memory], dim=1) + + masked_queries = query_tokens[mask_bool].view(batch_size, masked_per_sample, dim) + for block in self.decoder: + masked_queries = block(masked_queries, visible_memory) + # AMP may run attention/norm in fp32 for stability. Cast back before + # scattering into the decoder-state tensor, which may still be bf16. + masked_queries = self.decoder_norm(masked_queries).to(dtype=module_feats.dtype) + module_feats[mask_bool] = masked_queries.reshape(-1, dim) + + return self._predict_from_module_features(module_feats) + + def forward_predictor_from_encoder( + self, + x: torch.Tensor, + ) -> Union[List[torch.Tensor], torch.Tensor]: + module_feats = x[:, 1:, :] if self.use_cls else x + return self._predict_from_module_features(module_feats) + + def compute_loss( + self, + preds: Union[List[torch.Tensor], torch.Tensor], + targets: List[torch.Tensor], + mask: torch.Tensor, + ) -> torch.Tensor: # Module-level reconstruction loss and aggregation ablations. + """Compute reconstruction loss over masked modules. + + preds and targets contain module predictions and expression targets. + mask: (B, L), with 1 for masked modules and 0 for visible modules. + The decoder_gene_agg path accepts gene-level prediction tensors.""" + if self.recon_target_level == "gene" and self.decoder_gene_agg: + assert isinstance(preds, torch.Tensor) + # Use the original module mask to determine the reconstruction loss. + # For each masked module, select its genes from the gene-level predictions. + total_loss = torch.tensor(0.0, device=mask.device, dtype=torch.float32) + total_count = torch.tensor(0.0, device=mask.device, dtype=torch.float32) + for i, t in enumerate(targets): + module_mask = mask[:, i].float() # (B,) + module_gene_idx = torch.as_tensor(self.module_sets[i], dtype=torch.long, device=preds.device) + p = preds.index_select(dim=1, index=module_gene_idx) # (B, G_i) + diff = (p - t) ** 2 + if self.loss_mode == "gene_mean": + total_loss = total_loss + (diff * module_mask.unsqueeze(-1)).sum() + total_count = total_count + module_mask.sum() * t.shape[1] + else: + module_mse = diff.mean(dim=1) # (B,) + total_loss = total_loss + (module_mse * module_mask).sum() + total_count = total_count + module_mask.sum() + return total_loss / total_count.clamp(min=1.0) + + assert isinstance(preds, list) + if len(preds) != len(targets): + raise RuntimeError(f"pred/target module count mismatch: {len(preds)} vs {len(targets)}") + total_loss = torch.tensor(0.0, device=mask.device, dtype=torch.float32) + total_count = torch.tensor(0.0, device=mask.device, dtype=torch.float32) + + for i, (p, t) in enumerate(zip(preds, targets)): + module_mask = mask[:, i].float() # (B,) + if self.recon_target_level == "module_mean_value": + # Predict a module mean and compare it with the mean of its target genes. + # p: (B,1), t_mean: (B,1) + t_mean = t.mean(dim=1, keepdim=True) + diff = (p - t_mean) ** 2 + total_loss = total_loss + (diff.squeeze(-1) * module_mask).sum() + total_count = total_count + module_mask.sum() + else: + diff = (p - t) ** 2 # (B, G_i) + if self.loss_mode == "gene_mean": + # Weight each gene occurrence in masked modules equally. + total_loss = total_loss + (diff * module_mask.unsqueeze(-1)).sum() + total_count = total_count + module_mask.sum() * t.shape[1] + else: + # Average within each module, then across masked modules with equal module weights. + module_mse = diff.mean(dim=1) # (B,) + total_loss = total_loss + (module_mse * module_mask).sum() + total_count = total_count + module_mask.sum() + + if total_count.item() == 0: + return torch.tensor(0.0, device=mask.device, dtype=torch.float32) + return total_loss / total_count + + def forward( + self, cells: torch.Tensor, mask_ratio: Optional[float] = None, + return_encodings: bool = False, return_module_targets: bool = True, + ): + """Run representation extraction or reconstruction. + + cells: (B, num_genes). + mask_ratio=None uses the configured ratio. + When return_encodings=True, return module_encodings, mask, ids_restore, + optional module_targets, and cls_encoding when CLS is enabled. + Otherwise return total loss (reconstruction plus weighted CLS loss), + predictions (per-module tensors or a gene-level tensor), and the mask.""" + latent, mask, ids_restore, module_targets = self.forward_encoder( + cells, + mask_ratio=mask_ratio, + # Reconstruction always needs targets; representation callers can + # opt out explicitly while preserving the original API default. + return_module_targets=(return_module_targets or not return_encodings), + ) + + if return_encodings: + # Return representations for zero-shot tasks. + # drop-mask returns visible tokens; replace-mask returns every module token. + if self.use_cls: + seq_encodings = latent[:, 1:, :] + else: + seq_encodings = latent + + out = { + 'module_encodings': seq_encodings, + 'mask': mask, + 'ids_restore': ids_restore, + 'module_targets': module_targets + } + if self.use_cls: + out['cls_encoding'] = latent[:, 0, :] + return out + + if self.decoder_type == "performer": + preds = self.forward_decoder_performer(latent, ids_restore) + elif self.decoder_type == "cross_attn": + preds = self.forward_decoder_cross_attn(latent, mask, ids_restore) + elif self.decoder_type == "simple": + preds = self.forward_decoder_simple(latent, ids_restore) + elif self.decoder_type == "none": + preds = self.forward_predictor_from_encoder(latent) + else: + raise RuntimeError(f"Unsupported decoder_type at forward time: {self.decoder_type}") + recon_loss = self.compute_loss(preds, module_targets, mask) + cls_loss, _ = self.compute_cls_loss(latent, module_targets) + total_loss = recon_loss + self.cls_loss_weight * cls_loss + self.last_loss_breakdown = { + "recon_loss": float(recon_loss.detach().cpu()), + "cls_loss": float(cls_loss.detach().cpu()), + "total_loss": float(total_loss.detach().cpu()), + } + return total_loss, preds, mask + + def get_last_loss_breakdown(self) -> dict: + return dict(self.last_loss_breakdown) + + def load_state_dict(self, state_dict, strict: bool = True, assign: bool = False): + """Load current or pre-rename weights under the module-named interface.""" + return super().load_state_dict( + migrate_legacy_module_state_dict(state_dict), + strict=strict, + assign=assign, + ) + + def check_redraw_projections(self): + """Check whether projection matrices need redrawing and redraw them.""" + self.encoder.check_redraw_projections() + + def fix_projection_matrices_(self): + """Disable automatic projection-matrix redraws.""" + self.encoder.fix_projection_matrices_() diff --git a/BioMod/module_data_all/create_uncovered_modules.py b/BioMod/module_data_all/create_uncovered_modules.py index 3537b6f4ed698c2a13ffdf9e81169bd7c14a461b..06b3139c99d2ffbac95360c171cd880df9a45cb8 100644 --- a/BioMod/module_data_all/create_uncovered_modules.py +++ b/BioMod/module_data_all/create_uncovered_modules.py @@ -1,146 +1,151 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" -Create additional modules for genes not covered by existing modules -Ensure that all model genes can participate in training - -How to use: - python create_uncovered_modules.py \ - --merged_modules module_data_all/merged_modules.json \ - --output module_data_all/merged_modules_full.json \ - --module_size 50 -""" - -import json -import os -import csv -import argparse - -def create_uncovered_modules( - merged_modules_path, - gene_list_path, - gene_mapping_path, - output_path, - module_size=50, - verbose=True -): - """ - Create additional modules for uncovered genes - - Args: - merged_modules_path: merged module file path - gene_list_path: model gene list path - gene_mapping_path: Gene mapping file path - output_path: output file path - module_size: the size of the new module - verbose: whether to print detailed information - """ - # Load model gene list - with open(gene_list_path, 'r') as f: - model_genes = set(line.strip() for line in f) - - # Load existing modules - with open(merged_modules_path, 'r') as f: - modules = json.load(f) - - # Load gene map (symbol -> ensembl) - symbol_to_ens = {} - ens_to_symbol = {} - with open(gene_mapping_path, 'r') as f: - reader = csv.DictReader(f) - for row in reader: - symbol = row.get('Approved symbol', '').strip() - ens_id = row.get('Ensembl gene ID', '').strip() - if symbol and ens_id: - symbol_to_ens[symbol] = ens_id - ens_to_symbol[ens_id] = symbol - - # Find genes that are covered - covered_genes_ens = set() - for module_name, genes in modules.items(): - for g in genes: - ens_id = symbol_to_ens.get(g) - if ens_id and ens_id in model_genes: - covered_genes_ens.add(ens_id) - elif g in model_genes: - covered_genes_ens.add(g) - - # Find uncovered genes - uncovered_genes = model_genes - covered_genes_ens - uncovered_genes = sorted(list(uncovered_genes)) - - if verbose: - print(f"Total number of model genes: {len(model_genes)}") - print(f"Genes covered: {len(covered_genes_ens)} ({len(covered_genes_ens)/len(model_genes)*100:.1f}%)") - print(f"Genes not covered: {len(uncovered_genes)} ({len(uncovered_genes)/len(model_genes)*100:.1f}%)") - - # Create new modules for uncovered genes - # Use Ensembl ID (because the model uses Ensembl ID) - new_modules = {} - num_new_modules = (len(uncovered_genes) + module_size - 1) // module_size - - for i in range(num_new_modules): - start_idx = i * module_size - end_idx = min((i + 1) * module_size, len(uncovered_genes)) - module_genes = uncovered_genes[start_idx:end_idx] - - # Try converting to symbol if there is a mapping, otherwise keep the Ensembl ID - module_genes_final = [] - for g in module_genes: - symbol = ens_to_symbol.get(g) - if symbol: - module_genes_final.append(symbol) - else: - module_genes_final.append(g) - - new_modules[f"uncovered_module_{i+1:04d}"] = module_genes_final - - if verbose: - print(f"\nCreated {len(new_modules)} new modules (each approx. {module_size} genes)") - - # Merge existing modules and new modules - merged_modules = {**modules, **new_modules} - - # save - os.makedirs(os.path.dirname(output_path) if os.path.dirname(output_path) else '.', exist_ok=True) - with open(output_path, 'w', encoding='utf-8') as f: - json.dump(merged_modules, f, indent=2) - - if verbose: - print(f"\nFinal result:") - print(f" Original modules: {len(modules)}") - print(f" New modules: {len(new_modules)}") - print(f" Total modules: {len(merged_modules)}") - print(f" Save to: {output_path}") - - return merged_modules - -def main(): - parser = argparse.ArgumentParser(description='Create modules for uncovered genes') - parser.add_argument('--merged_modules', type=str, - default='module_data_all/merged_modules.json', - help='Path to merged modules JSON file') - parser.add_argument('--gene_list', type=str, - default='../selected_genes_27k.txt', - help='Path to model gene list file') - parser.add_argument('--gene_mapping', type=str, - default='../gene2ens.csv', - help='Path to gene mapping CSV file') - parser.add_argument('--output', '-o', type=str, - default='module_data_all/merged_modules_full.json', - help='Output file path') - parser.add_argument('--module_size', type=int, default=50, - help='Size of new modules for uncovered genes') - - args = parser.parse_args() - - create_uncovered_modules( - merged_modules_path=args.merged_modules, - gene_list_path=args.gene_list, - gene_mapping_path=args.gene_mapping, - output_path=args.output, - module_size=args.module_size - ) - -if __name__ == '__main__': - main() +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +Create additional modules for genes not covered by existing modules +Ensure that all model genes can participate in training + +Run from the repository root: + python BioMod/module_data_all/create_uncovered_modules.py \ + --merged_modules model_dependencies/module_data_all/merged_modules.json \ + --output tutorial_runs/module_data/merged_modules_full.json \ + --module_size 50 +""" + +import json +import os +import csv +import argparse +from pathlib import Path + +PROJECT_ROOT = Path(__file__).resolve().parents[2] +ASSET_ROOT = PROJECT_ROOT / 'model_dependencies' +MODULE_DIR = Path(__file__).resolve().parent + +def create_uncovered_modules( + merged_modules_path, + gene_list_path, + gene_mapping_path, + output_path, + module_size=50, + verbose=True +): + """ + Create additional modules for uncovered genes + + Args: + merged_modules_path: merged module file path + gene_list_path: model gene list path + gene_mapping_path: Gene mapping file path + output_path: output file path + module_size: the size of the new module + verbose: whether to print detailed information + """ + # Load model gene list + with open(gene_list_path, 'r') as f: + model_genes = set(line.strip() for line in f) + + # Load existing modules + with open(merged_modules_path, 'r') as f: + modules = json.load(f) + + # Load gene map (symbol -> ensembl) + symbol_to_ens = {} + ens_to_symbol = {} + with open(gene_mapping_path, 'r') as f: + reader = csv.DictReader(f) + for row in reader: + symbol = row.get('Approved symbol', '').strip() + ens_id = row.get('Ensembl gene ID', '').strip() + if symbol and ens_id: + symbol_to_ens[symbol] = ens_id + ens_to_symbol[ens_id] = symbol + + # Find genes that are covered + covered_genes_ens = set() + for module_name, genes in modules.items(): + for g in genes: + ens_id = symbol_to_ens.get(g) + if ens_id and ens_id in model_genes: + covered_genes_ens.add(ens_id) + elif g in model_genes: + covered_genes_ens.add(g) + + # Find uncovered genes + uncovered_genes = model_genes - covered_genes_ens + uncovered_genes = sorted(list(uncovered_genes)) + + if verbose: + print(f"Total number of model genes: {len(model_genes)}") + print(f"Genes covered: {len(covered_genes_ens)} ({len(covered_genes_ens)/len(model_genes)*100:.1f}%)") + print(f"Genes not covered: {len(uncovered_genes)} ({len(uncovered_genes)/len(model_genes)*100:.1f}%)") + + # Create new modules for uncovered genes + # Use Ensembl ID (because the model uses Ensembl ID) + new_modules = {} + num_new_modules = (len(uncovered_genes) + module_size - 1) // module_size + + for i in range(num_new_modules): + start_idx = i * module_size + end_idx = min((i + 1) * module_size, len(uncovered_genes)) + module_genes = uncovered_genes[start_idx:end_idx] + + # Try converting to symbol if there is a mapping, otherwise keep the Ensembl ID + module_genes_final = [] + for g in module_genes: + symbol = ens_to_symbol.get(g) + if symbol: + module_genes_final.append(symbol) + else: + module_genes_final.append(g) + + new_modules[f"uncovered_module_{i+1:04d}"] = module_genes_final + + if verbose: + print(f"\nCreated {len(new_modules)} new modules (each approx. {module_size} genes)") + + # Merge existing modules and new modules + merged_modules = {**modules, **new_modules} + + # save + os.makedirs(os.path.dirname(output_path) if os.path.dirname(output_path) else '.', exist_ok=True) + with open(output_path, 'w', encoding='utf-8') as f: + json.dump(merged_modules, f, indent=2) + + if verbose: + print(f"\nFinal result:") + print(f" Original modules: {len(modules)}") + print(f" New modules: {len(new_modules)}") + print(f" Total modules: {len(merged_modules)}") + print(f" Save to: {output_path}") + + return merged_modules + +def main(): + parser = argparse.ArgumentParser(description='Create modules for uncovered genes') + parser.add_argument('--merged_modules', type=str, + default=str(MODULE_DIR / 'merged_modules.json'), + help='Path to merged modules JSON file') + parser.add_argument('--gene_list', type=str, + default=str(ASSET_ROOT / 'selected_genes_27k.txt'), + help='Path to model gene list file') + parser.add_argument('--gene_mapping', type=str, + default=str(ASSET_ROOT / 'gene2ens.csv'), + help='Path to gene mapping CSV file') + parser.add_argument('--output', '-o', type=str, + default=str(MODULE_DIR / 'merged_modules_full.json'), + help='Output file path') + parser.add_argument('--module_size', type=int, default=50, + help='Size of new modules for uncovered genes') + + args = parser.parse_args() + + create_uncovered_modules( + merged_modules_path=args.merged_modules, + gene_list_path=args.gene_list, + gene_mapping_path=args.gene_mapping, + output_path=args.output, + module_size=args.module_size + ) + +if __name__ == '__main__': + main() diff --git a/BioMod/module_data_all/merge_modules.py b/BioMod/module_data_all/merge_modules.py index abe14b1cabf19b99a37cc144741a00c45debb82c..bae26048dcaf97610cb5613b019e8fca673c60e5 100644 --- a/BioMod/module_data_all/merge_modules.py +++ b/BioMod/module_data_all/merge_modules.py @@ -1,96 +1,96 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" -Merge multiple module JSON files into one file -Used to combine module data from different sources for training - -How to use: - python merge_modules.py --output module_data_all/merged_modules.json -""" - -import json -import os -import argparse - -def merge_module_files(input_files, output_file, verbose=True): - """ - Merge multiple module JSON files - - Args: - input_files: list of input file paths - output_file: output file path - verbose: whether to print detailed information - """ - merged_modules = {} - total_modules = 0 - all_genes = set() - - for input_file in input_files: - if not os.path.exists(input_file): - print(f"Warning: File not found: {input_file}") - continue - - with open(input_file, 'r', encoding='utf-8') as f: - data = json.load(f) - - file_modules = 0 - file_genes = set() - - # Get the file name as a prefix to avoid key conflicts - basename = os.path.splitext(os.path.basename(input_file))[0] - - for key, genes in data.items(): - if isinstance(genes, (list, tuple)): - # Add file name prefix to avoid key conflicts - new_key = f"{basename}__{key}" - merged_modules[new_key] = list(genes) - file_modules += 1 - file_genes.update(genes) - - total_modules += file_modules - all_genes.update(file_genes) - - if verbose: - print(f"{os.path.basename(input_file)}:") - print(f" Modules: {file_modules}") - print(f" Unique genes: {len(file_genes)}") - - # Save the merged file - os.makedirs(os.path.dirname(output_file) if os.path.dirname(output_file) else '.', exist_ok=True) - with open(output_file, 'w', encoding='utf-8') as f: - json.dump(merged_modules, f, indent=2) - - if verbose: - print("=" * 50) - print(f"Total modules: {total_modules}") - print(f"Total unique genes: {len(all_genes)}") - print(f"Saved to: {output_file}") - - return merged_modules - -def main(): - parser = argparse.ArgumentParser(description='Merge multiple module JSON files') - parser.add_argument('--input', '-i', nargs='+', - default=[ - 'module_data/module1_hallmarks.json', - 'module_data/module2_reactome_2022_filtered.json', - 'module_data_2/module3_hotspot_260k.json', - # Use absolute paths when adding external files - ], - help='Input JSON files to merge') - parser.add_argument('--output', '-o', - default='module_data_all/merged_modules.json', - help='Output merged JSON file') - parser.add_argument('--add-external', type=str, default=None, - help='Additional external module file to include') - - args = parser.parse_args() - - input_files = args.input - if args.add_external: - input_files.append(args.add_external) - - merge_module_files(input_files, args.output) - -if __name__ == '__main__': - main() +#!/usr/bin/env python +# -*- coding: utf-8 -*- +"""Merge explicitly selected module JSON files. + +Run from the repository root, choosing the module sources for your experiment: + python BioMod/module_data_all/merge_modules.py --input first.json second.json \ + --output tutorial_runs/module_data/merged_modules.json + +The example input filenames are placeholders for your module definitions. +""" + +import json +import os +import argparse +from pathlib import Path + +PROJECT_ROOT = Path(__file__).resolve().parents[2] +ASSET_ROOT = PROJECT_ROOT / 'model_dependencies' +MODULE_DIR = Path(__file__).resolve().parent + +def merge_module_files(input_files, output_file, verbose=True): + """ + Merge multiple module JSON files + + Args: + input_files: list of input file paths + output_file: output file path + verbose: whether to print detailed information + """ + merged_modules = {} + total_modules = 0 + all_genes = set() + + for input_file in input_files: + if not os.path.exists(input_file): + print(f"Warning: File not found: {input_file}") + continue + + with open(input_file, 'r', encoding='utf-8') as f: + data = json.load(f) + + file_modules = 0 + file_genes = set() + + # Get the file name as a prefix to avoid key conflicts + basename = os.path.splitext(os.path.basename(input_file))[0] + + for key, genes in data.items(): + if isinstance(genes, (list, tuple)): + # Add file name prefix to avoid key conflicts + new_key = f"{basename}__{key}" + merged_modules[new_key] = list(genes) + file_modules += 1 + file_genes.update(genes) + + total_modules += file_modules + all_genes.update(file_genes) + + if verbose: + print(f"{os.path.basename(input_file)}:") + print(f" Modules: {file_modules}") + print(f" Unique genes: {len(file_genes)}") + + # Save the merged file + os.makedirs(os.path.dirname(output_file) if os.path.dirname(output_file) else '.', exist_ok=True) + with open(output_file, 'w', encoding='utf-8') as f: + json.dump(merged_modules, f, indent=2) + + if verbose: + print("=" * 50) + print(f"Total modules: {total_modules}") + print(f"Total unique genes: {len(all_genes)}") + print(f"Saved to: {output_file}") + + return merged_modules + +def main(): + parser = argparse.ArgumentParser(description='Merge multiple module JSON files') + parser.add_argument('--input', '-i', nargs='+', required=True, + help='Explicit input module JSON files; historical source collections are not all bundled.') + parser.add_argument('--output', '-o', + default=str(MODULE_DIR / 'merged_modules.json'), + help='Output merged JSON file') + parser.add_argument('--add-external', type=str, default=None, + help='Additional external module file to include') + + args = parser.parse_args() + + input_files = args.input + if args.add_external: + input_files.append(args.add_external) + + merge_module_files(input_files, args.output) + +if __name__ == '__main__': + main() diff --git a/BioMod/performer_module.py b/BioMod/performer_module.py index 3826010e0364f13c3c9a4ff48a96b57028cbbc2c..132c07f1a3afe7a3d5917984bcaa1722b64a7912 100644 --- a/BioMod/performer_module.py +++ b/BioMod/performer_module.py @@ -8,7 +8,7 @@ Main features: 1. Use module data input (refer to MAE) 2. Mask the module (refer to MAE’s random_masking) -3. Only use encoder (no decoder) +3. Use a configurable reconstruction decoder or direct prediction heads 4. Use Performer as encoder (instead of standard Transformer Block) ================================================================================ @@ -56,7 +56,13 @@ from torch.nn.parallel import DistributedDataParallel as DDP import torch.distributed as dist import sys -sys.path.append('../') +from pathlib import Path + +SOURCE_DIR = Path(__file__).resolve().parent +PROJECT_ROOT = SOURCE_DIR.parent +ASSET_ROOT = PROJECT_ROOT / 'model_dependencies' +if str(SOURCE_DIR) not in sys.path: + sys.path.insert(0, str(SOURCE_DIR)) from models_performer_module import ( PerformerEncoderOnly_Module, sanitize_performer_module_config, @@ -82,7 +88,7 @@ parser = argparse.ArgumentParser() parser.add_argument("--local-rank", type=int, default=-1, help='Local process rank.') parser.add_argument("--gene_list_path", type=str, - default = "../selected_genes_27k.txt") + default=str(ASSET_ROOT / 'selected_genes_27k.txt')) parser.add_argument("--gene_num", type=int, default=27874, help='Number of genes.') parser.add_argument("--cell_num", type=int, default=49738024, help='Number of cells.') parser.add_argument("--epoch", type=int, default=100, help='Number of epochs.') @@ -118,11 +124,10 @@ parser.add_argument("--dec_depth", type=int, default=4, help='Decoder layer number (valid when performer/cross_attn)') parser.add_argument("--dec_heads", type=int, default=8, help='Number of decoder attention heads (valid when performer/cross_attn)') parser.add_argument("--dec_dim_head", type=int, default=64, help='Performer decoder dim per head (ignored when cross_attn)') -parser.add_argument("--ckpt_dir", type=str, default='./ckpts_performer_module/', help='Directory of checkpoint to save.') +parser.add_argument("--ckpt_dir", type=str, default=str(PROJECT_ROOT / 'tutorial_runs' / 'pretraining'), help='Directory for generated checkpoints; user relative paths use the current working directory.') parser.add_argument("--model_name", type=str, default='performer_module_27k', help='Pretrained model name.') -parser.add_argument("--data_path", type=str, - default='./cell_files', - help='Path of data for pretraining. When npz/txt is the directory, when h5/h5ad is the file path.') +parser.add_argument("--data_path", type=str, required=True, + help='Pretraining input: a directory for npz/txt or a file for h5/h5ad. Supply your own pretraining data; it is not bundled.') parser.add_argument("--data_format", type=str, default="npz", choices=["npz", "h5", "h5ad", "txt"], help='Pre-training data format: npz=one npz per cell; h5=single HDF5 file; h5ad=AnnData file; txt=one txt per cell') parser.add_argument("--h5_dataset_key", type=str, default="expression", @@ -130,15 +135,15 @@ # Module related parameters parser.add_argument("--module_data_path", type=str, default=None, help='Path to module JSON file (single file).') parser.add_argument("--module_data_dir", type=str, default=None, help='Path to directory containing module JSON files.') -parser.add_argument("--gene_mapping_path", type=str, default='../gene2ens.csv', help='Path to gene symbol to Ensembl ID mapping CSV.') -parser.add_argument("--gene2vec_data_path", type=str, default='../selected_gene2vec_27k.npy') +parser.add_argument("--gene_mapping_path", type=str, default=str(ASSET_ROOT / 'gene2ens.csv'), help='Path to gene symbol to Ensembl ID mapping CSV.') +parser.add_argument("--gene2vec_data_path", type=str, default=str(ASSET_ROOT / 'selected_gene2vec_27k.npy')) parser.add_argument("--fusion_method", type=str, default="multiply", choices=["multiply", "add", "gated_residual"], help='Fusion method: "multiply"=multiply; "add"=add to gene2vec after expressing MLP; "gated_residual"=inject expression information into the gated residual') parser.add_argument("-use_wandb", action='store_true', default=False) parser.add_argument("-compile", action='store_true') parser.add_argument("--model_path", type=str, default='None', - help='Directory of checkpoint to load.') + help='Checkpoint file to load; use its matching configuration for continued pretraining.') parser.add_argument( "--model_config_path", type=str, @@ -963,8 +968,9 @@ def any_rank_true(local_flag: bool) -> bool: wandb.log(wandb_payload)# train every LOG_EVERY step train_log_step train_loss, learning_rate train_log_step += 1 if args.latest_ckpt_every > 0 and (index % args.latest_ckpt_every == 0): - os.makedirs('./latest_checkpoint_performer_module/', exist_ok=True) - save_scaler_ckpt(i, model, optimizer, scheduler, scaler, loss.item(), model_name, './latest_checkpoint_performer_module/') + latest_ckpt_dir = os.path.join(ckpt_dir, 'latest') + os.makedirs(latest_ckpt_dir, exist_ok=True) + save_scaler_ckpt(i, model, optimizer, scheduler, scaler, loss.item(), model_name, latest_ckpt_dir) running_loss += loss.item() diff --git a/BioMod/zero_shot_batch/README.md b/BioMod/zero_shot_batch/README.md index 717a0607c16fd57ba05d5960cd6ba265f12de062..456c6719ae819ff7c7b2745780d795484c1ba62c 100644 --- a/BioMod/zero_shot_batch/README.md +++ b/BioMod/zero_shot_batch/README.md @@ -1,193 +1,159 @@ -# BioMod Zero-Shot Embeddings and Batch-Effect Evaluation - -This directory contains code for extracting BioMod cell embeddings and -evaluating batch effects without fine-tuning. Use the -[BioMod model weights and matching configuration](../../README.md#biomod-model-weights). - -## Features - -- **Task objective**: evaluate batch effects in cell embeddings extracted with the BioMod model weights. -- **Evaluation metrics**: compute ASW_label/batch metrics with the scib package. -- **Data format**: supports single-cell data in h5ad format. - -## Usage - -### Basic Usage - -```bash -python performer_module_zero_shot.py \ - --scfm_hyper_params_path \ - --scfm_ckpt_path \ - --module_data_path \ - --data_path -``` - -### Full Argument Example - -```bash -python performer_module_zero_shot.py \ - --gene2vec_path ../../selected_gene2vec_27k.npy \ - --scfm_hyper_params_path ../performer_module_multiply_all_module_48Mcell/performer_module_multiply_all_module_48Mcell.pkl \ - --scfm_ckpt_path ../performer_module_multiply_all_module_48Mcell/performer_module_multiply_all_module_48Mcell_epoch2.pth \ - --module_data_dir ../module_data_all \ - --gene_mapping_path ../../gene2ens.csv \ - --scfm_genes_file ../../selected_genes_27k.txt \ - --data_path ../../zero-shot-batch/pancreas_scib.h5ad \ - --output_key X_sclong_module \ - --n_cells all \ - --batch_size 256 \ - --fusion_method multiply \ - --output_dir res/performer_module_multiply_all_module_48Mcell_first_module \ - --metrics_mode simple \ - --pool_method first_module -``` - -```bash -python performer_module_zero_shot.py \ - --gene2vec_path ../../selected_gene2vec_27k.npy \ - --scfm_hyper_params_path ../performer_module_multiply_all_module_48Mcell/performer_module_multiply_all_module_48Mcell.pkl \ - --scfm_ckpt_path ../performer_module_multiply_all_module_48Mcell/performer_module_multiply_all_module_48Mcell_epoch2.pth \ - --module_data_dir ../module_data_all \ - --gene_mapping_path ../../gene2ens.csv \ - --scfm_genes_file ../../selected_genes_27k.txt \ - --dataset_preset lung \ - --data_path Kim2020_Lung.h5ad \ - --output_key X_sclong_module \ - --batch_size 128 \ - --fusion_method multiply \ - --metrics_mode scgpt \ - --output_dir res_Lung/performer_module_multiply_all_module_48Mcell_max \ - --pool_method max -``` - -### Arguments - -| Argument | Description | Default | -| --- | --- | --- | -| `--scfm_hyper_params_path` | Path to the matching BioMod model configuration; required | - | -| `--scfm_ckpt_path` | Path to the BioMod model weights; required | - | -| `--module_data_path` | Path to one module JSON file | None | -| `--module_data_dir` | Directory containing multiple module JSON files | None | -| `--gene2vec_path` | Path to the Gene2Vec file | ../selected_gene2vec_27k.npy | -| `--gene_mapping_path` | CSV mapping gene symbols to Ensembl IDs | ../gene2ens.csv | -| `--scfm_genes_file` | BioMod gene list file | ../selected_genes_27k.txt | -| `--data_path` | Input h5ad data file | ../zero-shot-batch/pancreas_scib.h5ad | -| `--output_key` | Key name for embeddings in `adata.obsm` | X_sclong_module | -| `--n_cells` | Number of cells to process; `all` means all cells | all | -| `--batch_size` | Inference batch size | 4 | -| `--output_dir` | Output directory | ./res | - -## Outputs - -After the run finishes, the following files are generated under `--output_dir`: - -1. **`sclong_module_batch_cell_emb_mode.csv`**: evaluation metrics in CSV format. - - Contains three columns: metric, label, and value. - - Main metric: ASW_label/batch. Larger values are better and indicate stronger batch-correction performance. - -2. **`sclong_module_batch_cell_emb_mode_ce.npy`**: NumPy array of cell embeddings. - - Shape: `(n_cells, embed_dim)`. - - Can be used for downstream analysis or visualization. - -## Workflow - -1. **Data loading**: load single-cell data in h5ad format. -2. **Gene mapping**: convert input gene symbols to Ensembl IDs and map them to the BioMod gene list. -3. **Model inference**: - - Reindex gene expression values to the BioMod gene order. - - Obtain encodings with the BioMod model. - - Extract cell embeddings by averaging `module_encodings`. -4. **Evaluation**: compute batch-correction metrics with scib. - -## Cell Embedding Extraction - -The code extracts cell embeddings in the following priority order: - -1. **`module_encodings`** (preferred): average the module encodings from the encoder output. - ```python - cell_emb = out['module_encodings'].mean(dim=1) # (batch_size, embed_dim) - ``` - -2. **`merged_decodings`**: if `module_encodings` is absent, use the first position of decoder output. - ```python - cell_emb = out['merged_decodings'][:, 0, :] # (batch_size, embed_dim) - ``` - -3. **`module_decodings`**: average module decodings. - ```python - cell_emb = out['module_decodings'].mean(dim=1) # (batch_size, embed_dim) - ``` - -## Comparison with Original scLong - -| Feature | scLong | BioMod | -| --- | --- | --- | -| Model architecture | DualEncoder (mini + large) | Single Encoder | -| Input format | Flat gene expression | Module-based | -| Embedding extraction | `merged_decodings[:, 0, :]` | `module_encodings.mean(dim=1)` | -| Gene handling | Directly uses 27k genes | Aggregates through modules | - -## Notes - -1. **Module data**: make sure the correct module data path is provided with `--module_data_path` or `--module_data_dir`. -2. **Gene mapping**: make sure the mapping file (`gene2ens.csv`) contains the required gene-symbol to Ensembl-ID mappings. -3. **Memory usage**: for large datasets, set `--n_cells` to limit the number of processed cells. -4. **Batch size**: adjust `--batch_size` according to GPU memory. - -## Example Output - -The following legacy log illustrates the output format; it does not document a -validated run with the bundled BioMod model weights. - -```text -Using device: cuda -Loading hyper parameters from ./checkpoints/model_params.pkl -Loading module data... -Creating sclong_module model... -Loading checkpoint from ./checkpoints/model_epoch6.pth -Total model parameters: 16,140,337 -Processing 1000 cells with 2000 genes -Input genes and scfm genes intersect: 1800 genes -Extracting cell embeddings... -Cell embeddings shape: (1000, 512) -Evaluating batch correction... -Using batch_key: batch -Using label_keys: ['celltype'] - -Evaluation Results: -================================================== - metric label value -0 ASW_label/batch celltype 0.8234 -================================================== -Zero-shot evaluation completed! -``` - -## Troubleshooting - -### Issue 1: Module Data Not Found - -**Error**: `KeyError: 'module_sets'` - -**Fix**: make sure `--module_data_path` or `--module_data_dir` is provided. - -### Issue 2: Gene Mapping Failed - -**Error**: `KeyError: 'gene_symbol'` - -**Fix**: check the format of `--gene_mapping_path` and make sure it contains the `Ensembl gene ID` column. - -### Issue 3: Model Output Key Missing - -**Error**: `KeyError: 'module_encodings'` - -**Fix**: use the BioMod model weights and their matching configuration, and -check that the model provides the required output. - -### Issue 4: Out of Memory - -**Error**: `RuntimeError: CUDA out of memory` - -**Fix**: - -- Decrease `--batch_size`. -- Use `--n_cells` to limit the number of processed cells. +# BioMod Zero-Shot Embeddings and Batch-Effect Evaluation + +This directory contains code for extracting BioMod cell embeddings and +evaluating batch effects without fine-tuning. Use the +[BioMod model weights and matching configuration](../../README.md#biomod-model-weights). + +## Features + +- **Task objective**: evaluate batch effects in cell embeddings extracted with the BioMod model weights. +- **Evaluation metrics**: compute ASW_label/batch metrics with the scib package. +- **Data format**: supports single-cell data in h5ad format. + +## Usage + +Run these commands from the repository root. Built-in resource defaults resolve from the script location; explicit relative paths resolve from the working directory. + +### Basic Usage + +```bash +python BioMod/zero_shot_batch/performer_module_zero_shot.py \ + --scfm_hyper_params_path weight/0810model/mae_tost_70M_75mask_ec2.pkl \ + --scfm_ckpt_path weight/0810model/mae_tost_70M_75mask_ec2_epoch0.pth \ + --module_data_path model_dependencies/module_data_all/merged_modules.json \ + --data_path data/downstream/final_data/Kim2020_Lung.h5ad \ + --dataset_preset lung +``` + +### Full Argument Example + +```bash +python BioMod/zero_shot_batch/performer_module_zero_shot.py \ + --gene2vec_path model_dependencies/selected_gene2vec_27k.npy \ + --scfm_hyper_params_path weight/0810model/mae_tost_70M_75mask_ec2.pkl \ + --scfm_ckpt_path weight/0810model/mae_tost_70M_75mask_ec2_epoch0.pth \ + --module_data_dir model_dependencies/module_data_all \ + --gene_mapping_path model_dependencies/gene2ens.csv \ + --scfm_genes_file model_dependencies/selected_genes_27k.txt \ + --data_path data/downstream/final_data/Kim2020_Lung.h5ad \ + --dataset_preset lung \ + --output_key X_sclong_module \ + --n_cells all \ + --batch_size 256 \ + --output_dir tutorial_runs/zero_shot_first_module \ + --metrics_mode simple \ + --pool_method first_module +``` + +```bash +python BioMod/zero_shot_batch/performer_module_zero_shot.py \ + --gene2vec_path model_dependencies/selected_gene2vec_27k.npy \ + --scfm_hyper_params_path weight/0810model/mae_tost_70M_75mask_ec2.pkl \ + --scfm_ckpt_path weight/0810model/mae_tost_70M_75mask_ec2_epoch0.pth \ + --module_data_dir model_dependencies/module_data_all \ + --gene_mapping_path model_dependencies/gene2ens.csv \ + --scfm_genes_file model_dependencies/selected_genes_27k.txt \ + --dataset_preset lung \ + --data_path data/downstream/final_data/Kim2020_Lung.h5ad \ + --output_key X_sclong_module \ + --batch_size 128 \ + --metrics_mode scgpt \ + --output_dir tutorial_runs/zero_shot_max \ + --pool_method max +``` + +### Arguments + +| Argument | Description | Default | +| --- | --- | --- | +| `--scfm_hyper_params_path` | Path to the matching BioMod model configuration; required | - | +| `--scfm_ckpt_path` | Path to the BioMod model weights; required | - | +| `--module_data_path` | Path to one module JSON file | None | +| `--module_data_dir` | Directory containing multiple module JSON files | None | +| `--gene2vec_path` | Path to the Gene2Vec file | model_dependencies/selected_gene2vec_27k.npy | +| `--gene_mapping_path` | CSV mapping gene symbols to Ensembl IDs | model_dependencies/gene2ens.csv | +| `--scfm_genes_file` | BioMod gene list file | model_dependencies/selected_genes_27k.txt | +| `--data_path` | Input h5ad data file | Required | +| `--output_key` | Key name for embeddings in `adata.obsm` | X_sclong_module | +| `--n_cells` | Number of cells to process; `all` means all cells | all | +| `--batch_size` | Inference batch size | 4 | +| `--output_dir` | Output directory | `BioMod/zero_shot_batch/res/` under the repository | + +## Outputs + +After the run finishes, the following files are generated under `--output_dir`: + +1. **`sclong_module_batch_cell_emb_mode.csv`**: evaluation metrics in CSV format. + - Contains three columns: metric, label, and value. + - Main metric: ASW_label/batch. Larger values are better and indicate stronger batch-correction performance. + +2. **`sclong_module_batch_cell_emb_mode_ce.npy`**: NumPy array of cell embeddings. + - Shape: `(n_cells, embed_dim)`. + - Can be used for downstream analysis or visualization. + +## Workflow + +1. **Data loading**: load single-cell data in h5ad format. +2. **Gene mapping**: convert input gene symbols to Ensembl IDs and map them to the BioMod gene list. +3. **Model inference**: + - Reindex gene expression values to the BioMod gene order. + - Obtain encodings with the BioMod model. + - Extract cell embeddings using the selected `--pool_method`. +4. **Evaluation**: compute batch-correction metrics with scib. + +## Cell Embedding Extraction + +The model output must contain `module_encodings`. Pooling is selected explicitly: + +- `auto`: use `cls_encoding` when present, otherwise mean-pool module encodings. +- `cls`: use `cls_encoding`; if absent, log a warning and use mean pooling. +- `mean` or `max`: pool module encodings along the module dimension. +- `first_module`: restore the original module order with `ids_restore` and select the first module. + +The current script does not fall back to decoder outputs when module encodings are absent. + +## Comparison with Original scLong + +| Feature | scLong | BioMod | +| --- | --- | --- | +| Model architecture | DualEncoder (mini + large) | Single Encoder | +| Input format | Flat gene expression | Module-based | +| Embedding extraction | `merged_decodings[:, 0, :]` | `module_encodings.mean(dim=1)` | +| Gene handling | Directly uses 27k genes | Aggregates through modules | + +## Notes + +1. **Module data**: make sure the correct module data path is provided with `--module_data_path` or `--module_data_dir`. +2. **Gene mapping**: make sure the mapping file (`gene2ens.csv`) contains the required gene-symbol to Ensembl-ID mappings. +3. **Memory usage**: for large datasets, set `--n_cells` to limit the number of processed cells. +4. **Batch size**: adjust `--batch_size` according to GPU memory. + +## Troubleshooting + +### Issue 1: Module Data Not Found + +**Error**: `KeyError: 'module_sets'` + +**Fix**: make sure `--module_data_path` or `--module_data_dir` is provided. + +### Issue 2: Gene Mapping Failed + +**Error**: `KeyError: 'gene_symbol'` + +**Fix**: check the format of `--gene_mapping_path` and make sure it contains the `Ensembl gene ID` column. + +### Issue 3: Model Output Key Missing + +**Error**: `KeyError: 'module_encodings'` + +**Fix**: use the BioMod model weights and their matching configuration, and +check that the model provides the required output. + +### Issue 4: Out of Memory + +**Error**: `RuntimeError: CUDA out of memory` + +**Fix**: + +- Decrease `--batch_size`. +- Use `--n_cells` to limit the number of processed cells. diff --git a/BioMod/zero_shot_batch/performer_module_zero_shot.py b/BioMod/zero_shot_batch/performer_module_zero_shot.py index d71e22e79949a9c03b44fd07c3b5baa2641a52bf..5bdf7e33642a6d01c502ebf3b6e0a6b69799482b 100644 --- a/BioMod/zero_shot_batch/performer_module_zero_shot.py +++ b/BioMod/zero_shot_batch/performer_module_zero_shot.py @@ -1,689 +1,694 @@ -""" -Adaptation code of the PerformerEncoderOnly_Module model on the zero-shot batch correction task. - -The current script mainly supports: -1. Use the PerformerEncoderOnly_Module model (encoder-only) to extract cell embeddings. -2. Use mean pooling of module encodings as cell embedding. -3. Supports evaluation based on scIB indicators consistent with the scGPT tutorial. -4. Supports Lung data set preset and retains all genes by default without going through the HVG 3000 process. -""" - -import argparse -import importlib.util -import os -import pickle -import sys -import tempfile -from copy import deepcopy -from typing import Dict, List, Optional, Sequence, Set, Union - -os.environ.setdefault("NUMBA_DISABLE_JIT", "1") -os.environ.setdefault("NUMBA_CACHE_DIR", tempfile.mkdtemp(prefix="numba_cache_")) - -import numpy as np -import pandas as pd -import scanpy as sc -import scib -import torch -from scanpy import AnnData -from tqdm import tqdm - -sys.path.append("../../") -sys.path.append("../") - - -DATASET_PRESETS = { - "auto": {}, - "lung": { - "gene_col": "gene_name", - "batch_key": "sample", - "label_keys": ["cell_type"], - }, -} - - -def print_sys(message: str) -> None: - print(message, flush=True, file=sys.stderr) - - -def strip_gene_version(gene_id: str) -> str: - return gene_id.split(".", 1)[0] - - -def canonicalize_target_gene_id( - raw_gene: object, - symbol_to_ensembl: Dict[str, str], - target_gene_set: Set[str], -) -> Optional[str]: - if pd.isna(raw_gene): - return None - - raw_gene_str = str(raw_gene).strip() - if not raw_gene_str: - return None - - direct_candidates = [raw_gene_str, strip_gene_version(raw_gene_str)] - for candidate in direct_candidates: - if candidate in target_gene_set: - return candidate - - mapped_gene = symbol_to_ensembl.get(raw_gene_str) - if mapped_gene is None: - mapped_gene = symbol_to_ensembl.get(strip_gene_version(raw_gene_str)) - - if mapped_gene is None or pd.isna(mapped_gene): - return None - - mapped_gene_str = strip_gene_version(str(mapped_gene).strip()) - return mapped_gene_str or None - - -def resolve_input_gene_ids( - adata: AnnData, - gene_col: Optional[str], - symbol_to_ensembl: Dict[str, str], - target_gene_set: Set[str], -) -> List[str]: - if gene_col is None: - raw_genes = [str(gene) for gene in adata.var.index.tolist()] - source_name = "adata.var.index" - else: - if gene_col not in adata.var.columns: - raise KeyError(f"gene_col '{gene_col}' not found in adata.var.columns") - raw_genes = [str(gene) if not pd.isna(gene) else "" for gene in adata.var[gene_col].tolist()] - source_name = f"adata.var['{gene_col}']" - - resolved_gene_ids: List[str] = [] - direct_hits = 0 - mapped_hits = 0 - unresolved = 0 - - for index, raw_gene in enumerate(raw_genes): - resolved_gene = canonicalize_target_gene_id(raw_gene, symbol_to_ensembl, target_gene_set) - if resolved_gene is None: - resolved_gene = f"unknown{index}" - unresolved += 1 - elif strip_gene_version(raw_gene) == resolved_gene: - direct_hits += 1 - else: - mapped_hits += 1 - resolved_gene_ids.append(resolved_gene) - - print_sys(f"Using genes from {source_name}") - print_sys( - "Gene ID resolution summary: " - f"direct_hits={direct_hits}, mapped_hits={mapped_hits}, unresolved={unresolved}" - ) - return resolved_gene_ids - - -def clear_neighbors(adata: AnnData) -> None: - neighbors = adata.uns.pop("neighbors", None) - if neighbors is not None: - dist_key = neighbors.get("distances_key", "distances") - conn_key = neighbors.get("connectivities_key", "connectivities") - adata.obsp.pop(dist_key, None) - adata.obsp.pop(conn_key, None) - adata.obsp.pop("distances", None) - adata.obsp.pop("connectivities", None) - - -def prepare_eval_adata( - adata: AnnData, - batch_key: str, - label_key: str, - embedding_key: str, - build_neighbors: bool = False, -) -> AnnData: - if embedding_key not in adata.obsm: - raise ValueError(f"Embeddings '{embedding_key}' not found in adata.obsm") - - if batch_key not in adata.obs.columns: - raise ValueError(f"batch_key '{batch_key}' not found in adata.obs") - - if label_key not in adata.obs.columns: - raise ValueError(f"label_key '{label_key}' not found in adata.obs") - - valid_mask = adata.obs[batch_key].notna() & adata.obs[label_key].notna() - dropped = int((~valid_mask).sum()) - if dropped > 0: - print_sys( - f"Dropping {dropped} cells with missing batch/label values " - f"for evaluation (batch_key={batch_key}, label_key={label_key})." - ) - - if int(valid_mask.sum()) == 0: - raise ValueError( - f"No labeled cells remain after filtering batch_key={batch_key}, label_key={label_key}" - ) - - eval_adata = adata[valid_mask].copy() - eval_adata.obs[batch_key] = eval_adata.obs[batch_key].astype("category") - eval_adata.obs[label_key] = eval_adata.obs[label_key].astype("category") - - if build_neighbors: - clear_neighbors(eval_adata) - sc.pp.neighbors(eval_adata, use_rep=embedding_key) - return eval_adata - - -def eval_scib_metrics_simple( - adata: AnnData, - batch_key: str, - label_key: str, - embedding_key: str, -) -> Dict[str, float]: - eval_adata = prepare_eval_adata( - adata, - batch_key, - label_key, - embedding_key, - build_neighbors=False, - ) - - results_dict: Dict[str, float] = {} - if len(eval_adata.obs[batch_key].unique()) > 1: - results_dict["ASW_label/batch"] = scib.metrics.silhouette_batch( - eval_adata, - batch_key, - label_key, - embed=embedding_key, - metric="euclidean", - return_all=False, - verbose=False, - ) - - results_dict = {key: value for key, value in results_dict.items() if not np.isnan(value)} - return results_dict - - -def eval_scib_metrics_scgpt( - adata: AnnData, - batch_key: str, - label_key: str, - embedding_key: str, -) -> Dict[str, float]: - eval_adata = prepare_eval_adata( - adata, - batch_key, - label_key, - embedding_key, - build_neighbors=True, - ) - - results = scib.metrics.metrics( - eval_adata, - adata_int=eval_adata, - batch_key=batch_key, - label_key=label_key, - embed=embedding_key, - isolated_labels_asw_=False, - silhouette_=True, - hvg_score_=False, - graph_conn_=True, - pcr_=True, - isolated_labels_f1_=False, - trajectory_=False, - nmi_=True, - ari_=True, - cell_cycle_=False, - kBET_=False, - ilisi_=False, - clisi_=False, - ) - result_dict = results[0].to_dict() - - result_dict["avg_bio"] = np.mean( - [ - result_dict["NMI_cluster/label"], - result_dict["ARI_cluster/label"], - result_dict["ASW_label"], - ] - ) - result_dict["avg_batch"] = np.mean( - [ - result_dict["graph_conn"], - result_dict["ASW_label/batch"], - ] - ) - - result_dict = {key: value for key, value in result_dict.items() if not np.isnan(value)} - return result_dict - - -def evaluate( - adata: AnnData, - batch_key: str, - label_keys: Sequence[str], - embedding_key: str, - metrics_mode: str, - res_path: str, -) -> pd.DataFrame: - met_df = pd.DataFrame(columns=["metric", "label", "value"]) - - unique_label_keys: List[str] = [] - for key in label_keys: - if key not in unique_label_keys: - unique_label_keys.append(key) - - if not unique_label_keys: - raise ValueError("No label keys were provided for evaluation") - - for label_key in unique_label_keys: - if metrics_mode == "scgpt": - metrics = eval_scib_metrics_scgpt( - adata=adata, - batch_key=batch_key, - label_key=label_key, - embedding_key=embedding_key, - ) - elif metrics_mode == "simple": - metrics = eval_scib_metrics_simple( - adata=adata, - batch_key=batch_key, - label_key=label_key, - embedding_key=embedding_key, - ) - else: - raise ValueError(f"Unknown metrics_mode: {metrics_mode}") - - print_sys( - "\n".join( - [f"[{label_key}] {metric_name}: {metric_value:.4f}" for metric_name, metric_value in metrics.items()] - ) - ) - for metric_name, metric_value in metrics.items(): - met_df.loc[len(met_df)] = [metric_name, label_key, metric_value] - - met_df.to_csv(res_path, index=False) - np.save(f"{res_path[:-4]}_ce.npy", adata.obsm[embedding_key]) - return met_df - - -def resolve_with_preset( - cli_value: Optional[object], - preset_value: Optional[object], -) -> Optional[object]: - return cli_value if cli_value is not None else preset_value - - -def resolve_batch_key(adata: AnnData, requested_batch_key: Optional[str]) -> str: - if requested_batch_key is not None: - if requested_batch_key not in adata.obs.columns: - raise KeyError(f"batch_key '{requested_batch_key}' not found in adata.obs.columns") - return requested_batch_key - - fallback_keys = ["batch", "str_batch", "sample", "Sample Characteristic[individual]"] - for key in fallback_keys: - if key in adata.obs.columns: - print_sys(f"Auto-selected batch_key='{key}'") - return key - - raise ValueError( - "Unable to infer batch_key. Please pass --batch_key explicitly." - ) - - -def resolve_label_keys(adata: AnnData, requested_label_keys: Optional[Sequence[str]]) -> List[str]: - if requested_label_keys: - missing_keys = [key for key in requested_label_keys if key not in adata.obs.columns] - if missing_keys: - raise KeyError(f"label_key(s) not found in adata.obs.columns: {missing_keys}") - return list(requested_label_keys) - - fallback_keys = ["celltype", "cell_type", "celltype_original"] - resolved_keys = [key for key in fallback_keys if key in adata.obs.columns] - if resolved_keys: - print_sys(f"Auto-selected label_keys={resolved_keys}") - return resolved_keys - - raise ValueError( - "Unable to infer label_key. Please pass --label_key explicitly." - ) - - -def reindex_tensor_universal( - tensor: torch.Tensor, - index_positions: Sequence[int], - dim: int, - filler: str = "0", - device: Union[str, torch.device] = "cpu", -) -> torch.Tensor: - index_tensor = torch.tensor(index_positions, device=device) - tensor_shape = list(tensor.shape) - tensor_shape[dim] = len(index_positions) - index_shape = [1] * len(tensor_shape) - index_shape[dim] = len(index_positions) - index_tensor = index_tensor.view(*index_shape).expand(*tensor_shape) - - if filler == "0": - padder_shape = deepcopy(tensor_shape) - padder_shape[dim] = 1 - padder = torch.zeros(padder_shape, dtype=tensor.dtype, device=device) - elif filler == "mean": - padder = tensor.mean(dim=dim, keepdim=True) - else: - raise ValueError("filler should be 0 or mean") - expanded_tensor = torch.cat((tensor, padder), dim=dim) - return torch.gather(expanded_tensor, dim, index_tensor) - - -project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -sys.path.insert(0, project_root) -sys.path.insert(0, os.path.dirname(project_root)) - -performer_module_dir = project_root - -def load_module_modules(): - models_path = os.path.join(performer_module_dir, "models_performer_module.py") - if not os.path.exists(models_path): - raise FileNotFoundError(f"Model file not found: {models_path}") - models_spec = importlib.util.spec_from_file_location("models_performer_module", models_path) - models_performer_module = importlib.util.module_from_spec(models_spec) - models_spec.loader.exec_module(models_performer_module) - - dataset_path = os.path.join(performer_module_dir, "module_dataset.py") - if not os.path.exists(dataset_path): - raise FileNotFoundError(f"Dataset file not found: {dataset_path}") - dataset_spec = importlib.util.spec_from_file_location("module_dataset", dataset_path) - module_dataset = importlib.util.module_from_spec(dataset_spec) - dataset_spec.loader.exec_module(module_dataset) - - return ( - models_performer_module.PerformerEncoderOnly_Module, - models_performer_module.sanitize_performer_module_config, - module_dataset.load_module_data, - ) - -OUTPUT_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "res") -os.makedirs(OUTPUT_DIR, exist_ok=True) - -parser = argparse.ArgumentParser() -parser.add_argument("--gene2vec_path", type=str, default="../selected_gene2vec_27k.npy", - help="Path to gene2vec file") -parser.add_argument("--scfm_hyper_params_path", type=str, required=True, - help="Path to sclong_module model hyper parameters pkl file") -parser.add_argument("--scfm_ckpt_path", type=str, required=True, - help="Path to sclong_module model checkpoint") -parser.add_argument("--module_data_path", type=str, default=None, - help="Path to module JSON file (single file)") -parser.add_argument("--module_data_dir", type=str, default=None, - help="Path to directory containing module JSON files") -parser.add_argument("--gene_mapping_path", type=str, default="../gene2ens.csv", - help="Path to gene symbol to Ensembl ID mapping CSV") -parser.add_argument("--scfm_genes_file", type=str, default="../selected_genes_27k.txt", - help="Path to scfm genes list file") -parser.add_argument("--data_path", type=str, default="../zero-shot-batch/pancreas_scib.h5ad", - help="Path to input h5ad file") -parser.add_argument("--dataset_preset", type=str, default="auto", choices=sorted(DATASET_PRESETS.keys()), - help="Dataset preset for batch/label/gene column defaults") -parser.add_argument("--gene_col", type=str, default=None, - help="Column in adata.var used as the input gene symbol column. " - "If not set, use adata.var.index.") -parser.add_argument("--batch_key", type=str, default=None, - help="Column in adata.obs used as the batch label") -parser.add_argument("--label_key", nargs="+", default=None, - help="One or more columns in adata.obs used as evaluation labels") -parser.add_argument("--output_key", type=str, default="X_sclong_module", - help="Key name for storing embeddings in adata.obsm") -parser.add_argument("--pool_method", type=str, default="auto", choices=["auto", "mean", "cls", "max", "first_module"], - help="Cell embedding pooling. auto prefers cls_encoding when the checkpoint has a trained CLS token; " - "first_module uses the first module after restoring original module order.") -parser.add_argument("--n_cells", type=str, default="all", - help="Number of cells to process (default: all)") -parser.add_argument("--batch_size", type=int, default=4, - help="Batch size for inference") -parser.add_argument("--output_dir", type=str, default=OUTPUT_DIR, - help="Output directory for results") -parser.add_argument("--metrics_mode", type=str, default="scgpt", choices=["scgpt", "simple"], - help="Evaluation metrics mode. Default uses the scGPT tutorial metric bundle.") -parser.add_argument("--min_genes_per_cell", type=int, default=0, - help="Optional Scanpy cell filter. Default 0 keeps all cells.") -parser.add_argument("--min_cells_per_gene", type=int, default=0, - help="Optional Scanpy gene filter. Default 0 keeps all genes.") -parser.add_argument("--shuffle_module_genes", action="store_true", - help="Optional downstream-only extra reshuffle on top of module_sets restored from PKL. " - "Do not pass this when reproducing the pretrained backbone as-is.") -parser.add_argument("--shuffle_seed", type=int, default=2021, - help="Random seed for the downstream extra reshuffle when --shuffle_module_genes is enabled.") -parser.add_argument("--save_raw_module_emb", action="store_true", - help="Save raw module_encodings with shape (num_cells, num_modules, embed_dim)") - -args = parser.parse_args() -os.makedirs(args.output_dir, exist_ok=True) - -device = torch.device("cuda" if torch.cuda.is_available() else "cpu") -print_sys(f"Using device: {device}") - -preset_config = DATASET_PRESETS[args.dataset_preset] -resolved_gene_col = resolve_with_preset(args.gene_col, preset_config.get("gene_col")) -resolved_batch_key = resolve_with_preset(args.batch_key, preset_config.get("batch_key")) -resolved_label_keys = resolve_with_preset(args.label_key, preset_config.get("label_keys")) - -print_sys( - f"dataset_preset={args.dataset_preset}, " - f"gene_col={resolved_gene_col}, batch_key={resolved_batch_key}, " - f"label_keys={resolved_label_keys}, metrics_mode={args.metrics_mode}, " - f"pool_method={args.pool_method}" -) - -PerformerEncoderOnly_Module, sanitize_performer_module_config, load_module_data = load_module_modules() - -print_sys(f"Loading hyper parameters from {args.scfm_hyper_params_path}") -with open(args.scfm_hyper_params_path, "rb") as handle: - scfm_hyper_params = pickle.load(handle) -scfm_hyper_params.pop("module_embed_chunk_padded_gene_budget", None) -scfm_hyper_params = sanitize_performer_module_config( - scfm_hyper_params, - source=args.scfm_hyper_params_path, - log_fn=print_sys, -) - -scfm_hyper_params["gene2vec_file"] = args.gene2vec_path -# Read fusion_method from pkl to ensure consistency with pre-training -fusion_method = scfm_hyper_params.get("fusion_method", "multiply") -print_sys(f"Using fusion_method from pkl: {fusion_method}") - -if "module_sets" not in scfm_hyper_params or scfm_hyper_params.get("module_sets") is None: - if args.module_data_path or args.module_data_dir: - print_sys("Loading module data...") - module_sets = load_module_data( - module_data_path=args.module_data_path, - module_data_dir=args.module_data_dir, - gene_list_path=args.scfm_genes_file, - gene_mapping_path=args.gene_mapping_path, - verbose=True, - ) - scfm_hyper_params["module_sets"] = module_sets - else: - raise ValueError("Model requires module_sets, but neither --module_data_path nor --module_data_dir was provided") -else: - module_sets = scfm_hyper_params["module_sets"] - print_sys(f"Using module_sets from hyper parameters ({len(module_sets)} modules)") - -if args.shuffle_module_genes: - import random - - print_sys( - f"Applying downstream-only extra module reshuffle (seed={args.shuffle_seed}). " - f"Pretraining-time module_sets from PKL will be reshuffled again." - ) - all_gene_indices: List[int] = [] - module_sizes: List[int] = [] - for module in module_sets: - all_gene_indices.extend(module) - module_sizes.append(len(module)) - - rng = random.Random(args.shuffle_seed) - rng.shuffle(all_gene_indices) - - shuffled_module_sets = [] - start_idx = 0 - for size in module_sizes: - shuffled_module_sets.append(all_gene_indices[start_idx:start_idx + size]) - start_idx += size - module_sets = shuffled_module_sets - scfm_hyper_params["module_sets"] = module_sets - print_sys(f"Shuffled {len(all_gene_indices)} genes across {len(module_sets)} modules") - -print_sys(f"Model hyper parameters: {list(scfm_hyper_params.keys())}") - -pkl_use_gene2vec = scfm_hyper_params.get("use_gene2vec", True) -scfm_hyper_params["use_gene2vec"] = pkl_use_gene2vec -print_sys(f"use_gene2vec={pkl_use_gene2vec} (from PKL)") - -print_sys("Creating PerformerEncoderOnly_Module model...") -model = PerformerEncoderOnly_Module(**scfm_hyper_params).to(device) - -print_sys(f"Loading checkpoint from {args.scfm_ckpt_path}") -ckpt = torch.load(args.scfm_ckpt_path, map_location=device) -model_state_dict = ckpt.get("model_state_dict", ckpt) -filtered_state_dict = { - key: value - for key, value in model_state_dict.items() - if "go_conv" not in key and "G_go" not in key.lower() -} -model.load_state_dict(filtered_state_dict, strict=True) - -for parameter in model.parameters(): - parameter.requires_grad = False -model.eval() - -all_params = sum(parameter.numel() for parameter in model.parameters()) -trainable_params = sum(parameter.numel() for parameter in model.parameters() if parameter.requires_grad) -print_sys(f"Total model parameters: {all_params:,}") -print_sys(f"Trainable parameters during zero-shot: {trainable_params:,}") - -gene2ens = pd.read_csv(args.gene_mapping_path, index_col=0) -symbol_to_ensembl = gene2ens["Ensembl gene ID"].to_dict() - -with open(args.scfm_genes_file, "r") as handle: - scfm_genes = [line.rstrip("\n") for line in handle.readlines()] -scfm_gene_set = set(scfm_genes) - -print_sys(f"Loading data from {args.data_path}") -adata = sc.read_h5ad(args.data_path) - -if args.min_genes_per_cell > 0: - before_cells = adata.n_obs - sc.pp.filter_cells(adata, min_genes=args.min_genes_per_cell) - print_sys(f"Filtered cells by min_genes={args.min_genes_per_cell}: {before_cells} -> {adata.n_obs}") - -if args.min_cells_per_gene > 0: - before_genes = adata.n_vars - sc.pp.filter_genes(adata, min_cells=args.min_cells_per_gene) - print_sys(f"Filtered genes by min_cells={args.min_cells_per_gene}: {before_genes} -> {adata.n_vars}") - -if args.n_cells == "all": - n_cells = adata.n_obs -else: - n_cells = int(args.n_cells) - if adata.n_obs > n_cells: - print_sys(f"Subsampling to {n_cells} cells") - sc.pp.subsample(adata, n_obs=n_cells, copy=False) - n_cells = adata.n_obs - -batch_key = resolve_batch_key(adata, resolved_batch_key) -label_keys = resolve_label_keys(adata, resolved_label_keys) - -print_sys(f"Processing {n_cells} cells with {adata.n_vars} genes") -print_sys(f"Evaluation batch_key={batch_key}, label_keys={label_keys}") - -input_gene_ids = resolve_input_gene_ids( - adata=adata, - gene_col=resolved_gene_col, - symbol_to_ensembl=symbol_to_ensembl, - target_gene_set=scfm_gene_set, -) - -input_mapping = {gene_id: index for index, gene_id in enumerate(input_gene_ids)} -input_gene_num = len(input_gene_ids) - -intersect_genes = np.intersect1d(input_gene_ids, scfm_genes) -print_sys(f"Input genes and scfm genes intersect: {len(intersect_genes)} genes") - -scfm_genes_pad = scfm_genes + ["PAD"] -scfm_seq_len = len(scfm_genes_pad) -scfm_mapping = {gene_id: index for index, gene_id in enumerate(scfm_genes_pad)} - -scfm_index_positions = [input_mapping.get(gene_id, input_gene_num) for gene_id in scfm_genes_pad] -_ = [scfm_mapping.get(gene_id, scfm_seq_len) for gene_id in input_gene_ids] - -print_sys("Extracting cell embeddings...") -cell_embeddings = [] -raw_module_embeddings = [] if args.save_raw_module_emb else None -embedding_key = args.output_key - -with torch.no_grad(): - for start in tqdm(range(0, n_cells, args.batch_size)): - batch_end = min(start + args.batch_size, n_cells) - batch_matrix = adata.X[start:batch_end, :] - if hasattr(batch_matrix, "toarray"): - batch_matrix = batch_matrix.toarray() - x = torch.tensor(batch_matrix, dtype=torch.float32).to(device) - - x_scfm = reindex_tensor_universal(x, scfm_index_positions, dim=1, filler="0", device=device) - out = model(x_scfm, mask_ratio=0.0, return_encodings=True) - - if "module_encodings" not in out: - raise ValueError(f"Expected 'module_encodings' in output, got: {list(out.keys())}") - - module_encodings = out["module_encodings"] - ids_restore = out.get("ids_restore") - if args.pool_method == "cls": - if "cls_encoding" not in out: - print_sys("Warning: pool_method='cls' but checkpoint has no cls_encoding, using mean pooling instead") - cell_emb = module_encodings.mean(dim=1) - else: - cell_emb = out["cls_encoding"] - elif args.pool_method == "mean": - cell_emb = module_encodings.mean(dim=1) - elif args.pool_method == "max": - cell_emb = module_encodings.max(dim=1)[0] - elif args.pool_method == "first_module": - if ids_restore is None: - raise ValueError("pool_method='first_module' requires ids_restore in model outputs") - restore_index = ids_restore.unsqueeze(-1).expand(-1, -1, module_encodings.shape[-1]) - restored_module_encodings = torch.gather(module_encodings, dim=1, index=restore_index) - cell_emb = restored_module_encodings[:, 0, :] - else: - cell_emb = out["cls_encoding"] if "cls_encoding" in out else module_encodings.mean(dim=1) - - if args.save_raw_module_emb: - raw_module_embeddings.append(module_encodings.cpu().numpy()) - - cell_embeddings.append(cell_emb.cpu().numpy()) - -cell_embeddings = np.concatenate(cell_embeddings, axis=0) -print_sys(f"Cell embeddings shape: {cell_embeddings.shape}") - -if args.save_raw_module_emb and raw_module_embeddings is not None: - raw_module_embeddings = np.concatenate(raw_module_embeddings, axis=0) - print_sys(f"Raw module embeddings shape: {raw_module_embeddings.shape}") - raw_module_path = os.path.join(args.output_dir, "sclong_module_raw_module_emb.npy") - np.save(raw_module_path, raw_module_embeddings) - print_sys(f"Raw module embeddings saved to {raw_module_path}") - -adata.obsm[embedding_key] = cell_embeddings - -print_sys("Evaluating batch correction...") -res = evaluate( - adata=adata, - batch_key=batch_key, - label_keys=label_keys, - embedding_key=embedding_key, - metrics_mode=args.metrics_mode, - res_path=os.path.join(args.output_dir, "sclong_module_batch_cell_emb_mode.csv"), -) - -print_sys("\nEvaluation Results:") -print_sys("=" * 50) -print_sys(res.to_string(index=False)) -print_sys("=" * 50) - -np.save(os.path.join(args.output_dir, "sclong_module_batch_cell_emb_mode_ce.npy"), cell_embeddings) -print_sys( - f"Cell embeddings saved to " - f"{os.path.join(args.output_dir, 'sclong_module_batch_cell_emb_mode_ce.npy')}" -) -print_sys("Zero-shot evaluation completed!") +""" +Adaptation code of the PerformerEncoderOnly_Module model on the zero-shot batch correction task. + +The current script mainly supports: +1. Use the PerformerEncoderOnly_Module model (encoder-only) to extract cell embeddings. +2. Use mean pooling of module encodings as cell embedding. +3. Supports evaluation based on scIB indicators consistent with the scGPT tutorial. +4. Supports Lung data set preset and retains all genes by default without going through the HVG 3000 process. +""" + +import argparse +import importlib.util +import os +import pickle +import sys +import tempfile +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +ASSET_DIR = REPO_ROOT / "model_dependencies" +DOWNSTREAM_DATA_DIR = REPO_ROOT / "data" / "downstream" / "final_data" +WEIGHTS_DIR = REPO_ROOT / "weight" / "0810model" +from copy import deepcopy +from typing import Dict, List, Optional, Sequence, Set, Union + +os.environ.setdefault("NUMBA_DISABLE_JIT", "1") +os.environ.setdefault("NUMBA_CACHE_DIR", tempfile.mkdtemp(prefix="numba_cache_")) + +import numpy as np +import pandas as pd +import scanpy as sc +import scib +import torch +from scanpy import AnnData +from tqdm import tqdm + +# Model imports below are anchored to this file, independent of the working directory. + + +DATASET_PRESETS = { + "auto": {}, + "lung": { + "gene_col": "gene_name", + "batch_key": "sample", + "label_keys": ["cell_type"], + }, +} + + +def print_sys(message: str) -> None: + print(message, flush=True, file=sys.stderr) + + +def strip_gene_version(gene_id: str) -> str: + return gene_id.split(".", 1)[0] + + +def canonicalize_target_gene_id( + raw_gene: object, + symbol_to_ensembl: Dict[str, str], + target_gene_set: Set[str], +) -> Optional[str]: + if pd.isna(raw_gene): + return None + + raw_gene_str = str(raw_gene).strip() + if not raw_gene_str: + return None + + direct_candidates = [raw_gene_str, strip_gene_version(raw_gene_str)] + for candidate in direct_candidates: + if candidate in target_gene_set: + return candidate + + mapped_gene = symbol_to_ensembl.get(raw_gene_str) + if mapped_gene is None: + mapped_gene = symbol_to_ensembl.get(strip_gene_version(raw_gene_str)) + + if mapped_gene is None or pd.isna(mapped_gene): + return None + + mapped_gene_str = strip_gene_version(str(mapped_gene).strip()) + return mapped_gene_str or None + + +def resolve_input_gene_ids( + adata: AnnData, + gene_col: Optional[str], + symbol_to_ensembl: Dict[str, str], + target_gene_set: Set[str], +) -> List[str]: + if gene_col is None: + raw_genes = [str(gene) for gene in adata.var.index.tolist()] + source_name = "adata.var.index" + else: + if gene_col not in adata.var.columns: + raise KeyError(f"gene_col '{gene_col}' not found in adata.var.columns") + raw_genes = [str(gene) if not pd.isna(gene) else "" for gene in adata.var[gene_col].tolist()] + source_name = f"adata.var['{gene_col}']" + + resolved_gene_ids: List[str] = [] + direct_hits = 0 + mapped_hits = 0 + unresolved = 0 + + for index, raw_gene in enumerate(raw_genes): + resolved_gene = canonicalize_target_gene_id(raw_gene, symbol_to_ensembl, target_gene_set) + if resolved_gene is None: + resolved_gene = f"unknown{index}" + unresolved += 1 + elif strip_gene_version(raw_gene) == resolved_gene: + direct_hits += 1 + else: + mapped_hits += 1 + resolved_gene_ids.append(resolved_gene) + + print_sys(f"Using genes from {source_name}") + print_sys( + "Gene ID resolution summary: " + f"direct_hits={direct_hits}, mapped_hits={mapped_hits}, unresolved={unresolved}" + ) + return resolved_gene_ids + + +def clear_neighbors(adata: AnnData) -> None: + neighbors = adata.uns.pop("neighbors", None) + if neighbors is not None: + dist_key = neighbors.get("distances_key", "distances") + conn_key = neighbors.get("connectivities_key", "connectivities") + adata.obsp.pop(dist_key, None) + adata.obsp.pop(conn_key, None) + adata.obsp.pop("distances", None) + adata.obsp.pop("connectivities", None) + + +def prepare_eval_adata( + adata: AnnData, + batch_key: str, + label_key: str, + embedding_key: str, + build_neighbors: bool = False, +) -> AnnData: + if embedding_key not in adata.obsm: + raise ValueError(f"Embeddings '{embedding_key}' not found in adata.obsm") + + if batch_key not in adata.obs.columns: + raise ValueError(f"batch_key '{batch_key}' not found in adata.obs") + + if label_key not in adata.obs.columns: + raise ValueError(f"label_key '{label_key}' not found in adata.obs") + + valid_mask = adata.obs[batch_key].notna() & adata.obs[label_key].notna() + dropped = int((~valid_mask).sum()) + if dropped > 0: + print_sys( + f"Dropping {dropped} cells with missing batch/label values " + f"for evaluation (batch_key={batch_key}, label_key={label_key})." + ) + + if int(valid_mask.sum()) == 0: + raise ValueError( + f"No labeled cells remain after filtering batch_key={batch_key}, label_key={label_key}" + ) + + eval_adata = adata[valid_mask].copy() + eval_adata.obs[batch_key] = eval_adata.obs[batch_key].astype("category") + eval_adata.obs[label_key] = eval_adata.obs[label_key].astype("category") + + if build_neighbors: + clear_neighbors(eval_adata) + sc.pp.neighbors(eval_adata, use_rep=embedding_key) + return eval_adata + + +def eval_scib_metrics_simple( + adata: AnnData, + batch_key: str, + label_key: str, + embedding_key: str, +) -> Dict[str, float]: + eval_adata = prepare_eval_adata( + adata, + batch_key, + label_key, + embedding_key, + build_neighbors=False, + ) + + results_dict: Dict[str, float] = {} + if len(eval_adata.obs[batch_key].unique()) > 1: + results_dict["ASW_label/batch"] = scib.metrics.silhouette_batch( + eval_adata, + batch_key, + label_key, + embed=embedding_key, + metric="euclidean", + return_all=False, + verbose=False, + ) + + results_dict = {key: value for key, value in results_dict.items() if not np.isnan(value)} + return results_dict + + +def eval_scib_metrics_scgpt( + adata: AnnData, + batch_key: str, + label_key: str, + embedding_key: str, +) -> Dict[str, float]: + eval_adata = prepare_eval_adata( + adata, + batch_key, + label_key, + embedding_key, + build_neighbors=True, + ) + + results = scib.metrics.metrics( + eval_adata, + adata_int=eval_adata, + batch_key=batch_key, + label_key=label_key, + embed=embedding_key, + isolated_labels_asw_=False, + silhouette_=True, + hvg_score_=False, + graph_conn_=True, + pcr_=True, + isolated_labels_f1_=False, + trajectory_=False, + nmi_=True, + ari_=True, + cell_cycle_=False, + kBET_=False, + ilisi_=False, + clisi_=False, + ) + result_dict = results[0].to_dict() + + result_dict["avg_bio"] = np.mean( + [ + result_dict["NMI_cluster/label"], + result_dict["ARI_cluster/label"], + result_dict["ASW_label"], + ] + ) + result_dict["avg_batch"] = np.mean( + [ + result_dict["graph_conn"], + result_dict["ASW_label/batch"], + ] + ) + + result_dict = {key: value for key, value in result_dict.items() if not np.isnan(value)} + return result_dict + + +def evaluate( + adata: AnnData, + batch_key: str, + label_keys: Sequence[str], + embedding_key: str, + metrics_mode: str, + res_path: str, +) -> pd.DataFrame: + met_df = pd.DataFrame(columns=["metric", "label", "value"]) + + unique_label_keys: List[str] = [] + for key in label_keys: + if key not in unique_label_keys: + unique_label_keys.append(key) + + if not unique_label_keys: + raise ValueError("No label keys were provided for evaluation") + + for label_key in unique_label_keys: + if metrics_mode == "scgpt": + metrics = eval_scib_metrics_scgpt( + adata=adata, + batch_key=batch_key, + label_key=label_key, + embedding_key=embedding_key, + ) + elif metrics_mode == "simple": + metrics = eval_scib_metrics_simple( + adata=adata, + batch_key=batch_key, + label_key=label_key, + embedding_key=embedding_key, + ) + else: + raise ValueError(f"Unknown metrics_mode: {metrics_mode}") + + print_sys( + "\n".join( + [f"[{label_key}] {metric_name}: {metric_value:.4f}" for metric_name, metric_value in metrics.items()] + ) + ) + for metric_name, metric_value in metrics.items(): + met_df.loc[len(met_df)] = [metric_name, label_key, metric_value] + + met_df.to_csv(res_path, index=False) + np.save(f"{res_path[:-4]}_ce.npy", adata.obsm[embedding_key]) + return met_df + + +def resolve_with_preset( + cli_value: Optional[object], + preset_value: Optional[object], +) -> Optional[object]: + return cli_value if cli_value is not None else preset_value + + +def resolve_batch_key(adata: AnnData, requested_batch_key: Optional[str]) -> str: + if requested_batch_key is not None: + if requested_batch_key not in adata.obs.columns: + raise KeyError(f"batch_key '{requested_batch_key}' not found in adata.obs.columns") + return requested_batch_key + + fallback_keys = ["batch", "str_batch", "sample", "Sample Characteristic[individual]"] + for key in fallback_keys: + if key in adata.obs.columns: + print_sys(f"Auto-selected batch_key='{key}'") + return key + + raise ValueError( + "Unable to infer batch_key. Please pass --batch_key explicitly." + ) + + +def resolve_label_keys(adata: AnnData, requested_label_keys: Optional[Sequence[str]]) -> List[str]: + if requested_label_keys: + missing_keys = [key for key in requested_label_keys if key not in adata.obs.columns] + if missing_keys: + raise KeyError(f"label_key(s) not found in adata.obs.columns: {missing_keys}") + return list(requested_label_keys) + + fallback_keys = ["celltype", "cell_type", "celltype_original"] + resolved_keys = [key for key in fallback_keys if key in adata.obs.columns] + if resolved_keys: + print_sys(f"Auto-selected label_keys={resolved_keys}") + return resolved_keys + + raise ValueError( + "Unable to infer label_key. Please pass --label_key explicitly." + ) + + +def reindex_tensor_universal( + tensor: torch.Tensor, + index_positions: Sequence[int], + dim: int, + filler: str = "0", + device: Union[str, torch.device] = "cpu", +) -> torch.Tensor: + index_tensor = torch.tensor(index_positions, device=device) + tensor_shape = list(tensor.shape) + tensor_shape[dim] = len(index_positions) + index_shape = [1] * len(tensor_shape) + index_shape[dim] = len(index_positions) + index_tensor = index_tensor.view(*index_shape).expand(*tensor_shape) + + if filler == "0": + padder_shape = deepcopy(tensor_shape) + padder_shape[dim] = 1 + padder = torch.zeros(padder_shape, dtype=tensor.dtype, device=device) + elif filler == "mean": + padder = tensor.mean(dim=dim, keepdim=True) + else: + raise ValueError("filler should be 0 or mean") + expanded_tensor = torch.cat((tensor, padder), dim=dim) + return torch.gather(expanded_tensor, dim, index_tensor) + + +project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, project_root) +sys.path.insert(0, os.path.dirname(project_root)) + +performer_module_dir = project_root + +def load_module_modules(): + models_path = os.path.join(performer_module_dir, "models_performer_module.py") + if not os.path.exists(models_path): + raise FileNotFoundError(f"Model file not found: {models_path}") + models_spec = importlib.util.spec_from_file_location("models_performer_module", models_path) + models_performer_module = importlib.util.module_from_spec(models_spec) + models_spec.loader.exec_module(models_performer_module) + + dataset_path = os.path.join(performer_module_dir, "module_dataset.py") + if not os.path.exists(dataset_path): + raise FileNotFoundError(f"Dataset file not found: {dataset_path}") + dataset_spec = importlib.util.spec_from_file_location("module_dataset", dataset_path) + module_dataset = importlib.util.module_from_spec(dataset_spec) + dataset_spec.loader.exec_module(module_dataset) + + return ( + models_performer_module.PerformerEncoderOnly_Module, + models_performer_module.sanitize_performer_module_config, + module_dataset.load_module_data, + ) + +OUTPUT_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "res") +os.makedirs(OUTPUT_DIR, exist_ok=True) + +parser = argparse.ArgumentParser() +parser.add_argument("--gene2vec_path", type=str, default=str(ASSET_DIR / "selected_gene2vec_27k.npy"), + help="Path to gene2vec file") +parser.add_argument("--scfm_hyper_params_path", type=str, required=True, + help="Path to the matching BioMod model configuration pkl file") +parser.add_argument("--scfm_ckpt_path", type=str, required=True, + help="Path to the BioMod model weights") +parser.add_argument("--module_data_path", type=str, default=None, + help="Path to module JSON file (single file)") +parser.add_argument("--module_data_dir", type=str, default=None, + help="Path to directory containing module JSON files") +parser.add_argument("--gene_mapping_path", type=str, default=str(ASSET_DIR / "gene2ens.csv"), + help="Path to gene symbol to Ensembl ID mapping CSV") +parser.add_argument("--scfm_genes_file", type=str, default=str(ASSET_DIR / "selected_genes_27k.txt"), + help="Path to scfm genes list file") +parser.add_argument("--data_path", type=str, required=True, + help="Path to input h5ad file") +parser.add_argument("--dataset_preset", type=str, default="auto", choices=sorted(DATASET_PRESETS.keys()), + help="Dataset preset for batch/label/gene column defaults") +parser.add_argument("--gene_col", type=str, default=None, + help="Column in adata.var used as the input gene symbol column. " + "If not set, use adata.var.index.") +parser.add_argument("--batch_key", type=str, default=None, + help="Column in adata.obs used as the batch label") +parser.add_argument("--label_key", nargs="+", default=None, + help="One or more columns in adata.obs used as evaluation labels") +parser.add_argument("--output_key", type=str, default="X_sclong_module", + help="Key name for storing embeddings in adata.obsm") +parser.add_argument("--pool_method", type=str, default="auto", choices=["auto", "mean", "cls", "max", "first_module"], + help="Cell embedding pooling. auto prefers cls_encoding when the checkpoint has a trained CLS token; " + "first_module uses the first module after restoring original module order.") +parser.add_argument("--n_cells", type=str, default="all", + help="Number of cells to process (default: all)") +parser.add_argument("--batch_size", type=int, default=4, + help="Batch size for inference") +parser.add_argument("--output_dir", type=str, default=OUTPUT_DIR, + help="Output directory for results") +parser.add_argument("--metrics_mode", type=str, default="scgpt", choices=["scgpt", "simple"], + help="Evaluation metrics mode. Default uses the scGPT tutorial metric bundle.") +parser.add_argument("--min_genes_per_cell", type=int, default=0, + help="Optional Scanpy cell filter. Default 0 keeps all cells.") +parser.add_argument("--min_cells_per_gene", type=int, default=0, + help="Optional Scanpy gene filter. Default 0 keeps all genes.") +parser.add_argument("--shuffle_module_genes", action="store_true", + help="Optional downstream-only extra reshuffle on top of module_sets restored from PKL. " + "Do not pass this when reproducing the pretrained backbone as-is.") +parser.add_argument("--shuffle_seed", type=int, default=2021, + help="Random seed for the downstream extra reshuffle when --shuffle_module_genes is enabled.") +parser.add_argument("--save_raw_module_emb", action="store_true", + help="Save raw module_encodings with shape (num_cells, num_modules, embed_dim)") + +args = parser.parse_args() +os.makedirs(args.output_dir, exist_ok=True) + +device = torch.device("cuda" if torch.cuda.is_available() else "cpu") +print_sys(f"Using device: {device}") + +preset_config = DATASET_PRESETS[args.dataset_preset] +resolved_gene_col = resolve_with_preset(args.gene_col, preset_config.get("gene_col")) +resolved_batch_key = resolve_with_preset(args.batch_key, preset_config.get("batch_key")) +resolved_label_keys = resolve_with_preset(args.label_key, preset_config.get("label_keys")) + +print_sys( + f"dataset_preset={args.dataset_preset}, " + f"gene_col={resolved_gene_col}, batch_key={resolved_batch_key}, " + f"label_keys={resolved_label_keys}, metrics_mode={args.metrics_mode}, " + f"pool_method={args.pool_method}" +) + +PerformerEncoderOnly_Module, sanitize_performer_module_config, load_module_data = load_module_modules() + +print_sys(f"Loading hyper parameters from {args.scfm_hyper_params_path}") +with open(args.scfm_hyper_params_path, "rb") as handle: + scfm_hyper_params = pickle.load(handle) +scfm_hyper_params.pop("module_embed_chunk_padded_gene_budget", None) +scfm_hyper_params = sanitize_performer_module_config( + scfm_hyper_params, + source=args.scfm_hyper_params_path, + log_fn=print_sys, +) + +scfm_hyper_params["gene2vec_file"] = args.gene2vec_path +# Read fusion_method from pkl to ensure consistency with pre-training +fusion_method = scfm_hyper_params.get("fusion_method", "multiply") +print_sys(f"Using fusion_method from pkl: {fusion_method}") + +if "module_sets" not in scfm_hyper_params or scfm_hyper_params.get("module_sets") is None: + if args.module_data_path or args.module_data_dir: + print_sys("Loading module data...") + module_sets = load_module_data( + module_data_path=args.module_data_path, + module_data_dir=args.module_data_dir, + gene_list_path=args.scfm_genes_file, + gene_mapping_path=args.gene_mapping_path, + verbose=True, + ) + scfm_hyper_params["module_sets"] = module_sets + else: + raise ValueError("Model requires module_sets, but neither --module_data_path nor --module_data_dir was provided") +else: + module_sets = scfm_hyper_params["module_sets"] + print_sys(f"Using module_sets from hyper parameters ({len(module_sets)} modules)") + +if args.shuffle_module_genes: + import random + + print_sys( + f"Applying downstream-only extra module reshuffle (seed={args.shuffle_seed}). " + f"Pretraining-time module_sets from PKL will be reshuffled again." + ) + all_gene_indices: List[int] = [] + module_sizes: List[int] = [] + for module in module_sets: + all_gene_indices.extend(module) + module_sizes.append(len(module)) + + rng = random.Random(args.shuffle_seed) + rng.shuffle(all_gene_indices) + + shuffled_module_sets = [] + start_idx = 0 + for size in module_sizes: + shuffled_module_sets.append(all_gene_indices[start_idx:start_idx + size]) + start_idx += size + module_sets = shuffled_module_sets + scfm_hyper_params["module_sets"] = module_sets + print_sys(f"Shuffled {len(all_gene_indices)} genes across {len(module_sets)} modules") + +print_sys(f"Model hyper parameters: {list(scfm_hyper_params.keys())}") + +pkl_use_gene2vec = scfm_hyper_params.get("use_gene2vec", True) +scfm_hyper_params["use_gene2vec"] = pkl_use_gene2vec +print_sys(f"use_gene2vec={pkl_use_gene2vec} (from PKL)") + +print_sys("Creating PerformerEncoderOnly_Module model...") +model = PerformerEncoderOnly_Module(**scfm_hyper_params).to(device) + +print_sys(f"Loading checkpoint from {args.scfm_ckpt_path}") +ckpt = torch.load(args.scfm_ckpt_path, map_location=device) +model_state_dict = ckpt.get("model_state_dict", ckpt) +filtered_state_dict = { + key: value + for key, value in model_state_dict.items() + if "go_conv" not in key and "G_go" not in key.lower() +} +model.load_state_dict(filtered_state_dict, strict=True) + +for parameter in model.parameters(): + parameter.requires_grad = False +model.eval() + +all_params = sum(parameter.numel() for parameter in model.parameters()) +trainable_params = sum(parameter.numel() for parameter in model.parameters() if parameter.requires_grad) +print_sys(f"Total model parameters: {all_params:,}") +print_sys(f"Trainable parameters during zero-shot: {trainable_params:,}") + +gene2ens = pd.read_csv(args.gene_mapping_path, index_col=0) +symbol_to_ensembl = gene2ens["Ensembl gene ID"].to_dict() + +with open(args.scfm_genes_file, "r") as handle: + scfm_genes = [line.rstrip("\n") for line in handle.readlines()] +scfm_gene_set = set(scfm_genes) + +print_sys(f"Loading data from {args.data_path}") +adata = sc.read_h5ad(args.data_path) + +if args.min_genes_per_cell > 0: + before_cells = adata.n_obs + sc.pp.filter_cells(adata, min_genes=args.min_genes_per_cell) + print_sys(f"Filtered cells by min_genes={args.min_genes_per_cell}: {before_cells} -> {adata.n_obs}") + +if args.min_cells_per_gene > 0: + before_genes = adata.n_vars + sc.pp.filter_genes(adata, min_cells=args.min_cells_per_gene) + print_sys(f"Filtered genes by min_cells={args.min_cells_per_gene}: {before_genes} -> {adata.n_vars}") + +if args.n_cells == "all": + n_cells = adata.n_obs +else: + n_cells = int(args.n_cells) + if adata.n_obs > n_cells: + print_sys(f"Subsampling to {n_cells} cells") + sc.pp.subsample(adata, n_obs=n_cells, copy=False) + n_cells = adata.n_obs + +batch_key = resolve_batch_key(adata, resolved_batch_key) +label_keys = resolve_label_keys(adata, resolved_label_keys) + +print_sys(f"Processing {n_cells} cells with {adata.n_vars} genes") +print_sys(f"Evaluation batch_key={batch_key}, label_keys={label_keys}") + +input_gene_ids = resolve_input_gene_ids( + adata=adata, + gene_col=resolved_gene_col, + symbol_to_ensembl=symbol_to_ensembl, + target_gene_set=scfm_gene_set, +) + +input_mapping = {gene_id: index for index, gene_id in enumerate(input_gene_ids)} +input_gene_num = len(input_gene_ids) + +intersect_genes = np.intersect1d(input_gene_ids, scfm_genes) +print_sys(f"Input genes and scfm genes intersect: {len(intersect_genes)} genes") + +scfm_genes_pad = scfm_genes + ["PAD"] +scfm_seq_len = len(scfm_genes_pad) +scfm_mapping = {gene_id: index for index, gene_id in enumerate(scfm_genes_pad)} + +scfm_index_positions = [input_mapping.get(gene_id, input_gene_num) for gene_id in scfm_genes_pad] +_ = [scfm_mapping.get(gene_id, scfm_seq_len) for gene_id in input_gene_ids] + +print_sys("Extracting cell embeddings...") +cell_embeddings = [] +raw_module_embeddings = [] if args.save_raw_module_emb else None +embedding_key = args.output_key + +with torch.no_grad(): + for start in tqdm(range(0, n_cells, args.batch_size)): + batch_end = min(start + args.batch_size, n_cells) + batch_matrix = adata.X[start:batch_end, :] + if hasattr(batch_matrix, "toarray"): + batch_matrix = batch_matrix.toarray() + x = torch.tensor(batch_matrix, dtype=torch.float32).to(device) + + x_scfm = reindex_tensor_universal(x, scfm_index_positions, dim=1, filler="0", device=device) + out = model(x_scfm, mask_ratio=0.0, return_encodings=True) + + if "module_encodings" not in out: + raise ValueError(f"Expected 'module_encodings' in output, got: {list(out.keys())}") + + module_encodings = out["module_encodings"] + ids_restore = out.get("ids_restore") + if args.pool_method == "cls": + if "cls_encoding" not in out: + print_sys("Warning: pool_method='cls' but checkpoint has no cls_encoding, using mean pooling instead") + cell_emb = module_encodings.mean(dim=1) + else: + cell_emb = out["cls_encoding"] + elif args.pool_method == "mean": + cell_emb = module_encodings.mean(dim=1) + elif args.pool_method == "max": + cell_emb = module_encodings.max(dim=1)[0] + elif args.pool_method == "first_module": + if ids_restore is None: + raise ValueError("pool_method='first_module' requires ids_restore in model outputs") + restore_index = ids_restore.unsqueeze(-1).expand(-1, -1, module_encodings.shape[-1]) + restored_module_encodings = torch.gather(module_encodings, dim=1, index=restore_index) + cell_emb = restored_module_encodings[:, 0, :] + else: + cell_emb = out["cls_encoding"] if "cls_encoding" in out else module_encodings.mean(dim=1) + + if args.save_raw_module_emb: + raw_module_embeddings.append(module_encodings.cpu().numpy()) + + cell_embeddings.append(cell_emb.cpu().numpy()) + +cell_embeddings = np.concatenate(cell_embeddings, axis=0) +print_sys(f"Cell embeddings shape: {cell_embeddings.shape}") + +if args.save_raw_module_emb and raw_module_embeddings is not None: + raw_module_embeddings = np.concatenate(raw_module_embeddings, axis=0) + print_sys(f"Raw module embeddings shape: {raw_module_embeddings.shape}") + raw_module_path = os.path.join(args.output_dir, "sclong_module_raw_module_emb.npy") + np.save(raw_module_path, raw_module_embeddings) + print_sys(f"Raw module embeddings saved to {raw_module_path}") + +adata.obsm[embedding_key] = cell_embeddings + +print_sys("Evaluating batch correction...") +res = evaluate( + adata=adata, + batch_key=batch_key, + label_keys=label_keys, + embedding_key=embedding_key, + metrics_mode=args.metrics_mode, + res_path=os.path.join(args.output_dir, "sclong_module_batch_cell_emb_mode.csv"), +) + +print_sys("\nEvaluation Results:") +print_sys("=" * 50) +print_sys(res.to_string(index=False)) +print_sys("=" * 50) + +np.save(os.path.join(args.output_dir, "sclong_module_batch_cell_emb_mode_ce.npy"), cell_embeddings) +print_sys( + f"Cell embeddings saved to " + f"{os.path.join(args.output_dir, 'sclong_module_batch_cell_emb_mode_ce.npy')}" +) +print_sys("Zero-shot evaluation completed!") diff --git a/BioMod/zero_shot_batch/sequencing_depth/cmd.txt b/BioMod/zero_shot_batch/sequencing_depth/cmd.txt index 4eddf12c03a5bd52e374c7725e8517a4fef804e9..6ada3ddefa64d53fceafa27a068e55bd1fc11d4b 100644 --- a/BioMod/zero_shot_batch/sequencing_depth/cmd.txt +++ b/BioMod/zero_shot_batch/sequencing_depth/cmd.txt @@ -1,41 +1,42 @@ -python make_low_depth_h5ad.py \ ---input_path ../Kim2020_Lung.h5ad \ ---output_dir ./data/Kim2020_Lung \ ---keep_ratios 0.5,0.25,0.1,0.05 \ ---seeds 0,1,2 - -python run_zero_shot_low_depth.py \ ---data_paths \ -../Kim2020_Lung.h5ad \ -./data/Kim2020_Lung/*.h5ad \ ---scfm_ckpt_path ../../init_models/performer_module_multiply_all_module_init.pth \ ---scfm_hyper_params_path ../../init_models/performer_module_multiply_all_module_init.pkl \ ---output_root ./results \ ---gene_mapping_path ../../gene2ens.csv \ ---scfm_genes_file ../../../selected_genes_27k.txt \ ---module_data_dir ../../../module_data_all \ ---dataset_preset lung \ ---output_key X_sclong_module \ ---batch_size 256 \ ---metrics_mode scgpt \ ---pool_method max - - -python run_zero_shot_low_depth.py \ ---data_paths \ -../Kim2020_Lung.h5ad \ -./data/Kim2020_Lung/Kim2020_Lung_ld0p5_seed0.h5ad \ -./data/Kim2020_Lung/Kim2020_Lung_ld0p25_seed0.h5ad \ -./data/Kim2020_Lung/Kim2020_Lung_ld0p1_seed0.h5ad \ -./data/Kim2020_Lung/Kim2020_Lung_ld0p05_seed0.h5ad \ ---scfm_ckpt_path ../../init_models/performer_module_multiply_all_module_init.pth \ ---scfm_hyper_params_path ../../init_models/performer_module_multiply_all_module_init.pkl \ ---output_root ./results \ ---gene_mapping_path ../../../gene2ens.csv \ ---scfm_genes_file ../../../selected_genes_27k.txt \ ---module_data_dir ../../module_data_all \ ---dataset_preset lung \ ---output_key X_sclong_module \ ---batch_size 256 \ ---metrics_mode scgpt \ +# Run from the repository root. The selected input/layer must contain raw integer counts. +python BioMod/zero_shot_batch/sequencing_depth/make_low_depth_h5ad.py \ +--input_path data/downstream/final_data/Kim2020_Lung.h5ad \ +--output_dir tutorial_runs/sequencing_depth/inputs \ +--keep_ratios 0.5,0.25,0.1,0.05 \ +--seeds 0,1,2 + +python BioMod/zero_shot_batch/sequencing_depth/run_zero_shot_low_depth.py \ +--data_paths \ +data/downstream/final_data/Kim2020_Lung.h5ad \ +tutorial_runs/sequencing_depth/inputs/*.h5ad \ +--scfm_ckpt_path weight/0810model/mae_tost_70M_75mask_ec2_epoch0.pth \ +--scfm_hyper_params_path weight/0810model/mae_tost_70M_75mask_ec2.pkl \ +--output_root tutorial_runs/sequencing_depth/results \ +--gene_mapping_path model_dependencies/gene2ens.csv \ +--scfm_genes_file model_dependencies/selected_genes_27k.txt \ +--module_data_dir model_dependencies/module_data_all \ +--dataset_preset lung \ +--output_key X_sclong_module \ +--batch_size 256 \ +--metrics_mode scgpt \ +--pool_method max + + +python BioMod/zero_shot_batch/sequencing_depth/run_zero_shot_low_depth.py \ +--data_paths \ +data/downstream/final_data/Kim2020_Lung.h5ad \ +tutorial_runs/sequencing_depth/inputs/Kim2020_Lung_ld0p5_seed0.h5ad \ +tutorial_runs/sequencing_depth/inputs/Kim2020_Lung_ld0p25_seed0.h5ad \ +tutorial_runs/sequencing_depth/inputs/Kim2020_Lung_ld0p1_seed0.h5ad \ +tutorial_runs/sequencing_depth/inputs/Kim2020_Lung_ld0p05_seed0.h5ad \ +--scfm_ckpt_path weight/0810model/mae_tost_70M_75mask_ec2_epoch0.pth \ +--scfm_hyper_params_path weight/0810model/mae_tost_70M_75mask_ec2.pkl \ +--output_root tutorial_runs/sequencing_depth/results \ +--gene_mapping_path model_dependencies/gene2ens.csv \ +--scfm_genes_file model_dependencies/selected_genes_27k.txt \ +--module_data_dir model_dependencies/module_data_all \ +--dataset_preset lung \ +--output_key X_sclong_module \ +--batch_size 256 \ +--metrics_mode scgpt \ --pool_method max \ No newline at end of file diff --git a/BioMod/zero_shot_batch/sequencing_depth/run_zero_shot_low_depth.py b/BioMod/zero_shot_batch/sequencing_depth/run_zero_shot_low_depth.py index 9edfc3f96be518a6e5b41112ca9695043cfeda90..0ce82f0343fb30030bf11bcc1c284ab0b5488bfd 100644 --- a/BioMod/zero_shot_batch/sequencing_depth/run_zero_shot_low_depth.py +++ b/BioMod/zero_shot_batch/sequencing_depth/run_zero_shot_low_depth.py @@ -1,196 +1,196 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -"""Batch-run performer_module zero-shot on full/low-depth h5ad files and aggregate metrics.""" - -import argparse -import re -import subprocess -import sys -from pathlib import Path -from typing import Iterable, List, Optional - -import pandas as pd - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser( - description="Run performer_module zero-shot on multiple full/low-depth h5ad files and aggregate metrics." - ) - parser.add_argument("--script_path", type=str, default=None, help="Path to performer_module_zero_shot.py") - parser.add_argument("--data_paths", nargs="+", required=True, help="One or more full/low-depth h5ad files") - parser.add_argument("--output_root", type=str, required=True, help="Root directory for per-run outputs") - parser.add_argument("--scfm_hyper_params_path", type=str, required=True) - parser.add_argument("--scfm_ckpt_path", type=str, required=True) - parser.add_argument("--module_data_path", type=str, default=None) - parser.add_argument("--module_data_dir", type=str, default=None) - parser.add_argument("--gene_mapping_path", type=str, required=True) - parser.add_argument("--scfm_genes_file", type=str, required=True) - parser.add_argument("--gene2vec_path", type=str, default="selected_gene2vec_27k.npy") - parser.add_argument("--dataset_preset", type=str, default="lung") - parser.add_argument("--gene_col", type=str, default=None) - parser.add_argument("--batch_key", type=str, default=None) - parser.add_argument("--label_key", nargs="+", default=None) - parser.add_argument("--output_key", type=str, default="X_sclong_module") - parser.add_argument("--pool_method", type=str, default="auto") - parser.add_argument("--n_cells", type=str, default="all") - parser.add_argument("--batch_size", type=int, default=4) - parser.add_argument("--metrics_mode", type=str, default="scgpt") - parser.add_argument("--min_genes_per_cell", type=int, default=0) - parser.add_argument("--min_cells_per_gene", type=int, default=0) - parser.add_argument("--shuffle_module_genes", action="store_true") - parser.add_argument("--shuffle_seed", type=int, default=2021) - parser.add_argument("--save_raw_module_emb", action="store_true") - parser.add_argument("--python_bin", type=str, default=sys.executable) - parser.add_argument("--overwrite", action="store_true", help="Re-run even if result CSV already exists") - return parser.parse_args() - - -def resolve_script_path(script_path: Optional[str]) -> Path: - if script_path is not None: - return Path(script_path).resolve() - return (Path(__file__).resolve().parent.parent / "performer_module_zero_shot.py").resolve() - - -def slugify_path(path: Path) -> str: - return re.sub(r"[^A-Za-z0-9._-]+", "_", path.stem) - - -def build_command(args: argparse.Namespace, script_path: Path, data_path: Path, output_dir: Path) -> List[str]: - cmd = [ - args.python_bin, - str(script_path), - "--data_path", - str(data_path), - "--output_dir", - str(output_dir), - "--scfm_hyper_params_path", - args.scfm_hyper_params_path, - "--scfm_ckpt_path", - args.scfm_ckpt_path, - "--gene_mapping_path", - args.gene_mapping_path, - "--scfm_genes_file", - args.scfm_genes_file, - "--gene2vec_path", - args.gene2vec_path, - "--dataset_preset", - args.dataset_preset, - "--output_key", - args.output_key, - "--pool_method", - args.pool_method, - "--n_cells", - args.n_cells, - "--batch_size", - str(args.batch_size), - "--metrics_mode", - args.metrics_mode, - "--min_genes_per_cell", - str(args.min_genes_per_cell), - "--min_cells_per_gene", - str(args.min_cells_per_gene), - ] - - if args.module_data_path: - cmd.extend(["--module_data_path", args.module_data_path]) - if args.module_data_dir: - cmd.extend(["--module_data_dir", args.module_data_dir]) - if args.gene_col: - cmd.extend(["--gene_col", args.gene_col]) - if args.batch_key: - cmd.extend(["--batch_key", args.batch_key]) - if args.label_key: - cmd.extend(["--label_key", *args.label_key]) - if args.shuffle_module_genes: - cmd.append("--shuffle_module_genes") - cmd.extend(["--shuffle_seed", str(args.shuffle_seed)]) - if args.save_raw_module_emb: - cmd.append("--save_raw_module_emb") - - return cmd - - -def parse_low_depth_tags(path: Path) -> dict: - stem = path.stem - keep_ratio = None - seed = None - match = re.search(r"_ld([0-9]+p[0-9]+|[0-9]+)_seed([0-9]+)", stem) - if match: - keep_ratio = float(match.group(1).replace("p", ".")) - seed = int(match.group(2)) - return { - "input_file": path.name, - "input_stem": stem, - "keep_ratio": keep_ratio, - "seed": seed, - "is_low_depth": keep_ratio is not None, - } - - -def load_metrics(csv_path: Path) -> pd.DataFrame: - df = pd.read_csv(csv_path) - required_columns = {"metric", "label", "value"} - missing = required_columns - set(df.columns) - if missing: - raise ValueError(f"Metrics CSV missing required columns {sorted(missing)}: {csv_path}") - return df - - -def build_summary(records: Iterable[pd.DataFrame]) -> pd.DataFrame: - frames = list(records) - if not frames: - return pd.DataFrame() - summary = pd.concat(frames, ignore_index=True) - - baseline = summary[summary["is_low_depth"] == False][["metric", "label", "value"]].copy() - baseline = baseline.rename(columns={"value": "baseline_value"}) - baseline = baseline.drop_duplicates(subset=["metric", "label"]) - summary = summary.merge(baseline, on=["metric", "label"], how="left") - summary["relative_to_full"] = summary["value"] / summary["baseline_value"] - return summary - - -def main() -> None: - args = parse_args() - script_path = resolve_script_path(args.script_path) - output_root = Path(args.output_root).resolve() - output_root.mkdir(parents=True, exist_ok=True) - - if not script_path.exists(): - raise FileNotFoundError(f"Zero-shot script not found: {script_path}") - - per_run_tables: List[pd.DataFrame] = [] - for raw_data_path in args.data_paths: - data_path = Path(raw_data_path).resolve() - if not data_path.exists(): - raise FileNotFoundError(f"Input h5ad not found: {data_path}") - - run_name = slugify_path(data_path) - run_dir = output_root / run_name - run_dir.mkdir(parents=True, exist_ok=True) - metrics_csv = run_dir / "sclong_module_batch_cell_emb_mode.csv" - - if args.overwrite or (not metrics_csv.exists()): - cmd = build_command(args, script_path, data_path, run_dir) - print(f"[runner] running {' '.join(cmd)}", flush=True) - subprocess.run(cmd, check=True) - else: - print(f"[runner] skip existing run: {run_dir}", flush=True) - - metrics_df = load_metrics(metrics_csv) - tag_info = parse_low_depth_tags(data_path) - metrics_df["run_name"] = run_name - metrics_df["data_path"] = str(data_path) - for key, value in tag_info.items(): - metrics_df[key] = value - per_run_tables.append(metrics_df) - - summary = build_summary(per_run_tables) - summary_csv = output_root / "zero_shot_low_depth_summary.csv" - summary.to_csv(summary_csv, index=False) - print(f"[runner] saved summary -> {summary_csv}", flush=True) - - -if __name__ == "__main__": - main() +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +"""Batch-run performer_module zero-shot on full/low-depth h5ad files and aggregate metrics.""" + +import argparse +import re +import subprocess +import sys +from pathlib import Path +from typing import Iterable, List, Optional + +import pandas as pd + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Run performer_module zero-shot on multiple full/low-depth h5ad files and aggregate metrics." + ) + parser.add_argument("--script_path", type=str, default=None, help="Path to performer_module_zero_shot.py") + parser.add_argument("--data_paths", nargs="+", required=True, help="One or more full/low-depth h5ad files") + parser.add_argument("--output_root", type=str, required=True, help="Root directory for per-run outputs") + parser.add_argument("--scfm_hyper_params_path", type=str, required=True) + parser.add_argument("--scfm_ckpt_path", type=str, required=True) + parser.add_argument("--module_data_path", type=str, default=None) + parser.add_argument("--module_data_dir", type=str, default=None) + parser.add_argument("--gene_mapping_path", type=str, required=True) + parser.add_argument("--scfm_genes_file", type=str, required=True) + parser.add_argument("--gene2vec_path", type=str, default=str(Path(__file__).resolve().parents[3] / "model_dependencies" / "selected_gene2vec_27k.npy")) + parser.add_argument("--dataset_preset", type=str, default="lung") + parser.add_argument("--gene_col", type=str, default=None) + parser.add_argument("--batch_key", type=str, default=None) + parser.add_argument("--label_key", nargs="+", default=None) + parser.add_argument("--output_key", type=str, default="X_sclong_module") + parser.add_argument("--pool_method", type=str, default="auto") + parser.add_argument("--n_cells", type=str, default="all") + parser.add_argument("--batch_size", type=int, default=4) + parser.add_argument("--metrics_mode", type=str, default="scgpt") + parser.add_argument("--min_genes_per_cell", type=int, default=0) + parser.add_argument("--min_cells_per_gene", type=int, default=0) + parser.add_argument("--shuffle_module_genes", action="store_true") + parser.add_argument("--shuffle_seed", type=int, default=2021) + parser.add_argument("--save_raw_module_emb", action="store_true") + parser.add_argument("--python_bin", type=str, default=sys.executable) + parser.add_argument("--overwrite", action="store_true", help="Re-run even if result CSV already exists") + return parser.parse_args() + + +def resolve_script_path(script_path: Optional[str]) -> Path: + if script_path is not None: + return Path(script_path).resolve() + return (Path(__file__).resolve().parent.parent / "performer_module_zero_shot.py").resolve() + + +def slugify_path(path: Path) -> str: + return re.sub(r"[^A-Za-z0-9._-]+", "_", path.stem) + + +def build_command(args: argparse.Namespace, script_path: Path, data_path: Path, output_dir: Path) -> List[str]: + cmd = [ + args.python_bin, + str(script_path), + "--data_path", + str(data_path), + "--output_dir", + str(output_dir), + "--scfm_hyper_params_path", + args.scfm_hyper_params_path, + "--scfm_ckpt_path", + args.scfm_ckpt_path, + "--gene_mapping_path", + args.gene_mapping_path, + "--scfm_genes_file", + args.scfm_genes_file, + "--gene2vec_path", + args.gene2vec_path, + "--dataset_preset", + args.dataset_preset, + "--output_key", + args.output_key, + "--pool_method", + args.pool_method, + "--n_cells", + args.n_cells, + "--batch_size", + str(args.batch_size), + "--metrics_mode", + args.metrics_mode, + "--min_genes_per_cell", + str(args.min_genes_per_cell), + "--min_cells_per_gene", + str(args.min_cells_per_gene), + ] + + if args.module_data_path: + cmd.extend(["--module_data_path", args.module_data_path]) + if args.module_data_dir: + cmd.extend(["--module_data_dir", args.module_data_dir]) + if args.gene_col: + cmd.extend(["--gene_col", args.gene_col]) + if args.batch_key: + cmd.extend(["--batch_key", args.batch_key]) + if args.label_key: + cmd.extend(["--label_key", *args.label_key]) + if args.shuffle_module_genes: + cmd.append("--shuffle_module_genes") + cmd.extend(["--shuffle_seed", str(args.shuffle_seed)]) + if args.save_raw_module_emb: + cmd.append("--save_raw_module_emb") + + return cmd + + +def parse_low_depth_tags(path: Path) -> dict: + stem = path.stem + keep_ratio = None + seed = None + match = re.search(r"_ld([0-9]+p[0-9]+|[0-9]+)_seed([0-9]+)", stem) + if match: + keep_ratio = float(match.group(1).replace("p", ".")) + seed = int(match.group(2)) + return { + "input_file": path.name, + "input_stem": stem, + "keep_ratio": keep_ratio, + "seed": seed, + "is_low_depth": keep_ratio is not None, + } + + +def load_metrics(csv_path: Path) -> pd.DataFrame: + df = pd.read_csv(csv_path) + required_columns = {"metric", "label", "value"} + missing = required_columns - set(df.columns) + if missing: + raise ValueError(f"Metrics CSV missing required columns {sorted(missing)}: {csv_path}") + return df + + +def build_summary(records: Iterable[pd.DataFrame]) -> pd.DataFrame: + frames = list(records) + if not frames: + return pd.DataFrame() + summary = pd.concat(frames, ignore_index=True) + + baseline = summary[summary["is_low_depth"] == False][["metric", "label", "value"]].copy() + baseline = baseline.rename(columns={"value": "baseline_value"}) + baseline = baseline.drop_duplicates(subset=["metric", "label"]) + summary = summary.merge(baseline, on=["metric", "label"], how="left") + summary["relative_to_full"] = summary["value"] / summary["baseline_value"] + return summary + + +def main() -> None: + args = parse_args() + script_path = resolve_script_path(args.script_path) + output_root = Path(args.output_root).resolve() + output_root.mkdir(parents=True, exist_ok=True) + + if not script_path.exists(): + raise FileNotFoundError(f"Zero-shot script not found: {script_path}") + + per_run_tables: List[pd.DataFrame] = [] + for raw_data_path in args.data_paths: + data_path = Path(raw_data_path).resolve() + if not data_path.exists(): + raise FileNotFoundError(f"Input h5ad not found: {data_path}") + + run_name = slugify_path(data_path) + run_dir = output_root / run_name + run_dir.mkdir(parents=True, exist_ok=True) + metrics_csv = run_dir / "sclong_module_batch_cell_emb_mode.csv" + + if args.overwrite or (not metrics_csv.exists()): + cmd = build_command(args, script_path, data_path, run_dir) + print(f"[runner] running {' '.join(cmd)}", flush=True) + subprocess.run(cmd, check=True) + else: + print(f"[runner] skip existing run: {run_dir}", flush=True) + + metrics_df = load_metrics(metrics_csv) + tag_info = parse_low_depth_tags(data_path) + metrics_df["run_name"] = run_name + metrics_df["data_path"] = str(data_path) + for key, value in tag_info.items(): + metrics_df[key] = value + per_run_tables.append(metrics_df) + + summary = build_summary(per_run_tables) + summary_csv = output_root / "zero_shot_low_depth_summary.csv" + summary.to_csv(summary_csv, index=False) + print(f"[runner] saved summary -> {summary_csv}", flush=True) + + +if __name__ == "__main__": + main() diff --git a/BioMod/zero_shot_batch/visualize_results.py b/BioMod/zero_shot_batch/visualize_results.py index 9d615412859193e1f79fcdf89f2817195b6a30cf..c80dea853a4b77499de1c05dcf1a1cce24e02760 100644 --- a/BioMod/zero_shot_batch/visualize_results.py +++ b/BioMod/zero_shot_batch/visualize_results.py @@ -1,294 +1,294 @@ -""" -Zero-shot batch correction visual code - -Used to generate the following images: -1. umap_cell_embeddings.png - UMAP visualization of cell embeddings (colored by cell type and batch) -2. umap_pca50.png - PCA dimensionality reduction to 50 dimensions and then UMAP visualization -3. batch_ASW_comparison.png - Batch ASW score comparison bar chart for different models/settings - -Enter data: -- sclong_module_batch_cell_emb_mode_ce.npy: cell embedding vector -- sclong_module_batch_cell_emb_mode.csv: ASW score -- pancreas_scib.h5ad: original data (including celltype and batch information) - -Calculation method: -1. umap_cell_embeddings: directly perform UMAP dimensionality reduction on cell embedding vectors -2. umap_pca50: First use PCA to reduce the dimension to 50 dimensions, and then perform UMAP dimension reduction. -3. batch_ASW: Use scib.metrics.silhouette_batch to calculate the batch ASW score -""" - -import os -import numpy as np -import pandas as pd -import scanpy as sc -import matplotlib.pyplot as plt -import seaborn as sns -from sklearn.decomposition import PCA -import argparse - -# Set drawing style -plt.rcParams['font.family'] = 'DejaVu Sans' -plt.rcParams['font.size'] = 10 -plt.rcParams['figure.dpi'] = 150 - -def plot_umap_cell_embeddings( - cell_embeddings: np.ndarray, - adata: sc.AnnData, - output_path: str, - embedding_key: str = "X_sclong_module" -): - """ - Draw cell-embedded UMAP visualization - - parameter: - - cell_embeddings: cell embedding vector (n_cells, embed_dim) - - adata: AnnData object, containing celltype and batch information - - output_path: output image path - - embedding_key: the key name embedded in adata.obsm - """ - print(f"Plotting UMAP of cell embeddings...") - - # Store embedding vector into adata - adata.obsm[embedding_key] = cell_embeddings - - # Calculate neighbors and UMAP - sc.pp.neighbors(adata, use_rep=embedding_key, n_neighbors=30, n_pcs=None) - sc.tl.umap(adata, min_dist=0.1) - - # Create graph - two subgraphs: color by celltype and by batch - fig, axes = plt.subplots(1, 2, figsize=(14, 6)) - - # Subfigure 1: Coloring by celltype - sc.pl.umap( - adata, - color='celltype', - ax=axes[0], - show=False, - title='Cell Type', - frameon=True - ) - - # Subpicture 2: Coloring by batch - sc.pl.umap( - adata, - color='batch', - ax=axes[1], - show=False, - title='Batch', - frameon=True - ) - - plt.tight_layout() - plt.savefig(output_path, dpi=150, bbox_inches='tight') - plt.close() - - print(f"Saved: {output_path}") - - -def plot_umap_pca50( - cell_embeddings: np.ndarray, - adata: sc.AnnData, - output_path: str, - embedding_key: str = "X_sclong_module_pca50" -): - """ - Plot PCA50 + UMAP visualization - - parameter: - - cell_embeddings: cell embedding vector (n_cells, embed_dim) - - adata: AnnData object - - output_path: output image path - - embedding_key: the key name embedded in adata.obsm - - Calculation method: - 1. Perform PCA dimensionality reduction on the cell embedding vector to 50 dimensions. - 2. Perform UMAP dimensionality reduction on the vector after PCA dimensionality reduction. - """ - print(f"Plotting UMAP after PCA50...") - - # PCA reduces dimensionality to 50 dimensions - n_components = min(50, cell_embeddings.shape[1], cell_embeddings.shape[0] - 1) - print(f" PCA components: {n_components}") - - pca = PCA(n_components=n_components) - cell_embeddings_pca50 = pca.fit_transform(cell_embeddings) - - # Calculate proportion of explained variance - explained_variance = np.sum(pca.explained_variance_ratio_) * 100 - print(f" Explained variance: {explained_variance:.2f}%") - - # Store PCA50 embedding into adata - adata.obsm[embedding_key] = cell_embeddings_pca50 - - # Calculate neighbors and UMAP - sc.pp.neighbors(adata, use_rep=embedding_key, n_neighbors=15, n_pcs=None) - sc.tl.umap(adata, min_dist=0.1) - - # Create a graph - two subplots - fig, axes = plt.subplots(1, 2, figsize=(14, 6)) - - # Subfigure 1: Coloring by celltype - sc.pl.umap( - adata, - color='celltype', - ax=axes[0], - show=False, - title=f'Cell Type (PCA{n_components})', - frameon=True - ) - - # Subpicture 2: Coloring by batch - sc.pl.umap( - adata, - color='batch', - ax=axes[1], - show=False, - title=f'Batch (PCA{n_components})', - frameon=True - ) - - plt.tight_layout() - plt.savefig(output_path, dpi=150, bbox_inches='tight') - plt.close() - - print(f"Saved: {output_path}") - - -def plot_batch_asw_comparison( - results_dict: dict, - output_path: str, - output_path_svg: str = None -): - """ - Plot a bar chart comparing batch ASW scores for different models/setups - - parameter: - - results_dict: dictionary, key is the model name, value is the ASW score - - output_path: output image path (.png) - - output_path_svg: output SVG image path (optional) - - Example results_dict: - { - 'scLong (multiply_27k)': 0.982, - 'scLong (no_gene2vec)': 0.778, - 'scGPT': 0.95, - ... - } - """ - print(f"Plotting batch ASW comparison...") - - # Color settings - color_dict = { - 'Raw': '#6FD1B2', - 'HVG': '#FF99CC', - 'scVI': '#FFCC99', - 'Geneformer': '#99CC99', - 'scGPT': '#FF9999', - 'scFoundation': '#D1B26F', - 'UCE': '#99FFCC', - 'scLong': '#CC99FF', - 'scLong (multiply_27k)': '#CC99FF', - 'scLong (no_gene2vec)': '#9966CC', - } - - models = list(results_dict.keys()) - values = list(results_dict.values()) - colors = [color_dict.get(model, '#888888') for model in models] - - fig, ax = plt.subplots(1, 1, figsize=(max(6, len(models) * 0.8), 4)) - - # Draw a bar chart - bars = ax.bar(models, values, color=colors, width=0.6) - - # Display values on bars - for bar, value in zip(bars, values): - height = bar.get_height() - ax.text(bar.get_x() + bar.get_width()/2., height + 0.01, - f'{value:.4f}', - ha='center', va='bottom', fontsize=9) - - ax.set_ylim((min(values) - 0.05, 1.02)) - ax.set_ylabel('batch ASW', fontsize=12) - ax.set_title('Batch ASW Comparison', fontsize=14) - - # Rotate x-axis label - plt.xticks(rotation=30, ha='right', fontsize=10) - - # Remove top and right borders - ax.spines['right'].set_visible(False) - ax.spines['top'].set_visible(False) - - plt.tight_layout() - plt.savefig(output_path, dpi=150, bbox_inches='tight') - - if output_path_svg: - plt.savefig(output_path_svg, format='svg', bbox_inches='tight') - - plt.close() - - print(f"Saved: {output_path}") - if output_path_svg: - print(f"Saved: {output_path_svg}") - - -def main(): - parser = argparse.ArgumentParser(description='Zero-shot batch correction visualization') - parser.add_argument('--res_dir', type=str, required=True, - help='Path to result directory (e.g., res/multiply_27k)') - parser.add_argument('--data_path', type=str, default='../zero-shot-batch/pancreas_scib.h5ad', - help='Path to original h5ad data file') - parser.add_argument('--embedding_file', type=str, default='sclong_module_batch_cell_emb_mode_ce.npy', - help='Name of embedding numpy file') - parser.add_argument('--csv_file', type=str, default='sclong_module_batch_cell_emb_mode.csv', - help='Name of ASW score csv file') - parser.add_argument('--embedding_key', type=str, default='X_sclong_module', - help='Key name for storing embeddings') - - args = parser.parse_args() - - # Load data - print(f"Loading data from {args.data_path}...") - adata = sc.read_h5ad(args.data_path) - - # Load cell embedding - embedding_path = os.path.join(args.res_dir, args.embedding_file) - print(f"Loading embeddings from {embedding_path}...") - cell_embeddings = np.load(embedding_path) - print(f" Cell embeddings shape: {cell_embeddings.shape}") - - # Load ASW scores - csv_path = os.path.join(args.res_dir, args.csv_file) - print(f"Loading ASW scores from {csv_path}...") - asw_df = pd.read_csv(csv_path) - asw_score = asw_df['value'].values[0] - print(f" ASW score: {asw_score:.4f}") - - # Make sure the number of cells in adata matches the number of embeddings - if adata.n_obs != cell_embeddings.shape[0]: - print(f"Warning: adata has {adata.n_obs} cells, but embeddings have {cell_embeddings.shape[0]} cells") - # If there are more cells in adata, only the first n are taken. - if adata.n_obs > cell_embeddings.shape[0]: - print(f" Subsampling adata to {cell_embeddings.shape[0]} cells") - adata = adata[:cell_embeddings.shape[0]].copy() - - # 1. Draw UMAP of cell embeddings - plot_umap_cell_embeddings( - cell_embeddings=cell_embeddings, - adata=adata, - output_path=os.path.join(args.res_dir, 'umap_cell_embeddings.png'), - embedding_key=args.embedding_key - ) - - # 2. Draw UMAP after PCA50 - plot_umap_pca50( - cell_embeddings=cell_embeddings, - adata=adata, - output_path=os.path.join(args.res_dir, 'umap_pca50.png'), - embedding_key=f"{args.embedding_key}_pca50" - ) - - print("\nVisualization completed!") - - -if __name__ == '__main__': +""" +Zero-shot batch correction visual code + +Used to generate the following images: +1. umap_cell_embeddings.png - UMAP visualization of cell embeddings (colored by cell type and batch) +2. umap_pca50.png - PCA dimensionality reduction to 50 dimensions and then UMAP visualization +3. batch_ASW_comparison.png - Batch ASW score comparison bar chart for different models/settings + +Enter data: +- sclong_module_batch_cell_emb_mode_ce.npy: cell embedding vector +- sclong_module_batch_cell_emb_mode.csv: ASW score +- Caller-supplied h5ad: the original data corresponding to the saved embeddings. + +Calculation method: +1. umap_cell_embeddings: directly perform UMAP dimensionality reduction on cell embedding vectors +2. umap_pca50: First use PCA to reduce the dimension to 50 dimensions, and then perform UMAP dimension reduction. +3. batch_ASW: Use scib.metrics.silhouette_batch to calculate the batch ASW score +""" + +import os +import numpy as np +import pandas as pd +import scanpy as sc +import matplotlib.pyplot as plt +import seaborn as sns +from sklearn.decomposition import PCA +import argparse + +# Set drawing style +plt.rcParams['font.family'] = 'DejaVu Sans' +plt.rcParams['font.size'] = 10 +plt.rcParams['figure.dpi'] = 150 + +def plot_umap_cell_embeddings( + cell_embeddings: np.ndarray, + adata: sc.AnnData, + output_path: str, + embedding_key: str = "X_sclong_module" +): + """ + Draw cell-embedded UMAP visualization + + parameter: + - cell_embeddings: cell embedding vector (n_cells, embed_dim) + - adata: AnnData object, containing celltype and batch information + - output_path: output image path + - embedding_key: the key name embedded in adata.obsm + """ + print(f"Plotting UMAP of cell embeddings...") + + # Store embedding vector into adata + adata.obsm[embedding_key] = cell_embeddings + + # Calculate neighbors and UMAP + sc.pp.neighbors(adata, use_rep=embedding_key, n_neighbors=30, n_pcs=None) + sc.tl.umap(adata, min_dist=0.1) + + # Create graph - two subgraphs: color by celltype and by batch + fig, axes = plt.subplots(1, 2, figsize=(14, 6)) + + # Subfigure 1: Coloring by celltype + sc.pl.umap( + adata, + color='celltype', + ax=axes[0], + show=False, + title='Cell Type', + frameon=True + ) + + # Subpicture 2: Coloring by batch + sc.pl.umap( + adata, + color='batch', + ax=axes[1], + show=False, + title='Batch', + frameon=True + ) + + plt.tight_layout() + plt.savefig(output_path, dpi=150, bbox_inches='tight') + plt.close() + + print(f"Saved: {output_path}") + + +def plot_umap_pca50( + cell_embeddings: np.ndarray, + adata: sc.AnnData, + output_path: str, + embedding_key: str = "X_sclong_module_pca50" +): + """ + Plot PCA50 + UMAP visualization + + parameter: + - cell_embeddings: cell embedding vector (n_cells, embed_dim) + - adata: AnnData object + - output_path: output image path + - embedding_key: the key name embedded in adata.obsm + + Calculation method: + 1. Perform PCA dimensionality reduction on the cell embedding vector to 50 dimensions. + 2. Perform UMAP dimensionality reduction on the vector after PCA dimensionality reduction. + """ + print(f"Plotting UMAP after PCA50...") + + # PCA reduces dimensionality to 50 dimensions + n_components = min(50, cell_embeddings.shape[1], cell_embeddings.shape[0] - 1) + print(f" PCA components: {n_components}") + + pca = PCA(n_components=n_components) + cell_embeddings_pca50 = pca.fit_transform(cell_embeddings) + + # Calculate proportion of explained variance + explained_variance = np.sum(pca.explained_variance_ratio_) * 100 + print(f" Explained variance: {explained_variance:.2f}%") + + # Store PCA50 embedding into adata + adata.obsm[embedding_key] = cell_embeddings_pca50 + + # Calculate neighbors and UMAP + sc.pp.neighbors(adata, use_rep=embedding_key, n_neighbors=15, n_pcs=None) + sc.tl.umap(adata, min_dist=0.1) + + # Create a graph - two subplots + fig, axes = plt.subplots(1, 2, figsize=(14, 6)) + + # Subfigure 1: Coloring by celltype + sc.pl.umap( + adata, + color='celltype', + ax=axes[0], + show=False, + title=f'Cell Type (PCA{n_components})', + frameon=True + ) + + # Subpicture 2: Coloring by batch + sc.pl.umap( + adata, + color='batch', + ax=axes[1], + show=False, + title=f'Batch (PCA{n_components})', + frameon=True + ) + + plt.tight_layout() + plt.savefig(output_path, dpi=150, bbox_inches='tight') + plt.close() + + print(f"Saved: {output_path}") + + +def plot_batch_asw_comparison( + results_dict: dict, + output_path: str, + output_path_svg: str = None +): + """ + Plot a bar chart comparing batch ASW scores for different models/setups + + parameter: + - results_dict: dictionary, key is the model name, value is the ASW score + - output_path: output image path (.png) + - output_path_svg: output SVG image path (optional) + + Example results_dict: + { + 'scLong (multiply_27k)': 0.982, + 'scLong (no_gene2vec)': 0.778, + 'scGPT': 0.95, + ... + } + """ + print(f"Plotting batch ASW comparison...") + + # Color settings + color_dict = { + 'Raw': '#6FD1B2', + 'HVG': '#FF99CC', + 'scVI': '#FFCC99', + 'Geneformer': '#99CC99', + 'scGPT': '#FF9999', + 'scFoundation': '#D1B26F', + 'UCE': '#99FFCC', + 'scLong': '#CC99FF', + 'scLong (multiply_27k)': '#CC99FF', + 'scLong (no_gene2vec)': '#9966CC', + } + + models = list(results_dict.keys()) + values = list(results_dict.values()) + colors = [color_dict.get(model, '#888888') for model in models] + + fig, ax = plt.subplots(1, 1, figsize=(max(6, len(models) * 0.8), 4)) + + # Draw a bar chart + bars = ax.bar(models, values, color=colors, width=0.6) + + # Display values on bars + for bar, value in zip(bars, values): + height = bar.get_height() + ax.text(bar.get_x() + bar.get_width()/2., height + 0.01, + f'{value:.4f}', + ha='center', va='bottom', fontsize=9) + + ax.set_ylim((min(values) - 0.05, 1.02)) + ax.set_ylabel('batch ASW', fontsize=12) + ax.set_title('Batch ASW Comparison', fontsize=14) + + # Rotate x-axis label + plt.xticks(rotation=30, ha='right', fontsize=10) + + # Remove top and right borders + ax.spines['right'].set_visible(False) + ax.spines['top'].set_visible(False) + + plt.tight_layout() + plt.savefig(output_path, dpi=150, bbox_inches='tight') + + if output_path_svg: + plt.savefig(output_path_svg, format='svg', bbox_inches='tight') + + plt.close() + + print(f"Saved: {output_path}") + if output_path_svg: + print(f"Saved: {output_path_svg}") + + +def main(): + parser = argparse.ArgumentParser(description='Zero-shot batch correction visualization') + parser.add_argument('--res_dir', type=str, required=True, + help='Path to result directory (e.g., res/multiply_27k)') + parser.add_argument('--data_path', type=str, required=True, + help='Path to original h5ad data file') + parser.add_argument('--embedding_file', type=str, default='sclong_module_batch_cell_emb_mode_ce.npy', + help='Name of embedding numpy file') + parser.add_argument('--csv_file', type=str, default='sclong_module_batch_cell_emb_mode.csv', + help='Name of ASW score csv file') + parser.add_argument('--embedding_key', type=str, default='X_sclong_module', + help='Key name for storing embeddings') + + args = parser.parse_args() + + # Load data + print(f"Loading data from {args.data_path}...") + adata = sc.read_h5ad(args.data_path) + + # Load cell embedding + embedding_path = os.path.join(args.res_dir, args.embedding_file) + print(f"Loading embeddings from {embedding_path}...") + cell_embeddings = np.load(embedding_path) + print(f" Cell embeddings shape: {cell_embeddings.shape}") + + # Load ASW scores + csv_path = os.path.join(args.res_dir, args.csv_file) + print(f"Loading ASW scores from {csv_path}...") + asw_df = pd.read_csv(csv_path) + asw_score = asw_df['value'].values[0] + print(f" ASW score: {asw_score:.4f}") + + # Make sure the number of cells in adata matches the number of embeddings + if adata.n_obs != cell_embeddings.shape[0]: + print(f"Warning: adata has {adata.n_obs} cells, but embeddings have {cell_embeddings.shape[0]} cells") + # If there are more cells in adata, only the first n are taken. + if adata.n_obs > cell_embeddings.shape[0]: + print(f" Subsampling adata to {cell_embeddings.shape[0]} cells") + adata = adata[:cell_embeddings.shape[0]].copy() + + # 1. Draw UMAP of cell embeddings + plot_umap_cell_embeddings( + cell_embeddings=cell_embeddings, + adata=adata, + output_path=os.path.join(args.res_dir, 'umap_cell_embeddings.png'), + embedding_key=args.embedding_key + ) + + # 2. Draw UMAP after PCA50 + plot_umap_pca50( + cell_embeddings=cell_embeddings, + adata=adata, + output_path=os.path.join(args.res_dir, 'umap_pca50.png'), + embedding_key=f"{args.embedding_key}_pca50" + ) + + print("\nVisualization completed!") + + +if __name__ == '__main__': main() \ No newline at end of file diff --git a/README.md b/README.md index 24b8230411521267ceac695528b935a2fb1638f3..d2f9b721711f662f32c67a1c7e2c39ea68229993 100644 --- a/README.md +++ b/README.md @@ -97,10 +97,10 @@ The repository also includes the following supporting workflows. | Pretraining and continued pretraining | [Pretraining script](BioMod/performer_module.py) | [Source guide](BioMod/README.md), [launcher reference](BioMod/command.txt) | | Module interpretability analysis | [Module geometry](BioMod/interpretability/module_geometry.py), [module importance](BioMod/interpretability/module_importance/module_importance.py), [text alignment](BioMod/interpretability/module_text_alignment/module_text_alignment.py) | [Overview](BioMod/interpretability/README.md), [attention](BioMod/interpretability/attention/README.md), [module importance guide](BioMod/interpretability/module_importance/README.md) | -Tutorials and launcher references retain some historical checkpoint paths, -server paths, and arguments. Adapt them to the current assets above and check -the selected script's `--help` before running. Some notebook plotting steps -also refer to external scripts or reference results that are not bundled here. +Bundled input defaults resolve from each script's location. Command examples +state their working directory; explicit relative command-line paths are resolved +from that directory. Pretraining corpora, legacy reference checkpoints, and +generated analysis results must be supplied separately where indicated. ## Usage notes @@ -111,22 +111,27 @@ The project root contains `BioMod/`, `model_dependencies/`, `weight/`, and may assume different working directories; adjust relative paths when following a guide. -Before running a notebook, update its configuration cells: +Start Jupyter from `BioMod/toturials/`. Each notebook sets +`PROJECT_ROOT = Path('../..').resolve()` and derives absolute runtime paths from +the outer repository root. If Jupyter starts at the repository root, use +`Path('.').resolve()` instead. Review the configuration cells before running: | Setting | Location | | --- | --- | -| `BASE` or `ASSET_ROOT` | Project root | -| `PROJECT_ROOT` | Inner `BioMod/` source directory | -| `MODULE_MODEL` or `LEGACY_MODULE_MODEL` | `model_dependencies/` under the project root | +| `PROJECT_ROOT` | Outer project root containing `BioMod/`, `model_dependencies/`, `weight/`, and `data/` | +| `SOURCE_DIR` | `PROJECT_ROOT / 'BioMod'` | +| Asset or dependency directory | `PROJECT_ROOT / 'model_dependencies'` | | Weight and configuration paths | The BioMod model weights and matching configuration listed above | -| `MODULE_DATA_DIR` | `BioMod/module_data_all/` under the project root | +| `MODULE_DATA_DIR` | `model_dependencies/module_data_all/` under the project root | | Data and output paths | Your task inputs and a writable output directory | Checkpoint variable names differ by notebook: `CKPT_PATH` / `CONFIG_PKL` for cell annotation, drug response, and zero-shot integration; `SCFM_CKPT` / `SCFM_HYPER` for gene perturbation; and `CHECKPOINT_PATH` / `HYPER_PARAMS_PATH` for drug combination. -Updating environment variables alone does not replace hard-coded notebook paths. +Training defaults to disabled (`RUN_TRAINING = False`). The zero-shot notebook +uses `RUN_EVALUATION = False` and enables UMAP export separately with `MAKE_UMAP`. +Output directories are created when the corresponding operation is enabled. ### Checkpoint arguments diff --git a/data/downstream/final_data/drug_combination_synergy/README.md b/data/downstream/final_data/drug_combination_synergy/README.md index 023937ec8b54fbd48dbba3dc2234cd680df1dd39..f7ce3ed36240f72a9426ba352f573aad70b297bb 100644 --- a/data/downstream/final_data/drug_combination_synergy/README.md +++ b/data/downstream/final_data/drug_combination_synergy/README.md @@ -1,48 +1,50 @@ -# Drug-combination synergy data - -This folder is a cleaned data layout for drug-combination synergy prediction. -The original collaborator package is kept unchanged at: - -```text -C:\Users\JXZ\Desktop\文章的相关材料\data\downstream\final_data\drug_response_pytorch\drug_combination -``` - -## Runnable data path - -Use this directory as `--data_dir` for `drug_combination/train_drug_combination_performer_patch.py`: - -```text -C:\Users\JXZ\Desktop\文章的相关材料\data\downstream\final_data\drug_combination_synergy\data_summary -``` - -Required files used by the current script: - -```text -data_summary/ -|-- new_labels_0_10.csv -|-- new_cell_features_954.csv -|-- smiles_0_10.csv -`-- test_data/ - |-- independent_input.csv - `-- independent_cell_features_954.csv -``` - -## Checked statistics - -- Training labels: (12415, 4) -- Training cell features: (32, 955) -- SMILES table: (35, 2) -- Independent test labels: (668, 4) -- Independent test cell features: (1020, 955) -- Training label counts: {'0': 6299, '1': 6116} -- Test label counts: {'1': 480, '0': 188} -- Unique training cells: 31 -- Unique test cells: 24 -- Unique training drug columns: drug1=35, drug2=35 -- Unique test drug columns: drug1=44, drug2=39 - -Additional files in `data_summary/` are retained for provenance and regeneration of train/test labels. - -## Files - -See `MANIFEST.csv` for file sizes and checksums. See `SUMMARY.json` for machine-readable shape checks. +# Drug-combination synergy data + +This folder is a cleaned data layout for drug-combination synergy prediction. +The bundled legacy collaborator reference package is located at this repository-relative path: + +```text +data/downstream/final_data/drug_response_pytorch/drug_combination +``` + +## Runnable data path + +From the repository root, use this directory as `--data_dir` for `BioMod/drug_combination/train_drug_combination_performer_module.py`: + +```text +data/downstream/final_data/drug_combination_synergy/data_summary +``` + +Required files used by the current script: + +```text +data_summary/ +|-- new_labels_0_10.csv +|-- new_cell_features_954.csv +|-- smiles_0_10.csv +`-- test_data/ + |-- independent_input.csv + `-- independent_cell_features_954.csv +``` + +## Previously recorded statistics + +The figures below are retained from the existing dataset documentation; this path/comment audit did not recompute them. + +- Training labels: (12415, 4) +- Training cell features: (32, 955) +- SMILES table: (35, 2) +- Independent test labels: (668, 4) +- Independent test cell features: (1020, 955) +- Training label counts: {'0': 6299, '1': 6116} +- Test label counts: {'1': 480, '0': 188} +- Unique training cells: 31 +- Unique test cells: 24 +- Unique training drug columns: drug1=35, drug2=35 +- Unique test drug columns: drug1=44, drug2=39 + +Additional files in `data_summary/` are retained for provenance and regeneration of train/test labels. + +## Files + +See `MANIFEST.csv` for file sizes and checksums. See `SUMMARY.json` for machine-readable shape checks. diff --git a/data/downstream/final_data/drug_combination_synergy/data_summary/README.md b/data/downstream/final_data/drug_combination_synergy/data_summary/README.md index a97eec301cc0e97f7d9179a21f05278a45b562ba..93ce42754d5c0c1450d465e9b4c89750e1cb246a 100644 --- a/data/downstream/final_data/drug_combination_synergy/data_summary/README.md +++ b/data/downstream/final_data/drug_combination_synergy/data_summary/README.md @@ -1,109 +1,108 @@ -# DeepDDS 核心数据文件说明 +# DeepDDS core data files -## 数据来源 +The provenance and counts below are translated from the existing dataset notes. +This path/comment audit did not recompute counts or validate the source studies. -- **药物协同数据**:来自 O'Neil et al. (2016) 的高通量药物组合筛选实验,涵盖 38 种抗癌药物在 39 个人类癌症细胞系上的两两组合。 -- **协同评分**:基于 4×4 剂量-响应矩阵,使用 Combenefit 工具计算 Loewe Additivity Score。同一药物组合若有多个重复实验,取平均值作为最终协同评分。 -- **细胞系表达数据**:来自 CCLE (Cancer Cell Line Encyclopedia) 2018年发布的 RNA-seq TPM 数据 (`CCLE_RNAseq_rsem_transcripts_tpm_20180929`),为细胞系在**未经药物处理、正常培养条件**下的基线基因表达谱。 +## Data sources -## 标签定义(二值化阈值) +- **Drug synergy:** the O'Neil et al. (2016) high-throughput screen of pairwise combinations of 38 anticancer drugs across 39 human cancer cell lines. +- **Synergy scores:** Loewe Additivity Scores computed with Combenefit from 4 x 4 dose-response matrices. Replicate measurements for the same combination were averaged. +- **Cell-line expression:** CCLE RNA-seq TPM data released in 2018 (`CCLE_RNAseq_rsem_transcripts_tpm_20180929`), describing baseline expression under standard culture conditions without drug treatment. -| 条件 | 标签 | 含义 | -|------|------|------| -| Loewe score > 10 | 1 (正样本) | 协同 (synergistic) | -| Loewe score < 0 | 0 (负样本) | 拮抗 (antagonistic) | -| 0 ≤ score ≤ 10 | 排除 | 模糊区间,不参与建模 | +## Binary labels -使用严格阈值(>10 而非 >0)的目的是:(1) 减少噪声附近样本的干扰;(2) 获得正负样本基本平衡的数据集。 +| Condition | Label | Meaning | +|---|---|---| +| Loewe score > 10 | 1 | Synergistic | +| Loewe score < 0 | 0 | Antagonistic | +| 0 <= score <= 10 | Excluded | Ambiguous range excluded from modeling | -## 最终数据规模 +The original notes describe the stricter positive threshold as intended to reduce near-zero noise and obtain roughly balanced classes. -- **12,415** 个唯一的 drug pair–cell line 组合 -- **36** 种抗癌药物 -- **31** 个人类癌症细胞系 -- 正样本 6,116 个,负样本 6,299 个 +## Previously recorded dataset size ---- +- 12,415 unique drug-pair/cell-line combinations. +- 36 anticancer drugs and 31 human cancer cell lines. +- 6,116 positive and 6,299 negative samples. -## 文件清单 +## File descriptions -### 1. new_labels_0_10.csv — 二值化标签文件(模型训练核心输入) +### `new_labels_0_10.csv`: binary training labels -- **行数**:12,415 条记录 + 1 行表头 -- **列**: - | 列名 | 类型 | 说明 | - |------|------|------| - | drug1 | string | 药物 A 的 SMILES 分子式 | - | drug2 | string | 药物 B 的 SMILES 分子式 | - | cell | string | 细胞系名称(如 A375, HCT116) | - | label | int | 0 = 拮抗 (score < 0),1 = 协同 (score > 10) | +The recorded layout is 12,415 records plus one header row. -### 2. score_0_10.csv — 连续协同评分文件 +| Column | Type | Description | +|---|---|---| +| drug1 | string | SMILES representation of drug A | +| drug2 | string | SMILES representation of drug B | +| cell | string | Cell-line name, such as A375 or HCT116 | +| label | int | 0 = antagonistic (score < 0); 1 = synergistic (score > 10) | -- **行数**:12,415 条记录 + 1 行表头(与 new_labels_0_10.csv 一一对应) -- **列**: - | 列名 | 类型 | 说明 | - |------|------|------| - | drug1 | string | 药物 A 名称(如 5-FU) | - | drug2 | string | 药物 B 名称(如 ABT-888) | - | cell | string | 细胞系名称 | - | score | float | Loewe Additivity 连续协同评分 | - | nor | float | 归一化后的评分 | - | index | int | 样本索引 | - | true_label | int | 真实二值标签 | - | predict_label | int | 模型预测标签(作者预训练结果) | - | predict_score | float | 模型预测概率(作者预训练结果) | - | smile1 | string | 药物 A 的 SMILES | - | smile2 | string | 药物 B 的 SMILES | +### `score_0_10.csv`: continuous synergy scores -### 3. new_cell_features_954.csv — 细胞系基因表达矩阵 +The existing notes record 12,415 records plus a header, corresponding to `new_labels_0_10.csv`. -- **维度**:31 个细胞系 × 954 个基因 -- **数据来源**:CCLE RNA-seq baseline TPM(未经药物处理的基线表达) -- **格式**: - - 第 1 行:gene_id(ENSG 编号) - - 第 2 行:transcript_id(ENST 编号) - - 第 3~33 行:31 个细胞系的表达值,第 1 列为细胞系名称 -- **用途**:作为模型中细胞系特征的输入,经全连接网络编码后与药物图特征融合 +| Column | Type | Description | +|---|---|---| +| drug1 | string | Drug A name, such as 5-FU | +| drug2 | string | Drug B name, such as ABT-888 | +| cell | string | Cell-line name | +| score | float | Continuous Loewe Additivity Score | +| nor | float | Normalized score | +| index | int | Sample index | +| true_label | int | Binary ground-truth label | +| predict_label | int | Legacy predicted label | +| predict_score | float | Legacy prediction probability | +| smile1 | string | Drug A SMILES | +| smile2 | string | Drug B SMILES | -### 4. cell_gene954.csv — 954 基因 ID 对照表 +The stored prediction columns are historical reference results, not results from the current BioMod checkout. -- **行数**:954 条记录 + 1 行表头 -- **列**:gene_id (ENSG), transcript_id (ENST) -- **用途**:记录 new_cell_features_954.csv 中 954 个基因的 Ensembl 编号 +### `new_cell_features_954.csv`: cell-line expression -### 5. smiles_0_10.csv — 药物 SMILES 对照表 +- Recorded dimensions: 31 cell lines x 954 genes. +- Source: CCLE baseline RNA-seq TPM, without drug treatment. +- Row 1: gene IDs (ENSG). +- Row 2: transcript IDs (ENST). +- Rows 3-33: expression values, with the cell-line name in the first column. +- Purpose: cell features encoded by fully connected layers and combined with drug graph features. -- **行数**:36 条(无表头) -- **列**:药物名称, SMILES 分子式 -- **用途**:药物名称与 SMILES 的映射关系,SMILES 用于构建分子图作为 GNN 输入 +### `cell_gene954.csv`: gene identifier table -### 6. drugs_0_10.csv — 药物信息表 +- Recorded layout: 954 records plus a header. +- Columns: `gene_id` (ENSG), `transcript_id` (ENST). +- Purpose: identify the genes in `new_cell_features_954.csv`. -- **行数**:36 条(无表头) -- **列**:药物名称, SMILES 分子式, 数值编码 -- **用途**:药物列表及其编码索引 +### `smiles_0_10.csv`: drug SMILES table -### 7. cell_line_0_10.csv — 细胞系信息表 +- The legacy notes record 36 headerless rows; counts may differ from the parser-based summary in the parent directory. +- Columns: drug name and SMILES representation. +- Purpose: map drug names to SMILES used for molecular-graph construction. -- **行数**:31 条(无表头) -- **列**:细胞系名称, 癌症类型(如 melanoma, ovarian, colon) -- **用途**:记录 31 个细胞系对应的癌症类型 +### `drugs_0_10.csv`: drug table ---- +- Recorded layout: 36 headerless rows. +- Columns: drug name, SMILES, numeric encoding. +- Purpose: enumerate drugs and their encoded indices. -## DeepDDS 模型输入结构 +### `cell_line_0_10.csv`: cell-line table -``` -Drug A (SMILES) ──→ 分子图 ──→ GAT/GCN ──→ Drug A embedding ─┐ - ├──→ 拼接 ──→ 全连接层 ──→ 协同/拮抗预测 -Drug B (SMILES) ──→ 分子图 ──→ GAT/GCN ──→ Drug B embedding ─┤ - │ -Cell line (954 genes TPM) ──→ FC layers ──→ Cell embedding ────┘ +- Recorded layout: 31 headerless rows. +- Columns: cell-line name and cancer type, such as melanoma, ovarian, or colon. +- Purpose: associate cell lines with cancer types. + +## DeepDDS inputs + +```text +Drug A SMILES -> Molecular graph -> GAT/GCN -> Drug A embedding --+ + | +Drug B SMILES -> Molecular graph -> GAT/GCN -> Drug B embedding --+-> Concatenate -> FC layers -> Synergy prediction + | +Cell line (954 genes, TPM) -> FC layers -> Cell embedding -------+ ``` -## 参考文献 +## References retained from the original notes - Wang et al. "DeepDDS: deep graph neural network with attention mechanism to predict synergistic drug combinations." Briefings in Bioinformatics, 2022. - O'Neil et al. "An unbiased oncology compound screen to identify novel combination strategies." Molecular Cancer Therapeutics, 2016. -- CCLE: Cancer Cell Line Encyclopedia (Broad Institute) +- CCLE: Cancer Cell Line Encyclopedia, Broad Institute. diff --git a/data/downstream/final_data/drug_combination_synergy/data_summary/test_data/README.md b/data/downstream/final_data/drug_combination_synergy/data_summary/test_data/README.md index b40f7e69e13f14bf07b0de3ee5e45b192629d5eb..539329f79d9c1c7220989f269780a6bfd03244aa 100644 --- a/data/downstream/final_data/drug_combination_synergy/data_summary/test_data/README.md +++ b/data/downstream/final_data/drug_combination_synergy/data_summary/test_data/README.md @@ -1,62 +1,54 @@ -# DeepDDS 独立测试集(AstraZeneca-Sanger DREAM Challenge) +# DeepDDS independent test data -## 数据来源 +## Source -来自 AstraZeneca-Sanger Drug Combination DREAM Challenge 数据集(Menden et al., Nature Communications, 2019)。 -原始数据文件:`41467_2019_9799_MOESM3_ESM.xlsx` +The original notes attribute these data to the AstraZeneca-Sanger Drug Combination DREAM Challenge (Menden et al., Nature Communications, 2019), with source supplement `41467_2019_9799_MOESM3_ESM.xlsx`. -该数据集与训练集(O'Neil et al. 2016)完全独立,用于评估模型的泛化能力。 +This collection is intended for evaluation against the separate O'Neil et al. (2016) training collection. The path/comment audit did not verify overlap, counts, or the original source study; those checks are required before making claims about benchmark independence. -## 数据规模 +## Previously recorded size -- **668** 个 drug pair–cell line 组合 -- **52** 种药物(部分为训练集中未见过的新药) -- **24** 个细胞系(部分为训练集中未见过的新细胞系) -- 正样本 480 个,负样本 188 个 +- 668 drug-pair/cell-line combinations. +- 52 drugs and 24 cell lines, including entries described in the original notes as absent from training. +- 480 positive and 188 negative samples. ---- +## Core inputs -## 文件清单 +| File | Purpose | +|---|---| +| `independent_input.csv` | Formatted model inputs; recorded as 668 rows with drug1 (SMILES), drug2 (SMILES), cell, and label | +| `independent_cell_features_954.csv` | CCLE baseline expression for 954 genes; recorded as 1,019 cell lines, including lines outside the training collection | -### 核心输入文件 +## Source and intermediate files -| 文件 | 说明 | -|------|------| -| `independent_input.csv` | 最终格式化的模型输入文件(668 条),列:drug1(SMILES), drug2(SMILES), cell, label | -| `independent_cell_features_954.csv` | 细胞系 954 基因 CCLE 基线表达矩阵(1019 个细胞系,含训练集外的新细胞系) | +| File | Purpose | +|---|---| +| `41467_2019_9799_MOESM3_ESM.xlsx` | Original publication supplement with the full AZ-DREAM data | +| `independent_allin.csv` | Extracted combinations; recorded as 76,607 rows with drug1, drug2, and cell | +| `independent_allin_smiles.csv` | Extracted combinations with SMILES annotations | +| `independent_set_combinatioin.csv` | Combination details, including synergy scores, IC50, Hill slope, and Einf | +| `independent_set_newcombinatioin.csv` | Filtered combinations described in the original notes as excluding overlap with training | -### 原始数据与中间处理文件 +## Supporting metadata -| 文件 | 说明 | -|------|------| -| `41467_2019_9799_MOESM3_ESM.xlsx` | 原始论文附件(AZ-DREAM Challenge 全量数据) | -| `independent_allin.csv` | 从原始数据提取的全量组合(76,607 条),列:drug1, drug2, cell | -| `independent_allin_smiles.csv` | 附加 SMILES 信息的全量组合 | -| `independent_set_combinatioin.csv` | 原始组合详情,含 synergy score、IC50、Hill slope、Einf 等药理学参数 | -| `independent_set_newcombinatioin.csv` | 筛选后的组合(去除与训练集重叠的部分) | +| File | Purpose | +|---|---| +| `independent_set_drug.csv` | Drug names, targets, pathways, SMILES, molecular weights, LogP, and Lipinski descriptors | +| `independent_set_cell_line.csv` | Metadata recorded for 144 cell lines: COSMIC IDs, tissue, TCGA labels, MSI status, and AZ-DREAM/O'Neil/ALMANAC membership | +| `independent_drug.csv` | Filtered drugs available for testing | +| `independent_cell.csv` | Filtered cell lines available for testing | +| `independent_input_drug_cell.csv` | Drug-to-cell-line mapping for the formatted inputs | -### 辅助信息文件 +## Drug-pair subsets -| 文件 | 说明 | -|------|------| -| `independent_set_drug.csv` | 药物信息表(药物名、靶点、通路、SMILES、分子量、LogP、Lipinski 等) | -| `independent_set_cell_line.csv` | 144 个细胞系信息(COSMIC ID、组织类型、TCGA 标签、MSI 状态、是否在 AZ-DREAM/O'Neil/ALMANAC 中) | -| `independent_drug.csv` | 筛选后可用于测试的药物列表 | -| `independent_cell.csv` | 筛选后可用于测试的细胞系列表 | -| `independent_input_drug_cell.csv` | 输入数据中的药物-细胞系映射关系 | +| File | Drug pair | +|---|---| +| `AZD2014_6244.csv` | AZD2014 + AZD6244 | +| `AZD8931-AZD5363.csv` | AZD8931 + AZD5363 | +| `Afatinib-MK2206.csv` | Afatinib + MK2206 | +| `GDC0941-AZD6244.csv` | GDC0941 + AZD6244 | +| `GDC0941-MK2206.csv` | GDC0941 + MK2206 | -### 子集文件(按药物组合拆分) - -| 文件 | 说明 | -|------|------| -| `AZD2014_6244.csv` | AZD2014 + AZD6244 组合的子集 | -| `AZD8931-AZD5363.csv` | AZD8931 + AZD5363 组合的子集 | -| `Afatinib-MK2206.csv` | Afatinib + MK2206 组合的子集 | -| `GDC0941-AZD6244.csv` | GDC0941 + AZD6244 组合的子集 | -| `GDC0941-MK2206.csv` | GDC0941 + MK2206 组合的子集 | - ---- - -## 参考文献 +## Reference retained from the original notes - Menden et al. "Community assessment to advance computational prediction of cancer drug combinations in a pharmacogenomic screen." Nature Communications, 2019. diff --git a/data/downstream/final_data/drug_response_pytorch/drug_combination/README.md b/data/downstream/final_data/drug_response_pytorch/drug_combination/README.md index 224e0013ee5461173e51ecd8bb6cf79a814902d5..520ac662796ace7d943be3604871343578751a67 100644 --- a/data/downstream/final_data/drug_response_pytorch/drug_combination/README.md +++ b/data/downstream/final_data/drug_response_pytorch/drug_combination/README.md @@ -1,9 +1,40 @@ -# DeepDDs -tips -Step1: create dataset for input -creat_data_DC.py could deal with the file like data/new_labels_0_10.csv - -Step2: input the train data to the modle -training_GAT.py is a train model by GAT \\ -trianing_GCN.py is a train model by GCN \\ -training_validation.py is a validation model for predicting novel drug pairs +# Legacy DeepDDS reference code + +This directory preserves the collaborator's DeepDDS/scLong/Geneformer reference +workflows. It is separate from the current BioMod drug-combination implementation +in `BioMod/drug_combination` at the repository root. Its model names and training +settings are retained; BioMod model weights are not interchangeable with these +legacy artifacts. + +## Paths + +Input defaults resolve from the source directory, independent of the shell's +working directory. `data_summary/` contains the bundled CSV inputs. The legacy +`independent_set/` input name maps to `data_summary/test_data/`. + +- `DEEPDDS_DATA_DIR`: optional directory containing compatible input CSVs. +- `DEEPDDS_ARTIFACT_DIR`: generated cache, embedding, result, and figure directory; + defaults to `outputs/reference_deepdds` at the repository root. +- Shared gene IDs resolve from the repository's `model_dependencies/` directory. + +The legacy scripts also require artifacts that are not bundled: a `smiles.csv` +with a `smile` column, matching scLong cell embeddings, optional Geneformer +embeddings under `geneformer/`, and saved benchmark results for plotting. +`smiles_0_10.csv` has a different schema and is not silently substituted for +`smiles.csv`. Missing inputs raise a path-specific error. Generate or supply +these artifacts with the corresponding reference model before running. + +## Entry points + +- `creat_data_DC.py`: prepare PyTorch Geometric datasets from compatible CSVs. +- `training_GAT.py` and `training_GCN.py`: legacy graph-model training workflows. +- `training_validation.py`: evaluate held-out drug combinations. +- `process.py` and `process.ipynb`: align previously generated scLong embeddings + to the DeepDDS gene order. +- `plot.ipynb`: plot previously generated ROC results. Its example filenames must + be adjusted to the chosen run and checkpoint identifier. + +Start the reference notebooks with this directory as the Jupyter working +directory. The stored execution outputs have been cleared because they refer to +the original machine and do not validate this checkout. This path/comment audit +did not run the reference training workflows or verify their numerical behavior. diff --git a/data/downstream/final_data/drug_response_pytorch/drug_combination/_reference_paths.py b/data/downstream/final_data/drug_response_pytorch/drug_combination/_reference_paths.py new file mode 100644 index 0000000000000000000000000000000000000000..6c174f3684154176e596b07c8ff0566f226fa4b4 --- /dev/null +++ b/data/downstream/final_data/drug_response_pytorch/drug_combination/_reference_paths.py @@ -0,0 +1,64 @@ +"""Portable paths for the bundled legacy DeepDDS reference scripts. + +DEEPDDS_DATA_DIR can override the bundled data_summary directory. +DEEPDDS_ARTIFACT_DIR stores generated caches, embeddings, metrics, and figures. +External legacy artifacts must be supplied by the user; BioMod weights are not +substituted for scLong or Geneformer artifacts. +""" + +import os +from pathlib import Path + +REFERENCE_DIR = Path(__file__).resolve().parent +PROJECT_ROOT = REFERENCE_DIR.parents[4] +DATA_DIR = Path(os.environ.get("DEEPDDS_DATA_DIR", str(REFERENCE_DIR / "data_summary"))).expanduser().resolve() +ARTIFACT_DIR = Path(os.environ.get("DEEPDDS_ARTIFACT_DIR", str(PROJECT_ROOT / "outputs" / "reference_deepdds"))).expanduser().resolve() +ASSET_DIR = PROJECT_ROOT / "model_dependencies" + + +def data_path(relative): + """Resolve a bundled input, preserving the legacy independent_set layout.""" + relative = relative.replace("\\", "/") + supplied_path = DATA_DIR / relative + if supplied_path.is_file(): + return str(supplied_path) + if relative.startswith("independent_set/"): + relative = "test_data/" + relative[len("independent_set/"):] + path = DATA_DIR / relative + if not path.is_file(): + raise FileNotFoundError( + "Legacy DeepDDS input is not bundled at {}. Set DEEPDDS_DATA_DIR to " + "a compatible data directory; retain the expected CSV schema.".format(path) + ) + return str(path) + + +def artifact_path(relative): + """Resolve a generated or externally supplied reference artifact.""" + return str(ARTIFACT_DIR / relative) + + +def output_path(relative): + """Resolve an output and create only its containing directory when run.""" + path = ARTIFACT_DIR / relative + path.parent.mkdir(parents=True, exist_ok=True) + return str(path) + + +def output_dir(relative): + """Return a generated-output directory with a trailing path separator.""" + path = ARTIFACT_DIR / relative + path.mkdir(parents=True, exist_ok=True) + return str(path) + os.sep + + +def required_artifact(relative): + """Locate an external legacy result without loading its contents.""" + path = Path(artifact_path(relative)) + if not path.is_file(): + raise FileNotFoundError( + "Legacy reference artifact is missing: {}. Generate it with the " + "matching reference model or set DEEPDDS_ARTIFACT_DIR to its directory." + .format(path) + ) + return str(path) diff --git a/data/downstream/final_data/drug_response_pytorch/drug_combination/cluster.py b/data/downstream/final_data/drug_response_pytorch/drug_combination/cluster.py index c0059a1e6720778d831406d0a3ff0a066c110e78..d1002e40443368ddda6d308bc205bd00893fe18b 100644 --- a/data/downstream/final_data/drug_response_pytorch/drug_combination/cluster.py +++ b/data/downstream/final_data/drug_response_pytorch/drug_combination/cluster.py @@ -1,25 +1,26 @@ -from sklearn.feature_extraction.text import TfidfVectorizer -from sklearn.cluster import KMeans -from sklearn.cluster import AffinityPropagation -from sklearn.cluster import DBSCAN -from sklearn.cluster import MeanShift, estimate_bandwidth -from sklearn.cluster import AgglomerativeClustering -from sklearn.cluster import Birch -from sklearn.cluster import SpectralClustering -from sklearn.mixture import GaussianMixture -from sklearn.metrics.cluster import normalized_mutual_info_score -import pandas as pd -import numpy as np - -AZD6244_weight = pd.read_csv('data/result/AZD6244_weight.csv', header=None) -AZD6244_weight = np.array(AZD6244_weight.T) -print('AZD6244_weight.shape', AZD6244_weight.shape) - -AZD2014_weight = pd.read_csv('data/result/AZD2014_weight.csv', header=None) -AZD2014_weight = np.array(AZD2014_weight.T) -print('AZD2014_weight.shape', AZD2014_weight.shape) - -#AffinityPropagation -af = AffinityPropagation().fit(AZD6244_weight) -result = af.labels_ -print(result) +from _reference_paths import ASSET_DIR, artifact_path, data_path, output_dir, output_path, required_artifact +from sklearn.feature_extraction.text import TfidfVectorizer +from sklearn.cluster import KMeans +from sklearn.cluster import AffinityPropagation +from sklearn.cluster import DBSCAN +from sklearn.cluster import MeanShift, estimate_bandwidth +from sklearn.cluster import AgglomerativeClustering +from sklearn.cluster import Birch +from sklearn.cluster import SpectralClustering +from sklearn.mixture import GaussianMixture +from sklearn.metrics.cluster import normalized_mutual_info_score +import pandas as pd +import numpy as np + +AZD6244_weight = pd.read_csv(output_path('result/AZD6244_weight.csv'), header=None) +AZD6244_weight = np.array(AZD6244_weight.T) +print('AZD6244_weight.shape', AZD6244_weight.shape) + +AZD2014_weight = pd.read_csv(output_path('result/AZD2014_weight.csv'), header=None) +AZD2014_weight = np.array(AZD2014_weight.T) +print('AZD2014_weight.shape', AZD2014_weight.shape) + +#AffinityPropagation +af = AffinityPropagation().fit(AZD6244_weight) +result = af.labels_ +print(result) diff --git a/data/downstream/final_data/drug_response_pytorch/drug_combination/creat_data_DC.py b/data/downstream/final_data/drug_response_pytorch/drug_combination/creat_data_DC.py index bbfdbc4bfd16aaa72a0d3e32311fb478f6d99b1a..4123effacc8a739059f4df8607a42b239cc9227d 100644 --- a/data/downstream/final_data/drug_response_pytorch/drug_combination/creat_data_DC.py +++ b/data/downstream/final_data/drug_response_pytorch/drug_combination/creat_data_DC.py @@ -1,123 +1,124 @@ -import csv -from itertools import islice - -import pandas as pd -import numpy as np -import os -import json, pickle -from collections import OrderedDict -from rdkit import Chem -# from rdkit.Chem import MolFromSmiles -import networkx as nx -from utils_test import * - - - -def get_cell_feature(cellId, cell_features): - for row in islice(cell_features, 0, None): - if row[0] == cellId: - return row[1: ] - -def atom_features(atom): - return np.array(one_of_k_encoding_unk(atom.GetSymbol(), - ['C', 'N', 'O', 'S', 'F', 'Si', 'P', 'Cl', 'Br', 'Mg', 'Na', 'Ca', 'Fe', 'As', - 'Al', 'I', 'B', 'V', 'K', 'Tl', 'Yb', 'Sb', 'Sn', 'Ag', 'Pd', 'Co', 'Se', - 'Ti', 'Zn', 'H', 'Li', 'Ge', 'Cu', 'Au', 'Ni', 'Cd', 'In', 'Mn', 'Zr', 'Cr', - 'Pt', 'Hg', 'Pb', 'Unknown']) + - one_of_k_encoding(atom.GetDegree(), [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) + - one_of_k_encoding_unk(atom.GetTotalNumHs(), [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) + - one_of_k_encoding_unk(atom.GetImplicitValence(), [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) + - [atom.GetIsAromatic()]) - - -def one_of_k_encoding(x, allowable_set): - if x not in allowable_set: - raise Exception("input {0} not in allowable set{1}:".format(x, allowable_set)) - return list(map(lambda s: x == s, allowable_set)) - - -def one_of_k_encoding_unk(x, allowable_set): - """Maps inputs not in the allowable set to the last element.""" - if x not in allowable_set: - x = allowable_set[-1] - return list(map(lambda s: x == s, allowable_set)) - - -def smile_to_graph(smile): - mol = Chem.MolFromSmiles(smile) - - c_size = mol.GetNumAtoms() - - features = [] - for atom in mol.GetAtoms(): - feature = atom_features(atom) - features.append(feature / sum(feature)) - - edges = [] - for bond in mol.GetBonds(): - edges.append([bond.GetBeginAtomIdx(), bond.GetEndAtomIdx()]) - g = nx.Graph(edges).to_directed() - edge_index = [] - for e1, e2 in g.edges: - edge_index.append([e1, e2]) - - return c_size, features, edge_index - - - -def creat_data(datafile, cellfile): - file2 = cellfile - cell_features = [] - with open(file2) as csvfile: - csv_reader = csv.reader(csvfile) # 使用csv.reader读取csvfile中的文件 - for row in csv_reader: - cell_features.append(row) - cell_features = np.array(cell_features) - print('cell_features',len(cell_features), cell_features) - #exit() - compound_iso_smiles = [] - df = pd.read_csv('/data1/ruiyi/deepdds/data/smiles.csv') - compound_iso_smiles += list(df['smile']) - compound_iso_smiles = set(compound_iso_smiles) - smile_graph = {} - print('compound_iso_smiles', compound_iso_smiles) - for smile in compound_iso_smiles: - print('smiles', smile) - g = smile_to_graph(smile) - smile_graph[smile] = g - - datasets = datafile - # convert to PyTorch data format - processed_data_file_train = '/data1/ruiyi/deepdds/data/processed/' + datasets + '_train.pt' - - if ((not os.path.isfile(processed_data_file_train))): - df = pd.read_csv('/data1/ruiyi/deepdds/data/' + datasets + '.csv') - drug1, drug2, cell, label = list(df['drug1']), list(df['drug2']), list(df['cell']), list(df['label']) - print(cell) - - drug1, drug2, cell, label = np.asarray(drug1), np.asarray(drug2), np.asarray(cell), np.asarray(label) - # make data PyTorch Geometric ready - - print('开始创建数据') - TestbedDataset(root='/data1/ruiyi/deepdds/data', dataset=datafile + '_drug1_combined_{}'.format(ckpt), xd=drug1, xt=cell, xt_featrue=cell_features, y=label,smile_graph=smile_graph) - TestbedDataset(root='/data1/ruiyi/deepdds/data', dataset=datafile + '_drug2_combined_{}'.format(ckpt), xd=drug2, xt=cell, xt_featrue=cell_features, y=label,smile_graph=smile_graph) - print('创建数据成功') - print('preparing ', datasets + '_.pt in pytorch format!') - # - # print(processed_data_file_train, ' have been created') - # - # else: - # print(processed_data_file_train, ' are already created') - -if __name__ == "__main__": - # datafile = 'prostate' - - if ind: - cellfile = '/data1/ruiyi/deepdds/data/independent_set/independent_cell_features_954.csv' - da = ['independent_set/independent_input'] - else: - cellfile = '/data1/ruiyi/deepdds/data/new_cell_features_954.csv' - da = ['new_labels_0_10'] - - for datafile in da: - creat_data(datafile, cellfile) +from _reference_paths import ASSET_DIR, artifact_path, data_path, output_dir, output_path, required_artifact +import csv +from itertools import islice + +import pandas as pd +import numpy as np +import os +import json, pickle +from collections import OrderedDict +from rdkit import Chem +# from rdkit.Chem import MolFromSmiles +import networkx as nx +from utils_test import * + + + +def get_cell_feature(cellId, cell_features): + for row in islice(cell_features, 0, None): + if row[0] == cellId: + return row[1: ] + +def atom_features(atom): + return np.array(one_of_k_encoding_unk(atom.GetSymbol(), + ['C', 'N', 'O', 'S', 'F', 'Si', 'P', 'Cl', 'Br', 'Mg', 'Na', 'Ca', 'Fe', 'As', + 'Al', 'I', 'B', 'V', 'K', 'Tl', 'Yb', 'Sb', 'Sn', 'Ag', 'Pd', 'Co', 'Se', + 'Ti', 'Zn', 'H', 'Li', 'Ge', 'Cu', 'Au', 'Ni', 'Cd', 'In', 'Mn', 'Zr', 'Cr', + 'Pt', 'Hg', 'Pb', 'Unknown']) + + one_of_k_encoding(atom.GetDegree(), [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) + + one_of_k_encoding_unk(atom.GetTotalNumHs(), [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) + + one_of_k_encoding_unk(atom.GetImplicitValence(), [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) + + [atom.GetIsAromatic()]) + + +def one_of_k_encoding(x, allowable_set): + if x not in allowable_set: + raise Exception("input {0} not in allowable set{1}:".format(x, allowable_set)) + return list(map(lambda s: x == s, allowable_set)) + + +def one_of_k_encoding_unk(x, allowable_set): + """Maps inputs not in the allowable set to the last element.""" + if x not in allowable_set: + x = allowable_set[-1] + return list(map(lambda s: x == s, allowable_set)) + + +def smile_to_graph(smile): + mol = Chem.MolFromSmiles(smile) + + c_size = mol.GetNumAtoms() + + features = [] + for atom in mol.GetAtoms(): + feature = atom_features(atom) + features.append(feature / sum(feature)) + + edges = [] + for bond in mol.GetBonds(): + edges.append([bond.GetBeginAtomIdx(), bond.GetEndAtomIdx()]) + g = nx.Graph(edges).to_directed() + edge_index = [] + for e1, e2 in g.edges: + edge_index.append([e1, e2]) + + return c_size, features, edge_index + + + +def creat_data(datafile, cellfile): + file2 = cellfile + cell_features = [] + with open(file2) as csvfile: + csv_reader = csv.reader(csvfile) # Read CSV rows with csv.reader. + for row in csv_reader: + cell_features.append(row) + cell_features = np.array(cell_features) + print('cell_features',len(cell_features), cell_features) + #exit() + compound_iso_smiles = [] + df = pd.read_csv(data_path('smiles.csv')) + compound_iso_smiles += list(df['smile']) + compound_iso_smiles = set(compound_iso_smiles) + smile_graph = {} + print('compound_iso_smiles', compound_iso_smiles) + for smile in compound_iso_smiles: + print('smiles', smile) + g = smile_to_graph(smile) + smile_graph[smile] = g + + datasets = datafile + # convert to PyTorch data format + processed_data_file_train = output_dir("cache/processed") + datasets + '_train.pt' + + if ((not os.path.isfile(processed_data_file_train))): + df = pd.read_csv(data_path(datasets + '.csv')) + drug1, drug2, cell, label = list(df['drug1']), list(df['drug2']), list(df['cell']), list(df['label']) + print(cell) + + drug1, drug2, cell, label = np.asarray(drug1), np.asarray(drug2), np.asarray(cell), np.asarray(label) + # make data PyTorch Geometric ready + + print('Creating datasets') + TestbedDataset(root=artifact_path("cache"), dataset=datafile + '_drug1_combined_{}'.format(ckpt), xd=drug1, xt=cell, xt_featrue=cell_features, y=label,smile_graph=smile_graph) + TestbedDataset(root=artifact_path("cache"), dataset=datafile + '_drug2_combined_{}'.format(ckpt), xd=drug2, xt=cell, xt_featrue=cell_features, y=label,smile_graph=smile_graph) + print('Datasets created') + print('preparing ', datasets + '_.pt in pytorch format!') + # + # print(processed_data_file_train, ' have been created') + # + # else: + # print(processed_data_file_train, ' are already created') + +if __name__ == "__main__": + # datafile = 'prostate' + + if ind: + cellfile = data_path('independent_set/independent_cell_features_954.csv') + da = ['independent_set/independent_input'] + else: + cellfile = data_path('new_cell_features_954.csv') + da = ['new_labels_0_10'] + + for datafile in da: + creat_data(datafile, cellfile) diff --git a/data/downstream/final_data/drug_response_pytorch/drug_combination/data_summary/README.md b/data/downstream/final_data/drug_response_pytorch/drug_combination/data_summary/README.md index a97eec301cc0e97f7d9179a21f05278a45b562ba..93ce42754d5c0c1450d465e9b4c89750e1cb246a 100644 --- a/data/downstream/final_data/drug_response_pytorch/drug_combination/data_summary/README.md +++ b/data/downstream/final_data/drug_response_pytorch/drug_combination/data_summary/README.md @@ -1,109 +1,108 @@ -# DeepDDS 核心数据文件说明 +# DeepDDS core data files -## 数据来源 +The provenance and counts below are translated from the existing dataset notes. +This path/comment audit did not recompute counts or validate the source studies. -- **药物协同数据**:来自 O'Neil et al. (2016) 的高通量药物组合筛选实验,涵盖 38 种抗癌药物在 39 个人类癌症细胞系上的两两组合。 -- **协同评分**:基于 4×4 剂量-响应矩阵,使用 Combenefit 工具计算 Loewe Additivity Score。同一药物组合若有多个重复实验,取平均值作为最终协同评分。 -- **细胞系表达数据**:来自 CCLE (Cancer Cell Line Encyclopedia) 2018年发布的 RNA-seq TPM 数据 (`CCLE_RNAseq_rsem_transcripts_tpm_20180929`),为细胞系在**未经药物处理、正常培养条件**下的基线基因表达谱。 +## Data sources -## 标签定义(二值化阈值) +- **Drug synergy:** the O'Neil et al. (2016) high-throughput screen of pairwise combinations of 38 anticancer drugs across 39 human cancer cell lines. +- **Synergy scores:** Loewe Additivity Scores computed with Combenefit from 4 x 4 dose-response matrices. Replicate measurements for the same combination were averaged. +- **Cell-line expression:** CCLE RNA-seq TPM data released in 2018 (`CCLE_RNAseq_rsem_transcripts_tpm_20180929`), describing baseline expression under standard culture conditions without drug treatment. -| 条件 | 标签 | 含义 | -|------|------|------| -| Loewe score > 10 | 1 (正样本) | 协同 (synergistic) | -| Loewe score < 0 | 0 (负样本) | 拮抗 (antagonistic) | -| 0 ≤ score ≤ 10 | 排除 | 模糊区间,不参与建模 | +## Binary labels -使用严格阈值(>10 而非 >0)的目的是:(1) 减少噪声附近样本的干扰;(2) 获得正负样本基本平衡的数据集。 +| Condition | Label | Meaning | +|---|---|---| +| Loewe score > 10 | 1 | Synergistic | +| Loewe score < 0 | 0 | Antagonistic | +| 0 <= score <= 10 | Excluded | Ambiguous range excluded from modeling | -## 最终数据规模 +The original notes describe the stricter positive threshold as intended to reduce near-zero noise and obtain roughly balanced classes. -- **12,415** 个唯一的 drug pair–cell line 组合 -- **36** 种抗癌药物 -- **31** 个人类癌症细胞系 -- 正样本 6,116 个,负样本 6,299 个 +## Previously recorded dataset size ---- +- 12,415 unique drug-pair/cell-line combinations. +- 36 anticancer drugs and 31 human cancer cell lines. +- 6,116 positive and 6,299 negative samples. -## 文件清单 +## File descriptions -### 1. new_labels_0_10.csv — 二值化标签文件(模型训练核心输入) +### `new_labels_0_10.csv`: binary training labels -- **行数**:12,415 条记录 + 1 行表头 -- **列**: - | 列名 | 类型 | 说明 | - |------|------|------| - | drug1 | string | 药物 A 的 SMILES 分子式 | - | drug2 | string | 药物 B 的 SMILES 分子式 | - | cell | string | 细胞系名称(如 A375, HCT116) | - | label | int | 0 = 拮抗 (score < 0),1 = 协同 (score > 10) | +The recorded layout is 12,415 records plus one header row. -### 2. score_0_10.csv — 连续协同评分文件 +| Column | Type | Description | +|---|---|---| +| drug1 | string | SMILES representation of drug A | +| drug2 | string | SMILES representation of drug B | +| cell | string | Cell-line name, such as A375 or HCT116 | +| label | int | 0 = antagonistic (score < 0); 1 = synergistic (score > 10) | -- **行数**:12,415 条记录 + 1 行表头(与 new_labels_0_10.csv 一一对应) -- **列**: - | 列名 | 类型 | 说明 | - |------|------|------| - | drug1 | string | 药物 A 名称(如 5-FU) | - | drug2 | string | 药物 B 名称(如 ABT-888) | - | cell | string | 细胞系名称 | - | score | float | Loewe Additivity 连续协同评分 | - | nor | float | 归一化后的评分 | - | index | int | 样本索引 | - | true_label | int | 真实二值标签 | - | predict_label | int | 模型预测标签(作者预训练结果) | - | predict_score | float | 模型预测概率(作者预训练结果) | - | smile1 | string | 药物 A 的 SMILES | - | smile2 | string | 药物 B 的 SMILES | +### `score_0_10.csv`: continuous synergy scores -### 3. new_cell_features_954.csv — 细胞系基因表达矩阵 +The existing notes record 12,415 records plus a header, corresponding to `new_labels_0_10.csv`. -- **维度**:31 个细胞系 × 954 个基因 -- **数据来源**:CCLE RNA-seq baseline TPM(未经药物处理的基线表达) -- **格式**: - - 第 1 行:gene_id(ENSG 编号) - - 第 2 行:transcript_id(ENST 编号) - - 第 3~33 行:31 个细胞系的表达值,第 1 列为细胞系名称 -- **用途**:作为模型中细胞系特征的输入,经全连接网络编码后与药物图特征融合 +| Column | Type | Description | +|---|---|---| +| drug1 | string | Drug A name, such as 5-FU | +| drug2 | string | Drug B name, such as ABT-888 | +| cell | string | Cell-line name | +| score | float | Continuous Loewe Additivity Score | +| nor | float | Normalized score | +| index | int | Sample index | +| true_label | int | Binary ground-truth label | +| predict_label | int | Legacy predicted label | +| predict_score | float | Legacy prediction probability | +| smile1 | string | Drug A SMILES | +| smile2 | string | Drug B SMILES | -### 4. cell_gene954.csv — 954 基因 ID 对照表 +The stored prediction columns are historical reference results, not results from the current BioMod checkout. -- **行数**:954 条记录 + 1 行表头 -- **列**:gene_id (ENSG), transcript_id (ENST) -- **用途**:记录 new_cell_features_954.csv 中 954 个基因的 Ensembl 编号 +### `new_cell_features_954.csv`: cell-line expression -### 5. smiles_0_10.csv — 药物 SMILES 对照表 +- Recorded dimensions: 31 cell lines x 954 genes. +- Source: CCLE baseline RNA-seq TPM, without drug treatment. +- Row 1: gene IDs (ENSG). +- Row 2: transcript IDs (ENST). +- Rows 3-33: expression values, with the cell-line name in the first column. +- Purpose: cell features encoded by fully connected layers and combined with drug graph features. -- **行数**:36 条(无表头) -- **列**:药物名称, SMILES 分子式 -- **用途**:药物名称与 SMILES 的映射关系,SMILES 用于构建分子图作为 GNN 输入 +### `cell_gene954.csv`: gene identifier table -### 6. drugs_0_10.csv — 药物信息表 +- Recorded layout: 954 records plus a header. +- Columns: `gene_id` (ENSG), `transcript_id` (ENST). +- Purpose: identify the genes in `new_cell_features_954.csv`. -- **行数**:36 条(无表头) -- **列**:药物名称, SMILES 分子式, 数值编码 -- **用途**:药物列表及其编码索引 +### `smiles_0_10.csv`: drug SMILES table -### 7. cell_line_0_10.csv — 细胞系信息表 +- The legacy notes record 36 headerless rows; counts may differ from the parser-based summary in the parent directory. +- Columns: drug name and SMILES representation. +- Purpose: map drug names to SMILES used for molecular-graph construction. -- **行数**:31 条(无表头) -- **列**:细胞系名称, 癌症类型(如 melanoma, ovarian, colon) -- **用途**:记录 31 个细胞系对应的癌症类型 +### `drugs_0_10.csv`: drug table ---- +- Recorded layout: 36 headerless rows. +- Columns: drug name, SMILES, numeric encoding. +- Purpose: enumerate drugs and their encoded indices. -## DeepDDS 模型输入结构 +### `cell_line_0_10.csv`: cell-line table -``` -Drug A (SMILES) ──→ 分子图 ──→ GAT/GCN ──→ Drug A embedding ─┐ - ├──→ 拼接 ──→ 全连接层 ──→ 协同/拮抗预测 -Drug B (SMILES) ──→ 分子图 ──→ GAT/GCN ──→ Drug B embedding ─┤ - │ -Cell line (954 genes TPM) ──→ FC layers ──→ Cell embedding ────┘ +- Recorded layout: 31 headerless rows. +- Columns: cell-line name and cancer type, such as melanoma, ovarian, or colon. +- Purpose: associate cell lines with cancer types. + +## DeepDDS inputs + +```text +Drug A SMILES -> Molecular graph -> GAT/GCN -> Drug A embedding --+ + | +Drug B SMILES -> Molecular graph -> GAT/GCN -> Drug B embedding --+-> Concatenate -> FC layers -> Synergy prediction + | +Cell line (954 genes, TPM) -> FC layers -> Cell embedding -------+ ``` -## 参考文献 +## References retained from the original notes - Wang et al. "DeepDDS: deep graph neural network with attention mechanism to predict synergistic drug combinations." Briefings in Bioinformatics, 2022. - O'Neil et al. "An unbiased oncology compound screen to identify novel combination strategies." Molecular Cancer Therapeutics, 2016. -- CCLE: Cancer Cell Line Encyclopedia (Broad Institute) +- CCLE: Cancer Cell Line Encyclopedia, Broad Institute. diff --git a/data/downstream/final_data/drug_response_pytorch/drug_combination/data_summary/test_data/README.md b/data/downstream/final_data/drug_response_pytorch/drug_combination/data_summary/test_data/README.md index b40f7e69e13f14bf07b0de3ee5e45b192629d5eb..539329f79d9c1c7220989f269780a6bfd03244aa 100644 --- a/data/downstream/final_data/drug_response_pytorch/drug_combination/data_summary/test_data/README.md +++ b/data/downstream/final_data/drug_response_pytorch/drug_combination/data_summary/test_data/README.md @@ -1,62 +1,54 @@ -# DeepDDS 独立测试集(AstraZeneca-Sanger DREAM Challenge) +# DeepDDS independent test data -## 数据来源 +## Source -来自 AstraZeneca-Sanger Drug Combination DREAM Challenge 数据集(Menden et al., Nature Communications, 2019)。 -原始数据文件:`41467_2019_9799_MOESM3_ESM.xlsx` +The original notes attribute these data to the AstraZeneca-Sanger Drug Combination DREAM Challenge (Menden et al., Nature Communications, 2019), with source supplement `41467_2019_9799_MOESM3_ESM.xlsx`. -该数据集与训练集(O'Neil et al. 2016)完全独立,用于评估模型的泛化能力。 +This collection is intended for evaluation against the separate O'Neil et al. (2016) training collection. The path/comment audit did not verify overlap, counts, or the original source study; those checks are required before making claims about benchmark independence. -## 数据规模 +## Previously recorded size -- **668** 个 drug pair–cell line 组合 -- **52** 种药物(部分为训练集中未见过的新药) -- **24** 个细胞系(部分为训练集中未见过的新细胞系) -- 正样本 480 个,负样本 188 个 +- 668 drug-pair/cell-line combinations. +- 52 drugs and 24 cell lines, including entries described in the original notes as absent from training. +- 480 positive and 188 negative samples. ---- +## Core inputs -## 文件清单 +| File | Purpose | +|---|---| +| `independent_input.csv` | Formatted model inputs; recorded as 668 rows with drug1 (SMILES), drug2 (SMILES), cell, and label | +| `independent_cell_features_954.csv` | CCLE baseline expression for 954 genes; recorded as 1,019 cell lines, including lines outside the training collection | -### 核心输入文件 +## Source and intermediate files -| 文件 | 说明 | -|------|------| -| `independent_input.csv` | 最终格式化的模型输入文件(668 条),列:drug1(SMILES), drug2(SMILES), cell, label | -| `independent_cell_features_954.csv` | 细胞系 954 基因 CCLE 基线表达矩阵(1019 个细胞系,含训练集外的新细胞系) | +| File | Purpose | +|---|---| +| `41467_2019_9799_MOESM3_ESM.xlsx` | Original publication supplement with the full AZ-DREAM data | +| `independent_allin.csv` | Extracted combinations; recorded as 76,607 rows with drug1, drug2, and cell | +| `independent_allin_smiles.csv` | Extracted combinations with SMILES annotations | +| `independent_set_combinatioin.csv` | Combination details, including synergy scores, IC50, Hill slope, and Einf | +| `independent_set_newcombinatioin.csv` | Filtered combinations described in the original notes as excluding overlap with training | -### 原始数据与中间处理文件 +## Supporting metadata -| 文件 | 说明 | -|------|------| -| `41467_2019_9799_MOESM3_ESM.xlsx` | 原始论文附件(AZ-DREAM Challenge 全量数据) | -| `independent_allin.csv` | 从原始数据提取的全量组合(76,607 条),列:drug1, drug2, cell | -| `independent_allin_smiles.csv` | 附加 SMILES 信息的全量组合 | -| `independent_set_combinatioin.csv` | 原始组合详情,含 synergy score、IC50、Hill slope、Einf 等药理学参数 | -| `independent_set_newcombinatioin.csv` | 筛选后的组合(去除与训练集重叠的部分) | +| File | Purpose | +|---|---| +| `independent_set_drug.csv` | Drug names, targets, pathways, SMILES, molecular weights, LogP, and Lipinski descriptors | +| `independent_set_cell_line.csv` | Metadata recorded for 144 cell lines: COSMIC IDs, tissue, TCGA labels, MSI status, and AZ-DREAM/O'Neil/ALMANAC membership | +| `independent_drug.csv` | Filtered drugs available for testing | +| `independent_cell.csv` | Filtered cell lines available for testing | +| `independent_input_drug_cell.csv` | Drug-to-cell-line mapping for the formatted inputs | -### 辅助信息文件 +## Drug-pair subsets -| 文件 | 说明 | -|------|------| -| `independent_set_drug.csv` | 药物信息表(药物名、靶点、通路、SMILES、分子量、LogP、Lipinski 等) | -| `independent_set_cell_line.csv` | 144 个细胞系信息(COSMIC ID、组织类型、TCGA 标签、MSI 状态、是否在 AZ-DREAM/O'Neil/ALMANAC 中) | -| `independent_drug.csv` | 筛选后可用于测试的药物列表 | -| `independent_cell.csv` | 筛选后可用于测试的细胞系列表 | -| `independent_input_drug_cell.csv` | 输入数据中的药物-细胞系映射关系 | +| File | Drug pair | +|---|---| +| `AZD2014_6244.csv` | AZD2014 + AZD6244 | +| `AZD8931-AZD5363.csv` | AZD8931 + AZD5363 | +| `Afatinib-MK2206.csv` | Afatinib + MK2206 | +| `GDC0941-AZD6244.csv` | GDC0941 + AZD6244 | +| `GDC0941-MK2206.csv` | GDC0941 + MK2206 | -### 子集文件(按药物组合拆分) - -| 文件 | 说明 | -|------|------| -| `AZD2014_6244.csv` | AZD2014 + AZD6244 组合的子集 | -| `AZD8931-AZD5363.csv` | AZD8931 + AZD5363 组合的子集 | -| `Afatinib-MK2206.csv` | Afatinib + MK2206 组合的子集 | -| `GDC0941-AZD6244.csv` | GDC0941 + AZD6244 组合的子集 | -| `GDC0941-MK2206.csv` | GDC0941 + MK2206 组合的子集 | - ---- - -## 参考文献 +## Reference retained from the original notes - Menden et al. "Community assessment to advance computational prediction of cancer drug combinations in a pharmacogenomic screen." Nature Communications, 2019. diff --git a/data/downstream/final_data/drug_response_pytorch/drug_combination/heatmap.py b/data/downstream/final_data/drug_response_pytorch/drug_combination/heatmap.py index bd86158925630b219dcc5f6c0371b8fceabb94b0..bdd078c5348c5dd8f5b025805ba8686b1b847ed2 100644 --- a/data/downstream/final_data/drug_response_pytorch/drug_combination/heatmap.py +++ b/data/downstream/final_data/drug_response_pytorch/drug_combination/heatmap.py @@ -1,113 +1,114 @@ -import matplotlib.pyplot as plt -import numpy as np -import csv -import numpy as np -from numpy import random -import pandas as pd - -import matplotlib.pyplot as plt - -import seaborn as sns - - -def get_map(AZD6244_weight, file): - confusion = AZD6244_weight - # print(confusion.shape) - # 热度图,后面是指定的颜色块,可设置其他的不同颜色 - plt.figure(figsize=(10, 8), dpi=1000) - - plt.imshow(confusion, cmap=plt.cm.Blues) - # ticks 坐标轴的坐标点 - # label 坐标轴标签说明 - indices = range(len(confusion[0])) - # print(indices) - # 第一个是迭代对象,表示坐标的显示顺序,第二个参数是坐标轴显示列表 - #plt.xticks(indices, [0, 1, 2]) - #plt.yticks(indices, [0, 1, 2]) - idx = range(len(confusion[0])) - # print(idx) - plt.xticks(indices, idx, fontsize=25) - plt.yticks(indices, idx, fontsize=25) - - plt.colorbar() - - font = {'family' : 'Times New Roman', - 'weight' : 'normal', - 'size' : 30, - } - - plt.xlabel('Predict', font) - plt.ylabel('True', font) - plt.title(file, font) - - # plt.rcParams两行是用于解决标签不能显示汉字的问题 - plt.rcParams['font.sans-serif']=['SimHei'] - plt.rcParams['axes.unicode_minus'] = False - - # 显示数据 - # for first_index in range(len(confusion)): #第几行 - # for second_index in range(len(confusion[first_index])): #第几列 - # plt.text(first_index, second_index, confusion[second_index][first_index], fontsize=30) - # 在matlab里面可以对矩阵直接imagesc(confusion) - # 显示 - plt.savefig('fig/' + file + '.png') - plt.close() - # plt.show() -if __name__ == '__main__': - - tem = np.array(([[126, 62], [289, 191]])) - get_map(tem, 'DeepSynergy') - - # AZD6244_weight = [] - # csvFile = open('data/result/AZD2014_begin_x1.csv', 'r') - # reader = csv.reader(csvFile) - # for it in reader: - # for num in range(len(it)): - # it[num] = round(float(it[num]), 3) - # AZD6244_weight.append(it) - # AZD6244_weight = np.array(AZD6244_weight) - # - # AZD2014_weight = [] - # csvFile = open('data/result/AZD2014_fin_x1.csv', 'r') - # reader = csv.reader(csvFile) - # for it in reader: - # for num in range(len(it)): - # it[num] = round(float(it[num]), 3) - # AZD2014_weight.append(it) - # AZD2014_weight = np.array(AZD2014_weight) - # - # print(AZD6244_weight) - # print(AZD2014_weight) - # - # dt = pd.DataFrame(AZD6244_weight.T) - # dt = np.array(dt.corr(method='pearson')) # 默认为'pearson'检验,可选'kendall','spearman' - # dt = np.around(dt, decimals=2) - # get_map(dt, '1') - # - # dt = pd.DataFrame(AZD2014_weight.T) - # dt = np.array(dt.corr(method='pearson')) # 默认为'pearson'检验,可选'kendall','spearman' - # dt = np.around(dt, decimals=2) - # get_map(dt, '2') - - # datafile = 'data/independent_set/independent_cell.csv' - # independent_cell = pd.read_csv(datafile) - # independent_cell = np.array(independent_cell)[:, 0] - # print(independent_cell) - # - # datafile = 'data/independent_set/independent_drug.csv' - # independent_drug = pd.read_csv(datafile) - # independent_drug = np.array(independent_drug)[:, 0] - # print(independent_drug) - # - # lc = len(independent_cell) - # ld = len(independent_drug) - # - # tem = [] - # for i in range(ld): - # for j in range(ld): - # for k in range(lc): - # if i != j: - # tem.append([independent_drug[i], independent_drug[j], independent_cell[k]]) - # - # df = pd.DataFrame(tem) - # df.to_csv('data/independent_set/independent_allin.csv', header=None, index=None) +from _reference_paths import ASSET_DIR, artifact_path, data_path, output_dir, output_path, required_artifact +import matplotlib.pyplot as plt +import numpy as np +import csv +import numpy as np +from numpy import random +import pandas as pd + +import matplotlib.pyplot as plt + +import seaborn as sns + + +def get_map(AZD6244_weight, file): + confusion = AZD6244_weight + # print(confusion.shape) + # Draw a heatmap using the selected color map. + plt.figure(figsize=(10, 8), dpi=1000) + + plt.imshow(confusion, cmap=plt.cm.Blues) + # ticks: axis tick positions. + # label: axis tick labels. + indices = range(len(confusion[0])) + # print(indices) + # The first argument sets tick positions; the second supplies their labels. + #plt.xticks(indices, [0, 1, 2]) + #plt.yticks(indices, [0, 1, 2]) + idx = range(len(confusion[0])) + # print(idx) + plt.xticks(indices, idx, fontsize=25) + plt.yticks(indices, idx, fontsize=25) + + plt.colorbar() + + font = {'family' : 'Times New Roman', + 'weight' : 'normal', + 'size' : 30, + } + + plt.xlabel('Predict', font) + plt.ylabel('True', font) + plt.title(file, font) + + # These rcParams settings support Chinese glyphs in labels. + plt.rcParams['font.sans-serif']=['SimHei'] + plt.rcParams['axes.unicode_minus'] = False + + # Display values. + # for first_index in range(len(confusion)): #Row index. + # for second_index in range(len(confusion[first_index])): #Column index. + # plt.text(first_index, second_index, confusion[second_index][first_index], fontsize=30) + # MATLAB offers imagesc(confusion) for the same matrix view. + # Display. + plt.savefig(output_dir("fig") + file + '.png') + plt.close() + # plt.show() +if __name__ == '__main__': + + tem = np.array(([[126, 62], [289, 191]])) + get_map(tem, 'DeepSynergy') + + # AZD6244_weight = [] + # csvFile = open('data/result/AZD2014_begin_x1.csv', 'r') + # reader = csv.reader(csvFile) + # for it in reader: + # for num in range(len(it)): + # it[num] = round(float(it[num]), 3) + # AZD6244_weight.append(it) + # AZD6244_weight = np.array(AZD6244_weight) + # + # AZD2014_weight = [] + # csvFile = open('data/result/AZD2014_fin_x1.csv', 'r') + # reader = csv.reader(csvFile) + # for it in reader: + # for num in range(len(it)): + # it[num] = round(float(it[num]), 3) + # AZD2014_weight.append(it) + # AZD2014_weight = np.array(AZD2014_weight) + # + # print(AZD6244_weight) + # print(AZD2014_weight) + # + # dt = pd.DataFrame(AZD6244_weight.T) + # dt = np.array(dt.corr(method='pearson')) # Use Pearson correlation; Kendall and Spearman are alternatives. + # dt = np.around(dt, decimals=2) + # get_map(dt, '1') + # + # dt = pd.DataFrame(AZD2014_weight.T) + # dt = np.array(dt.corr(method='pearson')) # Use Pearson correlation; Kendall and Spearman are alternatives. + # dt = np.around(dt, decimals=2) + # get_map(dt, '2') + + # datafile = 'data/independent_set/independent_cell.csv' + # independent_cell = pd.read_csv(datafile) + # independent_cell = np.array(independent_cell)[:, 0] + # print(independent_cell) + # + # datafile = 'data/independent_set/independent_drug.csv' + # independent_drug = pd.read_csv(datafile) + # independent_drug = np.array(independent_drug)[:, 0] + # print(independent_drug) + # + # lc = len(independent_cell) + # ld = len(independent_drug) + # + # tem = [] + # for i in range(ld): + # for j in range(ld): + # for k in range(lc): + # if i != j: + # tem.append([independent_drug[i], independent_drug[j], independent_cell[k]]) + # + # df = pd.DataFrame(tem) + # df.to_csv('data/independent_set/independent_allin.csv', header=None, index=None) diff --git a/data/downstream/final_data/drug_response_pytorch/drug_combination/load_data.py b/data/downstream/final_data/drug_response_pytorch/drug_combination/load_data.py index d6112a3c737db65828320ab61a69d42de0ac1a12..be132877fe751c186e0815a58a49f9a34dc92db8 100644 --- a/data/downstream/final_data/drug_response_pytorch/drug_combination/load_data.py +++ b/data/downstream/final_data/drug_response_pytorch/drug_combination/load_data.py @@ -1,128 +1,129 @@ -import csv -from itertools import islice - -import numpy as np -import pandas as pd -from rdkit import Chem -import networkx as nx - - - -def atom_features(atom): - return np.array(one_of_k_encoding_unk(atom.GetSymbol(), - ['C', 'N', 'O', 'S', 'F', 'Si', 'P', 'Cl', 'Br', 'Mg', 'Na', 'Ca', 'Fe', 'As', - 'Al', 'I', 'B', 'V', 'K', 'Tl', 'Yb', 'Sb', 'Sn', 'Ag', 'Pd', 'Co', 'Se', - 'Ti', 'Zn', 'H', 'Li', 'Ge', 'Cu', 'Au', 'Ni', 'Cd', 'In', 'Mn', 'Zr', 'Cr', - 'Pt', 'Hg', 'Pb', 'Unknown']) + - one_of_k_encoding(atom.GetDegree(), [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) + - one_of_k_encoding_unk(atom.GetTotalNumHs(), [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) + - one_of_k_encoding_unk(atom.GetImplicitValence(), [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) + - [atom.GetIsAromatic()]) - - -def one_of_k_encoding(x, allowable_set): - if x not in allowable_set: - raise Exception("input {0} not in allowable set{1}:".format(x, allowable_set)) - return list(map(lambda s: x == s, allowable_set)) - - -def one_of_k_encoding_unk(x, allowable_set): - """Maps inputs not in the allowable set to the last element.""" - if x not in allowable_set: - x = allowable_set[-1] - return list(map(lambda s: x == s, allowable_set)) - - -def smile_to_graph(smile): - mol = Chem.MolFromSmiles(smile) - - c_size = mol.GetNumAtoms() - - features = [] - for atom in mol.GetAtoms(): - feature = atom_features(atom) - features.append(feature / sum(feature)) - - edges = [] - for bond in mol.GetBonds(): - edges.append([bond.GetBeginAtomIdx(), bond.GetEndAtomIdx()]) - g = nx.Graph(edges).to_directed() - edge_index = [] - for e1, e2 in g.edges: - edge_index.append([e1, e2]) - - return c_size, features, edge_index - - -# get drug -compound_iso_smiles = [] -df = pd.read_csv('data/smiles.csv') -compound_iso_smiles += list(df['smile']) -compound_iso_smiles = set(compound_iso_smiles) -smile_graph = {} -for smile in compound_iso_smiles: - g = smile_to_graph(smile) - smile_graph[smile] = g - - -# get cell -cell_featrues = [] -cell_file = 'data/independent_set/independent_cell_features_954.csv' -with open(cell_file) as csvfile: - csv_reader = csv.reader(csvfile) # 使用csv.reader读取csvfile中的文件 - for row in csv_reader: - cell_featrues.append(row) -cell_featrues = np.array(cell_featrues) - - - - -def get_cell_feature(cellId, cell_features): - for row in islice(cell_features, 0, None): - if cellId in row[0]: - return row[1:] - - - - -def load(datafile): - train_featrues = [] - train_labels = [] - i = 1 - with open(datafile, 'r') as f: - next(f) - - reader = csv.reader(f) - for row in reader: - c_size, drug1, edge_index = smile_graph[row[0]] - drug1 = np.sum(drug1, axis=0) - - c_size, drug2, edge_index = smile_graph[row[1]] - drug2 = np.sum(drug2, axis=0) - - drug_con = np.hstack((drug1, drug2)) - cell_featrue = np.array(get_cell_feature(row[2], cell_featrues)) - - feature = np.hstack((drug_con, cell_featrue)) - temp = [] - for i in feature: - temp.append(float(i)) - train_featrues.append(temp) - train_labels.append(int(row[3])) - - featrues, labels = np.array(train_featrues), np.array(train_labels) - print(featrues.shape) - print(labels.shape) - return featrues, labels - - -if __name__ == "__main__": - a, b = load('data/new_labels_0_10.csv') - print(a.shape) - print(b.shape) - txtDF = pd.DataFrame(data=a) - txtDF.to_csv('data/new_labels_0_10_features.csv', index=False, header=False) - - txtDF = pd.DataFrame(data=b) - txtDF.to_csv('data/new_labels_0_10_labels.csv', index=False, header=False) - +from _reference_paths import ASSET_DIR, artifact_path, data_path, output_dir, output_path, required_artifact +import csv +from itertools import islice + +import numpy as np +import pandas as pd +from rdkit import Chem +import networkx as nx + + + +def atom_features(atom): + return np.array(one_of_k_encoding_unk(atom.GetSymbol(), + ['C', 'N', 'O', 'S', 'F', 'Si', 'P', 'Cl', 'Br', 'Mg', 'Na', 'Ca', 'Fe', 'As', + 'Al', 'I', 'B', 'V', 'K', 'Tl', 'Yb', 'Sb', 'Sn', 'Ag', 'Pd', 'Co', 'Se', + 'Ti', 'Zn', 'H', 'Li', 'Ge', 'Cu', 'Au', 'Ni', 'Cd', 'In', 'Mn', 'Zr', 'Cr', + 'Pt', 'Hg', 'Pb', 'Unknown']) + + one_of_k_encoding(atom.GetDegree(), [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) + + one_of_k_encoding_unk(atom.GetTotalNumHs(), [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) + + one_of_k_encoding_unk(atom.GetImplicitValence(), [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) + + [atom.GetIsAromatic()]) + + +def one_of_k_encoding(x, allowable_set): + if x not in allowable_set: + raise Exception("input {0} not in allowable set{1}:".format(x, allowable_set)) + return list(map(lambda s: x == s, allowable_set)) + + +def one_of_k_encoding_unk(x, allowable_set): + """Maps inputs not in the allowable set to the last element.""" + if x not in allowable_set: + x = allowable_set[-1] + return list(map(lambda s: x == s, allowable_set)) + + +def smile_to_graph(smile): + mol = Chem.MolFromSmiles(smile) + + c_size = mol.GetNumAtoms() + + features = [] + for atom in mol.GetAtoms(): + feature = atom_features(atom) + features.append(feature / sum(feature)) + + edges = [] + for bond in mol.GetBonds(): + edges.append([bond.GetBeginAtomIdx(), bond.GetEndAtomIdx()]) + g = nx.Graph(edges).to_directed() + edge_index = [] + for e1, e2 in g.edges: + edge_index.append([e1, e2]) + + return c_size, features, edge_index + + +# get drug +compound_iso_smiles = [] +df = pd.read_csv(data_path('smiles.csv')) +compound_iso_smiles += list(df['smile']) +compound_iso_smiles = set(compound_iso_smiles) +smile_graph = {} +for smile in compound_iso_smiles: + g = smile_to_graph(smile) + smile_graph[smile] = g + + +# get cell +cell_featrues = [] +cell_file = data_path('independent_set/independent_cell_features_954.csv') +with open(cell_file) as csvfile: + csv_reader = csv.reader(csvfile) # Read CSV rows with csv.reader. + for row in csv_reader: + cell_featrues.append(row) +cell_featrues = np.array(cell_featrues) + + + + +def get_cell_feature(cellId, cell_features): + for row in islice(cell_features, 0, None): + if cellId in row[0]: + return row[1:] + + + + +def load(datafile): + train_featrues = [] + train_labels = [] + i = 1 + with open(datafile, 'r') as f: + next(f) + + reader = csv.reader(f) + for row in reader: + c_size, drug1, edge_index = smile_graph[row[0]] + drug1 = np.sum(drug1, axis=0) + + c_size, drug2, edge_index = smile_graph[row[1]] + drug2 = np.sum(drug2, axis=0) + + drug_con = np.hstack((drug1, drug2)) + cell_featrue = np.array(get_cell_feature(row[2], cell_featrues)) + + feature = np.hstack((drug_con, cell_featrue)) + temp = [] + for i in feature: + temp.append(float(i)) + train_featrues.append(temp) + train_labels.append(int(row[3])) + + featrues, labels = np.array(train_featrues), np.array(train_labels) + print(featrues.shape) + print(labels.shape) + return featrues, labels + + +if __name__ == "__main__": + a, b = load(data_path('new_labels_0_10.csv')) + print(a.shape) + print(b.shape) + txtDF = pd.DataFrame(data=a) + txtDF.to_csv(output_path('new_labels_0_10_features.csv'), index=False, header=False) + + txtDF = pd.DataFrame(data=b) + txtDF.to_csv(output_path('new_labels_0_10_labels.csv'), index=False, header=False) + diff --git a/data/downstream/final_data/drug_response_pytorch/drug_combination/models/gat.py b/data/downstream/final_data/drug_response_pytorch/drug_combination/models/gat.py index 1cd376d0c61504f6d894bfbcd0c853f19bb75cfb..55457fd439adf4258890d02ab8d9f4ab92b6804a 100644 --- a/data/downstream/final_data/drug_response_pytorch/drug_combination/models/gat.py +++ b/data/downstream/final_data/drug_response_pytorch/drug_combination/models/gat.py @@ -1,187 +1,191 @@ -import torch -import torch.nn as nn -import torch.nn.functional as F -from torch.nn import Sequential, Linear, ReLU -from torch_geometric.nn import GATConv -from torch_geometric.nn import global_max_pool as gmp -import pandas as pd -import numpy as np -from heatmap import get_map - - - -# GAT model -class GATNet(torch.nn.Module): - def __init__(self, num_features_xd=78, n_output=2, num_features_xt=954, output_dim=128, dropout=0.2, file=None): - super(GATNet, self).__init__() - - # graph drug layers - self.drug1_gcn1 = GATConv(num_features_xd, output_dim, heads=10, dropout=dropout) - self.drug1_gcn2 = GATConv(output_dim * 10, output_dim, dropout=dropout) - # self.drug1_gcn3 = GATConv(output_dim, output_dim, dropout=dropout) - self.drug1_fc_g1 = nn.Linear(output_dim, output_dim) - # self.drug1_fc_g2 = nn.Linear(2048, output_dim) - self.filename = file - - - # DL cell featrues - self.reduction = nn.Sequential( - nn.Linear(num_features_xt, 2048), - nn.ReLU(), - nn.Dropout(dropout), - nn.Linear(2048, 512), - nn.ReLU(), - nn.Dropout(dropout), - nn.Linear(512, output_dim * 2), - nn.ReLU() - ) - - # 1D convolution on protein sequence - # self.embedding_xt = nn.Embedding(num_features_xt + 1, embed_dim) - # self.conv_xt1 = nn.Conv1d(in_channels=1000, out_channels=n_filters, kernel_size=8) - # self.fc_xt1 = nn.Linear(32*121, output_dim) - - # combined layers - self.fc1 = nn.Linear(output_dim * 4, 2048) - self.fc2 = nn.Linear(2048, 512) - self.fc3 = nn.Linear(512, 128) - self.out = nn.Linear(128, n_output) - - # activation and regularization - self.relu = nn.ReLU() - self.dropout = nn.Dropout(dropout) - self.output_dim = output_dim - - def get_col_index(self, x): - row_size = len(x[:, 0]) - row = np.zeros(row_size) - col_size = len(x[0, :]) - for i in range(col_size): - row[np.argmax(x[:, i])] += 1 - return row - - def save_num(self, d, path): - d = d.cpu().numpy() - ind = self.get_col_index(d) - ind = pd.DataFrame(ind) - ind.to_csv('data/case_study/' + path + '_index.csv', header=0, index=0) - # 下面是load操作 - # read_dictionary = np.load('my_file.npy').item() - # d = pd.DataFrame(d) - # d.to_csv('data/result/' + path + '.csv', header=0, index=0) - - def forward(self, data1, data2): - x1, edge_index1, batch1, cell = data1.x, data1.edge_index, data1.batch, data1.cell - - x2, edge_index2, batch2 = data2.x, data2.edge_index, data2.batch - - # deal drug1 - # begin_x1 = np.array(x1.cpu().detach().numpy()) - x1, arr = self.drug1_gcn1(x1, edge_index1) - x1 = F.elu(x1) - x1 = F.dropout(x1, p=0.2, training=self.training) - x1, arr = self.drug1_gcn2(x1, edge_index1) - x1 = F.elu(x1) - x1 = F.dropout(x1, p=0.2, training=self.training) - # fin_x1 = np.array(x1.cpu().detach().numpy()) - - - # if len(batch1) <1000: - # dt = pd.DataFrame(begin_x1.T) - # # dt = pd.DataFrame(begin_x1) - # # dt.to_csv('data/case_study/begin_fin/' + self.filename + '_drug1_begin_x1.csv', header=None, index=None) - # # dt = pd.DataFrame(fin_x1) - # # dt.to_csv('data/case_study/begin_fin/' + self.filename + '_drug1_fin_x1.csv', header=None, index=None) - # - # # begin_x1 = np.array(dt.corr(method='pearson')) # 默认为'pearson'检验,可选'kendall','spearman' - # # begin_x1 = np.around(begin_x1, decimals=2) - # # # dt.to_csv('data/result/begin_x1.csv', index=None, header=None) - # # get_map(begin_x1, 'data/case_study/' + self.filename + '_drug1_begin_x1') - # # - # # dt = pd.DataFrame(fin_x1.T) - # # fin_x1 = np.array(dt.corr(method='pearson')) # 默认为'pearson'检验,可选'kendall','spearman' - # # fin_x1 = np.around(fin_x1, decimals=2) - # # # dt.to_csv('data/result/fin_x1.csv', index=None, header=None) - # # get_map(fin_x1, 'data/case_study/' + self.filename + '_drug1_fin_x1') - # - # dt = pd.DataFrame(arr) - # dt = np.around(dt, decimals=2) - # get_map(dt, 'data/case_study/' + self.filename + '_drug1_att_x1') - # - # p = self.filename + '_drug1' - # self.save_num(x1, p) - # - x1 = gmp(x1, batch1) # global max pooling - - - x1 = self.drug1_fc_g1(x1) - x1 = self.relu(x1) - # x1 = self.drug1_fc_g2(x1) - # x1 = self.relu(x1) - - # deal drug2 - # begin_x2 = np.array(x2.cpu().detach().numpy()) - x2, arr = self.drug1_gcn1(x2, edge_index2) - x2 = F.elu(x2) - x2 = F.dropout(x2, p=0.2, training=self.training) - x2, arr = self.drug1_gcn2(x2, edge_index2) - x2 = F.elu(x2) - x2 = F.dropout(x2, p=0.2, training=self.training) - # fin_x2 = np.array(x2.cpu().detach().numpy()) - - - # if len(batch1) < 1000: - # dt = pd.DataFrame(begin_x2) - # # dt.to_csv('data/case_study/begin_fin/' + self.filename + '_drug1_begin_x2.csv', header=None, index=None) - # # dt = pd.DataFrame(fin_x2) - # # dt.to_csv('data/case_study/begin_fin/' + self.filename + '_drug1_fin_x2.csv', header=None, index=None) - # - # # dt = pd.DataFrame(begin_x2.T) - # # begin_x2 = np.array(dt.corr(method='pearson')) # 默认为'pearson'检验,可选'kendall','spearman' - # # begin_x2 = np.around(begin_x2, decimals=2) - # # # dt.to_csv('data/result/begin_x1.csv', index=None, header=None) - # # get_map(begin_x2, 'data/case_study/' + self.filename + '_drug2_begin_x2') - # # - # # dt = pd.DataFrame(fin_x2.T) - # # fin_x2 = np.array(dt.corr(method='pearson')) # 默认为'pearson'检验,可选'kendall','spearman' - # # fin_x2 = np.around(fin_x2, decimals=2) - # # # dt.to_csv('data/result/fin_x1.csv', index=None, header=None) - # # get_map(fin_x2, 'data/case_study/' + self.filename + '_drug2_fin_x2') - # - # dt = pd.DataFrame(arr) - # dt = np.around(dt, decimals=2) - # get_map(dt, 'data/case_study/' + self.filename + '_drug2_att_x1') - # - # p = self.filename + '_drug2' - # self.save_num(x2, p) - - - x2 = gmp(x2, batch2) # global max pooling - - - x2 = self.drug1_fc_g1(x2) - x2 = self.relu(x2) - # x2 = self.drug1_fc_g2(x2) - # x2 = self.relu(x2) - - # deal cell - cell = F.normalize(cell, 2, 1) - cell_vector = self.reduction(cell) - - - - # concat - xc = torch.cat((x1, x2, cell_vector), 1) - xc = F.normalize(xc, 2, 1) - # add some dense layers - xc = self.fc1(xc) - xc = self.relu(xc) - xc = self.dropout(xc) - xc = self.fc2(xc) - xc = self.relu(xc) - xc = self.dropout(xc) - xc = self.fc3(xc) - xc = self.relu(xc) - xc = self.dropout(xc) - out = self.out(xc) - return out +import sys +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from _reference_paths import ASSET_DIR, artifact_path, data_path, output_dir, output_path, required_artifact +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.nn import Sequential, Linear, ReLU +from torch_geometric.nn import GATConv +from torch_geometric.nn import global_max_pool as gmp +import pandas as pd +import numpy as np +from heatmap import get_map + + + +# GAT model +class GATNet(torch.nn.Module): + def __init__(self, num_features_xd=78, n_output=2, num_features_xt=954, output_dim=128, dropout=0.2, file=None): + super(GATNet, self).__init__() + + # graph drug layers + self.drug1_gcn1 = GATConv(num_features_xd, output_dim, heads=10, dropout=dropout) + self.drug1_gcn2 = GATConv(output_dim * 10, output_dim, dropout=dropout) + # self.drug1_gcn3 = GATConv(output_dim, output_dim, dropout=dropout) + self.drug1_fc_g1 = nn.Linear(output_dim, output_dim) + # self.drug1_fc_g2 = nn.Linear(2048, output_dim) + self.filename = file + + + # DL cell featrues + self.reduction = nn.Sequential( + nn.Linear(num_features_xt, 2048), + nn.ReLU(), + nn.Dropout(dropout), + nn.Linear(2048, 512), + nn.ReLU(), + nn.Dropout(dropout), + nn.Linear(512, output_dim * 2), + nn.ReLU() + ) + + # 1D convolution on protein sequence + # self.embedding_xt = nn.Embedding(num_features_xt + 1, embed_dim) + # self.conv_xt1 = nn.Conv1d(in_channels=1000, out_channels=n_filters, kernel_size=8) + # self.fc_xt1 = nn.Linear(32*121, output_dim) + + # combined layers + self.fc1 = nn.Linear(output_dim * 4, 2048) + self.fc2 = nn.Linear(2048, 512) + self.fc3 = nn.Linear(512, 128) + self.out = nn.Linear(128, n_output) + + # activation and regularization + self.relu = nn.ReLU() + self.dropout = nn.Dropout(dropout) + self.output_dim = output_dim + + def get_col_index(self, x): + row_size = len(x[:, 0]) + row = np.zeros(row_size) + col_size = len(x[0, :]) + for i in range(col_size): + row[np.argmax(x[:, i])] += 1 + return row + + def save_num(self, d, path): + d = d.cpu().numpy() + ind = self.get_col_index(d) + ind = pd.DataFrame(ind) + ind.to_csv(output_dir("case_study") + path + '_index.csv', header=0, index=0) + # Load the saved values. + # read_dictionary = np.load('my_file.npy').item() + # d = pd.DataFrame(d) + # d.to_csv('data/result/' + path + '.csv', header=0, index=0) + + def forward(self, data1, data2): + x1, edge_index1, batch1, cell = data1.x, data1.edge_index, data1.batch, data1.cell + + x2, edge_index2, batch2 = data2.x, data2.edge_index, data2.batch + + # deal drug1 + # begin_x1 = np.array(x1.cpu().detach().numpy()) + x1, arr = self.drug1_gcn1(x1, edge_index1) + x1 = F.elu(x1) + x1 = F.dropout(x1, p=0.2, training=self.training) + x1, arr = self.drug1_gcn2(x1, edge_index1) + x1 = F.elu(x1) + x1 = F.dropout(x1, p=0.2, training=self.training) + # fin_x1 = np.array(x1.cpu().detach().numpy()) + + + # if len(batch1) <1000: + # dt = pd.DataFrame(begin_x1.T) + # # dt = pd.DataFrame(begin_x1) + # # dt.to_csv('data/case_study/begin_fin/' + self.filename + '_drug1_begin_x1.csv', header=None, index=None) + # # dt = pd.DataFrame(fin_x1) + # # dt.to_csv('data/case_study/begin_fin/' + self.filename + '_drug1_fin_x1.csv', header=None, index=None) + # + # # begin_x1 = np.array(dt.corr(method='pearson')) # Use Pearson correlation; Kendall and Spearman are alternatives. + # # begin_x1 = np.around(begin_x1, decimals=2) + # # # dt.to_csv('data/result/begin_x1.csv', index=None, header=None) + # # get_map(begin_x1, 'data/case_study/' + self.filename + '_drug1_begin_x1') + # # + # # dt = pd.DataFrame(fin_x1.T) + # # fin_x1 = np.array(dt.corr(method='pearson')) # Use Pearson correlation; Kendall and Spearman are alternatives. + # # fin_x1 = np.around(fin_x1, decimals=2) + # # # dt.to_csv('data/result/fin_x1.csv', index=None, header=None) + # # get_map(fin_x1, 'data/case_study/' + self.filename + '_drug1_fin_x1') + # + # dt = pd.DataFrame(arr) + # dt = np.around(dt, decimals=2) + # get_map(dt, 'data/case_study/' + self.filename + '_drug1_att_x1') + # + # p = self.filename + '_drug1' + # self.save_num(x1, p) + # + x1 = gmp(x1, batch1) # global max pooling + + + x1 = self.drug1_fc_g1(x1) + x1 = self.relu(x1) + # x1 = self.drug1_fc_g2(x1) + # x1 = self.relu(x1) + + # deal drug2 + # begin_x2 = np.array(x2.cpu().detach().numpy()) + x2, arr = self.drug1_gcn1(x2, edge_index2) + x2 = F.elu(x2) + x2 = F.dropout(x2, p=0.2, training=self.training) + x2, arr = self.drug1_gcn2(x2, edge_index2) + x2 = F.elu(x2) + x2 = F.dropout(x2, p=0.2, training=self.training) + # fin_x2 = np.array(x2.cpu().detach().numpy()) + + + # if len(batch1) < 1000: + # dt = pd.DataFrame(begin_x2) + # # dt.to_csv('data/case_study/begin_fin/' + self.filename + '_drug1_begin_x2.csv', header=None, index=None) + # # dt = pd.DataFrame(fin_x2) + # # dt.to_csv('data/case_study/begin_fin/' + self.filename + '_drug1_fin_x2.csv', header=None, index=None) + # + # # dt = pd.DataFrame(begin_x2.T) + # # begin_x2 = np.array(dt.corr(method='pearson')) # Use Pearson correlation; Kendall and Spearman are alternatives. + # # begin_x2 = np.around(begin_x2, decimals=2) + # # # dt.to_csv('data/result/begin_x1.csv', index=None, header=None) + # # get_map(begin_x2, 'data/case_study/' + self.filename + '_drug2_begin_x2') + # # + # # dt = pd.DataFrame(fin_x2.T) + # # fin_x2 = np.array(dt.corr(method='pearson')) # Use Pearson correlation; Kendall and Spearman are alternatives. + # # fin_x2 = np.around(fin_x2, decimals=2) + # # # dt.to_csv('data/result/fin_x1.csv', index=None, header=None) + # # get_map(fin_x2, 'data/case_study/' + self.filename + '_drug2_fin_x2') + # + # dt = pd.DataFrame(arr) + # dt = np.around(dt, decimals=2) + # get_map(dt, 'data/case_study/' + self.filename + '_drug2_att_x1') + # + # p = self.filename + '_drug2' + # self.save_num(x2, p) + + + x2 = gmp(x2, batch2) # global max pooling + + + x2 = self.drug1_fc_g1(x2) + x2 = self.relu(x2) + # x2 = self.drug1_fc_g2(x2) + # x2 = self.relu(x2) + + # deal cell + cell = F.normalize(cell, 2, 1) + cell_vector = self.reduction(cell) + + + + # concat + xc = torch.cat((x1, x2, cell_vector), 1) + xc = F.normalize(xc, 2, 1) + # add some dense layers + xc = self.fc1(xc) + xc = self.relu(xc) + xc = self.dropout(xc) + xc = self.fc2(xc) + xc = self.relu(xc) + xc = self.dropout(xc) + xc = self.fc3(xc) + xc = self.relu(xc) + xc = self.dropout(xc) + out = self.out(xc) + return out diff --git a/data/downstream/final_data/drug_response_pytorch/drug_combination/models/gat_r.py b/data/downstream/final_data/drug_response_pytorch/drug_combination/models/gat_r.py index b9b68314af5f5903ca0f405722e23c8c02a74513..613a3ab22aa321447a07f15167ed8ad4017565c2 100644 --- a/data/downstream/final_data/drug_response_pytorch/drug_combination/models/gat_r.py +++ b/data/downstream/final_data/drug_response_pytorch/drug_combination/models/gat_r.py @@ -1,176 +1,180 @@ -import torch -import torch.nn as nn -import torch.nn.functional as F -from torch.nn import Sequential, Linear, ReLU -from torch_geometric.nn import GATConv -from torch_geometric.nn import global_max_pool as gmp -import pandas as pd -import numpy as np - - - -# GAT model -class GATNet(torch.nn.Module): - def __init__(self, num_features_xd=78, n_output=1, num_features_xt=954, output_dim=128, dropout=0.2, file=None): - super(GATNet, self).__init__() - - # graph drug layers - self.drug1_gcn1 = GATConv(num_features_xd, output_dim, heads=10, dropout=dropout) - self.drug1_gcn2 = GATConv(output_dim * 10, output_dim, dropout=dropout) - # self.drug1_gcn3 = GATConv(output_dim, output_dim, dropout=dropout) - self.drug1_fc_g1 = nn.Linear(output_dim, output_dim) - # self.drug1_fc_g2 = nn.Linear(2048, output_dim) - self.filename = file - - - # DL cell featrues - self.reduction = nn.Sequential( - nn.Linear(num_features_xt, 2048), - nn.ReLU(), - nn.Dropout(dropout), - nn.Linear(2048, 512), - nn.ReLU(), - nn.Dropout(dropout), - nn.Linear(512, output_dim * 2), - nn.ReLU() - ) - - # 1D convolution on protein sequence - # self.embedding_xt = nn.Embedding(num_features_xt + 1, embed_dim) - # self.conv_xt1 = nn.Conv1d(in_channels=1000, out_channels=n_filters, kernel_size=8) - # self.fc_xt1 = nn.Linear(32*121, output_dim) - - # combined layers - self.fc1 = nn.Linear(output_dim * 4, 2048) - self.fc2 = nn.Linear(2048, 512) - self.fc3 = nn.Linear(512, 256) - self.out = nn.Linear(256, n_output) - - # activation and regularization - self.relu = nn.ReLU() - self.tanh = nn.Tanh() - self.prelu = nn.PReLU(num_parameters=1, init=0.25) - self.dropout = nn.Dropout(dropout) - self.output_dim = output_dim - - def get_col_index(self, x): - row_size = len(x[:, 0]) - row = np.zeros(row_size) - col_size = len(x[0, :]) - for i in range(col_size): - row[np.argmax(x[:, i])] += 1 - return row - - def save_num(self, d, path): - d = d.cpu().numpy() - ind = self.get_col_index(d) - ind = pd.DataFrame(ind) - ind.to_csv('data/case_study/' + path + '_index.csv', header=0, index=0) - # 下面是load操作 - # read_dictionary = np.load('my_file.npy').item() - # d = pd.DataFrame(d) - # d.to_csv('data/result/' + path + '.csv', header=0, index=0) - - def forward(self, data1, data2): - x1, edge_index1, batch1, cell = data1.x, data1.edge_index, data1.batch, data1.cell - - x2, edge_index2, batch2 = data2.x, data2.edge_index, data2.batch - - # deal drug1 - # begin_x1 = np.array(x1.cpu().detach().numpy()) - x1, att = self.drug1_gcn1(x1, edge_index1) - x1 = F.elu(x1) - x1 = F.dropout(x1, p=0.2, training=self.training) - x1, att = self.drug1_gcn2(x1, edge_index1) - x1 = F.elu(x1) - x1 = F.dropout(x1, p=0.2, training=self.training) - # fin_x1 = np.array(x1.cpu().detach().numpy()) - - - # if len(batch1) <1000: - # dt = pd.DataFrame(begin_x1.T) - # begin_x1 = np.array(dt.corr(method='pearson')) # 默认为'pearson'检验,可选'kendall','spearman' - # begin_x1 = np.around(begin_x1, decimals=2) - # # dt.to_csv('data/result/begin_x1.csv', index=None, header=None) - # get_map(begin_x1, 'data/case_study/' + self.filename + '_drug1_begin_x1') - # - # dt = pd.DataFrame(fin_x1.T) - # fin_x1 = np.array(dt.corr(method='pearson')) # 默认为'pearson'检验,可选'kendall','spearman' - # fin_x1 = np.around(fin_x1, decimals=2) - # # dt.to_csv('data/result/fin_x1.csv', index=None, header=None) - # get_map(fin_x1, 'data/case_study/' + self.filename + '_drug1_fin_x1') - # - # dt = pd.DataFrame(arr) - # get_map(dt, 'data/case_study/' + self.filename + '_drug1_att_x1') - # - # p = self.filename + '_drug1' - # self.save_num(x1, p) - - x1 = gmp(x1, batch1) # global max pooling - - - x1 = self.drug1_fc_g1(x1) - x1 = self.relu(x1) - # x1 = self.drug1_fc_g2(x1) - # x1 = self.relu(x1) - - # deal drug2 - # begin_x2 = np.array(x2.cpu().detach().numpy()) - x2, att = self.drug1_gcn1(x2, edge_index2) - x2 = F.elu(x2) - x2 = F.dropout(x2, p=0.2, training=self.training) - x2, att = self.drug1_gcn2(x2, edge_index2) - x2 = F.elu(x2) - x2 = F.dropout(x2, p=0.2, training=self.training) - # fin_x2 = np.array(x2.cpu().detach().numpy()) - - - # if len(batch1) < 1000: - # dt = pd.DataFrame(begin_x2.T) - # begin_x2 = np.array(dt.corr(method='pearson')) # 默认为'pearson'检验,可选'kendall','spearman' - # begin_x2 = np.around(begin_x2, decimals=2) - # # dt.to_csv('data/result/begin_x1.csv', index=None, header=None) - # get_map(begin_x2, 'data/case_study/' + self.filename + '_drug2_begin_x2') - # - # dt = pd.DataFrame(fin_x2.T) - # fin_x2 = np.array(dt.corr(method='pearson')) # 默认为'pearson'检验,可选'kendall','spearman' - # fin_x2 = np.around(fin_x2, decimals=2) - # # dt.to_csv('data/result/fin_x1.csv', index=None, header=None) - # get_map(fin_x2, 'data/case_study/' + self.filename + '_drug2_fin_x2') - # - # dt = pd.DataFrame(arr) - # get_map(dt, 'data/case_study/' + self.filename + '_drug2_att_x1') - # - # p = self.filename + '_drug2' - # self.save_num(x2, p) - - - x2 = gmp(x2, batch2) # global max pooling - - - x2 = self.drug1_fc_g1(x2) - x2 = self.relu(x2) - # x2 = self.drug1_fc_g2(x2) - # x2 = self.relu(x2) - - # deal cell - # cell = F.normalize(cell, 2, 1) - cell_vector = self.reduction(cell) - - - - # concat - xc = torch.cat((x1, x2, cell_vector), 1) - xc = F.normalize(xc, 2, 1) - # add some dense layers - xc = self.fc1(xc) - xc = self.relu(xc) - xc = self.dropout(xc) - xc = self.fc2(xc) - xc = self.relu(xc) - xc = self.dropout(xc) - xc = self.fc3(xc) - xc = self.relu(xc) - xc = self.dropout(xc) - out = self.out(xc) - return out +import sys +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from _reference_paths import ASSET_DIR, artifact_path, data_path, output_dir, output_path, required_artifact +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.nn import Sequential, Linear, ReLU +from torch_geometric.nn import GATConv +from torch_geometric.nn import global_max_pool as gmp +import pandas as pd +import numpy as np + + + +# GAT model +class GATNet(torch.nn.Module): + def __init__(self, num_features_xd=78, n_output=1, num_features_xt=954, output_dim=128, dropout=0.2, file=None): + super(GATNet, self).__init__() + + # graph drug layers + self.drug1_gcn1 = GATConv(num_features_xd, output_dim, heads=10, dropout=dropout) + self.drug1_gcn2 = GATConv(output_dim * 10, output_dim, dropout=dropout) + # self.drug1_gcn3 = GATConv(output_dim, output_dim, dropout=dropout) + self.drug1_fc_g1 = nn.Linear(output_dim, output_dim) + # self.drug1_fc_g2 = nn.Linear(2048, output_dim) + self.filename = file + + + # DL cell featrues + self.reduction = nn.Sequential( + nn.Linear(num_features_xt, 2048), + nn.ReLU(), + nn.Dropout(dropout), + nn.Linear(2048, 512), + nn.ReLU(), + nn.Dropout(dropout), + nn.Linear(512, output_dim * 2), + nn.ReLU() + ) + + # 1D convolution on protein sequence + # self.embedding_xt = nn.Embedding(num_features_xt + 1, embed_dim) + # self.conv_xt1 = nn.Conv1d(in_channels=1000, out_channels=n_filters, kernel_size=8) + # self.fc_xt1 = nn.Linear(32*121, output_dim) + + # combined layers + self.fc1 = nn.Linear(output_dim * 4, 2048) + self.fc2 = nn.Linear(2048, 512) + self.fc3 = nn.Linear(512, 256) + self.out = nn.Linear(256, n_output) + + # activation and regularization + self.relu = nn.ReLU() + self.tanh = nn.Tanh() + self.prelu = nn.PReLU(num_parameters=1, init=0.25) + self.dropout = nn.Dropout(dropout) + self.output_dim = output_dim + + def get_col_index(self, x): + row_size = len(x[:, 0]) + row = np.zeros(row_size) + col_size = len(x[0, :]) + for i in range(col_size): + row[np.argmax(x[:, i])] += 1 + return row + + def save_num(self, d, path): + d = d.cpu().numpy() + ind = self.get_col_index(d) + ind = pd.DataFrame(ind) + ind.to_csv(output_dir("case_study") + path + '_index.csv', header=0, index=0) + # Load the saved values. + # read_dictionary = np.load('my_file.npy').item() + # d = pd.DataFrame(d) + # d.to_csv('data/result/' + path + '.csv', header=0, index=0) + + def forward(self, data1, data2): + x1, edge_index1, batch1, cell = data1.x, data1.edge_index, data1.batch, data1.cell + + x2, edge_index2, batch2 = data2.x, data2.edge_index, data2.batch + + # deal drug1 + # begin_x1 = np.array(x1.cpu().detach().numpy()) + x1, att = self.drug1_gcn1(x1, edge_index1) + x1 = F.elu(x1) + x1 = F.dropout(x1, p=0.2, training=self.training) + x1, att = self.drug1_gcn2(x1, edge_index1) + x1 = F.elu(x1) + x1 = F.dropout(x1, p=0.2, training=self.training) + # fin_x1 = np.array(x1.cpu().detach().numpy()) + + + # if len(batch1) <1000: + # dt = pd.DataFrame(begin_x1.T) + # begin_x1 = np.array(dt.corr(method='pearson')) # Use Pearson correlation; Kendall and Spearman are alternatives. + # begin_x1 = np.around(begin_x1, decimals=2) + # # dt.to_csv('data/result/begin_x1.csv', index=None, header=None) + # get_map(begin_x1, 'data/case_study/' + self.filename + '_drug1_begin_x1') + # + # dt = pd.DataFrame(fin_x1.T) + # fin_x1 = np.array(dt.corr(method='pearson')) # Use Pearson correlation; Kendall and Spearman are alternatives. + # fin_x1 = np.around(fin_x1, decimals=2) + # # dt.to_csv('data/result/fin_x1.csv', index=None, header=None) + # get_map(fin_x1, 'data/case_study/' + self.filename + '_drug1_fin_x1') + # + # dt = pd.DataFrame(arr) + # get_map(dt, 'data/case_study/' + self.filename + '_drug1_att_x1') + # + # p = self.filename + '_drug1' + # self.save_num(x1, p) + + x1 = gmp(x1, batch1) # global max pooling + + + x1 = self.drug1_fc_g1(x1) + x1 = self.relu(x1) + # x1 = self.drug1_fc_g2(x1) + # x1 = self.relu(x1) + + # deal drug2 + # begin_x2 = np.array(x2.cpu().detach().numpy()) + x2, att = self.drug1_gcn1(x2, edge_index2) + x2 = F.elu(x2) + x2 = F.dropout(x2, p=0.2, training=self.training) + x2, att = self.drug1_gcn2(x2, edge_index2) + x2 = F.elu(x2) + x2 = F.dropout(x2, p=0.2, training=self.training) + # fin_x2 = np.array(x2.cpu().detach().numpy()) + + + # if len(batch1) < 1000: + # dt = pd.DataFrame(begin_x2.T) + # begin_x2 = np.array(dt.corr(method='pearson')) # Use Pearson correlation; Kendall and Spearman are alternatives. + # begin_x2 = np.around(begin_x2, decimals=2) + # # dt.to_csv('data/result/begin_x1.csv', index=None, header=None) + # get_map(begin_x2, 'data/case_study/' + self.filename + '_drug2_begin_x2') + # + # dt = pd.DataFrame(fin_x2.T) + # fin_x2 = np.array(dt.corr(method='pearson')) # Use Pearson correlation; Kendall and Spearman are alternatives. + # fin_x2 = np.around(fin_x2, decimals=2) + # # dt.to_csv('data/result/fin_x1.csv', index=None, header=None) + # get_map(fin_x2, 'data/case_study/' + self.filename + '_drug2_fin_x2') + # + # dt = pd.DataFrame(arr) + # get_map(dt, 'data/case_study/' + self.filename + '_drug2_att_x1') + # + # p = self.filename + '_drug2' + # self.save_num(x2, p) + + + x2 = gmp(x2, batch2) # global max pooling + + + x2 = self.drug1_fc_g1(x2) + x2 = self.relu(x2) + # x2 = self.drug1_fc_g2(x2) + # x2 = self.relu(x2) + + # deal cell + # cell = F.normalize(cell, 2, 1) + cell_vector = self.reduction(cell) + + + + # concat + xc = torch.cat((x1, x2, cell_vector), 1) + xc = F.normalize(xc, 2, 1) + # add some dense layers + xc = self.fc1(xc) + xc = self.relu(xc) + xc = self.dropout(xc) + xc = self.fc2(xc) + xc = self.relu(xc) + xc = self.dropout(xc) + xc = self.fc3(xc) + xc = self.relu(xc) + xc = self.dropout(xc) + out = self.out(xc) + return out diff --git a/data/downstream/final_data/drug_response_pytorch/drug_combination/models/training_GAT_r.py b/data/downstream/final_data/drug_response_pytorch/drug_combination/models/training_GAT_r.py index 7318e8415e8536a07c1265143fa3c88f7b5c58a3..d230d4b2e8fce8d4bb4c70436c5b09140915f48c 100644 --- a/data/downstream/final_data/drug_response_pytorch/drug_combination/models/training_GAT_r.py +++ b/data/downstream/final_data/drug_response_pytorch/drug_combination/models/training_GAT_r.py @@ -1,197 +1,201 @@ -import random -import torch.nn.functional as F -import torch.nn as nn -from models.gat_r import GATNet -from models.gat_gcn_test import GAT_GCN -from models.gcn import GCNNet -from models.ginconv import GINConvNet -import math -from utils_test import * -from sklearn.metrics import roc_curve, confusion_matrix -from sklearn.metrics import cohen_kappa_score, accuracy_score, roc_auc_score, precision_score, recall_score, balanced_accuracy_score -from sklearn import metrics -import pandas as pd - - - -# training function at each epoch -def train(model, device, drug1_loader_train, drug2_loader_train, optimizer, epoch): - print('Training on {} samples...'.format(len(drug1_loader_train.dataset))) - model.train() - # train_loader = np.array(train_loader) - for batch_idx, data in enumerate(zip(drug1_loader_train, drug2_loader_train)): - data1 = data[0] - data2 = data[1] - data1 = data1.to(device) - data2 = data2.to(device) - y = data[0].y.view(-1, 1).float().to(device) - y = y.squeeze(1) - output = model(data1, data2) - loss = loss_fn(output, y.unsqueeze(1)) - # print('loss', loss) - optimizer.zero_grad() - loss.backward() - optimizer.step() - if batch_idx % LOG_INTERVAL == 0: - print('Train epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(epoch, - batch_idx * len(data1.x), - len(drug1_loader_train.dataset), - 100. * batch_idx / len(drug1_loader_train), - loss.item())) - - -def predicting(model, device, drug1_loader_test, drug2_loader_test): - model.eval() - total_preds = torch.Tensor() - total_labels = torch.Tensor() - total_prelabels = torch.Tensor() - print('Make prediction for {} samples...'.format(len(drug1_loader_test.dataset))) - with torch.no_grad(): - for data in zip(drug1_loader_test, drug2_loader_test): - data1 = data[0] - data2 = data[1] - data1 = data1.to(device) - data2 = data2.to(device) - output = model(data1, data2) - ys = output.to('cpu').data.numpy() - # predicted_labels = list(map(lambda x: np.argmax(x), ys)) - total_preds = torch.cat((total_preds, torch.Tensor(ys)), 0) - # total_prelabels = torch.cat((total_prelabels, torch.Tensor(predicted_labels)), 0) - total_labels = torch.cat((total_labels, data1.y.view(-1, 1).cpu()), 0) - return total_preds.numpy().flatten(), total_labels.numpy().flatten() - - -def shuffle_dataset(dataset, seed): - np.random.seed(seed) - np.random.shuffle(dataset) - return dataset - - -def split_dataset(dataset, ratio): - n = int(len(dataset) * ratio) - dataset_1, dataset_2 = dataset[:n], dataset[n:] - return dataset_1, dataset_2 - -def compute_mae_mse_rmse(target,prediction): - error = [] - for i in range(len(target)): - error.append(target[i] - prediction[i]) - squaredError = [] - absError = [] - for val in error: - squaredError.append(val * val) # target-prediction之差平方 - absError.append(abs(val)) # 误差绝对值 - mae=sum(absError)/len(absError) # 平均绝对误差MAE - mse=sum(squaredError)/len(squaredError) # 均方误差MSE - RMSE=sum(absError)/len(absError) - return mae,mse,RMSE - -def compute_rsquared(X, Y): - xBar = np.mean(X) - yBar = np.mean(Y) - SSR = 0 - varX = 0 - varY = 0 - for i in range(0, len(X)): - diffXXBar = X[i] - xBar - diffYYBar = Y[i] - yBar - SSR += (diffXXBar * diffYYBar) - varX += diffXXBar ** 2 - varY += diffYYBar ** 2 - - SST = math.sqrt(varX * varY) - r2=round((SSR / SST) ** 2,3) - return r2 - -# modeling = [GINConvNet, GATNet, GAT_GCN, GCNNet][int(sys.argv[2])] -# datasets = [['davis', 'kiba'][int(sys.argv[1])]] -# model_st = modeling.__name__ -modeling = GATNet - -TRAIN_BATCH_SIZE = 128 -TEST_BATCH_SIZE = 128 -LR = 0.0001 -LOG_INTERVAL = 20 -NUM_EPOCHS = 2000 - -print('Learning rate: ', LR) -print('Epochs: ', NUM_EPOCHS) -datafile = 'score_0_10' - -# CPU or GPU - -if torch.cuda.is_available(): - device = torch.device('cuda') - print('The code uses GPU...') -else: - device = torch.device('cpu') - print('The code uses CPU!!!') - -drug1_data = TestbedDataset(root='data', dataset=datafile + '_drug1') -drug2_data = TestbedDataset(root='data', dataset=datafile + '_drug2') - -lenth = len(drug1_data) -pot = int(lenth/5) -print('lenth', lenth) -print('pot', pot) - - -random_num = random.sample(range(0, lenth), lenth) -for i in range(5): - test_num = random_num[pot*i:pot*(i+1)] - train_num = random_num[:pot*i] + random_num[pot*(i+1):] - - drug1_data_train = drug1_data[train_num] - drug1_data_test = drug1_data[test_num] - drug1_loader_train = DataLoader(drug1_data_train, batch_size=TRAIN_BATCH_SIZE, shuffle=None) - drug1_loader_test = DataLoader(drug1_data_test, batch_size=TRAIN_BATCH_SIZE, shuffle=None) - - - drug2_data_test = drug2_data[test_num] - drug2_data_train = drug2_data[train_num] - drug2_loader_train = DataLoader(drug2_data_train, batch_size=TRAIN_BATCH_SIZE, shuffle=None) - drug2_loader_test = DataLoader(drug2_data_test, batch_size=TRAIN_BATCH_SIZE, shuffle=None) - - model = modeling().to(device) - loss_fn = nn.MSELoss() - optimizer = torch.optim.Adam(model.parameters(), lr=LR) - - - # model_file_name = 'data/result/GATNet(DrugA_DrugB)' + str(i) + '--model_' + datafile + '.model' - result_file_name = 'data/result/GATNet(DrugA_DrugB)_R' + str(i) + '--result_' + datafile + '.csv' - file_AUCs = 'data/result/GATNet(DrugA_DrugB)_R' + str(i) + '--AUCs--' + datafile + '.txt' - AUCs = ('Epoch\tepoch\tmse\tmae\trmse\tr2\tP') - with open(file_AUCs, 'w') as f: - f.write(AUCs + '\n') - - best_auc = 10000000000000 - for epoch in range(NUM_EPOCHS): - train(model, device, drug1_loader_train, drug2_loader_train, optimizer, epoch + 1) - S, T = predicting(model, device, drug1_loader_test, drug2_loader_test) - # T is correct score - # S is predict score - # Y is predict label - print(S) - print(T) - - # compute preformence - - mae, mse, rmse = compute_mae_mse_rmse(S, T) - - # save data - - if best_auc > mse: - r2 = compute_rsquared(S, T) - best_auc = mse - AUCs = [epoch, mse, mae, rmse, r2, math.sqrt(r2)] - save_AUCs(AUCs, file_AUCs) - # torch.save(model.state_dict(), model_file_name) - independent_num = [] - # independent_num.append(test_num) - independent_num.append(T) - # independent_num.append(Y) - independent_num.append(S) - txtDF = pd.DataFrame(data=independent_num) - txtDF.to_csv(result_file_name, index=False, header=False) - - print('best_mse', best_auc) +import sys +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from _reference_paths import ASSET_DIR, artifact_path, data_path, output_dir, output_path, required_artifact +import random +import torch.nn.functional as F +import torch.nn as nn +from models.gat_r import GATNet +from models.gat_gcn_test import GAT_GCN +from models.gcn import GCNNet +from models.ginconv import GINConvNet +import math +from utils_test import * +from sklearn.metrics import roc_curve, confusion_matrix +from sklearn.metrics import cohen_kappa_score, accuracy_score, roc_auc_score, precision_score, recall_score, balanced_accuracy_score +from sklearn import metrics +import pandas as pd + + + +# training function at each epoch +def train(model, device, drug1_loader_train, drug2_loader_train, optimizer, epoch): + print('Training on {} samples...'.format(len(drug1_loader_train.dataset))) + model.train() + # train_loader = np.array(train_loader) + for batch_idx, data in enumerate(zip(drug1_loader_train, drug2_loader_train)): + data1 = data[0] + data2 = data[1] + data1 = data1.to(device) + data2 = data2.to(device) + y = data[0].y.view(-1, 1).float().to(device) + y = y.squeeze(1) + output = model(data1, data2) + loss = loss_fn(output, y.unsqueeze(1)) + # print('loss', loss) + optimizer.zero_grad() + loss.backward() + optimizer.step() + if batch_idx % LOG_INTERVAL == 0: + print('Train epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(epoch, + batch_idx * len(data1.x), + len(drug1_loader_train.dataset), + 100. * batch_idx / len(drug1_loader_train), + loss.item())) + + +def predicting(model, device, drug1_loader_test, drug2_loader_test): + model.eval() + total_preds = torch.Tensor() + total_labels = torch.Tensor() + total_prelabels = torch.Tensor() + print('Make prediction for {} samples...'.format(len(drug1_loader_test.dataset))) + with torch.no_grad(): + for data in zip(drug1_loader_test, drug2_loader_test): + data1 = data[0] + data2 = data[1] + data1 = data1.to(device) + data2 = data2.to(device) + output = model(data1, data2) + ys = output.to('cpu').data.numpy() + # predicted_labels = list(map(lambda x: np.argmax(x), ys)) + total_preds = torch.cat((total_preds, torch.Tensor(ys)), 0) + # total_prelabels = torch.cat((total_prelabels, torch.Tensor(predicted_labels)), 0) + total_labels = torch.cat((total_labels, data1.y.view(-1, 1).cpu()), 0) + return total_preds.numpy().flatten(), total_labels.numpy().flatten() + + +def shuffle_dataset(dataset, seed): + np.random.seed(seed) + np.random.shuffle(dataset) + return dataset + + +def split_dataset(dataset, ratio): + n = int(len(dataset) * ratio) + dataset_1, dataset_2 = dataset[:n], dataset[n:] + return dataset_1, dataset_2 + +def compute_mae_mse_rmse(target,prediction): + error = [] + for i in range(len(target)): + error.append(target[i] - prediction[i]) + squaredError = [] + absError = [] + for val in error: + squaredError.append(val * val) # Squared target-prediction error. + absError.append(abs(val)) # Absolute error. + mae=sum(absError)/len(absError) # Mean absolute error (MAE). + mse=sum(squaredError)/len(squaredError) # Mean squared error (MSE). + RMSE=sum(absError)/len(absError) + return mae,mse,RMSE + +def compute_rsquared(X, Y): + xBar = np.mean(X) + yBar = np.mean(Y) + SSR = 0 + varX = 0 + varY = 0 + for i in range(0, len(X)): + diffXXBar = X[i] - xBar + diffYYBar = Y[i] - yBar + SSR += (diffXXBar * diffYYBar) + varX += diffXXBar ** 2 + varY += diffYYBar ** 2 + + SST = math.sqrt(varX * varY) + r2=round((SSR / SST) ** 2,3) + return r2 + +# modeling = [GINConvNet, GATNet, GAT_GCN, GCNNet][int(sys.argv[2])] +# datasets = [['davis', 'kiba'][int(sys.argv[1])]] +# model_st = modeling.__name__ +modeling = GATNet + +TRAIN_BATCH_SIZE = 128 +TEST_BATCH_SIZE = 128 +LR = 0.0001 +LOG_INTERVAL = 20 +NUM_EPOCHS = 2000 + +print('Learning rate: ', LR) +print('Epochs: ', NUM_EPOCHS) +datafile = 'score_0_10' + +# CPU or GPU + +if torch.cuda.is_available(): + device = torch.device('cuda') + print('The code uses GPU...') +else: + device = torch.device('cpu') + print('The code uses CPU!!!') + +drug1_data = TestbedDataset(root=artifact_path("cache"), dataset=datafile + '_drug1') +drug2_data = TestbedDataset(root=artifact_path("cache"), dataset=datafile + '_drug2') + +lenth = len(drug1_data) +pot = int(lenth/5) +print('lenth', lenth) +print('pot', pot) + + +random_num = random.sample(range(0, lenth), lenth) +for i in range(5): + test_num = random_num[pot*i:pot*(i+1)] + train_num = random_num[:pot*i] + random_num[pot*(i+1):] + + drug1_data_train = drug1_data[train_num] + drug1_data_test = drug1_data[test_num] + drug1_loader_train = DataLoader(drug1_data_train, batch_size=TRAIN_BATCH_SIZE, shuffle=None) + drug1_loader_test = DataLoader(drug1_data_test, batch_size=TRAIN_BATCH_SIZE, shuffle=None) + + + drug2_data_test = drug2_data[test_num] + drug2_data_train = drug2_data[train_num] + drug2_loader_train = DataLoader(drug2_data_train, batch_size=TRAIN_BATCH_SIZE, shuffle=None) + drug2_loader_test = DataLoader(drug2_data_test, batch_size=TRAIN_BATCH_SIZE, shuffle=None) + + model = modeling().to(device) + loss_fn = nn.MSELoss() + optimizer = torch.optim.Adam(model.parameters(), lr=LR) + + + # model_file_name = 'data/result/GATNet(DrugA_DrugB)' + str(i) + '--model_' + datafile + '.model' + result_file_name = output_path('result/GATNet(DrugA_DrugB)_R') + str(i) + '--result_' + datafile + '.csv' + file_AUCs = output_path('result/GATNet(DrugA_DrugB)_R') + str(i) + '--AUCs--' + datafile + '.txt' + AUCs = ('Epoch\tepoch\tmse\tmae\trmse\tr2\tP') + with open(file_AUCs, 'w') as f: + f.write(AUCs + '\n') + + best_auc = 10000000000000 + for epoch in range(NUM_EPOCHS): + train(model, device, drug1_loader_train, drug2_loader_train, optimizer, epoch + 1) + S, T = predicting(model, device, drug1_loader_test, drug2_loader_test) + # T is correct score + # S is predict score + # Y is predict label + print(S) + print(T) + + # compute preformence + + mae, mse, rmse = compute_mae_mse_rmse(S, T) + + # save data + + if best_auc > mse: + r2 = compute_rsquared(S, T) + best_auc = mse + AUCs = [epoch, mse, mae, rmse, r2, math.sqrt(r2)] + save_AUCs(AUCs, file_AUCs) + # torch.save(model.state_dict(), model_file_name) + independent_num = [] + # independent_num.append(test_num) + independent_num.append(T) + # independent_num.append(Y) + independent_num.append(S) + txtDF = pd.DataFrame(data=independent_num) + txtDF.to_csv(result_file_name, index=False, header=False) + + print('best_mse', best_auc) diff --git a/data/downstream/final_data/drug_response_pytorch/drug_combination/myDataset.py b/data/downstream/final_data/drug_response_pytorch/drug_combination/myDataset.py index 8bf24b9df403dbf84727bb4ad0f3569a33b47667..20ca12e0ad5fe647283f4a92b9567e069680feaf 100644 --- a/data/downstream/final_data/drug_response_pytorch/drug_combination/myDataset.py +++ b/data/downstream/final_data/drug_response_pytorch/drug_combination/myDataset.py @@ -1,111 +1,112 @@ -import random -from utils_test import * -from sklearn.model_selection import StratifiedKFold, KFold -import torch -import numpy as np -import pandas as pd -import csv - -t, f = 0, 0 -with open('D:\GraphDTA-master\data\independent_set\independent_set_newcombinatioin.csv', 'r') as f: - next(f) - reader = csv.reader(f) - for row in reader: - if float(row[11]) > 20: - t = 1 + int(t) - -print('t', t) - - - - - - - - - - - - - - - - - - - - - - - - - -# i = 0 -# datafile = 'new_labels_0_10' -# model_file = 'GAT_GCN' -# last_best = [] -# -# for enpo in range(5): -# data = [] -# file_AUCs = 'data/result/0_10/' + model_file + '(shared weight)' + str(i) + '--AUCs--' + datafile + '.txt' -# with open(file_AUCs, 'r') as f: -# data = f.read().strip().split('\n') -# best_auc = 0 -# best_data = [] -# print('len(data[1:])', len(data[1:])) -# for pot in data[1:]: -# Epoch, AUC_dev, PR_AUC, ACC, BACC, PREC, TPR, KAPPA, RECALL = pot.strip().split() -# if float(AUC_dev) > best_auc: -# best_auc = float(AUC_dev) -# best_data = [Epoch, AUC_dev, PR_AUC, ACC, BACC, PREC, TPR, KAPPA, RECALL] -# i += 1 -# print('best_data', best_data) -# last_best.append(best_data) -# print('last_best', last_best) -# AUC_dev0, PR_AUC0, ACC0, BACC0, PREC0, TPR0, KAPPA0, RECALL0 = 0, 0, 0, 0, 0, 0, 0, 0 -# for da in last_best: -# AUC_dev0 += float(da[1]) -# PR_AUC0 += float(da[2]) -# ACC0 += float(da[3]) -# BACC0 += float(da[4]) -# PREC0 += float(da[5]) -# TPR0 += float(da[6]) -# KAPPA0 += float(da[7]) -# RECALL0 += float(da[8]) -# -# gapAUC_dev, gapPR_AUC, gapACC, gapBACC, gapPREC, gapTPR, gapKAPPA, gapRECALL = 110, 110, 110, 110, 110, 110, 110, 110 -# -# for da in last_best: -# if abs(AUC_dev0/5 - float(da[1])) < gapAUC_dev : -# gapAUC_dev = abs(AUC_dev0/5 - float(da[1])) -# -# if abs(PR_AUC0/5 - float(da[2])) < gapPR_AUC : -# gapPR_AUC = abs(PR_AUC0/5 - float(da[1])) -# -# if abs(ACC0/5 - float(da[3])) < gapACC : -# gapACC = abs(ACC0/5 - float(da[1])) -# -# if abs(BACC0/5 - float(da[4])) < gapBACC : -# gapBACC = abs(BACC0/5 - float(da[1])) -# -# if abs(PREC0/5 - float(da[5])) < gapPREC : -# gapPREC = abs(PREC0/5 - float(da[1])) -# -# if abs(TPR0/5 - float(da[6])) < gapTPR : -# gapTPR = abs(TPR0/5 - float(da[1])) -# -# if abs(KAPPA0/5 - float(da[7])) < gapKAPPA : -# gapKAPPA = abs(KAPPA0/5 - float(da[1])) -# -# if abs(RECALL0/5 - float(da[8])) < gapRECALL : -# gapRECALL = abs(RECALL0/5 - float(da[1])) -# -# print(datafile + model_file, -# '\nAUC_dev0', AUC_dev0 / 5, gapAUC_dev, -# '\nPR_AUC0', PR_AUC0 / 5, gapPR_AUC, -# '\nACC0', ACC0 / 5, gapACC, -# '\nBACC0', BACC0 / 5, gapBACC, -# '\nPREC0', PREC0 / 5, gapPREC, -# '\nTPR0', TPR0 / 5, gapTPR, -# '\nKAPPA0', KAPPA0 / 5, gapKAPPA, -# '\nRECALL0', RECALL0 / 5, gapRECALL) \ No newline at end of file +from _reference_paths import ASSET_DIR, artifact_path, data_path, output_dir, output_path, required_artifact +import random +from utils_test import * +from sklearn.model_selection import StratifiedKFold, KFold +import torch +import numpy as np +import pandas as pd +import csv + +t, f = 0, 0 +with open(data_path('independent_set/independent_set_newcombinatioin.csv'), 'r') as f: + next(f) + reader = csv.reader(f) + for row in reader: + if float(row[11]) > 20: + t = 1 + int(t) + +print('t', t) + + + + + + + + + + + + + + + + + + + + + + + + + +# i = 0 +# datafile = 'new_labels_0_10' +# model_file = 'GAT_GCN' +# last_best = [] +# +# for enpo in range(5): +# data = [] +# file_AUCs = 'data/result/0_10/' + model_file + '(shared weight)' + str(i) + '--AUCs--' + datafile + '.txt' +# with open(file_AUCs, 'r') as f: +# data = f.read().strip().split('\n') +# best_auc = 0 +# best_data = [] +# print('len(data[1:])', len(data[1:])) +# for pot in data[1:]: +# Epoch, AUC_dev, PR_AUC, ACC, BACC, PREC, TPR, KAPPA, RECALL = pot.strip().split() +# if float(AUC_dev) > best_auc: +# best_auc = float(AUC_dev) +# best_data = [Epoch, AUC_dev, PR_AUC, ACC, BACC, PREC, TPR, KAPPA, RECALL] +# i += 1 +# print('best_data', best_data) +# last_best.append(best_data) +# print('last_best', last_best) +# AUC_dev0, PR_AUC0, ACC0, BACC0, PREC0, TPR0, KAPPA0, RECALL0 = 0, 0, 0, 0, 0, 0, 0, 0 +# for da in last_best: +# AUC_dev0 += float(da[1]) +# PR_AUC0 += float(da[2]) +# ACC0 += float(da[3]) +# BACC0 += float(da[4]) +# PREC0 += float(da[5]) +# TPR0 += float(da[6]) +# KAPPA0 += float(da[7]) +# RECALL0 += float(da[8]) +# +# gapAUC_dev, gapPR_AUC, gapACC, gapBACC, gapPREC, gapTPR, gapKAPPA, gapRECALL = 110, 110, 110, 110, 110, 110, 110, 110 +# +# for da in last_best: +# if abs(AUC_dev0/5 - float(da[1])) < gapAUC_dev : +# gapAUC_dev = abs(AUC_dev0/5 - float(da[1])) +# +# if abs(PR_AUC0/5 - float(da[2])) < gapPR_AUC : +# gapPR_AUC = abs(PR_AUC0/5 - float(da[1])) +# +# if abs(ACC0/5 - float(da[3])) < gapACC : +# gapACC = abs(ACC0/5 - float(da[1])) +# +# if abs(BACC0/5 - float(da[4])) < gapBACC : +# gapBACC = abs(BACC0/5 - float(da[1])) +# +# if abs(PREC0/5 - float(da[5])) < gapPREC : +# gapPREC = abs(PREC0/5 - float(da[1])) +# +# if abs(TPR0/5 - float(da[6])) < gapTPR : +# gapTPR = abs(TPR0/5 - float(da[1])) +# +# if abs(KAPPA0/5 - float(da[7])) < gapKAPPA : +# gapKAPPA = abs(KAPPA0/5 - float(da[1])) +# +# if abs(RECALL0/5 - float(da[8])) < gapRECALL : +# gapRECALL = abs(RECALL0/5 - float(da[1])) +# +# print(datafile + model_file, +# '\nAUC_dev0', AUC_dev0 / 5, gapAUC_dev, +# '\nPR_AUC0', PR_AUC0 / 5, gapPR_AUC, +# '\nACC0', ACC0 / 5, gapACC, +# '\nBACC0', BACC0 / 5, gapBACC, +# '\nPREC0', PREC0 / 5, gapPREC, +# '\nTPR0', TPR0 / 5, gapTPR, +# '\nKAPPA0', KAPPA0 / 5, gapKAPPA, +# '\nRECALL0', RECALL0 / 5, gapRECALL) diff --git a/data/downstream/final_data/drug_response_pytorch/drug_combination/plot.ipynb b/data/downstream/final_data/drug_response_pytorch/drug_combination/plot.ipynb index fe84e0251e0f8aacfbae498575df7fbe5eff3e08..00b7605eed4b3090818bda1fc4e00f414f7fc37d 100644 --- a/data/downstream/final_data/drug_response_pytorch/drug_combination/plot.ipynb +++ b/data/downstream/final_data/drug_response_pytorch/drug_combination/plot.ipynb @@ -1,106 +1,101 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/home/ruiyi/anaconda3/lib/python3.10/site-packages/scipy/__init__.py:146: UserWarning: A NumPy version >=1.17.3 and <1.25.0 is required for this version of SciPy (detected version 1.26.1\n", - " warnings.warn(f\"A NumPy version >={np_minversion} and <{np_maxversion}\"\n" - ] - } - ], - "source": [ - "import pickle\n", - "from sklearn import metrics\n", - "import matplotlib.pyplot as plt\n", - "import numpy as np" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "with open('fpr_tpr_scfm_run2.pkl','rb') as f:\n", - " fpr,tpr=pickle.load(f)\n", - "with open('fpr_tpr_bsl_run4.pkl','rb') as f:\n", - " fpr_b,tpr_b=pickle.load(f)" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Text(0.5, 1.0, 'ROC Curve')" - ] - }, - "execution_count": 22, - "metadata": {}, - "output_type": "execute_result" - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAA6EAAAPHCAYAAADZ9k9tAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy88F64QAAAACXBIWXMAAB7CAAAewgFu0HU+AAD2YUlEQVR4nOzdd1RU19oG8GfovSooiiD23lEsARWsiNixIBpNMZZoYnozyZdremKi8ZobC9iNsWPHCooFFUWjKHYFBQGVzjDn+4MwYZwZmD6U55fFWsM+++z9DhHl4Zyzt0gQBAFEREREREREBmBi7AKIiIiIiIio9mAIJSIiIiIiIoNhCCUiIiIiIiKDYQglIiIiIiIig2EIJSIiIiIiIoNhCCUiIiIiIiKDYQglIiIiIiIig2EIJSIiIiIiIoNhCCUiIiIiIiKDYQglIiIiIiIig2EIJSIiIiIiIoNhCCUiIiIiIiKDYQglIiIiIiIig2EIJSIiIiIiIoNhCCUiIiIiIiKDYQglIiIiIiIig2EIJSIiIiIiIoNhCCUiIiIiIiKDYQglIiIiIiIig2EIJSIiIiIiIoMxM3YBRERElbl58yaSk5Nx584dPH/+HPn5+bC2toadnR28vLzQrFkzNG3a1NhlSmVnZ+Pq1au4c+cOHj9+jLy8PIjFYjg6OsLZ2RnOzs5o2bIlfHx8jF2qRgRBwPXr13Hr1i3cu3cPz549Q15eHqysrODs7AwnJyfUr18fHTp0gK2trbHLJSKiKoYhlIiohlqwYAE+//xztc6xtraGg4MDHB0d0bBhQ3Tq1AmdO3fGwIED4erqqqdK5RUVFSE6OhobN27E4cOH8fjx40rPqVOnDgICAjBu3DgEBwfDysrKAJWWEgQBx44dw19//YWDBw/i6tWrEASh0vNcXFzQtWtX9O/fHxMmTEDDhg0NUK1mHj9+jC1btmD79u04efIknj59Wuk5pqamaNWqFbp3747Ro0cjKCgIpqamBqiWiIiqMpGgyr+SRERU7WgSQpWxsLBASEgI3n77bfTo0UMnYypSVFSEJUuW4Pvvv8fDhw81HqdevXqYP38+Zs2aBUtLSx1WKEssFmPFihX48ccfce3aNa3GEolE8Pf3x4wZMzBmzBiIRCIdVamdq1evYuHChdiwYQOKioq0GsvNzQ0TJkzA/Pnz0aBBAx1VSERE1Q1DKBFRDaXLEFrGxMQEs2bNwsKFC2FjY6PTsU+cOIFp06bh6tWrOhuzRYsWWL58OXr16qWzMcvExsbi9ddfx+XLl3U+dteuXfHNN9+gX79+Oh9bVQUFBfj444/xyy+/oLi4WKdjW1tbY86cOXj//ffh5OSk07GJiKjqYwglIqqh9BFCy/Tv3x+7d++GhYWFTsZbsmQJ5s2bV2HYMTExgaenJxo2bAhra2vk5+fj4cOHuHPnDiQSidLzzMzM8NNPP2HWrFk6qRUAFi5ciE8++QQlJSUV9rO2toaXlxdcXV1hbW2NgoICZGVl4datW8jLy6t0nosXL6Jdu3a6Kltl169fx+jRo3Hx4sUK+4lEIri5ucHT0xN2dnYwMTFBTk4O7t+/j9TU1EpvSR45ciT++usvXZZORETVAJ8JJSKqZU6ePKn0WF5eHrKyspCUlIQjR47gyJEjCvvFxMRg2rRpWL16tdb1fPHFF/jss88UHhOJRAgODkZERAQCAwPh6Ogo1+f58+c4fPgwoqKisHXrVrlAKhaLMXv2bGRmZuLTTz/VqlaJRII33ngDy5YtU9rH19cXEyZMQP/+/dG6dWuYmMgvRC8IApKTk3HkyBH89ddfOHTokMJAW1nI1YcLFy5gwIABSE9PV3jc0dERY8aMwYgRI+Dn5wdnZ2eF/XJycnDixAns3bsXGzduVHh7tTHeHxERVQECERHVSJ999pkAQO5DHZcvXxZ69OihcBwAQlxcnFY1Llq0SOnYXbt2Fc6dO6fWeJcuXRJ69uypdMxffvlFq3pnzZpVYb3Hjx/XaNzbt28Ls2fPFiwsLGTGPH/+vFb1qis5OVlwcXFR+P4sLS2Fzz77THj69Kna44rFYmHjxo1C+/btZcYcPny47t8EERFVedwnlIiIlGrdujWOHTuG/v37Kzyu7AqmKo4cOYK33npL4bFXXnkFJ06cQKdOndQas23btjh27Bhmzpyp8Phbb72Fo0ePql0rAPz3v//F4sWL5dpFIhE++ugjnDp1Cr1799ZobC8vL/zyyy9ISkpCYGCgRmNo6/nz5xg6dCgyMzPljjVt2hQJCQlYsGABHBwc1B7b1NQUY8eOxblz57BkyRLY2dnpomQiIqqmGEKJiKhC5ubmWLduncKFiI4cOYJnz56pPWZOTg6mTp2q8HbM1157Db///jvMzc01qtfU1BSLFy9W+AyoWCzG1KlTkZubq9aYycnJSgPz4sWL8X//938Kb7tVV7NmzbB//3589913Bt/KZO7cubh+/bpce5s2bXDixAm0adNG6zlMTU3xxhtv4Pz58+jcubPW4xERUfXEEEpERJVyc3NDeHi4XLtYLFb63GhFvvrqK9y+fVuu3dfXF0uWLNGgQnmLFi2Cn5+fXPutW7fwn//8R62x3njjDeTn5ytsf+ONNzSuURGRSIT58+dj586dsLW11enYysTGxmLFihVy7XZ2dtixYwfq1q2r0/maNm2K48ePY9iwYTodl4iIqgeujktEVEMpWx1X07/2N2/ejDFjxsi1f/vtt3jnnXdUHufx48fw8fGRuxppZmaGhIQEtG/fXqP6FElKSkLnzp3lVt21tbXFrVu3VApXBw8eRFBQkFx706ZNceHCBYMFRX3q3bs34uLi5NqXLVuGV1991QgVVX9PnjzB+fPnkZKSguzsbBQXF8Pe3h5t27ZVent7VVdUVIQLFy7gxo0bePToEfLy8mBnZwd3d3d4eHjA19cXVlZWxi6TiKoBro5LREQqady4scL2jIwMtcZZtmyZwtth33zzTZ0GUKD0GdG5c+fiu+++k2nPzc3F77//jo8++qjSMX744QeF7YsXL64RAfTUqVMKA6ifn5/RA6iiX6R89tlnWLBggcZjikQiuTZVfzFT2blisRhr167FsmXLcOrUKYVbB/n7+6N///7Ys2cPhgwZInPMxsYGaWlpsLe3V6meiqxatQpTp06VafPw8MC9e/fUunW8pKQEmzdvRlRUFGJiYlBYWKi0r7W1Nfr374+IiAiMHj1a49qJqObj7bhERKQSZQvSKFrIRhlBEBTe9ikSiXS6j2d5M2fOVPhD9/Llyys99+7du9i/f79ce7NmzTBgwACd1Gds//vf/xS2K1vciRQ7f/48OnXqhClTpuDkyZMV7l0LAAMGDICHh4dMW15eHjZv3qyTeiIjI+XawsPD1QqgO3fuRPPmzREWFobdu3dXGEABID8/H7t27cKYMWPQs2dPnD17Vu26iah2YAglIiKVPH36VGG7ogWLlLlw4YLCZ0EDAwPh7e2tYWUV8/LyUnj7461bt3DhwoUKz922bZvCMPHaa68pvCpW3UgkEmzfvl2uvU6dOrySpYZ9+/ahd+/eSEpKUvkcU1NTTJo0Sa5dUXhU1507dxSuAj1lyhSVzi8qKsLrr7+OkJAQ3Lx5U6MaTp48id69e2Pr1q0anU9ENRtDKBERqSQlJUVhu5ubm8pj7N27V2H7xIkTNapJVYp+2AeU11Nmz549CtvDwsK0rqkqOHPmjMLbqUNDQ2FpaWmEiqqfxMREjBo1Cnl5eTLtdnZ2aNOmDQICAtC5c2e4u7vLnRsRESHXduzYMdy6dUurmqKiouRuMfb19UXLli0rPbewsBAjRozAsmXLlPZxcHBA+/bt0bdvX/To0QOenp5KxxozZgzWrl2r3hsgohqPIZSIiFSi6LZUAGrt5RkfH6+wvUePHhrVpKru3bsrbD916lSF5yk67uHhgQYNGuikLmNT9v/D19fXwJVUXxMnTpR5xnn48OE4dOgQMjMzkZSUhMOHDyMhIQFpaWm4efMmXnnlFWnf1q1bo1u3bjLjCYKA1atXa1VTVFSUXJuqV0HnzZuH3bt3y7VbW1tj5syZOHv2LDIzM5GYmIhDhw7h5MmTuHv3LpKTk/Hee+/JLUxUUlKCV199FdeuXdPovRBRzcQQSkRElXr48CHWrVsn125tbY3evXurPI6i218dHBzQvHlzbcqrVPPmzRU+03r+/Hml59y6dQtZWVly7TUpoJ07d05he016j/p2+fJlAKXfCzt27MC2bdvQt29fhfvcNm7cWO6qv6JwqChEqiouLg43btyQabO0tFTp6v1ff/2FpUuXyrV3794dV69exeLFi9GlSxeFe9g2a9YMX3/9Nc6fPy93a31eXh4mTpyocF9gIqqdGEKJiKhCRUVFGD9+PAoKCuSOjRs3TumCRS8qLi7GvXv35No7duyo9+crRSIROnbsKNd+7949iMVihecoexZO0TjVlaL3aGZmhjZt2hihmurLxMQEu3bt0mjf07CwMLlbn1NSUhAbG6tRLYqeKQ0JCYGzs3OF5z1//lzmKm0Zf39/HDt2DI0aNVJp/pYtWyI2NlZu+6OEhASFzx8TUe3EEEpEREpdunQJvXr1wrFjx+SO2djY4IsvvlB5rIcPHyrcCkPRs3L6oOjZVYlEgocPHyrsf//+fYXtderU0WldxqToPTo5OcHMjDu4qWPu3Lno16+fRue6uLgoDK+aLFBUUFCATZs2ybWrcivu//73P7kr//Xq1cOWLVtgYWGhVh0NGjTA77//Ltf+888/qzUOEdVc/FeGiKiWUfYcIFC6xUJWVhaSkpIQExOjMHwCpVcWIyMjlS5IooiyrVycnJxUHkMbyubJyspSeJVH0a24FY1THSl6jzXp/RmCpaUlPvzwQ63GiIiIkNua5c8//8Svv/4q94xlRbZt2ya3inW9evUwcODACs8Ti8UKA+KCBQvg4uKi8vzlhYaGon379rh48aK07fjx47h+/TqaNWum0ZhEVHMwhBIR1TJ+fn5anW9jY4MVK1aovYVHfn6+wnZHR0et6lGVsnmU1aWsvSaFNEXvsSa9P0MYPnw4XF1dtRpj0KBBqFevHtLS0qRtT58+xbZt29RaiVnR1dNJkyYpfIazvLi4OLlb5e3t7VVezEiZSZMm4d1335Vpi42NZQglIt6OS0REqjEzM0NYWBiSkpIwbtw4tc9XtiiJobYCUTZPcXGxwnZlz4rWpK1LFL3HmvT+DKFv375aj2FmZqZwmyJ1bslNTU3FgQMH5NpVCZKK7ngYOnSo1n8W+vTpI9cWFxen1ZhEVDMwhBIRUaVsbGwQFRWF9evXo3HjxhqNoewH2mfPnmlTmsqUzWNtba2wXdltkIaq1xAUvcea9P4MoUuXLjoZR9GeoQcOHEBqaqpK569Zs0buFz1dunRRaZGp48ePy7V17dpVpXkr8uIquQBkbs8lotqLt+MSEVGl8vLyMGHCBOzfvx/Lli1Te6ESALC1tVXYnp2drWV1qlE2j7K6jF2vIdja2iIvL0+mrSa9P0No2LChTsZp164dOnfuLLNtTklJCdasWYN33nmn0vO12Rs0KSlJrq2kpKTC58dVoeguA2XPhhNR7cIQSkRUyyhaoRYoXVkzMzMTly9fRnR0NCIjI+UCyapVq5Ceno6tW7cq3AexIvXq1VPYbqjQ8+KCLWXq16+vsN3Y9RpCvXr1kJ6eLtNWk96fIejymeaIiAi5vVsjIyMrDaEJCQlyQdLCwgLjx49XaV5FwfC9995T6Vx1MYQSEcDbcYmI6B9WVlbw8PBAUFAQfv75Z1y/fh1Dhw6V6xcdHY233npL7fFdXV1hY2Mj156cnKxRveq6du2aXJutra3ShXiUrfx769YtXZZlVIre4/Pnz/HkyRMjVFM9KfozramJEyfK3WVw+fJlJCQkVHieomdHg4ODVVowKScnB4WFheoVqgVlvwwiotqFIZSIiBSqU6cOtm/frjCILl68GDt27FB7zNatW8u1JScn6/05xGfPnuH69ety7W3btlV6TqtWrWBiIv/P5JkzZ3RamzEpe16wJr3H6sTV1VXh91tFCxQVFxdj/fr1cu2q3opr6FAokUgMOh8RVU0MoUREpJSpqSnWrVuncIGR1157Te3w2LlzZ7k2QRAqvdKjrYSEBIW3IXfq1EnpOXZ2dmjRooVc+4ULF5SuqFvdKFt85vTp0wauhMooWqBo/fr1Sv/MRUdHIyMjQ6bNzc0NgwcPVmm+yrZvISLSB4ZQIiKqkIODA5YuXSrXnpaWhi+++EKtsQICAhS279u3T5PSVKZs/H79+lV4nqLtNwoLC3HixAmd1GVsAQEBEIlEcu2HDx82QjX6Vx1+eTBkyBDUrVtXpi0jIwO7d+9W2F/RVdKJEyfCzEy1ZT/s7e0VticnJ0MQBL18EBExhBIRUaUGDRqEIUOGyLUvXrwYd+/eVWscRT8cR0ZGKt2XU1tisVjhyqFmZmYICgqq8Nxhw4YpbP/f//6nk9qMzc3NDb6+vnLtR48eVXj7sqEp+rOizZ+T6vCsq7m5OSZMmCDXrihsPnnyBNHR0XLtiq6mKmNrawtnZ2e59pSUFJXHICJSF0MoERGpZOHChXJXzQoLC9W6Gurs7KzwNsG0tDTs2rVL6xoV2b17t8K9FocNG6Z0UaIygYGBClfP3bx5s9wtkNVVeHi4XJsgCFi2bJkRqpGl6Crd8+fPNR7vwYMH2pRjMIqe54yOjpYL0evWrZO7utuxY0d06NBBrfkUPaudmJio1hhEROpgCCUiIpW0b98eo0aNkmuPjIzEzZs3VR7ntddeU9j+6aef6vx2SbFYjE8//VStOsozMzPD9OnT5doLCwuxcOFCreurCsLDw2FnZyfX/scff+D+/ftGqOhfirY/SUtL03i8uLg4bcoxGEVBsqioSG4BIkVXR1VdkKg8Pz8/uTZNFh4jIlIVQygREans008/lbsaKhaL8eWXX6o8xtChQxUuUHTp0iV89913WtdY3vfff6/wik737t0xcOBAlcaYO3euwjD0888/4/jx41rXqExxcbFWV/1U5eDggHnz5sm1P336FNOmTdPrM3yV3R7r4eEh13bx4kWN59u+fbvG5xqaoltqy4dORVu3KLuVtzLBwcFybfHx8VXilmwiqpkYQomISGXt2rVDaGioXPvq1atx48YNlcdRFja//PJLnDt3TtPyZCQmJiq8VVgkEuHbb79VeRwXFxeFV1MlEgkiIiL0cltuVlYWBg4caLDn8t555x00aNBArn3//v346aef9DLnxo0bMWvWrAr7KPplxbVr13D79m215zt//jwOHTqk9nnGomhxobNnz+LKlSsAFF8FVbSokSr69OkjtwK2RCLBhx9+qPZYRESqYAglIiK1KLoaWlJSotbV0H79+im8zbWgoABDhw7F5cuXtarx2rVrGDJkCPLz8+WOvfbaa3jppZfUGm/u3Lno1auXXPutW7cQGBio8JlTTcXFxaFTp04GXaHW3t4ef/zxh8Jj8+fPx5IlS3Q2V0FBAWbPno2wsDCF/3/Kq1Onjlw4EgRB7YWhioqKFP55q8qUbbMSFRWFkpISrFmzRu6YOgsSlWdiYqLwavjmzZsr3KOUiEhjAhER1UifffaZAEDuQxdCQkLkxjU1NRWuXbum8hg5OTlChw4dFNbo5OQkbNu2TaPadu/eLbi6uioct3PnzkJeXp5G4967d0/w8PBQOG79+vWF6OhojcYtk5aWJrzyyiuCiYmJdNzz589rNaa6Pv30U4XvD4AQEREhZGZmajX+1q1bBR8fH+mYw4cPr/Sc+fPny9ViZWUlJCUlqTRnQUGBMHbsWKXvS53vCX19Pynz119/yc3XoEEDYdeuXXLtderUEYqKijSeq6ioSGjevLncuObm5sKGDRu0fi8PHjwQfvvtN63HIaKagSGUiKiG0mcITUhIUDj2xIkT1Rrnzp07QqNGjZSGg9DQUOHcuXMqjXXp0iVh3LhxgkgkUjiWt7e3cO/ePU3ertT58+cFFxcXpfUGBQUJe/fuFcRiscpjnjp1Snj99dcFKysrufEMHUIFQRBefvllpe/P1dVV+OKLL4QHDx6oPF5WVpbwxx9/CO3bt5cbT5UQevXqVYW11KtXTzhx4kSF5549e1bw8/OTnmNjY1OtQmhhYaHCX6g0btxYrm3OnDlaz3fq1CnB3Nxc4fucPn262t8/+fn5ws6dO4Xx48cL5ubmQosWLbSukYhqBpEgcNdgIqKaaMGCBfj888/l2nX11/6wYcPktlUxMTFBUlISWrVqpfI4Zbe0VrTCbvv27REUFIT27dujQYMGsLa2Rn5+PlJTU3Hp0iUcPHiwwmdJmzRpgpiYGHh5ealclzJJSUkYNGhQhdt91K1bFwEBAejSpQsaN24MV1dXWFtbo7CwEFlZWUhJSUFiYiKOHj1a4Qq058+fR8eOHbWuWR2CIGDevHlYtGiR0j4ikQjdunVDr1690LJlS3h6esLW1hYmJibIycnBgwcPkJycjBMnTuD06dMoKipSOM7w4cOxbdu2Smt6+eWXsXLlSoV1DB48GMHBwfDx8YG1tTUyMjJw7do17N27F8ePH5f5875q1SqFq8eq+j3x4m3o6pyrqdmzZ2Px4sWV9ktISFD4DK26Vq9ejYiICIXvy9zcHMOHD0f//v3RvXt31K9fH87OzhAEAU+fPkV2djaSk5Nx4cIFJCQk4ODBg8jNzZWe36JFC1y9elXrGomoBjBa/CUiIr3S55VQQRCE06dPKxw/LCxM7bGePHkiDBo0qMJbJjX9GDRokNa3kb7o8ePHwsCBA/VSb/m6nzx5otO61bFmzRrB3t5eb+/PxcVFiIqKUqmWrKwswdPTU6v5PvzwQ0EQtLuaqc/vJ2XOnj1b6Xtr166dTueMjIxUekVUmw9eCSWiMlyYiIiINNKtWzeFC6ds2rRJ7YWFXFxcsGfPHqxYsUKj1T0VqVu3LlasWIE9e/bA2dlZJ2OWH3vv3r1Yu3YtPD09dTp2t27dcOjQIezZswcuLi46HVsdEydOxNWrVzFx4kSYmOjuxwVra2u8//77uHnzJsLDw1U6x8nJCUeOHIGPj4/a85mYmGDhwoX46quv1D63KujSpQvatm1bYR9NFyRSZvLkyTh27BiaNm2q03HNzc11Oh4RVV8MoUREpLHPPvtMrk0ikSi8DVgVU6dOxe3bt7Fo0aJKf/BWpnXr1vjpp59w+/ZtTJ06VaMxVDVhwgSkpKRgzZo1CAgI0Disubq6YubMmYiPj8fp06fRt29fHVeqGQ8PD6xZswbXrl3DnDlzUK9ePY3GEYlE6N27N5YtW4bU1FQsXLhQ4d6rFfHx8cHJkycxffp0mJqaqnRO586dcfz4cbz//vualF1lVBQyzczMMGnSJJ3P2aNHDyQlJeH7779XuH2PqkxMTPDSSy/hjz/+QFxcnA4rJKLqjM+EEhHVUPfv31f4vGGPHj10Ok9CQgKKi4tl2kxMTODr66v12FeuXMGhQ4cQHx+P5ORk3L17F8+fP0dBQQEsLS1hb2+PRo0aoVmzZujRowf69euncXjVhcePH+Pw4cM4deoULl++jNu3byM9PR25ubmQSCRwdHSEk5MTXFxc0KpVK3Tr1g3dunVD586dq8VVIolEgtOnTyMuLg4JCQlISUnBvXv38OzZMxQUFMDKygpOTk5wdnaGh4cHunTpgq5du6JHjx7w8PDQWR03btzA1q1bsX//fty8eRPp6enIz8+Ho6MjmjRpgh49emDkyJHw9/fX2ZzG9PjxY/z2228KjzVo0ACvvPKKXucXi8U4cOAAduzYgRMnTuDKlSsQi8UK+9atWxctW7ZEhw4d0LdvX/Tt21fndyIQUfXHEEpEREREKispKcHDhw+RlZWFwsJCWFlZwcHBAc7OznBwcDB2eURUDTCEEhERERERkcHwmVAiIiIiIiIyGIZQIiIiIiIiMhiGUCIiIiIiIjIYhlAiIiIiIiIyGIZQIiIiIiIiMhiGUCIiIiIiIjIYhlAiIiIiIiIyGIZQIiIiIiIiMhgzYxdAVUNBQQFSUlIq7NOkSRNYWVkZqCIiIiIiIqqJGEIJAJCSkoK2bdtW2CcpKQlt2rQxUEVERERERFQT8XZcIiIiIiIiMhiGUCIiIiIiIjIYhlAiIiIiIiIyGIZQIiIiIiIiMhiGUCIiIiIiIjIYhlAiIiIiIiIyGIZQIiIiIiIiMhiGUCIiIiIiIjIYhlAiIiIiIiIyGIZQIiIiIiIiMhiGUCIiIiIiIjIYhlAiIiIiIiIyGIZQIiIiIiIiMhiGUCIiIiIiIjIYhlAiIiIiIiIyGIZQIiIiIiIiMhgzYxdANVdBQQGys7ORl5eHkpISY5dDVGuZmprCxsYGTk5OsLKyMnY5REREVMsxhJLOCYKA1NRUPH361NilEBEAsViMwsJCZGVlwdHREfXr14dIJDJ2WURERFRLMYSSzj158kQugJqZ8Y8akbGIxWLp66dPn8LCwgJ16tQxYkVERERUmzEZkE4VFRUhPT1d+rmbmxucnJxgampqxKqIareSkhJkZ2fj8ePHAID09HQ4ODjAwsLCyJURERFRbcSFiUincnJypK9dXV3h6urKAEpkZKamptLvxzLlv1eJiIiIDIkhlHQqNzdX+trBwcGIlRDRi8p/T5b/XiUiIiIyJIZQ0qmioiIAgEgkgqWlpZGrIaLyLC0tpQsSlX2vEhERERkaQyjplEQiAVB6+x9X3ySqWkQikfT2+LLvVSIiIiJDYwglIiIiIiIig2EIJSIiIiIiIoNhCCUiIiIiIiKDYQglIiIiIiIig2EIJSIiIiIiIoNhCFXR48ePsWvXLnz66acYPHgw6tSpA5FIBJFIhClTpuhlzg0bNmDgwIGoX78+rKys4O3tjfDwcMTHx+tlPiIiIiIiIn0zM3YB1YW7u7vB5iooKMCYMWOwa9cumfY7d+7gzp07WLduHRYsWIBPPvnEYDVR1Zabm4u1a9di+/btSExMREZGBszMzODm5gZ3d3d06NABAQEB8Pf3R/369XU276pVqzB16lSV+x8+fBgBAQFKz/fx8UFKSkql49y7dw/e3t4y24zcunUL3t7eKtdCRERERMbBEKoBT09PtGrVCvv379fL+NOmTZMG0L59++LNN9+Eh4cHLl26hP/85z9ISUnBp59+ivr162P69Ol6qYGqj9OnT2PcuHG4ffu2THthYSFu3bqFW7duIT4+HsuWLYO7uzvS0tKMU6gKbt68iRMnTqBnz54V9lu7di33uSQiIiKqphhCVfTpp5+iW7du6NatG9zd3XH79m00btxY5/McPXoU69atAwAMGzYMW7dulW4u361bN4SEhKBLly64e/cu3n33XYwePRpOTk46r4Oqhxs3biAoKAjPnj0DAISEhGD06NFo3rw5LCwskJGRgcTERBw4cACHDx/Way3/93//h+HDh1fYp6LvGSsrKxQUFGD16tWVhtDVq1fLnENERERE1QdDqIo+//xzg8zz7bffAgBMTU3x22+/SQNomTp16uCbb77B+PHjkZWVheXLl+Ptt982SG1U9Xz00UfSALpixQqFt8YGBQVh/vz5SE9Px6ZNm/RWS4MGDdC2bVuNzw8JCcGmTZuwadMmLFq0CBYWFgr7nTt3DleuXAEADB8+HBs3btR4TiIiIiIyPC5MVIXk5OQgJiYGQGlwaNiwocJ+I0eOhIODAwBgy5YtBquPqpaSkhLpbdtdu3at9NnMunXrYubMmYYoTSPjxo2DhYUFMjMzER0drbRf2VXQrl27omXLloYqj4iIiEivBIkEhbnPlH7UJAyhVcjp06dRWFgIAPD391faz8LCAj169JCeU1xcbJD6qGpJT09HXl4eAKBp06Zaj1dUVITff/8dQ4cORYMGDWBpaQk3Nzd06dIFs2bNwvHjxyEIgtbzKOPi4oKhQ4cC+DdovkgsFmP9+vUAgPDwcL3VQkRERGRo4qICWH3vqPDD40snY5enU7wdtwr5+++/pa8ru8LTsmVL7N+/H2KxGNevX0fr1q1Vnuf+/ftybVV5sRpSrPztquX/7GjiwoULGDlyJG7duiXTnp6ejvT0dJw7dw5LlizR+wq04eHh2Lp1K6Kjo5GZmQkXFxeZ4/v378ejR49gZmaGsLAw/Pbbb3qrhYiIiEhn0tKAffuAxERA2S/1iwuAuoYty1gYQquQe/fuSV8ruxW3jKenp8x56oTQ8udS9eXi4gIvLy/cuXMHiYmJ+Oabb/DOO+/AxES9GxyuXLmCPn36ICcnBwAwYsQIhIWFwcfHByUlJbh27RoOHDiArVu36uNtyBg6dChcXFyQmZmJTZs24fXXX5c5XnaFdODAgXBzc9N7PURERERaO3cOCAwEsrIq7mcC4FODVGR0DKFVyPPnz6Wv7ezsKuxra2srfV0WHqoNe3ugqMjYVRiWhQVQ7v+vrsyePRvz588HALz//vtYunQphg0bBj8/P3Tv3h1NmjSpdIxJkyYhJycHJiYmWLt2LcLCwmSOd+/eHZMnT8aTJ09gY2OjdJwHDx4gKSlJ6fGGDRtWupKzhYUFxo0bh6VLl2L16tUyIfT58+fYvn07AN6KS0RERNWEIAAjR1YeQGsZhtAqpPxWE8pWBi1jaWkpfZ2fn6/WPOWvuJa5du0aAgMD1RpHY0VFtS+E6sm8efNw5coVrFixAgBw584dLF68GIsXLwYAuLu7IyAgABMnTkRwcDBEIpHM+fv27cP58+cBlAbaFwNoea6urhXW8vHHH+Pjjz9WenzlypWYMmVKpe8pPDwcS5cuxYkTJ3Dz5k34+PgAADZv3oz8/Hw4ODggJCSk0nGIiIiIjC45Gbhzp8IuhabAjhbAPu2X+Kg2uDBRFWJlZSV9XVRJSCtbwAgArK2t1ZqnYcOGch/16tVTr1iqEkxMTLB8+XLs2bMHQUFBcrfiPnr0CBs3bkRISAh8fX2RkpIic7z8KrTz5s0zSM2V8fPzky60tGbNGml72a24o0ePVvvPPBEREZFRKFiLpbxCU2DMWGDsWGB55wo6vrBtY3XHEFqF2NvbS19Xdottbm6u9HVlt+5SzTdo0CDs378fGRkZ2LlzJz777DMEBwfD0dFR2ufs2bPo06cPUlNTpW1lV0EbNWoELy8vrWpYuXIlBEFQ+qHKVdAyZbfblgXPe/fu4ciRIzLHiIiIiKq75Z2BnS1U6FjJXZLVDUNoFVJ+MSJFK9iWV/6WWi40RGWcnZ0RHByMBQsWYOfOnXj06BFWrFgBZ2dnAEBqaio++eQTaf+MjAwAQP369Y1SrzKTJk0CANy4cQPx8fFYs2YNBEFAo0aNKty+iIiIiKiqEwBsbg1MCwFmDlXtHFENuwuMIbQKKb/C7dWrVyvsW3bczMxMJ3tEUs1kaWmJqVOnSvfWBIAtW7ZAIpHI9HvxWVFj8/HxQa9evQCUXg0tuy134sSJVa5WIiIiIlWk2QH7mwDt3ii9BXdFRbffvqC9e3v9FWYEXJioCunWrRssLCxQVFSEo0eP4v3331fYr6ioCPHx8TLnVCvVrV5dMPJ7HjhwIDw9PXHv3j1kZWXhyZMnqFu3LurUqQMAePjwoVHrU2Ty5MmIi4vDihUrpIt28VZcIiIiqm6eWgIjwoDDjUs/F0GEYAxFNrIRi1iVxhjbZqweKzQ8htAqxN7eHv3798eePXtw8OBB3L9/X+F+oVu2bMGzZ88AlO7pWO3oYasSqpyHh4f0Nu6yBYw6d+6M2NhY3L17F3fu3NH6uVBdGjt2LObMmSMNoF26dEGrVq2MXBURERGRAs+fAytXAocOyW7HkpWFUeNeDKDB6IIuAAABAuIQp3RYC1MLvN/rfbzW5TV9Vm9wvB3XgFatWgWRSASRSIQFCxYo7FO256NYLMbMmTNRUlIiczwjIwPvvfceAMDJyQnTp0/Xa81UM+Tl5eHKlSsAAAcHB7i4uAAAhg0bJu3z008/GaU2ZZycnBAaGgpLS0tYWloiIiLC2CURERERyXv+HBg8GHjzTWD7duDYMeDYMTw4fwxLrC4hpnS3OYggwjAMkwZQAAhCEHqip8Jh903ah8x3M/F5389r3ONIvBKqotjYWNy4cUP6edmCLkDp4imrVq2S6a/OSqDl9evXD2FhYdiwYQN27NiBoKAgzJ07Fx4eHrh06RK++uor3L17FwDw9ddfSxecodonJycH/fv3xyeffIIhQ4bIbc9SRiKRYPbs2Xj+zxXokJAQ6V9kgYGB6NKlCxISEvDrr7+iR48eSvcKzczMhLW1tUG3R9mwYYPB5iIiIiLSyIoVQNy/VzPFJkBEKLCu3GOcZQG0M2QfBC1BCTKRKTfkK51fwYAmA/RVsdExhKrojz/+QGRkpMJjcXFxiIuTvYyuaQgFgBUrVuDZs2fYvXs3Dh8+jMOHD8scNzExwSeffILXXqtZl+VJfadPn8awYcPQoEEDhIaGws/PD15eXrC3t0d2djbOnz+PFStW4NKlSwAAR0dHfPnllzJjrF69Gr6+vsjJycH48ePx559/IiwsDD4+PigpKcGNGzdw4MABbN68GZcuXYK3t7cR3ikRERFRFSKRAMuWAbt3A7t2yRz6pK98AA1BCDqhk0y/EpRgEzbhGq5J26zMrDCh7QQsGbJEr+UbG0NoFWRtbY3o6GisW7cOq1atQmJiIrKzs+Hu7o4+ffpg1qxZ8PPzM3aZZGRmZmaoV68e0tLS8ODBAyxZsgRLlij/C6tZs2ZYv369XIhs1aoVjhw5ghEjRuDevXvYsmULtmzZoufqiYiIiKopiQSYPr30GVAFtpRbwkIEEYZjODqio0yfFwPotE7T8Ea3N9C6bmtYmVnpq/IqgyFURatWrZK75VZdU6ZMUesK6YQJEzBhwgSt5qSay8rKCg8ePEB8fDwOHjyI+Ph4XLt2DY8ePUJBQQFsbW3h4eGBDh06YPjw4Rg1apTSlZS7dOmCa9eu4Y8//sC2bduQlJSErKwsuLq6okGDBujduzfCwsJ4FZSIiIjo/HmlAVQiApJLNx+ACCKEIhQd0EGmTwlKsBEbkYxkfND7AwQ3D0ZPT8XPhdZUIkEQBGMXQcZ3+fJltG3btsI+SUlJaNOmTYV9rl+/DrFYDDMzMzRr1kyXJRKRDvB7lIiISEMbNgDr1gE7d8odSnYtvQ33oA+QaaM8gIohxiZsQjKSsT1sO0JahBiq+iqFV0KJiIiIiIgq8ssvePL+m4jxAS70B8pfxbvtBGxoJ9vdEpaoh3oybWKIsREbcR3X8Wb3NzGs+TDUVgyhREREREREyhQX4/JPH6DvLCDdVrVTClCASEQiAhFwh7s0gE7wn4BAn0D08uxV47ZdUQdDKBERERERkTLnz2PEsDyVA2iZPOQhClGYgAk4jMPItcvFgoAFeimxumEIJSIiIiIiUiIlIxnXXTU7Nxe5+AN/QICAsV5jdVtYNaZ4d3siIiIiIiLC3cLHFR43+ec/ZQQIsDW3xdzuc3VcWfXFEEpERERERKQBE5hg1D//KQqirtauGNFyBPaH74efp58RKqyaeDsuERERERGRmkxggtEYjdZoDaD0iucWbIEEErRza4eYyTGoa1vXyFVWTQyhREREREREChSVFOHM82ty7aYwxWiMRiu0kra1RVt0bdgV/Qb1Q+f6nWFqYmrIUqsVhlAiIiIiIqJyBEHAZ0c+w48nf0Ruca7MMVOYYgzGoCVayrSbm5tjbL+xaNygsSFLrZYYQomIiIiIiMr5Kf4nfHnsS7n2igLohAkT4O3tbaAKqzeGUCIiIiIiojLp6VhzcplcsylMMRZj0QItZNrNzc0xceJEeHl5GarCao+r4xIRERERER07BrRvD7i54fzzZJlDygKoRChmANUAr4QSEREREVGtI5aI8cupX7D7+m7cfpwM3L8P9BWAvrL9TGGKcRiH5mgu016IQviknoGX1/8ZsOqagSGUiIiIiIhqFYkgQcS2CKy7tO7fRmf5fmYww1iMVRhA795egwUeE/Rcac3EEEpERERERLXKudRzsgFUATOYYRzGoRmaybRLSgrR+PgaLIh7AIu4Ufoss8ZiCCUiIiIiopohLw9YtQo4dAjIzFTa7WjDu0CTiocajuFyAbRIKMTrK1fD88kTYPU6oFs3HRRd+zCEEhERERFR9ZebCwwbBhw+XGnXHH9UGkJP4ASaoimsYQ0AKEABUkRH4bl6NdCjB2BlpYOiayeujktERERERNVfVJRKAVRVqUhFFKKQj3wUoABrsRbhYR8AAQEMoFrilVAiIiIiIqo+Dh4E1q8Hzp8HBOHf9gsXdD5VKlKxGqvRwqUFfur/E4a1GKbzOWojhlAiIiIiIjIsQQBSUkqDpESi+nnr1gE7duivLgALDgNvnPnnEysrWN66AAeHunqds7ZhCCWqxo4cOYK+ffvKtZuamsLBwQGOjo7w9PREly5d0Lt3bwwbNgwWFhZGqNR4RCKRwjY7Ozs4OjrCzc0NnTp1gq+vL0aNGgVXV9dKx5wyZQoiIyPl2i0tLeHo6AgnJye0bt0aXbt2xeDBg9G5c2eV683NzcXatWuxfft2JCYmIiMjA2ZmZnBzc4O7uzs6dOiAgIAA+Pv7o379+iqPS0REVGVkZQFDhgDx8UaZXiICHjj8+7k5zNEczXEZlwEAdkVA3bx/Do4bDzCA6pxIEMpfw6ba6vLly2jbtm2FfZKSktCmTZsK+1y/fh1isRhmZmZo1qxZhX1Je8pCqDJ169bFnDlz8P7778PMrHb8DkpRCFXGysoKYWFh+O6771CnTh2l/ZSFUGW6deuGb775ptL/V6dPn8a4ceNw+/btSsd0d3dHWlqayjWU4fcoEREZXVBQ6S21BlYiAuYPANa2B9JtS9vMYY4JmIDGaIx92IeTOInv9wFvn7cCwsKA338HzM0NXmtNVzt+CiWqBWbMmIE33nhD+nlOTg6ysrJw8eJFxMTE4ODBg0hPT8cnn3yCnTt3YteuXahbt/b8Zq9r165YuXKl9PPCwkJkZWXh+vXriI2NxdatW5Gfn49Vq1Zh79692Lp1K3r06FHpuPv27YOHhwcAQCKRIDs7G2lpaTh16hS2bduGmzdv4syZMwgMDMTHH3+Mzz//XOE4N27cQFBQEJ49ewYACAkJwejRo9G8eXNYWFggIyMDiYmJOHDgAA7rcNEFIiIig8rKAmJiDDbdE2tgSysg7rXBiEzdI3PMAhaYgAnwhjcAYCAGQoAAzOsBDFvIxYf0iCGUqIZwc3NTeDV78ODBeO+993D58mWEh4fj/PnzOH36NEaOHImYmJhac3uura2twq9PYGAgZsyYgYyMDMydOxdr165FWloaQkJCcObMGXh5eVU4bvPmzeHt7S3XPnbsWHz33XeIjIzErFmzkJeXhy+++ALu7u4yvywo89FHH0kD6IoVKzB16lS5PkFBQZg/fz7S09OxadMmFd85ERFRFXL1quxiQnpy2wm4XBcInvhPg4IAOhET4QXZf+dfwksQ6tZlANUzbtFCVEu0adMGcXFx6NSpEwAgNjYWv/32m5Grqjrq1KmDNWvW4PXXXwcApKen480339RqTBMTE0ydOhX79u2T3v789ttvy91GW1JSgl27dgEovWKrKICWV7duXcycOVOr2oiIiGqiAz6Az5tA47nlAugLlAXQPOQhEpFwtHfUf6G1HEMoUS1ibW2N1atXS5+T/P7771FcXKywb1ZWFv7v//4Pfn5+qFOnDiwtLeHh4YHhw4djy5YtKs2Xl5eHn3/+GX379oW7uzssLCzg5uaGAQMGYOXKlSgpKVF6rre3N0QiEaZMmQIAOHPmDMaPHw9PT09YWVnB09MTU6ZMwd9//63eF6ESP//8Mzw9PQEAO3bswOXLl7Ues3fv3pg3bx4AoKCgAD/99JPM8fT0dOTlla6A0LRpU63nIyIiqlUCAoDkZJw/vR0hUy1xy1l5V0tYYhImyQXQXOQiEpF4hEcI8A7Qa7nE23GJap02bdogKCgI+/fvx4MHD3DmzBn07NlTps/u3bsxceJEZGdny7SnpqZix44d2LFjB4YOHYoNGzbAzs5O4TxnzpzBiBEj8ODBA5n29PR0HDhwAAcOHMB///tf7NixA+7u7hXWvGLFCrz22msQi8XStvv37yMyMhIbNmxAZGQkxo0bp8ZXQTlLS0vMmDEDH374IQRBwPbt2ytdkEsVc+bMwQ8//ACJRIKtW7fim2++kR4rf0u0rkM1ERFRtdGgATB4sOr969QB+vUD+vcHTEyw8eByFJQUKu1eFkA94SnTXhZAH+Mx3uj6Bpq68BfC+sYQSlQLBQYGYv/+/QCA48ePy4TQAwcOICQkBCUlJfD29saMGTPQvXt3ODg44MGDB9i4cSPWrFmD6OhoRERE4K+//pIb/9KlS+jbty9yc3Ph5uaGGTNmoE+fPnB1dcXjx4+xY8cOLFu2DKdPn8bw4cNx/PhxmCtZee7ChQtYt24d3Nzc8MEHH8DX1xcFBQXYvXs3fv75ZxQWFmLSpElo3LgxfH19dfb1+fDDD6VfH11o2LAhWrZsiStXruD69etIS0tDvXr1AAAuLi7w8vLCnTt3kJiYiG+++QbvvPMOTEx4swoREdUibdoA//ufxqcnpCYoPVZZAO3RogeCmwVjeufpaq2sT5phCCWDs19oj6KSImOXYVAWphZ4/sFzY5chVX7fyuTkZOnr3NxchIeHo6SkBAMGDMDWrVthY2MjPd6pUycEBwfjpZdewquvvootW7YgJiYG/fv3l/YRBAGTJk1Cbm4uOnTogIMHD8ptdzJgwAAEBwdj6NChOHXqFKKiojBt2jSFtSYmJsLLywvx8fHS0AYAL730EgYOHIgBAwZALBZj5syZOHPmjMIx1NWhQweYmJhAIpHIfH201blzZ1y5cgVA6VYp5d/P7NmzMX/+fADA+++/j6VLl2LYsGHw8/ND9+7d0aRJE53VQUREZHCFhcCxY8B33+ltCokgUdhuCUuEIxwN0VCmPRe56DSwE77t8a3eaiLF+Gt2MriikqJa+VGVuLq6Sl9nZWVJX69cuRKPHj2ClZUVVq9eLRNAy3vllVekVx3Lb3sCANHR0bh48SIAICoqSul+m4MGDcLo0aMVjvGiH374QSawlenbty9eeeUVAMDZs2d1FkItLCxgb28PQPbroy1lX3cAmDdvHl5++WXp53fu3MHixYsxceJENG3aFPXq1UNYWBh27twJbu9MRERVVklJ6TYs5T/u3QO6dAEGDAAOHDBoOVawUhhALawsMO/1eRjfY7xB66FSDKFEtVD55zifP//3Cu327dsBAP7+/nBzc6twjJdeegkAcPLkSZn2sjFatGiB9u3bqzTGmTNnlC5S5OzsjOHDhysdo3xwO6jDja/Lvkblvz66GlPRuCYmJli+fDn27NmDoKAguVtxHz16hI0bNyIkJAS+vr5ISUnRWV1ERERae/oUmDQJcHMDXFxkPxo1AnSw0J+6lAVQW1tbTH95Ouq71zd4TVSKt+MS1ULlA5CDg4P09dmzZwEA+/btU/l5iBe3Gykb49q1ayqPUVRUhMzMTNStW1fuWKdOnaTbmyjSsWNHWFhYoKioCElJSSrNp4qyr1H5r4+uxqxo3EGDBmHQoEHIyspCXFwczp49i4SEBBw/fhxPnz4FUPo17tOnDxISElC/Pv8BJSIiIxOLgUGDgPh4Y1ciwwte8ICHTJudnR0iIiKU3qlFhsEQSlQLZWRkSF+7uLgAAIqLi+VWw1VF2dYiZR4/fqxRTS+OU6ayK7JmZmZwcXFBWloaMjMzNZr7RYWFhdLAWPb10QVFX3dlnJ2dERwcjODgYGlN69atw9tvv42srCykpqbik08+wR9//KGz+oiIiNRWUgJs2qSbANqihcJmiSDBpUeXEHcvDgXiAqWn3316V+bza7iGbdiGUIRCBBEDaBXCEEpUC50/f176usU/f+GXvx127Nix+OSTTzQau2ycXr164b///a/K53l4eChsV+Vqqq6fkUxMTJSO2ULJP4iaKP91b968uVrnWlpaYurUqfDw8MCgQYMAAFu2bMHvv//OVXSJiMjwSkqAt94C1qwBdPRLYAwbJtdUKC7EqE2jEH09WqMhE5EIEUQIswljAK1CGELJ4CxMLSrvVMNUtfd8oNyiAL179wYAWFlZwcbGBnl5ecjOzkbbtm01GtvV1RWPHj1Cenq6xmOU9+jRowqPi8Vi6SI/urpqqejro6179+7h2rVrAEqDraJbj1UxcOBAeHp64t69e8jKysKTJ080HouIiEgtT54AW7YAx48Dq1frduyffwaCggAAN7Nu4qNDH+HQrUN4nKvZHVblXcAF/DbmNwbQKoQhlAyuKm1VUhslJSUhJiYGAODp6YmuXbtKj3Xq1AlxcXGIi4tDXl6e0tVxK9KpUydcuXIFycnJuHPnDry8vLSq98KFCxCLxUqfC01MTERRUenqw7oIvQUFBdIruCKRqMJFkdTxyy+/QCIpXTp+xIgRWo3l4eGBe/fuAQCvghIRke49eQLcvi3blpYG/POIiM68/DIwcCDQty/wzy9U7z+7j4BVAbj37J5OpzIzZ+ypSvjTC1Etkp+fj8mTJ0tvNZ0/f75MuAsJCQFQul/okiVLNJqjbAwA+PZb7ffdyszMxM6dO5UeX7FihfR1YGCg1vPNmzcP9+/fBwCEhoaiVatWWo8ZGxuLn3/+GUDpFee5c+dqPFZeXp50r1EHBwedPrNKRES13IMHQJ8+pYGwa1fZD20CqJfXvx8tWgDTpwPnzgHLlwNjx0oDaImkBL+e+lXtAGoDG0zDNDRGY+UlOGn3S3HSLf5KgKiWuHLlCsLDw6XPJfr7+2PGjBkyfV5//XV89913yMjIwCeffIK2bdti8ODBSseMi4tDSUmJdKsVABg1ahRatWqFv//+G0uXLkXnzp0xbdo0pWMkJSXh1q1bGKbgOZAyb731Fnr27Al3d3eZ9qNHj+L3338HAHTp0gXdunVT/gWoREZGBubOnYu1a9cCANzd3bFo0SKNxwMAiUSCqKgozJw5E2KxGACwaNEiufeRk5OD/v3745NPPsGQIUOUXt2USCSYPXu2dNGkkJAQlVcgJiKiWqqwsPTW2ZgYoNwCeQr7HT+u+/mTkoA2bSrskl2QjTei38DeG3uRVaDe/tw2sMFkTEY91MMETMBarMVt3Jbp09WjK9xsK17okAyLIZSohnj8+LHMFiW5ubnIysrCxYsXERMTgwMHDkivgPbo0QObN2+Gubm5zBgODg5Yv349Bg8ejMLCQgQHB2PUqFEYNWoUmjRpAgBITU1FQkICtm7diosXL+LXX3+VCaGmpqbYuHEjevbsiZycHEyfPh1//vknJkyYgBYtWsDc3ByPHz/G+fPnsWvXLpw4cQJvv/220hDaoUMHXLlyBV26dMEHH3wAX19fFBYWYvfu3fjpp5+kt+pWduU2NzdX5utTWFiI7OxsXL9+HbGxsdiyZQvy8/MBlN7uum3bNnh6elb6dU9OTkZOTg6A0pD49OlTpKWl4dSpU9i6dStu3rwJoPS22c8++wyvvvqqwnFOnz6NYcOGoUGDBggNDYWfnx+8vLxgb2+P7OxsnD9/HitWrMClS5cAAI6Ojvjyyy8rrY+IiGqxwkJg1CggWrNFfbQ2YwbQunWFXYpLijFwzUCcfnBa7eFtYIMIRMAdpb/cNYc5JmIi1mAN7uAOAKCRYyNsHL1R/dpJrxhCiWqIpUuXYunSpRX2qVu3LubOnYt3331X6TOWgYGB2LdvHyZOnIi0tDT8+eef+PPPP5WOqWi/y3bt2iEuLg6jR4/G9evXsW/fPuzbt0+tMcp07NgRs2bNwowZMzBr1iy54xYWFoiMjET37t2VjgGU7q3Zrl27CvtYWVlhwoQJ+Pbbb+Hq6lph3zIDBw6stI+vry++/fZb+Pv7KzxuZmaGevXqIS0tDQ8ePMCSJUsqDNXNmjXD+vXr4e3trVKNRERUS23YYJwAOmIEMHRo6TOf5e7YEQQBd5/eldlm5fjd4xoF0AGeA+CX6QdRruwdQWaWZpjSYQpMbEzQpX4X+Hv7w87CTvP3QnrBEEpUA5mYmMDe3h6Ojo7w8vJCly5d0KdPHwQHB8PCovKVevv164eUlBSsXLkSu3btQmJiIp48eQITExPUrVsXrVq1gr+/P0aNGqV0C5P27dvjypUrWLduHbZu3YqEhASkp6dDIpHA1dUVLVq0QO/evTFixAh07ty5wnqmT5+Otm3b4qeffkJsbCwyMjJQt25d9O/fH++99x5aV/JbVkXs7Ozg4OAAd3d3dO7cGd27d8eoUaO0esbSwsICjo6OcHJyQuvWrdGtWzcMHToUHTt2rPA8KysrPHjwAPHx8Th48CDi4+Nx7do1PHr0CAUFBbC1tYWHhwc6dOiA4cOHY9SoUSr9fyQioloiNhZYuxZISAD+WQQPQOnnhuLoCPzxBzB6tMLD/z37X3x1/Cvcf3Zfq2mGtxiOWR1nIflQMtJz018owRERERFwdnbWag7SP5Gg6w32qFq6fPlypSuLJiUloU0l9/Rfv35dentks2bNdFki1TLe3t64c+cOIiIisGrVKmOXU2Pwe5SIqIbZtg0YMwb4Z+0Bg/v1V6B7d6BTJ0DJXVarE1dj8rbJWk1T16Yu0uanIS83D1FRUUhPlw+gU6ZMgZOTk1bzkGHwSigRERERUXUkCMCcOfoPoN26Aba2/35uZQX07g2EhwONGlVQnoCrGVcxZfsUrUv4T///IC83D5GRkch4YYElJycnREREMIBWIwyhRERERETV0c2bwD3d7qcpo0sXYPNmQIM1CPZc34NpO6YhNSdVqxJGtRqFSe0noX+D/koD6JQpU+Do6KjVPGRYDKFERERERNXR06ean3vwIGBXwYI93t7AC1uKqepc6jmEbgxFUUmRZrX94+rMq2hRpwWeP3+OyMhIPHnyROa4s7MzIiIiGECrIYZQIiIiIqLaJDwc6N9fb8NvvrJZ6wA6t/tcNHdtzgBaQzGEEhERERHVBg4OQFgYsHixXqc5n3Ze5b5dPbrKfN7EuQmGNBuC8PbhKCgoUBhAXVxcEBERUeEWb1S1MYQSUZV0+/ZtY5dARERUfXXrBqxZ8+/nIhHQuLHSFWx1SdXNNya1n4TVI1YrPW5lZYXGjRvLhFAG0JqBIZSIiIiIqKaxsQGaNzd2FUo1dmqMr/p9VWEfkUiEIUOGQBAEJCQkwNXVFZMnT2YArQEYQomIiIiIyCDq2dXDe73ew7g241Dfvn6l/UUiEYYOHQp7e3t07twZ9vb2BqiS9I0hlIiIiIiIDOKvsX+hp2dPtc4RiUTw9/fXU0VkDCbGLoCIiIiIiGq37OxsZGdnG7sMMhCGUCIiIiIiMprs7GxERkYiMjIST7XZ+5SqDYZQ0ilTU1MAQElJCSQSiZGrIaLyJBIJSkpKAPz7vUpERGRM2dnZWLVqlfRKKINo7cAQSjplZWUFoHRp7pycHCNXQ0Tl5eTkSJfNt7a2NnI1RERU22VlZWHVqlUyoTMrKwt79uwxYlVkCAyhpFPll8xOS0vDs2fPeEWUyMgkEgmePXuGtLQ0aRtXFyQiImPKyspSeNXTzc0Nw4YNM1JVZChcHZd0ytbWFtbW1sjPz0dJSQkePHgAkUjEW/+IjKikpERm43Bra2vY2toasSIiIqrNMjMzERkZiWfPnsm0u7u7Izw8nP9G1QIMoaRTIpEIjRo1wt27d5Gfnw+g9NZcsVhs5MqICCgNoI0aNYJIJDJ2KUREVAvlPctD5GbFAXTy5MmwsbExUmVkSAyhpHMmJibw8vJCbm4unj9/Lr0qSkTGYWpqCmtra9jb28PW1pYBlIiI9OZJ3hOcSz2n8JgLXHB291kU5hXKtNerVw/h4eEMoLUIQyjphUgkgp2dHezs7IxdChERERHp2f1n9zFu8zicvHcSAgS54y5wwRRMYQAlAAyhRERERERVnyAAly8DGRn/tiUn63XKAnEBohKjEHMrBk/ynlTYL+5enNLjrnDFFEyBPWQXxatXrx4mT57MFdtrIYZQIiIiIqKqbP9+YPJk4NEjvU8lCAJSslJw+fFljNg4QuFVTXUoC6D169dHeHg4A2gtxRBKRERERFRVXbwIhIQAhYWV99XS9SfXMWz9MFx7ck1nY/ZDP7kA6uHhgUmTJjGA1mIMoUREREREVcH9+0BCAlB+j/XPPzdIAM0rzkNAZAAePn+o03G3YzvsYY9GaASgNICGh4fDyspKp/NQ9cIQSkRERERkTDk5wIgRwMGDuhuzTh21usfejdV5AAWAIhRhLdYiHOFo7tKcAZQAMIQSERERERlXRIRuAygABAaq1f3io4u6nf8fPRr2QM+GPTGi2Qh09ejKAEoAGEKJiIiIiIwnLw/YtUu3Yw4aBEyZotYpgqDdAkQv8m3gi02jN8HLyUun41LNwBBKRERERGQs168DRUWandu0KdCq1b+fu7gA/frh6fCBOHJ7H1KyUlQe6tjdYyr1+y7oO/h7+cu1P8t8hpTEFLTv0x5N6jRBXdu6Ks9NtQ9DKBERERFRddOgQek+oSKRTPOxO8cwZEkT5Bbn6nS6hf0XIrRlKFrWaSl37NGjR4jaG4W8vDzYwAZdw7rqdG6qeUyMXQAREREREanB1hbYsEEugD4teIoha4foPIB+EfAF3u/9vsIAmpaWhsjISOTl5QEAUlJSsGHDBojFYp3WQDULr4QSEREREVVFdesCgwf/+7lIBHTsCAwbBjRpItf9yO0jOg+gFUlLS0NUVBTy8/Nl2ouLi1FSUgIzM0YNUox/MoiIiIiIqqIWLYDISJW7q/MMqDq8nbzl2lJTU7F69Wq5AOrl5YUJEybAwsJCL7VQzcAQSkRERERECpmZmCHQR3a7l9TUVERFRaGgoECmnQGUVMUQSkRERERkLJqujGsAZiZm+HPMn6hvX1/a9vDhQ6xevVougHp7e2P8+PEMoKQShlAiIiIiIkPbvRv46ivgxAmdDCeWiHH9yXWlx/29/BHgHaDyeI2dGiPQJxANHBpI2yoKoBMmTIC5ubnadVPtxBBKRERERGRIBw8CoaFAcbHWQ5VISjBv3zxEJUbhaeFTpf36evfFZwGfaTzPgwcPsHr1ahQWFsq0N27cGOPHj2cAJbUwhBIRERERGdLixToJoAAwe89sLD27VCdjKaMsgPr4+CAsLIwBlNTGEEpEREREZEgnT6rWz9ZW6aHikmKcfXiWAZSqJYZQIiIiIiJDUnUxIn9/hc3fxH6Dr45/hedFz1We0tHKUeW+5VlaWsLc3FwmhDZp0gTjxo1jACWNMYQSEREREVU1TZsCU6YAAPKK8/B7wu+IuRWDXcm7NBquX+N+Gp1Xp04dREREIDIyEjk5OWjSpAnCwsJgZsYYQZrjnx4iIiIioqqieXMgKAh4/32gfn3kF+dj+IbhOHjzoMZDzug6A+3c2ml8flkQjY2NRXBwMAMoaY1/goiIiIiI9K24GNi4EYiJAbKzlfe7cgUwNZV+uunyJo0D6OjWozGk6RBM6TgFIpFIozHK1KlTB6GhoVqNQVSGIZSIiIiISJ+Ki4EJE4DNm9U+9fDtwxpNuWTIErzR7Q21zsnKyoKTk5PWgZWoMibGLoCIiIiIqEb76y+NAigAZBdkq31OkE8QpnacqtY5d+7cwdKlS3Ho0CEIgqD2nETq4JVQIiIiIiJ9iokxyDST2k9CP+9+CGsbBmtza5XPu337NtatW4fi4mLExsYCAPr168croqQ3DKFERERERPr05Ilq/Tp0kHkeVBXutu54t9e7eMvvLQ0Kkw2gZWJjY+Hm5oZ27TRfzIioIgyhRERERERVwYQJ0pfpuemIuRWD7de2K+3+e/DvmN55usZXLG/duoV169ZBLBbLtLds2RKtW7fWaEwiVTCEEhERERHpmiAAy5cDO3cCO3ZU3NfGBnjzTeCddwAAMTdjELw+GAXiggpPa+rSVOMAevPmTaxfv14ugLZq1QqjRo2CqZpXZInUwRBKRERERKRrb70F/Pxz5f2++gp4+23A0hIA8LTgKYatH1ZpANUGAygZG0MoEREREZEu3b6tWgAFAF9faQAFSrdkyRfn66UsAEhJScGGDRvkAmjr1q0xcuRIBlAyCIZQIiIiIiJdOnJE5a5PhFzkP7sv/Tz2bqzK5zZxaaJOVUhJScH69etRUlIi096mTRuMGDGCAZQMhiGUiIiIiEiXnj+vtMvGNsDnAcDfJ0KBE+pP0c6tHRo5NlK5/40bN7BhwwaFAXTkyJEwMTFRvwgiDTGEEhEREREZ0I4WwIRRgETD3Ofp4IltYdtU7n/9+nVs3LhRLoC2bdsWI0aMYAAlg2MIJSIiIiIykHsOwOshJpCYSNQ+19rMGjvG70Avz16wNrdW6Zz8/Hxs3rxZLoC2a9cOoaGhDKBkFPxTR0RERESkC7m5wJ49wJdfyh2K8wSazwYavQWk2qofQAHgi75fINAnUOUACgDW1tZyt9sygJKx8U8eEREREZE6SkqAggLZj+vXgebNgSFDgPR0me7XXIFBk4DrrtpNG+AdoNF5LVq0wJgxY2BiYoL27dszgJLR8XZcIiIiIiJV3L4NzJsHHDoEPHum8mlbWgE5lpX3q0h4+3B0rt9Z4/NbtmyJl19+GfXr12cAJaNjCCUiIiIiqkxaGhAQANy5o/apCR6q921VpxVEIpH0c28nbwxuOhgzus6AiUi78NigQQOtzifSFYZQIiIiIqLyBAHIyCi97bbMkiUaBVAAkIgq7wMAoS1DsXXcVo3mAIC///4bANCqVSuNxyAyBIZQIiIiIiKgNHx++SWwfDlw965Bp3a3dcd/+v1H4/P//vtvbN68GQAwZswYtGzZUlelEekcQygRERER1WwXLgDR0UBycsX9oqIMUk55JiITfOb/GSa1nwQfZx+Nxrhy5Qo2b94MQRAAAH/++SeDKFVpDKFEREREVHOtWweEhwMSzbZF0VSJCPjRD9jZrwGOmz1Q2m/fpH0I9AnUeJ7Lly/jr7/+kgZQAJBIJEhJSWEIpSqLIZSIiIiIaqbsbGDyZMME0Lp1pS8FZydMDzXBKptrAJQHUG0pCqAA0KVLFwwZMkRv8xJpi+szExEREVHNcv068J//AC4usosL6YOtLVBYCDx+LP1IObn7nwCqP0lJSQoDaNeuXTF06FCZFXaJqhpeCSUiIiKimuP0aSAoSK19PLXy9deAhYVM07E7x1Q+3cLUovJOL7h06RK2bt2qMIAOGTKEAZSqPIZQIiIiIqo5XnlFPwG0efN/X4tEQIcOwLhxwMiR0uanBU9x6NYhzN07V6UhLUwt0KV+F7XKUBZAu3XrhsGDBzOAUrXAEEpERERENUN6OnDxou7Gs7ICfv4ZkldfQXZBtuI++ZkAgFtZtxAQGYCcohyVh3+186uwtbBVuf/Fixexbds2uQDq6+uLQYMGMYBStcEQSkREREQ1w5MnqvcdN670QxknJ+R2bofZR97Fzu8/RkZehvb1/aOZSzOMbzsenwV8pvI5iYmJ2L59u1wA7d69OwYOHMgAStUKQygRERER1S516gBLlgCurkq7CIKAYVH9cfj2YZ1OPa/HPPw48Ee1zklMTMS2bdvk2hlAqbri6rhEREREVHs0bQocPlxhAAWAS48v6TyAAkDHeh3VPiczM1OurUePHgygVG3xSigRERER1Q43bgBNmqjUNf5+vM6ntzKzQpBPkNrnBQQEQBAEHD9+HADg5+eHoKAgBlCqthhCiYiIiKjm69ZN5QAKAMUlxTqd3srMCjvCdqC+fX21zxWJROjbty8EQYBEIkFgYCADKFVrDKFERERERHrycZ+P0b1hd/h7+cPe0l7jcUQiEfr16yd9TVSdMYQSERERUfWXkgKsXGmQqRwsHeBs5az0uLO1M/p598PbPd+Gh72HWmMXFhbC0tJS4TGGT6opGEKJiIiIqPqSSIBXXgFWrDDYlG90fQMLAxfqfNyEhAQcOXIEkydPRt26dXU+PlFVwdVxiYiIiKj6+vFHgwZQfTl79ix27dqFnJwcREZGIiNDd/uSElU1DKFEREREVH3t2GHsCrR25swZREdHSz/Pzc1FZGQk8vLyjFgVkf7wdlwN3L17F7/88guio6Nx9+5dWFpaomnTphg7dizeeOMN2NjYaD3HlStXsHTpUhw5cgR37txBQUEBHB0d0bZtW4SEhGD69Omwt9f84XYiIiKiGuH6ddX6NWqksPlqxlUsPr0Yx+4cQ17xv6EvuyBbB8VV7vTp09izZ49ce5cuXXTyMyVRVcQQqqbo6GhMnDgRT58+lbbl5eXhzJkzOHPmDP744w/s3r0bPj4+Gs/xww8/4P3334dYLJZpz8jIwJEjR3DkyBEsWrQIO3bsQPv27TWeh4iIiKjWGDxYrunv9L8REBmAx7mPDV8PlAdQf39/BAQEGL4gIgPh7bhqSExMxNixY/H06VPY2dnhq6++wokTJxATE4NXXnkFAHDt2jUMHToUOTk5Gs2xadMmzJ8/H2KxGBYWFpg3bx6io6Nx6tQprFu3Dr179wYA3LlzB4MGDZIJw0RERESkwLRpwNSpcs3fnvjWaAH01KlTCgNoQEAAAyjVeLwSqoa5c+ciLy8PZmZm2L9/P/z8/KTH+vXrh2bNmuHdd9/F1atX8eOPP+LTTz9Ve44vv/xS+nrLli0YOnSo9HNfX1+MHz8eo0aNwpYtW5Camorly5fjrbfe0u6NEREREdVEn34KBAcDXboAJvLXXo7ePqrRsOam5lqVFR8fj3379sm19+3bFy+99JJWYxNVB7wSqqIzZ87gyJEjAIBp06bJBNAyb7/9Nlq1agUA+Pnnn1FcXKzWHM+ePUNSUhIAoHPnzjIBtLzPPvtM+vrEiRNqzUFERERUa7z7LtCtm1wATclMwe8Jv+NW9i2Nhu3RsIfGJZ08eZIBlGo9hlAVbdu2Tfp6qoLbOQDAxMQEkydPBgBkZWVJQ6uqioqKpK8reqa0SZMm0teFhYVqzUFERERUm/1w4gc0/bUpXtv1mkbnt3VrC38vf43OPXnyJPbv3y/X3q9fPwZQqlUYQlV0/PhxAICtrS26dOmitJ+//79/KcXGxqo1R506deDi4gIAuHnzptJ+KSkp0tfNmzdXaw4iIiKiGiMzE0hLU7n7sTvHMP/AfI2msrewx4iWIxAzOQa2FrZqn3/ixAmFAbR///7o06ePRjURVVd8JlRFf//9NwCgadOmMDNT/mVr2bKl3DnqePXVV/H111/j3Llz2LNnDwYrWMmt7LlRU1NTTJ8+Xe057t+/L9eWpsZf4ERERERGdfFi6WJDCQlqnRadHF15p3/cnXsXVmZW0s+drJw0fhb0ypUrOHDggFx7YGAgevXqpdGYRNUZQ6gKCgoKkJGRAQBo2LBhhX2dnZ1ha2uL3Nxc3Lt3T+25PvroI5w9exYHDx7EiBEjMGvWLPTv3x916tTBzZs3sXTpUhw9ehSmpqb45ZdfpM+gqsPT01Ptc4iIiIiM6sQJYOdO4MABtcNnmdtPb6vUr4F9AzR0aAiRSKTRPC9q3rw5mjZtihs3bkjbgoKC0LNnT52MT1TdMISq4Pnz59LXdnZ2lfYvC6GabNNiZ2eHPXv2YNWqVfj666/xww8/4IcffpDpM3LkSLz77rvo3r272uMTERERVTvLlwMa3P2lqaHNhuosgAKAmZkZxo0bhw0bNiAlJQUDBgxQuMglUW3BEKqCgoIC6WsLC4tK+1taWgIA8vPzNZrv7NmzWL9+vdLnQg8ePAh3d3e0atUKDg4Oao+v6ArttWvXEBgYqPZYRERERHqVlQW8+qr651lZASr83PYify9//DjwR/Xnq4SZmRnCwsKQnJyM1q1b63x8ouqECxOpwMrq3+cByq9gq0zZirXW1tZqz7V582YEBATg0KFDaNeuHbZu3YonT56gqKgIKSkp+M9//oPi4mIsXboUPXv21OhZzoYNG8p91KtXT+1xiIiIiPTuyBFAIlH/vD59AHPVn+Ec2GQgzrxyRuOFh1RhZmbGAEoEhlCV2NvbS1+rcottbm4uANVu3S3v0aNHmDJlCgoLC9GmTRucOHECoaGhcHFxgbm5OXx8fPDBBx9g586dEIlEuHz5MmbPnq3emyEiIiKqTtLT1T/H0RH4z3/UOuXN7m+iq0dXmJqYqj9fOefOnZO5i46I5DGEqsDKygp16tQBoHhl2fKysrKkIVTdBYA2bNggPffDDz+Era3i38L1798f/fv3BwBs2bIFWVlZas1DREREVCP5+QFvvw3ExgJduxp8+iNHjmDnzp1Yu3Yt93InqgBDqIrKVqG9ceMGxGKx0n5Xr16VO0dV5bd06dy5c4V9y/YqlUgkSE5OVmseIiIioholIKB0z9ATJ4DvvwfatpXrcivrFvbd2KeX6QVBwOHDh3H06FEApRct1qxZwyBKpARDqIp69+4NoPRW24QKlgUv+8sHgNr7PpXff7SioAsAxcXFCs8jIiIiqjU6dgTS0oDDhwFnZ4Vd7mTfQbul7eDziw+eFj7VeQllAfTYsWMy7ffv30dKSorO5yOqCZheVBQaGoqFCxcCAFauXKlwexSJRIKoqCgAgJOTE/r27avWHI0bN5a+Pn78ONoq+C1embK/6EQiEby9vdWah4iIiKg6yzMHfvUFDvTNxOMtQUr7iSVi/J3xt9Lj2ioLoMePH5c7NnToUC5CRKQEr4SqyNfXF3369AEALF++HCdPnpTr88MPP0hvqX3zzTdh/sKKbKtWrYJIJIJIJMKCBQvkzh869N89qb766is8ePBAYS2///47zp49CwDo0aMHXF1dNX5fRERERFXSvn1AWBjw2msyzflmQMh44P0gIMbsLi49vqT0Q98B9NChQwoDaHBwMLoa4ZlUouqCV0LVsGjRIvTq1Qv5+fkYMGAAPvzwQ/Tt2xf5+fnYsGEDfv/9dwBA8+bN8fbbb6s9fsuWLTF16lSsWLECDx48QKdOnTB37lz06dMH9vb2uHfvHjZs2IB169YBAExNTfEfNVd+IyIiIqryNm8uDaAlJXKHtrcEYnx0P6WbrZvKfQVBQExMDOLi4uSOBQcHS9fuICLFGELV0KlTJ2zcuBGTJk3Cs2fP8OGHH8r1ad68OaKjo2W2dVHHb7/9htzcXGzcuBHp6en46KOPFPaztbXF77//joCAAI3mISIiIqqyPvxQJoBKRMCi7qUB9Ki37qdzsXZBx3odVeorCAIOHjyIEydOyB0bNmxYpYtLEhFDqNqGDRuGixcvYtGiRYiOjsb9+/dhYWGBpk2bYsyYMZg1axZsbGw0Ht/S0hIbNmzAa6+9hlWrViE+Ph4PHjxAYWEhHBwc0KJFCwQGBuLVV19Fw4YNdfjOiIiIiAzoyRPg00+BAweAR4/+bX/2DABQbAKcaQCc8QDmDtZfGZamllg7cq1K+4MKgoADBw4ofCwrJCQEnTp10keJRDWOSBAEwdhFkPFdvny5woWQACApKQlt2rQxUEVERERUYz19WrqtyoULCg9n2AD9JwMX6+mvhEFNB6GXZy+EtgxFW7eKfwYCSgPo/v37ER8fL3ds+PDh6Nixox6qJKqZeCWUiIiIiAxr+3alARQAIkL1E0AtTC3g7+WPFcNXoKGD6neUCYKAffv24dSpU3LHGECJ1McQSkRERESGpWBFWbEJcMIT2NsU2N1cs2EvvHYBjlaOSo+72brBxlz9x6YKCgqQnJws1x4aGooOHTqoPR5RbccQSkRERESGlZ8v82mhKTB6LLCrheZDvtL5FXSop59AaG1tjYiICERGRiIrKwsikQihoaFo3769XuYjqum4TygRERERGY5EAmRmyjR93VvzANrEuQk+6P0Blg5dqoPilHN0dERERARcXFwwYsQIBlAiLfBKKBERERHpX34+MGdO6fOg6ekyh/Y2VX+4/w79L8I7hGt0e62mHB0dMWPGDJiZ8UdoIm3wO4iIiIiI9EsQgBEjgH37kG8G3HWVPRzvqfpQfwz7A0FNgtDIsZFua/xH2cYRIpFI4XEGUCLt8buIiIiIiPTr+nU8PbIPEWGlVz0LNfwJdP2o9QhrG6bb2soRBAHR0dEQBAHBwcFKgygRaYchlIiIiIj0Sjh5EoMmqXfF80Uf9fkI49qM011RLxAEAbt27cK5c+ekbQyiRPrBEEpEREREenU5/47GAfS3Ib8hpEUIGjg00G1R5SgKoOfOnYOpqSmGDBmit3mJaiuGUCIiIiLSn6IinL1xDLBX/1RLU0tM6zwNFqYWuq/rH4IgYOfOnTh//rxMu4mJCby9vfU2L1Ftxi1aiIiIiEj3SkqAWbMAZ2eUHIrRaIjejXrrPYDu2LFDYQAdPXo0Wrdurbe5iWozXgklIiIiIt2bPx9YskTj0x0tHbGw/0IdFiRLIpFg586duHDhgky7iYkJxowZg5YtW+ptbqLajiGUiIiIiHSrpASIilKpq5vEBo0a/nvF0cLUAn4N/TCl4xS0dWurl/IkEgl27NiBxMREmXYTExOMHTsWLVq00Mu8RFSKIZSIiIiIdOvuXSAzU6WuE0074sdX4vRc0L8kEgm2b9+OixcvyrQzgBIZDkMoEREREamsUFyI+PvxuJpxFRAXA0ePAX//DTx/9m+n9HSgy7+fxjaqYEBPLfZtUZOyAGpqaoqxY8eiefPmBquFqDZjCCUiIiKiij14AJw+jaTcW+h/+ws8Lnkqe7yxFmPXr69VaaqSSCTYtm0bLl26JNPOAEpkeAyhRERERKRYfj4wdiywaxeKTYDAt4DHdjqeQyTS8YCK7dq1S2EAHTduHJo1a2aQGoioFEMoERERUU135AiwfDlw+jQgFqt+3s2b0penGwCPdB1ADah9+/ZISkpCcXExgNIAGhYWhqZNmxq5MqLahyGUiIiIqCZ49gw4ehRIS5Nt37UL2LFD6+GTXbUeQiFPB8M8E+rt7Y0JEyZg7dq1EASBAZTIiBhCiYiIiKq7zZuBCROAf67yVScDmw402FxlQVQikaBJkyYGm5eIZDGEEhEREVUFz54BCxYA+/cDqamqn6fiVihV0X+H/het67auvKMONW6szSpKRKQLDKFERERExpabCwwcCMTHG7sSqUxrIM9c9nNlOqUC/cseHw3sD3h5yXawsARsbaWfNnRoiAFNBqBV3Va6K/gfJSUluHPnDnx8fHQ+NhHpBkMoERERkbHt3VtlAujO5sBH/YFL7qqf08euNb4b8gowdChgxJVmS0pKsHnzZly9ehUhISHo1KmT0WohIuUYQomIiIgMZf164LffgFOnZFepFQTj1VTOocbAyHGA2FTNE/sHAoPn6qMklZUPoACwY8cOiEQidOzY0ah1EZE8E2MXQERERFQrrFpVunhQbGzpAkKC8O+HoQUEAA8fAo8fSz8e3rqIN95sqn4ArQJKSkrw559/SgNomT179iA3N9dIVRGRMrwSSkRERKRrYnHpc57lffih4et46y3Zzx0cgL59gZ49AbPSHwPPp57HpK2TcCX9isbTOFg6aFOlVsRiMf78808kJyfLtJubm2PChAmwLfcsKhFVDQyhRERERLry8CEwcyYQEwM8f26cGlq0AEaNAr74AjBVfFkz5mYMtl7digM3DyD5SbLCPuoI8A7QegxNVBRAJ06cCK8XF0gioiqBIZSIiIhIF7KySm9zvX5dt+OmpgJ16qje30zxj3fZBdlIeJiAX0//iu3XtuuoOGBEyxF4yeslnY2nKrFYjE2bNuH6C19vCwsLTJw4EY0aNTJ4TUSkGoZQIiIiIkVyc4GMDNX7r1un+wA6cyZQr55WQxSXFGPK9inYkLQBEkGi8Tg+zj6wMrOSft7QoSEGNhmI2b6zYW5qXsGZuicWi7Fx40bcuHFDpp0BlKh6YAglIiIiKu/WLeDll4G4uNIFhAzhxecW27QpvaV2/nyNhrv39B5+O/Mbjt45ipP3T2pdXl/vvjgUcUjrcXShogA6adIkeHp6GqkyIlIVQygRERFRmSdPSm+pvXvXcHPOmwf8+KPOhrv79C4CVgXgVvYtnYznbOWM7wd8r5OxtCUWi7FhwwakpKTItDOAElUvDKFEREREZfbuNWwAbdCg9JZbHfrp5E86CaCBPoHwa+iHie0mokWdFjqoTDvFxcXYuHGjXAC1tLTEpEmT0LBhQyNVRkTqYgglIiIiKnP6tP7GLh+S7O1Lr7i+/TbQpInWQz8vfI7/O/Z/2H9zPy6kXdB4HFORKQY1HYR1o9YZddsVRW7cuKEwgIaHh6NBgwZGqoqINMEQSkRERBQbC2zdCvzyi+7HNjUtvc3X0VH3YwPILcrFoLWDcOLeCY3HCPAOwLeB36JFnRZVLnyWadWqFQYOHIh9+/YBYAAlqs4YQomIiKh2ys0Fzp8HVq4EVqzQ3zxffqm3AAoAe27s0SiA1rerj7FtxmJgk4EY1HQQRCKRHqrTrR49ekAQBBw7dgzh4eHw8PAwdklEpAGGUCIiIqpdSkqA2bOBP/5Qf/Xbtm1V79umDTByJDB2rHpzqOnYnWNq9W/v3h7fBH6DQU0H6aki/fLz80P79u1h++KKwkRUbTCEEhERUe3y4YfA0qXqn7dmDTBxou7rUaJEUoL4+/E4cvsIcopylPb79fSvKo1nY26Du3PvwtXGVVclGg0DKFH1xhBKREREtUNeHnDyJPDtt5qd36mTbuupQHFJMSZsmYDNVzbrbMxFgxZViwBaVFSELVu2oE+fPnzek6iGMjF2AURERER699NPgLMzEBio2fldugCtWum2pgr8HP+zzgJop3qdsDxkOaZ3nq6T8fSpqKgIa9euxbVr17B69Wo8fPjQ2CURkR4whBIREVHNtmUL8NZbQFGR+ueKREDfvsC+faWvDWRvyl6djBMVGoVzr53Dy51e1sl4+lRYWIi1a9fi7j/7tBYWFjKIEtVQvB2XiIiIarY//1T/nDZtgGXLgJYtAVfD38J69+ldnYzT3r29TsbRt7IAeu/ePZl2kUgEExNeMyGqaRhCiYiIqGZLSlK972uvAUFBwIgRQDUPP16OXmjn3s7YZVSqsLAQa9aswf3792XabWxsMHnyZLi7uxupMiLSF4ZQIiIiIgD49FPg88+NXYVO1Lerj32T9sFEVLWDdEFBAdauXaswgEZERMDNzc1IlRGRPjGEEhEREb30EvDuu8auAoIg4GbWTdzIvKG0z3/6/QcOlg5Kj7d1aws/Tz9YmFroo0SdKSgowJo1a/DgwQOZdltbW0yePJkBlKgGYwglIiKi2is4GJg8GRg6FLCxMWopZx+exbjN43Az62aF/V7u9DLc7ar3LaoVBdCIiAjUrVvXSJURkSEwhBIREVHttWQJ0KiR0aaPTo7Glr+3YF/KPjx4/qDyE2qAgoIChaveMoAS1R4MoURERFQ95ecDx48D8fFAcbHyfmlphqtJBWk5aTj94DR+PPkjjt45auxyDCo/Px9r1qyRC6B2dnaIiIhAnTp1jFQZERkSQygRERFVL8XFQGoqEBgIXL9u7GpUVlRShIlbJmLzlc0aj2FuYg57S3sdVmU4Zft+pqamyrQzgBLVPlV7yTQiIiKiMqdOAf37A/b2gJdXtQqgADBnzxytAigA+Hn6wcbcuM+uasrc3ByuL+y5ygBKVDvxSigRERFVXefOAfv3A9HRQGyssavRyINnD3D49mEsS1im1Tiu1q74dfCvOqrK8ExMTDBixAgAQFJSEuzt7RERESEXTImo5mMIJSIiIuN79Ej+uc4ffwR++kl/c9raAvXq6W14QRDw7oF38f3J77UaZ3iL4ejRsAfGthkLH2cfHVVnHGVB1MbGBr6+vgygRLUUQygREREZz6JFpR+3bhl+7kGDAAvd7qX5tOApPjn8Cfan7Me1J9c0HsfS1BJDmw/FquGrqu0zoMqYmJhg8ODBxi6DiIyIIZSIiIiMY8kSYO5c48zdrRvwxx86HTK3KBcD1wzEqQenNB5jeIvh+KrfV/Bx9oG1ubUOqzOsvLw8CIIAW1tbY5dCRFUQQygREREZ3t27wKxZuh/39deBli2VHzczA7p3Bzp1AkxN1RpaLBGjRFKi9PiOazs0DqAf9P4AA5oMgL+XP0QikUZjVBW5ubmIioqCIAiIiIhgECUiOQyhREREZDg7dpRe/dTl7bfNmgH+/sBbbwGtWuluXACPcx8jKjEK6y6tw6XHlyCWiHU6PgB8F/Qd5vecr/NxjaEsgD5+/BgAEBUVhcmTJzOIEpEMhlAiIiIyjNhYYPRo+QWINNWvX2mo1UPAOX7nOH4+9TN2XNuhl+BZZlSrUXiz+5t6G9+QcnNzERkZifT0dGnb48ePsWPHDowfP96IlRFRVcMQSkRERIaxZo32AbRRI2DhQsDXF2jSBNDDrasfxnyIhbELdT5uGVdrV8zpPgcB3gHo6dkTZibV/8exnJwcREVFyQRQAHBycuIiREQkp/r/rUdERETVw9mzqvd98bbaVq2A0FBg0iS9BM8ycXfj9BZAe3n2woyuMzCx/US9jG8sOTk5iIyMREZGhky7k5MTIiIi4OTkZJzCiKjKYgglIiIi/ZFIgFWrgL17gYQE1c55803g55+1mlYQBBy8eRD7UvbhwfMHKp+3IWmDVvMqIoIIj995jDo2dXQ+trE9f/4cUVFRCgPolClT4OjoaKTKiKgqYwglIiIi/ZBIgOnTgZUrVT+nWTPgnXc0njKnKAfJT5LxQcwH2J+yX+NxdGluj7k1NoBGRkbiyZMnMu3Ozs6IiIhgACUipRhCiYiISD/i4lQPoE2aAC+/DEydCtSvr/ZUzwufY+KWidhzY49eFxJysnJS+RnONnXbYETLEZjTfY7e6jEWBlAi0gZDKBEREaknLQ345Rfg8GHg6VPl/f7+W/UxDx4EvL01Lil0YygO3Tqk8fmqeKPrG1g8ZHG138dTW8+ePUNkZCQyMzNl2l1cXBAREQEHBwcjVUZE1QVDKBEREZW6fRs4eRIoKlLeJy0NeP993c5bt27pqrcauvv0rt4DKAC81vU1BlAGUCLSAYZQIiKi2q6oCAgLA7ZuNc78U6YAJiYan34u9ZzualGiZZ2WaOvWVu/zVHV79+5lACUirWn+Nz4RERHVDO++a5wA6ugIvPUW8PXXGp2e8DABM6NnYsTGETouTFYH9w44GH4QJiL+2BQcHAx3d3fp566urpgyZQoDKBGphVdCiYiIajNBALZsMfy8W7YAw4YBZhX/KFIoLsTJ+ydx/9l9mfbYu7FYlrBMrSkHNhmI2b6z1Tqndd3WaOzcWK1zajIbGxtMnjwZkZGRKCkpQUREBOzt7Y1dFhFVMwyhREREtVl+PnDvnmHnDAgAQkOBSp6vjLkZg9CNocgpytFqute7vI7QlqEY2HSgVuNQqbIgKpFIGECJSCMMoURERGQYDg6lVz//+99KA+id7DsIXh+MAnGBVlNO6zQNS4OXajUGybO1tTV2CURUjTGEEhERkfaOHCnd67Mi9epVevttmb039modQAHAt4Gv1mPURtnZ2Th58iQGDBgAU1NTY5dDRDUMQygREVFtIghAYeG/nxdUEPSsrYFp05QfNzUFfH2BgQMBV1fd1Qjg7ww19hhVor5dfQxrPkwH1dQuWVlZiIyMxNOnT5GTk4NRo0bBRIvVi4mIXsQQSkREVBtkZgJz5wJ79wLp6aqd4+AA/PqrXssqTyJI8HXs19h0eRMSHyVqPI6ZiRn8Gvrhf8P+h/r29XVYYc2XlZWFVatW4dmzZwCAK1euQCQSYeTIkQyiRKQzDKFEREQ1XV4eEBQEnNP/fpramLNnDpacWaLx+c1cmmF72HY0dGgIe0sumKOuzMxMREZGSgNomfT0dBQWFsLa2tpIlRFRTcMQSkREVJNJJMDGjVUugOYX5yNfnC/9PCs/S+UAamVmhZc7viz93M7CDv7e/ujfuD8szSx1XmttoCyAuru7Y/LkyQygRKRTDKFEREQ1UXEx8NZbwKZNwOPHmo1hY6PbmgCcuHcC7x18D/H34yGWiDUaY0XICoxvN17HldVeT548QWRkJJ4/fy7TXhZAbfTw54CIajeGUCIioppo2jRg9WrtxvD3100t/7iQdgED1wzUet/PPl59dFQRKQug9erVQ3h4OAMoEekFQygREVFNk5UFrF2r3RjOzsC8ebqp5x8rz6/UOoBO6TgFDR0a6qii2i0jIwORkZHIyZH9f1KvXj3egktEesUQSkREVNMkJJQ+C6qqunX/fW1rC7z0UumtvO3b67Ss+AfxGp/bpX4XDG8xHB/2+VCHFdVeygJo/fr1ER4ezgBKRHrFEEpERFTTiNV41vLePaCh/q4sCoKAnck7sffGXpx+cFqjMTaM2oBxbcfpuLLaiwGUiIyNIZSIiKi2mjdPZwG0QFyAxLREFEuKpW2CIOCdA+/g1INTGo/bu1FvBDcP1kWJhNLtViIjI5GbmyvT7uHhgfDwcFhZWRmpMiKqTRhCiYiIapthw0o/pk/XeqiyoPnbmd9ktlzRhJejl/R1Pbt6CPQJxHu93oOtha22ZdI/xGIxSkpKZNoaNGiASZMmMYASkcEwhBIREVUXglC64NDevcDdu8r7ZWYqPzZmTOm2LVrIzM/E/xL+h8O3D2Nfyj6txiqTMicFPs4+OhmLlCu75Xb16tUoKChgACUio2AIJSIiquru3wcSE4GZM4E7d4xaSmZ+JgKjAnE+7bzOxnSycpK5Ckr6VXbr7eHDhzFq1CgGUCIyOIZQIiKiqiorCwgJAWJjjV2J1NIzS3UaQAEgrE0YTE1MdTomVczDwwMTJ040dhlEVEsxhBIREVUlFy8Cf/wBnDhRutWKrpmYqH1KUUkRfjr5E/am7MWR20d0VoqZiRkmtJuAXwb/orMx6V95eXmwsbExdhlERHIYQomIiKqKM2eAoCDg6VP9zdG1q1rdxRIxxm0eh21Xt+mshHFtxmFmt5noWK8j7C3tdTYu/SstLQ1RUVHo3bs3evbsaexyiIhkMIQSERFVFZ9+qt8A6ugIjBih1imxd2M1DqBDmg2R+byla0uEtAiBv7e/RuORasoCaH5+Pg4cOACRSAQ/Pz9jl0VEJMUQSkREZEwPHgDffAMcPAj8/bf+5mndGli5EmjSRK3Tjt4+qlb/Ro6N8H99/w/hHcLVOo90IzU1FatXr0Z+/r/b5ezfvx/29vZo27atESsjIvoXQygREZGxpKYCAQHAjRuajzFnDvDyyxX3qVsX8PDQaPicohyV+96ccxPeTt4QiUQazUXaSU1NRVRUFAoKCmTavby80Lx5cyNVRUQkjyGUiIjIWFas0CyA+vmVPjs6dCjg66v7usopLClUqd8XAV+gsXNjvdZCyj18+FC692d53t7eGD9+PCwsLIxUGRGRPIZQIiIiYzl2TPW+pqalgXPZMqBdO/3VBEAiSLDgyAJsvLwRyU+SK+zbxLkJpneejvd6vafXmkg5BlAiqm4YQomIiIwlN1e1fp07A4cPAw4OeivleeFz7E/Zj5P3T+KHkz+odM7C/gvxfu/39VYTVe7BgwdYvXo1Cgtlr1g3btwY48ePh7m5uZEqIyJSjiGUiIioKjM3B77+Wq8B9HHuY/SL7IfL6ZfVOs/F2kVPFZEqGECJqLpSf8dqIiIiMgw/P2DbttLnP/Vozp45agdQALAxt9FDNaSK+/fvKwygPj4+DKBEVOXxSigREVFVdPo00K2b3qeRCBIcuHlAo3Nf8npJx9WQKu7du4c1a9agqKhIpr1JkyYYN24cAygRVXkMoURERFWRlZVBpikQFyAzP1Pt817p/AoaOTbSQ0VUkfz8fKxbt05hAA0LC4OZGX+0I6Kqj7fjEhERkcp6efbCt4Hf4r/B/zV2KbWStbU1Bg0aJNPWtGlTBlAiqlb4txURERFVyNzEHFEjohDWNszYpRCADh06QBAEbN++Hc2aNcPYsWMZQImoWuHfWERERLVUcUkx4u/HV9hn94Td6OPVB3YWdgaqilTRsWNH2Nvbw8vLiwGUiKod/q1FRERUC626sApz9szB86LnSvu427pjcLPBBqyK1NGkSRNjl0BEpBGGUCIiIkPbvx/YvRuIizPK9AdvHsTL21+GAMEo85Nqbt++DYlEAh8fH2OXQkSkUwyhREREhrRgAfD550YtYdPlTQygVdzt27exbt06CIKACRMmoHHjxsYuiYhIZ7g6LhERkaEkJRk9gALAxUcXVernZuum50pIkVu3bmHt2rUoLi6GWCzGunXrcOvWLWOXRUSkMwyhREREhhITo3pfkUhvZah6FTTQJ1BvNZBiN2/exLp16yAWi6VtYrEYFy5cMF5RREQ6xttxiYiIDCE1FVizRrW+1tZAs2ZaT1kgLkDc3ThcfHRRJnimPk+t9NxO9Trh45c+1roGUt3Nmzexfv16mQAKAK1atUJISIiRqiIi0j2GUCIiIn16+hQYMQI4fFj1c0aNAiwttZr2RuYNBKwKwIPnD9Q6r3uD7ni1y6sY1WoUHK0ctaqBVJeSkoINGzbIBdDWrVtj5MiRMDU1NVJlRES6xxBKRESkT6NGqR5ATUyAkSOB//1Po6l2XtuJRacW4cS9E8gX52s0xqrQVWhZp6VG55JmUlJSsH79epSUlMi0t2nTBiNGjGAAJaIahyGUiIhIXx4/Vv050BEjgBUrACenCrulZKZgf8p+3H16V6Z9+7Xt+Dvjbw0LJWO5ceMGNmzYIBdA27ZtixEjRsDEhMt3EFHNwxBKRESkL0lJqvcdO7bSALo6cTUitkXobXsVMxMzNLBvoJexSd7169exceNGBlAiqnUYQomIiLSVmgp8913pbbdZWf+237mj+hj9+lV4OCUzRa8BFAD6NOoDe0t7vY1P/1IWQNu1a4fQ0FAGUCKq0RhCiYiItPHoUWmAvHpV8zGWLQPcSvfkzMrPwu7ru3Hi3gkUlhRKuyw/v1zbSivU1KUpVo9Yrdc5qFRycjI2bdokF0Dbt2+P4cOHM4ASUY3HEEpERKSNlSs1D6AffggEBwN+fniU8whXM66ib2RfvV7tnO07Gy7WLtLPTUWm6NagG17yegk25jZ6m5f+dfPmTQZQIqrVGEKJiIi0cfSoZueNGQN89RVibsZgzm9tcCX9im7rKsfbyRuBjQPxf/3+D+527nqbh1QzcOBAlJSU4OzZswCADh06ICQkhAGUiGoNhlAiIiJt5Oaq1V0AsKcZsKdTKrb+2FDtfTzVMaXjFPwx7A+YmnCLj6pEJBJhyJAhEAQBJSUlGDZsGAMoEdUqDKFERESaEgRAIlG9O4DP+gJf+gMoigWKdFPGmNZj4O/lL/3cxtwG/t7+8HH20c0EpHMikQhDhw6FIAgMoERU6zCEEhERqev5c2D+fGDPHuDePZVPS/K0xJf+hZV3VEN9u/pYM3INLEwtdDou6UZJSQlMTRVfiRaJRBCJRAauiIjI+PirNyIiInUUFQGDBwO//155AD16FMjMlH7ErPs/nZbi5eiFwxGHGUCrqCtXruC3335Ddna2sUshIqpSeCWUiIhIHSdOAHFxqvV1cACcnaWfZhRlazTl1/2/lru1trlrc7RzbwcTEX+fXBVduXIFmzdvhiAIiIyMxJQpU+Do6GjssoiIqgSGUCIiInWcOKF6X2trjad5yeslvNToJUzvPB1eTl4aj0OGd/nyZfz1118QhNKtdrKzs7Fq1SpMnz4dtra2Rq6OiMj4GEI1cPfuXfzyyy+Ijo7G3bt3YWlpiaZNm2Ls2LF44403YGOju33WDh48iDVr1iA2NhapqakwMzODu7s72rdvj/79+yM8PBx2dnY6m4+IiCpRqOIzne7uQPPmag1tKjLF6NajERkaCUszSw2KI2NLSkrCli1bpAG0TJMmTXT68wERUXXGEKqm6OhoTJw4EU+fPpW25eXl4cyZMzhz5gz++OMP7N69Gz4+2q1ImJWVhalTp2L79u1yx549e4br16/jr7/+gp+fHzp27KjVXEREpIQgAA8eyAbPzEzVzv30U0CNRWdGtx6N5SHL4WDpoGaRVFUoC6Bdu3bFkCFDuAgREdE/GELVkJiYiLFjxyIvLw92dnb44IMP0LdvX+Tn52PDhg343//+h2vXrmHo0KE4c+aMxlconz59iqCgICQkJAAAhg4dirCwMDRt2hQlJSW4c+cOzpw5g82bN+vy7RERUXm//QZ88w1w96565wUFAVOmABMmqHVan0Z9GECrsUuXLmHr1q1yAbRbt24YPHgwAygRUTkMoWqYO3cu8vLyYGZmhv3798PPz096rF+/fmjWrBneffddXL16FT/++CM+/fRTjeaZPXs2EhISYGZmhjVr1mDcuHEyx3v16oUJEybgxx9/RElJiVbviYiIFFixApg5U/3zfvoJmDtX5+VQ1Xbx4kVs27ZNLoD6+vpi0KBBDKBERC/gknoqOnPmDI4cOQIAmDZtmkwALfP222+jVatWAICff/4ZxcXFas8TGxuL1atXAwA+/vhjuQBankgkgpkZf49ARKRzv/9u7AqommAAJSJSH0OoirZt2yZ9PXXqVIV9TExMMHnyZAClz3SWhVZ1LF68GABgZ2eHt99+W+3ziYhIB86c0ew8bsFRqyQmJiq8Bbd79+4MoEREFWAIVdHx48cBALa2tujSpYvSfv7+/tLXsbGxas1RVFQkXYho8ODB0mdKxWIx7ty5g7t376KoqEjd0omISF0SiWbn9e2r2zqoyrpw4YLML6jL9OjRAwMHDmQAJSKqAEOoiv7++28AQNOmTSu8BbZly5Zy56gqMTERBQUFAAA/Pz+kpaVh6tSpcHJygre3N7y8vODo6IghQ4bghDr71L3g/v37ch9paWkaj0dERADmzwe8vY1dBRlAcnKywtXr/fz8MGDAAAZQIqJK8IFCFRQUFCAjIwMA0LBhwwr7Ojs7w9bWFrm5ubh3755a81y5ckVmznbt2knnLd++Z88e7Nu3Dz/88APmarAAhqenp9rnEBERSrdc6dBBtq1ZMyA4GAgPN05NZHBeXl7w9PSU+Xfez88PQUFBDKBERCrglVAVPH/+XPpalW1XbG1tAQA5OTlqzZNZbu+5zz//HBkZGQgODsbZs2dRUFCAR48e4bfffoODgwMkEgneeust7NmzR605iIhIC3Z2wPnzsh+bNgGTJ6u1JyhVb5aWlpg4caL0l7o9e/ZkACUiUgOvhKqg7BZZALCwsKi0v6WlJQAgPz9frXlyc3OlrwsLCzFs2DBs27YNJialvytwc3PDjBkz0K5dO/j7+0MikeDdd99Ve/EDRVdor127hsDAQLXqJSIiqq3KguilS5fQpUsXBlAiIjXwSqgKrKyspK9VWRiosLAQAGBtba3xPADw3XffSQNoeb1798bIkSMBAElJSUhKSlJrnoYNG8p91KtXT60xiIiIajtLS0t07dqVAZSISE0MoSqwt7eXvlblFtuyK5qq3LqrbJ7GjRujRYsWSvsOHDhQ+vqMplsJEBHRvx4/Bt59F+ja1diVUBVy7do1jfb9JiIi5Xg7rgqsrKxQp04dZGRk4P79+xX2zcrKkoZQdRcAKt+/sgWQyvd9/PixWvMQEdELMjKAfv2Ay5eNXQlVIWfOnMHu3bvh4+ODsLAwmJubG7skIqIagVdCVdSqVSsAwI0bNyAWi5X2u3r1qtw5qmrTpo30dUlJSYV9yx+vaMsYIiJSwapVDKAk4/Tp09i9ezcA4ObNm9i4cWOF//4TEZHqGEJV1Lt3bwClt9omJCQo7Xf06FHp6169eqk1h5eXFxo1agQASElJqbBv+eMNGjRQax4iInrBkSOq9eOVsFrh1KlTcqvPp6SkyGylRkREmmMIVVFoaKj09cqVKxX2kUgkiIqKAgA4OTmhb9++as8zatQoAMCjR49w4sQJpf22bNkifd2nTx+15yEiIpTehrtpExAdrVr/Hj00mmb39d2YvHUyvjr+lUbnk+HEx8dj7969cu0BAQFo3769ESoiIqp5GEJV5OvrKw17y5cvx8mTJ+X6/PDDD/j7778BAG+++abcsyOrVq2CSCSCSCTCggULFM4zd+5c6Sq5c+bMkdm2pcyaNWtw5J/f2g8dOrTS50eJiEiB3buBBg2AceNU629qCsyYofY0y88tx9B1Q7H64mq1zyXDOnnyJPbt2yfX3rdvX/j7+xuhIiKimokhVA2LFi2CtbU1xGIxBgwYgIULFyI+Ph6HDx/Ga6+9hnfffRcA0Lx5c7z99tsazdGoUSN88cUXAICEhAT4+voiMjISCQkJOHToEGbNmoUpU6YAABwcHPDTTz/p5L0REdUqqanAiBGACttuAQA6dQI2bgSCg9WaRiwR4639b2lQIBnayZMnsX//frn2fv364aWXXjJCRURENRdXtFFDp06dsHHjRkyaNAnPnj3Dhx9+KNenefPmiI6OltluRV3vvPMOMjMz8c033+DKlSvS0Fmem5sbtm3bhmbNmmk8DxFRjSMIwLlzwKFDQGam8n6rV6seQA8fBgICKuxy/9l9LDy+EIdvH0ZWQZa0PS0nTbU5AJiZ8J9kYzlx4gQOHDgg196vXz8+8kJEpAf8F09Nw4YNw8WLF7Fo0SJER0fj/v37sLCwQNOmTTFmzBjMmjULNjY2Ws+zcOFChISEYOnSpTh+/DhSU1NhZWWF5s2bIyQkBLNnz4ajo6MO3hERUQ0hCMDcucAvv+huTJEIaNdO4aGs/CwcunUI8ffj8f3J77WeyreBr9ZjkPri4uJw8OBBufb+/ftLFyUkIiLdEgmCIBi7CDK+y5cvo23bthX2SUpKktlGhoioStm4EQgL0+2Yfn6AgkXi9t7Yi9ANoSgsKdTJNF09uiJ+WjxMTUx1Mh6pJjY2FjExMXLtgYGBaq9wT0REquMzoUREVDMoWNFUKz4+wIYNcs2pz1N1FkBtzG0wouUIHAg/wABqYMoCaFBQEAMoEZGe8XZcIiKqGe7e1c04fn7AggVA796AgscrDtw8oJMAOrf7XHwd+DUszSy1HovUU1BQgNOnT8u1DxgwAH5+fkaoiIiodmEIJSKiWkUAkG0FCCIFB+3sgG3rAXt7AAVAfoFclzMPzmhdQ0OHhnin1zsMoEZiZWWFyZMnIzIyEjk5OQAYQImIDIkhlIiIagWJCPikL7CmPXDXSVmvHGCpt95qsLOwQ1/vvvhp4E/wsPfQ2zxUuTp16iAiIgKRkZHo1asXevToYeySiIhqDYZQIiKq+d55B2+6nsLigmMGmc7N1g2HIw7Dy9FLpt3SzJJbsVQhderUwcyZM2FlZWXsUoiIahX+S0hERDWaAODq4K5YfOw7vc/lbOWMNSPXwN/LH7YWtnqfj1QjCAJEIkX3X4MBlIjICLg6LhER1ViLfQHPt4DWx8YZZL5vg77FkGZDGECrCEEQcPjwYcTExIA70hERVR28EkpERDVGgRmwqQ1wqDEQ2dHw8/t7+Rt+UlJIEAQcOXIEx479ewt2//79lV4RJSIiw2EIJSKiau/+s/u46pSJUfOBZ0a6u3Jap2lo6tLUOJOTjLIroMePH5e2xcXFwcTEBP369TNiZUREBDCEEhFRNZaSmYKRm0bi4qOLQHvNx3lxASF1NHNthuBmwZjdfTavslUBgiDg0KFDiI2NlTvm6OhohIqIiOhFDKFERFQt5RTlICAyAPef3ddqnOUhy/Fyp5d1UxQZlSAIiImJQVxcnNyx4OBgdOnSxQhVERHRi7gwERERVTu3s29j3t55WgdQfy9/jGtjmEWLSL8EQcDBgwcVBtBhw4YxgBIRVSG8EkpERNVGdkE2gtcFI+6efNBQx6Cmg9C/cX/M6DqDK9nWAIIg4MCBAzh58qTcsZCQEHTq1MkIVRERkTIMoUREVG2M+XOMxgHU1twW7/V6Dx+/9DGf3axBBEHA/v37ER8fL3ds+PDh6Nixo+GLIiKiCjGEEhFRtfAk7wlibsaofd6nR4AhH65Ap34TYWFqofvCyGgEQcC+fftw6tQpuWMMoEREVRdDKBERVQt/Z/wNAYLK/T87Aoy6ArR7DODbtgADaI1SUQANDQ1Fhw4djFAVERGpgiGUiIhqnIffA/VzjF0F6dOBAwfkAqhIJEJoaOj/t3ff4VFV+R/HP5OEJJAEAgQkNOlFkKp0hFBFRClKTwKuZdddy6prYXcVdV0Vsf3WRdeykAQkoNKLIKGLSBVEell6CwRCQnru7w+WWcYUZpKZuTOT9+t58uzMueee8x33MplPzp171apVKe7XAwBwOa6OCwDwKZ8tIICWBbfeeqv8/P73McZisWjIkCEEUADwAoRQAIBPeP0/9bXlX9LD28yuBO7QtGlTDR8+XH5+ftYAevvtt5tdFgDADj5xOu7u3bt12223mV0GAMAkA4Ja6i/TdpldBtysadOmevDBB5Wbm6uWLVuaXQ4AwE5evRK6f/9+jRo1iosPAIAPS89O1/y98/X8d88X3WkXAbSsatasGQEUALyMV66EHj58WK+++qq+/PJL5eXlcb83APBRlzIvqW9CX205tcXsUmASwzB0+vRp1axZ0+xSAABO4lUroUePHtXDDz+s5s2ba/r06crLyzO7JACAC73w3QsE0DLMMAwtWrRIX3zxhfbs2WN2OQAAJ3HbSuiePXt0/PhxXbx4UcHBwWrQoIFatmxpc2W7opw8eVJ/+9vfNHXqVOXk5Mgwrt0nzmKxWB8DALxcVpZ07pxN07L9S0o/rsUiNWhQ+nHgVoZhaOHChdq+fbsk6euvv9aDDz6oZs2amVwZAKC0XBpCL1++rLfeekvx8fE6c+ZMge1Vq1bVo48+qhdffFGhoaEFtqelpem1117TRx99pKysrALh0zAM9ezZU6+88oorXwYAwJWSk6WHH5a++066etVm08m/SvK/+RDNzxezsWtXqWrVUpUI9zIMQwsWLNBPP/1kbcvPz9ecOXP01FNPKSQkxLziAACl5rIQunPnTt199906e/ZskauVycnJevPNNzVnzhwtW7ZMderUsW5buXKlxo0bp5MnTxYaPnv16qVXXnlF3bt3d9VLAAC4WlaW1KePtGNHqYa5f18RG5o1k2bPLtXYcK/8/HwtXLjQJoBKkp+fn4YOHUoABQAf4JLvhJ46dUpRUVE6c+aMDMOQxWIp8scwDO3du1f333+/cnNzJUlTpkxR//79deLECev+0rW/jPbu3Vtr167VihUrCKAA4O1++KFUAdQ/X/p0gXTX0V9tqFtX2rBB+vlnKTKydDXCbfLz8wusgErXAujw4cM5FRcAfIRLVkKffPJJpaSk2ITHolwPojt27NBnn32myMhI/eEPf7DZZhiG+vbtq1deeUVdunRxRckAADNs2lTiXRd+eS18VswqZOOECVLnziWvC26Xn5+v+fPna+fOnTbt1wNo06ZNTaoMAOBsTl8JPX78uObNm2cTIKOiojR//nydPXtW2dnZSk5O1rJlyzR48GCblc6PP/5YTz75pKT/BdC2bdtq5cqVWrZsGQEUAHzNf8+AcVRwjnTv/iICqCT17FnikuB+RQVQf39/jRgxggAKAD7G6SF01qxZys/Pl3QtSD711FNKSkrSoEGDVK1aNQUEBKhKlSrq27ev5syZo8mTJ1tXSn/55RedOHHCOtarr76qzZs3qycfJgAA9vrd76QmTcyuAnbKz8/XvHnzigygTfj/EgB8jtNPx928ebP1cbNmzfTuu+8W2/+ZZ57R0qVLlZSUZF0RtVgs+uSTT/TII484uzwAgAfK8ZM21ZKOVL6hMSxM+ZYrhe9gsUi3Nbdta9RIGjjw2pV2//v7BJ4tPz9fc+fO1a5du2zarwfQxo0bm1QZAMCVnB5Cf/75Z+vjhx56yK77gD722GNKSkqSdC2A9uzZkwAKAGXElprS3WOlCxV+vaWIACpJwUHSL7+4siy4WHEBdOTIkWrUqJFJlQEAXM3pIfTChQvWxx07drRrn1/3Gzt2rFNrAgB4lsRdiZq+c7pW5SzX1UfNrgZm2Lt3LwEUAMoop38n9PLly9bHt9xyi137XO93/buhrVu3dnZZAAAP8dGmjzTqm1FafGCxrirH7HJgkttuu0133XWX9XlAQIBGjRpFAAWAMsDpK6HZ2dnWx0FBQXbtExgYaPO8cuXKRfQEAHizvPw8vbzq5VKPUyuslhOqgdl69uwpwzD0ww8/aOTIkWrYsKHZJQEA3MAl9wktLX9/f7NLAAC4wC/nf1FKZkqpx+nfsL8TqoHZLBaLoqKi1KZNG1WpUsXscgAAbuL003EBAChKenZ6qcfoUKuD3uj9hhOqgSewWCwEUAAoYzxyJRQAgEf9Oiig3R3W58EBwep+a3f1bdBXIYEhJlYGR+Tl5Wnx4sXq3LmzqlWrZnY5AAAP4NIQainhfdpKuh8AwLvdkiYN3C998K0U9uoQaeCLZpeEUsjLy9NXX32lffv2af/+/YqNjSWIAgBcE0Kvh8iuXbsqIMDxKRzdz2Kx6NChQw7PAwDwHN2OSmunSvwZ0jfk5ubqq6++0v79+yVJ6enpiouL07hx4xQREWFydQAAM7lsJdQwDJ04ccIt+7FyCgDeLySHAOorfh1Ar8vOztbVq1dNqgoA4ClcFkLdFQyv31sUAODBcnKkH3+U5n8khZpdDFwpNzdXs2fP1oEDB2zaAwMDNWbMGNWtW9ekygAAnsIlIZRgCACwunpVGjRIWrlSqi3pYTv3q1fPhUXBFXJzczVr1iwdPHjQpp0ACgC4kdND6KpVq5w9JADAm02YcC2AOqJcOal3b9fUA5coLoCOHTtWderUMakyAICncXoI7dGjh7OHBAB4g337pG+/lQ4ftm3/v/9zbJxy5aRvvpG4iqrXyMnJ0axZswpcJJAACgAoDPcJBQCUnGFI585JM2ZIzz5b+vGmTZP69JFq1Sr9WHCLnJwcJSYm6vCv/vgQFBSksWPHqnbt2iZVBgDwVIRQAIDj8vOvnWabkCCdOuWcMdu3l2JjnTMW3KK4ABodHa1a/DEBAFAIQigAwHF//KPjp9neDPeO9Co5OTmaOXOmjhw5YtNOAAUA3IzbQmheXp5OnDih5ORkSVJERIRq164tf39/d5UAACgtw7j23c9fBdB8i/SPDtLiJtLhykXvnsGfPn2GxWIp8Ds8ODhY0dHRqlmzpklVAQC8gcs/DqxevVoffvihVq1apStXrthsCw0NVVRUlJ544gn15iqIAODZPv5Y+tvfrKff5lmk7ZHStkjpsUEm1wa3CwgI0IgRI6xXxCWAAgDs5bIQmpWVpZiYGH399deSCr936JUrV7Rw4UItXLhQQ4YM0fTp0xUcHOyqkgAA585Js2ZJ69dLGRn277dwoc3TsyFSnxhp1y1Org9e5XoQnT9/vrp06aLIyEizSwIAeAGXhND8/Hzdc889Wr16tTV8WiyWQvte3z537lwNGDBASUlJ8vPzc0VZAFA2GYa0f7+0c6c0fLhThnxwuPMDaDn/cs4dEG4REBCgYcOGmV0GAMCLuCTtvfPOO1q1apWka+HTYrHIMIxCf27cvnbtWr399tuuKAkAyqbvvpPq1JGaNXNaAL1QXlpf1ylD2ehUq5PzB4VTZGdnKzMz0+wyAAA+wukhNDMzU2+//bZ15dMwDIWFhenZZ5/VsmXLtHv3bv3yyy9atmyZnn32WVWsWNEaRg3D0KRJk5ThyCliAIDC7dwp3XefdPKkU4c9WEUyCj+5pcSqlq+qkS1HOndQOEVWVpZmzJih6dOnE0QBAE7h9BD69ddf69KlS9bn7dq10549e/TOO++ob9++atasmZo3b66+ffvqnXfe0Z49e9SuXTtr/9TUVOv3SAEAJWAY0t690kMPSR4eGsr5lVPPej2VFJOkhlUaml0OfuV6AD127JhOnjypGTNmKCsry+yyAABezunfCV27dq2kayugFStW1IIFC4q9UEGNGjW0YMEC3Xbbbdar565du1bR0dHOLg0AfN/+/dLAgdLBg2ZXoo8GfKThLYo/BTg0MFTly5V3U0VwxPUAevz4cWvbiRMnNHfuXI0cyao1AKDknB5Ct27dKunad0Efeughuy7VXrNmTT300EP64IMPZLFYrGMAAByQkSFFRVlvoeIqyRWK3z7lninq17AfK5teLDMzUzNmzNCJEyds2itUqKBevXqZVBUAwFc4PYSePn3a+rhfv35279e/f3998MEHMgxDZ86ccXZZAOD71q8vWQD997+lBg1u2i3hzHK9cTRB+zKOF9mnT4M++t2dv3O8BniMzMxMTZ8+XSd/9V3ikJAQxcTEqHr16iZVBgDwFU4PoZcvX7Y+rlvX/ssn3tj3xjEAAHb6+Wf7+zZrJv3mN9KoUVKtWjft/tUvXyl29ZsyVPCez/AdxQXQ2NhYVatWzaTKAAC+xOkh9MYr24aGhtq9X4UK/zu/i6vvAYAD5s6V4uOlefPs61+rlrR7t1TE/ZsL86+t/yKA+rjMzEwlJCTo1K9W0wmgAABnc3oIBQC40RdfSA8/bH//kBDpyy8dCqCStOnkJrv6VQyq6NC48AwZGRmaPn16gQAaGhqq2NhYRUREmFQZAMAXOf0WLQAANzEM6c9/tr//pEnStm3SXXc5PFVufq5d/Xre2tPhsWGujIyMQldACaAAAFdhJRQAvNX+/dLZs/b1/fvfpT/9yaXltI9sr+jW3F7Lm1wPoDdeVFAigAIAXIsQCgDeKj3d/r6dO7usjG51u6l3/d56quNTCg8Od9k8cL5FixYVCKBhYWGKjY1V1apVTaoKAODrXBJCLf/9rtHIkSMVHBxs1z6/vhiRI/chs1gsSkpKsr9AAChL7rxT6tLFJUP7W/y1bvw6l4wN1+vfv79Onz6tlJQUSQRQAIB7uGwl1DAM/fjjjyXed82aNXb3tTh4gQ0AKBOqVpUGDJD+8Q8pMNDsauCBKlasqNjYWMXFxSkvL0+xsbGqUqWK2WUBAHycy0JoSYIhYRIAnKRtW2nLFsmP68+heJUqVVJsbKzy8vIIoAAAt3BJCDUM7iUHAKaqWLHUAfRc+jm9sfYNJR1JUkZuxs13gNeqVKmS2SUAAMoQp4fQqVOnOntIAICbJV9NVlRclHaf3212KXCC9PR0bdmyRXfddRdnHQEATOf0EBobG+vsIQEAbjbz55kEUB+Rnp6uuLg4nT9/Xqmpqbr33nsJogAAU3GLFgDwRoYhHTvm1CEzczM1IWmCFh9YrP0X9tu1T6A/FzzyZGlpaYqPj9f58+clSdu2bZMkgigAwFROD6EPPfSQ9fHkyZO5yAEAONsXX0ivveaUEHop85KSDidpy6kteuv7txzev0sd19z6BaWXlpamuLg4JScn27QfPnxYV69eVUhIiEmVAQDKOqeH0GnTpln/ujpx4kRCKAA40+zZ0sMPl3qYtOw0/XLuF0XFRZX4okP+Fn892fHJUtcC57ty5Yri4+MLBNDw8HCNGzeOAAoAMJXLro7LaT4A4AKff16q3efsmaM31r2h7ae3y1DJr2TeqXYnPd/led3X9L5S1QPnu3LliuLi4nThwgWb9sqVKys2NpYr4QIATMd3QgHAm2zZYl+/QoLGov2LNPyr4coz8ko8vZ/FT6kvpiokkJU0T0QABQB4A0IoAHi6zExp6lQpKUlKSbFvn169CjT9c/M/SxVAJen1qNcJoB4qNTVVcXFxunjxok17lSpVFBsbq4oVK5pUGQAAtgihAODJMjOloUOlpUvt36dDB2ncuALNG09sLHEZ3et215jbx+ixOx4r8RhwHQIoAMCbEEIBwJPNnm1/AO3X79oK6G9/W+jpuDl5OQ5NHVIuRItGL1LPej0d2g/udfnyZcXFxSnlV6vkVatWVUxMDAEUAOBxCKEA4MlWrrSvn8UiLVvmlClf6fGKOtTqoLtuvUuhgaFOGROuUVwAjY2NVVhYmEmVAQBQNEIoAHgye78D2r59qaZpW6OtetfvrT91/ZOqh1Qv1Vhwn8uXLystLc2mLSIiQjExMQRQAIDHIoQCgC8o5Dug9rLIom2PbXNeLXCbunXravTo0fryyy+Vk5OjiIgIxcbGKjSUFWwAgOfyM7sAAEApVK4s/f3v0u9/X2SXrNwsrT26Vuk56W4sDO5Sr149jR49WjVr1iSAAgC8gktWQi0WiyRp5MiRCg4OdsUUBeZLSkpy+TwA4FH+8Q/pd7+T/P2L7DJr1yw9tOAhXc256sbC4G716tXTww8/bP39CwCAJ3PZ6biGYejHH3901fA28/BLF0CZ1KJFsQF008lNGjNnTKnvDQrPkJOTo3LlyhW5nd+FAABvwem4AOAJpk+XeveWwsOlihX/97N4cYmHnLtnLgHUR6SkpGjKlCnato3v7gIAvJ9LL0xkGIYrhwcA73L4sJSUJJ0/b9v+z39Kp045fbpd53fZ1S8yLNLpc8N5UlJSNG3aNKWmpmrhwoWyWCxq27at2WUBAFBiLguhFotFkyZNUkREhKumAADvMWmS9MILZldRqD4N+phdAopw8eJFxcXFKTU11dq2YMECBQUF6bbbbjOxMgAASs4lIfT69zQffPBB1a1b1xVTAID3WLnSNQE0KKjUQ9xW7TZN6jPJCcXA2S5evKhp06bpypUrNu233HKL6tWrZ05RAAA4AfcJBQBXK8X3OosUHCy1a1fi3ZtWbao/d/+z7m1yryqXr+zEwuAMFy5cUFxcXIEAWqNGDUVHR6tChQomVQYAQOkRQgHA1Y4ccf6YTz11LYiW0L/u/Zd61OvhxILgLMnJyYqLi1NaWppNe40aNRQTE6Py5cubVBkAAM5BCAUAb9KmjTR6tPTcc2ZXAhcoKoBGRkYqOjqaAAoA8AmEUABwllOnrp16u2WLlJ//v/aS3FbjpZekN96QuPdjmUEABQCUFYRQAHCGXbukXr0K3n7lZh56SGrc+H/PK1e+Nk6jRgTQMuT8+fOKi4tTenq6TXvNmjUVHR2t4FKceg0AgKchhAKAM0RHOx5AJWnUKKlP6W6RcjjlsObtnaetp7cq3/jfCuzWU1tLNS7co6gAWqtWLY0dO5YACgDwOYRQACit8+eln34yZepNJzepb0JfpWal3rwzPE5mZiYBFABQ5vi5YlALp5ABKEvOnCn5vi1alGrqmLkxBFAvFhwcrO7du9u01a5dmwAKAPBpLlkJNQyDIAoAN9OlixQZ6fBup66c0vSd05W4K1H7Luwr0dS8R3uOjh07yjAMLVu2zBpAg4KCzC4LAACXcXoIPXLD/fBq1arl7OEBwHOcOSMlJEj//Kfj+7Zvr4yvZ2rz0bXKyMmwe7djl4/p0UWPOj7fDSyyqEW10q3Awrk6deqk0NBQNW7cmAAKAPB5Tg+ht956q7OHBADPc/Kk1LOndPBg8f0aN5Zef/1/zy0WGa1b67XTM/XmF02UlZfl0jIL06dBH1WtUNXt86J4LVu2NLsEAADcggsTAUBJvP76zQOoJFWrJo0YYdP0ry2faOKaV11UWPGi6kXpqwe/MmXusu7MmTPKy8vjLCEAQJlHCAWAopw7J02fLq1aJZ06Zbtt2zb7xvAreP23xF2JTiiueM0jmuudvu9Yn1ssFrWp0UY1w2q6fG4UdPr0acXHx8swDEVHRxNEAQBlGiEUAG5kGNLy5dLnn0vz50s5OaUbr337Ak07z+4s3Zg3EegfqLjBcbqz1p0unQf2uR5AMzMzJUkJCQmKiYlRzZr8QQAAUDa55BYtAOCVzp6VevWS7r5b+vrr0gfQcuUKnIorSYaM0o1bjLqV6mreiHkEUA9x6tQpmwAqSVlZWVq3bp2JVQEAYC5WQgHguscek1avds5YtWtLU6ZInTs7Z7ybmDN8jlrd0koNKjfg9ise4tSpU0pISLAJoJJUr149DR061KSqAAAwHyEUAI4dk5YsuXb6rTMcOCA1bCiVIAwOaz7M7hAZ5B+kbnW7aWjzoaoeUt3hueA6J0+eVEJCgrKybK9+XL9+fY0aNUrlypUzqTIAAMxHCAVQduXnS7/9rfTZZ84b85NPpEaNCjSfSD2hF1e8qKQjSbqUeanI3Wc9MEv+fv7OqwduV1QAbdCggUaOHEkABQCUeYRQAGXXe+85L4DWqiU999y1U3p/5Vz6OfWc1lOHUg45Zy54rBMnTmj69OkEUAAAikEIBeB7srKkH36QtmyR8vKK7vfii/aPuXq1VNTVTAMDpbp1izz99qtfviKAlgEnTpxQQkKCsrOzbdobNmyoESNGEEABAPgvQigA33L2rBQVJe3Z47wxH3xQ6tGjxLuvP77ern6B/oFcVMhLHT9+XNOnTy80gI4cOVIBAfy6BQDgOn4rAvAt0dHODaDStVu2lEJWbtbNO0nqVreb/CzcOcvbHDt2TDNmzCgQQBs1aqQRI0YQQAEA+BV+MwLwfjk5106/XbJE+u475449frw0bpxzxyxEOb9y+lOXP7l8HjiXYRjasWNHgQDauHFjDR8+nAAKAEAh+O0IwLtlZUlDh14LoM70+utSr15Sp06Sn2tXJ+9tcq+e7PCk+jbs69J54HwWi0UDBw5Udna2du3aJYkACgDAzfAbEoB3+/vfnR9Af/Mb6S9/cXi3vcl79fXur7Xl1BblG/nW9k0nNxW5z3fR36lPgz4lKhOewc/PT0OGDJFhGMrJydGDDz5IAAUAoBj8lgTgfQxDWr/+2qm3r7/u3LG7dZPef9/h3TYc36D+0/srLTvNufXAK/j5+Wno0KEyDEP+/tznFQCA4hBCAXgXw5D+9Cfp3Xcd3/dvfyt6W2Cg1LWrdOedUglupfHwgocJoGWAYRhFXsHYz8WnbQMA4CsIoQC8y/LlJQugs2ZJw4c7vx5Jp6+c1p7kkl2R1yJuyeItjhw5ohUrVmjUqFEKDQ01uxwAALwWIRSAZ0hNlebMkdauldLTi+43e7bjY7/yissCqCQlX00u0X4WWdS6RmsnVwNXOHLkiL788kvl5uYqPj5esbGxCgkJMbssAAC8EiEUgHulpkqHDtm2Xbok9e0r5eU5d64ZM6SePaWaNZ07rpP0b9RfERUizC4DN3H48GHNnDlTubm5kqTz588rPj5e48aNU/ny5U2uDgAA78MXWErg2LFjeu6559S8eXOFhISoSpUq6tChgyZPnqyrV6+6ZM7Tp08rPDxcFotFFotFPXv2dMk8gMtcuCD16ydVqSK1a2f706uX8wPoM89Io0d7ZAC1yKIBjQZo9gMlWNWFWx06dMgmgF4XERGhwMBAk6oCAMC7sRLqoMWLF2vMmDG6fPmyte3q1avavHmzNm/erM8//1xLlixRgwYNnDrvE088YTMn4BX27JESE6VNm6Rvv3XfvCNGSG+95b75itGiWgt9dM9H1ucWWXT7LberSvkqJlYFe1wPoHm/+gNJixYtNGTIEK6CCwBACRFCHbBjxw4NHz5cV69eVWhoqF566SVFRUUpIyNDiYmJ+uyzz7Rv3z4NHDhQmzdvdtqFKxYuXKhvvvlG1atX17lz55wyJuByGzdeW/m8csV9c77/vvTgg1KtWu6b8yYqBVdSz3o9zS4DDjp48KASExMLBNCWLVtqyJAhXAkXAIBS4LeoA55++mldvXpVAQEBWr58uSZMmKDOnTurV69e+vTTTzVp0iRJ0t69e/Xee+85Zc60tDT9/ve/lyRNnjzZKWMCbvH44+4NoF26SE8/7VEBFN7pwIEDBFAAAFyI36R22rx5s1avXi1J+s1vfqPOnTsX6PPss8+qefPmkqQPPvhAOTk5pZ53woQJOn78uKKiohQdHV3q8QC3uHhR2r7dffN17SotWOC++eCzDhw4oFmzZhUIoLfffjsBFAAAJ+F0XDvNmzfP+nj8+PGF9vHz81NMTIxeeuklpaSkaPXq1erbt2+J59y0aZP++c9/KjAwUB9//HGJxwHcLiXFOeO8+abUuphbmFgs0u23s/oJp9i/f79mz55dIIC2atVK999/PwEUAAAnIYTaad26dZKkkJAQtW/fvsh+PXr0sD5ev359iUNobm6uHn30UeXn5+uFF15Q06ZNSzQO4PGaN5eqVfvf88BAqXNnaexYqUkT8+pCmbJv3z7Nnj1b+fn5Nu0EUAAAnI8Qaqc9e/ZIkho1aqSAgKL/szVr1qzAPiUxefJk7dixQw0bNtSECRNKPE5hTpw4UaDtzJkzTp0DuKkWLaRZs679L2Ciw4cPFxpAW7durfvuu48ACgCAkxFC7ZCZmank5GRJUu3atYvtW7lyZYWEhCg9PV3Hjx8v0XyHDx/Wa6+9JkmaMmWKgoODSzROUerUqePU8VDGXbkirVkjnT37v7abXcX5+HHpJv+WPElOXo7Opp8tcvuZNP6I481q1KihatWq6ewNx3CbNm00aNAgAigAAC5ACLXDlRuu8GnPbVeuh9C0tLQSzffYY48pIyNDI0aMUL9+/Uo0BuAWs2dfO23WkYtwtWnjNQE0NStVjyx8REsPLNWVbDde6RduVaFCBcXExCg+Pl5nz55VmzZtdN9998lisZhdGgAAPokQaofMzEzr48DAwJv2DwoKkiRlZGQ4PFd8fLxWrFihihUr6v3333d4f3sUtkK7b98+9enTxyXzwUft2CGNHi396iIu3u4/l/6juXvmavOpzZq5a6bZ5cBNrgfRzZs366677iKAAgDgQoRQO9x4Omx2dvZN+2dlZUmSypcv79A8ycnJevbZZyVJb7zxhiIjIx3a316FnVJ8+fJll8wFH7Zokc8F0O2nt6t3fG+lZDrp6r7wKhUqVLC5uBwAAHANvuxih7CwMOtje06xTU9Pl2Tfqbs3euaZZ5ScnKw77rhDjz/+uGNFAu62b1/J9vPgU3HHzx/v9ABaMaiiU8dD6Rw7dqzALVgAAIB7sRJqh+DgYEVERCg5ObnQK8veKCUlxRpCHbkA0KlTp5SQkCBJ6tWrl2bPnl1s/3PnzikxMVGSVL9+fXXs2NHuuQBT3X23qdPn5OUoYWeCVhxeoXPp/7uA0vmr57Xz7E6nz9fz1p5OHxMl88svv+ibb75Rs2bNNGzYMPn7+5tdEgAAZRIh1E7NmzfXunXrdPDgQeXm5hZ5m5a9e/fa7GOvG0/znTRp0k3779mzR6NGjZIkxcbGEkLhHUaOlB57zLTpc/JyNOLrEZq7d65b5ru9+u36TbvfuGUuFG/Xrl2aM2eODMPQnj17NGfOHA0bNoyr3wIAYAJCqJ26deumdevWKT09XVu3bi0y9K1Zs8b6uGvXru4qD/As994rNWr0v+eVKklRUVK3bpKJq0/z9s5zSgCtHlJdERUiitxeObiyetXvpSc6PFFsP7jHzz//rLlz58owDGvb7t271bhxY7Vp08a8wgAAKKMIoXYaPHiw3nzzTUnS1KlTCw2h+fn5io+PlySFh4crKirK7vHr1atn8wGpKNev2NijRw+tXr3a7vEBt/rNb6TBg82uooCVR1aWav/aFWtrwcgFahvZ1kkVwdUKC6CSdOedd6p169YmVQUAQNnGeUh26tChg7p37y5J+uKLL/TDDz8U6PPuu+9qz549kqSnnnpK5cqVs9k+bdo0WSwWWSwWTZw40eU1A7B1MfNiifZrULmBfv7dzzr29DECqBfZuXNnoQG0Q4cOGjBgALdhAQDAJKyEOuDDDz9U165dlZGRoX79+mnChAmKiopSRkaGEhMT9emnn0qSmjRpYr3VCgDvFuAXoIQhCWpZvaXZpcABO3bs0Pz58wsE0I4dO6p///4EUAAATEQIdUDbtm01a9YsjR07VqmpqZowYUKBPk2aNNHixYttbusCwDvVrlhbHw34SF3qdDG7FDjgp59+0vz58wu0E0ABAPAMhFAHDRo0SDt37tSHH36oxYsX68SJEwoMDFSjRo304IMP6g9/+IMqVKhgdplAmZaTl6NNJzdpy6ktyjP+d0/IfclF39v0g/4f2ITN8OBwNarSiMDiZYoKoJ06dVK/fv34/xMAAA9ACC2BW2+9Ve+9957ee+89h/YbN26cxo0bV6q57bl4EVCWnU8/r17xvbTr3C6H9mtStYnurHWni6qCO2zfvl0LFiwo0N65c2f17duXAAoAgIcghALwehuOb9Cb69/U2qNrlZqVanY5MEFRAbRLly7q06cPARQAAA9CCAXg1X488aP6JfRTek662aXAJJmZmVqxYkWB9q5du6p3794EUAAAPAy3aAHg1f6x6R9OCaD1K9d3QjUwQ3BwsMaOHavg4GBrW7du3QigAAB4KEIoAK+2/tj6Uo9RP7y+mlZt6oRqYJbIyEjFxMQoODhY3bp1U69evQigAAB4KE7HBeDVsvKySrV/tQrVtGDUAgKLD4iMjNTvfvc7hYWF8f8nAAAejBAKoEyIbhWtBpUbWJ9bZFGrW1opqn6UwoPDzSsMTlWxYkWzSwAAADdBCAXgs8KDwzWw8UC93ONlNanaxOxy4ASbNm1STk6OunbtanYpAACghAihAHzWyWdOqkK5CmaXASfZtGmTli5dKkmyWCzq0qWLyRUBAICSIIQC8Fjp2enaeXan8oy8Ivtk52W7sSKY5ccff9S3335rff7dd9/JYrGoc+fOJlYFAABKghAKwOPk5OXo0UWPasbOGcrJzzG7HJhs48aNWrZsWYH2nByODQAAvBEhFIDHeXLpk5r20zSzy4AH+OGHH7R8+fIC7VFRUbrrrrtMqAgAAJQWIRSAx0jNStW6o+v0ydZPzC4FHmDDhg367rvvCrT36tVL3bt3N6EiAADgDIRQAKYzDEMTV0/UG+veKPb7n44IDQxVoH+gU8aC+xUVQHv37q1u3bqZUBEAAHAWQigAU6Rlp+md79/RiiMrtOH4BqeP3+PWHgrw4y3OG33//fdasWJFgfY+ffpwaxYAAHwAn9AAuF16drrumXGP1h1b55LxI0Mj9W6/d10yNlxr/fr1SkpKKtDet29fbskCAICPIIQCcJxhSBkZJd59wb4FJQ6g9zS+p8htgf6B6lK7i0a0HKG6leqWtDyYZN26dVq5cmWB9n79+nErFgAAfAghFID9srOlP/1JmjdPOnasxMOsObqmRPt90P8DPdXpqRLPC89FAAUAoOwghAKw34gR1wJoKaVlpzm8z71N7tVv7/htqeeGZ6pUqVKBtv79+6tTp04mVAMAAFyJEArAPocPOyWAOurJDk+qV/1eGtB4AFe79WGtWrWSYRia999j7O6771bHjh3NLQoAALgEIRSAfX74wf6+FSqUaqrbqt2mpzo+pUfbP1qqceBdWrduLUnKyspShw4dTK4GAAC4CiEUgH2ys+3rFxAgFfEdvsMph7X0wFLN+HlGkbvHD45XdOvoklQIH3A9iAIAAN9FCAXgXC++KIWFFWhecmCJhswaouy84sNseHC4iwqDJzAMQykpKapSpYrZpQAAAJP4mV0AAB/Rp4/0r39Jr71WYFN6drqGzR520wAK32YYhlatWqWPP/5Yhw8fNrscAABgElZCAZTek09KH36oK1lXNG/ndK05ukbpOenWzUsPLFVmbqaJBcJshmFo5cqVWr9+vSRp5syZGjVqlBo0aGByZQAAwN0IoQCc4nLmZfWf3l8/nvyxVOM0r9bcSRXBUxiGoaSkJH3//ffWttzcXCUmJurJJ59UaGioidUBAAB3I4QCcIo3179Z6gDaoloLNazc0EkVwRMYhqEVK1Zow4YNBbYNGDCAAAoAQBlECAVQYrl+0vRW0nchy/Tl9/tKNVajKo20ZMwSWSwWJ1UHsxmGoe+++04/FHJ7n/vuu09t27Y1oSoAAGA2QiiAEsn1k8YOlWa1lKSSB9CoelGa3G+y2tRoIz8L10rzFcUF0Pvvv19t2rRxf1EAAMAjEEIBlMiSxtcDaMlVCqqkGUNnKDIs0jlFwSMYhqHly5dr48aNBbYRQAEAAMsOAEpkZf3S7R8ZGqnl0csJoD7GMAwtW7as0AA6ePBgAigAAGAlFEDJXCxvf99FoxbZPK9dsbZuv+V2Tr/1MYZh6Ntvv9WmTZts2i0WiwYPHqxWrVqZVBkAAPAkhFAALvV61Osa2GSg2WXAxQzD0NKlS7V582abdovFoiFDhuj22283qTIAAOBpCKEAXCIsMEyP3/m4/tz9z2aXAjfYs2cPARQAANiFEApAunBBWrlSOnOm6D6FfMevKBN7TNSE7hNUzr+cE4qDN2jevLnuvPNOaxC1WCwaOnSoWrYs5dWrAACAzyGEAmXd9OlSbKyUn++0IVvXaE0ALWMsFosGDBggwzC0detWAigAACgSIRQoa/LypLfekubMkbZtM7sa+BCLxaJ77rlHbdu2Vc2aNc0uBwAAeChCKFDWPPqo9O9/m10FfJTFYiGAAgCAYnF/BKAsOXqUAIpSMQxDSUlJunTpktmlAAAAL0UIBcqCzz+X7rhDqlfP7ErgxQzD0IIFC7R+/XrFxcURRAEAQIkQQgFflJMjJSVJL78sBQdLjzwibd3qlKF3V5Mmd5ESWjtlOHiJ/Px8LViwQD/99JMk6dKlS4qLi9Ply5fNLQwAAHgdvhMK+JqsLOnBB6WFC0s3zoQJBZrmWfbpwYC5yrU470q68HzXA+iOHTts2lNTU3Xu3DlVqlTJpMoAAIA3IoQCvubdd0seQNu0kR54QHrpJcnP9kSJ9Ox0jZxUVbl5BNCyJD8/X/Pnz9fOnTtt2v38/DR8+HA1btzYpMoAAIC3IoQCvmbZspLtt2iRNHBgkZvXHl2rrLysEhYFb1RUAPX399fw4cPVpEkTkyoDAADejBAK+JKUFGntWsf3u+suqXfvYrucSTtj93Atq7d0vAZ4lPz8fM2bN08///yzTTsBFAAAlBYXJgJ8we7dUseOUtWqju3XqpX0pz9JS5Zcu4CRE7S6pZUaVm7olLFgjvz8fM2dO7fQADpixAgCKAAAKBVWQgFvd/q01KuXdPas/fv85S/Sa69JFotTS2lZvaWWjlkqi5PHhftcD6C7du2yaff399fIkSPVqFEjkyoDAAC+ghAKeLuFC+0PoFOmSMOGSdWrF9vtStYVbTi+Qek56da2raeLvsVLZGikVsauVNOqTQmgXiw/P19z5szRL7/8YtNOAAUAAM5ECAW83ZYt9vULCJBiY6UKFYrt9tqa1/T62teVm59rdwkP3vagmkU0s7s/PE9eXp7mzJmj3bt327QHBARo5MiRatiQU6wBAIBzEEIBb7V8uTR3rvTZZ/b179TppgE07qc4vbL6FScUB2+Tl5entLQ0m7aAgACNGjVKDRo0MKkqAADgi7gwEeCN/vlPqX9/6ZNP7OsfFiZNmnTTbl/t/qqUhcFbBQYGavTo0apTp44kAigAAHAdQijgbVJTpaeesr//s89K69dLnTvftOuuc7tu2qcwkWGRJdoPniUoKEhjxoxRgwYNNHr0aAIoAABwCU7HBbzNunVSXp59fb/++tqFiFysb4O+Lp8D7hEUFKSxY8dygSkAAOAyrIQC3iYlxf6+d9zhujr+69Wer6p9zfYunwfOk5eXp5ycnCK3E0ABAIArsRIK+KoePaRbb3XacCNbjlSQf5D1ee2KtdW/YX91v7W70+aA6+Xm5uqrr75Sbm6uRo4cqXLlypldEgAAKGMIoYCvKVdOuvtuafp0pw77z3v+qSrlqzh1TLjX9QC6f/9+SVJiYiJBFAAAuB2n4wK+pGNH6eJFacECqWJFs6uBB8nNzdXs2bOtAVSSDh8+rLlz55pYFQAAKItYCQV8Sbt2Umio2VXAw+Tm5mrWrFk6ePCgTXtgYKA6depkUlUAAKCsIoQCgA8rLoCOHTvWel9QAAAAdyGEAoCPysnJ0axZs3To0CGbdgIoAAAwEyEUAHxQTk6OEhMTdfjwYZv26/cBrV27tkmVAQCAso4QCgA+prgAGh0drVq1aplUGQAAACEUAHxKTk6OZs6cqSNHjti0E0ABAICnIIQCgI8oKoAGBwcrOjpaNWvWNKkyAACA/+E+oQDgIxYuXEgABQAAHo8QCgA+okePHgq94T6xwcHBiomJIYACAACPQggFvIlhSP/5j9lVwENVrVpVsbGxCg0NtQbQyMhIs8sCAACwwXdCAW9gGNKrr0offyydO2d2NfBgERERio2NVW5urmrUqGF2OQAAAAUQQgFv8Prr10IoYIeIiAizSwAAACgSp+MCns4wpE8/NbsKeJCsrCxt3rxZhmGYXQoAAIDDWAkFPN2ZM9LJk/b1rVTJtbXAdFlZWZo+fbpOnDihy5cvq3fv3rJYLGaXBQAAYDdCKODp8vLs79url+vqgOkyMzM1Y8YMnThxQpL0/fffSxJBFAAAeBVOxwV8xQMPSFFRZlcBF8nMzLSugN5o+/btSk9PN6kqAAAAx7ESCni7e++V+vWTfvtbKYB/0r7oegA9+avTskNCQqy3ZAEAAPAWfGIFvFnz5tLChWZXARfKzMxUQkKCTp06ZdN+PYBWq1bNpMoAAABKhhAKAB4qIyND06dPJ4ACAACfQggFAA+UkZGhhIQEnT592qY9NDRUsbGx3AsUAAB4LS5MBAAehgAKAAB8GSuhAOBBMjIyFB8frzNnzti0h4WFKTY2VlWrVjWpMgAAAOcghAKAh7h69aoSEhIIoAAAwKcRQgHAQ5w9e1bnzp2zaatYsaJiY2NVpUoVk6oCAABwLr4TCgAeon79+ho+fLj8/K69NRNAAQCALyKEAlBOXo42HN+go5ePml1Kmde0aVMNHz5cVapU0bhx4wigAADA53A6LlDGLTu4TMO/Hq7UrFSzS8F/NW3aVI0aNZK/v7/ZpQAAADgdIRTwVLm50saN0vTpLptiX/I+3Z94v7Lyslw2BwqXn59vPe22MARQAADgqwihgCfKyZFGjZK++cal0yzav4gAaoK0tDRNnz5dd911l2677TazywEAAHArQijgiT74wOUBVJJ2nd9lV7+QciEKDQx1cTVlQ1pamuLi4pScnKxvvvlGFotFzZs3N7ssAAAAt+HCRIAn+eEH6dVXpeefN7sSGz3q9VCgf6DZZXi9K1euWAOodO2U3K+//lr79+83uTIAAAD3YSUU8BQTJ14LoI5o2tQlpdyoXng9/fOef7p8Hl93PYBeuHDBpr1SpUq65ZZbTKoKAADA/QihgCfYsMHxACpJgwY5v5b/qlK+it7p+44GNRmkaiHVXDZPWVBUAK1cubJiY2NVqVIlkyoDAABwP0IoYKZNm6SZM699B9RRzz4rjR/v9JKu+2zQZxrafKjLxi8rUlNTFRcXp4sXL9q0V6lSRbGxsapYsaJJlQEAAJiDEAqYITdXevFF6d13Hd83Lk6KipLq1HF+XXAqAigAAEBBhFDA3c6elUaMkNascXzfRx6RYmKcXxOc7vLly4qLi1NKSopNe9WqVRUbG6uwsDCTKgMAADAXIRRwpw0bpAcflE6dcnzf++6T/u//nF8TnI4ACgAAUDRCKOBqOTnSTz9Jq1eX7NYrr78uxcZy+q2XuHz5sqZNm6ZLly7ZtEdERCgmJoYACgAAyjxCKOBKn3xyLXheuVKy/Vu3lv78Z8licW5dcInMzMwiA2hsbKxCQ0PNKQwAAMCD+JldAOCz5s+Xfve7kgfQO++Uli0jgHqR4OBgtW7d2qatWrVqBFAAAIAbsBIKuMqXX9rft0ED6e23pZCQa89btJDq1nVNXXCpHj16yDAMrV27VtWqVVNMTAwBFAAA4AaEUMBVduywv++WLVLlyq6rBW5jsVjUs2dPhYSEqEWLFgq5/ocFAAAASCKEAq5jGPb1+8tfCKA+xmKxqEOHDmaXAQAA4JH4Tihgpk6dpOeeM7sKlMDFixd17tw5s8sAAADwOoRQwAy9e0uffy4tXy5VquT26fPy87Tl1BZN3znd7XP7gosXLyouLk7x8fE6f/682eUAAAB4FU7HBcyQkCBFRpoy9YELB9Q3oa+OXj5qyvze7sKFC4qLi9OV/171OC4uTrGxsapWrZrJlQEAAHgHVkKBMiQvP099EvoQQEvo1wFUktLT07V8+XITqwIAAPAurIQCZUBufq62nNqijzZ9pGOXj5ldjldKTk5WXFyc0tLSbNpr1KihoUOHmlQVAACA9yGEAs504YL0wgvXvut5/LjZ1UiSUjJS1Dehr7ae3urQfrXCarmoIu9TVACNjIxUdHS0ypcvb1JlAAAA3ocQCjjLlStSnz7STz+ZXYmN3yz4jcMBtHJwZbWv2d5FFXkXAigAAIBzEUIBZ/n2W48LoFm5WVp2aJlD+1QoV0FzRsxRgB9vD+fPn1dcXJzS09Nt2mvWrKmxY8cSQAEAAEqAT5mAs3z/vf19g4NdV8cNzqaf1dWcq3b3/3jgx7q70d2qF17PdUV5ieICaHR0tILd9P8hAACAryGEAqV15Ii0dKn04Yf29W/ZUqpc2bU1Ocgii37+3c9qUb2F2aV4hHPnzik+Pr5AAK1Vq5bGjh1LAAUAACgFQihQGklJ0r33SpmZ9vW3WKQJE1xbk4Mssujz+z4ngP7XuXPnFBcXp6tXbVeQCaAAAADOQQgFSio7Wxo61P4A2q+f9Nhj1/bxEJ/e+6n6NuzL6bc32LBhQ4EAWrt2bY0dO1ZBQUEmVQUAAOA7CKFASf34o5Saal/fGTOk0aNdW4+DWlRroUfaP2J2GR7n3nvv1dWrV3XgwAFJUp06dTRmzBgCKAAAgJP4mV0A4LVOnbK/7223ua4OOFVAQICGDx+uRo0aEUABAABcgJVQwNUaNJBatTK7CjggICBAI0aMUF5eHgEUAADAyQihgKMOHZISE6W//OXmfevXl777TvLjpANvExAQoIAA3iIBAACcjU9YgCN27JB695YuXCi+X/ny1+4b2ro1AdRDnT59WitXrtSwYcO44i0AAIAb8ekYcMQTT9w8gErS2LFS27YEUA916tQpxcfH6+DBg5o+fboy7b3CMQAAAEqNT8iAva5elX74wb6+hE+PderUKSUkJFiD58mTJwmiAAAAbsQn5RI4duyYnnvuOTVv3lwhISGqUqWKOnTooMmTJxe4v6CjUlNTlZiYqEceeUTt2rVTeHi4AgMDVa1aNfXs2VOTJ0/WpUuXnPNC4JgrV6TcXPv6tm3r2lpQIidPnlR8fHyBwBkYGCh/f3+TqgIAAChbLIZhGGYX4U0WL16sMWPG6PLly4Vub9q0qZYsWaIGDRo4PPbSpUs1ZMgQZWVlFdvvlltu0cyZMxUVFeXwHEX55Zdf1LJly2L77Nq1Sy1atHDanF7n7FmpRo2b9wsPl37+Wapd2+Ul3cyxy8d06we3FrqtRbUW2vX4LjdXZJ6TJ08qISGhwL+vBg0aaOTIkSpXrpxJlQEAAJQtrIQ6YMeOHRo+fLguX76s0NBQvfHGG9qwYYOSkpL0yCOPSJL27dungQMHKi0tzeHxL1y4oKysLPn5+al///56//33tXLlSm3btk0LFizQiBEjJElnz57Vvffeq59++smZLw/O0LDhtavhekAAxf+cOHGCAAoAAOAhuDquA55++mldvXpVAQEBWr58uTp37mzd1qtXLzVu3FjPP/+89u7dq/fee08vv/yyQ+OXK1dOjz32mCZMmKC6devabGvbtq0GDRqkrl276sknn9TVq1f17LPPKikpySmvDU5w6NC1e4LCo1wPoNnZ2TbtDRs21IgRIwigAAAAbsbpuHbavHmzOnToIEl67LHH9MknnxTok5+fr5YtW2rPnj2qXLmyzp4965IPuHfeeae2bNkiPz8/nTt3TlWrVi31mJyOa4fiTsdt1Eg6cMC99dihrJ+Oe/z4cU2fPr1AAG3UqJFGjBjBfUABAABMwOm4dpo3b5718fjx4wvt4+fnp5iYGElSSkqKVq9e7ZJaevbsKela6D1y5IhL5gC83bFjxwigAAAAHogQaqd169ZJkkJCQtS+ffsi+/Xo0cP6eP369S6p5cbvtflxKxCggGPHjmnGjBkFAmjjxo0JoAAAACbjk5id9uzZI+naKkpxH2CbNWtWYB9nW7NmjSQpICBAjRo1cnj/EydOFGg7c+ZMqesCPMH1U3BzcnJs2hs3bqzhw4cTQAEAAEzGpzE7ZGZmKjk5WZJU+yZXPa1cubJCQkKUnp6u48ePO72WxYsXa+fOnZKk/v37q2LFig6PUadOHWeXBXiMSpUqKSwsTBcvXrS2NWnSRA8++CABFAAAwANwLqcdrly5Yn0cGhp60/4hISGSVKLbtBTn4sWL+v3vfy9J8vf31+uvv+7U8VGE/HxpyxZp8mSzK4EdKlasqNjYWFWuXFnStXv3EkABAAA8B5/K7JCZmWl9HBgYeNP+QUFBkqSMjAyn1ZCXl6cxY8bo6NGjkqS//OUvatu2bYnGKmyFdt++ferTp0+pavRJeXnS+PFSQoLZlcAB14Po999/r/79+8vf39/skgAAAPBfhFA7BAcHWx//+kInhbl+4aDy5cs7rYbHH39c3377rSRp4MCB+utf/1risQo7pfjy5cslHs+nTZlCAPVSFStW1IABA2SxWMwuBQAAADfgdFw7hIWFWR/bc4ptenq6JPtO3bXHSy+9pE8//VSS1K1bN3311Ves7LjLkiVmV4BinDt3TkXd6thisRBAAQAAPBAh1A7BwcGKiIiQVPiVZW+UkpJiDaHOuADQ22+/rbfeekuS1K5dOy1atMipK6y4if+e/nxTjRu7tg4UcPjwYX322WdauHChTRAtKpQCAADAMxBC7dS8eXNJ0sGDB5Wbm1tkv7179xbYp6SmTJmiF1980TrWsmXLVKlSpVKNCRe5916zKyggKzdL3x/73uwyXOLQoUOaOXOmcnNztX37dmsQJYACAAB4PkKonbp16ybp2qm2W7duLbLf9Xt4SlLXrl1LPF9CQoL+8Ic/SJIaNGigFStWWFdj4WF++9trPx7kg40fqMqkKho9Z7TZpTjdjQH0uu3bt2v79u2SxCm4AAAAHo4QaqfBgwdbH0+dOrXQPvn5+YqPj5ckhYeHKyoqqkRzzZkzR+PHj5dhGKpdu7aSkpJUs2bNEo0FF5o4UTp48NrFi/w855/SrF2z9Mdlf9TVnKtml+J0Bw8e1MyZM5WXl2fT3qJFC7Vp04YACgAA4AU855Ozh+vQoYO6d+8uSfriiy/0ww8/FOjz7rvvas+ePZKkp556SuXKlbPZPm3aNOvFUiZOnFjoPMuXL9eoUaOUl5en6tWra8WKFapXr55TXwuc5De/kRo2lEwKPvlGvrae2qrpO6fb/Iz8ZqQp9bjawYMHlZiYWCCAtmzZUkOHDpWfB/0hAAAAAEXjFi0O+PDDD9W1a1dlZGSoX79+mjBhgqKiopSRkaHExETrFWybNGmiZ5991uHxN27cqCFDhig7O1vlypXT+++/r5ycHO3atavIfWrXrq3w8PCSviR4qUMXD6lPQh/959J/SjxG9ZDqzivIxQ4cOKBZs2YVCKC33367Bg8eTAAFAADwIoRQB7Rt21azZs3S2LFjlZqaqgkTJhTo06RJEy1evNjmti72+vbbb3X16rVTKHNycjRmzJib7jN16lSNGzfO4bngvfKN/FIHUEnq06CPcwpysf3792v27NkEUAAAAB/BpzcHDRo0SDt37tQf//hHNWnSRBUqVFB4eLjuuOMOvf3229q+fbsaNWpkdpnwYdtPby91AL3r1rv0VMennFOQC+3fv7/QFdBWrVoRQAEAALyUxeCeBpD0yy+/qGXLlsX22bVrl1q0aOGmijzEbbdJ//2ebwHHj0u1a7u3HknTd05X9NzoEu0b3Spa/Rr205BmQxQSGOLkypxr3759mj17tvLz823aW7durfvuu48ACgAA4KU4HRfwUFtObdHf1v5Nq/+zWmnZadb2PCOvmL2Kdm+TexU/JN5Z5bnU3r179dVXXxUIoG3atNGgQYMIoAAAAF6MEAqY6EzaGX136DvtPr/bpn3H2R1aenCp0+ZpVKWRPh74sdPGc6XMzEzNnz+fAAoAAOCjCKGASdYfW68BMwbYrHI6w+N3PG59XM6/nDrX7qx+DfupcvnKTp3HVYKDgzVy5EjNmDFDOTk5kq5dFGzQoEHcBxQAAMAHEEIBE+Tk5WjQzEFOD6DPdHpG7/Z/16ljmuHWW2/V6NGj9eWXX6ply5YEUAAAAB9CCAVM8OPJH3Up85LTx+1+a3enj2mWevXq6ZFHHlFERAQBFAAAwIcQQgETHLt8zOljdqjVQX0b9HX6uGaqVq2a2SUAAADAyQihgJepUK6CzfNaYbXUr2E/vdHrDY+/7cqv7dq1S9nZ2WrXrp3ZpQAAAMBNCKGAl+het7uWjFmi0MBQs0txip9//llz586VYRiyWCxq27at2SUBAADADQihgJtl5GRo44mNRW7vWqerRrUcZX1ezr+cOtXupJbVW8rP4hu3J7kxgErSggULJIkgCgAAUAYQQgE3ycvP0xNLn9C/t/9bWXlZRfZrWb2lft/h926szL127typefPmWQPodWfPnjWpIgAAALgTIRRwotz8XH2x7QstP7xcp66cstlW3OpnWbFjxw7NmzevQHvHjh3Vv39/9xcEAAAAtyOEAk6Sm5+rMXPGaPYvs80uxSP99NNPmj9/foH2Tp06qV+/ftyGBQAAoIzwjS+YAR4g6XCSUwJoRIUIJ1TjWQigAAAAuI6VUKAoWVmSA99TXPWfVU6Ztlf9Xk4Zx1Ns377deuGhG3Xu3Fl9+/YlgAIAAJQxrIQCv3b8uDRggFS5snTxot27rTyystRTP9ruUfWs17PU43iKbdu2FRpAu3TpQgAFAAAoo1gJBW6UkiL17CkdPuzQbl9s+0KbT20u0ZSB/oGKaRWjfg37adhtw3zmNizbtm3TwoULC7R37dpVvXv3JoACAACUUYRQ4LqsLOmDDwoE0F+qSX/vLq2uJ10JumFDfHPJYpEhQ2nZaQ5PVz+8vib1naQHbnugVGV7oq1bt2rRokUF2rt166ZevXoRQAEAAMowQiiQnq5LD4/Vqt1L9Et4tnTX/zbtjZBmtCpivxz7g+fcEXPVuEpj6/OwoDDVrVS3hAV7NsMwlJubW6C9e/fuioqKIoACAACUcYRQlHnbo/uod5ONSmnmmvHvbXKv7m96f5kIX4ZhSLp2309J+vbbbyURQAEAAPA/hFCUaXn/OaIBDTcqpbxrxu9Vv5dmDJ1RZsLXja+zY8eOMgxDGRkZ6tmzZ5n5bwAAAIDiEUJRpm1b/5XOhrpm7Be7vqjXe72uAD/f/2dmGEahIbNTp04mVAMAAABP5vufjoEiGIahH9P3OX3cikEVFTc4ToObDXb62J4oPT1dFSpUMLsMAAAAeAnfuBcE4KB3N7yr+h/W1xNn/u20MSsFVdKw5sO07dFtZSaAbty4Uf/4xz908uRJs0sBAACAl2AlFGXOBxs/0HPfPVfi/dtFttOK6BWqXL6yE6vyPhs3btSyZcskSdOnT1d0dLRq1aplclUAAADwdIRQlDlTNk+xu++kPpOsjwP8AtSxdkd1qNWhTHzPszg//PCDli9fbn2elZWlhIQE/f73v1dYWJiJlQEAAMDTle1P0ihzLmVe0oGLB+zq+3RqC/2p659cXJH32bBhg7777rsC7V27diWAAgAA4KYIoShTcvNz7e7bM6uGCyvxTt9//71WrFhRoL13797q1q2bCRUBAADA2xBCgUL0PyjdHVjb7DI8SlEBtE+fPuratasJFQEAAMAbEUJRdpw4ISUtLLZL78NS30PSUz9KQTH+birM861fv15JSUkF2vv27asuXbqYUBEAAAC8FSEUvs8wpBdekN55R6og6fnCu9VPkVbEu7Uyr7Bu3TqtXLmyQHu/fv3UuXNnEyoCAACANyOEwvfFxV0LoHDY2rVrtWrVqgLtBFAAAACUFCEUvuvECWnFCmn8eLMr8UpFBdD+/furU6dOJlQEAAAAX0AIhe/Jy5Pi46WHHirdOHXrOqceL7R3795CA+jdd9+tjh07mlARAAAAfIWf2QUATpGXJ/31r1Lz5lJAQOkDqCTdfXfpx/BSTZo00e23327TNmDAAAIoAAAASo2VUPiG3/5W+vxz54337rtSGQ5cfn5+Gjx4sAzD0K5duzRgwAB16NDB7LIAAADgAwih8H4XLkhffFH6cSpVkj58Terb99qKahnn5+enIUOGqE2bNmrYsKHZ5QAAAMBHEELh/TZuvHYbltKqUkV68snSj+ND/Pz8CKAAAABwKr4TCu+XlWV2BV7LMAxt3LhRaWlpZpcCAACAMoIQirLl1VfNrsBjGIahpKQkLVu2TPHx8QRRAAAAuAUhFL6vSxfpnXeuXUH38cfNrsYjGIahFStW6Pvvv5cknT9/XvHx8UpPTze5MgAAAPg6vhMK3zZunDR1qtlVeBTDMPTdd9/phx9+sGk/f/68jh49qttuu82kygAAAFAWEEKBMsQwDC1fvlwbN24ssO3+++8ngAIAAMDlCKFAGWEYhpYtW6Yff/yxwLb7779fbdq0cX9RAAAAKHMIoSgz8o187Ty70+wyTFFcAB08eLBat25tQlUAAAAoiwihKBPid8TrmWXP6ELGBbNLcTvDMPTtt99q06ZNNu0Wi0WDBw9Wq1atTKoMAAAAZREhFD7v24Pfaty8cTJkmF2K2xUXQIcMGaLbb7/dpMoAAABQVhFC4bNy/aStgef1wOwHymwAXbp0qTZv3mzTTgAFAACAmQih8Elrb5WGjJAuVlgs5di3T0SFCNcW5UaGYWjJkiXasmWLTbvFYtHQoUPVsmVLkyoDAABAWUcIhc85VkkaMEa6GujYfr3r93ZNQSbIzs7WiRMnbNosFouGDRumFi1amFQVAAAAIPmZXQDgbEsbOR5A29Zoqz91/ZNrCjJBUFCQoqOjdcstt0i6FkAfeOABAigAAABMRwiFz9lV3f6+AxsP1JR7pigpJklVyldxXVEmqFChgmJiYhQZGakHHnhAt912m9klAQAAAJyOi7Lrrlvv0qLRi8wuw6UqVKighx9+WH5+/L0JAAAAnoFPpiiTIkMj9dmgz8wuwykMw1BeXl6R2wmgAAAA8CSshKJMCfAL0JR7pmhQ00GqEVrD7HJKLT8/XwsXLlRWVpaGDRsmf39/s0sCAAAAisUSCcqUzwd9rkfaP+IzAXTBggX66aeftGfPHn3zzTfFrogCAAAAnoAQCnih/Px8zZ8/Xzt27LC27dmzR/PnzzexKgAAAODmCKGAl7keQHfu3GnT7u/vr5YtW5pUFQAAAGAfvhMKeJH8/HzNmzdPP//8s027v7+/hg8friZNmphUGQAAAGAfQijgJfLz8zV37lzt2rXLpt3f318jRoxQ48aNTaoMAAAAsB8hFPACxQXQkSNHqlGjRiZVBgAAADiGEAp4uPz8fM2ZM0e//PKLTTsBFAAAAN6IEAp4sLy8PM2ZM0e7d++2aQ8ICNDIkSPVsGFDkyoDAAAASoYQCniovLw8ffPNN9qzZ49Ne0BAgEaNGqUGDRqYVBkAAABQctyiBfBQixcvJoACAADA5xBCAQ/VoUMHBQcHW58HBARo9OjRBFAAAAB4NUIo4KFq1KihmJgYBQcHWwNo/fr1zS4LAAAAKBW+EwrvlpYmrV1rdhUuExkZqZiYGGVlZalevXpmlwMAAACUGiEU3ikvT/rDH6QvvpBycsyuxqUiIyPNLgEAAABwGk7HhXf685+lTz6xCaBXy0nLG0ofdTSxrhLIzc0tcAsWAAAAwFexEgrvYxjSl1/aNO2uJvWKlc6GmlRTCeXm5mr27Nk6cOCA+vTpo65du5pdEgAAAOBSrITC+1y6JB0/bn2ab5EGjPHuACpJK1as0IYNG0yuCgAAAHAtQii8j2HYPP25unQs3JxSSio3N1ezZs2yBtDr1qxZoytXrphUFQAAAOB6nI4Lr3eoiv1964XXc1kd9srNzVViYqIOHTpk0x4YGKixY8cqLCzMpMoAAAAA1yOEoswIDw5X5zqdTa0hJydHs2bNKhBAg4KCNHbsWNWuXdukygAAAAD3IISiTAgLDNPi0YsV6B9oWg05OTlKTEzU4cOHbdqDgoIUHR2tWrVqmVQZAAAA4D6EUPi8xGGJ6t2gtyIqRJhWQ05OjmbOnKkjR47YtBNAAQAAUNYQQuHTxrUZpxEtR5haQ1EBNDg4WNHR0apZs6ZJlQEAAADuRwgFXCg7O1szZ87Uf/7zH5t2AigAAADKKkIo4CLFBdCYmBhFRkaaUxgAAABgIu4TCrjIsWPHdPToUZs2AigAAADKOlZC4TWSrybrs62fafXB75T6m/+1X6hgXk3FadSokQYPHqx58+bJMAyVL19eMTExqlGjhtmlAQAAAKYhhMIrJF9NVq+4Xvr53M/XGuqYW4+9WrVqJUlavny5xo4dSwAFAABAmUcIhVf4aNNH/wugXqZVq1Zq2rSpgoKCzC4FAAAAMB0hFB4rKzdL7/3wnpYdWqY1R9eUaAx/i7+TqyqcYRiyWCxFbieAAgAAANcQQuGRcvJy9MBXD2jR/kWlGueOmnc4qaKiZWVladasWerWrZsaNGjg8vkAAAAAb8bVceGRVv9ndakDaESFCA1qMshJFRUuMzNT06dP15EjRzRz5kwdPnzYpfMBAAAA3o4QCo9U0tNvr2sX2U6rYlepVsVaTqqooOsB9MSJE5Kk3NxczZw5U0eOHHHZnAAAAIC343RceKS07DS7+/48RbIY/31SM1K3bNipiAoRrinsvzIzM5WQkKBTp07ZtAcFBSk0NNSlcwMAAADejBAKr/bmCqnluRsaKgRKLg6gGRkZmj59eoEAGhoaqtjYWEVEuHZ+AAAAwJsRQuGVGl2QHtsqPbvhVxsqV3bpvBkZGUpISNDp06dt2gmgAAAAgH0IofA67y6TnvmhiI29erlsXgIoAAAAUHqEUHidyhlFbGjaVHruOZfMmZGRofj4eJ05c8amPSwsTLGxsapatapL5gUAAAB8DVfHhfdr2lR6801pzRopMtLpw1+9epUACgAAADgJK6HwfkuXSvXru2To6wH07NmzNu0VK1ZUbGysqlSp4pJ5AQAAAF/FSihQhKysLAIoAAAA4GSEUKAIgYGBuvXWW23aKlWqpHHjxhFAAQAAgBIihAJFsFgsuvvuu3XnnXdKuhZAY2NjVdnFt4EBAAAAfBnfCQWKYbFYNGDAAFWoUEGtW7cmgAIAAAClRAgFbsJisahnz55mlwEAAAD4BE7HBSSlpaUpJSXF7DIAAAAAn0cIRZl35coVxcXFKS4uTpcuXTK7HAAAAMCnEUJRpl0PoMnJybp8+bKmTZtGEAUAAABciBCKMut6AL1w4YK17fLly1q4cKGJVQEAAAC+jQsToUxKTU1VXFycLl68aNNepUoV3X///SZVBQAAAPg+VkJR5hQXQGNjY1WxYkWTKgMAAAB8HyuhKFMuX76suLi4AlfCrVq1qmJjYxUWFmZSZQAAAEDZwEooygwCKAAAAGA+VkJRJhQVQCMiIhQTE0MABQAAANyEEAqfd+nSpULvARoREaHY2FiFhoaaUxgAAABQBhFC4dOKCqDVqlVTTEwMARQAAABwM74TCp+2cuVKAigAAADgQQih8Gn33nuv6tata31erVo1TsEFAAAATEQIhWe6etUpwwQGBmr06NGqU6eOqlevrtjYWIWEhDhlbAAAAACO4zuh8ExHjzptqKCgII0ZM0a5ubkEUAAAAMBkhFB4pvPnpCA7+lksUuXKN+0WFBSkoCB7BgQAAADgSpyOC89k2Nnvjjuk8HBdvHhRc+fOVU5OjkvLAgAAAFA6rITCe4WHS1Om6MKFC4qLi9OVK1eUlpamkSNHqly5cmZXBwAAAKAQrITCO73+urRpk5Lr1dO0adN05coVSdLhw4eVmJjIiigAAADgoQihJXDs2DE999xzat68uUJCQlSlShV16NBBkydP1lUnXdVVkhITE9W/f39FRkYqODhY9erVU3R0tDZu3Oi0ObzSQ+Olv/xFyZUrKy4uTmlpaTabMzIylJuba1JxAAAAAIrD6bgOWrx4scaMGaPLly9b265evarNmzdr8+bN+vzzz7VkyRI1aNCgxHNkZmbqwQcf1KJFi2zajx49qqNHj+rLL7/UxIkT9de//rXEc3i75OTkQgNozZo1NXbsWJUvX96kygAAAAAUh5VQB+zYsUPDhw/X5cuXFRoaqjfeeEMbNmxQUlKSHnnkEUnSvn37NHDgwALhyBG/+c1vrAE0KipK8+bN06ZNm/TFF1+oYcOGys/P18svv6zPP//cKa/L2+TkBWjatGmFBtDo6GgCKAAAAODBWAl1wNNPP62rV68qICBAy5cvV+fOna3bevXqpcaNG+v555/X3r179d577+nll192eI41a9boyy+/lCQNGjRIc+fOlb+/vyTpzjvv1H333af27dvr2LFjev755/XAAw8oPDzcKa/PG1RTNZ1NraF8I92mvVatWho7dqyCg4NNqgwAAACAPVgJtdPmzZu1evVqSddWKm8MoNc9++yzat68uSTpgw8+KNHFcSZNmiRJ8vf315QpU6wB9LqIiAi9/fbbkqSUlBR98cUXDs/hraqpmmIVq3zD9r8JARQAAADwHoRQO82bN8/6ePz48YX28fPzU0xMjKRrAfF6aLVXWlqakpKSJEl9+/ZV7dq1C+03dOhQVaxYUZI0Z84ch+bwVtVVXeM0TqEKtWmvXbs2ARQAAADwIoRQO61bt06SFBISovbt2xfZr0ePHtbH69evd2iOTZs2KSsrq8A4vxYYGKhOnTpZ9/H125FUV3XFKlYhCrFpJ4ACAAAA3ofvhNppz549kqRGjRopIKDo/2zNmjUrsI+jc/x6nKLmWb58uXJzc3XgwAHddtttds9z4sSJAm1nzpyxv1A3KiqA1qlTR2PGjFFQUJBJlQEAAAAoCUKoHTIzM5WcnCxJRZ4ie13lypUVEhKi9PR0HT9+3KF5bux/s3nq1Kljs58jIfTGfT1drnKVpzybtqCATAIoAAAA4KU4HdcOV65csT4ODQ0tpuc1ISHXVu0cvU2LI/Ncn6Mk83iTi7qoaZqmK7r23+aojqpa2DkCKAAAAOClWAm1Q2ZmpvVxYGDgTftfD0gZGRkum+fGEOboPIWt0O7bt099+vRxaBx3uR5EoxSlBVqg7pYxZpcEAAAAoIQIoXa48cI32dnZN+1//eJC5cuXd9k81+coyTyFnep7+fJlh8Zwtwu6oK/1tdllAAAAACglQqgdwsLCrI/tOfU1PT1dkn2n7pZ0nutzlGQebxBSroIiMiyFbgsq51joBgAAAOA5CKF2CA4OVkREhJKTkwu9suyNUlJSrAHR0QsA3bhCeeLECd1xxx1F9r3xlFpvutCQvd547Xu9YXYRAAAAAJyOCxPZqXnz5pKkgwcPKjc3t8h+e/fuLbCPvW68wu2N4xQ3T0BAgBo1auTQPAAAAABgFkKonbp16ybp2mmwW7duLbLfmjVrrI+7du3q0Bx33nmn9YJEN47za9nZ2dq4cWOBfQAAAADA0xFC7TR48GDr46lTpxbaJz8/X/Hx8ZKk8PBwRUVFOTRHWFiYevfuLUlasWJFkaf+zpkzR6mpqZKkIUOGODQHAAAAAJiJEGqnDh06qHv37pKkL774Qj/88EOBPu+++6727NkjSXrqqadUrlw5m+3Tpk2TxWKRxWLRxIkTC53nueeekyTl5ubq97//vfLy8my2Jycn64UXXpB0Leg+/PDDpXpdAAAAAOBOhFAHfPjhhypfvrxyc3PVr18/vfnmm9q4caNWrVqlxx57TM8//7wkqUmTJnr22WdLNEevXr00cuRISdKCBQvUt29fLViwQFu2bNHUqVPVqVMnHTt2TJL01ltvqXLlys55cQAAAADgBlwd1wFt27bVrFmzNHbsWKWmpmrChAkF+jRp0kSLFy+2ud2Ko/79738rNTVVS5Ys0apVq7Rq1Sqb7X5+fvrrX/+qxx57rMRzAAAAAIAZWAl10KBBg7Rz50798Y9/VJMmTVShQgWFh4frjjvu0Ntvv63t27eX+mq15cuX1+LFizVjxgz17dtX1atXV2BgoOrUqaPRo0dr/fr1RZ7OCwAAAACezGIYhmF2ETDfL7/8opYtWxbbZ9euXWrRooWbKgIAAADgi1gJBQAAAAC4DSEUAAAAAOA2hFAAAAAAgNsQQgEAAAAAbkMIBQAAAAC4DSEUAAAAAOA2hFAAAAAAgNsQQgEAAAAAbkMIBQAAAAC4DSEUAAAAAOA2hFAAAAAAgNsQQgEAAAAAbkMIBQAAAAC4DSEUAAAAAOA2hFAAAAAAgNsQQgEAAAAAbkMIBQAAAAC4TYDZBcAzZGVl3bTPwYMH3VAJAAAAAE/XsGFDBQcHl2hfQigkScePH79pn8GDB7u+EAAAAAAeb9euXWrRokWJ9uV0XAAAAACA2xBCAQAAAABuQwgFAAAAALgNIRQAAAAA4DaEUAAAAACA2xBCAQAAAABuQwgFAAAAALiNxTAMw+wiYL5Lly5pzZo1xfapU6eOgoKC3FTRNWfOnFGfPn0kSStWrFCNGjXcOj+8D8cMHMUxA0dxzMBRHDNwhLccLw0bNlRwcHCJ9g1wci3wUuHh4br//vvNLqOASpUqWR83bdpUtWvXNrEaeAOOGTiKYwaO4piBozhm4IiycLxwOi4AAAAAwG0IoQAAAAAAtyGEAgAAAADchhAKAAAAAHAbro4LAAAAAHAbVkIBAAAAAG5DCAUAAAAAuA0hFAAAAADgNoRQAAAAAIDbEEIBAAAAAG5DCAUAAAAAuA0hFAAAAADgNoRQAAAAAIDbEEIBAAAAAG5DCAUAAAAAuA0hFG5x7NgxPffcc2revLlCQkJUpUoVdejQQZMnT9bVq1edNk9iYqL69++vyMhIBQcHq169eoqOjtbGjRudNgfcw5XHTGpqqhITE/XII4+oXbt2Cg8PV2BgoKpVq6aePXtq8uTJunTpknNeCNzGXe8zNzp9+rTCw8NlsVhksVjUs2dPl8wD53Pn8bJixQqNGzdOjRo1UkhIiCpVqqQmTZrogQce0Mcff6y0tDSnzgfXcMcxs3v3bj3xxBO6/fbbVbFiRevvpqioKL3//vu6cuWKU+aB65w7d06LFi3Syy+/rAEDBigiIsL6O2LcuHEumdMrP/8agIstWrTIqFSpkiGp0J+mTZsahw4dKtUcGRkZxr333lvkHH5+fsZrr73mpFcEV3PlMbNkyRIjKCioyLGv/9xyyy3GypUrnfzK4CrueJ8pzLBhw2zm6dGjh9PngPO563i5ePGicf/999/0/Wb79u2lf1FwKXccM5MnTzYCAgKKPVZuvfVWY8eOHU56VXCF4v7/i42Ndepc3vz5lxAKl/rpp5+MChUqGJKM0NBQ44033jA2bNhgJCUlGY888oj1H0mzZs2MK1eulHie0aNHW8eKiooy5s2bZ2zatMn44osvjIYNG1q3ffbZZ058dXAFVx8zCQkJ1jfm/v37G++//76xcuVKY9u2bcaCBQuMESNGWOeoUKECHw69gLveZ35twYIFhiSjevXqhFAv4q7j5dKlS0b79u2t4w0cONBISEgwfvjhB2P9+vXGjBkzjKefftqoXbs27zMezh3HzKxZs6zjBAYGGn/84x+NxYsXGz/++KPx5ZdfGt26dbNuj4yMNC5duuTkVwlnuTEE1qlTx+jXr5/LQqg3f/4lhMKlevbsaUgyAgICjA0bNhTYPmnSJOs/kFdffbVEc6xevdo6xqBBg4zc3Fyb7efPnzfq1q1rSDIqV65spKSklGgeuIerj5nExETjscceM44ePVpkn//7v/+zztGrVy+H54B7ueN95teuXLli1KlTx5BkxMfHE0K9iLuOl+joaOs8iYmJRfbLz883cnJySjwPXM8dx0zLli2tYyxatKjQPkOHDrX2effdd0s0D1zv5ZdfNhYuXGicOXPGMAzDOHLkiEtCqLd//iWEwmU2bdpk/cfx2GOPFdonLy/PaN68ufUfSHZ2tsPz3HPPPYYkw9/f3zh+/HihfWbOnGmtZfLkyQ7PAfdw1zFjjzvuuMO6YpqcnOySOVB6Zh0zTzzxhPUvz4ZhEEK9hLuOl3Xr1lnnmThxYmnLhoncccxcvnzZOke7du2K7Ldjxw5rv2HDhjk0B8zjqhDq7Z9/uTARXGbevHnWx+PHjy+0j5+fn2JiYiRJKSkpWr16tUNzpKWlKSkpSZLUt29f1a5du9B+Q4cOVcWKFSVJc+bMcWgOuI87jhl7Xb/ATH5+vo4cOeKSOVB6ZhwzmzZt0j//+U8FBgbq448/LtVYcC93HS8fffSRJCk0NFTPPvusw/vDc7jjmMnOzrY+btCgQZH9GjZsaH2clZXl0BzwLb7w+ZcQCpdZt26dJCkkJETt27cvsl+PHj2sj9evX+/QHJs2bbK+Ed84zq8FBgaqU6dO1n1ycnIcmgfu4Y5jxl43/oL38+Ot0lO5+5jJzc3Vo48+qvz8fL3wwgtq2rRpiceC+7njeMnOztb8+fMlSQMGDFBoaKika8fO0aNHdezYMZvQAc/mjmMmIiJCVapUkSQdPny4yH6HDh2yPm7SpIlDc8C3+MLnXz5ZwWX27NkjSWrUqJECAgKK7NesWbMC+zg6x6/HKW6e3NxcHThwwKF54B7uOGbstWbNGklSQECAGjVq5JI5UHruPmYmT56sHTt2qGHDhpowYUKJx4E53HG87NixQ5mZmZKkzp0768yZMxo/frzCw8NVr1493XrrrapUqZLuuecebdiwoQSvAu7krveYRx99VJK0bds2LV26tNA+r7/+uiTJ399fDz/8sMNzwHf4wudfQihcIjMzU8nJyZJU5CkC11WuXFkhISGSpOPHjzs0z439bzZPnTp1Ct0PnsFdx4w9Fi9erJ07d0qS+vfvbz2VBZ7F3cfM4cOH9dprr0mSpkyZouDg4BKNA3O463jZvXu3zZy33367pk2bpvT0dJv2pUuXqnv37vrggw8cGh/u4873mD//+c/q06ePJGnIkCF67rnntHTpUm3evFmzZs1Sz5499fXXX8vf31//93//p+bNmzs8B3yHL3z+JYTCJW68mfL1U5GKc/2N29Ebdjsyz/U5SjIPXM9dx8zNXLx4Ub///e8lXftr8/W/PMPzuPuYeeyxx5SRkaERI0aoX79+JRoD5nHX8XLx4kXr41dffVXJycm69957tWXLFmVmZurs2bOaMmWKKlasqPz8fD3zzDNFrnzBXO58jwkNDdXSpUv12WefqXbt2nr33Xd1zz33qEOHDho5cqTWrFmjoUOH6vvvv9fjjz/u8PjwLb7w+ZcQCpe4fiqSdO189JsJCgqSJGVkZLhsnutzlGQeuJ67jpni5OXlacyYMTp69Kgk6S9/+Yvatm3rtPHhXO48ZuLj47VixQpVrFhR77//vsP7w3zuOl5uXPHMysrSoEGDNH/+fLVv315BQUGqXr26fve732nx4sXy8/OTYRh6/vnnZRiGQ/PA9dz9e2nLli2aOXNmkd8LXbFiheLi4pSamlqi8eE7fOHzLyEULnHjaWr2XIDh+pery5cv77J5brzQjKPzwPXcdcwU5/HHH9e3334rSRo4cKD++te/Om1sOJ+7jpnk5GTrFU7feOMNRUZGOrQ/PIMZv5ck6Z133in04mbdunXT0KFDJUm7du3Srl27HJoHrufO30tff/21evbsqZUrV+r222/X3LlzdeHCBWVnZ+vQoUP6+9//rpycHH388cfq0qWLzpw54/Ac8B2+8PmXEAqXCAsLsz62Z+n/+l+O7TndpaTz3PjXaUfngeu565gpyksvvaRPP/1U0rUPh1999ZX8/f2dMjZcw13HzDPPPKPk5GTdcccdnAbnxcz4vVS/fv1ir6Dcv39/6+PNmzc7NA9cz13HzNmzZzVu3DhlZWWpRYsW2rBhgwYPHqwqVaqoXLlyatCggV566SUtXLhQFotFv/zyi5544gnHXgx8ii98/i36Ml9AKQQHBysiIkLJyck6ceJEsX1TUlKs/0Bu/PK0PW78MvaJEyd0xx13FNn3xi9jOzoPXM9dx0xh3n77bb311luSpHbt2mnRokUe9ddCFM4dx8ypU6eUkJAgSerVq5dmz55dbP9z584pMTFR0rUA0rFjR7vngmu56z3mxv6OXDDk3LlzDs0D13PXMZOYmGjdd8KECTbf4btR79691bt3b61YsUJz5sxRSkqKKleu7NBc8A2+8PmXEAqXad68udatW6eDBw8qNze3yEub792712YfR9x2222FjlPcPNxyw3O545j5tSlTpujFF1+0jrVs2TJVqlSpVGPCfVx9zNx4mtOkSZNu2n/Pnj0aNWqUJCk2NpYQ6mHc8R7TokUL6+O8vLxi+964vbjbf8A87jhmbrzdRrt27Yrt2759e61YsUL5+fnav38/7zFllC98/uV0XLhMt27dJF07DWDr1q1F9rt+P0ZJ6tq1q0Nz3HnnndYvZN84zq9lZ2dr48aNBfaBZ3HHMXOjhIQE/eEPf5AkNWjQQCtWrFBERESJx4P7ufuYgXdzx/Fy6623qm7dupKkQ4cOFdv3xu21atVyaB64hzuOmRuDbW5ubrF9c3JyCt0PZYsvfP4lhMJlBg8ebH08derUQvvk5+crPj5ekhQeHq6oqCiH5ggLC1Pv3r0lXbtqXFGny8yZM8d6NbkhQ4Y4NAfcxx3HzHVz5szR+PHjZRiGateuraSkJNWsWbNEY8E8rj5m6tWrJ8MwbvpzXY8ePaxt06ZNK9Frguu46z1m2LBhkq5912/Dhg1F9pszZ471cffu3R2eB67njmOmfv361sfr1q0rtu/atWslSRaLRfXq1XNoHvgOn/j8awAu1L17d0OSERAQYGzYsKHA9kmTJhmSDEnGK6+8UmD71KlTi91uGIaRlJRk7XPfffcZubm5NtvPnz9v1K1b15BkhIeHGxcvXnTGS4OLuOOYWbZsmREYGGhIMqpXr27s3bvXya8C7uSOY+Zmru/fo0ePEu0P93HH8XL06FEjODjYkGS0b9/eSEtLK9AnISHBOs7AgQNL+7LgQq4+Zvbs2WNYLBZDklGrVi3jxIkThdbxr3/9yzpO586dS/uy4CZHjhyx/v8WGxtr1z5l4fMv6/hwqQ8//FBdu3ZVRkaG+vXrpwkTJigqKkoZGRlKTEy0Xo20SZMm1lsgOKpXr14aOXKkEhMTtWDBAvXt21dPP/20atasqZ9//llvvPGGjh07Jkl66623+BK/h3P1MbNx40YNGTJE2dnZKleunN5//33l5OQUe3uE2rVrKzw8vKQvCS7mjvcZ+A53HC9169bVa6+9pueff15bt25Vhw4d9Pzzz6tly5a6fPmy5syZo08++USSuPesF3D1MdOsWTONHz9e//73v3Xy5Em1bdtWTz/9tLp3766wsDAdP35ciYmJ+vLLLyVJ/v7++vvf/+7U1wjnWb9+vQ4ePGh9npycbH188ODBAmfJjBs3rkTzeP3nX7NTMHzfggULjIoVK1r/WvPrnyZNmhgHDhwodF97VyiuXr1q3HPPPUXO4efnV+IVDrifK4+ZV155pchxi/qZOnWqa18wSs0d7zPFub4/K6HewV3Hy4svvmhd4Srsp3r16oWurMHzuPqYyczMNEaMGHHT30chISHGjBkzXPhKUVqxsbEOfcYoTFn4/Mt3QuFygwYN0s6dO/XHP/5RTZo0UYUKFRQeHq477rhDb7/9trZv317qq3WVL19eixcv1owZM9S3b19Vr15dgYGBqlOnjkaPHq3169dr4sSJznlBcDl3HDPwLRwzcIS7jpc333xT33//vaKjo1WvXj0FBQWpUqVKuvPOO/X6669r//796ty5sxNeEVzN1cdMUFCQEhMTtXLlSsXExKhJkyYKCQlRQECAqlSpos6dO+uvf/2r9u7dq9GjRzvxlcGbefPnX4th3HBFBQAAAAAAXIiVUAAAAACA2xBCAQAAAABuQwgFAAAAALgNIRQAAAAA4DaEUAAAAACA2xBCAQAAAABuQwgFAAAAALgNIRQAAAAA4DaEUAAAAACA2xBCAQAAAABuQwgFAAAAALgNIRQAAAAA4DaEUAAAAACA2xBCAQAAAABuQwgFAAAAALgNIRQAAAAA4DaEUAAAAACA2xBCAQDwMPXq1ZPFYinVz9NPP13isf39/VWlShU1aNBAAwcO1MSJE/Xzzz/bXX/Pnj1vOoefn58qVaqkOnXq6K677tKTTz6phQsXKjc310n/FQEAnooQCgAAbOTn5yslJUVHjhzRkiVL9Oqrr6pVq1bq0aOH9uzZ45Q5DMNQamqqTpw4oXXr1ukf//iH7rvvPtWpU0fx8fFOmQMA4JkIoQAAwC5r165Vu3btNG/ePJfNcebMGcXGxuqhhx6SYRgumwcAYJ4AswsAAADFmzlzpjp16uTQPhUrVizx2Hl5ebpw4YK2bdumuLg4bdy40botMzNTo0aN0rp163THHXfYNcc777yjBx54wKbt+kro3r17tWTJEiUmJio7O9u6ferUqWrcuLFeeuklu+YAAHgPi8GfGQEA8Cj16tXT0aNHrc9XrVqlnj17mjb2Rx99pCeffNJmZbJbt25at25dof179uypNWvWWJ9PnTpV48aNK3aOHTt2aMCAATp9+rS1LTQ0VIcOHVL16tWL3RcA4F04HRcAABTrD3/4g5555hmbtvXr12v79u1Om6N169aaOXOmTVtaWpq++eYbp80BAPAMhFAAAHBTEyZMULly5Wzali9f7tQ5evToobZt29q0rVy50qlzAADMRwgFAAA3VaVKlQLfAd29e7fT5/n191OPHz/u9DkAAOYihAIAALvUqVPH5nlycrLT56hcubLN84sXLzp9DgCAuQihAADALu64lmFqaqrN8+DgYJfPCQBwL0IoAACwy4kTJ2yeR0REOH2OrVu32jxv0KCB0+cAAJiLEAoAAG4qJSWlQEBs3ry5U+fYsWOHzT1JJTnt1jQAAM9BCAUAwMNFRUXJYrHY/bN69Wqn1/DWW28pOzvbpq1v375OG//UqVMaOXKkzSm/YWFhGjNmjNPmAAB4hgCzCwAAAJ7tk08+0TvvvGPT1qVLF7Vv377EYxqGodTUVO3bt0+LFy/WRx99VOAiRJMmTVK1atVKPAcAwDMRQgEAKMPOnDmj//znPzZteXl5SklJ0bZt2xQfH6/vv//eZntQUJDee+89u+cYP368xo8fb3d/f39/vfnmm/rtb39r9z4AAO9BCAUAwMPNnDmzwP0zi1OjRg27+44aNcqhWoKCgjRjxgx17NjRof3sUb58eQ0bNkwvvPCCWrZs6fTxAQCegRAKAICHq1GjhurVq2d2GerSpYv+9a9/uSwgZmZmKjc3V7Vq1XLJ+AAAz0AIBQAANvz8/BQWFqbw8HA1a9ZMHTp00NChQ9WmTZsSjffOO+/ogQcesGnLyMjQsWPHtHLlSn366ae6dOmSDMNQYmKidu7cqTVr1rjkFjAAAPMRQgEAKMNWrVrl8tugREREFLqS27x5c/Xv31/PPPOM7rnnHm3btk2StHv3bg0fPlwrVqyQnx8X8gcAX8M7OwAAMNUtt9yixYsXKzIy0tq2atUqvf/++yZWBQBwFUIoAAAwXY0aNfTRRx/ZtE2cOFFnzpwxqSIAgKsQQgEAgEcYOnSoevToYX2elpam119/3cSKAACuQAgFAAAe4+9//7vN888//1xHjx41qRoAgCsQQgEAgMfo0qWL+vTpY32enZ2tv/3tbyZWBABwNkIoAADwKK+88orN82nTpunw4cMmVQMAcDZCKAAA8CjdunVTr169rM9zc3P12muvmVgRAMCZCKEAAMDjTJw40eb59OnTtX//fnOKAQA4FSEUAAB4nO7duysqKsr6PC8vT6+++qqJFQEAnIUQCgAAPNKvvxuamJio3bt3m1QNAMBZCKEAAMAj9ejRw+a+ofn5+QVO0wUAeB+LYRiG2UUAAAAAAMoGVkIBAAAAAG5DCAUAAAAAuA0hFAAAAADgNoRQAAAAAIDbEEIBAAAAAG5DCAUAAAAAuA0hFAAAAADgNoRQAAAAAIDbEEIBAAAAAG5DCAUAAAAAuA0hFAAAAADgNoRQAAAAAIDbEEIBAAAAAG5DCAUAAAAAuA0hFAAAAADgNoRQAAAAAIDbEEIBAAAAAG5DCAUAAAAAuA0hFAAAAADgNoRQAAAAAIDbEEIBAAAAAG5DCAUAAAAAuA0hFAAAAADgNoRQAAAAAIDbEEIBAAAAAG5DCAUAAAAAuA0hFAAAAADgNoRQAAAAAIDb/D+yOcwBGCRs3AAAAABJRU5ErkJggg==", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "fig = plt.figure(figsize=(5,5),dpi=200)\n", - "ax = plt.subplot(111)\n", - "ax.plot(fpr,tpr,c='r',label='ScFM',lw=2.5)\n", - "ax.plot(fpr_b,tpr_b,c='g',label='DeepDDS',lw=2.5)\n", - "ax.plot(np.arange(0,1,1/100),np.arange(0,1,1/100),c='grey',ls='--')\n", - "ax.spines[['right','top']].set_visible(False)\n", - "for axis in ['bottom','left']:\n", - " ax.spines[axis].set_linewidth(3)\n", - "ax.set_xlabel('FPR',fontdict={'size':15})\n", - "ax.set_ylabel('TPR',fontdict={'size':15})\n", - "ax.legend()\n", - "ax.set_title('ROC Curve',fontdict={'size':20})\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "base", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.9" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Legacy DeepDDS reference workflow\n", + "This notebook uses legacy scLong/DeepDDS artifacts, not BioMod model weights. Set `DEEPDDS_ARTIFACT_DIR` before starting the kernel if your generated embeddings or ROC results are stored elsewhere. These external results are not included in this repository.\n" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from pathlib import Path\n", + "import sys\n", + "\n", + "# Start Jupyter in this drug_combination directory.\n", + "REFERENCE_DIR = Path(\".\").resolve()\n", + "if not (REFERENCE_DIR / \"_reference_paths.py\").is_file():\n", + " raise FileNotFoundError(\"Start this notebook from the legacy drug_combination directory.\")\n", + "sys.path.insert(0, str(REFERENCE_DIR))\n", + "from _reference_paths import ASSET_DIR, artifact_path, data_path, output_dir, output_path, required_artifact\n" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import pickle\n", + "from sklearn import metrics\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "with open(required_artifact('fpr_tpr/fpr_tpr_scfm_run2.pkl'),'rb') as f:\n", + " fpr,tpr=pickle.load(f)\n", + "with open(required_artifact('fpr_tpr/fpr_tpr_bsl_run4.pkl'),'rb') as f:\n", + " fpr_b,tpr_b=pickle.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "fig = plt.figure(figsize=(5,5),dpi=200)\n", + "ax = plt.subplot(111)\n", + "ax.plot(fpr,tpr,c='r',label='ScFM',lw=2.5)\n", + "ax.plot(fpr_b,tpr_b,c='g',label='DeepDDS',lw=2.5)\n", + "ax.plot(np.arange(0,1,1/100),np.arange(0,1,1/100),c='grey',ls='--')\n", + "ax.spines[['right','top']].set_visible(False)\n", + "for axis in ['bottom','left']:\n", + " ax.spines[axis].set_linewidth(3)\n", + "ax.set_xlabel('FPR',fontdict={'size':15})\n", + "ax.set_ylabel('TPR',fontdict={'size':15})\n", + "ax.legend()\n", + "ax.set_title('ROC Curve',fontdict={'size':20})\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "base", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.9" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/data/downstream/final_data/drug_response_pytorch/drug_combination/process.ipynb b/data/downstream/final_data/drug_response_pytorch/drug_combination/process.ipynb index 930351880fdd79393f1ee15d9dc8f02f4917b571..e76f45e580196d1dfe0637d3f951589a0f6b8456 100644 --- a/data/downstream/final_data/drug_response_pytorch/drug_combination/process.ipynb +++ b/data/downstream/final_data/drug_response_pytorch/drug_combination/process.ipynb @@ -1,1165 +1,156 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "import numpy as np\n", - "import pandas as pd\n", - "from sklearn.preprocessing import normalize" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "954" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# Test intersect genes\n", - "\n", - "gene_list_scfm=[]\n", - "with open('/data1/ruiyi/scfm/selected_genes_27k.txt') as f:\n", - " for l in f.readlines():\n", - " gene_list_scfm.append(str(l.strip()))\n", - "len(gene_list_scfm)\n", - "\n", - "gene_list_deepdds=[]\n", - "with open('data/independent_set/independent_cell_features_954.csv') as f:\n", - " for l in f.readlines():\n", - " l=l.strip().split(',')\n", - " #print(l)\n", - " break\n", - "gene_list_deepdds=l[1:]\n", - "len(gene_list_deepdds)\n", - "\n", - "cnt=0\n", - "for gene in gene_list_deepdds:\n", - " if gene in gene_list_scfm:cnt+=1\n", - "cnt\n" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "deepdds_scfm_output=np.load('/data1/ruiyi/deepdds/deepdds_ind_scfm1227_output.npy')\n", - "deepdds_scfm_output.shape" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(1019, 954, 200)" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "deepdds_scfm_output_processed=np.zeros((deepdds_scfm_output.shape[0],len(gene_list_deepdds),deepdds_scfm_output.shape[-1]))\n", - "gene_deepdds2idx={v:k for k,v in enumerate(gene_list_deepdds)}\n", - "for k,v in enumerate(gene_list_scfm):\n", - " if v in gene_deepdds2idx:\n", - " idx=gene_deepdds2idx[v]\n", - " deepdds_scfm_output_processed[:,idx,:]=deepdds_scfm_output[:,k,:]\n", - "deepdds_scfm_output_processed.shape " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "np.save('/data1/ruiyi/deepdds/deepdds_ind_scfm1227_output_processd.npy', deepdds_scfm_output_processed)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(['X22RV1_PROSTATE',\n", - " 'X2313287_STOMACH',\n", - " 'X253JBV_URINARY_TRACT',\n", - " 'X253J_URINARY_TRACT',\n", - " 'X42MGBA_CENTRAL_NERVOUS_SYSTEM',\n", - " 'X5637_URINARY_TRACT',\n", - " 'X59M_OVARY',\n", - " 'X639V_URINARY_TRACT',\n", - " 'X647V_URINARY_TRACT',\n", - " 'X697_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'X769P_KIDNEY',\n", - " 'X786O_KIDNEY',\n", - " 'X8305C_THYROID',\n", - " 'X8505C_THYROID',\n", - " 'X8MGBA_CENTRAL_NERVOUS_SYSTEM',\n", - " 'A101D_SKIN',\n", - " 'A1207_CENTRAL_NERVOUS_SYSTEM',\n", - " 'A172_CENTRAL_NERVOUS_SYSTEM',\n", - " 'A204_SOFT_TISSUE',\n", - " 'A2058_SKIN',\n", - " 'A253_SALIVARY_GLAND',\n", - " 'A2780_OVARY',\n", - " 'A375_SKIN',\n", - " 'A3KAW_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'A427_LUNG',\n", - " 'A498_KIDNEY',\n", - " 'A4FUK_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'A549_LUNG',\n", - " 'A673_BONE',\n", - " 'A704_KIDNEY',\n", - " 'ABC1_LUNG',\n", - " 'ACCMESO1_PLEURA',\n", - " 'ACHN_KIDNEY',\n", - " 'AGS_STOMACH',\n", - " 'ALLSIL_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'AM38_CENTRAL_NERVOUS_SYSTEM',\n", - " 'AML193_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'AMO1_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'AN3CA_ENDOMETRIUM',\n", - " 'ASPC1_PANCREAS',\n", - " 'AU565_BREAST',\n", - " 'BC3C_URINARY_TRACT',\n", - " 'BCP1_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'BCPAP_THYROID',\n", - " 'BDCM_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'BEN_LUNG',\n", - " 'BFTC905_URINARY_TRACT',\n", - " 'BFTC909_KIDNEY',\n", - " 'BHT101_THYROID',\n", - " 'BHY_UPPER_AERODIGESTIVE_TRACT',\n", - " 'BICR16_UPPER_AERODIGESTIVE_TRACT',\n", - " 'BICR18_UPPER_AERODIGESTIVE_TRACT',\n", - " 'BICR22_UPPER_AERODIGESTIVE_TRACT',\n", - " 'BICR31_UPPER_AERODIGESTIVE_TRACT',\n", - " 'BICR56_UPPER_AERODIGESTIVE_TRACT',\n", - " 'BICR6_UPPER_AERODIGESTIVE_TRACT',\n", - " 'BL41_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'BL70_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'BT12_SOFT_TISSUE',\n", - " 'BT20_BREAST',\n", - " 'BT474_BREAST',\n", - " 'BT483_BREAST',\n", - " 'BT549_BREAST',\n", - " 'BV173_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'BXPC3_PANCREAS',\n", - " 'C2BBE1_LARGE_INTESTINE',\n", - " 'C32_SKIN',\n", - " 'C8166_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'CA46_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'CACO2_LARGE_INTESTINE',\n", - " 'CADOES1_BONE',\n", - " 'CAKI1_KIDNEY',\n", - " 'CAKI2_KIDNEY',\n", - " 'CAL120_BREAST',\n", - " 'CAL12T_LUNG',\n", - " 'CAL148_BREAST',\n", - " 'CAL27_UPPER_AERODIGESTIVE_TRACT',\n", - " 'CAL29_URINARY_TRACT',\n", - " 'CAL33_UPPER_AERODIGESTIVE_TRACT',\n", - " 'CAL51_BREAST',\n", - " 'CAL54_KIDNEY',\n", - " 'CAL62_THYROID',\n", - " 'CAL78_BONE',\n", - " 'CAL851_BREAST',\n", - " 'CALU1_LUNG',\n", - " 'CALU3_LUNG',\n", - " 'CALU6_LUNG',\n", - " 'CAMA1_BREAST',\n", - " 'CAOV3_OVARY',\n", - " 'CAOV4_OVARY',\n", - " 'CAPAN1_PANCREAS',\n", - " 'CAPAN2_PANCREAS',\n", - " 'CAS1_CENTRAL_NERVOUS_SYSTEM',\n", - " 'CCFSTTG1_CENTRAL_NERVOUS_SYSTEM',\n", - " 'CCK81_LARGE_INTESTINE',\n", - " 'CFPAC1_PANCREAS',\n", - " 'CH157MN_CENTRAL_NERVOUS_SYSTEM',\n", - " 'CHAGOK1_LUNG',\n", - " 'CHP126_AUTONOMIC_GANGLIA',\n", - " 'CHP212_AUTONOMIC_GANGLIA',\n", - " 'CI1_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'CJM_SKIN',\n", - " 'CL11_LARGE_INTESTINE',\n", - " 'CL14_LARGE_INTESTINE',\n", - " 'CL34_LARGE_INTESTINE',\n", - " 'CL40_LARGE_INTESTINE',\n", - " 'CMK_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'CMLT1_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'COLO201_LARGE_INTESTINE',\n", - " 'COLO320_LARGE_INTESTINE',\n", - " 'COLO668_LUNG',\n", - " 'COLO678_LARGE_INTESTINE',\n", - " 'COLO679_SKIN',\n", - " 'COLO680N_OESOPHAGUS',\n", - " 'COLO684_ENDOMETRIUM',\n", - " 'COLO741_SKIN',\n", - " 'COLO783_SKIN',\n", - " 'COLO792_SKIN',\n", - " 'COLO800_SKIN',\n", - " 'COLO829_SKIN',\n", - " 'CORL105_LUNG',\n", - " 'CORL23_LUNG',\n", - " 'CORL24_LUNG',\n", - " 'CORL279_LUNG',\n", - " 'CORL311_LUNG',\n", - " 'CORL47_LUNG',\n", - " 'CORL88_LUNG',\n", - " 'CORL95_LUNG',\n", - " 'COV318_OVARY',\n", - " 'COV362_OVARY',\n", - " 'COV434_OVARY',\n", - " 'COV644_OVARY',\n", - " 'CW2_LARGE_INTESTINE',\n", - " 'D283MED_CENTRAL_NERVOUS_SYSTEM',\n", - " 'D341MED_CENTRAL_NERVOUS_SYSTEM',\n", - " 'DANG_PANCREAS',\n", - " 'DAOY_CENTRAL_NERVOUS_SYSTEM',\n", - " 'DAUDI_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'DBTRG05MG_CENTRAL_NERVOUS_SYSTEM',\n", - " 'DB_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'DEL_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'DETROIT562_UPPER_AERODIGESTIVE_TRACT',\n", - " 'DKMG_CENTRAL_NERVOUS_SYSTEM',\n", - " 'DM3_FIBROBLAST',\n", - " 'DMS114_LUNG',\n", - " 'DMS153_LUNG',\n", - " 'DMS273_LUNG',\n", - " 'DMS454_LUNG',\n", - " 'DMS53_LUNG',\n", - " 'DMS79_LUNG',\n", - " 'DND41_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'DOHH2_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'DU145_PROSTATE',\n", - " 'DU4475_BREAST',\n", - " 'DV90_LUNG',\n", - " 'EB1_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'EBC1_LUNG',\n", - " 'ECC10_STOMACH',\n", - " 'ECC12_STOMACH',\n", - " 'ECGI10_OESOPHAGUS',\n", - " 'EFE184_ENDOMETRIUM',\n", - " 'EFM192A_BREAST',\n", - " 'EFM19_BREAST',\n", - " 'EFO21_OVARY',\n", - " 'EFO27_OVARY',\n", - " 'EHEB_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'EJM_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'EKVX_LUNG',\n", - " 'EM2_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'EN_ENDOMETRIUM',\n", - " 'EOL1_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'EPLC272H_LUNG',\n", - " 'ES2_OVARY',\n", - " 'ESS1_ENDOMETRIUM',\n", - " 'EW8_BONE',\n", - " 'EWS502_BONE',\n", - " 'F36P_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'FADU_UPPER_AERODIGESTIVE_TRACT',\n", - " 'FTC133_THYROID',\n", - " 'FTC238_THYROID',\n", - " 'FU97_STOMACH',\n", - " 'FUOV1_OVARY',\n", - " 'G292CLONEA141B1_BONE',\n", - " 'G361_SKIN',\n", - " 'G401_SOFT_TISSUE',\n", - " 'G402_SOFT_TISSUE',\n", - " 'GA10_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'GAMG_CENTRAL_NERVOUS_SYSTEM',\n", - " 'GB1_CENTRAL_NERVOUS_SYSTEM',\n", - " 'GCIY_STOMACH',\n", - " 'GCT_SOFT_TISSUE',\n", - " 'GDM1_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'GI1_CENTRAL_NERVOUS_SYSTEM',\n", - " 'GMS10_CENTRAL_NERVOUS_SYSTEM',\n", - " 'GOS3_CENTRAL_NERVOUS_SYSTEM',\n", - " 'GP2D_LARGE_INTESTINE',\n", - " 'GRANTA519_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'GSS_STOMACH',\n", - " 'GSU_STOMACH',\n", - " 'H4_CENTRAL_NERVOUS_SYSTEM',\n", - " 'HARA_LUNG',\n", - " 'HCC1143_BREAST',\n", - " 'HCC1171_LUNG',\n", - " 'HCC1187_BREAST',\n", - " 'HCC1195_LUNG',\n", - " 'HCC1359_LUNG',\n", - " 'HCC1395_BREAST',\n", - " 'HCC1419_BREAST',\n", - " 'HCC1428_BREAST',\n", - " 'HCC1438_LUNG',\n", - " 'HCC1500_BREAST',\n", - " 'HCC1569_BREAST',\n", - " 'HCC1588_LUNG',\n", - " 'HCC1599_BREAST',\n", - " 'HCC15_LUNG',\n", - " 'HCC1806_BREAST',\n", - " 'HCC1833_LUNG',\n", - " 'HCC1937_BREAST',\n", - " 'HCC1954_BREAST',\n", - " 'HCC202_BREAST',\n", - " 'HCC2108_LUNG',\n", - " 'HCC2157_BREAST',\n", - " 'HCC2218_BREAST',\n", - " 'HCC2279_LUNG',\n", - " 'HCC2429_LUNG',\n", - " 'HCC2450_LUNG',\n", - " 'HCC2814_LUNG',\n", - " 'HCC2935_LUNG',\n", - " 'HCC33_LUNG',\n", - " 'HCC364_LUNG',\n", - " 'HCC366_LUNG',\n", - " 'HCC38_BREAST',\n", - " 'HCC4006_LUNG',\n", - " 'HCC44_LUNG',\n", - " 'HCC461_LUNG',\n", - " 'HCC515_LUNG',\n", - " 'HCC56_LARGE_INTESTINE',\n", - " 'HCC70_BREAST',\n", - " 'HCC78_LUNG',\n", - " 'HCC827GR5_LUNG',\n", - " 'HCC827_LUNG',\n", - " 'HCC95_LUNG',\n", - " 'HCT116_LARGE_INTESTINE',\n", - " 'HCT15_LARGE_INTESTINE',\n", - " 'HDLM2_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'HDMYZ_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'HDQP1_BREAST',\n", - " 'HEC108_ENDOMETRIUM',\n", - " 'HEC151_ENDOMETRIUM',\n", - " 'HEC1A_ENDOMETRIUM',\n", - " 'HEC1B_ENDOMETRIUM',\n", - " 'HEC251_ENDOMETRIUM',\n", - " 'HEC265_ENDOMETRIUM',\n", - " 'HEC50B_ENDOMETRIUM',\n", - " 'HEC59_ENDOMETRIUM',\n", - " 'HEC6_ENDOMETRIUM',\n", - " 'HEKTE_KIDNEY',\n", - " 'HEL9217_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'HELA_CERVIX',\n", - " 'HEL_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'HEP3B217_LIVER',\n", - " 'HEPG2_LIVER',\n", - " 'HEYA8_OVARY',\n", - " 'HGC27_STOMACH',\n", - " 'HH_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'HL60_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'HLFA_FIBROBLAST',\n", - " 'HLF_LIVER',\n", - " 'HMC18_BREAST',\n", - " 'HMCB_SKIN',\n", - " 'HMEL_BREAST',\n", - " 'HNT34_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'HOP62_LUNG',\n", - " 'HOP92_LUNG',\n", - " 'HOS_BONE',\n", - " 'HPAC_PANCREAS',\n", - " 'HPAFII_PANCREAS',\n", - " 'HPBALL_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'HS172T_FIBROBLAST',\n", - " 'HS229T_FIBROBLAST',\n", - " 'HS255T_FIBROBLAST',\n", - " 'HS274T_FIBROBLAST',\n", - " 'HS281T_FIBROBLAST',\n", - " 'HS294T_SKIN',\n", - " 'HS343T_FIBROBLAST',\n", - " 'HS578T_BREAST',\n", - " 'HS600T_FIBROBLAST',\n", - " 'HS606T_FIBROBLAST',\n", - " 'HS611T_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'HS616T_FIBROBLAST',\n", - " 'HS618T_FIBROBLAST',\n", - " 'HS675T_FIBROBLAST',\n", - " 'HS683_CENTRAL_NERVOUS_SYSTEM',\n", - " 'HS688AT_FIBROBLAST',\n", - " 'HS695T_SKIN',\n", - " 'HS698T_FIBROBLAST',\n", - " 'HS706T_BONE',\n", - " 'HS729_SOFT_TISSUE',\n", - " 'HS737T_FIBROBLAST',\n", - " 'HS739T_FIBROBLAST',\n", - " 'HS742T_FIBROBLAST',\n", - " 'HS746T_STOMACH',\n", - " 'HS751T_FIBROBLAST',\n", - " 'HS766T_PANCREAS',\n", - " 'HS819T_FIBROBLAST',\n", - " 'HS821T_FIBROBLAST',\n", - " 'HS822T_FIBROBLAST',\n", - " 'HS834T_FIBROBLAST',\n", - " 'HS839T_FIBROBLAST',\n", - " 'HS840T_FIBROBLAST',\n", - " 'HS852T_SKIN',\n", - " 'HS863T_FIBROBLAST',\n", - " 'HS870T_FIBROBLAST',\n", - " 'HS888T_FIBROBLAST',\n", - " 'HS895T_FIBROBLAST',\n", - " 'HS934T_FIBROBLAST',\n", - " 'HS936T_SKIN',\n", - " 'HS939T_SKIN',\n", - " 'HS940T_FIBROBLAST',\n", - " 'HS944T_SKIN',\n", - " 'HSC2_UPPER_AERODIGESTIVE_TRACT',\n", - " 'HSC3_UPPER_AERODIGESTIVE_TRACT',\n", - " 'HSC4_UPPER_AERODIGESTIVE_TRACT',\n", - " 'HT1080_SOFT_TISSUE',\n", - " 'HT115_LARGE_INTESTINE',\n", - " 'HT1197_URINARY_TRACT',\n", - " 'HT1376_URINARY_TRACT',\n", - " 'HT144_SKIN',\n", - " 'HT29_LARGE_INTESTINE',\n", - " 'HT55_LARGE_INTESTINE',\n", - " 'HT_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'HUCCT1_BILIARY_TRACT',\n", - " 'HUG1N_STOMACH',\n", - " 'HUH1_LIVER',\n", - " 'HUH28_BILIARY_TRACT',\n", - " 'HUH6_LIVER',\n", - " 'HUH7_LIVER',\n", - " 'HUNS1_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'HUPT3_PANCREAS',\n", - " 'HUPT4_PANCREAS',\n", - " 'HUT102_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'HUT78_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'HUTU80_SMALL_INTESTINE',\n", - " 'IALM_LUNG',\n", - " 'IGR1_SKIN',\n", - " 'IGR37_SKIN',\n", - " 'IGR39_SKIN',\n", - " 'IGROV1_OVARY',\n", - " 'IM95_STOMACH',\n", - " 'IMR32_AUTONOMIC_GANGLIA',\n", - " 'INA6_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'IOMMLEE_CENTRAL_NERVOUS_SYSTEM',\n", - " 'IPC298_SKIN',\n", - " 'ISHIKAWAHERAKLIO02ER_ENDOMETRIUM',\n", - " 'ISTMES1_PLEURA',\n", - " 'ISTMES2_PLEURA',\n", - " 'J82_URINARY_TRACT',\n", - " 'JEKO1_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'JHESOAD1_OESOPHAGUS',\n", - " 'JHH1_LIVER',\n", - " 'JHH2_LIVER',\n", - " 'JHH4_LIVER',\n", - " 'JHH5_LIVER',\n", - " 'JHH6_LIVER',\n", - " 'JHH7_LIVER',\n", - " 'JHOC5_OVARY',\n", - " 'JHOM1_OVARY',\n", - " 'JHOM2B_OVARY',\n", - " 'JHOS2_OVARY',\n", - " 'JHOS4_OVARY',\n", - " 'JHUEM1_ENDOMETRIUM',\n", - " 'JHUEM2_ENDOMETRIUM',\n", - " 'JHUEM3_ENDOMETRIUM',\n", - " 'JHUEM7_ENDOMETRIUM',\n", - " 'JIMT1_BREAST',\n", - " 'JJN3_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'JK1_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'JL1_PLEURA',\n", - " 'JM1_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'JMSU1_URINARY_TRACT',\n", - " 'JURKAT_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'JURLMK1_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'JVM2_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'JVM3_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'K029AX_SKIN',\n", - " 'K562_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'KALS1_CENTRAL_NERVOUS_SYSTEM',\n", - " 'KARPAS299_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'KARPAS422_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'KARPAS620_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'KASUMI1_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'KASUMI2_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'KASUMI6_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'KATOIII_STOMACH',\n", - " 'KCL22_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'KD_SOFT_TISSUE',\n", - " 'KE37_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'KE39_STOMACH',\n", - " 'KE97_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'KELLY_AUTONOMIC_GANGLIA',\n", - " 'KG1C_CENTRAL_NERVOUS_SYSTEM',\n", - " 'KG1_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'KHM1B_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'KIJK_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'KLE_ENDOMETRIUM',\n", - " 'KM12_LARGE_INTESTINE',\n", - " 'KMBC2_URINARY_TRACT',\n", - " 'KMH2_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'KMM1_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'KMRC1_KIDNEY',\n", - " 'KMRC20_KIDNEY',\n", - " 'KMRC2_KIDNEY',\n", - " 'KMRC3_KIDNEY',\n", - " 'KMS11_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'KMS12BM_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'KMS18_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'KMS20_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'KMS21BM_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'KMS26_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'KMS27_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'KMS28BM_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'KMS34_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'KNS42_CENTRAL_NERVOUS_SYSTEM',\n", - " 'KNS60_CENTRAL_NERVOUS_SYSTEM',\n", - " 'KNS62_LUNG',\n", - " 'KNS81_CENTRAL_NERVOUS_SYSTEM',\n", - " 'KO52_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'KOPN8_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'KP2_PANCREAS',\n", - " 'KP3_PANCREAS',\n", - " 'KP4_PANCREAS',\n", - " 'KPL1_BREAST',\n", - " 'KPMRTRY_SOFT_TISSUE',\n", - " 'KPNRTBM1_AUTONOMIC_GANGLIA',\n", - " 'KPNSI9S_AUTONOMIC_GANGLIA',\n", - " 'KPNYN_AUTONOMIC_GANGLIA',\n", - " 'KS1_CENTRAL_NERVOUS_SYSTEM',\n", - " 'KU1919_URINARY_TRACT',\n", - " 'KU812_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'KURAMOCHI_OVARY',\n", - " 'KYM1_SOFT_TISSUE',\n", - " 'KYO1_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'KYSE140_OESOPHAGUS',\n", - " 'KYSE150_OESOPHAGUS',\n", - " 'KYSE180_OESOPHAGUS',\n", - " 'KYSE270_OESOPHAGUS',\n", - " 'KYSE30_OESOPHAGUS',\n", - " 'KYSE410_OESOPHAGUS',\n", - " 'KYSE450_OESOPHAGUS',\n", - " 'KYSE510_OESOPHAGUS',\n", - " 'KYSE520_OESOPHAGUS',\n", - " 'KYSE70_OESOPHAGUS',\n", - " 'L1236_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'L33_PANCREAS',\n", - " 'L363_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'L428_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'L540_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'LAMA84_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'LC1F_LUNG',\n", - " 'LCLC103H_LUNG',\n", - " 'LCLC97TM1_LUNG',\n", - " 'LI7_LIVER',\n", - " 'LK2_LUNG',\n", - " 'LMSU_STOMACH',\n", - " 'LN18_CENTRAL_NERVOUS_SYSTEM',\n", - " 'LN215_CENTRAL_NERVOUS_SYSTEM',\n", - " 'LN229_CENTRAL_NERVOUS_SYSTEM',\n", - " 'LN235_CENTRAL_NERVOUS_SYSTEM',\n", - " 'LN319_CENTRAL_NERVOUS_SYSTEM',\n", - " 'LN340_CENTRAL_NERVOUS_SYSTEM',\n", - " 'LN382_CENTRAL_NERVOUS_SYSTEM',\n", - " 'LN428_CENTRAL_NERVOUS_SYSTEM',\n", - " 'LN443_CENTRAL_NERVOUS_SYSTEM',\n", - " 'LN464_CENTRAL_NERVOUS_SYSTEM',\n", - " 'LNCAPCLONEFGC_PROSTATE',\n", - " 'LNZ308_CENTRAL_NERVOUS_SYSTEM',\n", - " 'LOUCY_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'LOUNH91_LUNG',\n", - " 'LOVO_LARGE_INTESTINE',\n", - " 'LOXIMVI_SKIN',\n", - " 'LP1_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'LS1034_LARGE_INTESTINE',\n", - " 'LS123_LARGE_INTESTINE',\n", - " 'LS180_LARGE_INTESTINE',\n", - " 'LS411N_LARGE_INTESTINE',\n", - " 'LS513_LARGE_INTESTINE',\n", - " 'LU65_LUNG',\n", - " 'LU99_LUNG',\n", - " 'LUDLU1_LUNG',\n", - " 'LXF289_LUNG',\n", - " 'M059K_CENTRAL_NERVOUS_SYSTEM',\n", - " 'M07E_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'MALME3M_SKIN',\n", - " 'MC116_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'MCAS_OVARY',\n", - " 'MCF7_BREAST',\n", - " 'MDAMB134VI_BREAST',\n", - " 'MDAMB157_BREAST',\n", - " 'MDAMB175VII_BREAST',\n", - " 'MDAMB231_BREAST',\n", - " 'MDAMB361_BREAST',\n", - " 'MDAMB415_BREAST',\n", - " 'MDAMB435S_SKIN',\n", - " 'MDAMB436_BREAST',\n", - " 'MDAMB453_BREAST',\n", - " 'MDAMB468_BREAST',\n", - " 'MDAPCA2B_PROSTATE',\n", - " 'MDST8_LARGE_INTESTINE',\n", - " 'ME1_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'MEC1_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'MEG01_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'MELHO_SKIN',\n", - " 'MELJUSO_SKIN',\n", - " 'MESSA_SOFT_TISSUE',\n", - " 'MEWO_SKIN',\n", - " 'MFE280_ENDOMETRIUM',\n", - " 'MFE296_ENDOMETRIUM',\n", - " 'MFE319_ENDOMETRIUM',\n", - " 'MG63_BONE',\n", - " 'MHHCALL2_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'MHHCALL3_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'MHHCALL4_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'MHHES1_BONE',\n", - " 'MHHNB11_AUTONOMIC_GANGLIA',\n", - " 'MIAPACA2_PANCREAS',\n", - " 'MINO_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'MJ_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'MKN1_STOMACH',\n", - " 'MKN45_STOMACH',\n", - " 'MKN74_STOMACH',\n", - " 'MKN7_STOMACH',\n", - " 'ML1_THYROID',\n", - " 'MM1S_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'MOLM13_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'MOLM16_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'MOLM6_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'MOLP2_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'MOLP8_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'MOLT13_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'MOLT16_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'MOLT3_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'MONOMAC1_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'MONOMAC6_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'MON_SOFT_TISSUE',\n", - " 'MORCPR_LUNG',\n", - " 'MPP89_PLEURA',\n", - " 'MSTO211H_PLEURA',\n", - " 'MUTZ3_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'MUTZ5_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'MV411_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'NALM19_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'NALM1_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'NALM6_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'NAMALWA_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'NB1_AUTONOMIC_GANGLIA',\n", - " 'NB4_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'NCCSTCK140_STOMACH',\n", - " 'NCIH1048_LUNG',\n", - " 'NCIH1092_LUNG',\n", - " 'NCIH1105_LUNG',\n", - " 'NCIH1155_LUNG',\n", - " 'NCIH1184_LUNG',\n", - " 'NCIH1299_LUNG',\n", - " 'NCIH1339_LUNG',\n", - " 'NCIH1341_LUNG',\n", - " 'NCIH1355_LUNG',\n", - " 'NCIH1373_LUNG',\n", - " 'NCIH1385_LUNG',\n", - " 'NCIH1395_LUNG',\n", - " 'NCIH1435_LUNG',\n", - " 'NCIH1436_LUNG',\n", - " 'NCIH1437_LUNG',\n", - " 'NCIH146_LUNG',\n", - " 'NCIH1563_LUNG',\n", - " 'NCIH1568_LUNG',\n", - " 'NCIH1573_LUNG',\n", - " 'NCIH1581_LUNG',\n", - " 'NCIH1618_LUNG',\n", - " 'NCIH1623_LUNG',\n", - " 'NCIH1648_LUNG',\n", - " 'NCIH1650_LUNG',\n", - " 'NCIH1651_LUNG',\n", - " 'NCIH1666_LUNG',\n", - " 'NCIH1693_LUNG',\n", - " 'NCIH1694_LUNG',\n", - " 'NCIH1703_LUNG',\n", - " 'NCIH1734_LUNG',\n", - " 'NCIH1755_LUNG',\n", - " 'NCIH1781_LUNG',\n", - " 'NCIH1792_LUNG',\n", - " 'NCIH1793_LUNG',\n", - " 'NCIH1819_LUNG',\n", - " 'NCIH1836_LUNG',\n", - " 'NCIH1838_LUNG',\n", - " 'NCIH1869_LUNG',\n", - " 'NCIH1876_LUNG',\n", - " 'NCIH1915_LUNG',\n", - " 'NCIH1930_LUNG',\n", - " 'NCIH1944_LUNG',\n", - " 'NCIH1963_LUNG',\n", - " 'NCIH196_LUNG',\n", - " 'NCIH1975_LUNG',\n", - " 'NCIH2004RT_SOFT_TISSUE',\n", - " 'NCIH2009_LUNG',\n", - " 'NCIH2023_LUNG',\n", - " 'NCIH2029_LUNG',\n", - " 'NCIH2030_LUNG',\n", - " 'NCIH2052_PLEURA',\n", - " 'NCIH2066_LUNG',\n", - " 'NCIH2073_LUNG',\n", - " 'NCIH2077_LUNG',\n", - " 'NCIH2081_LUNG',\n", - " 'NCIH2085_LUNG',\n", - " 'NCIH2087_LUNG',\n", - " 'NCIH209_LUNG',\n", - " 'NCIH2106_LUNG',\n", - " 'NCIH2110_LUNG',\n", - " 'NCIH211_LUNG',\n", - " 'NCIH2122_LUNG',\n", - " 'NCIH2126_LUNG',\n", - " 'NCIH2170_LUNG',\n", - " 'NCIH2171_LUNG',\n", - " 'NCIH2172_LUNG',\n", - " 'NCIH2196_LUNG',\n", - " 'NCIH2227_LUNG',\n", - " 'NCIH2228_LUNG',\n", - " 'NCIH226_LUNG',\n", - " 'NCIH2286_LUNG',\n", - " 'NCIH2291_LUNG',\n", - " 'NCIH2342_LUNG',\n", - " 'NCIH2347_LUNG',\n", - " 'NCIH23_LUNG',\n", - " 'NCIH2405_LUNG',\n", - " 'NCIH2444_LUNG',\n", - " 'NCIH2452_PLEURA',\n", - " 'NCIH2882_LUNG',\n", - " 'NCIH2887_LUNG',\n", - " 'NCIH28_PLEURA',\n", - " 'NCIH292_LUNG',\n", - " 'NCIH3122_LUNG',\n", - " 'NCIH322_LUNG',\n", - " 'NCIH3255_LUNG',\n", - " 'NCIH358_LUNG',\n", - " 'NCIH441_LUNG',\n", - " 'NCIH446_LUNG',\n", - " 'NCIH460_LUNG',\n", - " 'NCIH508_LARGE_INTESTINE',\n", - " 'NCIH510_LUNG',\n", - " 'NCIH520_LUNG',\n", - " 'NCIH522_LUNG',\n", - " 'NCIH524_LUNG',\n", - " 'NCIH526_LUNG',\n", - " 'NCIH596_LUNG',\n", - " 'NCIH647_LUNG',\n", - " 'NCIH650_LUNG',\n", - " 'NCIH660_PROSTATE',\n", - " 'NCIH661_LUNG',\n", - " 'NCIH684_LIVER',\n", - " 'NCIH69_LUNG',\n", - " 'NCIH716_LARGE_INTESTINE',\n", - " 'NCIH727_LUNG',\n", - " 'NCIH747_LARGE_INTESTINE',\n", - " 'NCIH810_LUNG',\n", - " 'NCIH82_LUNG',\n", - " 'NCIH838_LUNG',\n", - " 'NCIH841_LUNG',\n", - " 'NCIH854_LUNG',\n", - " 'NCIH889_LUNG',\n", - " 'NCIH929_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'NCIN87_STOMACH',\n", - " 'NCO2_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'NH6_AUTONOMIC_GANGLIA',\n", - " 'NHAHTDD_CENTRAL_NERVOUS_SYSTEM',\n", - " 'NIHOVCAR3_OVARY',\n", - " 'NMCG1_CENTRAL_NERVOUS_SYSTEM',\n", - " 'NOMO1_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'NUDHL1_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'NUDUL1_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'NUGC2_STOMACH',\n", - " 'NUGC3_STOMACH',\n", - " 'NUGC4_STOMACH',\n", - " 'OAW28_OVARY',\n", - " 'OAW42_OVARY',\n", - " 'OC314_OVARY',\n", - " 'OCIAML2_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'OCIAML3_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'OCIAML5_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'OCILY19_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'OCILY3_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'OCIM1_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'OCIMY5_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'OCIMY7_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'OCUM1_STOMACH',\n", - " 'OE19_OESOPHAGUS',\n", - " 'OE21_OESOPHAGUS',\n", - " 'OE33_OESOPHAGUS',\n", - " 'OELE_OVARY',\n", - " 'ONCODG1_OVARY',\n", - " 'ONS76_CENTRAL_NERVOUS_SYSTEM',\n", - " 'OPM1_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'OPM2_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'OSRC2_KIDNEY',\n", - " 'OUMS23_LARGE_INTESTINE',\n", - " 'OV56_OVARY',\n", - " 'OV7_OVARY',\n", - " 'OV90_OVARY',\n", - " 'OVCAR4_OVARY',\n", - " 'OVCAR8_OVARY',\n", - " 'OVISE_OVARY',\n", - " 'OVK18_OVARY',\n", - " 'OVKATE_OVARY',\n", - " 'OVMANA_OVARY',\n", - " 'OVSAHO_OVARY',\n", - " 'OVTOKO_OVARY',\n", - " 'P12ICHIKAWA_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'P31FUJ_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'P3HR1_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'PANC0203_PANCREAS',\n", - " 'PANC0213_PANCREAS',\n", - " 'PANC0327_PANCREAS',\n", - " 'PANC0403_PANCREAS',\n", - " 'PANC0504_PANCREAS',\n", - " 'PANC0813_PANCREAS',\n", - " 'PANC1005_PANCREAS',\n", - " 'PANC1_PANCREAS',\n", - " 'PATU8902_PANCREAS',\n", - " 'PATU8988S_PANCREAS',\n", - " 'PATU8988T_PANCREAS',\n", - " 'PC14_LUNG',\n", - " 'PC3_PROSTATE',\n", - " 'PC9_LUNG',\n", - " 'PECAPJ15_UPPER_AERODIGESTIVE_TRACT',\n", - " 'PECAPJ34CLONEC12_UPPER_AERODIGESTIVE_TRACT',\n", - " 'PECAPJ41CLONED2_UPPER_AERODIGESTIVE_TRACT',\n", - " 'PECAPJ49_UPPER_AERODIGESTIVE_TRACT',\n", - " 'PEER_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'PF382_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'PFEIFFER_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'PK1_PANCREAS',\n", - " 'PK45H_PANCREAS',\n", - " 'PK59_PANCREAS',\n", - " 'PL21_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'PLB985_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'PLCPRF5_LIVER',\n", - " 'PRECLH_PROSTATE',\n", - " 'PSN1_PANCREAS',\n", - " 'QGP1_PANCREAS',\n", - " 'RAJI_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'RCC10RGB_KIDNEY',\n", - " 'RCHACV_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'RCM1_LARGE_INTESTINE',\n", - " 'RDES_BONE',\n", - " 'RD_SOFT_TISSUE',\n", - " 'REC1_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'REH_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'RERFGC1B_STOMACH',\n", - " 'RERFLCAD1_LUNG',\n", - " 'RERFLCAD2_LUNG',\n", - " 'RERFLCAI_LUNG',\n", - " 'RERFLCKJ_LUNG',\n", - " 'RERFLCMS_LUNG',\n", - " 'RERFLCSQ1_LUNG',\n", - " 'RH18_SOFT_TISSUE',\n", - " 'RH30_SOFT_TISSUE',\n", - " 'RH41_SOFT_TISSUE',\n", - " 'RI1_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'RKN_SOFT_TISSUE',\n", - " 'RKO_LARGE_INTESTINE',\n", - " 'RL952_ENDOMETRIUM',\n", - " 'RL_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'RMGI_OVARY',\n", - " 'RMUGS_OVARY',\n", - " 'RPMI7951_SKIN',\n", - " 'RPMI8226_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'RPMI8402_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'RS411_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'RS5_FIBROBLAST',\n", - " 'RT112_URINARY_TRACT',\n", - " 'RT4_URINARY_TRACT',\n", - " 'RVH421_SKIN',\n", - " 'S117_THYROID',\n", - " 'SALE_LUNG',\n", - " 'SAOS2_BONE',\n", - " 'SBC5_LUNG',\n", - " 'SCABER_URINARY_TRACT',\n", - " 'SCC15_UPPER_AERODIGESTIVE_TRACT',\n", - " 'SCC25_UPPER_AERODIGESTIVE_TRACT',\n", - " 'SCC4_UPPER_AERODIGESTIVE_TRACT',\n", - " 'SCC9_UPPER_AERODIGESTIVE_TRACT',\n", - " 'SCLC21H_LUNG',\n", - " 'SEM_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'SET2_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'SF126_CENTRAL_NERVOUS_SYSTEM',\n", - " 'SF172_CENTRAL_NERVOUS_SYSTEM',\n", - " 'SF268_CENTRAL_NERVOUS_SYSTEM',\n", - " 'SF295_CENTRAL_NERVOUS_SYSTEM',\n", - " 'SF539_CENTRAL_NERVOUS_SYSTEM',\n", - " 'SF767_CERVIX',\n", - " 'SH10TC_STOMACH',\n", - " 'SH4_SKIN',\n", - " 'SHP77_LUNG',\n", - " 'SIGM5_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'SIHA_CERVIX',\n", - " 'SIMA_AUTONOMIC_GANGLIA',\n", - " 'SJSA1_BONE',\n", - " 'SKBR3_BREAST',\n", - " 'SKCO1_LARGE_INTESTINE',\n", - " 'SKES1_BONE',\n", - " 'SKHEP1_LIVER',\n", - " 'SKLMS1_SOFT_TISSUE',\n", - " 'SKLU1_LUNG',\n", - " 'SKM1_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'SKMEL1_SKIN',\n", - " 'SKMEL24_SKIN',\n", - " 'SKMEL28_SKIN',\n", - " 'SKMEL30_SKIN',\n", - " 'SKMEL31_SKIN',\n", - " 'SKMEL3_SKIN',\n", - " 'SKMEL5_SKIN',\n", - " 'SKMES1_LUNG',\n", - " 'SKMM2_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'SKNAS_AUTONOMIC_GANGLIA',\n", - " 'SKNBE2_AUTONOMIC_GANGLIA',\n", - " 'SKNDZ_AUTONOMIC_GANGLIA',\n", - " 'SKNEP1_BONE',\n", - " 'SKNFI_AUTONOMIC_GANGLIA',\n", - " 'SKNMC_BONE',\n", - " 'SKNO1_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'SKNSH_AUTONOMIC_GANGLIA',\n", - " 'SKOV3_OVARY',\n", - " 'SKPNDW_BONE',\n", - " 'SKRC20_KIDNEY',\n", - " 'SKUT1_SOFT_TISSUE',\n", - " 'SLR20_KIDNEY',\n", - " 'SLR21_KIDNEY',\n", - " 'SLR23_KIDNEY',\n", - " 'SLR24_KIDNEY',\n", - " 'SLR25_KIDNEY',\n", - " 'SLR26_KIDNEY',\n", - " 'SNB75_CENTRAL_NERVOUS_SYSTEM',\n", - " 'SNGM_ENDOMETRIUM',\n", - " 'SNU1033_LARGE_INTESTINE',\n", - " 'SNU1040_LARGE_INTESTINE',\n", - " 'SNU1041_UPPER_AERODIGESTIVE_TRACT',\n", - " 'SNU1066_UPPER_AERODIGESTIVE_TRACT',\n", - " 'SNU1076_UPPER_AERODIGESTIVE_TRACT',\n", - " 'SNU1077_ENDOMETRIUM',\n", - " 'SNU1079_BILIARY_TRACT',\n", - " 'SNU1105_CENTRAL_NERVOUS_SYSTEM',\n", - " 'SNU1196_BILIARY_TRACT',\n", - " 'SNU1197_LARGE_INTESTINE',\n", - " 'SNU119_OVARY',\n", - " 'SNU1214_UPPER_AERODIGESTIVE_TRACT',\n", - " 'SNU1272_KIDNEY',\n", - " 'SNU16_STOMACH',\n", - " 'SNU175_LARGE_INTESTINE',\n", - " 'SNU182_LIVER',\n", - " 'SNU1_STOMACH',\n", - " 'SNU201_CENTRAL_NERVOUS_SYSTEM',\n", - " 'SNU213_PANCREAS',\n", - " 'SNU216_STOMACH',\n", - " 'SNU245_BILIARY_TRACT',\n", - " 'SNU283_LARGE_INTESTINE',\n", - " 'SNU308_BILIARY_TRACT',\n", - " 'SNU324_PANCREAS',\n", - " 'SNU349_KIDNEY',\n", - " 'SNU387_LIVER',\n", - " 'SNU398_LIVER',\n", - " 'SNU407_LARGE_INTESTINE',\n", - " 'SNU410_PANCREAS',\n", - " 'SNU423_LIVER',\n", - " 'SNU449_LIVER',\n", - " 'SNU466_CENTRAL_NERVOUS_SYSTEM',\n", - " 'SNU46_UPPER_AERODIGESTIVE_TRACT',\n", - " 'SNU475_LIVER',\n", - " 'SNU478_BILIARY_TRACT',\n", - " 'SNU489_CENTRAL_NERVOUS_SYSTEM',\n", - " 'SNU503_LARGE_INTESTINE',\n", - " 'SNU520_STOMACH',\n", - " 'SNU5_STOMACH',\n", - " 'SNU601_STOMACH',\n", - " 'SNU61_LARGE_INTESTINE',\n", - " 'SNU620_STOMACH',\n", - " 'SNU626_CENTRAL_NERVOUS_SYSTEM',\n", - " 'SNU668_STOMACH',\n", - " 'SNU685_ENDOMETRIUM',\n", - " 'SNU719_STOMACH',\n", - " 'SNU738_CENTRAL_NERVOUS_SYSTEM',\n", - " 'SNU761_LIVER',\n", - " 'SNU81_LARGE_INTESTINE',\n", - " 'SNU840_OVARY',\n", - " 'SNU869_BILIARY_TRACT',\n", - " 'SNU878_LIVER',\n", - " 'SNU886_LIVER',\n", - " 'SNU899_UPPER_AERODIGESTIVE_TRACT',\n", - " 'SNU8_OVARY',\n", - " 'SNUC1_LARGE_INTESTINE',\n", - " 'SNUC2A_LARGE_INTESTINE',\n", - " 'SNUC4_LARGE_INTESTINE',\n", - " 'SNUC5_LARGE_INTESTINE',\n", - " 'SQ1_LUNG',\n", - " 'SR786_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'ST486_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'STM9101_SOFT_TISSUE',\n", - " 'SU8686_PANCREAS',\n", - " 'SUDHL10_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'SUDHL1_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'SUDHL4_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'SUDHL5_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'SUDHL6_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'SUDHL8_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'SUIT2_PANCREAS',\n", - " 'SUPB15_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'SUPM2_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'SUPT11_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'SUPT1_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'SW1088_CENTRAL_NERVOUS_SYSTEM',\n", - " 'SW1116_LARGE_INTESTINE',\n", - " 'SW1271_LUNG',\n", - " 'SW1353_BONE',\n", - " 'SW1417_LARGE_INTESTINE',\n", - " 'SW1463_LARGE_INTESTINE',\n", - " 'SW1573_LUNG',\n", - " 'SW1710_URINARY_TRACT',\n", - " 'SW1783_CENTRAL_NERVOUS_SYSTEM',\n", - " 'SW1990_PANCREAS',\n", - " 'SW403_LARGE_INTESTINE',\n", - " 'SW480_LARGE_INTESTINE',\n", - " 'SW48_LARGE_INTESTINE',\n", - " 'SW579_THYROID',\n", - " 'SW620_LARGE_INTESTINE',\n", - " 'SW780_URINARY_TRACT',\n", - " 'SW837_LARGE_INTESTINE',\n", - " 'SW900_LUNG',\n", - " 'SW948_LARGE_INTESTINE',\n", - " 'T173_FIBROBLAST',\n", - " 'T24_URINARY_TRACT',\n", - " 'T3M10_LUNG',\n", - " 'T3M4_PANCREAS',\n", - " 'T47D_BREAST',\n", - " 'T84_LARGE_INTESTINE',\n", - " 'T98G_CENTRAL_NERVOUS_SYSTEM',\n", - " 'TALL1_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'TC71_BONE',\n", - " 'TCCPAN2_PANCREAS',\n", - " 'TCCSUP_URINARY_TRACT',\n", - " 'TE10_OESOPHAGUS',\n", - " 'TE11_OESOPHAGUS',\n", - " 'TE125T_FIBROBLAST',\n", - " 'TE14_OESOPHAGUS',\n", - " 'TE159T_FIBROBLAST',\n", - " 'TE15_OESOPHAGUS',\n", - " 'TE1_OESOPHAGUS',\n", - " 'TE441T_SOFT_TISSUE',\n", - " 'TE4_OESOPHAGUS',\n", - " 'TE5_OESOPHAGUS',\n", - " 'TE617T_SOFT_TISSUE',\n", - " 'TE6_OESOPHAGUS',\n", - " 'TE8_OESOPHAGUS',\n", - " 'TE9_OESOPHAGUS',\n", - " 'TEN_ENDOMETRIUM',\n", - " 'TF1_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'TGBC11TKB_STOMACH',\n", - " 'THP1_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'TIG3TD_FIBROBLAST',\n", - " 'TM31_CENTRAL_NERVOUS_SYSTEM',\n", - " 'TM87_SOFT_TISSUE',\n", - " 'TO175T_FIBROBLAST',\n", - " 'TOLEDO_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'TOV112D_OVARY',\n", - " 'TOV21G_OVARY',\n", - " 'TT2609C02_THYROID',\n", - " 'TTC1240_SOFT_TISSUE',\n", - " 'TTC549_SOFT_TISSUE',\n", - " 'TTC642_SOFT_TISSUE',\n", - " 'TTC709_SOFT_TISSUE',\n", - " 'TT_OESOPHAGUS',\n", - " 'TT_THYROID',\n", - " 'TUHR10TKB_KIDNEY',\n", - " 'TUHR14TKB_KIDNEY',\n", - " 'TUHR4TKB_KIDNEY',\n", - " 'TYKNU_OVARY',\n", - " 'U118MG_CENTRAL_NERVOUS_SYSTEM',\n", - " 'U178_CENTRAL_NERVOUS_SYSTEM',\n", - " 'U251MG_CENTRAL_NERVOUS_SYSTEM',\n", - " 'U266B1_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'U2OS_BONE',\n", - " 'U343_CENTRAL_NERVOUS_SYSTEM',\n", - " 'U87MG_CENTRAL_NERVOUS_SYSTEM',\n", - " 'U937_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE',\n", - " 'UACC257_SKIN',\n", - " 'UACC62_SKIN',\n", - " 'UACC812_BREAST',\n", - " 'UACC893_BREAST',\n", - " 'UBLC1_URINARY_TRACT',\n", - " 'UMUC1_URINARY_TRACT',\n", - " 'UMUC3_URINARY_TRACT',\n", - " 'UO31_KIDNEY',\n", - " 'UOK101_KIDNEY',\n", - " 'VCAP_PROSTATE',\n", - " ...],\n", - " 1019)" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "cell_list_deepdds=[]\n", - "with open('data/independent_set/independent_cell_features_954.csv') as f:\n", - " for l in f.readlines():\n", - " l=l.strip().split(',')\n", - " cell_list_deepdds.append(l[0])\n", - "cell_list_deepdds=cell_list_deepdds[2:]\n", - "cell_list_deepdds,len(cell_list_deepdds)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "cell2embed={cell_list_deepdds[i].split('_')[0]:deepdds_scfm_output_processed[i] for i in range(len(cell_list_deepdds))}\n", - "np.save('/data1/ruiyi/deepdds/deepddsindcell2scfm1227embed.npy', cell2embed)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "deepce", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.18" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Legacy DeepDDS reference workflow\n", + "This notebook uses legacy scLong/DeepDDS artifacts, not BioMod model weights. Set `DEEPDDS_ARTIFACT_DIR` before starting the kernel if your generated embeddings or ROC results are stored elsewhere. These external results are not included in this repository.\n" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from pathlib import Path\n", + "import sys\n", + "\n", + "# Start Jupyter in this drug_combination directory.\n", + "REFERENCE_DIR = Path(\".\").resolve()\n", + "if not (REFERENCE_DIR / \"_reference_paths.py\").is_file():\n", + " raise FileNotFoundError(\"Start this notebook from the legacy drug_combination directory.\")\n", + "sys.path.insert(0, str(REFERENCE_DIR))\n", + "from _reference_paths import ASSET_DIR, artifact_path, data_path, output_dir, output_path, required_artifact\n" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import pandas as pd\n", + "from sklearn.preprocessing import normalize" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Test intersect genes\n", + "\n", + "gene_list_scfm=[]\n", + "with open(str(ASSET_DIR / \"selected_genes_27k.txt\")) as f:\n", + " for l in f.readlines():\n", + " gene_list_scfm.append(str(l.strip()))\n", + "len(gene_list_scfm)\n", + "\n", + "gene_list_deepdds=[]\n", + "with open(data_path('independent_set/independent_cell_features_954.csv')) as f:\n", + " for l in f.readlines():\n", + " l=l.strip().split(',')\n", + " #print(l)\n", + " break\n", + "gene_list_deepdds=l[1:]\n", + "len(gene_list_deepdds)\n", + "\n", + "cnt=0\n", + "for gene in gene_list_deepdds:\n", + " if gene in gene_list_scfm:cnt+=1\n", + "cnt\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "deepdds_scfm_output=np.load(artifact_path('deepdds_ind_scfm1227_output.npy'))\n", + "deepdds_scfm_output.shape" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "deepdds_scfm_output_processed=np.zeros((deepdds_scfm_output.shape[0],len(gene_list_deepdds),deepdds_scfm_output.shape[-1]))\n", + "gene_deepdds2idx={v:k for k,v in enumerate(gene_list_deepdds)}\n", + "for k,v in enumerate(gene_list_scfm):\n", + " if v in gene_deepdds2idx:\n", + " idx=gene_deepdds2idx[v]\n", + " deepdds_scfm_output_processed[:,idx,:]=deepdds_scfm_output[:,k,:]\n", + "deepdds_scfm_output_processed.shape " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "np.save(artifact_path('deepdds_ind_scfm1227_output_processd.npy'), deepdds_scfm_output_processed)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "cell_list_deepdds=[]\n", + "with open(data_path('independent_set/independent_cell_features_954.csv')) as f:\n", + " for l in f.readlines():\n", + " l=l.strip().split(',')\n", + " cell_list_deepdds.append(l[0])\n", + "cell_list_deepdds=cell_list_deepdds[2:]\n", + "cell_list_deepdds,len(cell_list_deepdds)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "cell2embed={cell_list_deepdds[i].split('_')[0]:deepdds_scfm_output_processed[i] for i in range(len(cell_list_deepdds))}\n", + "np.save(artifact_path('deepddsindcell2scfm1227embed.npy'), cell2embed)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "deepce", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.18" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/data/downstream/final_data/drug_response_pytorch/drug_combination/process.py b/data/downstream/final_data/drug_response_pytorch/drug_combination/process.py index beaf9027cd54ee1c5c4402d2c8d71827e84e49db..f0515fdac531b3b57442e7f885d992a76efd1f6e 100644 --- a/data/downstream/final_data/drug_response_pytorch/drug_combination/process.py +++ b/data/downstream/final_data/drug_response_pytorch/drug_combination/process.py @@ -1,73 +1,74 @@ -import numpy as np -import pandas as pd -from sklearn.preprocessing import normalize - -# Test intersect genes -ckpt='v20228' -ind=True -gene_list_scfm=[] -with open('/data3/ruiyi/scfm/selected_genes_27k.txt') as f: - for l in f.readlines(): - gene_list_scfm.append(str(l.strip())) -print(len(gene_list_scfm)) - -gene_list_deepdds=[] -if ind: - with open('/data1/ruiyi/deepdds/data/independent_set/independent_cell_features_954.csv') as f: - for l in f.readlines(): - l=l.strip().split(',') - #print(l) - break -else: - with open('/data1/ruiyi/deepdds/data/new_cell_features_954.csv') as f: - for l in f.readlines(): - l=l.strip().split(',') - #print(l) - break -gene_list_deepdds=l[1:] -print(len(gene_list_deepdds)) - -cnt=0 -for gene in gene_list_deepdds: - if gene in gene_list_scfm:cnt+=1 -print(cnt) - -if ind: - deepdds_scfm_output=np.load('/data1/ruiyi/deepdds/deepdds_ind_scfm{}_output.npy'.format(ckpt)) -else: - deepdds_scfm_output=np.load('/data1/ruiyi/deepdds/deepdds_scfm{}_output.npy'.format(ckpt)) -print(deepdds_scfm_output.shape) - -deepdds_scfm_output_processed=np.zeros((deepdds_scfm_output.shape[0],len(gene_list_deepdds),deepdds_scfm_output.shape[-1])) -gene_deepdds2idx={v:k for k,v in enumerate(gene_list_deepdds)} -for k,v in enumerate(gene_list_scfm): - if v in gene_deepdds2idx: - idx=gene_deepdds2idx[v] - deepdds_scfm_output_processed[:,idx,:]=deepdds_scfm_output[:,k,:] -print(deepdds_scfm_output_processed.shape) - -if ind: - np.save('/data1/ruiyi/deepdds/deepdds_ind_scfm{}_output_processd.npy'.format(ckpt), deepdds_scfm_output_processed) -else: - np.save('/data1/ruiyi/deepdds/deepdds_scfm{}_output_processd.npy'.format(ckpt), deepdds_scfm_output_processed) - -cell_list_deepdds=[] -if ind: - with open('/data1/ruiyi/deepdds/data/independent_set/independent_cell_features_954.csv') as f: - for l in f.readlines(): - l=l.strip().split(',') - cell_list_deepdds.append(l[0]) -else: - with open('/data1/ruiyi/deepdds/data/new_cell_features_954.csv') as f: - for l in f.readlines(): - l=l.strip().split(',') - cell_list_deepdds.append(l[0]) -cell_list_deepdds=cell_list_deepdds[2:] -#cell_list_deepdds,len(cell_list_deepdds) -print(len(cell_list_deepdds)) - -cell2embed={cell_list_deepdds[i].split('_')[0]:deepdds_scfm_output_processed[i] for i in range(len(cell_list_deepdds))} -if ind: - np.save('/data1/ruiyi/deepdds/deepddsindcell2scfm{}embed.npy'.format(ckpt), cell2embed) -else: - np.save('/data1/ruiyi/deepdds/deepddscell2scfm{}embed.npy'.format(ckpt), cell2embed) +from _reference_paths import ASSET_DIR, artifact_path, data_path, output_dir, output_path, required_artifact +import numpy as np +import pandas as pd +from sklearn.preprocessing import normalize + +# Test intersect genes +ckpt='v20228' +ind=True +gene_list_scfm=[] +with open(str(ASSET_DIR / "selected_genes_27k.txt")) as f: + for l in f.readlines(): + gene_list_scfm.append(str(l.strip())) +print(len(gene_list_scfm)) + +gene_list_deepdds=[] +if ind: + with open(data_path('independent_set/independent_cell_features_954.csv')) as f: + for l in f.readlines(): + l=l.strip().split(',') + #print(l) + break +else: + with open(data_path('new_cell_features_954.csv')) as f: + for l in f.readlines(): + l=l.strip().split(',') + #print(l) + break +gene_list_deepdds=l[1:] +print(len(gene_list_deepdds)) + +cnt=0 +for gene in gene_list_deepdds: + if gene in gene_list_scfm:cnt+=1 +print(cnt) + +if ind: + deepdds_scfm_output=np.load(artifact_path('deepdds_ind_scfm{}_output.npy').format(ckpt)) +else: + deepdds_scfm_output=np.load(artifact_path('deepdds_scfm{}_output.npy').format(ckpt)) +print(deepdds_scfm_output.shape) + +deepdds_scfm_output_processed=np.zeros((deepdds_scfm_output.shape[0],len(gene_list_deepdds),deepdds_scfm_output.shape[-1])) +gene_deepdds2idx={v:k for k,v in enumerate(gene_list_deepdds)} +for k,v in enumerate(gene_list_scfm): + if v in gene_deepdds2idx: + idx=gene_deepdds2idx[v] + deepdds_scfm_output_processed[:,idx,:]=deepdds_scfm_output[:,k,:] +print(deepdds_scfm_output_processed.shape) + +if ind: + np.save(artifact_path('deepdds_ind_scfm{}_output_processd.npy').format(ckpt), deepdds_scfm_output_processed) +else: + np.save(artifact_path('deepdds_scfm{}_output_processd.npy').format(ckpt), deepdds_scfm_output_processed) + +cell_list_deepdds=[] +if ind: + with open(data_path('independent_set/independent_cell_features_954.csv')) as f: + for l in f.readlines(): + l=l.strip().split(',') + cell_list_deepdds.append(l[0]) +else: + with open(data_path('new_cell_features_954.csv')) as f: + for l in f.readlines(): + l=l.strip().split(',') + cell_list_deepdds.append(l[0]) +cell_list_deepdds=cell_list_deepdds[2:] +#cell_list_deepdds,len(cell_list_deepdds) +print(len(cell_list_deepdds)) + +cell2embed={cell_list_deepdds[i].split('_')[0]:deepdds_scfm_output_processed[i] for i in range(len(cell_list_deepdds))} +if ind: + np.save(artifact_path('deepddsindcell2scfm{}embed.npy').format(ckpt), cell2embed) +else: + np.save(artifact_path('deepddscell2scfm{}embed.npy').format(ckpt), cell2embed) diff --git a/data/downstream/final_data/drug_response_pytorch/drug_combination/roc_auc.py b/data/downstream/final_data/drug_response_pytorch/drug_combination/roc_auc.py index 112281657b8c2ac8cc9c7d6ed2b5c78d5d86fc8e..0b219f642c4f1c12336f28a835b3a37031524608 100644 --- a/data/downstream/final_data/drug_response_pytorch/drug_combination/roc_auc.py +++ b/data/downstream/final_data/drug_response_pytorch/drug_combination/roc_auc.py @@ -1,70 +1,71 @@ -import numpy as np -from sklearn.metrics import roc_curve -from sklearn.metrics import auc -import pylab as plt -import pandas as pd -import matplotlib.pyplot as plt -import matplotlib -#; matplotlib.use('TkAgg') -#---自己按照公式实现 -def auc_calculate(labels,preds,n_bins=100): - postive_len = sum(labels) - negative_len = len(labels) - postive_len - total_case = postive_len * negative_len - pos_histogram = [0 for _ in range(n_bins)] - neg_histogram = [0 for _ in range(n_bins)] - bin_width = 1.0 / n_bins - for i in range(len(labels)): - nth_bin = int(preds[i]/bin_width) - if labels[i]==1: - pos_histogram[nth_bin] += 1 - else: - neg_histogram[nth_bin] += 1 - accumulated_neg = 0 - satisfied_pair = 0 - for i in range(n_bins): - satisfied_pair += (pos_histogram[i]*accumulated_neg + pos_histogram[i]*neg_histogram[i]*0.5) - accumulated_neg += neg_histogram[i] - - return satisfied_pair / float(total_case) - -if __name__ == '__main__': - - datafile = ['data/result/independent_set_predict/Independent_GATNet--result_new_labels_0_10.csv', 'data/result/independent_set_predict/Independent_GCNNet--result_new_labels_0_10.csv', - 'data/result/independent_set_predict/Independent_DeepSynergy--result_new_labels_0_10.csv', - 'data/result/independent_set_predict/RF--ROC--new_labels_0_10.csv', - 'data/result/independent_set_predict/SVM--ROC--new_labels_0_10.csv', - 'data/result/MLP/MLP_validation--ROC--new_labels_0_10.csv', - 'data/result/GBM/GBM_validation--ROC--new_labels_0_10.csv', - 'data/result/XGB/XGB_validation--ROC--new_labels_0_10.csv', - 'data/result/AdaBoost/AdaBoost_validation--ROC--new_labels_0_10.csv'] - methods = ['DeepDDS-GAT', 'DeepDDS-GCN', - 'DeepSynergy', - 'RF', - 'SVM', - 'MLP', - 'GBM', - 'XGB', - 'AdaBoost'] - for it in range(len(datafile)): - tem = pd.read_csv(datafile[it], header=None) - tem = np.array(tem) - print(tem[0].shape) - print(tem[2].shape) - print(type(tem[0][0])) - print(type(tem[2][0])) - - fpr, tpr, thresholds = roc_curve(tem[0], tem[2], pos_label=1) - print("-----sklearn:", auc(fpr, tpr)) - plt.plot(fpr, tpr, label=methods[it] + '(AUC = %0.3f)' % auc(fpr, tpr)) - - plt.legend(loc='lower right') - plt.plot([0, 1], [0, 1], 'r--') - plt.xlim([0, 1]) - plt.ylim([0, 1]) - plt.ylabel('True Positive Rate') - plt.xlabel('False Positive Rate') - plt.title('Predictive performance on novel drug combinations') - plt.rcParams['savefig.dpi'] = 1000 #图片像素 - plt.savefig('independence.png') - plt.show() \ No newline at end of file +from _reference_paths import ASSET_DIR, artifact_path, data_path, output_dir, output_path, required_artifact +import numpy as np +from sklearn.metrics import roc_curve +from sklearn.metrics import auc +import pylab as plt +import pandas as pd +import matplotlib.pyplot as plt +import matplotlib +#; matplotlib.use('TkAgg') +#---Implement the metric from its formula. +def auc_calculate(labels,preds,n_bins=100): + postive_len = sum(labels) + negative_len = len(labels) - postive_len + total_case = postive_len * negative_len + pos_histogram = [0 for _ in range(n_bins)] + neg_histogram = [0 for _ in range(n_bins)] + bin_width = 1.0 / n_bins + for i in range(len(labels)): + nth_bin = int(preds[i]/bin_width) + if labels[i]==1: + pos_histogram[nth_bin] += 1 + else: + neg_histogram[nth_bin] += 1 + accumulated_neg = 0 + satisfied_pair = 0 + for i in range(n_bins): + satisfied_pair += (pos_histogram[i]*accumulated_neg + pos_histogram[i]*neg_histogram[i]*0.5) + accumulated_neg += neg_histogram[i] + + return satisfied_pair / float(total_case) + +if __name__ == '__main__': + + datafile = [output_path('result/independent_set_predict/Independent_GATNet--result_new_labels_0_10.csv'), output_path('result/independent_set_predict/Independent_GCNNet--result_new_labels_0_10.csv'), + output_path('result/independent_set_predict/Independent_DeepSynergy--result_new_labels_0_10.csv'), + output_path('result/independent_set_predict/RF--ROC--new_labels_0_10.csv'), + output_path('result/independent_set_predict/SVM--ROC--new_labels_0_10.csv'), + output_path('result/MLP/MLP_validation--ROC--new_labels_0_10.csv'), + output_path('result/GBM/GBM_validation--ROC--new_labels_0_10.csv'), + output_path('result/XGB/XGB_validation--ROC--new_labels_0_10.csv'), + output_path('result/AdaBoost/AdaBoost_validation--ROC--new_labels_0_10.csv')] + methods = ['DeepDDS-GAT', 'DeepDDS-GCN', + 'DeepSynergy', + 'RF', + 'SVM', + 'MLP', + 'GBM', + 'XGB', + 'AdaBoost'] + for it in range(len(datafile)): + tem = pd.read_csv(datafile[it], header=None) + tem = np.array(tem) + print(tem[0].shape) + print(tem[2].shape) + print(type(tem[0][0])) + print(type(tem[2][0])) + + fpr, tpr, thresholds = roc_curve(tem[0], tem[2], pos_label=1) + print("-----sklearn:", auc(fpr, tpr)) + plt.plot(fpr, tpr, label=methods[it] + '(AUC = %0.3f)' % auc(fpr, tpr)) + + plt.legend(loc='lower right') + plt.plot([0, 1], [0, 1], 'r--') + plt.xlim([0, 1]) + plt.ylim([0, 1]) + plt.ylabel('True Positive Rate') + plt.xlabel('False Positive Rate') + plt.title('Predictive performance on novel drug combinations') + plt.rcParams['savefig.dpi'] = 1000 #Output resolution in DPI. + plt.savefig('independence.png') + plt.show() diff --git a/data/downstream/final_data/drug_response_pytorch/drug_combination/scfm_roc.py b/data/downstream/final_data/drug_response_pytorch/drug_combination/scfm_roc.py index 112281657b8c2ac8cc9c7d6ed2b5c78d5d86fc8e..0b219f642c4f1c12336f28a835b3a37031524608 100644 --- a/data/downstream/final_data/drug_response_pytorch/drug_combination/scfm_roc.py +++ b/data/downstream/final_data/drug_response_pytorch/drug_combination/scfm_roc.py @@ -1,70 +1,71 @@ -import numpy as np -from sklearn.metrics import roc_curve -from sklearn.metrics import auc -import pylab as plt -import pandas as pd -import matplotlib.pyplot as plt -import matplotlib -#; matplotlib.use('TkAgg') -#---自己按照公式实现 -def auc_calculate(labels,preds,n_bins=100): - postive_len = sum(labels) - negative_len = len(labels) - postive_len - total_case = postive_len * negative_len - pos_histogram = [0 for _ in range(n_bins)] - neg_histogram = [0 for _ in range(n_bins)] - bin_width = 1.0 / n_bins - for i in range(len(labels)): - nth_bin = int(preds[i]/bin_width) - if labels[i]==1: - pos_histogram[nth_bin] += 1 - else: - neg_histogram[nth_bin] += 1 - accumulated_neg = 0 - satisfied_pair = 0 - for i in range(n_bins): - satisfied_pair += (pos_histogram[i]*accumulated_neg + pos_histogram[i]*neg_histogram[i]*0.5) - accumulated_neg += neg_histogram[i] - - return satisfied_pair / float(total_case) - -if __name__ == '__main__': - - datafile = ['data/result/independent_set_predict/Independent_GATNet--result_new_labels_0_10.csv', 'data/result/independent_set_predict/Independent_GCNNet--result_new_labels_0_10.csv', - 'data/result/independent_set_predict/Independent_DeepSynergy--result_new_labels_0_10.csv', - 'data/result/independent_set_predict/RF--ROC--new_labels_0_10.csv', - 'data/result/independent_set_predict/SVM--ROC--new_labels_0_10.csv', - 'data/result/MLP/MLP_validation--ROC--new_labels_0_10.csv', - 'data/result/GBM/GBM_validation--ROC--new_labels_0_10.csv', - 'data/result/XGB/XGB_validation--ROC--new_labels_0_10.csv', - 'data/result/AdaBoost/AdaBoost_validation--ROC--new_labels_0_10.csv'] - methods = ['DeepDDS-GAT', 'DeepDDS-GCN', - 'DeepSynergy', - 'RF', - 'SVM', - 'MLP', - 'GBM', - 'XGB', - 'AdaBoost'] - for it in range(len(datafile)): - tem = pd.read_csv(datafile[it], header=None) - tem = np.array(tem) - print(tem[0].shape) - print(tem[2].shape) - print(type(tem[0][0])) - print(type(tem[2][0])) - - fpr, tpr, thresholds = roc_curve(tem[0], tem[2], pos_label=1) - print("-----sklearn:", auc(fpr, tpr)) - plt.plot(fpr, tpr, label=methods[it] + '(AUC = %0.3f)' % auc(fpr, tpr)) - - plt.legend(loc='lower right') - plt.plot([0, 1], [0, 1], 'r--') - plt.xlim([0, 1]) - plt.ylim([0, 1]) - plt.ylabel('True Positive Rate') - plt.xlabel('False Positive Rate') - plt.title('Predictive performance on novel drug combinations') - plt.rcParams['savefig.dpi'] = 1000 #图片像素 - plt.savefig('independence.png') - plt.show() \ No newline at end of file +from _reference_paths import ASSET_DIR, artifact_path, data_path, output_dir, output_path, required_artifact +import numpy as np +from sklearn.metrics import roc_curve +from sklearn.metrics import auc +import pylab as plt +import pandas as pd +import matplotlib.pyplot as plt +import matplotlib +#; matplotlib.use('TkAgg') +#---Implement the metric from its formula. +def auc_calculate(labels,preds,n_bins=100): + postive_len = sum(labels) + negative_len = len(labels) - postive_len + total_case = postive_len * negative_len + pos_histogram = [0 for _ in range(n_bins)] + neg_histogram = [0 for _ in range(n_bins)] + bin_width = 1.0 / n_bins + for i in range(len(labels)): + nth_bin = int(preds[i]/bin_width) + if labels[i]==1: + pos_histogram[nth_bin] += 1 + else: + neg_histogram[nth_bin] += 1 + accumulated_neg = 0 + satisfied_pair = 0 + for i in range(n_bins): + satisfied_pair += (pos_histogram[i]*accumulated_neg + pos_histogram[i]*neg_histogram[i]*0.5) + accumulated_neg += neg_histogram[i] + + return satisfied_pair / float(total_case) + +if __name__ == '__main__': + + datafile = [output_path('result/independent_set_predict/Independent_GATNet--result_new_labels_0_10.csv'), output_path('result/independent_set_predict/Independent_GCNNet--result_new_labels_0_10.csv'), + output_path('result/independent_set_predict/Independent_DeepSynergy--result_new_labels_0_10.csv'), + output_path('result/independent_set_predict/RF--ROC--new_labels_0_10.csv'), + output_path('result/independent_set_predict/SVM--ROC--new_labels_0_10.csv'), + output_path('result/MLP/MLP_validation--ROC--new_labels_0_10.csv'), + output_path('result/GBM/GBM_validation--ROC--new_labels_0_10.csv'), + output_path('result/XGB/XGB_validation--ROC--new_labels_0_10.csv'), + output_path('result/AdaBoost/AdaBoost_validation--ROC--new_labels_0_10.csv')] + methods = ['DeepDDS-GAT', 'DeepDDS-GCN', + 'DeepSynergy', + 'RF', + 'SVM', + 'MLP', + 'GBM', + 'XGB', + 'AdaBoost'] + for it in range(len(datafile)): + tem = pd.read_csv(datafile[it], header=None) + tem = np.array(tem) + print(tem[0].shape) + print(tem[2].shape) + print(type(tem[0][0])) + print(type(tem[2][0])) + + fpr, tpr, thresholds = roc_curve(tem[0], tem[2], pos_label=1) + print("-----sklearn:", auc(fpr, tpr)) + plt.plot(fpr, tpr, label=methods[it] + '(AUC = %0.3f)' % auc(fpr, tpr)) + + plt.legend(loc='lower right') + plt.plot([0, 1], [0, 1], 'r--') + plt.xlim([0, 1]) + plt.ylim([0, 1]) + plt.ylabel('True Positive Rate') + plt.xlabel('False Positive Rate') + plt.title('Predictive performance on novel drug combinations') + plt.rcParams['savefig.dpi'] = 1000 #Output resolution in DPI. + plt.savefig('independence.png') + plt.show() diff --git a/data/downstream/final_data/drug_response_pytorch/drug_combination/training.py b/data/downstream/final_data/drug_response_pytorch/drug_combination/training.py index 20e6ed353b45d96502eeb1735e76385256e9cfd8..9c811ab2f9f60ade97ca70f298e1edc8727ba4ef 100644 --- a/data/downstream/final_data/drug_response_pytorch/drug_combination/training.py +++ b/data/downstream/final_data/drug_response_pytorch/drug_combination/training.py @@ -1,176 +1,177 @@ -import random -import numpy as np -import pandas as pd -import sys, os -from random import shuffle -import torch.utils.data as Data -import torch -import torch.nn.functional as F -import torch.nn as nn -from torch.utils.data import TensorDataset, Dataset -from models.gat import GATNet -from models.gat_gcn_test import GAT_GCN -from models.gcn import GCNNet -from models.ginconv import GINConvNet -from utils_test import * -from sklearn.metrics import roc_curve, confusion_matrix -from sklearn.metrics import cohen_kappa_score, accuracy_score, roc_auc_score, precision_score, recall_score, balanced_accuracy_score -from sklearn import metrics -from sklearn.model_selection import StratifiedKFold, KFold -import pandas as pd -import matplotlib.pyplot as plt -import seaborn as sns - - -# training function at each epoch -def train(model, device, drug1_loader_train, drug2_loader_train, optimizer, epoch): - print('Training on {} samples...'.format(len(drug1_loader_train.dataset))) - model.train() - # train_loader = np.array(train_loader) - for batch_idx, data in enumerate(zip(drug1_loader_train, drug2_loader_train)): - data1 = data[0] - data2 = data[1] - data1 = data1.to(device) - data2 = data2.to(device) - y = data[0].y.view(-1, 1).long().to(device) - y = y.squeeze(1) - optimizer.zero_grad() - output = model(data1, data2) - loss = loss_fn(output, y) - # print('loss', loss) - loss.backward() - optimizer.step() - if batch_idx % LOG_INTERVAL == 0: - print('Train epoch: {} Loss: {:.6f}'.format(epoch, loss.item())) - - -def predicting(model, device, drug1_loader_test, drug2_loader_test): - model.eval() - total_preds = torch.Tensor() - total_labels = torch.Tensor() - total_prelabels = torch.Tensor() - print('Make prediction for {} samples...'.format(len(drug1_loader_test.dataset))) - with torch.no_grad(): - for data in zip(drug1_loader_test, drug2_loader_test): - data1 = data[0] - data2 = data[1] - data1 = data1.to(device) - data2 = data2.to(device) - output = model(data1, data2) - ys = F.softmax(output, 1).to('cpu').data.numpy() - predicted_labels = list(map(lambda x: np.argmax(x), ys)) - predicted_scores = list(map(lambda x: x[1], ys)) - total_preds = torch.cat((total_preds, torch.Tensor(predicted_scores)), 0) - total_prelabels = torch.cat((total_prelabels, torch.Tensor(predicted_labels)), 0) - total_labels = torch.cat((total_labels, data1.y.view(-1, 1).cpu()), 0) - return total_labels.numpy().flatten(), total_preds.numpy().flatten(), total_prelabels.numpy().flatten() - - -# modeling = [GINConvNet, GATNet, GAT_GCN, GCNNet][int(sys.argv[2])] - -modeling = GATNet - -TRAIN_BATCH_SIZE = 512 -TEST_BATCH_SIZE = 512 -LR = 0.001 -LOG_INTERVAL = 20 -NUM_EPOCHS = 1000 - -print('Learning rate: ', LR) -print('Epochs: ', NUM_EPOCHS) -file = ['AZD2014_6244'] - -# CPU or GPU - -if torch.cuda.is_available(): - device = torch.device('cuda:0') - print('The code uses GPU...') -else: - device = torch.device('cpu') - print('The code uses CPU!!!') - -for datafile in file: - - independent_drug1_data = TestbedDataset(root='data', dataset=datafile + '_drug1') - independent_drug2_data = TestbedDataset(root='data', dataset=datafile + '_drug2') - print('independent_drug1_data[0]', independent_drug1_data[0]) - lenth = len(independent_drug1_data) - - - - drug1_data = TestbedDataset(root='data', dataset='new_labels_0_10_drug1') - drug2_data = TestbedDataset(root='data', dataset='new_labels_0_10_drug2') - lenth = len(drug1_data) - random_num = random.sample(range(0, lenth), lenth) - drug1_data = drug1_data[random_num] - drug2_data = drug2_data[random_num] - - # 构建训练集 - drug1_loader_train = DataLoader(drug1_data, batch_size=TRAIN_BATCH_SIZE, shuffle=None) - drug2_loader_train = DataLoader(drug2_data, batch_size=TRAIN_BATCH_SIZE, shuffle=None) - - # 构建独立测试集 - independent_drug1_loader_test = DataLoader(independent_drug1_data, batch_size=TEST_BATCH_SIZE, shuffle=None) - independent_drug2_loader_test = DataLoader(independent_drug2_data, batch_size=TEST_BATCH_SIZE, shuffle=None) - - model = modeling().to(device) - loss_fn = nn.CrossEntropyLoss() - optimizer = torch.optim.Adam(model.parameters(), lr=LR) - - - # model_file_name = 'data/result/' + datafile + '_GATNet--model.pt' - # independent_num_file_name = 'data/result/' + datafile + '_GATNet--result.csv' - # file_AUCs = 'data/result/' + datafile + '_GATNet--AUCs.txt' - # AUCs = ('Epoch\tAUC_dev\tPR_AUC\tACC\tBACC\tPREC\tTPR\tKAPPA') - # with open(file_AUCs, 'w') as f: - # f.write(AUCs + '\n') - - best_auc = 0 - - for epoch in range(NUM_EPOCHS): - train(model, device, drug1_loader_train, drug2_loader_train, optimizer, epoch + 1) - T, S, Y = predicting(model, device, independent_drug1_loader_test, independent_drug2_loader_test) - - # T is correct label - # S is predict score - # Y is predict label - - # compute preformence - # AUC = roc_auc_score(T, S) - # precision, recall, threshold = metrics.precision_recall_curve(T, S) - # PR_AUC = metrics.auc(recall, precision) - # BACC = balanced_accuracy_score(T, Y) - # tn, fp, fn, tp = confusion_matrix(T, Y).ravel() - # TPR = tp / (tp + fn) - # PREC = precision_score(T, Y) - # ACC = accuracy_score(T, Y) - # KAPPA = cohen_kappa_score(T, Y) - - # save data - - # if best_auc < AUC: - # best_auc = AUC - # torch.save(model.state_dict(), model_file_name) - # independent_num = [] - # independent_num.append(T) - # independent_num.append(Y) - # independent_num.append(S) - # txtDF = pd.DataFrame(data=independent_num) - # txtDF.to_csv(independent_num_file_name, index=False, header=False) - - # sns.set() - # f, ax = plt.subplots() - # CM = confusion_matrix(T, Y) - # print(CM) - # sns.heatmap(CM, annot=True, ax=ax, fmt='.20g') - # ax.set_title('DeepDDS(GATNet) confusion matrix') # 标题 - # ax.set_xlabel('Predict') # x轴 - # ax.set_ylabel('True') # y轴 - # plt.savefig('D:\GraphDTA-master\data\\result\photo\GATNet_cm.png') - - # AUCs = [epoch, AUC, PR_AUC, ACC, BACC, PREC, TPR, KAPPA] - # save_AUCs(AUCs, file_AUCs) - # torch.save(model.state_dict(), model_file_name) - # print("BEST_AUC", best_auc) - - +from _reference_paths import ASSET_DIR, artifact_path, data_path, output_dir, output_path, required_artifact +import random +import numpy as np +import pandas as pd +import sys, os +from random import shuffle +import torch.utils.data as Data +import torch +import torch.nn.functional as F +import torch.nn as nn +from torch.utils.data import TensorDataset, Dataset +from models.gat import GATNet +from models.gat_gcn_test import GAT_GCN +from models.gcn import GCNNet +from models.ginconv import GINConvNet +from utils_test import * +from sklearn.metrics import roc_curve, confusion_matrix +from sklearn.metrics import cohen_kappa_score, accuracy_score, roc_auc_score, precision_score, recall_score, balanced_accuracy_score +from sklearn import metrics +from sklearn.model_selection import StratifiedKFold, KFold +import pandas as pd +import matplotlib.pyplot as plt +import seaborn as sns + + +# training function at each epoch +def train(model, device, drug1_loader_train, drug2_loader_train, optimizer, epoch): + print('Training on {} samples...'.format(len(drug1_loader_train.dataset))) + model.train() + # train_loader = np.array(train_loader) + for batch_idx, data in enumerate(zip(drug1_loader_train, drug2_loader_train)): + data1 = data[0] + data2 = data[1] + data1 = data1.to(device) + data2 = data2.to(device) + y = data[0].y.view(-1, 1).long().to(device) + y = y.squeeze(1) + optimizer.zero_grad() + output = model(data1, data2) + loss = loss_fn(output, y) + # print('loss', loss) + loss.backward() + optimizer.step() + if batch_idx % LOG_INTERVAL == 0: + print('Train epoch: {} Loss: {:.6f}'.format(epoch, loss.item())) + + +def predicting(model, device, drug1_loader_test, drug2_loader_test): + model.eval() + total_preds = torch.Tensor() + total_labels = torch.Tensor() + total_prelabels = torch.Tensor() + print('Make prediction for {} samples...'.format(len(drug1_loader_test.dataset))) + with torch.no_grad(): + for data in zip(drug1_loader_test, drug2_loader_test): + data1 = data[0] + data2 = data[1] + data1 = data1.to(device) + data2 = data2.to(device) + output = model(data1, data2) + ys = F.softmax(output, 1).to('cpu').data.numpy() + predicted_labels = list(map(lambda x: np.argmax(x), ys)) + predicted_scores = list(map(lambda x: x[1], ys)) + total_preds = torch.cat((total_preds, torch.Tensor(predicted_scores)), 0) + total_prelabels = torch.cat((total_prelabels, torch.Tensor(predicted_labels)), 0) + total_labels = torch.cat((total_labels, data1.y.view(-1, 1).cpu()), 0) + return total_labels.numpy().flatten(), total_preds.numpy().flatten(), total_prelabels.numpy().flatten() + + +# modeling = [GINConvNet, GATNet, GAT_GCN, GCNNet][int(sys.argv[2])] + +modeling = GATNet + +TRAIN_BATCH_SIZE = 512 +TEST_BATCH_SIZE = 512 +LR = 0.001 +LOG_INTERVAL = 20 +NUM_EPOCHS = 1000 + +print('Learning rate: ', LR) +print('Epochs: ', NUM_EPOCHS) +file = ['AZD2014_6244'] + +# CPU or GPU + +if torch.cuda.is_available(): + device = torch.device('cuda:0') + print('The code uses GPU...') +else: + device = torch.device('cpu') + print('The code uses CPU!!!') + +for datafile in file: + + independent_drug1_data = TestbedDataset(root=artifact_path("cache"), dataset=datafile + '_drug1') + independent_drug2_data = TestbedDataset(root=artifact_path("cache"), dataset=datafile + '_drug2') + print('independent_drug1_data[0]', independent_drug1_data[0]) + lenth = len(independent_drug1_data) + + + + drug1_data = TestbedDataset(root=artifact_path("cache"), dataset='new_labels_0_10_drug1') + drug2_data = TestbedDataset(root=artifact_path("cache"), dataset='new_labels_0_10_drug2') + lenth = len(drug1_data) + random_num = random.sample(range(0, lenth), lenth) + drug1_data = drug1_data[random_num] + drug2_data = drug2_data[random_num] + + # Construct the training dataset. + drug1_loader_train = DataLoader(drug1_data, batch_size=TRAIN_BATCH_SIZE, shuffle=None) + drug2_loader_train = DataLoader(drug2_data, batch_size=TRAIN_BATCH_SIZE, shuffle=None) + + # Construct the independent test dataset. + independent_drug1_loader_test = DataLoader(independent_drug1_data, batch_size=TEST_BATCH_SIZE, shuffle=None) + independent_drug2_loader_test = DataLoader(independent_drug2_data, batch_size=TEST_BATCH_SIZE, shuffle=None) + + model = modeling().to(device) + loss_fn = nn.CrossEntropyLoss() + optimizer = torch.optim.Adam(model.parameters(), lr=LR) + + + # model_file_name = 'data/result/' + datafile + '_GATNet--model.pt' + # independent_num_file_name = 'data/result/' + datafile + '_GATNet--result.csv' + # file_AUCs = 'data/result/' + datafile + '_GATNet--AUCs.txt' + # AUCs = ('Epoch\tAUC_dev\tPR_AUC\tACC\tBACC\tPREC\tTPR\tKAPPA') + # with open(file_AUCs, 'w') as f: + # f.write(AUCs + '\n') + + best_auc = 0 + + for epoch in range(NUM_EPOCHS): + train(model, device, drug1_loader_train, drug2_loader_train, optimizer, epoch + 1) + T, S, Y = predicting(model, device, independent_drug1_loader_test, independent_drug2_loader_test) + + # T is correct label + # S is predict score + # Y is predict label + + # compute preformence + # AUC = roc_auc_score(T, S) + # precision, recall, threshold = metrics.precision_recall_curve(T, S) + # PR_AUC = metrics.auc(recall, precision) + # BACC = balanced_accuracy_score(T, Y) + # tn, fp, fn, tp = confusion_matrix(T, Y).ravel() + # TPR = tp / (tp + fn) + # PREC = precision_score(T, Y) + # ACC = accuracy_score(T, Y) + # KAPPA = cohen_kappa_score(T, Y) + + # save data + + # if best_auc < AUC: + # best_auc = AUC + # torch.save(model.state_dict(), model_file_name) + # independent_num = [] + # independent_num.append(T) + # independent_num.append(Y) + # independent_num.append(S) + # txtDF = pd.DataFrame(data=independent_num) + # txtDF.to_csv(independent_num_file_name, index=False, header=False) + + # sns.set() + # f, ax = plt.subplots() + # CM = confusion_matrix(T, Y) + # print(CM) + # sns.heatmap(CM, annot=True, ax=ax, fmt='.20g') + # ax.set_title('DeepDDS(GATNet) confusion matrix') # Title. + # ax.set_xlabel('Predict') # X axis. + # ax.set_ylabel('True') # Y axis. + # plt.savefig('outputs/reference_deepdds/result/photo/GATNet_cm.png') + + # AUCs = [epoch, AUC, PR_AUC, ACC, BACC, PREC, TPR, KAPPA] + # save_AUCs(AUCs, file_AUCs) + # torch.save(model.state_dict(), model_file_name) + # print("BEST_AUC", best_auc) + + diff --git a/data/downstream/final_data/drug_response_pytorch/drug_combination/training_GAT.py b/data/downstream/final_data/drug_response_pytorch/drug_combination/training_GAT.py index 610d35293da9a647aa49fb10167fe28131a28a34..ea828348733a9105ab83dd36364203faa8448789 100644 --- a/data/downstream/final_data/drug_response_pytorch/drug_combination/training_GAT.py +++ b/data/downstream/final_data/drug_response_pytorch/drug_combination/training_GAT.py @@ -1,171 +1,172 @@ -import random -import torch.nn.functional as F -import torch.nn as nn -from models.gat import GATNet -from models.gat_gcn_test import GAT_GCN -from models.gcn import GCNNet -from models.ginconv import GINConvNet -from utils_test import * -from sklearn.metrics import roc_curve, confusion_matrix -from sklearn.metrics import cohen_kappa_score, accuracy_score, roc_auc_score, precision_score, recall_score, balanced_accuracy_score -from sklearn import metrics -import pandas as pd - - -# training function at each epoch -def train(model, device, drug1_loader_train, drug2_loader_train, optimizer, epoch): - print('Training on {} samples...'.format(len(drug1_loader_train.dataset))) - model.train() - # train_loader = np.array(train_loader) - for batch_idx, data in enumerate(zip(drug1_loader_train, drug2_loader_train)): - data1 = data[0] - data2 = data[1] - data1 = data1.to(device) - data2 = data2.to(device) - y = data[0].y.view(-1, 1).long().to(device) - y = y.squeeze(1) - optimizer.zero_grad() - output = model(data1, data2) - loss = loss_fn(output, y) - # print('loss', loss) - loss.backward() - optimizer.step() - if batch_idx % LOG_INTERVAL == 0: - print('Train epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(epoch, - batch_idx * len(data1.x), - len(drug1_loader_train.dataset), - 100. * batch_idx / len(drug1_loader_train), - loss.item())) - - -def predicting(model, device, drug1_loader_test, drug2_loader_test): - model.eval() - total_preds = torch.Tensor() - total_labels = torch.Tensor() - total_prelabels = torch.Tensor() - print('Make prediction for {} samples...'.format(len(drug1_loader_test.dataset))) - with torch.no_grad(): - for data in zip(drug1_loader_test, drug2_loader_test): - data1 = data[0] - data2 = data[1] - data1 = data1.to(device) - data2 = data2.to(device) - output = model(data1, data2) - ys = F.softmax(output, 1).to('cpu').data.numpy() - predicted_labels = list(map(lambda x: np.argmax(x), ys)) - predicted_scores = list(map(lambda x: x[1], ys)) - total_preds = torch.cat((total_preds, torch.Tensor(predicted_scores)), 0) - total_prelabels = torch.cat((total_prelabels, torch.Tensor(predicted_labels)), 0) - total_labels = torch.cat((total_labels, data1.y.view(-1, 1).cpu()), 0) - return total_labels.numpy().flatten(), total_preds.numpy().flatten(), total_prelabels.numpy().flatten() - - -def shuffle_dataset(dataset, seed): - np.random.seed(seed) - np.random.shuffle(dataset) - return dataset - - -def split_dataset(dataset, ratio): - n = int(len(dataset) * ratio) - dataset_1, dataset_2 = dataset[:n], dataset[n:] - return dataset_1, dataset_2 - -# modeling = [GINConvNet, GATNet, GAT_GCN, GCNNet][int(sys.argv[2])] -# datasets = [['davis', 'kiba'][int(sys.argv[1])]] -# model_st = modeling.__name__ -modeling = GATNet - -TRAIN_BATCH_SIZE = 128 -TEST_BATCH_SIZE = 128 -LR = 0.0005 -LOG_INTERVAL = 20 -NUM_EPOCHS = 200 - -print('Learning rate: ', LR) -print('Epochs: ', NUM_EPOCHS) -datafile = 'new_labels_0_10' - -# CPU or GPU - -if torch.cuda.is_available(): - device = torch.device('cuda') - print('The code uses GPU...') -else: - device = torch.device('cpu') - print('The code uses CPU!!!') - -drug1_data = TestbedDataset(root='data', dataset=datafile + '_drug1_bsl') -drug2_data = TestbedDataset(root='data', dataset=datafile + '_drug2_bsl') - -lenth = len(drug1_data) -pot = int(lenth/5) -print('lenth', lenth) -print('pot', pot) - - -random_num = random.sample(range(0, lenth), lenth) -for i in range(5): - test_num = random_num[pot*i:pot*(i+1)] - train_num = random_num[:pot*i] + random_num[pot*(i+1):] - - drug1_data_train = drug1_data[train_num] - drug1_data_test = drug1_data[test_num] - drug1_loader_train = DataLoader(drug1_data_train, batch_size=TRAIN_BATCH_SIZE, shuffle=None) - drug1_loader_test = DataLoader(drug1_data_test, batch_size=TRAIN_BATCH_SIZE, shuffle=None) - - - drug2_data_test = drug2_data[test_num] - drug2_data_train = drug2_data[train_num] - drug2_loader_train = DataLoader(drug2_data_train, batch_size=TRAIN_BATCH_SIZE, shuffle=None) - drug2_loader_test = DataLoader(drug2_data_test, batch_size=TRAIN_BATCH_SIZE, shuffle=None) - - model = modeling().to(device) - loss_fn = nn.CrossEntropyLoss() - optimizer = torch.optim.Adam(model.parameters(), lr=LR) - - - model_file_name = 'data/result/GATNet(DrugA_DrugB)' + str(i) + '--model_' + datafile + '.model' - result_file_name = 'data/result/GATNet(DrugA_DrugB)' + str(i) + '--result_' + datafile + '.csv' - file_AUCs = 'data/result/GATNet(DrugA_DrugB)' + str(i) + '--AUCs--' + datafile + '.txt' - AUCs = ('Epoch\tAUC_dev\tPR_AUC\tACC\tBACC\tPREC\tTPR\tKAPPA\tRECALL') - with open(file_AUCs, 'w') as f: - f.write(AUCs + '\n') - - best_auc = 0 - for epoch in range(NUM_EPOCHS): - train(model, device, drug1_loader_train, drug2_loader_train, optimizer, epoch + 1) - T, S, Y = predicting(model, device, drug1_loader_test, drug2_loader_test) - # T is correct label - # S is predict score - # Y is predict label - - # compute preformence - AUC = roc_auc_score(T, S) - precision, recall, threshold = metrics.precision_recall_curve(T, S) - PR_AUC = metrics.auc(recall, precision) - BACC = balanced_accuracy_score(T, Y) - tn, fp, fn, tp = confusion_matrix(T, Y).ravel() - TPR = tp / (tp + fn) - PREC = precision_score(T, Y) - ACC = accuracy_score(T, Y) - KAPPA = cohen_kappa_score(T, Y) - recall = recall_score(T, Y) - - - # save data - - if best_auc < AUC: - best_auc = AUC - AUCs = [epoch, AUC, PR_AUC, ACC, BACC, PREC, TPR, KAPPA, recall] - save_AUCs(AUCs, file_AUCs) - # torch.save(model.state_dict(), model_file_name) - # independent_num = [] - # independent_num.append(test_num) - # independent_num.append(T) - # independent_num.append(Y) - # independent_num.append(S) - # txtDF = pd.DataFrame(data=independent_num) - # txtDF.to_csv(result_file_name, index=False, header=False) - - print('best_auc', best_auc) +from _reference_paths import ASSET_DIR, artifact_path, data_path, output_dir, output_path, required_artifact +import random +import torch.nn.functional as F +import torch.nn as nn +from models.gat import GATNet +from models.gat_gcn_test import GAT_GCN +from models.gcn import GCNNet +from models.ginconv import GINConvNet +from utils_test import * +from sklearn.metrics import roc_curve, confusion_matrix +from sklearn.metrics import cohen_kappa_score, accuracy_score, roc_auc_score, precision_score, recall_score, balanced_accuracy_score +from sklearn import metrics +import pandas as pd + + +# training function at each epoch +def train(model, device, drug1_loader_train, drug2_loader_train, optimizer, epoch): + print('Training on {} samples...'.format(len(drug1_loader_train.dataset))) + model.train() + # train_loader = np.array(train_loader) + for batch_idx, data in enumerate(zip(drug1_loader_train, drug2_loader_train)): + data1 = data[0] + data2 = data[1] + data1 = data1.to(device) + data2 = data2.to(device) + y = data[0].y.view(-1, 1).long().to(device) + y = y.squeeze(1) + optimizer.zero_grad() + output = model(data1, data2) + loss = loss_fn(output, y) + # print('loss', loss) + loss.backward() + optimizer.step() + if batch_idx % LOG_INTERVAL == 0: + print('Train epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(epoch, + batch_idx * len(data1.x), + len(drug1_loader_train.dataset), + 100. * batch_idx / len(drug1_loader_train), + loss.item())) + + +def predicting(model, device, drug1_loader_test, drug2_loader_test): + model.eval() + total_preds = torch.Tensor() + total_labels = torch.Tensor() + total_prelabels = torch.Tensor() + print('Make prediction for {} samples...'.format(len(drug1_loader_test.dataset))) + with torch.no_grad(): + for data in zip(drug1_loader_test, drug2_loader_test): + data1 = data[0] + data2 = data[1] + data1 = data1.to(device) + data2 = data2.to(device) + output = model(data1, data2) + ys = F.softmax(output, 1).to('cpu').data.numpy() + predicted_labels = list(map(lambda x: np.argmax(x), ys)) + predicted_scores = list(map(lambda x: x[1], ys)) + total_preds = torch.cat((total_preds, torch.Tensor(predicted_scores)), 0) + total_prelabels = torch.cat((total_prelabels, torch.Tensor(predicted_labels)), 0) + total_labels = torch.cat((total_labels, data1.y.view(-1, 1).cpu()), 0) + return total_labels.numpy().flatten(), total_preds.numpy().flatten(), total_prelabels.numpy().flatten() + + +def shuffle_dataset(dataset, seed): + np.random.seed(seed) + np.random.shuffle(dataset) + return dataset + + +def split_dataset(dataset, ratio): + n = int(len(dataset) * ratio) + dataset_1, dataset_2 = dataset[:n], dataset[n:] + return dataset_1, dataset_2 + +# modeling = [GINConvNet, GATNet, GAT_GCN, GCNNet][int(sys.argv[2])] +# datasets = [['davis', 'kiba'][int(sys.argv[1])]] +# model_st = modeling.__name__ +modeling = GATNet + +TRAIN_BATCH_SIZE = 128 +TEST_BATCH_SIZE = 128 +LR = 0.0005 +LOG_INTERVAL = 20 +NUM_EPOCHS = 200 + +print('Learning rate: ', LR) +print('Epochs: ', NUM_EPOCHS) +datafile = 'new_labels_0_10' + +# CPU or GPU + +if torch.cuda.is_available(): + device = torch.device('cuda') + print('The code uses GPU...') +else: + device = torch.device('cpu') + print('The code uses CPU!!!') + +drug1_data = TestbedDataset(root=artifact_path("cache"), dataset=datafile + '_drug1_bsl') +drug2_data = TestbedDataset(root=artifact_path("cache"), dataset=datafile + '_drug2_bsl') + +lenth = len(drug1_data) +pot = int(lenth/5) +print('lenth', lenth) +print('pot', pot) + + +random_num = random.sample(range(0, lenth), lenth) +for i in range(5): + test_num = random_num[pot*i:pot*(i+1)] + train_num = random_num[:pot*i] + random_num[pot*(i+1):] + + drug1_data_train = drug1_data[train_num] + drug1_data_test = drug1_data[test_num] + drug1_loader_train = DataLoader(drug1_data_train, batch_size=TRAIN_BATCH_SIZE, shuffle=None) + drug1_loader_test = DataLoader(drug1_data_test, batch_size=TRAIN_BATCH_SIZE, shuffle=None) + + + drug2_data_test = drug2_data[test_num] + drug2_data_train = drug2_data[train_num] + drug2_loader_train = DataLoader(drug2_data_train, batch_size=TRAIN_BATCH_SIZE, shuffle=None) + drug2_loader_test = DataLoader(drug2_data_test, batch_size=TRAIN_BATCH_SIZE, shuffle=None) + + model = modeling().to(device) + loss_fn = nn.CrossEntropyLoss() + optimizer = torch.optim.Adam(model.parameters(), lr=LR) + + + model_file_name = output_path('result/GATNet(DrugA_DrugB)') + str(i) + '--model_' + datafile + '.model' + result_file_name = output_path('result/GATNet(DrugA_DrugB)') + str(i) + '--result_' + datafile + '.csv' + file_AUCs = output_path('result/GATNet(DrugA_DrugB)') + str(i) + '--AUCs--' + datafile + '.txt' + AUCs = ('Epoch\tAUC_dev\tPR_AUC\tACC\tBACC\tPREC\tTPR\tKAPPA\tRECALL') + with open(file_AUCs, 'w') as f: + f.write(AUCs + '\n') + + best_auc = 0 + for epoch in range(NUM_EPOCHS): + train(model, device, drug1_loader_train, drug2_loader_train, optimizer, epoch + 1) + T, S, Y = predicting(model, device, drug1_loader_test, drug2_loader_test) + # T is correct label + # S is predict score + # Y is predict label + + # compute preformence + AUC = roc_auc_score(T, S) + precision, recall, threshold = metrics.precision_recall_curve(T, S) + PR_AUC = metrics.auc(recall, precision) + BACC = balanced_accuracy_score(T, Y) + tn, fp, fn, tp = confusion_matrix(T, Y).ravel() + TPR = tp / (tp + fn) + PREC = precision_score(T, Y) + ACC = accuracy_score(T, Y) + KAPPA = cohen_kappa_score(T, Y) + recall = recall_score(T, Y) + + + # save data + + if best_auc < AUC: + best_auc = AUC + AUCs = [epoch, AUC, PR_AUC, ACC, BACC, PREC, TPR, KAPPA, recall] + save_AUCs(AUCs, file_AUCs) + # torch.save(model.state_dict(), model_file_name) + # independent_num = [] + # independent_num.append(test_num) + # independent_num.append(T) + # independent_num.append(Y) + # independent_num.append(S) + # txtDF = pd.DataFrame(data=independent_num) + # txtDF.to_csv(result_file_name, index=False, header=False) + + print('best_auc', best_auc) diff --git a/data/downstream/final_data/drug_response_pytorch/drug_combination/training_GAT_GCN.py b/data/downstream/final_data/drug_response_pytorch/drug_combination/training_GAT_GCN.py index eea4211e6de2a6c74bcc6da3fd8af90d5c528717..f7cfa09a2d361b086a1ce905d0e6dcd34a032d59 100644 --- a/data/downstream/final_data/drug_response_pytorch/drug_combination/training_GAT_GCN.py +++ b/data/downstream/final_data/drug_response_pytorch/drug_combination/training_GAT_GCN.py @@ -1,178 +1,179 @@ -import random -import numpy as np -import pandas as pd -import sys, os -from random import shuffle -import torch.utils.data as Data -import torch -import torch.nn.functional as F -import torch.nn as nn -from torch.utils.data import TensorDataset, Dataset -from models.gat import GATNet -from models.gat_gcn_test import GAT_GCN -from models.gcn import GCNNet -from models.ginconv import GINConvNet -from utils_test import * -from sklearn.metrics import roc_curve, confusion_matrix -from sklearn.metrics import cohen_kappa_score, accuracy_score, roc_auc_score, precision_score, recall_score, balanced_accuracy_score -from sklearn import metrics -from sklearn.model_selection import StratifiedKFold, KFold -import pandas as pd - - - -# training function at each epoch -def train(model, device, drug1_loader_train, drug2_loader_train, optimizer, epoch): - print('Training on {} samples...'.format(len(drug1_loader_train.dataset))) - model.train() - # train_loader = np.array(train_loader) - for batch_idx, data in enumerate(zip(drug1_loader_train, drug2_loader_train)): - data1 = data[0] - data2 = data[1] - data1 = data1.to(device) - data2 = data2.to(device) - y = data[0].y.view(-1, 1).long().to(device) - y = y.squeeze(1) - optimizer.zero_grad() - output = model(data1, data2) - loss = loss_fn(output, y) - # print('loss', loss) - loss.backward() - optimizer.step() - if batch_idx % LOG_INTERVAL == 0: - print('Train epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(epoch, - batch_idx * len(data1.x), - len(drug1_loader_train.dataset), - 100. * batch_idx / len(drug1_loader_train), - loss.item())) - - -def predicting(model, device, drug1_loader_test, drug2_loader_test): - model.eval() - total_preds = torch.Tensor() - total_labels = torch.Tensor() - total_prelabels = torch.Tensor() - print('Make prediction for {} samples...'.format(len(drug1_loader_test.dataset))) - with torch.no_grad(): - for data in zip(drug1_loader_test, drug2_loader_test): - data1 = data[0] - data2 = data[1] - data1 = data1.to(device) - data2 = data2.to(device) - output = model(data1, data2) - ys = F.softmax(output, 1).to('cpu').data.numpy() - predicted_labels = list(map(lambda x: np.argmax(x), ys)) - predicted_scores = list(map(lambda x: x[1], ys)) - total_preds = torch.cat((total_preds, torch.Tensor(predicted_scores)), 0) - total_prelabels = torch.cat((total_prelabels, torch.Tensor(predicted_labels)), 0) - total_labels = torch.cat((total_labels, data1.y.view(-1, 1).cpu()), 0) - return total_labels.numpy().flatten(), total_preds.numpy().flatten(), total_prelabels.numpy().flatten() - - -def shuffle_dataset(dataset, seed): - np.random.seed(seed) - np.random.shuffle(dataset) - return dataset - - -def split_dataset(dataset, ratio): - n = int(len(dataset) * ratio) - dataset_1, dataset_2 = dataset[:n], dataset[n:] - return dataset_1, dataset_2 - -# modeling = [GINConvNet, GATNet, GAT_GCN, GCNNet][int(sys.argv[2])] -# datasets = [['davis', 'kiba'][int(sys.argv[1])]] -# model_st = modeling.__name__ -modeling = GAT_GCN - -TRAIN_BATCH_SIZE = 256 -TEST_BATCH_SIZE = 256 -LR = 0.0005 -LOG_INTERVAL = 20 -NUM_EPOCHS = 2000 - -print('Learning rate: ', LR) -print('Epochs: ', NUM_EPOCHS) -datafile = 'new_labels_0_10' -cellfile = 'data/new_cell_features_954.csv' - -# CPU or GPU -import os -os.environ["CUDA_VISIBLE_DEVICES"] = "1" - -if torch.cuda.is_available(): - device = torch.device('cuda') - print('The code uses GPU...') -else: - device = torch.device('cpu') - print('The code uses CPU!!!') - -drug1_data = TestbedDataset(root='data', dataset=datafile + '_drug1') -drug2_data = TestbedDataset(root='data', dataset=datafile + '_drug2') -lenth = len(drug1_data) -pot = int(lenth/5) -print('lenth', lenth) -print('pot', pot) -best_auc = 0 - -random_num = random.sample(range(0, lenth), lenth) -for i in range(5): - test_num = random_num[pot*i:pot*(i+1)] - train_num = random_num[:pot*i] + random_num[pot*(i+1):] - - drug1_data_train = drug1_data[train_num] - drug1_data_test = drug1_data[test_num] - # print('type(drug1_data_train)', type(drug1_data_train)) - # print('drug1_data_train[0]', drug1_data_train[0]) - # print('len(drug1_data_train)', len(drug1_data_train)) - drug1_loader_train = DataLoader(drug1_data_train, batch_size=TRAIN_BATCH_SIZE, shuffle=None) - drug1_loader_test = DataLoader(drug1_data_test, batch_size=TRAIN_BATCH_SIZE, shuffle=None) - - - # print('len(drug2_data)', len(drug2_data)) - - drug2_data_test = drug2_data[test_num] - drug2_data_train = drug2_data[train_num] - drug2_loader_train = DataLoader(drug2_data_train, batch_size=TRAIN_BATCH_SIZE, shuffle=None) - drug2_loader_test = DataLoader(drug2_data_test, batch_size=TRAIN_BATCH_SIZE, shuffle=None) - - model = modeling().to(device) - loss_fn = nn.CrossEntropyLoss() - optimizer = torch.optim.Adam(model.parameters(), lr=LR) - - - model_file_name = 'data/result/GAT_GCN(shared weight)' + str(i) + '--model_' + datafile + '.model' - result_file_name = 'data/result/GAT_GCN(shared weight)' + str(i) + '--result_' + datafile + '.csv' - file_AUCs = 'data/result/GAT_GCN(shared weight)' + str(i) + '--AUCs--' + datafile + '.txt' - AUCs = ('Epoch\tAUC_dev\tPR_AUC\tACC\tBACC\tPREC\tTPR\tKAPPA\tRECALL') - with open(file_AUCs, 'w') as f: - f.write(AUCs + '\n') - - for epoch in range(NUM_EPOCHS): - train(model, device, drug1_loader_train, drug2_loader_train, optimizer, epoch + 1) - T, S, Y = predicting(model, device, drug1_loader_test, drug2_loader_test) - # T is correct label - # S is predict score - # Y is predict label - - # compute preformence - AUC = roc_auc_score(T, S) - precision, recall, threshold = metrics.precision_recall_curve(T, S) - PR_AUC = metrics.auc(recall, precision) - BACC = balanced_accuracy_score(T, Y) - tn, fp, fn, tp = confusion_matrix(T, Y).ravel() - TPR = tp / (tp + fn) - PREC = precision_score(T, Y) - ACC = accuracy_score(T, Y) - KAPPA = cohen_kappa_score(T, Y) - recall = recall_score(T, Y) - - # save data - AUCs = [epoch, AUC, PR_AUC, ACC, BACC, PREC, TPR, KAPPA, recall] - save_AUCs(AUCs, file_AUCs) - ret = [rmse(T, S), mse(T, S), pearson(T, S), spearman(T, S), ci(T, S)] - if best_auc < AUC: - best_auc = AUC - torch.save(model.state_dict(), model_file_name) - - +from _reference_paths import ASSET_DIR, artifact_path, data_path, output_dir, output_path, required_artifact +import random +import numpy as np +import pandas as pd +import sys, os +from random import shuffle +import torch.utils.data as Data +import torch +import torch.nn.functional as F +import torch.nn as nn +from torch.utils.data import TensorDataset, Dataset +from models.gat import GATNet +from models.gat_gcn_test import GAT_GCN +from models.gcn import GCNNet +from models.ginconv import GINConvNet +from utils_test import * +from sklearn.metrics import roc_curve, confusion_matrix +from sklearn.metrics import cohen_kappa_score, accuracy_score, roc_auc_score, precision_score, recall_score, balanced_accuracy_score +from sklearn import metrics +from sklearn.model_selection import StratifiedKFold, KFold +import pandas as pd + + + +# training function at each epoch +def train(model, device, drug1_loader_train, drug2_loader_train, optimizer, epoch): + print('Training on {} samples...'.format(len(drug1_loader_train.dataset))) + model.train() + # train_loader = np.array(train_loader) + for batch_idx, data in enumerate(zip(drug1_loader_train, drug2_loader_train)): + data1 = data[0] + data2 = data[1] + data1 = data1.to(device) + data2 = data2.to(device) + y = data[0].y.view(-1, 1).long().to(device) + y = y.squeeze(1) + optimizer.zero_grad() + output = model(data1, data2) + loss = loss_fn(output, y) + # print('loss', loss) + loss.backward() + optimizer.step() + if batch_idx % LOG_INTERVAL == 0: + print('Train epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(epoch, + batch_idx * len(data1.x), + len(drug1_loader_train.dataset), + 100. * batch_idx / len(drug1_loader_train), + loss.item())) + + +def predicting(model, device, drug1_loader_test, drug2_loader_test): + model.eval() + total_preds = torch.Tensor() + total_labels = torch.Tensor() + total_prelabels = torch.Tensor() + print('Make prediction for {} samples...'.format(len(drug1_loader_test.dataset))) + with torch.no_grad(): + for data in zip(drug1_loader_test, drug2_loader_test): + data1 = data[0] + data2 = data[1] + data1 = data1.to(device) + data2 = data2.to(device) + output = model(data1, data2) + ys = F.softmax(output, 1).to('cpu').data.numpy() + predicted_labels = list(map(lambda x: np.argmax(x), ys)) + predicted_scores = list(map(lambda x: x[1], ys)) + total_preds = torch.cat((total_preds, torch.Tensor(predicted_scores)), 0) + total_prelabels = torch.cat((total_prelabels, torch.Tensor(predicted_labels)), 0) + total_labels = torch.cat((total_labels, data1.y.view(-1, 1).cpu()), 0) + return total_labels.numpy().flatten(), total_preds.numpy().flatten(), total_prelabels.numpy().flatten() + + +def shuffle_dataset(dataset, seed): + np.random.seed(seed) + np.random.shuffle(dataset) + return dataset + + +def split_dataset(dataset, ratio): + n = int(len(dataset) * ratio) + dataset_1, dataset_2 = dataset[:n], dataset[n:] + return dataset_1, dataset_2 + +# modeling = [GINConvNet, GATNet, GAT_GCN, GCNNet][int(sys.argv[2])] +# datasets = [['davis', 'kiba'][int(sys.argv[1])]] +# model_st = modeling.__name__ +modeling = GAT_GCN + +TRAIN_BATCH_SIZE = 256 +TEST_BATCH_SIZE = 256 +LR = 0.0005 +LOG_INTERVAL = 20 +NUM_EPOCHS = 2000 + +print('Learning rate: ', LR) +print('Epochs: ', NUM_EPOCHS) +datafile = 'new_labels_0_10' +cellfile = data_path('new_cell_features_954.csv') + +# CPU or GPU +import os +os.environ["CUDA_VISIBLE_DEVICES"] = "1" + +if torch.cuda.is_available(): + device = torch.device('cuda') + print('The code uses GPU...') +else: + device = torch.device('cpu') + print('The code uses CPU!!!') + +drug1_data = TestbedDataset(root=artifact_path("cache"), dataset=datafile + '_drug1') +drug2_data = TestbedDataset(root=artifact_path("cache"), dataset=datafile + '_drug2') +lenth = len(drug1_data) +pot = int(lenth/5) +print('lenth', lenth) +print('pot', pot) +best_auc = 0 + +random_num = random.sample(range(0, lenth), lenth) +for i in range(5): + test_num = random_num[pot*i:pot*(i+1)] + train_num = random_num[:pot*i] + random_num[pot*(i+1):] + + drug1_data_train = drug1_data[train_num] + drug1_data_test = drug1_data[test_num] + # print('type(drug1_data_train)', type(drug1_data_train)) + # print('drug1_data_train[0]', drug1_data_train[0]) + # print('len(drug1_data_train)', len(drug1_data_train)) + drug1_loader_train = DataLoader(drug1_data_train, batch_size=TRAIN_BATCH_SIZE, shuffle=None) + drug1_loader_test = DataLoader(drug1_data_test, batch_size=TRAIN_BATCH_SIZE, shuffle=None) + + + # print('len(drug2_data)', len(drug2_data)) + + drug2_data_test = drug2_data[test_num] + drug2_data_train = drug2_data[train_num] + drug2_loader_train = DataLoader(drug2_data_train, batch_size=TRAIN_BATCH_SIZE, shuffle=None) + drug2_loader_test = DataLoader(drug2_data_test, batch_size=TRAIN_BATCH_SIZE, shuffle=None) + + model = modeling().to(device) + loss_fn = nn.CrossEntropyLoss() + optimizer = torch.optim.Adam(model.parameters(), lr=LR) + + + model_file_name = output_path('result/GAT_GCN(shared weight)') + str(i) + '--model_' + datafile + '.model' + result_file_name = output_path('result/GAT_GCN(shared weight)') + str(i) + '--result_' + datafile + '.csv' + file_AUCs = output_path('result/GAT_GCN(shared weight)') + str(i) + '--AUCs--' + datafile + '.txt' + AUCs = ('Epoch\tAUC_dev\tPR_AUC\tACC\tBACC\tPREC\tTPR\tKAPPA\tRECALL') + with open(file_AUCs, 'w') as f: + f.write(AUCs + '\n') + + for epoch in range(NUM_EPOCHS): + train(model, device, drug1_loader_train, drug2_loader_train, optimizer, epoch + 1) + T, S, Y = predicting(model, device, drug1_loader_test, drug2_loader_test) + # T is correct label + # S is predict score + # Y is predict label + + # compute preformence + AUC = roc_auc_score(T, S) + precision, recall, threshold = metrics.precision_recall_curve(T, S) + PR_AUC = metrics.auc(recall, precision) + BACC = balanced_accuracy_score(T, Y) + tn, fp, fn, tp = confusion_matrix(T, Y).ravel() + TPR = tp / (tp + fn) + PREC = precision_score(T, Y) + ACC = accuracy_score(T, Y) + KAPPA = cohen_kappa_score(T, Y) + recall = recall_score(T, Y) + + # save data + AUCs = [epoch, AUC, PR_AUC, ACC, BACC, PREC, TPR, KAPPA, recall] + save_AUCs(AUCs, file_AUCs) + ret = [rmse(T, S), mse(T, S), pearson(T, S), spearman(T, S), ci(T, S)] + if best_auc < AUC: + best_auc = AUC + torch.save(model.state_dict(), model_file_name) + + diff --git a/data/downstream/final_data/drug_response_pytorch/drug_combination/training_GAT_r.py b/data/downstream/final_data/drug_response_pytorch/drug_combination/training_GAT_r.py index 84ea8b9932b59564c5c68a7efd8643f1f4b7d1b4..7e42331b921bf93424e2fcbb8793584142b4108f 100644 --- a/data/downstream/final_data/drug_response_pytorch/drug_combination/training_GAT_r.py +++ b/data/downstream/final_data/drug_response_pytorch/drug_combination/training_GAT_r.py @@ -1,197 +1,198 @@ -import random -import torch.nn.functional as F -import torch.nn as nn -from models.gat_r import GATNet -from models.gat_gcn_test import GAT_GCN -from models.gcn import GCNNet -from models.ginconv import GINConvNet -import math -from utils_test import * -from sklearn.metrics import roc_curve, confusion_matrix -from sklearn.metrics import cohen_kappa_score, accuracy_score, roc_auc_score, precision_score, recall_score, balanced_accuracy_score -from sklearn import metrics -import pandas as pd -from sklearn import preprocessing - - - -# training function at each epoch -def train(model, device, drug1_loader_train, drug2_loader_train, optimizer, epoch): - print('Training on {} samples...'.format(len(drug1_loader_train.dataset))) - model.train() - # train_loader = np.array(train_loader) - for batch_idx, data in enumerate(zip(drug1_loader_train, drug2_loader_train)): - data1 = data[0] - data2 = data[1] - data1 = data1.to(device) - data2 = data2.to(device) - y = data[0].y.view(-1, 1).float().to(device) - y = y.squeeze(1) - output = model(data1, data2) - loss = loss_fn(output, y.unsqueeze(1)) - # print('loss', loss) - optimizer.zero_grad() - loss.backward() - optimizer.step() - if batch_idx % LOG_INTERVAL == 0: - print('Train epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(epoch, - batch_idx * len(data1.x), - len(drug1_loader_train.dataset), - 100. * batch_idx / len(drug1_loader_train), - loss.item())) - - -def predicting(model, device, drug1_loader_test, drug2_loader_test): - model.eval() - total_preds = torch.Tensor() - total_labels = torch.Tensor() - total_prelabels = torch.Tensor() - print('Make prediction for {} samples...'.format(len(drug1_loader_test.dataset))) - with torch.no_grad(): - for data in zip(drug1_loader_test, drug2_loader_test): - data1 = data[0] - data2 = data[1] - data1 = data1.to(device) - data2 = data2.to(device) - output = model(data1, data2) - ys = output.to('cpu').data.numpy() - # predicted_labels = list(map(lambda x: np.argmax(x), ys)) - total_preds = torch.cat((total_preds, torch.Tensor(ys)), 0) - # total_prelabels = torch.cat((total_prelabels, torch.Tensor(predicted_labels)), 0) - total_labels = torch.cat((total_labels, data1.y.view(-1, 1).cpu()), 0) - return total_preds.numpy().flatten(), total_labels.numpy().flatten() - - -def shuffle_dataset(dataset, seed): - np.random.seed(seed) - np.random.shuffle(dataset) - return dataset - - -def split_dataset(dataset, ratio): - n = int(len(dataset) * ratio) - dataset_1, dataset_2 = dataset[:n], dataset[n:] - return dataset_1, dataset_2 - -def compute_mae_mse_rmse(target,prediction): - error = [] - for i in range(len(target)): - error.append(target[i] - prediction[i]) - squaredError = [] - absError = [] - for val in error: - squaredError.append(val * val) # target-prediction之差平方 - absError.append(abs(val)) # 误差绝对值 - mae=sum(absError)/len(absError) # 平均绝对误差MAE - mse=sum(squaredError)/len(squaredError) # 均方误差MSE - RMSE=sum(absError)/len(absError) - return mae,mse,RMSE - -def compute_rsquared(X, Y): - xBar = np.mean(X) - yBar = np.mean(Y) - SSR = 0 - varX = 0 - varY = 0 - for i in range(0, len(X)): - diffXXBar = X[i] - xBar - diffYYBar = Y[i] - yBar - SSR += (diffXXBar * diffYYBar) - varX += diffXXBar ** 2 - varY += diffYYBar ** 2 - - SST = math.sqrt(varX * varY) - r2=round((SSR / SST) ** 2,3) - return r2 - -# modeling = [GINConvNet, GATNet, GAT_GCN, GCNNet][int(sys.argv[2])] -# datasets = [['davis', 'kiba'][int(sys.argv[1])]] -# model_st = modeling.__name__ -modeling = GATNet - -TRAIN_BATCH_SIZE = 64 -TEST_BATCH_SIZE = 64 -LR = 0.0001 -LOG_INTERVAL = 20 -NUM_EPOCHS = 1000 - -print('Learning rate: ', LR) -print('Epochs: ', NUM_EPOCHS) -datafile = 'score_0_10' - -# CPU or GPU - -if torch.cuda.is_available(): - device = torch.device('cuda') - print('The code uses GPU...') -else: - device = torch.device('cpu') - print('The code uses CPU!!!') - -drug1_data = TestbedDataset(root='data', dataset=datafile + '_drug1') -drug2_data = TestbedDataset(root='data', dataset=datafile + '_drug2') - -lenth = len(drug1_data) -pot = int(lenth/5) -print('lenth', lenth) -print('pot', pot) - - -random_num = random.sample(range(0, lenth), lenth) -for i in range(5): - test_num = random_num[pot*i:pot*(i+1)] - train_num = random_num[:pot*i] + random_num[pot*(i+1):] - - drug1_data_train = drug1_data[train_num] - drug1_data_test = drug1_data[test_num] - drug1_loader_train = DataLoader(drug1_data_train, batch_size=TRAIN_BATCH_SIZE, shuffle=None) - drug1_loader_test = DataLoader(drug1_data_test, batch_size=TRAIN_BATCH_SIZE, shuffle=None) - - - drug2_data_test = drug2_data[test_num] - drug2_data_train = drug2_data[train_num] - drug2_loader_train = DataLoader(drug2_data_train, batch_size=TRAIN_BATCH_SIZE, shuffle=None) - drug2_loader_test = DataLoader(drug2_data_test, batch_size=TRAIN_BATCH_SIZE, shuffle=None) - - model = modeling().to(device) - loss_fn = nn.MSELoss() - optimizer = torch.optim.Adam(model.parameters(), lr=LR) - - - # model_file_name = 'data/result/GATNet(DrugA_DrugB)' + str(i) + '--model_' + datafile + '.model' - result_file_name = 'data/result/GATNet(DrugA_DrugB)_R' + str(i) + '--result_' + datafile + '.csv' - file_AUCs = 'data/result/GATNet(DrugA_DrugB)_R' + str(i) + '--AUCs--' + datafile + '.txt' - AUCs = ('Epoch\tepoch\tmse\tmae\trmse\tr2') - with open(file_AUCs, 'w') as f: - f.write(AUCs + '\n') - - best_auc = 10000000000000 - for epoch in range(NUM_EPOCHS): - train(model, device, drug1_loader_train, drug2_loader_train, optimizer, epoch + 1) - S, T = predicting(model, device, drug1_loader_test, drug2_loader_test) - # T is correct score - # S is predict score - # Y is predict label - print(S) - print(T) - - # compute preformence - - mae, mse, rmse = compute_mae_mse_rmse(S, T) - r2 = compute_rsquared(S, T) - # save data - - if best_auc > mse: - best_auc = mse - AUCs = [epoch, mse, mae, rmse, r2] - save_AUCs(AUCs, file_AUCs) - # torch.save(model.state_dict(), model_file_name) - independent_num = [] - # independent_num.append(test_num) - independent_num.append(T) - # independent_num.append(Y) - independent_num.append(S) - txtDF = pd.DataFrame(data=independent_num) - txtDF.to_csv(result_file_name, index=False, header=False) - - print('best_mse', best_auc) +from _reference_paths import ASSET_DIR, artifact_path, data_path, output_dir, output_path, required_artifact +import random +import torch.nn.functional as F +import torch.nn as nn +from models.gat_r import GATNet +from models.gat_gcn_test import GAT_GCN +from models.gcn import GCNNet +from models.ginconv import GINConvNet +import math +from utils_test import * +from sklearn.metrics import roc_curve, confusion_matrix +from sklearn.metrics import cohen_kappa_score, accuracy_score, roc_auc_score, precision_score, recall_score, balanced_accuracy_score +from sklearn import metrics +import pandas as pd +from sklearn import preprocessing + + + +# training function at each epoch +def train(model, device, drug1_loader_train, drug2_loader_train, optimizer, epoch): + print('Training on {} samples...'.format(len(drug1_loader_train.dataset))) + model.train() + # train_loader = np.array(train_loader) + for batch_idx, data in enumerate(zip(drug1_loader_train, drug2_loader_train)): + data1 = data[0] + data2 = data[1] + data1 = data1.to(device) + data2 = data2.to(device) + y = data[0].y.view(-1, 1).float().to(device) + y = y.squeeze(1) + output = model(data1, data2) + loss = loss_fn(output, y.unsqueeze(1)) + # print('loss', loss) + optimizer.zero_grad() + loss.backward() + optimizer.step() + if batch_idx % LOG_INTERVAL == 0: + print('Train epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(epoch, + batch_idx * len(data1.x), + len(drug1_loader_train.dataset), + 100. * batch_idx / len(drug1_loader_train), + loss.item())) + + +def predicting(model, device, drug1_loader_test, drug2_loader_test): + model.eval() + total_preds = torch.Tensor() + total_labels = torch.Tensor() + total_prelabels = torch.Tensor() + print('Make prediction for {} samples...'.format(len(drug1_loader_test.dataset))) + with torch.no_grad(): + for data in zip(drug1_loader_test, drug2_loader_test): + data1 = data[0] + data2 = data[1] + data1 = data1.to(device) + data2 = data2.to(device) + output = model(data1, data2) + ys = output.to('cpu').data.numpy() + # predicted_labels = list(map(lambda x: np.argmax(x), ys)) + total_preds = torch.cat((total_preds, torch.Tensor(ys)), 0) + # total_prelabels = torch.cat((total_prelabels, torch.Tensor(predicted_labels)), 0) + total_labels = torch.cat((total_labels, data1.y.view(-1, 1).cpu()), 0) + return total_preds.numpy().flatten(), total_labels.numpy().flatten() + + +def shuffle_dataset(dataset, seed): + np.random.seed(seed) + np.random.shuffle(dataset) + return dataset + + +def split_dataset(dataset, ratio): + n = int(len(dataset) * ratio) + dataset_1, dataset_2 = dataset[:n], dataset[n:] + return dataset_1, dataset_2 + +def compute_mae_mse_rmse(target,prediction): + error = [] + for i in range(len(target)): + error.append(target[i] - prediction[i]) + squaredError = [] + absError = [] + for val in error: + squaredError.append(val * val) # Squared target-prediction error. + absError.append(abs(val)) # Absolute error. + mae=sum(absError)/len(absError) # Mean absolute error (MAE). + mse=sum(squaredError)/len(squaredError) # Mean squared error (MSE). + RMSE=sum(absError)/len(absError) + return mae,mse,RMSE + +def compute_rsquared(X, Y): + xBar = np.mean(X) + yBar = np.mean(Y) + SSR = 0 + varX = 0 + varY = 0 + for i in range(0, len(X)): + diffXXBar = X[i] - xBar + diffYYBar = Y[i] - yBar + SSR += (diffXXBar * diffYYBar) + varX += diffXXBar ** 2 + varY += diffYYBar ** 2 + + SST = math.sqrt(varX * varY) + r2=round((SSR / SST) ** 2,3) + return r2 + +# modeling = [GINConvNet, GATNet, GAT_GCN, GCNNet][int(sys.argv[2])] +# datasets = [['davis', 'kiba'][int(sys.argv[1])]] +# model_st = modeling.__name__ +modeling = GATNet + +TRAIN_BATCH_SIZE = 64 +TEST_BATCH_SIZE = 64 +LR = 0.0001 +LOG_INTERVAL = 20 +NUM_EPOCHS = 1000 + +print('Learning rate: ', LR) +print('Epochs: ', NUM_EPOCHS) +datafile = 'score_0_10' + +# CPU or GPU + +if torch.cuda.is_available(): + device = torch.device('cuda') + print('The code uses GPU...') +else: + device = torch.device('cpu') + print('The code uses CPU!!!') + +drug1_data = TestbedDataset(root=artifact_path("cache"), dataset=datafile + '_drug1') +drug2_data = TestbedDataset(root=artifact_path("cache"), dataset=datafile + '_drug2') + +lenth = len(drug1_data) +pot = int(lenth/5) +print('lenth', lenth) +print('pot', pot) + + +random_num = random.sample(range(0, lenth), lenth) +for i in range(5): + test_num = random_num[pot*i:pot*(i+1)] + train_num = random_num[:pot*i] + random_num[pot*(i+1):] + + drug1_data_train = drug1_data[train_num] + drug1_data_test = drug1_data[test_num] + drug1_loader_train = DataLoader(drug1_data_train, batch_size=TRAIN_BATCH_SIZE, shuffle=None) + drug1_loader_test = DataLoader(drug1_data_test, batch_size=TRAIN_BATCH_SIZE, shuffle=None) + + + drug2_data_test = drug2_data[test_num] + drug2_data_train = drug2_data[train_num] + drug2_loader_train = DataLoader(drug2_data_train, batch_size=TRAIN_BATCH_SIZE, shuffle=None) + drug2_loader_test = DataLoader(drug2_data_test, batch_size=TRAIN_BATCH_SIZE, shuffle=None) + + model = modeling().to(device) + loss_fn = nn.MSELoss() + optimizer = torch.optim.Adam(model.parameters(), lr=LR) + + + # model_file_name = 'data/result/GATNet(DrugA_DrugB)' + str(i) + '--model_' + datafile + '.model' + result_file_name = output_path('result/GATNet(DrugA_DrugB)_R') + str(i) + '--result_' + datafile + '.csv' + file_AUCs = output_path('result/GATNet(DrugA_DrugB)_R') + str(i) + '--AUCs--' + datafile + '.txt' + AUCs = ('Epoch\tepoch\tmse\tmae\trmse\tr2') + with open(file_AUCs, 'w') as f: + f.write(AUCs + '\n') + + best_auc = 10000000000000 + for epoch in range(NUM_EPOCHS): + train(model, device, drug1_loader_train, drug2_loader_train, optimizer, epoch + 1) + S, T = predicting(model, device, drug1_loader_test, drug2_loader_test) + # T is correct score + # S is predict score + # Y is predict label + print(S) + print(T) + + # compute preformence + + mae, mse, rmse = compute_mae_mse_rmse(S, T) + r2 = compute_rsquared(S, T) + # save data + + if best_auc > mse: + best_auc = mse + AUCs = [epoch, mse, mae, rmse, r2] + save_AUCs(AUCs, file_AUCs) + # torch.save(model.state_dict(), model_file_name) + independent_num = [] + # independent_num.append(test_num) + independent_num.append(T) + # independent_num.append(Y) + independent_num.append(S) + txtDF = pd.DataFrame(data=independent_num) + txtDF.to_csv(result_file_name, index=False, header=False) + + print('best_mse', best_auc) diff --git a/data/downstream/final_data/drug_response_pytorch/drug_combination/training_GCN.py b/data/downstream/final_data/drug_response_pytorch/drug_combination/training_GCN.py index 2ca0cc62e439b78d3148648b9007530c79476fcc..2a861bc984f5c74a13710158a1dcddcb6fb07384 100644 --- a/data/downstream/final_data/drug_response_pytorch/drug_combination/training_GCN.py +++ b/data/downstream/final_data/drug_response_pytorch/drug_combination/training_GCN.py @@ -1,202 +1,203 @@ -import random -import numpy as np -import pandas as pd -import sys, os -from random import shuffle -import torch.utils.data as Data -import torch -import torch.nn.functional as F -import torch.nn as nn -from torch.utils.data import TensorDataset, Dataset -from models.gat import GATNet -from models.gat_gcn_test import GAT_GCN -from models.gcn import GCNNet -from models.ginconv import GINConvNet -from utils_test import * -from sklearn.metrics import roc_curve, confusion_matrix -from sklearn.metrics import cohen_kappa_score, accuracy_score, roc_auc_score, precision_score, recall_score, balanced_accuracy_score -from sklearn import metrics -from sklearn.model_selection import StratifiedKFold, KFold -import pandas as pd - - - -# training function at each epoch -def train(model, device, drug1_loader_train, drug2_loader_train, optimizer, epoch): - print('Training on {} samples...'.format(len(drug1_loader_train.dataset))) - model.train() - # train_loader = np.array(train_loader) - for batch_idx, data in enumerate(zip(drug1_loader_train, drug2_loader_train)): - data1 = data[0] - data2 = data[1] - data1 = data1.to(device) - data2 = data2.to(device) - y = data[0].y.view(-1, 1).long().to(device) - y = y.squeeze(1) - optimizer.zero_grad() - output = model(data1, data2) - loss = loss_fn(output, y) - # print('loss', loss) - loss.backward() - optimizer.step() - if batch_idx % LOG_INTERVAL == 0: - print('Train epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(epoch, - batch_idx * len(data1.x), - len(drug1_loader_train.dataset), - 100. * batch_idx / len(drug1_loader_train), - loss.item())) - - -def predicting(model, device, drug1_loader_test, drug2_loader_test): - model.eval() - total_preds = torch.Tensor() - total_labels = torch.Tensor() - total_prelabels = torch.Tensor() - print('Make prediction for {} samples...'.format(len(drug1_loader_test.dataset))) - with torch.no_grad(): - for data in zip(drug1_loader_test, drug2_loader_test): - data1 = data[0] - data2 = data[1] - data1 = data1.to(device) - data2 = data2.to(device) - output = model(data1, data2) - ys = F.softmax(output, 1).to('cpu').data.numpy() - predicted_labels = list(map(lambda x: np.argmax(x), ys)) - predicted_scores = list(map(lambda x: x[1], ys)) - total_preds = torch.cat((total_preds, torch.Tensor(predicted_scores)), 0) - total_prelabels = torch.cat((total_prelabels, torch.Tensor(predicted_labels)), 0) - total_labels = torch.cat((total_labels, data1.y.view(-1, 1).cpu()), 0) - return total_labels.numpy().flatten(), total_preds.numpy().flatten(), total_prelabels.numpy().flatten() - - -def shuffle_dataset(dataset, seed): - np.random.seed(seed) - np.random.shuffle(dataset) - return dataset - - -def split_dataset(dataset, ratio): - n = int(len(dataset) * ratio) - dataset_1, dataset_2 = dataset[:n], dataset[n:] - return dataset_1, dataset_2 - -# modeling = [GINConvNet, GATNet, GAT_GCN, GCNNet][int(sys.argv[2])] -# datasets = [['davis', 'kiba'][int(sys.argv[1])]] -# model_st = modeling.__name__ -modeling = GCNNet - -TRAIN_BATCH_SIZE = 256 -TEST_BATCH_SIZE = 256 -LR = 0.0005 -LOG_INTERVAL = 20 -NUM_EPOCHS = 100 - -print('Learning rate: ', LR) -print('Epochs: ', NUM_EPOCHS) -datafile = 'new_labels_0_10' - -# CPU or GPU -import os -os.environ["CUDA_VISIBLE_DEVICES"] = "1" - -if torch.cuda.is_available(): - device = torch.device('cuda') - print('The code uses GPU...') -else: - device = torch.device('cpu') - print('The code uses CPU!!!') - -drug1_data = TestbedDataset(root='data', dataset=datafile + '_drug1') -drug2_data = TestbedDataset(root='data', dataset=datafile + '_drug2') - -ind_datafile='independent_set/independent_input' -ind_drug1_data = TestbedDataset(root='data', dataset=ind_datafile + '_drug1') -ind_drug2_data = TestbedDataset(root='data', dataset=ind_datafile + '_drug2') - -lenth = len(drug1_data) -pot = int(lenth/5) -print('lenth', lenth) -print('pot', pot) - -domain_gen=True -random_num = random.sample(range(0, lenth), lenth) -for i in range(5): - if domain_gen: - drug1_data_train = drug1_data - drug1_data_test = ind_drug1_data - - drug2_data_train = drug2_data - drug2_data_test = ind_drug2_data - else: - - test_num = random_num[pot*i:pot*(i+1)] - train_num = random_num[:pot*i] + random_num[pot*(i+1):] - - drug1_data_train = drug1_data[train_num] - drug1_data_test = drug1_data[test_num] - - drug2_data_test = drug2_data[test_num] - drug2_data_train = drug2_data[train_num] - # print('type(drug1_data_train)', type(drug1_data_train)) - # print('drug1_data_train[0]', drug1_data_train[0]) - # print('len(drug1_data_train)', len(drug1_data_train)) - drug1_loader_train = DataLoader(drug1_data_train, batch_size=TRAIN_BATCH_SIZE, shuffle=None) - drug1_loader_test = DataLoader(drug1_data_test, batch_size=TRAIN_BATCH_SIZE, shuffle=None) - - - - drug2_loader_train = DataLoader(drug2_data_train, batch_size=TRAIN_BATCH_SIZE, shuffle=None) - drug2_loader_test = DataLoader(drug2_data_test, batch_size=TRAIN_BATCH_SIZE, shuffle=None) - - model = modeling().to(device) - loss_fn = nn.CrossEntropyLoss() - optimizer = torch.optim.Adam(model.parameters(), lr=LR) - - - # model_file_name = 'data/result/GCNNet(DrugA_DrugB)' + str(i) + '--model_' + datafile + '.model' - # result_file_name = 'data/result/GCNNet(DrugA_DrugB)' + str(i) + '--result_' + datafile + '.csv' - # file_AUCs = 'data/result/GCNNet(DrugA_DrugB)' + str(i) + '--AUCs--' + datafile + '.txt' - # AUCs = ('Epoch\tAUC_dev\tPR_AUC\tACC\tBACC\tPREC\tTPR\tKAPPA\tRECALL') - # with open(file_AUCs, 'w') as f: - # f.write(AUCs + '\n') - - best_auc = 0 - for epoch in range(NUM_EPOCHS): - train(model, device, drug1_loader_train, drug2_loader_train, optimizer, epoch + 1) - T, S, Y = predicting(model, device, drug1_loader_test, drug2_loader_test) - # T is correct label - # S is predict score - # Y is predict label - #print(T,S,Y) - # compute preformence - try: - AUC = roc_auc_score(T, S) - except: - AUC=0 - precision, recall, threshold = metrics.precision_recall_curve(T, S) - PR_AUC = metrics.auc(recall, precision) - BACC = balanced_accuracy_score(T, Y) - tn, fp, fn, tp = confusion_matrix(T, Y).ravel() - TPR = tp / (tp + fn) - PREC = precision_score(T, Y) - ACC = accuracy_score(T, Y) - KAPPA = cohen_kappa_score(T, Y) - recall = recall_score(T, Y) - - # save data - AUCs = [epoch, AUC, PR_AUC, ACC, BACC, PREC, TPR, KAPPA, recall] - # save_AUCs(AUCs, file_AUCs) - ret = [rmse(T, S), mse(T, S), pearson(T, S), spearman(T, S), ci(T, S)] - if best_auc < AUC: - best_auc = AUC - print(best_auc) - # torch.save(model.state_dict(), model_file_name) - # independent_num = [] - # independent_num.append(test_num) - # independent_num.append(T) - # independent_num.append(Y) - # independent_num.append(S) - # txtDF = pd.DataFrame(data=independent_num) - # txtDF.to_csv(result_file_name, index=False, header=False) - - +from _reference_paths import ASSET_DIR, artifact_path, data_path, output_dir, output_path, required_artifact +import random +import numpy as np +import pandas as pd +import sys, os +from random import shuffle +import torch.utils.data as Data +import torch +import torch.nn.functional as F +import torch.nn as nn +from torch.utils.data import TensorDataset, Dataset +from models.gat import GATNet +from models.gat_gcn_test import GAT_GCN +from models.gcn import GCNNet +from models.ginconv import GINConvNet +from utils_test import * +from sklearn.metrics import roc_curve, confusion_matrix +from sklearn.metrics import cohen_kappa_score, accuracy_score, roc_auc_score, precision_score, recall_score, balanced_accuracy_score +from sklearn import metrics +from sklearn.model_selection import StratifiedKFold, KFold +import pandas as pd + + + +# training function at each epoch +def train(model, device, drug1_loader_train, drug2_loader_train, optimizer, epoch): + print('Training on {} samples...'.format(len(drug1_loader_train.dataset))) + model.train() + # train_loader = np.array(train_loader) + for batch_idx, data in enumerate(zip(drug1_loader_train, drug2_loader_train)): + data1 = data[0] + data2 = data[1] + data1 = data1.to(device) + data2 = data2.to(device) + y = data[0].y.view(-1, 1).long().to(device) + y = y.squeeze(1) + optimizer.zero_grad() + output = model(data1, data2) + loss = loss_fn(output, y) + # print('loss', loss) + loss.backward() + optimizer.step() + if batch_idx % LOG_INTERVAL == 0: + print('Train epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(epoch, + batch_idx * len(data1.x), + len(drug1_loader_train.dataset), + 100. * batch_idx / len(drug1_loader_train), + loss.item())) + + +def predicting(model, device, drug1_loader_test, drug2_loader_test): + model.eval() + total_preds = torch.Tensor() + total_labels = torch.Tensor() + total_prelabels = torch.Tensor() + print('Make prediction for {} samples...'.format(len(drug1_loader_test.dataset))) + with torch.no_grad(): + for data in zip(drug1_loader_test, drug2_loader_test): + data1 = data[0] + data2 = data[1] + data1 = data1.to(device) + data2 = data2.to(device) + output = model(data1, data2) + ys = F.softmax(output, 1).to('cpu').data.numpy() + predicted_labels = list(map(lambda x: np.argmax(x), ys)) + predicted_scores = list(map(lambda x: x[1], ys)) + total_preds = torch.cat((total_preds, torch.Tensor(predicted_scores)), 0) + total_prelabels = torch.cat((total_prelabels, torch.Tensor(predicted_labels)), 0) + total_labels = torch.cat((total_labels, data1.y.view(-1, 1).cpu()), 0) + return total_labels.numpy().flatten(), total_preds.numpy().flatten(), total_prelabels.numpy().flatten() + + +def shuffle_dataset(dataset, seed): + np.random.seed(seed) + np.random.shuffle(dataset) + return dataset + + +def split_dataset(dataset, ratio): + n = int(len(dataset) * ratio) + dataset_1, dataset_2 = dataset[:n], dataset[n:] + return dataset_1, dataset_2 + +# modeling = [GINConvNet, GATNet, GAT_GCN, GCNNet][int(sys.argv[2])] +# datasets = [['davis', 'kiba'][int(sys.argv[1])]] +# model_st = modeling.__name__ +modeling = GCNNet + +TRAIN_BATCH_SIZE = 256 +TEST_BATCH_SIZE = 256 +LR = 0.0005 +LOG_INTERVAL = 20 +NUM_EPOCHS = 100 + +print('Learning rate: ', LR) +print('Epochs: ', NUM_EPOCHS) +datafile = 'new_labels_0_10' + +# CPU or GPU +import os +os.environ["CUDA_VISIBLE_DEVICES"] = "1" + +if torch.cuda.is_available(): + device = torch.device('cuda') + print('The code uses GPU...') +else: + device = torch.device('cpu') + print('The code uses CPU!!!') + +drug1_data = TestbedDataset(root=artifact_path("cache"), dataset=datafile + '_drug1') +drug2_data = TestbedDataset(root=artifact_path("cache"), dataset=datafile + '_drug2') + +ind_datafile='independent_set/independent_input' +ind_drug1_data = TestbedDataset(root=artifact_path("cache"), dataset=ind_datafile + '_drug1') +ind_drug2_data = TestbedDataset(root=artifact_path("cache"), dataset=ind_datafile + '_drug2') + +lenth = len(drug1_data) +pot = int(lenth/5) +print('lenth', lenth) +print('pot', pot) + +domain_gen=True +random_num = random.sample(range(0, lenth), lenth) +for i in range(5): + if domain_gen: + drug1_data_train = drug1_data + drug1_data_test = ind_drug1_data + + drug2_data_train = drug2_data + drug2_data_test = ind_drug2_data + else: + + test_num = random_num[pot*i:pot*(i+1)] + train_num = random_num[:pot*i] + random_num[pot*(i+1):] + + drug1_data_train = drug1_data[train_num] + drug1_data_test = drug1_data[test_num] + + drug2_data_test = drug2_data[test_num] + drug2_data_train = drug2_data[train_num] + # print('type(drug1_data_train)', type(drug1_data_train)) + # print('drug1_data_train[0]', drug1_data_train[0]) + # print('len(drug1_data_train)', len(drug1_data_train)) + drug1_loader_train = DataLoader(drug1_data_train, batch_size=TRAIN_BATCH_SIZE, shuffle=None) + drug1_loader_test = DataLoader(drug1_data_test, batch_size=TRAIN_BATCH_SIZE, shuffle=None) + + + + drug2_loader_train = DataLoader(drug2_data_train, batch_size=TRAIN_BATCH_SIZE, shuffle=None) + drug2_loader_test = DataLoader(drug2_data_test, batch_size=TRAIN_BATCH_SIZE, shuffle=None) + + model = modeling().to(device) + loss_fn = nn.CrossEntropyLoss() + optimizer = torch.optim.Adam(model.parameters(), lr=LR) + + + # model_file_name = 'data/result/GCNNet(DrugA_DrugB)' + str(i) + '--model_' + datafile + '.model' + # result_file_name = 'data/result/GCNNet(DrugA_DrugB)' + str(i) + '--result_' + datafile + '.csv' + # file_AUCs = 'data/result/GCNNet(DrugA_DrugB)' + str(i) + '--AUCs--' + datafile + '.txt' + # AUCs = ('Epoch\tAUC_dev\tPR_AUC\tACC\tBACC\tPREC\tTPR\tKAPPA\tRECALL') + # with open(file_AUCs, 'w') as f: + # f.write(AUCs + '\n') + + best_auc = 0 + for epoch in range(NUM_EPOCHS): + train(model, device, drug1_loader_train, drug2_loader_train, optimizer, epoch + 1) + T, S, Y = predicting(model, device, drug1_loader_test, drug2_loader_test) + # T is correct label + # S is predict score + # Y is predict label + #print(T,S,Y) + # compute preformence + try: + AUC = roc_auc_score(T, S) + except: + AUC=0 + precision, recall, threshold = metrics.precision_recall_curve(T, S) + PR_AUC = metrics.auc(recall, precision) + BACC = balanced_accuracy_score(T, Y) + tn, fp, fn, tp = confusion_matrix(T, Y).ravel() + TPR = tp / (tp + fn) + PREC = precision_score(T, Y) + ACC = accuracy_score(T, Y) + KAPPA = cohen_kappa_score(T, Y) + recall = recall_score(T, Y) + + # save data + AUCs = [epoch, AUC, PR_AUC, ACC, BACC, PREC, TPR, KAPPA, recall] + # save_AUCs(AUCs, file_AUCs) + ret = [rmse(T, S), mse(T, S), pearson(T, S), spearman(T, S), ci(T, S)] + if best_auc < AUC: + best_auc = AUC + print(best_auc) + # torch.save(model.state_dict(), model_file_name) + # independent_num = [] + # independent_num.append(test_num) + # independent_num.append(T) + # independent_num.append(Y) + # independent_num.append(S) + # txtDF = pd.DataFrame(data=independent_num) + # txtDF.to_csv(result_file_name, index=False, header=False) + + diff --git a/data/downstream/final_data/drug_response_pytorch/drug_combination/training_GCN_r.py b/data/downstream/final_data/drug_response_pytorch/drug_combination/training_GCN_r.py index b6509e527c0aee2c69ce866f3742544e29b5a41d..c954bf2557294adf0295b20063dced6f97188e83 100644 --- a/data/downstream/final_data/drug_response_pytorch/drug_combination/training_GCN_r.py +++ b/data/downstream/final_data/drug_response_pytorch/drug_combination/training_GCN_r.py @@ -1,195 +1,196 @@ -import random -import torch.nn.functional as F -import torch.nn as nn -from models.gcn_r import GCNNet -from models.gat_gcn_test import GAT_GCN -from models.ginconv import GINConvNet -import math -from utils_test import * -from sklearn.metrics import roc_curve, confusion_matrix -from sklearn.metrics import cohen_kappa_score, accuracy_score, roc_auc_score, precision_score, recall_score, balanced_accuracy_score -from sklearn import metrics -import pandas as pd - - -# training function at each epoch -def train(model, device, drug1_loader_train, drug2_loader_train, optimizer, epoch): - print('Training on {} samples...'.format(len(drug1_loader_train.dataset))) - model.train() - # train_loader = np.array(train_loader) - for batch_idx, data in enumerate(zip(drug1_loader_train, drug2_loader_train)): - data1 = data[0] - data2 = data[1] - data1 = data1.to(device) - data2 = data2.to(device) - y = data[0].y.view(-1, 1).float().to(device) - y = y.squeeze(1) - output = model(data1, data2) - loss = loss_fn(output, y.unsqueeze(1)) - # print('loss', loss) - optimizer.zero_grad() - loss.backward() - optimizer.step() - if batch_idx % LOG_INTERVAL == 0: - print('Train epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(epoch, - batch_idx * len(data1.x), - len(drug1_loader_train.dataset), - 100. * batch_idx / len(drug1_loader_train), - loss.item())) - - -def predicting(model, device, drug1_loader_test, drug2_loader_test): - model.eval() - total_preds = torch.Tensor() - total_labels = torch.Tensor() - # total_prelabels = torch.Tensor() - print('Make prediction for {} samples...'.format(len(drug1_loader_test.dataset))) - with torch.no_grad(): - for data in zip(drug1_loader_test, drug2_loader_test): - data1 = data[0] - data2 = data[1] - data1 = data1.to(device) - data2 = data2.to(device) - output = model(data1, data2) - ys = output.to('cpu').data.numpy() - # predicted_labels = list(map(lambda x: np.argmax(x), ys)) - total_preds = torch.cat((total_preds, torch.Tensor(ys)), 0) - # total_prelabels = torch.cat((total_prelabels, torch.Tensor(predicted_labels)), 0) - total_labels = torch.cat((total_labels, data1.y.view(-1, 1).cpu()), 0) - return total_preds.numpy().flatten(), total_labels.numpy().flatten() - - -def shuffle_dataset(dataset, seed): - np.random.seed(seed) - np.random.shuffle(dataset) - return dataset - - -def split_dataset(dataset, ratio): - n = int(len(dataset) * ratio) - dataset_1, dataset_2 = dataset[:n], dataset[n:] - return dataset_1, dataset_2 - -def compute_mae_mse_rmse(target,prediction): - error = [] - for i in range(len(target)): - error.append(target[i] - prediction[i]) - squaredError = [] - absError = [] - for val in error: - squaredError.append(val * val) # target-prediction之差平方 - absError.append(abs(val)) # 误差绝对值 - mae=sum(absError)/len(absError) # 平均绝对误差MAE - mse=sum(squaredError)/len(squaredError) # 均方误差MSE - RMSE=sum(absError)/len(absError) - return mae,mse,RMSE - -def compute_rsquared(X, Y): - xBar = np.mean(X) - yBar = np.mean(Y) - SSR = 0 - varX = 0 - varY = 0 - for i in range(0, len(X)): - diffXXBar = X[i] - xBar - diffYYBar = Y[i] - yBar - SSR += (diffXXBar * diffYYBar) - varX += diffXXBar ** 2 - varY += diffYYBar ** 2 - - SST = math.sqrt(varX * varY) - r2=round((SSR / SST) ** 2,3) - return r2 - -# modeling = [GINConvNet, GATNet, GAT_GCN, GCNNet][int(sys.argv[2])] -# datasets = [['davis', 'kiba'][int(sys.argv[1])]] -# model_st = modeling.__name__ -modeling = GCNNet - -TRAIN_BATCH_SIZE = 128 -TEST_BATCH_SIZE = 128 -LR = 0.0001 -LOG_INTERVAL = 20 -NUM_EPOCHS = 1000 - -print('Learning rate: ', LR) -print('Epochs: ', NUM_EPOCHS) -datafile = 'score_0_10' - -# CPU or GPU - -if torch.cuda.is_available(): - device = torch.device('cuda') - print('The code uses GPU...') -else: - device = torch.device('cpu') - print('The code uses CPU!!!') - -drug1_data = TestbedDataset(root='data', dataset=datafile + '_drug1') -drug2_data = TestbedDataset(root='data', dataset=datafile + '_drug2') - -lenth = len(drug1_data) -pot = int(lenth/5) -print('lenth', lenth) -print('pot', pot) - - -random_num = random.sample(range(0, lenth), lenth) -for i in range(5): - test_num = random_num[pot*i:pot*(i+1)] - train_num = random_num[:pot*i] + random_num[pot*(i+1):] - - drug1_data_train = drug1_data[train_num] - drug1_data_test = drug1_data[test_num] - drug1_loader_train = DataLoader(drug1_data_train, batch_size=TRAIN_BATCH_SIZE, shuffle=None) - drug1_loader_test = DataLoader(drug1_data_test, batch_size=TRAIN_BATCH_SIZE, shuffle=None) - - - drug2_data_test = drug2_data[test_num] - drug2_data_train = drug2_data[train_num] - drug2_loader_train = DataLoader(drug2_data_train, batch_size=TRAIN_BATCH_SIZE, shuffle=None) - drug2_loader_test = DataLoader(drug2_data_test, batch_size=TRAIN_BATCH_SIZE, shuffle=None) - - model = modeling().to(device) - loss_fn = nn.MSELoss() - optimizer = torch.optim.Adam(model.parameters(), lr=LR) - - - # model_file_name = 'data/result/GATNet(DrugA_DrugB)' + str(i) + '--model_' + datafile + '.model' - result_file_name = 'data/result/GCNNet(DrugA_DrugB)_R' + str(i) + '--result_' + datafile + '.csv' - file_AUCs = 'data/result/GCNNet(DrugA_DrugB)_R' + str(i) + '--AUCs--' + datafile + '.txt' - AUCs = ('Epoch\tepoch\tmse\tmae\trmse\tr2\tP') - with open(file_AUCs, 'w') as f: - f.write(AUCs + '\n') - - best_auc = 10000000000000 - for epoch in range(NUM_EPOCHS): - train(model, device, drug1_loader_train, drug2_loader_train, optimizer, epoch + 1) - S, T = predicting(model, device, drug1_loader_test, drug2_loader_test) - # T is correct score - # S is predict score - # Y is predict label - print(S) - print(T) - - # compute preformence - - mae, mse, rmse = compute_mae_mse_rmse(S, T) - - # save data - - if best_auc > mse: - best_auc = mse - r2 = compute_rsquared(S, T) - AUCs = [epoch, mse, mae, rmse, r2, math.sqrt(r2)] - save_AUCs(AUCs, file_AUCs) - # torch.save(model.state_dict(), model_file_name) - independent_num = [] - # independent_num.append(test_num) - independent_num.append(T) - # independent_num.append(Y) - independent_num.append(S) - txtDF = pd.DataFrame(data=independent_num) - txtDF.to_csv(result_file_name, index=False, header=False) - - print('best_mse', best_auc) +from _reference_paths import ASSET_DIR, artifact_path, data_path, output_dir, output_path, required_artifact +import random +import torch.nn.functional as F +import torch.nn as nn +from models.gcn_r import GCNNet +from models.gat_gcn_test import GAT_GCN +from models.ginconv import GINConvNet +import math +from utils_test import * +from sklearn.metrics import roc_curve, confusion_matrix +from sklearn.metrics import cohen_kappa_score, accuracy_score, roc_auc_score, precision_score, recall_score, balanced_accuracy_score +from sklearn import metrics +import pandas as pd + + +# training function at each epoch +def train(model, device, drug1_loader_train, drug2_loader_train, optimizer, epoch): + print('Training on {} samples...'.format(len(drug1_loader_train.dataset))) + model.train() + # train_loader = np.array(train_loader) + for batch_idx, data in enumerate(zip(drug1_loader_train, drug2_loader_train)): + data1 = data[0] + data2 = data[1] + data1 = data1.to(device) + data2 = data2.to(device) + y = data[0].y.view(-1, 1).float().to(device) + y = y.squeeze(1) + output = model(data1, data2) + loss = loss_fn(output, y.unsqueeze(1)) + # print('loss', loss) + optimizer.zero_grad() + loss.backward() + optimizer.step() + if batch_idx % LOG_INTERVAL == 0: + print('Train epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(epoch, + batch_idx * len(data1.x), + len(drug1_loader_train.dataset), + 100. * batch_idx / len(drug1_loader_train), + loss.item())) + + +def predicting(model, device, drug1_loader_test, drug2_loader_test): + model.eval() + total_preds = torch.Tensor() + total_labels = torch.Tensor() + # total_prelabels = torch.Tensor() + print('Make prediction for {} samples...'.format(len(drug1_loader_test.dataset))) + with torch.no_grad(): + for data in zip(drug1_loader_test, drug2_loader_test): + data1 = data[0] + data2 = data[1] + data1 = data1.to(device) + data2 = data2.to(device) + output = model(data1, data2) + ys = output.to('cpu').data.numpy() + # predicted_labels = list(map(lambda x: np.argmax(x), ys)) + total_preds = torch.cat((total_preds, torch.Tensor(ys)), 0) + # total_prelabels = torch.cat((total_prelabels, torch.Tensor(predicted_labels)), 0) + total_labels = torch.cat((total_labels, data1.y.view(-1, 1).cpu()), 0) + return total_preds.numpy().flatten(), total_labels.numpy().flatten() + + +def shuffle_dataset(dataset, seed): + np.random.seed(seed) + np.random.shuffle(dataset) + return dataset + + +def split_dataset(dataset, ratio): + n = int(len(dataset) * ratio) + dataset_1, dataset_2 = dataset[:n], dataset[n:] + return dataset_1, dataset_2 + +def compute_mae_mse_rmse(target,prediction): + error = [] + for i in range(len(target)): + error.append(target[i] - prediction[i]) + squaredError = [] + absError = [] + for val in error: + squaredError.append(val * val) # Squared target-prediction error. + absError.append(abs(val)) # Absolute error. + mae=sum(absError)/len(absError) # Mean absolute error (MAE). + mse=sum(squaredError)/len(squaredError) # Mean squared error (MSE). + RMSE=sum(absError)/len(absError) + return mae,mse,RMSE + +def compute_rsquared(X, Y): + xBar = np.mean(X) + yBar = np.mean(Y) + SSR = 0 + varX = 0 + varY = 0 + for i in range(0, len(X)): + diffXXBar = X[i] - xBar + diffYYBar = Y[i] - yBar + SSR += (diffXXBar * diffYYBar) + varX += diffXXBar ** 2 + varY += diffYYBar ** 2 + + SST = math.sqrt(varX * varY) + r2=round((SSR / SST) ** 2,3) + return r2 + +# modeling = [GINConvNet, GATNet, GAT_GCN, GCNNet][int(sys.argv[2])] +# datasets = [['davis', 'kiba'][int(sys.argv[1])]] +# model_st = modeling.__name__ +modeling = GCNNet + +TRAIN_BATCH_SIZE = 128 +TEST_BATCH_SIZE = 128 +LR = 0.0001 +LOG_INTERVAL = 20 +NUM_EPOCHS = 1000 + +print('Learning rate: ', LR) +print('Epochs: ', NUM_EPOCHS) +datafile = 'score_0_10' + +# CPU or GPU + +if torch.cuda.is_available(): + device = torch.device('cuda') + print('The code uses GPU...') +else: + device = torch.device('cpu') + print('The code uses CPU!!!') + +drug1_data = TestbedDataset(root=artifact_path("cache"), dataset=datafile + '_drug1') +drug2_data = TestbedDataset(root=artifact_path("cache"), dataset=datafile + '_drug2') + +lenth = len(drug1_data) +pot = int(lenth/5) +print('lenth', lenth) +print('pot', pot) + + +random_num = random.sample(range(0, lenth), lenth) +for i in range(5): + test_num = random_num[pot*i:pot*(i+1)] + train_num = random_num[:pot*i] + random_num[pot*(i+1):] + + drug1_data_train = drug1_data[train_num] + drug1_data_test = drug1_data[test_num] + drug1_loader_train = DataLoader(drug1_data_train, batch_size=TRAIN_BATCH_SIZE, shuffle=None) + drug1_loader_test = DataLoader(drug1_data_test, batch_size=TRAIN_BATCH_SIZE, shuffle=None) + + + drug2_data_test = drug2_data[test_num] + drug2_data_train = drug2_data[train_num] + drug2_loader_train = DataLoader(drug2_data_train, batch_size=TRAIN_BATCH_SIZE, shuffle=None) + drug2_loader_test = DataLoader(drug2_data_test, batch_size=TRAIN_BATCH_SIZE, shuffle=None) + + model = modeling().to(device) + loss_fn = nn.MSELoss() + optimizer = torch.optim.Adam(model.parameters(), lr=LR) + + + # model_file_name = 'data/result/GATNet(DrugA_DrugB)' + str(i) + '--model_' + datafile + '.model' + result_file_name = output_path('result/GCNNet(DrugA_DrugB)_R') + str(i) + '--result_' + datafile + '.csv' + file_AUCs = output_path('result/GCNNet(DrugA_DrugB)_R') + str(i) + '--AUCs--' + datafile + '.txt' + AUCs = ('Epoch\tepoch\tmse\tmae\trmse\tr2\tP') + with open(file_AUCs, 'w') as f: + f.write(AUCs + '\n') + + best_auc = 10000000000000 + for epoch in range(NUM_EPOCHS): + train(model, device, drug1_loader_train, drug2_loader_train, optimizer, epoch + 1) + S, T = predicting(model, device, drug1_loader_test, drug2_loader_test) + # T is correct score + # S is predict score + # Y is predict label + print(S) + print(T) + + # compute preformence + + mae, mse, rmse = compute_mae_mse_rmse(S, T) + + # save data + + if best_auc > mse: + best_auc = mse + r2 = compute_rsquared(S, T) + AUCs = [epoch, mse, mae, rmse, r2, math.sqrt(r2)] + save_AUCs(AUCs, file_AUCs) + # torch.save(model.state_dict(), model_file_name) + independent_num = [] + # independent_num.append(test_num) + independent_num.append(T) + # independent_num.append(Y) + independent_num.append(S) + txtDF = pd.DataFrame(data=independent_num) + txtDF.to_csv(result_file_name, index=False, header=False) + + print('best_mse', best_auc) diff --git a/data/downstream/final_data/drug_response_pytorch/drug_combination/training_GCN_scfm.py b/data/downstream/final_data/drug_response_pytorch/drug_combination/training_GCN_scfm.py index c286502a0625fd3db1231bc228f0e090064492b6..292d8feca490208c2e783af24eb09f4e686512b1 100644 --- a/data/downstream/final_data/drug_response_pytorch/drug_combination/training_GCN_scfm.py +++ b/data/downstream/final_data/drug_response_pytorch/drug_combination/training_GCN_scfm.py @@ -1,248 +1,249 @@ -import random -import numpy as np -import pandas as pd -import sys, os -from random import shuffle -import torch.utils.data as Data -import torch -import torch.nn.functional as F -import torch.nn as nn -import pickle - -from torch.utils.data import TensorDataset, Dataset -from models.gat import GATNet -from models.gat_gcn_test import GAT_GCN -from models.gcn import GCNNet -from models.ginconv import GINConvNet -from utils_test import * -from sklearn.metrics import roc_curve, confusion_matrix -from sklearn.metrics import cohen_kappa_score, accuracy_score, roc_auc_score, precision_score, recall_score, balanced_accuracy_score -from sklearn import metrics -from sklearn.model_selection import StratifiedKFold, KFold -from torch.optim.lr_scheduler import StepLR -import pandas as pd - -# CPU or GPU -import os -os.environ["CUDA_VISIBLE_DEVICES"] = "0" - -if torch.cuda.is_available(): - device = torch.device('cuda') - print('The code uses GPU...') -else: - device = torch.device('cpu') - print('The code uses CPU!!!') - -# training function at each epoch -def train(model, device, drug1_loader_train, drug2_loader_train, optimizer, epoch): - print('Training on {} samples...'.format(len(drug1_loader_train.dataset))) - model.train() - # train_loader = np.array(train_loader) - for batch_idx, data in enumerate(zip(drug1_loader_train, drug2_loader_train)): - data1 = data[0] - data2 = data[1] - data1 = data1.to(device) - data2 = data2.to(device) - y = data[0].y.view(-1, 1).long().to(device) - y = y.squeeze(1) - optimizer.zero_grad() - output = model(data1, data2) - loss = loss_fn(output, y) - # print('loss', loss) - loss.backward() - optimizer.step() - if batch_idx % LOG_INTERVAL == 0: - print('Train epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(epoch, - batch_idx * len(data1.x), - len(drug1_loader_train.dataset), - 100. * batch_idx / len(drug1_loader_train), - loss.item())) - - -def predicting(model, device, drug1_loader_test, drug2_loader_test): - model.eval() - total_preds = torch.Tensor() - total_labels = torch.Tensor() - total_prelabels = torch.Tensor() - print('Make prediction for {} samples...'.format(len(drug1_loader_test.dataset))) - with torch.no_grad(): - for data in zip(drug1_loader_test, drug2_loader_test): - data1 = data[0] - data2 = data[1] - data1 = data1.to(device) - data2 = data2.to(device) - output = model(data1, data2) - ys = F.softmax(output, 1).to('cpu').data.numpy() - predicted_labels = list(map(lambda x: np.argmax(x), ys)) - predicted_scores = list(map(lambda x: x[1], ys)) - total_preds = torch.cat((total_preds, torch.Tensor(predicted_scores)), 0) - total_prelabels = torch.cat((total_prelabels, torch.Tensor(predicted_labels)), 0) - total_labels = torch.cat((total_labels, data1.y.view(-1, 1).cpu()), 0) - return total_labels.numpy().flatten(), total_preds.numpy().flatten(), total_prelabels.numpy().flatten() - - -def shuffle_dataset(dataset, seed): - np.random.seed(seed) - np.random.shuffle(dataset) - return dataset - - -def split_dataset(dataset, ratio): - n = int(len(dataset) * ratio) - dataset_1, dataset_2 = dataset[:n], dataset[n:] - return dataset_1, dataset_2 - -# modeling = [GINConvNet, GATNet, GAT_GCN, GCNNet][int(sys.argv[2])] -# datasets = [['davis', 'kiba'][int(sys.argv[1])]] -# model_st = modeling.__name__ -modeling = GCNNet - -TRAIN_BATCH_SIZE = 256 -TEST_BATCH_SIZE = 256 -LR = 0.0001 -LOG_INTERVAL = 1 -NUM_EPOCHS = 100 - -print('Learning rate: ', LR) -print('Epochs: ', NUM_EPOCHS) -datafile = 'new_labels_0_10' - - - -if scfm and not geneformer: - drug1_data = TestbedDataset(root='/data1/ruiyi/deepdds/data', dataset=datafile + '_drug1_combined_{}'.format(ckpt)) - drug2_data = TestbedDataset(root='/data1/ruiyi/deepdds/data', dataset=datafile + '_drug2_combined_{}'.format(ckpt)) - ind_datafile='independent_set/independent_input' - ind_drug1_data = TestbedDataset(root='/data1/ruiyi/deepdds/data', dataset=ind_datafile + '_drug1_combined_{}'.format(ckpt)) - ind_drug2_data = TestbedDataset(root='/data1/ruiyi/deepdds/data', dataset=ind_datafile + '_drug2_combined_{}'.format(ckpt)) -elif geneformer: - name = ckpt - drug1_data = TestbedDataset(root='/data1/ruiyi/deepdds/data', dataset=datafile + '_drug1_combined_{}'.format(name)) - drug2_data = TestbedDataset(root='/data1/ruiyi/deepdds/data', dataset=datafile + '_drug2_combined_{}'.format(name)) - ind_datafile='independent_set/independent_input' - ind_drug1_data = TestbedDataset(root='/data1/ruiyi/deepdds/data', dataset=ind_datafile + '_drug1_combined_{}'.format(name)) - ind_drug2_data = TestbedDataset(root='/data1/ruiyi/deepdds/data', dataset=ind_datafile + '_drug2_combined_{}'.format(name)) -else: - drug1_data = TestbedDataset(root='/data1/ruiyi/deepdds/data', dataset=datafile + '_drug1_bsl') - drug2_data = TestbedDataset(root='/data1/ruiyi/deepdds/data', dataset=datafile + '_drug2_bsl') - ind_datafile='independent_set/independent_input' - ind_drug1_data = TestbedDataset(root='/data1/ruiyi/deepdds/data', dataset=ind_datafile + '_drug1_bsl') - ind_drug2_data = TestbedDataset(root='/data1/ruiyi/deepdds/data', dataset=ind_datafile + '_drug2_bsl') - - -lenth = len(drug1_data) -pot = int(lenth/5) -print('lenth', lenth) -print('pot', pot) - -few_shot=False -domain_gen=True -random_num = random.sample(range(0, lenth), lenth) -for i in range(5): - if domain_gen: - drug1_data_train = drug1_data - drug1_data_test = ind_drug1_data - - drug2_data_train = drug2_data - drug2_data_test = ind_drug2_data - else: - if few_shot: - train_num = random_num[pot*i:pot*(i+1)] - test_num = random_num[:pot*i] + random_num[pot*(i+1):] - else: - test_num = random_num[pot*i:pot*(i+1)] - train_num = random_num[:pot*i] + random_num[pot*(i+1):] - - - drug1_data_train = drug1_data[train_num] - drug1_data_test = drug1_data[test_num] - - drug2_data_test = drug2_data[test_num] - drug2_data_train = drug2_data[train_num] - # print('type(drug1_data_train)', type(drug1_data_train)) - # print('drug1_data_train[0]', drug1_data_train[0]) - # print('len(drug1_data_train)', len(drug1_data_train)) - drug1_loader_train = DataLoader(drug1_data_train, batch_size=TRAIN_BATCH_SIZE, shuffle=None, pin_memory=True) - drug1_loader_test = DataLoader(drug1_data_test, batch_size=TRAIN_BATCH_SIZE, shuffle=None, pin_memory=True) - - - - drug2_loader_train = DataLoader(drug2_data_train, batch_size=TRAIN_BATCH_SIZE, shuffle=None, pin_memory=True) - drug2_loader_test = DataLoader(drug2_data_test, batch_size=TRAIN_BATCH_SIZE, shuffle=None, pin_memory=True) - - model = modeling().to(device) - for name, param in model.named_parameters(): - print(f"Parameter Name: {name}, Shape: {param.size()}") - total_params = sum(p.numel() for p in model.parameters()) - print("Total number of parameters:", total_params) - - loss_fn = nn.CrossEntropyLoss() - optimizer = torch.optim.Adam(model.parameters(), lr=LR) - # Create a StepLR scheduler - scheduler = StepLR(optimizer, step_size=1000, gamma=0.9) - - - # model_file_name = 'data/result/GCNNet(DrugA_DrugB)' + str(i) + '--model_' + datafile + '.model' - # result_file_name = 'data/result/GCNNet(DrugA_DrugB)' + str(i) + '--result_' + datafile + '.csv' - # file_AUCs = 'data/result/GCNNet(DrugA_DrugB)' + str(i) + '--AUCs--' + datafile + '.txt' - # AUCs = ('Epoch\tAUC_dev\tPR_AUC\tACC\tBACC\tPREC\tTPR\tKAPPA\tRECALL') - # with open(file_AUCs, 'w') as f: - # f.write(AUCs + '\n') - - best_auc = 0 - for epoch in range(NUM_EPOCHS): - train(model, device, drug1_loader_train, drug2_loader_train, optimizer, epoch + 1) - scheduler.step() - if epoch%5==0: - T, S, Y = predicting(model, device, drug1_loader_test, drug2_loader_test) - # T is correct label - # S is predict score - # Y is predict label - #print(T,S,Y) - # compute preformence - try: - AUC = roc_auc_score(T, S) - except: - AUC=0 - fpr, tpr, thres = metrics.roc_curve(T, S) - - precision, recall, threshold = metrics.precision_recall_curve(T, S) - PR_AUC = metrics.auc(recall, precision) - BACC = balanced_accuracy_score(T, Y) - tn, fp, fn, tp = confusion_matrix(T, Y).ravel() - TPR = tp / (tp + fn) - PREC = precision_score(T, Y) - ACC = accuracy_score(T, Y) - KAPPA = cohen_kappa_score(T, Y) - recall = recall_score(T, Y) - - # save data - AUCs = [epoch, AUC, PR_AUC, ACC, BACC, PREC, TPR, KAPPA, recall] - # save_AUCs(AUCs, file_AUCs) - ret = [rmse(T, S), mse(T, S), pearson(T, S), spearman(T, S), ci(T, S)] - if best_auc < AUC: - best_auc = AUC - print('best_auc:',best_auc) - print(fpr) - print(tpr) - print(thres) - print(AUCs) - print(ret) - ''' - # torch.save(model.state_dict(), model_file_name) - independent_num = [] - independent_num.append(test_num) - independent_num.append(T) - independent_num.append(Y) - independent_num.append(S) - txtDF = pd.DataFrame(data=independent_num) - result_file_name='gcn_scfm_results.csv' - txtDF.to_csv(result_file_name, index=False, header=False) - ''' - if scfm: - with open('fpr_tpr/fpr_tpr_scfm_run{}_{}.pkl'.format(str(i),ckpt),'wb') as fp: - pickle.dump((fpr,tpr), fp) - else: - with open('fpr_tpr/fpr_tpr_bsl_run{}_{}.pkl'.format(str(i),ckpt),'wb') as fp: - pickle.dump((fpr,tpr), fp) - +from _reference_paths import ASSET_DIR, artifact_path, data_path, output_dir, output_path, required_artifact +import random +import numpy as np +import pandas as pd +import sys, os +from random import shuffle +import torch.utils.data as Data +import torch +import torch.nn.functional as F +import torch.nn as nn +import pickle + +from torch.utils.data import TensorDataset, Dataset +from models.gat import GATNet +from models.gat_gcn_test import GAT_GCN +from models.gcn import GCNNet +from models.ginconv import GINConvNet +from utils_test import * +from sklearn.metrics import roc_curve, confusion_matrix +from sklearn.metrics import cohen_kappa_score, accuracy_score, roc_auc_score, precision_score, recall_score, balanced_accuracy_score +from sklearn import metrics +from sklearn.model_selection import StratifiedKFold, KFold +from torch.optim.lr_scheduler import StepLR +import pandas as pd + +# CPU or GPU +import os +os.environ["CUDA_VISIBLE_DEVICES"] = "0" + +if torch.cuda.is_available(): + device = torch.device('cuda') + print('The code uses GPU...') +else: + device = torch.device('cpu') + print('The code uses CPU!!!') + +# training function at each epoch +def train(model, device, drug1_loader_train, drug2_loader_train, optimizer, epoch): + print('Training on {} samples...'.format(len(drug1_loader_train.dataset))) + model.train() + # train_loader = np.array(train_loader) + for batch_idx, data in enumerate(zip(drug1_loader_train, drug2_loader_train)): + data1 = data[0] + data2 = data[1] + data1 = data1.to(device) + data2 = data2.to(device) + y = data[0].y.view(-1, 1).long().to(device) + y = y.squeeze(1) + optimizer.zero_grad() + output = model(data1, data2) + loss = loss_fn(output, y) + # print('loss', loss) + loss.backward() + optimizer.step() + if batch_idx % LOG_INTERVAL == 0: + print('Train epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(epoch, + batch_idx * len(data1.x), + len(drug1_loader_train.dataset), + 100. * batch_idx / len(drug1_loader_train), + loss.item())) + + +def predicting(model, device, drug1_loader_test, drug2_loader_test): + model.eval() + total_preds = torch.Tensor() + total_labels = torch.Tensor() + total_prelabels = torch.Tensor() + print('Make prediction for {} samples...'.format(len(drug1_loader_test.dataset))) + with torch.no_grad(): + for data in zip(drug1_loader_test, drug2_loader_test): + data1 = data[0] + data2 = data[1] + data1 = data1.to(device) + data2 = data2.to(device) + output = model(data1, data2) + ys = F.softmax(output, 1).to('cpu').data.numpy() + predicted_labels = list(map(lambda x: np.argmax(x), ys)) + predicted_scores = list(map(lambda x: x[1], ys)) + total_preds = torch.cat((total_preds, torch.Tensor(predicted_scores)), 0) + total_prelabels = torch.cat((total_prelabels, torch.Tensor(predicted_labels)), 0) + total_labels = torch.cat((total_labels, data1.y.view(-1, 1).cpu()), 0) + return total_labels.numpy().flatten(), total_preds.numpy().flatten(), total_prelabels.numpy().flatten() + + +def shuffle_dataset(dataset, seed): + np.random.seed(seed) + np.random.shuffle(dataset) + return dataset + + +def split_dataset(dataset, ratio): + n = int(len(dataset) * ratio) + dataset_1, dataset_2 = dataset[:n], dataset[n:] + return dataset_1, dataset_2 + +# modeling = [GINConvNet, GATNet, GAT_GCN, GCNNet][int(sys.argv[2])] +# datasets = [['davis', 'kiba'][int(sys.argv[1])]] +# model_st = modeling.__name__ +modeling = GCNNet + +TRAIN_BATCH_SIZE = 256 +TEST_BATCH_SIZE = 256 +LR = 0.0001 +LOG_INTERVAL = 1 +NUM_EPOCHS = 100 + +print('Learning rate: ', LR) +print('Epochs: ', NUM_EPOCHS) +datafile = 'new_labels_0_10' + + + +if scfm and not geneformer: + drug1_data = TestbedDataset(root=artifact_path("cache"), dataset=datafile + '_drug1_combined_{}'.format(ckpt)) + drug2_data = TestbedDataset(root=artifact_path("cache"), dataset=datafile + '_drug2_combined_{}'.format(ckpt)) + ind_datafile='independent_set/independent_input' + ind_drug1_data = TestbedDataset(root=artifact_path("cache"), dataset=ind_datafile + '_drug1_combined_{}'.format(ckpt)) + ind_drug2_data = TestbedDataset(root=artifact_path("cache"), dataset=ind_datafile + '_drug2_combined_{}'.format(ckpt)) +elif geneformer: + name = ckpt + drug1_data = TestbedDataset(root=artifact_path("cache"), dataset=datafile + '_drug1_combined_{}'.format(name)) + drug2_data = TestbedDataset(root=artifact_path("cache"), dataset=datafile + '_drug2_combined_{}'.format(name)) + ind_datafile='independent_set/independent_input' + ind_drug1_data = TestbedDataset(root=artifact_path("cache"), dataset=ind_datafile + '_drug1_combined_{}'.format(name)) + ind_drug2_data = TestbedDataset(root=artifact_path("cache"), dataset=ind_datafile + '_drug2_combined_{}'.format(name)) +else: + drug1_data = TestbedDataset(root=artifact_path("cache"), dataset=datafile + '_drug1_bsl') + drug2_data = TestbedDataset(root=artifact_path("cache"), dataset=datafile + '_drug2_bsl') + ind_datafile='independent_set/independent_input' + ind_drug1_data = TestbedDataset(root=artifact_path("cache"), dataset=ind_datafile + '_drug1_bsl') + ind_drug2_data = TestbedDataset(root=artifact_path("cache"), dataset=ind_datafile + '_drug2_bsl') + + +lenth = len(drug1_data) +pot = int(lenth/5) +print('lenth', lenth) +print('pot', pot) + +few_shot=False +domain_gen=True +random_num = random.sample(range(0, lenth), lenth) +for i in range(5): + if domain_gen: + drug1_data_train = drug1_data + drug1_data_test = ind_drug1_data + + drug2_data_train = drug2_data + drug2_data_test = ind_drug2_data + else: + if few_shot: + train_num = random_num[pot*i:pot*(i+1)] + test_num = random_num[:pot*i] + random_num[pot*(i+1):] + else: + test_num = random_num[pot*i:pot*(i+1)] + train_num = random_num[:pot*i] + random_num[pot*(i+1):] + + + drug1_data_train = drug1_data[train_num] + drug1_data_test = drug1_data[test_num] + + drug2_data_test = drug2_data[test_num] + drug2_data_train = drug2_data[train_num] + # print('type(drug1_data_train)', type(drug1_data_train)) + # print('drug1_data_train[0]', drug1_data_train[0]) + # print('len(drug1_data_train)', len(drug1_data_train)) + drug1_loader_train = DataLoader(drug1_data_train, batch_size=TRAIN_BATCH_SIZE, shuffle=None, pin_memory=True) + drug1_loader_test = DataLoader(drug1_data_test, batch_size=TRAIN_BATCH_SIZE, shuffle=None, pin_memory=True) + + + + drug2_loader_train = DataLoader(drug2_data_train, batch_size=TRAIN_BATCH_SIZE, shuffle=None, pin_memory=True) + drug2_loader_test = DataLoader(drug2_data_test, batch_size=TRAIN_BATCH_SIZE, shuffle=None, pin_memory=True) + + model = modeling().to(device) + for name, param in model.named_parameters(): + print(f"Parameter Name: {name}, Shape: {param.size()}") + total_params = sum(p.numel() for p in model.parameters()) + print("Total number of parameters:", total_params) + + loss_fn = nn.CrossEntropyLoss() + optimizer = torch.optim.Adam(model.parameters(), lr=LR) + # Create a StepLR scheduler + scheduler = StepLR(optimizer, step_size=1000, gamma=0.9) + + + # model_file_name = 'data/result/GCNNet(DrugA_DrugB)' + str(i) + '--model_' + datafile + '.model' + # result_file_name = 'data/result/GCNNet(DrugA_DrugB)' + str(i) + '--result_' + datafile + '.csv' + # file_AUCs = 'data/result/GCNNet(DrugA_DrugB)' + str(i) + '--AUCs--' + datafile + '.txt' + # AUCs = ('Epoch\tAUC_dev\tPR_AUC\tACC\tBACC\tPREC\tTPR\tKAPPA\tRECALL') + # with open(file_AUCs, 'w') as f: + # f.write(AUCs + '\n') + + best_auc = 0 + for epoch in range(NUM_EPOCHS): + train(model, device, drug1_loader_train, drug2_loader_train, optimizer, epoch + 1) + scheduler.step() + if epoch%5==0: + T, S, Y = predicting(model, device, drug1_loader_test, drug2_loader_test) + # T is correct label + # S is predict score + # Y is predict label + #print(T,S,Y) + # compute preformence + try: + AUC = roc_auc_score(T, S) + except: + AUC=0 + fpr, tpr, thres = metrics.roc_curve(T, S) + + precision, recall, threshold = metrics.precision_recall_curve(T, S) + PR_AUC = metrics.auc(recall, precision) + BACC = balanced_accuracy_score(T, Y) + tn, fp, fn, tp = confusion_matrix(T, Y).ravel() + TPR = tp / (tp + fn) + PREC = precision_score(T, Y) + ACC = accuracy_score(T, Y) + KAPPA = cohen_kappa_score(T, Y) + recall = recall_score(T, Y) + + # save data + AUCs = [epoch, AUC, PR_AUC, ACC, BACC, PREC, TPR, KAPPA, recall] + # save_AUCs(AUCs, file_AUCs) + ret = [rmse(T, S), mse(T, S), pearson(T, S), spearman(T, S), ci(T, S)] + if best_auc < AUC: + best_auc = AUC + print('best_auc:',best_auc) + print(fpr) + print(tpr) + print(thres) + print(AUCs) + print(ret) + ''' + # torch.save(model.state_dict(), model_file_name) + independent_num = [] + independent_num.append(test_num) + independent_num.append(T) + independent_num.append(Y) + independent_num.append(S) + txtDF = pd.DataFrame(data=independent_num) + result_file_name='gcn_scfm_results.csv' + txtDF.to_csv(result_file_name, index=False, header=False) + ''' + if scfm: + with open(output_path('fpr_tpr/fpr_tpr_scfm_run{}_{}.pkl').format(str(i),ckpt),'wb') as fp: + pickle.dump((fpr,tpr), fp) + else: + with open(output_path('fpr_tpr/fpr_tpr_bsl_run{}_{}.pkl').format(str(i),ckpt),'wb') as fp: + pickle.dump((fpr,tpr), fp) + diff --git a/data/downstream/final_data/drug_response_pytorch/drug_combination/training_validation.py b/data/downstream/final_data/drug_response_pytorch/drug_combination/training_validation.py index 1ac4f4de2dd68f12c9c57c0110a0e6e6c3375c17..78de6126227d40b81316e0709f61046afaed55bd 100644 --- a/data/downstream/final_data/drug_response_pytorch/drug_combination/training_validation.py +++ b/data/downstream/final_data/drug_response_pytorch/drug_combination/training_validation.py @@ -1,189 +1,190 @@ -import random -import numpy as np -import pandas as pd -import sys, os -from random import shuffle -import torch.utils.data as Data -import torch -import torch.nn.functional as F -import torch.nn as nn -from torch.utils.data import TensorDataset, Dataset -from models.gat import GATNet -from models.gat_gcn_test import GAT_GCN -from models.gcn import GCNNet -from models.ginconv import GINConvNet -from utils_test import * -from sklearn.metrics import roc_curve, confusion_matrix -from sklearn.metrics import cohen_kappa_score, accuracy_score, roc_auc_score, precision_score, recall_score, balanced_accuracy_score -from sklearn import metrics -from sklearn.model_selection import StratifiedKFold, KFold -import pandas as pd -import matplotlib.pyplot as plt -import seaborn as sns - - -# training function at each epoch -def train(model, device, drug1_loader_train, drug2_loader_train, optimizer, epoch): - print('Training on {} samples...'.format(len(drug1_loader_train.dataset))) - model.train() - # train_loader = np.array(train_loader) - for batch_idx, data in enumerate(zip(drug1_loader_train, drug2_loader_train)): - data1 = data[0] - data2 = data[1] - data1 = data1.to(device) - data2 = data2.to(device) - y = data[0].y.view(-1, 1).long().to(device) - y = y.squeeze(1) - optimizer.zero_grad() - output = model(data1, data2) - loss = loss_fn(output, y) - # print('loss', loss) - loss.backward() - optimizer.step() - if batch_idx % LOG_INTERVAL == 0: - print('Train epoch: {} Loss: {:.6f}'.format(epoch, loss.item())) - - -def predicting(model, device, drug1_loader_test, drug2_loader_test): - model.eval() - total_preds = torch.Tensor() - total_labels = torch.Tensor() - total_prelabels = torch.Tensor() - print('Make prediction for {} samples...'.format(len(drug1_loader_test.dataset))) - with torch.no_grad(): - for data in zip(drug1_loader_test, drug2_loader_test): - data1 = data[0] - data2 = data[1] - data1 = data1.to(device) - data2 = data2.to(device) - output = model(data1, data2) - ys = F.softmax(output, 1).to('cpu').data.numpy() - predicted_labels = list(map(lambda x: np.argmax(x), ys)) - predicted_scores = list(map(lambda x: x[1], ys)) - total_preds = torch.cat((total_preds, torch.Tensor(predicted_scores)), 0) - total_prelabels = torch.cat((total_prelabels, torch.Tensor(predicted_labels)), 0) - # total_labels = torch.cat((total_labels, data1.y.view(-1, 1).cpu()), 0) - return total_labels.numpy().flatten(), total_preds.numpy().flatten(), total_prelabels.numpy().flatten() - - -# modeling = [GINConvNet, GATNet, GAT_GCN, GCNNet][int(sys.argv[2])] - - - -TRAIN_BATCH_SIZE = 256 -TEST_BATCH_SIZE = 256 -LR = 0.0001 -LOG_INTERVAL = 20 -NUM_EPOCHS = 100 - -print('Learning rate: ', LR) -print('Epochs: ', NUM_EPOCHS) -file = ['new_labels_0_10_leave'] - -# CPU or GPU - -if torch.cuda.is_available(): - device = torch.device('cuda') - print('The code uses GPU...') -else: - device = torch.device('cpu') - print('The code uses CPU!!!') - -for datafile in file: - - independent_drug1_data = TestbedDataset(root='data', dataset=datafile + '_drug1') - independent_drug2_data = TestbedDataset(root='data', dataset=datafile + '_drug2') - print('independent_drug1_data[0]', independent_drug1_data[0]) - lenth = len(independent_drug1_data) - - - - drug1_data = TestbedDataset(root='data', dataset='new_labels_0_10_drug1') - drug2_data = TestbedDataset(root='data', dataset='new_labels_0_10_drug2') - lenth = len(drug1_data) - random_num = random.sample(range(0, lenth), lenth) - drug1_data = drug1_data[random_num] - drug2_data = drug2_data[random_num] - - # 构建训练集 - drug1_loader_train = DataLoader(drug1_data, batch_size=TRAIN_BATCH_SIZE, shuffle=None) - drug2_loader_train = DataLoader(drug2_data, batch_size=TRAIN_BATCH_SIZE, shuffle=None) - - # 构建独立测试集 - independent_drug1_loader_test = DataLoader(independent_drug1_data, batch_size=TEST_BATCH_SIZE, shuffle=None) - independent_drug2_loader_test = DataLoader(independent_drug2_data, batch_size=TEST_BATCH_SIZE, shuffle=None) - - modeling = GATNet - model = modeling(file=datafile).to(device) - loss_fn = nn.CrossEntropyLoss() - optimizer = torch.optim.Adam(model.parameters(), lr=LR) - - - # model_file_name = 'data/result/' + datafile + '_GATNet--model.pt' - - # file_AUCs = 'data/result/' + datafile + '_GATNet--AUCs.txt' - # AUCs = ('Epoch\tAUC_dev\tPR_AUC\tACC\tBACC\tPREC\tTPR\tKAPPA') - # with open(file_AUCs, 'w') as f: - # f.write(AUCs + '\n') - # - # best_auc = 0 - - for epoch in range(NUM_EPOCHS): - train(model, device, drug1_loader_train, drug2_loader_train, optimizer, epoch + 1) - - if (epoch+1) % 100 == 0: - - T, S, Y = predicting(model, device, independent_drug1_loader_test, independent_drug2_loader_test) - # torch.save(model.state_dict(), model_file_name) - independent_num_file_name = 'data/result/new_labels_0_10_leave/' + datafile + '--result' + str(epoch) + '.csv' - independent_num = [] - # independent_num.append(test_num) - # independent_num.append(T) - independent_num.append(Y) - independent_num.append(S) - txtDF = pd.DataFrame(data=list(map(list, zip(*independent_num)))) - txtDF.to_csv(independent_num_file_name, index=False, header=False) - - # T is correct label - # S is predict score - # Y is predict label - - # compute preformence - # AUC = roc_auc_score(T, S) - # precision, recall, threshold = metrics.precision_recall_curve(T, S) - # PR_AUC = metrics.auc(recall, precision) - # BACC = balanced_accuracy_score(T, Y) - # tn, fp, fn, tp = confusion_matrix(T, Y).ravel() - # TPR = tp / (tp + fn) - # PREC = precision_score(T, Y) - # ACC = accuracy_score(T, Y) - # KAPPA = cohen_kappa_score(T, Y) - # - # # save data - # - # if best_auc < AUC: - # best_auc = AUC - # # torch.save(model.state_dict(), model_file_name) - # # independent_num = [] - # # independent_num.append(T) - # # independent_num.append(Y) - # # independent_num.append(S) - # # txtDF = pd.DataFrame(data=independent_num) - # # txtDF.to_csv(independent_num_file_name, index=False, header=False) - # - # # sns.set() - # # f, ax = plt.subplots() - # # CM = confusion_matrix(T, Y) - # # print(CM) - # # sns.heatmap(CM, annot=True, ax=ax, fmt='.20g') - # # ax.set_title('DeepDDS(GATNet) confusion matrix') # 标题 - # # ax.set_xlabel('Predict') # x轴 - # # ax.set_ylabel('True') # y轴 - # # plt.savefig('D:\GraphDTA-master\data\\result\photo\GATNet_cm.png') - # - # AUCs = [epoch, AUC, PR_AUC, ACC, BACC, PREC, TPR, KAPPA] - # save_AUCs(AUCs, file_AUCs) - # # torch.save(model.state_dict(), model_file_name) - # print("BEST_AUC", best_auc) - - +from _reference_paths import ASSET_DIR, artifact_path, data_path, output_dir, output_path, required_artifact +import random +import numpy as np +import pandas as pd +import sys, os +from random import shuffle +import torch.utils.data as Data +import torch +import torch.nn.functional as F +import torch.nn as nn +from torch.utils.data import TensorDataset, Dataset +from models.gat import GATNet +from models.gat_gcn_test import GAT_GCN +from models.gcn import GCNNet +from models.ginconv import GINConvNet +from utils_test import * +from sklearn.metrics import roc_curve, confusion_matrix +from sklearn.metrics import cohen_kappa_score, accuracy_score, roc_auc_score, precision_score, recall_score, balanced_accuracy_score +from sklearn import metrics +from sklearn.model_selection import StratifiedKFold, KFold +import pandas as pd +import matplotlib.pyplot as plt +import seaborn as sns + + +# training function at each epoch +def train(model, device, drug1_loader_train, drug2_loader_train, optimizer, epoch): + print('Training on {} samples...'.format(len(drug1_loader_train.dataset))) + model.train() + # train_loader = np.array(train_loader) + for batch_idx, data in enumerate(zip(drug1_loader_train, drug2_loader_train)): + data1 = data[0] + data2 = data[1] + data1 = data1.to(device) + data2 = data2.to(device) + y = data[0].y.view(-1, 1).long().to(device) + y = y.squeeze(1) + optimizer.zero_grad() + output = model(data1, data2) + loss = loss_fn(output, y) + # print('loss', loss) + loss.backward() + optimizer.step() + if batch_idx % LOG_INTERVAL == 0: + print('Train epoch: {} Loss: {:.6f}'.format(epoch, loss.item())) + + +def predicting(model, device, drug1_loader_test, drug2_loader_test): + model.eval() + total_preds = torch.Tensor() + total_labels = torch.Tensor() + total_prelabels = torch.Tensor() + print('Make prediction for {} samples...'.format(len(drug1_loader_test.dataset))) + with torch.no_grad(): + for data in zip(drug1_loader_test, drug2_loader_test): + data1 = data[0] + data2 = data[1] + data1 = data1.to(device) + data2 = data2.to(device) + output = model(data1, data2) + ys = F.softmax(output, 1).to('cpu').data.numpy() + predicted_labels = list(map(lambda x: np.argmax(x), ys)) + predicted_scores = list(map(lambda x: x[1], ys)) + total_preds = torch.cat((total_preds, torch.Tensor(predicted_scores)), 0) + total_prelabels = torch.cat((total_prelabels, torch.Tensor(predicted_labels)), 0) + # total_labels = torch.cat((total_labels, data1.y.view(-1, 1).cpu()), 0) + return total_labels.numpy().flatten(), total_preds.numpy().flatten(), total_prelabels.numpy().flatten() + + +# modeling = [GINConvNet, GATNet, GAT_GCN, GCNNet][int(sys.argv[2])] + + + +TRAIN_BATCH_SIZE = 256 +TEST_BATCH_SIZE = 256 +LR = 0.0001 +LOG_INTERVAL = 20 +NUM_EPOCHS = 100 + +print('Learning rate: ', LR) +print('Epochs: ', NUM_EPOCHS) +file = ['new_labels_0_10_leave'] + +# CPU or GPU + +if torch.cuda.is_available(): + device = torch.device('cuda') + print('The code uses GPU...') +else: + device = torch.device('cpu') + print('The code uses CPU!!!') + +for datafile in file: + + independent_drug1_data = TestbedDataset(root=artifact_path("cache"), dataset=datafile + '_drug1') + independent_drug2_data = TestbedDataset(root=artifact_path("cache"), dataset=datafile + '_drug2') + print('independent_drug1_data[0]', independent_drug1_data[0]) + lenth = len(independent_drug1_data) + + + + drug1_data = TestbedDataset(root=artifact_path("cache"), dataset='new_labels_0_10_drug1') + drug2_data = TestbedDataset(root=artifact_path("cache"), dataset='new_labels_0_10_drug2') + lenth = len(drug1_data) + random_num = random.sample(range(0, lenth), lenth) + drug1_data = drug1_data[random_num] + drug2_data = drug2_data[random_num] + + # Construct the training dataset. + drug1_loader_train = DataLoader(drug1_data, batch_size=TRAIN_BATCH_SIZE, shuffle=None) + drug2_loader_train = DataLoader(drug2_data, batch_size=TRAIN_BATCH_SIZE, shuffle=None) + + # Construct the independent test dataset. + independent_drug1_loader_test = DataLoader(independent_drug1_data, batch_size=TEST_BATCH_SIZE, shuffle=None) + independent_drug2_loader_test = DataLoader(independent_drug2_data, batch_size=TEST_BATCH_SIZE, shuffle=None) + + modeling = GATNet + model = modeling(file=datafile).to(device) + loss_fn = nn.CrossEntropyLoss() + optimizer = torch.optim.Adam(model.parameters(), lr=LR) + + + # model_file_name = 'data/result/' + datafile + '_GATNet--model.pt' + + # file_AUCs = 'data/result/' + datafile + '_GATNet--AUCs.txt' + # AUCs = ('Epoch\tAUC_dev\tPR_AUC\tACC\tBACC\tPREC\tTPR\tKAPPA') + # with open(file_AUCs, 'w') as f: + # f.write(AUCs + '\n') + # + # best_auc = 0 + + for epoch in range(NUM_EPOCHS): + train(model, device, drug1_loader_train, drug2_loader_train, optimizer, epoch + 1) + + if (epoch+1) % 100 == 0: + + T, S, Y = predicting(model, device, independent_drug1_loader_test, independent_drug2_loader_test) + # torch.save(model.state_dict(), model_file_name) + independent_num_file_name = output_dir('result/new_labels_0_10_leave/') + datafile + '--result' + str(epoch) + '.csv' + independent_num = [] + # independent_num.append(test_num) + # independent_num.append(T) + independent_num.append(Y) + independent_num.append(S) + txtDF = pd.DataFrame(data=list(map(list, zip(*independent_num)))) + txtDF.to_csv(independent_num_file_name, index=False, header=False) + + # T is correct label + # S is predict score + # Y is predict label + + # compute preformence + # AUC = roc_auc_score(T, S) + # precision, recall, threshold = metrics.precision_recall_curve(T, S) + # PR_AUC = metrics.auc(recall, precision) + # BACC = balanced_accuracy_score(T, Y) + # tn, fp, fn, tp = confusion_matrix(T, Y).ravel() + # TPR = tp / (tp + fn) + # PREC = precision_score(T, Y) + # ACC = accuracy_score(T, Y) + # KAPPA = cohen_kappa_score(T, Y) + # + # # save data + # + # if best_auc < AUC: + # best_auc = AUC + # # torch.save(model.state_dict(), model_file_name) + # # independent_num = [] + # # independent_num.append(T) + # # independent_num.append(Y) + # # independent_num.append(S) + # # txtDF = pd.DataFrame(data=independent_num) + # # txtDF.to_csv(independent_num_file_name, index=False, header=False) + # + # # sns.set() + # # f, ax = plt.subplots() + # # CM = confusion_matrix(T, Y) + # # print(CM) + # # sns.heatmap(CM, annot=True, ax=ax, fmt='.20g') + # # ax.set_title('DeepDDS(GATNet) confusion matrix') # Title. + # # ax.set_xlabel('Predict') # X axis. + # # ax.set_ylabel('True') # Y axis. + # # plt.savefig('outputs/reference_deepdds/result/photo/GATNet_cm.png') + # + # AUCs = [epoch, AUC, PR_AUC, ACC, BACC, PREC, TPR, KAPPA] + # save_AUCs(AUCs, file_AUCs) + # # torch.save(model.state_dict(), model_file_name) + # print("BEST_AUC", best_auc) + + diff --git a/data/downstream/final_data/drug_response_pytorch/drug_combination/utils_test.py b/data/downstream/final_data/drug_response_pytorch/drug_combination/utils_test.py index 46b75f81c5de8dfc53ff266218eba8f3c772012e..c2318b940e42a5412038fdf554f530a283b47885 100644 --- a/data/downstream/final_data/drug_response_pytorch/drug_combination/utils_test.py +++ b/data/downstream/final_data/drug_response_pytorch/drug_combination/utils_test.py @@ -1,223 +1,224 @@ -import os -from itertools import islice -import sys -import numpy as np -from math import sqrt -from scipy import stats -from torch_geometric.data import InMemoryDataset -from torch_geometric.loader import DataLoader -from torch_geometric import data as DATA -import torch -from creat_data_DC import creat_data -from tqdm import tqdm - -import pandas as pd - -ind=True -ckpt='geneformer2' -ckpt_scfm = 'v20228' -#ckpt = ckpt_scfm -scfm=True -geneformer = True -class TestbedDataset(InMemoryDataset): - def __init__(self, root='/tmp', dataset='_drug1', - xd=None, xt=None, y=None, xt_featrue=None, transform=None, - pre_transform=None, smile_graph=None): - - #root is required for save preprocessed data, default is '/tmp' - super(TestbedDataset, self).__init__(root, transform, pre_transform) - # benchmark dataset, default = 'davis' - self.dataset = dataset - if ind: - self.cell2embed=np.load('/data1/ruiyi/deepdds/deepddsindcell2scfm{}embed.npy'.format(ckpt_scfm),allow_pickle=True).item() - print(len(self.cell2embed)) - if geneformer: - #self.processed_paths[0]='' - print(self.processed_paths) - celllist = list(self.cell2embed.keys()) - df = pd.read_csv('/data3/ruiyi/geneformer/output/deepdds_ind.csv', index_col=0, skiprows=1) - - # Convert the DataFrame to a dictionary where keys are row names and values are NumPy arrays - self.cell2geneformer = {celllist[k]: np.array(df.loc[row]) for k,row in enumerate(df.index)} - print(len(self.cell2geneformer),len(self.cell2embed)) - #exit() - print(self.cell2geneformer['SW948'].shape) - #print(self.cell2embed['SW948'].shape) - #exit() - else: - self.cell2embed=np.load('/data1/ruiyi/deepdds/deepddscell2scfm{}embed.npy'.format(ckpt_scfm),allow_pickle=True).item() - if geneformer: - print(self.processed_paths) - celllist = list(self.cell2embed.keys()) - df = pd.read_csv('/data3/ruiyi/geneformer/output/deepdds.csv', index_col=0, skiprows=1) - - # Convert the DataFrame to a dictionary where keys are row names and values are NumPy arrays - self.cell2geneformer = {celllist[k]: np.array(df.loc[row]) for k,row in enumerate(df.index)} - print(len(self.cell2geneformer),len(self.cell2embed)) - - if os.path.isfile(self.processed_paths[0]): - print('Pre-processed data found: {}, loading ...'.format(self.processed_paths[0])) - self.data, self.slices = torch.load(self.processed_paths[0]) - #print('data',self.data) - #print('slices',self.slices) - else: - print('Pre-processed data {} not found, doing pre-processing...'.format(self.processed_paths[0])) - #print(xd, xt, xt_featrue, y, smile_graph) - self.process(xd, xt, xt_featrue, y, smile_graph) - self.data, self.slices = torch.load(self.processed_paths[0]) - - @property - def raw_file_names(self): - pass - #return ['some_file_1', 'some_file_2', ...] - - @property - def processed_file_names(self): - return [self.dataset + '.pt'] - - def download(self): - # Download to `self.raw_dir`. - pass - - def _download(self): - pass - - def _process(self): - if not os.path.exists(self.processed_dir): - os.makedirs(self.processed_dir) - - def get_cell_feature(self, cellId, cell_features): - for row in islice(cell_features, 0, None): - if cellId in row[0]: - print(cellId) - return row[1:] - return False - - def get_cell_scfm_feature(self, cellId, cell_features): - return self.cell2embed.get(cellId) - - # Customize the process method to fit the task of drug-target affinity prediction - # Inputs: - # XD - list of SMILES, XT: list of encoded target (categorical or one-hot), - # Y: list of labels (i.e. affinity) - # Return: PyTorch-Geometric format processed data - def process(self, xd, xt, xt_featrue, y, smile_graph): - print(len(xd),len(xt),len(y)) - assert (len(xd) == len(xt) and len(xt) == len(y)), "The three lists must be the same length!" - data_list = [] - data_len = len(xd) - print('number of data', data_len) - for i in tqdm(range(data_len)): - # print('Converting SMILES to graph: {}/{}'.format(i+1, data_len)) - smiles = xd[i] - target = xt[i] - labels = y[i] - # convert SMILES to molecular representation using rdkit - c_size, features, edge_index = smile_graph[smiles] - # make the graph ready for PyTorch Geometrics GCN algorithms: - GCNData = DATA.Data(x=torch.Tensor(features), - edge_index=torch.LongTensor(edge_index).transpose(1, 0), - y=torch.Tensor([labels])) - #print(target,xt_featrue) - - if scfm and not geneformer: - #print(target,xt_featrue) - print('test') - exit() - cell_scfm = self.get_cell_scfm_feature(target, xt_featrue) - if cell_scfm is None: - print('fail: ',target) - continue - cell = self.get_cell_feature(target, xt_featrue) - new_cell = [] - for n in cell: - new_cell.append(float(n)) - cell_scfm=torch.FloatTensor([cell_scfm]) - cell=torch.FloatTensor([new_cell]) - cell_combined=torch.cat((cell_scfm,cell.unsqueeze(-1)),dim=-1) - - GCNData.cell = cell_combined - elif geneformer: - cell = self.get_cell_feature(target, xt_featrue) - #print(self.cell2geneformer) - cell_geneformer = self.cell2geneformer.get(target) - if cell_geneformer is None: - print('fail: ',target) - continue - #print(cell_geneformer) - #exit() - new_cell = [] - for n in cell: - new_cell.append(float(n)) - for n in cell_geneformer: - new_cell.append(n) - GCNData.cell = torch.FloatTensor([new_cell]) - else: - print('test2') - cell = self.get_cell_feature(target, xt_featrue) - new_cell = [] - for n in cell: - new_cell.append(float(n)) - GCNData.cell = torch.FloatTensor([new_cell]) - ''' - if cell == False : # 如果读取cell失败则中断程序 - print('cell', cell) - sys.exit() - ''' - - - # print('cell_feature', cell_feature) - - #print(GCNData.cell.shape) - #exit() - GCNData.__setitem__('c_size', torch.LongTensor([c_size])) - # append graph, label and target sequence to data list - data_list.append(GCNData) - - if self.pre_filter is not None: - data_list = [data for data in data_list if self.pre_filter(data)] - - if self.pre_transform is not None: - data_list = [self.pre_transform(data) for data in data_list] - print('Graph construction done. Saving to file.') - data, slices = self.collate(data_list) - # save preprocessed data: - torch.save((data, slices), self.processed_paths[0]) - -def rmse(y,f): - rmse = sqrt(((y - f)**2).mean(axis=0)) - return rmse -def save_AUCs(AUCs, filename): - with open(filename, 'a') as f: - f.write('\t'.join(map(str, AUCs)) + '\n') -def mse(y,f): - mse = ((y - f)**2).mean(axis=0) - return mse -def pearson(y,f): - rp = np.corrcoef(y, f)[0,1] - return rp -def spearman(y,f): - rs = stats.spearmanr(y, f)[0] - return rs -def ci(y,f): - ind = np.argsort(y) - y = y[ind] - f = f[ind] - i = len(y)-1 - j = i-1 - z = 0.0 - S = 0.0 - while i > 0: - while j >= 0: - if y[i] > y[j]: - z = z+1 - u = f[i] - f[j] - if u > 0: - S = S + 1 - elif u == 0: - S = S + 0.5 - j = j - 1 - i = i - 1 - j = i-1 - ci = S/z - return ci \ No newline at end of file +from _reference_paths import ASSET_DIR, artifact_path, data_path, output_dir, output_path, required_artifact +import os +from itertools import islice +import sys +import numpy as np +from math import sqrt +from scipy import stats +from torch_geometric.data import InMemoryDataset +from torch_geometric.loader import DataLoader +from torch_geometric import data as DATA +import torch +from creat_data_DC import creat_data +from tqdm import tqdm + +import pandas as pd + +ind=True +ckpt='geneformer2' +ckpt_scfm = 'v20228' +#ckpt = ckpt_scfm +scfm=True +geneformer = True +class TestbedDataset(InMemoryDataset): + def __init__(self, root=artifact_path("cache"), dataset='_drug1', + xd=None, xt=None, y=None, xt_featrue=None, transform=None, + pre_transform=None, smile_graph=None): + + #root is required for save preprocessed data, default is '/tmp' + super(TestbedDataset, self).__init__(root, transform, pre_transform) + # benchmark dataset, default = 'davis' + self.dataset = dataset + if ind: + self.cell2embed=np.load(artifact_path('deepddsindcell2scfm{}embed.npy').format(ckpt_scfm),allow_pickle=True).item() + print(len(self.cell2embed)) + if geneformer: + #self.processed_paths[0]='' + print(self.processed_paths) + celllist = list(self.cell2embed.keys()) + df = pd.read_csv(required_artifact('geneformer/deepdds_ind.csv'), index_col=0, skiprows=1) + + # Convert the DataFrame to a dictionary where keys are row names and values are NumPy arrays + self.cell2geneformer = {celllist[k]: np.array(df.loc[row]) for k,row in enumerate(df.index)} + print(len(self.cell2geneformer),len(self.cell2embed)) + #exit() + print(self.cell2geneformer['SW948'].shape) + #print(self.cell2embed['SW948'].shape) + #exit() + else: + self.cell2embed=np.load(artifact_path('deepddscell2scfm{}embed.npy').format(ckpt_scfm),allow_pickle=True).item() + if geneformer: + print(self.processed_paths) + celllist = list(self.cell2embed.keys()) + df = pd.read_csv(required_artifact('geneformer/deepdds.csv'), index_col=0, skiprows=1) + + # Convert the DataFrame to a dictionary where keys are row names and values are NumPy arrays + self.cell2geneformer = {celllist[k]: np.array(df.loc[row]) for k,row in enumerate(df.index)} + print(len(self.cell2geneformer),len(self.cell2embed)) + + if os.path.isfile(self.processed_paths[0]): + print('Pre-processed data found: {}, loading ...'.format(self.processed_paths[0])) + self.data, self.slices = torch.load(self.processed_paths[0]) + #print('data',self.data) + #print('slices',self.slices) + else: + print('Pre-processed data {} not found, doing pre-processing...'.format(self.processed_paths[0])) + #print(xd, xt, xt_featrue, y, smile_graph) + self.process(xd, xt, xt_featrue, y, smile_graph) + self.data, self.slices = torch.load(self.processed_paths[0]) + + @property + def raw_file_names(self): + pass + #return ['some_file_1', 'some_file_2', ...] + + @property + def processed_file_names(self): + return [self.dataset + '.pt'] + + def download(self): + # Download to `self.raw_dir`. + pass + + def _download(self): + pass + + def _process(self): + if not os.path.exists(self.processed_dir): + os.makedirs(self.processed_dir) + + def get_cell_feature(self, cellId, cell_features): + for row in islice(cell_features, 0, None): + if cellId in row[0]: + print(cellId) + return row[1:] + return False + + def get_cell_scfm_feature(self, cellId, cell_features): + return self.cell2embed.get(cellId) + + # Customize the process method to fit the task of drug-target affinity prediction + # Inputs: + # XD - list of SMILES, XT: list of encoded target (categorical or one-hot), + # Y: list of labels (i.e. affinity) + # Return: PyTorch-Geometric format processed data + def process(self, xd, xt, xt_featrue, y, smile_graph): + print(len(xd),len(xt),len(y)) + assert (len(xd) == len(xt) and len(xt) == len(y)), "The three lists must be the same length!" + data_list = [] + data_len = len(xd) + print('number of data', data_len) + for i in tqdm(range(data_len)): + # print('Converting SMILES to graph: {}/{}'.format(i+1, data_len)) + smiles = xd[i] + target = xt[i] + labels = y[i] + # convert SMILES to molecular representation using rdkit + c_size, features, edge_index = smile_graph[smiles] + # make the graph ready for PyTorch Geometrics GCN algorithms: + GCNData = DATA.Data(x=torch.Tensor(features), + edge_index=torch.LongTensor(edge_index).transpose(1, 0), + y=torch.Tensor([labels])) + #print(target,xt_featrue) + + if scfm and not geneformer: + #print(target,xt_featrue) + print('test') + exit() + cell_scfm = self.get_cell_scfm_feature(target, xt_featrue) + if cell_scfm is None: + print('fail: ',target) + continue + cell = self.get_cell_feature(target, xt_featrue) + new_cell = [] + for n in cell: + new_cell.append(float(n)) + cell_scfm=torch.FloatTensor([cell_scfm]) + cell=torch.FloatTensor([new_cell]) + cell_combined=torch.cat((cell_scfm,cell.unsqueeze(-1)),dim=-1) + + GCNData.cell = cell_combined + elif geneformer: + cell = self.get_cell_feature(target, xt_featrue) + #print(self.cell2geneformer) + cell_geneformer = self.cell2geneformer.get(target) + if cell_geneformer is None: + print('fail: ',target) + continue + #print(cell_geneformer) + #exit() + new_cell = [] + for n in cell: + new_cell.append(float(n)) + for n in cell_geneformer: + new_cell.append(n) + GCNData.cell = torch.FloatTensor([new_cell]) + else: + print('test2') + cell = self.get_cell_feature(target, xt_featrue) + new_cell = [] + for n in cell: + new_cell.append(float(n)) + GCNData.cell = torch.FloatTensor([new_cell]) + ''' + if cell == False : # Stop if the cell features cannot be read. + print('cell', cell) + sys.exit() + ''' + + + # print('cell_feature', cell_feature) + + #print(GCNData.cell.shape) + #exit() + GCNData.__setitem__('c_size', torch.LongTensor([c_size])) + # append graph, label and target sequence to data list + data_list.append(GCNData) + + if self.pre_filter is not None: + data_list = [data for data in data_list if self.pre_filter(data)] + + if self.pre_transform is not None: + data_list = [self.pre_transform(data) for data in data_list] + print('Graph construction done. Saving to file.') + data, slices = self.collate(data_list) + # save preprocessed data: + torch.save((data, slices), self.processed_paths[0]) + +def rmse(y,f): + rmse = sqrt(((y - f)**2).mean(axis=0)) + return rmse +def save_AUCs(AUCs, filename): + with open(filename, 'a') as f: + f.write('\t'.join(map(str, AUCs)) + '\n') +def mse(y,f): + mse = ((y - f)**2).mean(axis=0) + return mse +def pearson(y,f): + rp = np.corrcoef(y, f)[0,1] + return rp +def spearman(y,f): + rs = stats.spearmanr(y, f)[0] + return rs +def ci(y,f): + ind = np.argsort(y) + y = y[ind] + f = f[ind] + i = len(y)-1 + j = i-1 + z = 0.0 + S = 0.0 + while i > 0: + while j >= 0: + if y[i] > y[j]: + z = z+1 + u = f[i] - f[j] + if u > 0: + S = S + 1 + elif u == 0: + S = S + 0.5 + j = j - 1 + i = i - 1 + j = i-1 + ci = S/z + return ci diff --git a/data/downstream/final_data/drug_response_pytorch/drug_response_pytorch/DeepCDR/README.md b/data/downstream/final_data/drug_response_pytorch/drug_response_pytorch/DeepCDR/README.md index 479feeaf34014c15aeb52e4018d65c3a151e1507..3c00fd6e2c3ea76caee2247c9db367b474426b9d 100644 --- a/data/downstream/final_data/drug_response_pytorch/drug_response_pytorch/DeepCDR/README.md +++ b/data/downstream/final_data/drug_response_pytorch/drug_response_pytorch/DeepCDR/README.md @@ -1,4 +1,8 @@ # DeepCDR + +This is the legacy Keras/TensorFlow reference implementation. The two `prog/run_DeepCDR*.py` entrypoints retain Python 2 syntax and require their original environment; they are not BioMod training scripts. + +Bundled input paths resolve from the source file to the sibling `../data` directory (relative to this DeepCDR directory). Override with `DEEPCDR_DATA_DIR` if needed. Model outputs go to `checkpoint/` in this directory, configurable with `DEEPCDR_CHECKPOINT_DIR`. `prog/process_drug.py` writes regenerated graph features to `outputs/drug_graph_feat/`, configurable with `DEEPCDR_GRAPH_OUTPUT_DIR`, so it does not overwrite bundled features by default. Cancer Drug Response Prediction via a Hybrid Graph Convolutional Network This work has been accepted to [ECCB2020](https://eccb2020.info/)([slides](https://github.com/kimmo1019/DeepCDR/blob/master/DeepCDR_ECCB_presentation.pdf)) and was also published in the journal [Bioinformatics](https://academic.oup.com/bioinformatics/article/36/Supplement_2/i911/6055929). @@ -22,6 +26,8 @@ Installation has been tested in a Linux/MacOS platform. # Instructions We provide detailed step-by-step instructions for running DeepCDR model including data preprocessing, model training, and model test. +Run the following legacy commands from `prog/`. The upstream instructions below describe the original package layout. + ## Model implementation **Step 1: Data Preparing** @@ -82,7 +88,7 @@ python run_DeepCDR.py -gpu_id [gpu_id] -use_mut [use_mut] -use_gexp [use_gexp] - ``` One can run `python run_DeepCDR.py -gpu_id 0 -use_mut True -use_gexp True -use_methy True` to implement the DeepCDR regression model. -The trained model will be saved in `data/checkpoint` folder. The overall Pearson's correlation will be calculated. +The trained model will be saved in `checkpoint/` folder in this DeepCDR directory. The overall Pearson's correlation will be calculated. ### DeepCDR classification model @@ -95,7 +101,7 @@ python run_DeepCDR_classify.py -gpu_id [gpu_id] -use_mut [use_mut] -use_gexp [us ``` One can run `python run_DeepCDR_classify.py -gpu_id 0 -use_mut True -use_gexp True -use_methy True` to implement the DeepCDR classification model. -The trained model will be saved in `data/checkpoint` folder. The overall auROC and auPRC will be calculated. +The trained model will be saved in `checkpoint/` folder in this DeepCDR directory. The overall auROC and auPRC will be calculated. ## External patient data diff --git a/data/downstream/final_data/drug_response_pytorch/drug_response_pytorch/DeepCDR/prog/layers/graph.py b/data/downstream/final_data/drug_response_pytorch/drug_response_pytorch/DeepCDR/prog/layers/graph.py index 7ecc5023fbaa410bdbca1f6e3c88b2167e75c323..ef41c5170d8bf12b698833ac7134d2082dfeb940 100644 --- a/data/downstream/final_data/drug_response_pytorch/drug_response_pytorch/DeepCDR/prog/layers/graph.py +++ b/data/downstream/final_data/drug_response_pytorch/drug_response_pytorch/DeepCDR/prog/layers/graph.py @@ -62,7 +62,7 @@ class GraphLayer(keras.layers.Layer): class GraphConv(GraphLayer): - """Graph convolutional layer. + r"""Graph convolutional layer. h_i^{(t)} = \sigma \left ( \frac{ G_i^T (h_i^{(t - 1)} W + b)}{\sum G_i} \right ) """ diff --git a/data/downstream/final_data/drug_response_pytorch/drug_response_pytorch/DeepCDR/prog/process_drug.py b/data/downstream/final_data/drug_response_pytorch/drug_response_pytorch/DeepCDR/prog/process_drug.py index 1c34b80edbdc2a9c8bfd0ca9cd1d5e6a6c0a2f82..f8830886531c00d1ef29c22f55f5081bc27aae58 100644 --- a/data/downstream/final_data/drug_response_pytorch/drug_response_pytorch/DeepCDR/prog/process_drug.py +++ b/data/downstream/final_data/drug_response_pytorch/drug_response_pytorch/DeepCDR/prog/process_drug.py @@ -6,8 +6,11 @@ import numpy as np import hickle as hkl -drug_smiles_file='../data/223drugs_pubchem_smiles.txt' -save_dir='../data/GDSC/drug_graph_feat' +DATA_DIR = os.environ.get('DEEPCDR_DATA_DIR', os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', 'data'))) +drug_smiles_file=os.path.join(DATA_DIR, '223drugs_pubchem_smiles.txt') +save_dir=os.environ.get('DEEPCDR_GRAPH_OUTPUT_DIR', os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'outputs', 'drug_graph_feat'))) +if not os.path.isdir(save_dir): + os.makedirs(save_dir) pubchemid2smile = {item.split('\t')[0]:item.split('\t')[1].strip() for item in open(drug_smiles_file).readlines()} if not os.path.exists(save_dir): os.makedirs(save_dir) diff --git a/data/downstream/final_data/drug_response_pytorch/drug_response_pytorch/DeepCDR/prog/run_DeepCDR.py b/data/downstream/final_data/drug_response_pytorch/drug_response_pytorch/DeepCDR/prog/run_DeepCDR.py index 8f366d5611a6b91b4b1563b1a547022e71e7bd0f..0c56c248636a8468ac862e4483e8080fa7ea5958 100644 --- a/data/downstream/final_data/drug_response_pytorch/drug_response_pytorch/DeepCDR/prog/run_DeepCDR.py +++ b/data/downstream/final_data/drug_response_pytorch/drug_response_pytorch/DeepCDR/prog/run_DeepCDR.py @@ -59,7 +59,11 @@ TCGA_label_set = ["ALL","BLCA","BRCA","CESC","DLBC","LIHC","LUAD", "ESCA","GBM","HNSC","KIRC","LAML","LCML","LGG", "LUSC","MESO","MM","NB","OV","PAAD","SCLC","SKCM", "STAD","THCA",'COAD/READ'] -DPATH = '../data' +# Paths are relative to this source file, not the shell working directory. +DPATH = os.environ.get('DEEPCDR_DATA_DIR', os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', 'data'))) +CHECKPOINT_DIR = os.environ.get('DEEPCDR_CHECKPOINT_DIR', os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'checkpoint'))) +if not os.path.isdir(CHECKPOINT_DIR): + os.makedirs(CHECKPOINT_DIR) Drug_info_file = '%s/GDSC/1.Drug_listMon Jun 24 09_00_55 2019.csv'%DPATH Cell_line_info_file = '%s/CCLE/Cell_lines_annotations_20181226.txt'%DPATH Drug_feature_file = '%s/GDSC/drug_graph_feat'%DPATH @@ -211,7 +215,7 @@ class MyCallback(Callback): return def on_train_end(self, logs={}): self.model.set_weights(self.best_weight) - self.model.save('../checkpoint/MyBestDeepCDR_%s.h5'%model_suffix) + self.model.save(os.path.join(CHECKPOINT_DIR, 'MyBestDeepCDR_%s.h5')%model_suffix) if self.stopped_epoch > 0 : print('Epoch %05d: early stopping' % (self.stopped_epoch + 1)) return @@ -241,7 +245,7 @@ def ModelTraining(model,X_drug_data_train,X_mutation_data_train,X_gexpr_data_tra optimizer = Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False) model.compile(optimizer = optimizer,loss='mean_squared_error',metrics=['mse']) #EarlyStopping(monitor='val_loss',patience=5) - callbacks = [ModelCheckpoint('../checkpoint/best_DeepCDR_%s.h5'%model_suffix,monitor='val_loss',save_best_only=False, save_weights_only=False), + callbacks = [ModelCheckpoint(os.path.join(CHECKPOINT_DIR, 'best_DeepCDR_%s.h5')%model_suffix,monitor='val_loss',save_best_only=False, save_weights_only=False), MyCallback(validation_data=validation_data,patience=10)] X_drug_feat_data_train = [item[0] for item in X_drug_data_train] X_drug_adj_data_train = [item[1] for item in X_drug_data_train] diff --git a/data/downstream/final_data/drug_response_pytorch/drug_response_pytorch/DeepCDR/prog/run_DeepCDR_classify.py b/data/downstream/final_data/drug_response_pytorch/drug_response_pytorch/DeepCDR/prog/run_DeepCDR_classify.py index 34c6bf785bf52ac606633fb10ce1d300ac4ca97d..d19f4db766be000ee8f1ecdc269a0e796e2bb8e6 100644 --- a/data/downstream/final_data/drug_response_pytorch/drug_response_pytorch/DeepCDR/prog/run_DeepCDR_classify.py +++ b/data/downstream/final_data/drug_response_pytorch/drug_response_pytorch/DeepCDR/prog/run_DeepCDR_classify.py @@ -59,7 +59,11 @@ TCGA_label_set = ["ALL","BLCA","BRCA","CESC","DLBC","LIHC","LUAD", "ESCA","GBM","HNSC","KIRC","LAML","LCML","LGG", "LUSC","MESO","MM","NB","OV","PAAD","SCLC","SKCM", "STAD","THCA",'COAD/READ'] -DPATH = '../data' +# Paths are relative to this source file, not the shell working directory. +DPATH = os.environ.get('DEEPCDR_DATA_DIR', os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', 'data'))) +CHECKPOINT_DIR = os.environ.get('DEEPCDR_CHECKPOINT_DIR', os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'checkpoint'))) +if not os.path.isdir(CHECKPOINT_DIR): + os.makedirs(CHECKPOINT_DIR) Drug_info_file = '%s/GDSC/1.Drug_listMon Jun 24 09_00_55 2019.csv'%DPATH Cell_line_info_file = '%s/CCLE/Cell_lines_annotations_20181226.txt'%DPATH Drug_feature_file = '%s/GDSC/drug_graph_feat'%DPATH @@ -239,7 +243,7 @@ class MyCallback(Callback): return def on_train_end(self, logs={}): self.model.set_weights(self.best_weight) - self.model.save('../checkpoint/MyBestDeepCDR_classify_%s.h5'%model_suffix) + self.model.save(os.path.join(CHECKPOINT_DIR, 'MyBestDeepCDR_classify_%s.h5')%model_suffix) if self.stopped_epoch > 0 : print('Epoch %05d: early stopping' % (self.stopped_epoch + 1)) return @@ -272,7 +276,7 @@ def ModelTraining(model,X_drug_data_train,X_mutation_data_train,X_gexpr_data_tra optimizer = Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False) model.compile(optimizer = optimizer,loss='binary_crossentropy',metrics=['accuracy',precision,recall,f1_score,average_precision]) #EarlyStopping(monitor='val_loss',patience=5) - callbacks = [ModelCheckpoint('../checkpoint/best_DeepCDR_classify_%s.h5'%model_suffix,monitor='val_average_precision',save_best_only=True, save_weights_only=False), + callbacks = [ModelCheckpoint(os.path.join(CHECKPOINT_DIR, 'best_DeepCDR_classify_%s.h5')%model_suffix,monitor='val_average_precision',save_best_only=True, save_weights_only=False), MyCallback(validation_data=validation_data,patience=10)] X_drug_feat_data_train = [item[0] for item in X_drug_data_train] X_drug_adj_data_train = [item[1] for item in X_drug_data_train] diff --git a/data/downstream/final_data/drug_response_pytorch/drug_response_pytorch/DeepCDR/prog/utils.py b/data/downstream/final_data/drug_response_pytorch/drug_response_pytorch/DeepCDR/prog/utils.py index 03976a57661251c1e979380d3f9ea834289f8d14..d0c15001a7100f4eb7804d2b8f95be9ca57035d2 100644 --- a/data/downstream/final_data/drug_response_pytorch/drug_response_pytorch/DeepCDR/prog/utils.py +++ b/data/downstream/final_data/drug_response_pytorch/drug_response_pytorch/DeepCDR/prog/utils.py @@ -12,8 +12,11 @@ def encode_onehot(labels): return labels_onehot -def load_data(path="data/cora/", dataset="cora"): +def load_data(path=None, dataset="cora"): """Load citation network dataset (cora only for now)""" + if path is None: + raise ValueError('Supply the directory containing the external Cora .content and .cites files.') + path = str(path).rstrip('/\\') + '/' print('Loading {} dataset...'.format(dataset)) idx_features_labels = np.genfromtxt("{}{}.content".format(path, dataset), dtype=np.dtype(str)) @@ -135,4 +138,4 @@ def sparse_to_tuple(sparse_mx): coords = np.vstack((sparse_mx.row, sparse_mx.col)).transpose() values = sparse_mx.data shape = sparse_mx.shape - return coords, values, shape \ No newline at end of file + return coords, values, shape diff --git a/data/downstream/final_data/drug_response_pytorch/drug_response_pytorch/model.py b/data/downstream/final_data/drug_response_pytorch/drug_response_pytorch/model.py index 3270c87cd0320e4662e723315d07cd68d573417f..e4a940ce7eb919c0206ba08c8dad4e9df66db3bd 100644 --- a/data/downstream/final_data/drug_response_pytorch/drug_response_pytorch/model.py +++ b/data/downstream/final_data/drug_response_pytorch/drug_response_pytorch/model.py @@ -51,17 +51,17 @@ class MultiSourceGCNModelTorch(nn.Module): """ A **simplified** version of the original KerasMultiSourceGCNModel. - 为了保证训练脚本稳定运行,这里不再完全还原 DeepCDR 的复杂 GCN + CNN 结构, - 而是实现一个输入输出一致、结构尽量简单但能利用各模态信息的 MLP 模型: + This implementation uses a simpler multimodal MLP rather than reproducing + the full DeepCDR GCN + CNN architecture, while preserving its input/output interface: Inputs per batch: - - drug_feat: (B, N, F_d) 按原子维度做平均,得到 (B, F_d) - - drug_adj: (B, N, N) 当前模型中不使用,仅占位 - - mut_feat: (B, 1, 1, M) 展开后取平均,得到 (B, 1) 的一个 summary scalar + - drug_feat: (B, N, F_d); average over atoms to obtain (B, F_d). + - drug_adj: (B, N, N); retained as an unused interface placeholder. + - mut_feat: (B, 1, 1, M); flatten and average to obtain a (B, 1) summary. - gexpr_feat: (B, F_g) - methy_feat: (B, F_m) - 这些特征拼接后经过两层全连接,最终输出 (B, 1) 的回归值。 + Concatenate the features and apply two fully connected layers to predict (B, 1) regression values. """ def __init__( @@ -83,9 +83,9 @@ class MultiSourceGCNModelTorch(nn.Module): self.use_gexp = use_gexp self.use_methy = use_methy - in_dim = drug_feat_dim # 来自 drug_feat 的 pooled 特征 + in_dim = drug_feat_dim # Pooled drug features. if use_mut: - in_dim += 1 # mutation 分支只取一个 scalar summary + in_dim += 1 # The mutation branch contributes one scalar summary. if use_gexp: in_dim += gexpr_dim if use_methy: @@ -113,12 +113,12 @@ class MultiSourceGCNModelTorch(nn.Module): """ B = drug_feat.size(0) - # Drug branch: 对原子维度做平均池化,得到 (B, F_d) + # Drug branch: average over atoms to obtain (B, F_d). x_drug = drug_feat.mean(dim=1) # (B, F_d) feats = [x_drug] - # Mutation branch: 展开后对所有基因取平均,得到一个 scalar summary + # Mutation branch: flatten and average over all genes to obtain a scalar summary. if self.use_mut: m = mut_feat.view(B, -1).mean(dim=1, keepdim=True) # (B, 1) feats.append(m) diff --git a/data/downstream/final_data/drug_response_pytorch/drug_response_pytorch/train_deepcdr_torch.py b/data/downstream/final_data/drug_response_pytorch/drug_response_pytorch/train_deepcdr_torch.py index b0a1fe2674f96fcfa654039d67d8d2c117ef7aef..d2fda938bf93bd2766d58fbdde3e969de5b78662 100644 --- a/data/downstream/final_data/drug_response_pytorch/drug_response_pytorch/train_deepcdr_torch.py +++ b/data/downstream/final_data/drug_response_pytorch/drug_response_pytorch/train_deepcdr_torch.py @@ -14,6 +14,11 @@ Data pipeline (kept as close as possible to original): """ import argparse +from pathlib import Path + +SCRIPT_DIR = Path(__file__).resolve().parent +PROJECT_ROOT = SCRIPT_DIR.parents[4] +ASSET_DIR = PROJECT_ROOT / "model_dependencies" import os import random from typing import List, Tuple @@ -73,7 +78,7 @@ def metadata_generate( gene_expression_file: str, methylation_file: str, ) -> Tuple[pd.DataFrame, dict, pd.DataFrame, pd.DataFrame, list]: - # drug_id --> pubchem_id(使用 iloc 显式按位置取列,避免 Series.__getitem__ FutureWarning) + # Map drug_id to pubchem_id using positional iloc indexing. reader = pd.read_csv(drug_info_file) drugid2pubchemid = { str(row.iloc[0]): str(row.iloc[5]) @@ -326,7 +331,7 @@ def eval_epoch(model, loader, device): def main(): parser = argparse.ArgumentParser(description="PyTorch DeepCDR") parser.add_argument("--gpu_id", type=str, default="0") - parser.add_argument("--data_path", type=str, default="../data") + parser.add_argument("--data_path", type=str, default=str(SCRIPT_DIR / "data")) parser.add_argument("--epochs", type=int, default=100) parser.add_argument("--batch_size", type=int, default=64) parser.add_argument("--max_atoms", type=int, default=100) diff --git a/data/downstream/final_data/drug_response_pytorch/drug_response_pytorch/train_sclong_drug.py b/data/downstream/final_data/drug_response_pytorch/drug_response_pytorch/train_sclong_drug.py index cb39574a24c81f0a2a649bbcf194a5968cd070bb..6701fe1f138ea7c68ab91d8cd13779a9c07ca8c4 100644 --- a/data/downstream/final_data/drug_response_pytorch/drug_response_pytorch/train_sclong_drug.py +++ b/data/downstream/final_data/drug_response_pytorch/drug_response_pytorch/train_sclong_drug.py @@ -1,693 +1,698 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" -Use pretrained scLong (DualEncoderSCFM) cell embeddings + simple drug features -to predict drug response (IC50), reusing DeepCDR 的数据管线 + scLong zero-shot 的 embedding 逻辑。 - -数据流程: - 1) metadata_generate: 读取药物 / 细胞系信息,构造 (cell_line, pubchem_id, ln_ic50, cancer_type) 列表; - 2) build_cell_embeddings_with_sclong: 用 CCLE 表达矩阵 + scLong,把每个 cell line 映射到一个 cell embedding; - 3) feature_extract_drug_only: 为每个样本构造药物图特征 (feat, adj); - 4) SCFMDrugDataset: 返回 (x_drug, a_drug, cell_emb, y); - 5) SimpleSCFMDrugModel: 把药物特征和 cell embedding 拼接,经 MLP 预测 IC50。 -""" - -import argparse -import os -import random -from typing import Dict, List, Tuple - -import hickle as hkl -import numpy as np -import pandas as pd -import scipy.sparse as sp -import torch -import torch.nn as nn -from scipy.stats import pearsonr -from torch.utils.data import DataLoader, Dataset - - -# ---------------------- scLong / SCFM 相关 ---------------------- -import pickle - -import sys -sys.path.append("../") -from performer_pytorch_cont.ding_models import DualEncoderSCFM - - -def print_sys(*args, **kwargs): - """简单的打印包装,保持和 zero-shot 脚本风格一致。""" - print(*args, **kwargs, flush=True) - - -def reindex_tensor_universal(tensor: torch.Tensor, - index_positions: List[int], - dim: int, - filler: str = "0", - device: str = "cpu") -> torch.Tensor: - """ - 通用重排操作:沿着某一维 dim 按 index_positions 重新索引,超出索引位置用 filler 填充。 - 来自 zero-shot 脚本。 - """ - index_tensor = torch.tensor(index_positions, device=device) - tensor_shape = list(tensor.shape) - tensor_shape[dim] = len(index_positions) - index_shape = [1] * len(tensor_shape) - index_shape[dim] = len(index_positions) - index_tensor = index_tensor.view(*index_shape).expand(*tensor_shape) - - if filler == "0": - padder_shape = list(tensor_shape) - padder_shape[dim] = 1 - padder = torch.zeros(padder_shape, dtype=tensor.dtype, device=device) - elif filler == "mean": - padder = tensor.mean(dim=dim, keepdim=True) - else: - raise ValueError("filler should be '0' or 'mean'") - - expanded_tensor = torch.cat((tensor, padder), dim=dim) - reindexed = torch.gather(expanded_tensor, dim, index_tensor) - return reindexed - - -def load_sclong_model( - gene2vec_path: str, - scfm_hyper_params_path: str, - scfm_ckpt_path: str, - device: torch.device, -) -> DualEncoderSCFM: - """ - 加载 scLong (DualEncoderSCFM) 模型。 - - - 如果提供了 scfm_ckpt_path,则加载预训练权重; - - 如果未提供或为 'none',则仅根据 hyper_params 初始化一个随机权重的模型。 - """ - with open(scfm_hyper_params_path, "rb") as f: - scfm_hyper_params = pickle.load(f) - scfm_hyper_params["gene2vec_file"] = gene2vec_path - print_sys("SCFM hyper-params:", scfm_hyper_params) - - model = DualEncoderSCFM(**scfm_hyper_params).to(device) - - if scfm_ckpt_path and str(scfm_ckpt_path).lower() != "none": - print_sys(f"Loading SCFM checkpoint from: {scfm_ckpt_path}") - ckpt = torch.load(scfm_ckpt_path, map_location=device) - model.load_state_dict(ckpt["model_state_dict"]) - print_sys("Loaded pretrained SCFM weights.") - else: - print_sys("No SCFM checkpoint provided, using randomly initialized SCFM.") - - model.eval() - for p in model.parameters(): - p.requires_grad = False - return model - - -def build_scfm_gene_mapping( - symbol2ens_path: str, - scfm_genes_path: str, - expr_genes: List[str], -) -> Tuple[Dict[str, int], List[int]]: - """ - 构建从 CCLE 表达矩阵的列 (基因名) 到 scFM 27k 基因的重排索引。 - - 返回: - - input_mapping: dict[input_gene_id] -> 列索引 - - scfm_index_positions: 对于每个 scfm_genes_pad 中的基因,给出在 input_genes 中的列位置(找不到则指向末尾 pad)。 - """ - # 加载 symbol -> ENS 映射 - # with open(symbol2ens_path, "r") as f: - # symbol2ens = { - # line.split(",")[0]: line.split(",")[1].rstrip("\n") - # for line in f - # if line.strip() and line.split(",")[1].rstrip("\n") != "unknown" - # } - gene2ens = pd.read_csv('../gene2ens.csv',index_col=0) - symbol2ens = gene2ens['Ensembl gene ID'].to_dict() - - # 把 CCLE 表达矩阵的列名映射到 ENS,如果没有映射就保持原始名字 - input_genes = [ - symbol2ens.get(sym, sym) for sym in expr_genes - ] - input_gene_num = len(input_genes) - input_mapping = {gid: i for i, gid in enumerate(input_genes)} - - # 加载 scFM 预训练使用的 27k 基因 - with open(scfm_genes_path, "r") as f: - scfm_genes = [line.rstrip("\n") for line in f.readlines()] - scfm_genes_pad = scfm_genes + ["PAD"] - - # 对每个 scFM 基因,找到它在输入中的位置;若不存在则用 input_gene_num(pad) - scfm_index_positions = [ - input_mapping.get(gid, input_gene_num) for gid in scfm_genes_pad - ] - print_sys(f"target and scfm genes intersect: {len(np.intersect1d(input_genes, scfm_genes))}") - return input_mapping, scfm_index_positions - - -def build_cell_embeddings_with_sclong( - gexpr_feature: pd.DataFrame, - symbol2ens_path: str, - scfm_genes_path: str, - gene2vec_path: str, - scfm_hyper_params_path: str, - scfm_ckpt_path: str, - device: torch.device, -) -> Tuple[Dict[str, np.ndarray], int]: - """ - 用 scLong 为每个 cell line 构造一个 cell embedding。 - - gexpr_feature: index 是 cell_line_id,columns 是基因名。 - 返回: - - cell_emb_dict: {cell_line_id: embedding (np.ndarray, shape=(D,))} - - emb_dim: embedding 维度 D - """ - # 加载 SCFM 模型 - model = load_sclong_model( - gene2vec_path=gene2vec_path, - scfm_hyper_params_path=scfm_hyper_params_path, - scfm_ckpt_path=scfm_ckpt_path, - device=device, - ) - - # 构建基因重排映射 - _, scfm_index_positions = build_scfm_gene_mapping( - symbol2ens_path=symbol2ens_path, - scfm_genes_path=scfm_genes_path, - expr_genes=list(gexpr_feature.columns), - ) - - cell_emb_dict: Dict[str, np.ndarray] = {} - first_emb_dim: int = -1 - - with torch.no_grad(): - for cell_line_id, row in gexpr_feature.iterrows(): - x = torch.tensor(row.values[None, :], dtype=torch.float32, device=device) # (1, num_genes) - x_scfm = reindex_tensor_universal( - x, scfm_index_positions, dim=1, filler="0", device=device - ) # (1, seq_len=27k+1) - # cell_emb = model.get_cell_emb(x_scfm) # (1, D) - out = model(x_scfm, return_encodings=True) - cell_emb = out['merged_decodings'][:, 0, :] # 只取第0位的CLS标记 - cell_emb_np = cell_emb.squeeze(0).cpu().numpy() - cell_emb_dict[cell_line_id] = cell_emb_np - if first_emb_dim < 0: - first_emb_dim = cell_emb_np.shape[0] - - print_sys(f"Built scLong embeddings for {len(cell_emb_dict)} cell lines, dim={first_emb_dim}") - return cell_emb_dict, first_emb_dim - - -# ---------------------- DeepCDR 风格的数据准备(只保留 drug+IC50) ---------------------- - -TCGA_LABEL_SET = [ - "ALL", - "BLCA", - "BRCA", - "CESC", - "DLBC", - "LIHC", - "LUAD", - "ESCA", - "GBM", - "HNSC", - "KIRC", - "LAML", - "LCML", - "LGG", - "LUSC", - "MESO", - "MM", - "NB", - "OV", - "PAAD", - "SCLC", - "SKCM", - "STAD", - "THCA", - "COAD/READ", -] - - -def normalize_adj(adj: np.ndarray) -> np.ndarray: - adj = adj + np.eye(adj.shape[0]) - d = sp.diags(np.power(np.array(adj.sum(1)), -0.5).flatten(), 0).toarray() - a_norm = adj.dot(d).transpose().dot(d) - return a_norm - - -def calculate_graph_feat( - feat_mat: np.ndarray, - adj_list: List[List[int]], - max_atoms: int, -) -> Tuple[np.ndarray, np.ndarray]: - """和 train_deepcdr_torch.py 一致的药物图特征预处理。""" - assert feat_mat.shape[0] == len(adj_list) - feat = np.zeros((max_atoms, feat_mat.shape[-1]), dtype=np.float32) - adj_mat = np.zeros((max_atoms, max_atoms), dtype=np.float32) - - feat[: feat_mat.shape[0], :] = feat_mat - for i, nodes in enumerate(adj_list): - for each in nodes: - adj_mat[i, int(each)] = 1 - - assert np.allclose(adj_mat, adj_mat.T) - adj_ = adj_mat[: len(adj_list), : len(adj_list)] - norm_adj_ = normalize_adj(adj_) - adj_mat[: len(adj_list), : len(adj_list)] = norm_adj_ - return feat, adj_mat - - -def metadata_generate( - drug_info_file: str, - cell_line_info_file: str, - genomic_mutation_file: str, - drug_feature_dir: str, - gene_expression_file: str, - methylation_file: str, -) -> Tuple[pd.DataFrame, dict, pd.DataFrame, pd.DataFrame, list]: - """ - 与 train_deepcdr_torch.py 相同的元数据构造,只是最终模型只用 drug_feature + gexpr_feature + IC50。 - """ - # drug_id --> pubchem_id - reader = pd.read_csv(drug_info_file) - drugid2pubchemid = { - str(row.iloc[0]): str(row.iloc[5]) - for _, row in reader.iterrows() - if str(row.iloc[5]).isdigit() - } - - # map cellline --> cancer type - cellline2cancertype = {} - with open(cell_line_info_file, "r") as f: - for line in f.readlines()[1:]: - cellline_id = line.split("\t")[1] - tcga_label = line.strip().split("\t")[-1] - cellline2cancertype[cellline_id] = tcga_label - - mutation_feature = pd.read_csv( - genomic_mutation_file, sep=",", header=0, index_col=0 - ) - gexpr_feature = pd.read_csv( - gene_expression_file, sep=",", header=0, index_col=0 - ) - # only keep overlapped cell lines - mutation_feature = mutation_feature.loc[list(gexpr_feature.index)] - - methylation_feature = pd.read_csv( - methylation_file, sep=",", header=0, index_col=0 - ) - - # load drug graph features (.hkl) - drug_feature = {} - drug_pubchem_id_set = [] - for fname in os.listdir(drug_feature_dir): - if not fname.endswith(".hkl"): - continue - pubchem_id = fname.split(".")[0] - feat_mat, adj_list, degree_list = hkl.load( - os.path.join(drug_feature_dir, fname) - ) - drug_feature[pubchem_id] = [feat_mat, adj_list, degree_list] - drug_pubchem_id_set.append(pubchem_id) - - # IC50 - cancer_response_exp_file = os.path.join( - os.path.dirname(gene_expression_file), "GDSC_IC50.csv" - ) - experiment_data = pd.read_csv( - cancer_response_exp_file, sep=",", header=0, index_col=0 - ) - drug_match_list = [ - item - for item in experiment_data.index - if item.split(":")[1] in drugid2pubchemid.keys() - ] - experiment_data_filtered = experiment_data.loc[drug_match_list] - - data_idx = [] - for each_drug in experiment_data_filtered.index: - for each_cellline in experiment_data_filtered.columns: - pubchem_id = drugid2pubchemid[each_drug.split(":")[-1]] - if ( - str(pubchem_id) in drug_pubchem_id_set - and each_cellline in mutation_feature.index - ): - if ( - not np.isnan( - experiment_data_filtered.loc[each_drug, each_cellline] - ) - and each_cellline in cellline2cancertype.keys() - ): - ln_ic50 = float( - experiment_data_filtered.loc[each_drug, each_cellline] - ) - data_idx.append( - ( - each_cellline, - pubchem_id, - ln_ic50, - cellline2cancertype[each_cellline], - ) - ) - - nb_celllines = len(set([item[0] for item in data_idx])) - nb_drugs = len(set([item[1] for item in data_idx])) - print_sys( - f"{len(data_idx)} instances across {nb_celllines} cell lines and {nb_drugs} drugs were generated." - ) - return mutation_feature, drug_feature, gexpr_feature, methylation_feature, data_idx - - -def data_split(data_idx: list, ratio: float = 0.95): - """与 DeepCDR 相同的按 cancer type 分层划分。""" - train_idx, test_idx = [], [] - for each_type in TCGA_LABEL_SET: - data_subtype_idx = [item for item in data_idx if item[-1] == each_type] - if len(data_subtype_idx) == 0: - continue - k = max(1, int(ratio * len(data_subtype_idx))) - train_list = random.sample(data_subtype_idx, k) - test_list = [item for item in data_subtype_idx if item not in train_list] - train_idx += train_list - test_idx += test_list - return train_idx, test_idx - - -def feature_extract_drug_only( - data_idx: list, - drug_feature: dict, - max_atoms: int, -) -> Tuple[list, np.ndarray, np.ndarray, list]: - """ - 只为每个样本构造药物图特征 + 目标值。cell embedding 之后再从 dict 中查。 - """ - nb_instance = len(data_idx) - drug_data = [[] for _ in range(nb_instance)] - target = np.zeros(nb_instance, dtype=np.float32) - cancer_type_list = [] - - for idx in range(nb_instance): - cell_line_id, pubchem_id, ln_ic50, cancer_type = data_idx[idx] - feat_mat, adj_list, _ = drug_feature[str(pubchem_id)] - feat, adj = calculate_graph_feat(feat_mat, adj_list, max_atoms) - drug_data[idx] = [feat, adj] - target[idx] = ln_ic50 - cancer_type_list.append([cancer_type, cell_line_id, pubchem_id]) - - return drug_data, target, cancer_type_list, data_idx - - -# ---------------------- Dataset & Model ---------------------- - -class SCFMDrugDataset(Dataset): - """ - Dataset 返回: - x_drug : (N_atoms, F_d) - a_drug : (N_atoms, N_atoms) - cell_emb : (D,) - y : scalar IC50 - """ - - def __init__( - self, - drug_data: list, - targets: np.ndarray, - index_list: list, - cell_emb_dict: Dict[str, np.ndarray], - ): - super().__init__() - self.drug_data = drug_data - self.targets = targets.astype(np.float32) - self.index_list = index_list - self.cell_emb_dict = cell_emb_dict - - def __len__(self) -> int: - return int(self.targets.shape[0]) - - def __getitem__(self, idx: int): - feat, adj = self.drug_data[idx] - x_drug = torch.from_numpy(feat).float() # (N, F_d) - a_drug = torch.from_numpy(adj).float() # (N, N) - - cell_line_id, pubchem_id, ln_ic50, cancer_type = self.index_list[idx] - cell_emb_np = self.cell_emb_dict[cell_line_id] - cell_emb = torch.from_numpy(cell_emb_np).float() # (D,) - - y = torch.tensor(self.targets[idx], dtype=torch.float32) - return x_drug, a_drug, cell_emb, y - - -def collate_fn_sclong(batch): - x_drug, a_drug, cell_emb, y = zip(*batch) - x_drug = torch.stack(x_drug, dim=0) # (B, N, F_d) - a_drug = torch.stack(a_drug, dim=0) # (B, N, N) - cell_emb = torch.stack(cell_emb, dim=0) # (B, D) - y = torch.stack(y, dim=0) # (B,) - return x_drug, a_drug, cell_emb, y - - -class SimpleSCFMDrugModel(nn.Module): - """ - 简单的模型: - - 药物:按原子维平均池化 + Linear(F_d -> 128) - - 细胞:scLong embedding Linear(D -> 128) - - 拼接后:Linear(256 -> 256) -> Dropout -> Linear(256 -> 1) - """ - - def __init__(self, drug_feat_dim: int, cell_emb_dim: int, hidden_dim: int = 256): - super().__init__() - self.drug_fc = nn.Linear(drug_feat_dim, 128) - self.cell_fc = nn.Linear(cell_emb_dim, 128) - self.fc1 = nn.Linear(128 + 128, hidden_dim) - self.fc2 = nn.Linear(hidden_dim, 1) - self.dropout = nn.Dropout(0.1) - - def forward( - self, - drug_feat: torch.Tensor, # (B, N, F_d) - drug_adj: torch.Tensor, # (B, N, N) # 当前未使用,占位 - cell_emb: torch.Tensor, # (B, D) - ) -> torch.Tensor: - # 药物 branch: 对原子维做平均池化,得到 (B, F_d) - x_drug = drug_feat.mean(dim=1) # (B, F_d) - x_drug = torch.relu(self.drug_fc(x_drug)) # (B, 128) - - # 细胞 branch: 直接对 scLong embedding 做一层线性变换 - x_cell = torch.relu(self.cell_fc(cell_emb)) # (B, 128) - - h = torch.cat([x_drug, x_cell], dim=1) # (B, 256) - h = torch.relu(self.fc1(h)) - h = self.dropout(h) - out = self.fc2(h) # (B, 1) - return out - - -# ---------------------- 训练与评估 ---------------------- - -def train_epoch(model, loader, optimizer, criterion, device): - model.train() - total_loss = 0.0 - total = 0 - for x_drug, a_drug, cell_emb, y in loader: - x_drug = x_drug.to(device) - a_drug = a_drug.to(device) - cell_emb = cell_emb.to(device) - y = y.to(device) - - optimizer.zero_grad(set_to_none=True) - pred = model(x_drug, a_drug, cell_emb).squeeze(-1) - loss = criterion(pred, y) - loss.backward() - optimizer.step() - - total_loss += float(loss.item()) * y.size(0) - total += y.size(0) - return total_loss / total if total > 0 else 0.0 - - -def eval_epoch(model, loader, device): - model.eval() - preds = [] - trues = [] - with torch.no_grad(): - for x_drug, a_drug, cell_emb, y in loader: - x_drug = x_drug.to(device) - a_drug = a_drug.to(device) - cell_emb = cell_emb.to(device) - y = y.to(device) - - pred = model(x_drug, a_drug, cell_emb).squeeze(-1) - preds.append(pred.cpu().numpy()) - trues.append(y.cpu().numpy()) - preds = np.concatenate(preds, axis=0) - trues = np.concatenate(trues, axis=0) - pcc = pearsonr(preds, trues)[0] - return preds, trues, pcc - - -def main(): - parser = argparse.ArgumentParser(description="Use scLong (SCFM) cell embeddings for drug response prediction") - parser.add_argument("--gpu_id", type=str, default="0") - parser.add_argument("--data_path", type=str, default="./data") - parser.add_argument("--epochs", type=int, default=50) - parser.add_argument("--batch_size", type=int, default=64) - parser.add_argument("--max_atoms", type=int, default=100) - # 添加新的参数选项 - parser.add_argument("--use_graph_conv", action="store_true", help="是否使用图卷积处理药物特征") - parser.add_argument("--use_multi_source", action="store_true", help="是否使用多源数据(基因突变+甲基化)") - - # scLong / SCFM 相关路径(需要根据你的环境修改) - parser.add_argument("--gene2vec_path", type=str, - default="../selected_gene2vec_27k.npy") - parser.add_argument("--scfm_hyper_params_path", type=str, - default="../gocont_4096_48m_pretrain_1b_mix.pkl") - parser.add_argument( - "--scfm_ckpt_path", - type=str, - default="../gocont_4096_48m_pretrain_1b_mix_2024-02-05_16-23-37.pth", - help="Path to pretrained SCFM checkpoint .pth; use 'none' to initialize randomly.", - ) - parser.add_argument("--symbol2ens_path", type=str, - default="../gene2ens.csv") - parser.add_argument("--scfm_genes_path", type=str, - default="../selected_genes_27k.txt") - - args = parser.parse_args() - - os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu_id - device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - print_sys(f"Using device: {device}") - - dpath = args.data_path - drug_info_file = f"{dpath}/GDSC/1.Drug_listMon Jun 24 09_00_55 2019.csv" - cell_line_info_file = f"{dpath}/CCLE/Cell_lines_annotations_20181226.txt" - drug_feature_dir = f"{dpath}/GDSC/drug_graph_feat" - genomic_mutation_file = ( - f"{dpath}/CCLE/genomic_mutation_34673_demap_features.csv" - ) - gene_expression_file = ( - f"{dpath}/CCLE/genomic_expression_561celllines_697genes_demap_features.csv" - ) - methylation_file = ( - f"{dpath}/CCLE/genomic_methylation_561celllines_808genes_demap_features.csv" - ) - - random.seed(0) - np.random.seed(0) - torch.manual_seed(0) - - ( - mutation_feature, - drug_feature, - gexpr_feature, - methylation_feature, - data_idx, - ) = metadata_generate( - drug_info_file, - cell_line_info_file, - genomic_mutation_file, - drug_feature_dir, - gene_expression_file, - methylation_file, - ) - - # 用 scLong 预计算每个 cell line 的 embedding - cell_emb_dict, cell_emb_dim = build_cell_embeddings_with_sclong( - gexpr_feature=gexpr_feature, - symbol2ens_path=args.symbol2ens_path, - scfm_genes_path=args.scfm_genes_path, - gene2vec_path=args.gene2vec_path, - scfm_hyper_params_path=args.scfm_hyper_params_path, - scfm_ckpt_path=args.scfm_ckpt_path, - device=device, - ) - - train_idx, test_idx = data_split(data_idx) - - ( - X_drug_train, - Y_train, - train_cancer_info, - train_index_list, - ) = feature_extract_drug_only( - train_idx, - drug_feature, - args.max_atoms, - ) - - ( - X_drug_test, - Y_test, - test_cancer_info, - test_index_list, - ) = feature_extract_drug_only( - test_idx, - drug_feature, - args.max_atoms, - ) - - # 推导药物特征维度 - drug_dim = X_drug_train[0][0].shape[-1] - - train_ds = SCFMDrugDataset( - X_drug_train, Y_train, train_index_list, cell_emb_dict - ) - test_ds = SCFMDrugDataset( - X_drug_test, Y_test, test_index_list, cell_emb_dict - ) - num_workers = min(4, os.cpu_count() or 1) - train_loader = DataLoader( - train_ds, - batch_size=args.batch_size, - shuffle=True, - collate_fn=collate_fn_sclong, - num_workers=num_workers, - pin_memory=torch.cuda.is_available(), - ) - test_loader = DataLoader( - test_ds, - batch_size=args.batch_size, - shuffle=False, - collate_fn=collate_fn_sclong, - num_workers=num_workers, - pin_memory=torch.cuda.is_available(), - ) - - model = SimpleSCFMDrugModel( - drug_feat_dim=drug_dim, - cell_emb_dim=cell_emb_dim, - hidden_dim=256, - ).to(device) - - optimizer = torch.optim.Adam(model.parameters(), lr=1e-3) - criterion = torch.nn.MSELoss() - - best_pcc = -1.0 - best_pred = None - best_true = None - - for epoch in range(1, args.epochs + 1): - train_loss = train_epoch(model, train_loader, optimizer, criterion, device) - preds, trues, pcc = eval_epoch(model, test_loader, device) - print_sys( - f"Epoch {epoch:03d} | train_loss={train_loss:.4f} | val_pcc={pcc:.4f}" - ) - if pcc > best_pcc: - best_pcc = pcc - best_pred = preds - best_true = trues - - if best_pred is not None: - np.savez( - os.path.join(dpath, "SCFM_drug_response_results.npz"), - pred=best_pred, - test=best_true, - ) - print_sys(f"Best PCC: {best_pcc:.4f}") - - -if __name__ == "__main__": - main() - - +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +Use pretrained scLong (DualEncoderSCFM) cell embeddings + simple drug features +to predict drug response (IC50), reusing the DeepCDR data pipeline and scLong zero-shot embeddings. + +Data flow: + 1) metadata_generate: read drug/cell metadata and construct (cell_line, pubchem_id, ln_ic50, cancer_type) records. + 2) build_cell_embeddings_with_sclong: encode each cell line from its CCLE expression profile with scLong. + 3) feature_extract_drug_only: construct drug graph features (feat, adj) for each sample. + 4) SCFMDrugDataset: return (x_drug, a_drug, cell_emb, y). + 5) SimpleSCFMDrugModel: concatenate drug and cell features and predict IC50 with an MLP. +""" + +import argparse +from pathlib import Path + +SCRIPT_DIR = Path(__file__).resolve().parent +PROJECT_ROOT = SCRIPT_DIR.parents[4] +ASSET_DIR = PROJECT_ROOT / "model_dependencies" +import os +import random +from typing import Dict, List, Tuple + +import hickle as hkl +import numpy as np +import pandas as pd +import scipy.sparse as sp +import torch +import torch.nn as nn +from scipy.stats import pearsonr +from torch.utils.data import DataLoader, Dataset + + +# ---------------------- scLong / SCFM utilities ---------------------- +import pickle + +import sys +sys.path.insert(0, str(ASSET_DIR)) +from performer_pytorch_cont.ding_models import DualEncoderSCFM + + +def print_sys(*args, **kwargs): + """Print immediately, matching the zero-shot script logging style.""" + print(*args, **kwargs, flush=True) + + +def reindex_tensor_universal(tensor: torch.Tensor, + index_positions: List[int], + dim: int, + filler: str = "0", + device: str = "cpu") -> torch.Tensor: + """ + Reindex dimension dim using index_positions; fill missing positions with filler. + Adapted from the zero-shot script. + """ + index_tensor = torch.tensor(index_positions, device=device) + tensor_shape = list(tensor.shape) + tensor_shape[dim] = len(index_positions) + index_shape = [1] * len(tensor_shape) + index_shape[dim] = len(index_positions) + index_tensor = index_tensor.view(*index_shape).expand(*tensor_shape) + + if filler == "0": + padder_shape = list(tensor_shape) + padder_shape[dim] = 1 + padder = torch.zeros(padder_shape, dtype=tensor.dtype, device=device) + elif filler == "mean": + padder = tensor.mean(dim=dim, keepdim=True) + else: + raise ValueError("filler should be '0' or 'mean'") + + expanded_tensor = torch.cat((tensor, padder), dim=dim) + reindexed = torch.gather(expanded_tensor, dim, index_tensor) + return reindexed + + +def load_sclong_model( + gene2vec_path: str, + scfm_hyper_params_path: str, + scfm_ckpt_path: str, + device: torch.device, +) -> DualEncoderSCFM: + """ + Load the scLong (DualEncoderSCFM) model. + + - Load pretrained weights when scfm_ckpt_path is supplied. + - If omitted or set to 'none', initialize random weights from hyper_params. + """ + with open(scfm_hyper_params_path, "rb") as f: + scfm_hyper_params = pickle.load(f) + scfm_hyper_params["gene2vec_file"] = gene2vec_path + print_sys("SCFM hyper-params:", scfm_hyper_params) + + model = DualEncoderSCFM(**scfm_hyper_params).to(device) + + if scfm_ckpt_path and str(scfm_ckpt_path).lower() != "none": + print_sys(f"Loading SCFM checkpoint from: {scfm_ckpt_path}") + ckpt = torch.load(scfm_ckpt_path, map_location=device) + model.load_state_dict(ckpt["model_state_dict"]) + print_sys("Loaded pretrained SCFM weights.") + else: + print_sys("No SCFM checkpoint provided, using randomly initialized SCFM.") + + model.eval() + for p in model.parameters(): + p.requires_grad = False + return model + + +def build_scfm_gene_mapping( + symbol2ens_path: str, + scfm_genes_path: str, + expr_genes: List[str], +) -> Tuple[Dict[str, int], List[int]]: + """ + Build reindexing positions from CCLE expression columns to the scFM 27k gene order. + + Returns: + - input_mapping: dict[input_gene_id] -> column index + - scfm_index_positions: input column for each scfm_genes_pad entry; absent genes use the final padding column. + """ + # Load the gene-symbol to Ensembl mapping. + # with open(symbol2ens_path, "r") as f: + # symbol2ens = { + # line.split(",")[0]: line.split(",")[1].rstrip("\n") + # for line in f + # if line.strip() and line.split(",")[1].rstrip("\n") != "unknown" + # } + gene2ens = pd.read_csv(symbol2ens_path,index_col=0) + symbol2ens = gene2ens['Ensembl gene ID'].to_dict() + + # Map CCLE expression column names to Ensembl IDs; retain unmapped names. + input_genes = [ + symbol2ens.get(sym, sym) for sym in expr_genes + ] + input_gene_num = len(input_genes) + input_mapping = {gid: i for i, gid in enumerate(input_genes)} + + # Load the 27k genes used for scFM pretraining. + with open(scfm_genes_path, "r") as f: + scfm_genes = [line.rstrip("\n") for line in f.readlines()] + scfm_genes_pad = scfm_genes + ["PAD"] + + # Locate each scFM gene in the input; use input_gene_num (padding) for missing genes. + scfm_index_positions = [ + input_mapping.get(gid, input_gene_num) for gid in scfm_genes_pad + ] + print_sys(f"target and scfm genes intersect: {len(np.intersect1d(input_genes, scfm_genes))}") + return input_mapping, scfm_index_positions + + +def build_cell_embeddings_with_sclong( + gexpr_feature: pd.DataFrame, + symbol2ens_path: str, + scfm_genes_path: str, + gene2vec_path: str, + scfm_hyper_params_path: str, + scfm_ckpt_path: str, + device: torch.device, +) -> Tuple[Dict[str, np.ndarray], int]: + """ + Build a scLong embedding for each cell line. + + gexpr_feature: rows are cell_line_id values; columns are gene names. + Returns: + - cell_emb_dict: {cell_line_id: embedding (np.ndarray, shape=(D,))} + - emb_dim: embedding dimension D + """ + # Load the SCFM model. + model = load_sclong_model( + gene2vec_path=gene2vec_path, + scfm_hyper_params_path=scfm_hyper_params_path, + scfm_ckpt_path=scfm_ckpt_path, + device=device, + ) + + # Build the gene reindexing map. + _, scfm_index_positions = build_scfm_gene_mapping( + symbol2ens_path=symbol2ens_path, + scfm_genes_path=scfm_genes_path, + expr_genes=list(gexpr_feature.columns), + ) + + cell_emb_dict: Dict[str, np.ndarray] = {} + first_emb_dim: int = -1 + + with torch.no_grad(): + for cell_line_id, row in gexpr_feature.iterrows(): + x = torch.tensor(row.values[None, :], dtype=torch.float32, device=device) # (1, num_genes) + x_scfm = reindex_tensor_universal( + x, scfm_index_positions, dim=1, filler="0", device=device + ) # (1, seq_len=27k+1) + # cell_emb = model.get_cell_emb(x_scfm) # (1, D) + out = model(x_scfm, return_encodings=True) + cell_emb = out['merged_decodings'][:, 0, :] # Select the CLS token at position 0. + cell_emb_np = cell_emb.squeeze(0).cpu().numpy() + cell_emb_dict[cell_line_id] = cell_emb_np + if first_emb_dim < 0: + first_emb_dim = cell_emb_np.shape[0] + + print_sys(f"Built scLong embeddings for {len(cell_emb_dict)} cell lines, dim={first_emb_dim}") + return cell_emb_dict, first_emb_dim + + +# ---------------------- DeepCDR-style drug and IC50 inputs ---------------------- + +TCGA_LABEL_SET = [ + "ALL", + "BLCA", + "BRCA", + "CESC", + "DLBC", + "LIHC", + "LUAD", + "ESCA", + "GBM", + "HNSC", + "KIRC", + "LAML", + "LCML", + "LGG", + "LUSC", + "MESO", + "MM", + "NB", + "OV", + "PAAD", + "SCLC", + "SKCM", + "STAD", + "THCA", + "COAD/READ", +] + + +def normalize_adj(adj: np.ndarray) -> np.ndarray: + adj = adj + np.eye(adj.shape[0]) + d = sp.diags(np.power(np.array(adj.sum(1)), -0.5).flatten(), 0).toarray() + a_norm = adj.dot(d).transpose().dot(d) + return a_norm + + +def calculate_graph_feat( + feat_mat: np.ndarray, + adj_list: List[List[int]], + max_atoms: int, +) -> Tuple[np.ndarray, np.ndarray]: + """Preprocess drug graph features as in train_deepcdr_torch.py.""" + assert feat_mat.shape[0] == len(adj_list) + feat = np.zeros((max_atoms, feat_mat.shape[-1]), dtype=np.float32) + adj_mat = np.zeros((max_atoms, max_atoms), dtype=np.float32) + + feat[: feat_mat.shape[0], :] = feat_mat + for i, nodes in enumerate(adj_list): + for each in nodes: + adj_mat[i, int(each)] = 1 + + assert np.allclose(adj_mat, adj_mat.T) + adj_ = adj_mat[: len(adj_list), : len(adj_list)] + norm_adj_ = normalize_adj(adj_) + adj_mat[: len(adj_list), : len(adj_list)] = norm_adj_ + return feat, adj_mat + + +def metadata_generate( + drug_info_file: str, + cell_line_info_file: str, + genomic_mutation_file: str, + drug_feature_dir: str, + gene_expression_file: str, + methylation_file: str, +) -> Tuple[pd.DataFrame, dict, pd.DataFrame, pd.DataFrame, list]: + """ + Build metadata as in train_deepcdr_torch.py, using drug_feature, gexpr_feature, and IC50. + """ + # drug_id --> pubchem_id + reader = pd.read_csv(drug_info_file) + drugid2pubchemid = { + str(row.iloc[0]): str(row.iloc[5]) + for _, row in reader.iterrows() + if str(row.iloc[5]).isdigit() + } + + # map cellline --> cancer type + cellline2cancertype = {} + with open(cell_line_info_file, "r") as f: + for line in f.readlines()[1:]: + cellline_id = line.split("\t")[1] + tcga_label = line.strip().split("\t")[-1] + cellline2cancertype[cellline_id] = tcga_label + + mutation_feature = pd.read_csv( + genomic_mutation_file, sep=",", header=0, index_col=0 + ) + gexpr_feature = pd.read_csv( + gene_expression_file, sep=",", header=0, index_col=0 + ) + # only keep overlapped cell lines + mutation_feature = mutation_feature.loc[list(gexpr_feature.index)] + + methylation_feature = pd.read_csv( + methylation_file, sep=",", header=0, index_col=0 + ) + + # load drug graph features (.hkl) + drug_feature = {} + drug_pubchem_id_set = [] + for fname in os.listdir(drug_feature_dir): + if not fname.endswith(".hkl"): + continue + pubchem_id = fname.split(".")[0] + feat_mat, adj_list, degree_list = hkl.load( + os.path.join(drug_feature_dir, fname) + ) + drug_feature[pubchem_id] = [feat_mat, adj_list, degree_list] + drug_pubchem_id_set.append(pubchem_id) + + # IC50 + cancer_response_exp_file = os.path.join( + os.path.dirname(gene_expression_file), "GDSC_IC50.csv" + ) + experiment_data = pd.read_csv( + cancer_response_exp_file, sep=",", header=0, index_col=0 + ) + drug_match_list = [ + item + for item in experiment_data.index + if item.split(":")[1] in drugid2pubchemid.keys() + ] + experiment_data_filtered = experiment_data.loc[drug_match_list] + + data_idx = [] + for each_drug in experiment_data_filtered.index: + for each_cellline in experiment_data_filtered.columns: + pubchem_id = drugid2pubchemid[each_drug.split(":")[-1]] + if ( + str(pubchem_id) in drug_pubchem_id_set + and each_cellline in mutation_feature.index + ): + if ( + not np.isnan( + experiment_data_filtered.loc[each_drug, each_cellline] + ) + and each_cellline in cellline2cancertype.keys() + ): + ln_ic50 = float( + experiment_data_filtered.loc[each_drug, each_cellline] + ) + data_idx.append( + ( + each_cellline, + pubchem_id, + ln_ic50, + cellline2cancertype[each_cellline], + ) + ) + + nb_celllines = len(set([item[0] for item in data_idx])) + nb_drugs = len(set([item[1] for item in data_idx])) + print_sys( + f"{len(data_idx)} instances across {nb_celllines} cell lines and {nb_drugs} drugs were generated." + ) + return mutation_feature, drug_feature, gexpr_feature, methylation_feature, data_idx + + +def data_split(data_idx: list, ratio: float = 0.95): + """Stratify the split by cancer type, following DeepCDR.""" + train_idx, test_idx = [], [] + for each_type in TCGA_LABEL_SET: + data_subtype_idx = [item for item in data_idx if item[-1] == each_type] + if len(data_subtype_idx) == 0: + continue + k = max(1, int(ratio * len(data_subtype_idx))) + train_list = random.sample(data_subtype_idx, k) + test_list = [item for item in data_subtype_idx if item not in train_list] + train_idx += train_list + test_idx += test_list + return train_idx, test_idx + + +def feature_extract_drug_only( + data_idx: list, + drug_feature: dict, + max_atoms: int, +) -> Tuple[list, np.ndarray, np.ndarray, list]: + """ + Construct drug graph features and targets; look up cell embeddings from the dictionary later. + """ + nb_instance = len(data_idx) + drug_data = [[] for _ in range(nb_instance)] + target = np.zeros(nb_instance, dtype=np.float32) + cancer_type_list = [] + + for idx in range(nb_instance): + cell_line_id, pubchem_id, ln_ic50, cancer_type = data_idx[idx] + feat_mat, adj_list, _ = drug_feature[str(pubchem_id)] + feat, adj = calculate_graph_feat(feat_mat, adj_list, max_atoms) + drug_data[idx] = [feat, adj] + target[idx] = ln_ic50 + cancer_type_list.append([cancer_type, cell_line_id, pubchem_id]) + + return drug_data, target, cancer_type_list, data_idx + + +# ---------------------- Dataset & Model ---------------------- + +class SCFMDrugDataset(Dataset): + """ + Dataset returns: + x_drug : (N_atoms, F_d) + a_drug : (N_atoms, N_atoms) + cell_emb : (D,) + y : scalar IC50 + """ + + def __init__( + self, + drug_data: list, + targets: np.ndarray, + index_list: list, + cell_emb_dict: Dict[str, np.ndarray], + ): + super().__init__() + self.drug_data = drug_data + self.targets = targets.astype(np.float32) + self.index_list = index_list + self.cell_emb_dict = cell_emb_dict + + def __len__(self) -> int: + return int(self.targets.shape[0]) + + def __getitem__(self, idx: int): + feat, adj = self.drug_data[idx] + x_drug = torch.from_numpy(feat).float() # (N, F_d) + a_drug = torch.from_numpy(adj).float() # (N, N) + + cell_line_id, pubchem_id, ln_ic50, cancer_type = self.index_list[idx] + cell_emb_np = self.cell_emb_dict[cell_line_id] + cell_emb = torch.from_numpy(cell_emb_np).float() # (D,) + + y = torch.tensor(self.targets[idx], dtype=torch.float32) + return x_drug, a_drug, cell_emb, y + + +def collate_fn_sclong(batch): + x_drug, a_drug, cell_emb, y = zip(*batch) + x_drug = torch.stack(x_drug, dim=0) # (B, N, F_d) + a_drug = torch.stack(a_drug, dim=0) # (B, N, N) + cell_emb = torch.stack(cell_emb, dim=0) # (B, D) + y = torch.stack(y, dim=0) # (B,) + return x_drug, a_drug, cell_emb, y + + +class SimpleSCFMDrugModel(nn.Module): + """ + Simple prediction model: + - Drug: mean pooling over atoms, then Linear(F_d -> 128). + - Cell: project scLong embeddings with Linear(D -> 128). + - Concatenation: Linear(256 -> 256) -> Dropout -> Linear(256 -> 1). + """ + + def __init__(self, drug_feat_dim: int, cell_emb_dim: int, hidden_dim: int = 256): + super().__init__() + self.drug_fc = nn.Linear(drug_feat_dim, 128) + self.cell_fc = nn.Linear(cell_emb_dim, 128) + self.fc1 = nn.Linear(128 + 128, hidden_dim) + self.fc2 = nn.Linear(hidden_dim, 1) + self.dropout = nn.Dropout(0.1) + + def forward( + self, + drug_feat: torch.Tensor, # (B, N, F_d) + drug_adj: torch.Tensor, # (B, N, N); retained for interface compatibility and not used here. + cell_emb: torch.Tensor, # (B, D) + ) -> torch.Tensor: + # Drug branch: average over atoms to obtain (B, F_d). + x_drug = drug_feat.mean(dim=1) # (B, F_d) + x_drug = torch.relu(self.drug_fc(x_drug)) # (B, 128) + + # Cell branch: apply a linear projection to the scLong embedding. + x_cell = torch.relu(self.cell_fc(cell_emb)) # (B, 128) + + h = torch.cat([x_drug, x_cell], dim=1) # (B, 256) + h = torch.relu(self.fc1(h)) + h = self.dropout(h) + out = self.fc2(h) # (B, 1) + return out + + +# ---------------------- Training and evaluation ---------------------- + +def train_epoch(model, loader, optimizer, criterion, device): + model.train() + total_loss = 0.0 + total = 0 + for x_drug, a_drug, cell_emb, y in loader: + x_drug = x_drug.to(device) + a_drug = a_drug.to(device) + cell_emb = cell_emb.to(device) + y = y.to(device) + + optimizer.zero_grad(set_to_none=True) + pred = model(x_drug, a_drug, cell_emb).squeeze(-1) + loss = criterion(pred, y) + loss.backward() + optimizer.step() + + total_loss += float(loss.item()) * y.size(0) + total += y.size(0) + return total_loss / total if total > 0 else 0.0 + + +def eval_epoch(model, loader, device): + model.eval() + preds = [] + trues = [] + with torch.no_grad(): + for x_drug, a_drug, cell_emb, y in loader: + x_drug = x_drug.to(device) + a_drug = a_drug.to(device) + cell_emb = cell_emb.to(device) + y = y.to(device) + + pred = model(x_drug, a_drug, cell_emb).squeeze(-1) + preds.append(pred.cpu().numpy()) + trues.append(y.cpu().numpy()) + preds = np.concatenate(preds, axis=0) + trues = np.concatenate(trues, axis=0) + pcc = pearsonr(preds, trues)[0] + return preds, trues, pcc + + +def main(): + parser = argparse.ArgumentParser(description="Use scLong (SCFM) cell embeddings for drug response prediction") + parser.add_argument("--gpu_id", type=str, default="0") + parser.add_argument("--data_path", type=str, default=str(SCRIPT_DIR / "data")) + parser.add_argument("--epochs", type=int, default=50) + parser.add_argument("--batch_size", type=int, default=64) + parser.add_argument("--max_atoms", type=int, default=100) + # Configure optional model branches. + parser.add_argument("--use_graph_conv", action="store_true", help="Use graph convolution for drug features.") + parser.add_argument("--use_multi_source", action="store_true", help="Use additional mutation and methylation features.") + + # scLong / SCFM paths; supply the matching legacy checkpoint and configuration. + parser.add_argument("--gene2vec_path", type=str, + default=str(ASSET_DIR / "selected_gene2vec_27k.npy")) + parser.add_argument("--scfm_hyper_params_path", type=str, + required=True, help="Path to the matching legacy scLong hyperparameter file (not bundled).") + parser.add_argument( + "--scfm_ckpt_path", + type=str, + required=True, + help="Path to pretrained SCFM checkpoint .pth; use 'none' to initialize randomly.", + ) + parser.add_argument("--symbol2ens_path", type=str, + default=str(ASSET_DIR / "gene2ens.csv")) + parser.add_argument("--scfm_genes_path", type=str, + default=str(ASSET_DIR / "selected_genes_27k.txt")) + + args = parser.parse_args() + + os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu_id + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + print_sys(f"Using device: {device}") + + dpath = args.data_path + drug_info_file = f"{dpath}/GDSC/1.Drug_listMon Jun 24 09_00_55 2019.csv" + cell_line_info_file = f"{dpath}/CCLE/Cell_lines_annotations_20181226.txt" + drug_feature_dir = f"{dpath}/GDSC/drug_graph_feat" + genomic_mutation_file = ( + f"{dpath}/CCLE/genomic_mutation_34673_demap_features.csv" + ) + gene_expression_file = ( + f"{dpath}/CCLE/genomic_expression_561celllines_697genes_demap_features.csv" + ) + methylation_file = ( + f"{dpath}/CCLE/genomic_methylation_561celllines_808genes_demap_features.csv" + ) + + random.seed(0) + np.random.seed(0) + torch.manual_seed(0) + + ( + mutation_feature, + drug_feature, + gexpr_feature, + methylation_feature, + data_idx, + ) = metadata_generate( + drug_info_file, + cell_line_info_file, + genomic_mutation_file, + drug_feature_dir, + gene_expression_file, + methylation_file, + ) + + # Precompute a scLong embedding for each cell line. + cell_emb_dict, cell_emb_dim = build_cell_embeddings_with_sclong( + gexpr_feature=gexpr_feature, + symbol2ens_path=args.symbol2ens_path, + scfm_genes_path=args.scfm_genes_path, + gene2vec_path=args.gene2vec_path, + scfm_hyper_params_path=args.scfm_hyper_params_path, + scfm_ckpt_path=args.scfm_ckpt_path, + device=device, + ) + + train_idx, test_idx = data_split(data_idx) + + ( + X_drug_train, + Y_train, + train_cancer_info, + train_index_list, + ) = feature_extract_drug_only( + train_idx, + drug_feature, + args.max_atoms, + ) + + ( + X_drug_test, + Y_test, + test_cancer_info, + test_index_list, + ) = feature_extract_drug_only( + test_idx, + drug_feature, + args.max_atoms, + ) + + # Determine the drug feature dimension. + drug_dim = X_drug_train[0][0].shape[-1] + + train_ds = SCFMDrugDataset( + X_drug_train, Y_train, train_index_list, cell_emb_dict + ) + test_ds = SCFMDrugDataset( + X_drug_test, Y_test, test_index_list, cell_emb_dict + ) + num_workers = min(4, os.cpu_count() or 1) + train_loader = DataLoader( + train_ds, + batch_size=args.batch_size, + shuffle=True, + collate_fn=collate_fn_sclong, + num_workers=num_workers, + pin_memory=torch.cuda.is_available(), + ) + test_loader = DataLoader( + test_ds, + batch_size=args.batch_size, + shuffle=False, + collate_fn=collate_fn_sclong, + num_workers=num_workers, + pin_memory=torch.cuda.is_available(), + ) + + model = SimpleSCFMDrugModel( + drug_feat_dim=drug_dim, + cell_emb_dim=cell_emb_dim, + hidden_dim=256, + ).to(device) + + optimizer = torch.optim.Adam(model.parameters(), lr=1e-3) + criterion = torch.nn.MSELoss() + + best_pcc = -1.0 + best_pred = None + best_true = None + + for epoch in range(1, args.epochs + 1): + train_loss = train_epoch(model, train_loader, optimizer, criterion, device) + preds, trues, pcc = eval_epoch(model, test_loader, device) + print_sys( + f"Epoch {epoch:03d} | train_loss={train_loss:.4f} | val_pcc={pcc:.4f}" + ) + if pcc > best_pcc: + best_pcc = pcc + best_pred = preds + best_true = trues + + if best_pred is not None: + np.savez( + os.path.join(dpath, "SCFM_drug_response_results.npz"), + pred=best_pred, + test=best_true, + ) + print_sys(f"Best PCC: {best_pcc:.4f}") + + +if __name__ == "__main__": + main() + + diff --git a/data/downstream/final_data/drug_response_pytorch/drug_response_pytorch/train_sclong_drug_GCN_linear.py b/data/downstream/final_data/drug_response_pytorch/drug_response_pytorch/train_sclong_drug_GCN_linear.py index 5d69302fbf1ab8bb215cdd2c78ae5c2727491ca1..f27fa787b346874097fc8b8a110efa157105e8b2 100644 --- a/data/downstream/final_data/drug_response_pytorch/drug_response_pytorch/train_sclong_drug_GCN_linear.py +++ b/data/downstream/final_data/drug_response_pytorch/drug_response_pytorch/train_sclong_drug_GCN_linear.py @@ -2,17 +2,22 @@ # -*- coding: utf-8 -*- """ Use pretrained scLong (DualEncoderSCFM) cell embeddings + simple drug features -to predict drug response (IC50), reusing DeepCDR 的数据管线 + scLong zero-shot 的 embedding 逻辑。 - -数据流程: - 1) metadata_generate: 读取药物 / 细胞系信息,构造 (cell_line, pubchem_id, ln_ic50, cancer_type) 列表; - 2) build_cell_embeddings_with_sclong: 用 CCLE 表达矩阵 + scLong,把每个 cell line 映射到一个 cell embedding; - 3) feature_extract_drug_only: 为每个样本构造药物图特征 (feat, adj); - 4) SCFMDrugDataset: 返回 (x_drug, a_drug, cell_emb, y); - 5) SimpleSCFMDrugModel: 把药物特征和 cell embedding 拼接,经 MLP 预测 IC50。 +to predict drug response (IC50), reusing the DeepCDR data pipeline and scLong zero-shot embeddings. + +Data flow: + 1) metadata_generate: read drug/cell metadata and construct (cell_line, pubchem_id, ln_ic50, cancer_type) records. + 2) build_cell_embeddings_with_sclong: encode each cell line from its CCLE expression profile with scLong. + 3) feature_extract_drug_only: construct drug graph features (feat, adj) for each sample. + 4) SCFMDrugDataset: return (x_drug, a_drug, cell_emb, y). + 5) SimpleSCFMDrugModel: concatenate drug and cell features and predict IC50 with an MLP. """ import argparse +from pathlib import Path + +SCRIPT_DIR = Path(__file__).resolve().parent +PROJECT_ROOT = SCRIPT_DIR.parents[4] +ASSET_DIR = PROJECT_ROOT / "model_dependencies" import os import random from typing import Dict, List, Tuple @@ -27,16 +32,16 @@ from scipy.stats import pearsonr from torch.utils.data import DataLoader, Dataset -# ---------------------- scLong / SCFM 相关 ---------------------- +# ---------------------- scLong / SCFM utilities ---------------------- import pickle import sys -sys.path.append("../") +sys.path.insert(0, str(ASSET_DIR)) from performer_pytorch_cont.ding_models import DualEncoderSCFM def print_sys(*args, **kwargs): - """简单的打印包装,保持和 zero-shot 脚本风格一致。""" + """Print immediately, matching the zero-shot script logging style.""" print(*args, **kwargs, flush=True) @@ -46,8 +51,8 @@ def reindex_tensor_universal(tensor: torch.Tensor, filler: str = "0", device: str = "cpu") -> torch.Tensor: """ - 通用重排操作:沿着某一维 dim 按 index_positions 重新索引,超出索引位置用 filler 填充。 - 来自 zero-shot 脚本。 + Reindex dimension dim using index_positions; fill missing positions with filler. + Adapted from the zero-shot script. """ index_tensor = torch.tensor(index_positions, device=device) tensor_shape = list(tensor.shape) @@ -77,10 +82,10 @@ def load_sclong_model( device: torch.device, ) -> DualEncoderSCFM: """ - 加载 scLong (DualEncoderSCFM) 模型。 + Load the scLong (DualEncoderSCFM) model. - - 如果提供了 scfm_ckpt_path,则加载预训练权重; - - 如果未提供或为 'none',则仅根据 hyper_params 初始化一个随机权重的模型。 + - Load pretrained weights when scfm_ckpt_path is supplied. + - If omitted or set to 'none', initialize random weights from hyper_params. """ with open(scfm_hyper_params_path, "rb") as f: scfm_hyper_params = pickle.load(f) @@ -109,35 +114,35 @@ def build_scfm_gene_mapping( expr_genes: List[str], ) -> Tuple[Dict[str, int], List[int]]: """ - 构建从 CCLE 表达矩阵的列 (基因名) 到 scFM 27k 基因的重排索引。 + Build reindexing positions from CCLE expression columns to the scFM 27k gene order. - 返回: - - input_mapping: dict[input_gene_id] -> 列索引 - - scfm_index_positions: 对于每个 scfm_genes_pad 中的基因,给出在 input_genes 中的列位置(找不到则指向末尾 pad)。 + Returns: + - input_mapping: dict[input_gene_id] -> column index + - scfm_index_positions: input column for each scfm_genes_pad entry; absent genes use the final padding column. """ - # 加载 symbol -> ENS 映射 + # Load the gene-symbol to Ensembl mapping. # with open(symbol2ens_path, "r") as f: # symbol2ens = { # line.split(",")[0]: line.split(",")[1].rstrip("\n") # for line in f # if line.strip() and line.split(",")[1].rstrip("\n") != "unknown" # } - gene2ens = pd.read_csv('../gene2ens.csv',index_col=0) + gene2ens = pd.read_csv(symbol2ens_path,index_col=0) symbol2ens = gene2ens['Ensembl gene ID'].to_dict() - # 把 CCLE 表达矩阵的列名映射到 ENS,如果没有映射就保持原始名字 + # Map CCLE expression column names to Ensembl IDs; retain unmapped names. input_genes = [ symbol2ens.get(sym, sym) for sym in expr_genes ] input_gene_num = len(input_genes) input_mapping = {gid: i for i, gid in enumerate(input_genes)} - # 加载 scFM 预训练使用的 27k 基因 + # Load the 27k genes used for scFM pretraining. with open(scfm_genes_path, "r") as f: scfm_genes = [line.rstrip("\n") for line in f.readlines()] scfm_genes_pad = scfm_genes + ["PAD"] - # 对每个 scFM 基因,找到它在输入中的位置;若不存在则用 input_gene_num(pad) + # Locate each scFM gene in the input; use input_gene_num (padding) for missing genes. scfm_index_positions = [ input_mapping.get(gid, input_gene_num) for gid in scfm_genes_pad ] @@ -155,14 +160,14 @@ def build_cell_embeddings_with_sclong( device: torch.device, ) -> Tuple[Dict[str, np.ndarray], int]: """ - 用 scLong 为每个 cell line 构造一个 cell embedding。 + Build a scLong embedding for each cell line. - gexpr_feature: index 是 cell_line_id,columns 是基因名。 - 返回: + gexpr_feature: rows are cell_line_id values; columns are gene names. + Returns: - cell_emb_dict: {cell_line_id: embedding (np.ndarray, shape=(D,))} - - emb_dim: embedding 维度 D + - emb_dim: embedding dimension D """ - # 加载 SCFM 模型 + # Load the SCFM model. model = load_sclong_model( gene2vec_path=gene2vec_path, scfm_hyper_params_path=scfm_hyper_params_path, @@ -170,7 +175,7 @@ def build_cell_embeddings_with_sclong( device=device, ) - # 构建基因重排映射 + # Build the gene reindexing map. _, scfm_index_positions = build_scfm_gene_mapping( symbol2ens_path=symbol2ens_path, scfm_genes_path=scfm_genes_path, @@ -188,7 +193,7 @@ def build_cell_embeddings_with_sclong( ) # (1, seq_len=27k+1) # cell_emb = model.get_cell_emb(x_scfm) # (1, D) out = model(x_scfm, return_encodings=True) - cell_emb = out['merged_decodings'][:, 0, :] # 只取第0位的CLS标记 + cell_emb = out['merged_decodings'][:, 0, :] # Select the CLS token at position 0. # print(cell_emb.shape) cell_emb_np = cell_emb.squeeze(0).cpu().numpy() cell_emb_dict[cell_line_id] = cell_emb_np @@ -199,7 +204,7 @@ def build_cell_embeddings_with_sclong( return cell_emb_dict, first_emb_dim -# ---------------------- DeepCDR 风格的数据准备(只保留 drug+IC50) ---------------------- +# ---------------------- DeepCDR-style drug and IC50 inputs ---------------------- TCGA_LABEL_SET = [ "ALL", @@ -242,7 +247,7 @@ def calculate_graph_feat( adj_list: List[List[int]], max_atoms: int, ) -> Tuple[np.ndarray, np.ndarray]: - """和 train_deepcdr_torch.py 一致的药物图特征预处理。""" + """Preprocess drug graph features as in train_deepcdr_torch.py.""" assert feat_mat.shape[0] == len(adj_list) feat = np.zeros((max_atoms, feat_mat.shape[-1]), dtype=np.float32) adj_mat = np.zeros((max_atoms, max_atoms), dtype=np.float32) @@ -268,7 +273,7 @@ def metadata_generate( methylation_file: str, ) -> Tuple[pd.DataFrame, dict, pd.DataFrame, pd.DataFrame, list]: """ - 与 train_deepcdr_torch.py 相同的元数据构造,只是最终模型只用 drug_feature + gexpr_feature + IC50。 + Build metadata as in train_deepcdr_torch.py, using drug_feature, gexpr_feature, and IC50. """ # drug_id --> pubchem_id reader = pd.read_csv(drug_info_file) @@ -361,7 +366,7 @@ def metadata_generate( def data_split(data_idx: list, ratio: float = 0.95): - """与 DeepCDR 相同的按 cancer type 分层划分。""" + """Stratify the split by cancer type, following DeepCDR.""" train_idx, test_idx = [], [] for each_type in TCGA_LABEL_SET: data_subtype_idx = [item for item in data_idx if item[-1] == each_type] @@ -381,7 +386,7 @@ def feature_extract_drug_only( max_atoms: int, ) -> Tuple[list, np.ndarray, np.ndarray, list]: """ - 只为每个样本构造药物图特征 + 目标值。cell embedding 之后再从 dict 中查。 + Construct drug graph features and targets; look up cell embeddings from the dictionary later. """ nb_instance = len(data_idx) drug_data = [[] for _ in range(nb_instance)] @@ -408,61 +413,61 @@ def feature_extract_drug_only( # max_atoms: int, # ) -> Tuple[list, np.ndarray, np.ndarray, np.ndarray, np.ndarray, list, list]: # """ -# 为每个样本构造完整的多源特征:药物图特征、基因突变、基因表达和甲基化特征。 +# Construct drug graph, mutation, expression, and methylation features for each sample. -# 参数: -# data_idx: 样本索引列表,每个元素是(cell_line_id, pubchem_id, ln_ic50, cancer_type)元组 -# drug_feature: 药物特征字典,键为pubchem_id,值为[feat_mat, adj_list, degree_list] -# mutation_feature: 基因突变特征DataFrame -# gexpr_feature: 基因表达特征DataFrame -# methylation_feature: 甲基化特征DataFrame -# max_atoms: 药物分子最大原子数 +# Arguments: +# data_idx: list of (cell_line_id, pubchem_id, ln_ic50, cancer_type) tuples. +# drug_feature: pubchem_id -> [feat_mat, adj_list, degree_list]. +# mutation_feature: mutation-feature DataFrame. +# gexpr_feature: gene-expression DataFrame. +# methylation_feature: methylation-feature DataFrame. +# max_atoms: maximum number of atoms per drug. -# 返回: -# drug_data: 药物图特征列表,每个元素是[feat, adj] -# mutation_data: 基因突变特征数组,形状为(nb_instance, 1, 1, nb_mutation_feature) -# gexpr_data: 基因表达特征数组,形状为(nb_instance, nb_gexpr_features) -# methylation_data: 甲基化特征数组,形状为(nb_instance, nb_methylation_features) -# target: 目标值数组(IC50) -# cancer_type_list: 癌症类型列表 -# data_idx: 原始数据索引列表 +# Returns: +# drug_data: list of [feat, adj] drug graph features. +# mutation_data: array with shape (nb_instance, 1, 1, nb_mutation_feature). +# gexpr_data: array with shape (nb_instance, nb_gexpr_features). +# methylation_data: array with shape (nb_instance, nb_methylation_features). +# target: array of IC50 targets. +# cancer_type_list: cancer type for each sample. +# data_idx: original sample index list. # """ # nb_instance = len(data_idx) # nb_mutation_feature = mutation_feature.shape[1] # nb_gexpr_features = gexpr_feature.shape[1] # nb_methylation_features = methylation_feature.shape[1] -# # 初始化数据结构 +# Initialize the output containers. # drug_data = [[] for _ in range(nb_instance)] -# # mutation_data使用(1,1,D)的形状以兼容后续处理 +# Use mutation shape (1, 1, D) for compatibility with downstream code. # mutation_data = np.zeros((nb_instance, 1, 1, nb_mutation_feature), dtype=np.float32) # gexpr_data = np.zeros((nb_instance, nb_gexpr_features), dtype=np.float32) # methylation_data = np.zeros((nb_instance, nb_methylation_features), dtype=np.float32) # target = np.zeros(nb_instance, dtype=np.float32) # cancer_type_list = [] -# # 为每个样本填充特征 +# Extract features for each sample. # for idx in range(nb_instance): # cell_line_id, pubchem_id, ln_ic50, cancer_type = data_idx[idx] -# # 处理药物特征 +# Extract drug features. # feat_mat, adj_list, _ = drug_feature[str(pubchem_id)] # feat, adj = calculate_graph_feat(feat_mat, adj_list, max_atoms) # drug_data[idx] = [feat, adj] -# # 处理基因突变特征 +# Extract mutation features. # mutation_data[idx, 0, 0, :] = mutation_feature.loc[cell_line_id].values -# # 处理基因表达特征 +# Extract expression features. # gexpr_data[idx, :] = gexpr_feature.loc[cell_line_id].values -# # 处理甲基化特征 +# Extract methylation features. # methylation_data[idx, :] = methylation_feature.loc[cell_line_id].values -# # 目标值 +# Target value. # target[idx] = ln_ic50 -# # 记录癌症类型信息 +# Record the cancer type. # cancer_type_list.append([cancer_type, cell_line_id, pubchem_id]) # return ( @@ -480,7 +485,7 @@ def feature_extract_drug_only( class SCFMDrugDataset(Dataset): """ - Dataset 返回: + Dataset returns: x_drug : (N_atoms, F_d) a_drug : (N_atoms, N_atoms) cell_emb : (D,) @@ -527,13 +532,13 @@ def collate_fn_sclong(batch): class GraphConv(nn.Module): """ - 图卷积层,用于处理药物分子的图结构信息。 + Graph convolution layer for drug molecular graphs. - 计算方式:h_i = sum_j A_ji * (x_j W + b) - 其中: - - x: (B, N, F_in) 节点特征 - - A: (B, N, N) 邻接矩阵 - - out: (B, N, F_out) 输出特征 + Computation: h_i = sum_j A_ji * (x_j W + b). + Shapes: + - x: (B, N, F_in) node features + - A: (B, N, N) adjacency matrix + - out: (B, N, F_out) output features """ def __init__(self, in_features: int, out_features: int, bias: bool = True): @@ -554,8 +559,8 @@ class GraphConv(nn.Module): def forward(self, x: torch.Tensor, adj: torch.Tensor) -> torch.Tensor: """ - x: (B, N, F_in) 节点特征 - adj: (B, N, N) 邻接矩阵 + x: (B, N, F_in) node features + adj: (B, N, N) adjacency matrix """ # (B, N, F_out) = (B, N, F_in) @ (F_in, F_out) xw = torch.matmul(x, self.weight) @@ -568,23 +573,23 @@ class GraphConv(nn.Module): class SimpleSCFMDrugModel(nn.Module): """ - 基于图卷积的药物响应预测模型: - - 药物:使用GraphConv处理图结构 + 全局平均池化 - - 细胞:scLong embedding 特征 - - 拼接后:全连接层预测IC50 + Graph-convolution drug-response model: + - Drug: GraphConv over the molecular graph, followed by global mean pooling. + - Cell: project the scLong embedding. + - Predict IC50 from concatenated features using fully connected layers. """ def __init__(self, drug_feat_dim: int, cell_emb_dim: int, hidden_dim: int = 256): super().__init__() - # 图卷积层处理药物分子结构 + # Encode the drug molecular structure with graph convolutions. self.graph_conv1 = GraphConv(drug_feat_dim, 128) self.graph_conv2 = GraphConv(128, 128) self.relu = nn.ReLU() - # 细胞嵌入特征处理 + # Process cell embeddings. self.cell_fc = nn.Linear(cell_emb_dim, 128) - # 多模态融合和预测 + # Predict from fused modalities. self.fc1 = nn.Linear(128 + 128, hidden_dim) self.fc2 = nn.Linear(hidden_dim, 1) self.dropout = nn.Dropout(0.1) @@ -592,20 +597,20 @@ class SimpleSCFMDrugModel(nn.Module): def forward( self, drug_feat: torch.Tensor, # (B, N, F_d) - drug_adj: torch.Tensor, # (B, N, N) 邻接矩阵 + drug_adj: torch.Tensor, # (B, N, N) adjacency matrix cell_emb: torch.Tensor, # (B, D) ) -> torch.Tensor: - # 药物分支:使用图卷积处理分子结构 + # Drug branch: encode the molecular graph. x_drug = self.relu(self.graph_conv1(drug_feat, drug_adj)) # (B, N, 128) x_drug = self.relu(self.graph_conv2(x_drug, drug_adj)) # (B, N, 128) - # 全局平均池化得到药物表示 + # Global mean pooling produces the drug representation. x_drug = x_drug.mean(dim=1) # (B, 128) - # 细胞分支:处理scLong embedding + # Cell branch: process scLong embeddings. x_cell = self.relu(self.cell_fc(cell_emb)) # (B, 128) - # 特征融合 + # Fuse features. h = torch.cat([x_drug, x_cell], dim=1) # (B, 256) h = self.relu(self.fc1(h)) h = self.dropout(h) @@ -615,30 +620,30 @@ class SimpleSCFMDrugModel(nn.Module): # class MultiSourceSCFMDrugModel(nn.Module): # """ -# 多源数据融合的药物响应预测模型: -# - 药物:GraphConv处理图结构 -# - 细胞:scLong embedding + 基因突变 + 甲基化特征 -# - 所有特征融合后预测IC50 +# Drug-response model with multiple input modalities: +# - Drug: encode molecular graphs with GraphConv. +# - Cell: scLong embeddings, mutation features, and methylation features. +# - Fuse all features to predict IC50. # """ # def __init__(self, drug_feat_dim: int, cell_emb_dim: int, mutation_dim: int, # gexpr_dim: int, methylation_dim: int, hidden_dim: int = 256): # super().__init__() -# # 图卷积层处理药物分子结构 +# Encode the drug molecular structure with graph convolutions. # self.graph_conv1 = GraphConv(drug_feat_dim, 128) # self.graph_conv2 = GraphConv(128, 128) # self.relu = nn.ReLU() -# # 细胞嵌入特征处理 +# Process cell embeddings. # self.cell_fc = nn.Linear(cell_emb_dim, 128) -# # 基因突变特征处理 -# self.mut_fc = nn.Linear(mutation_dim, 64) # 将突变特征降维 +# Process mutation features. +# self.mut_fc = nn.Linear(mutation_dim, 64) # Reduce mutation feature dimensionality. -# # 甲基化特征处理 -# self.methy_fc = nn.Linear(methylation_dim, 64) # 将甲基化特征降维 +# Process methylation features. +# self.methy_fc = nn.Linear(methylation_dim, 64) # Reduce methylation feature dimensionality. -# # 多模态融合和预测 +# Predict from fused modalities. # # 128(drug) + 128(cell) + 64(mutation) + 64(methylation) = 384 # self.fc1 = nn.Linear(128 + 128 + 64 + 64, hidden_dim) # self.fc2 = nn.Linear(hidden_dim, 1) @@ -652,30 +657,30 @@ class SimpleSCFMDrugModel(nn.Module): # mutation_feat: torch.Tensor = None, # (B, 1, 1, M) # methylation_feat: torch.Tensor = None # (B, M) # ) -> torch.Tensor: -# # 药物分支 +# Drug branch. # x_drug = self.relu(self.graph_conv1(drug_feat, drug_adj)) # x_drug = self.relu(self.graph_conv2(x_drug, drug_adj)) # x_drug = x_drug.mean(dim=1) # (B, 128) -# # 细胞分支 +# Cell branch. # x_cell = self.relu(self.cell_fc(cell_emb)) # (B, 128) -# # 初始化特征列表 +# Initialize the feature list. # feats = [x_drug, x_cell] -# # 基因突变分支(如果提供) +# Mutation branch, when supplied. # if mutation_feat is not None: -# # 展开并降维 +# Flatten the input dimensions. # mutation_flat = mutation_feat.view(mutation_feat.size(0), -1) # (B, M) # x_mut = self.relu(self.mut_fc(mutation_flat)) # (B, 64) # feats.append(x_mut) -# # 甲基化分支(如果提供) +# Methylation branch, when supplied. # if methylation_feat is not None: # x_methy = self.relu(self.methy_fc(methylation_feat)) # (B, 64) # feats.append(x_methy) -# # 特征融合 +# Fuse features. # h = torch.cat(feats, dim=1) # h = self.relu(self.fc1(h)) # h = self.dropout(h) @@ -683,7 +688,7 @@ class SimpleSCFMDrugModel(nn.Module): # return out -# ---------------------- 训练与评估 ---------------------- +# ---------------------- Training and evaluation ---------------------- def train_epoch(model, loader, optimizer, criterion, device): model.train() @@ -729,29 +734,29 @@ def eval_epoch(model, loader, device): def main(): parser = argparse.ArgumentParser(description="Use scLong (SCFM) cell embeddings for drug response prediction") parser.add_argument("--gpu_id", type=str, default="0") - parser.add_argument("--data_path", type=str, default="./data") + parser.add_argument("--data_path", type=str, default=str(SCRIPT_DIR / "data")) parser.add_argument("--epochs", type=int, default=50) parser.add_argument("--batch_size", type=int, default=64) parser.add_argument("--max_atoms", type=int, default=100) - # 添加新的参数选项 - parser.add_argument("--use_graph_conv", action="store_false", help="是否使用图卷积处理药物特征") - parser.add_argument("--use_multi_source", action="store_true", help="是否使用多源数据(基因突变+甲基化)") + # Configure optional model branches. + parser.add_argument("--use_graph_conv", action="store_false", help="Disable graph convolution (legacy flag behavior).") + parser.add_argument("--use_multi_source", action="store_true", help="Use additional mutation and methylation features.") - # scLong / SCFM 相关路径(需要根据你的环境修改) + # scLong / SCFM paths; supply the matching legacy checkpoint and configuration. parser.add_argument("--gene2vec_path", type=str, - default="../selected_gene2vec_27k.npy") + default=str(ASSET_DIR / "selected_gene2vec_27k.npy")) parser.add_argument("--scfm_hyper_params_path", type=str, - default="../gocont_4096_48m_pretrain_1b_mix.pkl") + required=True, help="Path to the matching legacy scLong hyperparameter file (not bundled).") parser.add_argument( "--scfm_ckpt_path", type=str, - default="../gocont_4096_48m_pretrain_1b_mix_2024-02-05_16-23-37.pth", + required=True, help="Path to pretrained SCFM checkpoint .pth; use 'none' to initialize randomly.", ) parser.add_argument("--symbol2ens_path", type=str, - default="../gene2ens.csv") + default=str(ASSET_DIR / "gene2ens.csv")) parser.add_argument("--scfm_genes_path", type=str, - default="../selected_genes_27k.txt") + default=str(ASSET_DIR / "selected_genes_27k.txt")) args = parser.parse_args() @@ -759,9 +764,9 @@ def main(): device = torch.device("cuda" if torch.cuda.is_available() else "cpu") print_sys(f"Using device: {device}") - # 显示使用的功能选项 - print_sys(f"使用图卷积: {args.use_graph_conv}") - print_sys(f"使用多源数据: {args.use_multi_source}") + # Display the enabled model options. + print_sys(f"Use graph convolution: {args.use_graph_conv}") + print_sys(f"Use multiple input modalities: {args.use_multi_source}") dpath = args.data_path drug_info_file = f"{dpath}/GDSC/1.Drug_listMon Jun 24 09_00_55 2019.csv" @@ -796,7 +801,7 @@ def main(): methylation_file, ) - # 用 scLong 预计算每个 cell line 的 embedding + # Precompute a scLong embedding for each cell line. cell_emb_dict, cell_emb_dim = build_cell_embeddings_with_sclong( gexpr_feature=gexpr_feature, symbol2ens_path=args.symbol2ens_path, @@ -809,11 +814,11 @@ def main(): train_idx, test_idx = data_split(data_idx) - # 基因映射准备(用于feature_extract函数) + # Prepare the gene map for feature extraction. # gene_mapping = {gene: i for i, gene in enumerate(gexpr_feature['gene'].values)} - gene2ens = pd.read_csv('../gene2ens.csv',index_col=0) + gene2ens = pd.read_csv(args.symbol2ens_path,index_col=0) symbol2ens = gene2ens['Ensembl gene ID'].to_dict() - # 把 CCLE 表达矩阵的列名映射到 ENS,如果没有映射就保持原始名字 + # Map CCLE expression column names to Ensembl IDs; retain unmapped names. expr_genes = list(gexpr_feature.columns) input_genes = [ symbol2ens.get(sym, sym) for sym in expr_genes @@ -823,9 +828,9 @@ def main(): if args.use_multi_source: exit(0) - # # 使用完整的feature_extract函数,处理多源数据 - # print_sys("使用完整feature_extract函数提取多源特征...") - # # 训练集特征提取 + # # Extract all modalities with the complete feature_extract function. + # print_sys("Extract multiple input modalities with feature_extract...") + # # Extract training features. # train_drug_feat, train_drug_adj, train_target, \ # train_mutation_data, train_gexpr_data, train_methylation_data = feature_extract( # data_idx, train_idx, drug_feature, gene_mapping, @@ -833,7 +838,7 @@ def main(): # max_atoms=args.max_atoms # ) - # # 测试集特征提取 + # # Extract test features. # test_drug_feat, test_drug_adj, test_target, \ # test_mutation_data, test_gexpr_data, test_methylation_data = feature_extract( # data_idx, test_idx, drug_feature, gene_mapping, @@ -841,7 +846,7 @@ def main(): # max_atoms=args.max_atoms # ) - # # 构建数据集,包含基因突变和甲基化数据 + # # Construct datasets with mutation and methylation features. # train_ds = SCFMDrugDataset( # drug_feat=train_drug_feat, # drug_adj=train_drug_adj, @@ -859,8 +864,8 @@ def main(): # methylation_data=test_methylation_data # ) else: - # 使用原有的feature_extract_drug_only函数 - print_sys("使用feature_extract_drug_only函数提取特征...") + # Use the existing feature_extract_drug_only function. + print_sys("Extract features with feature_extract_drug_only...") ( X_drug_train, Y_train, @@ -883,7 +888,7 @@ def main(): args.max_atoms, ) - # 转换数据格式以适应新的数据集结构 + # Convert to the format expected by the dataset. # train_drug_feat = [x[0] for x in X_drug_train] # train_drug_adj = [x[1] for x in X_drug_train] # train_target = Y_train @@ -911,7 +916,7 @@ def main(): # cell_emb=cell_emb_dict # ) - # # 推导药物特征维度 + # # Determine the drug feature dimension. # drug_dim = train_drug_feat[0].shape[-1] train_ds = SCFMDrugDataset( drug_data=X_drug_train, @@ -926,10 +931,10 @@ def main(): cell_emb_dict=cell_emb_dict ) - # 推导药物特征维度 + # Determine the drug feature dimension. drug_dim = X_drug_train[0][0].shape[-1] - # 创建数据加载器 + # Display dataset information. num_workers = min(4, os.cpu_count() or 1) train_loader = DataLoader( train_ds, @@ -948,12 +953,12 @@ def main(): pin_memory=torch.cuda.is_available(), ) - # 根据参数选择模型 + # Select the model from the configured options. if args.use_multi_source and args.use_graph_conv: - # 使用多源数据的图卷积模型 - print_sys("使用MultiSourceSCFMDrugModel模型...") + # Use graph convolution with multiple input modalities. + print_sys("Use MultiSourceSCFMDrugModel...") exit(0) - # # 确定基因突变和甲基化特征维度 + # # Determine mutation and methylation feature dimensions. # mutation_dim = len(gene_mapping) if args.use_multi_source and train_mutation_data else 0 # methylation_dim = len(gene_mapping) if args.use_multi_source and train_methylation_data else 0 @@ -966,16 +971,16 @@ def main(): # hidden_dim=256, # ).to(device) elif args.use_graph_conv: - # 使用图卷积的简单模型 - print_sys("使用GraphConv版本的SimpleSCFMDrugModel模型...") + # Use the simple graph-convolution model. + print_sys("Use the GraphConv version of SimpleSCFMDrugModel...") model = SimpleSCFMDrugModel( drug_feat_dim=drug_dim, cell_emb_dim=cell_emb_dim, hidden_dim=256, ).to(device) else: - # 使用原始模型 - print_sys("使用原始SimpleSCFMDrugModel模型...") + # Use the original model. + print_sys("Use the original SimpleSCFMDrugModel...") model = SimpleSCFMDrugModel( drug_feat_dim=drug_dim, cell_emb_dim=cell_emb_dim, @@ -990,7 +995,7 @@ def main(): best_true = None for epoch in range(1, args.epochs + 1): - # 根据是否使用多源数据选择不同的训练和评估函数 + # Select training/evaluation functions for the enabled input modalities. if args.use_multi_source: exit(0) # train_loss = train_epoch_multi_source(model, train_loader, optimizer, criterion, device) @@ -1018,19 +1023,19 @@ def main(): def train_epoch_multi_source(model, loader, optimizer, criterion, device): """ - 支持多源数据的训练函数 + Train with multiple input modalities. """ model.train() total_loss = 0.0 total = 0 for batch in loader: - # 将数据移到指定设备 + # Move data to the selected device. drug_feat = batch['drug_feat'].to(device) drug_adj = batch['drug_adj'].to(device) cell_emb = batch['cell_emb'].to(device) target = batch['target'].to(device) - # 获取可选的多源数据 + # Retrieve optional input modalities. mutation_data = batch.get('mutation_data', None) methylation_data = batch.get('methylation_data', None) if mutation_data is not None: @@ -1040,7 +1045,7 @@ def train_epoch_multi_source(model, loader, optimizer, criterion, device): optimizer.zero_grad(set_to_none=True) - # 模型前向传播,传入可选的多源数据 + # Run the forward pass with the available input modalities. pred = model(drug_feat, drug_adj, cell_emb, mutation_feat=mutation_data, methylation_feat=methylation_data).squeeze(-1) @@ -1056,20 +1061,20 @@ def train_epoch_multi_source(model, loader, optimizer, criterion, device): def eval_epoch_multi_source(model, loader, device): """ - 支持多源数据的评估函数 + Evaluate with multiple input modalities. """ model.eval() preds = [] trues = [] with torch.no_grad(): for batch in loader: - # 将数据移到指定设备 + # Move data to the selected device. drug_feat = batch['drug_feat'].to(device) drug_adj = batch['drug_adj'].to(device) cell_emb = batch['cell_emb'].to(device) target = batch['target'].to(device) - # 获取可选的多源数据 + # Retrieve optional input modalities. mutation_data = batch.get('mutation_data', None) methylation_data = batch.get('methylation_data', None) if mutation_data is not None: @@ -1077,7 +1082,7 @@ def eval_epoch_multi_source(model, loader, device): if methylation_data is not None: methylation_data = methylation_data.to(device) - # 模型前向传播,传入可选的多源数据 + # Run the forward pass with the available input modalities. pred = model(drug_feat, drug_adj, cell_emb, mutation_feat=mutation_data, methylation_feat=methylation_data).squeeze(-1) diff --git a/data/downstream/final_data/drug_response_pytorch/drug_response_pytorch/train_tost_multisource.py b/data/downstream/final_data/drug_response_pytorch/drug_response_pytorch/train_tost_multisource.py index 8feda25943d162c647836398d69649ecb9ca21e9..89cb6be183d8e0d0dee22a8b0cc2eee1bde6b8e8 100644 --- a/data/downstream/final_data/drug_response_pytorch/drug_response_pytorch/train_tost_multisource.py +++ b/data/downstream/final_data/drug_response_pytorch/drug_response_pytorch/train_tost_multisource.py @@ -1,18 +1,23 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- """ -多源版本:使用 - - 药物图特征(GraphConv) - - scLong (DualEncoderSCFM) 细胞嵌入 - - 基因突变特征 (CCLE mutation) - - 甲基化特征 (CCLE methylation) -一起预测药物响应 (IC50)。 +Predict drug response (IC50) from multiple input modalities: + - Drug graph features (GraphConv). + - scLong (DualEncoderSCFM) cell embeddings. + - CCLE mutation features. + - CCLE methylation features. +Fuse these features for IC50 regression. - python train_tost_multisource.py --gpu_id 0 --data_path ./data + python train_tost_multisource.py --gpu_id 0 --scfm_hyper_params_path /path/to/scLong.pkl --scfm_ckpt_path /path/to/scLong.pth """ import argparse +from pathlib import Path + +SCRIPT_DIR = Path(__file__).resolve().parent +PROJECT_ROOT = SCRIPT_DIR.parents[4] +ASSET_DIR = PROJECT_ROOT / "model_dependencies" import os import random import sys @@ -28,15 +33,15 @@ import torch.nn as nn from scipy.stats import pearsonr from torch.utils.data import DataLoader, Dataset -# 让脚本能找到 DualEncoderSCFM -sys.path.append("../") +# Make the bundled DualEncoderSCFM implementation importable. +sys.path.insert(0, str(ASSET_DIR)) from performer_pytorch_cont.ding_models import DualEncoderSCFM -# ---------------------- 工具函数 / 打印 ---------------------- +# ---------------------- Utilities and logging ---------------------- def print_sys(*args, **kwargs): - """简单的打印包装,保持和其它脚本风格一致。""" + """Print immediately, matching the other training scripts.""" print(*args, **kwargs, flush=True) @@ -48,8 +53,8 @@ def reindex_tensor_universal( device: Union[torch.device, str] = "cpu", ) -> torch.Tensor: """ - 通用重排操作:沿着某一维 dim 按 index_positions 重新索引,超出索引位置用 filler 填充。 - 来自 zero-shot 脚本。 + Reindex dimension dim using index_positions; fill missing positions with filler. + Adapted from the zero-shot script. """ index_tensor = torch.tensor(index_positions, device=device) tensor_shape = list(tensor.shape) @@ -72,7 +77,7 @@ def reindex_tensor_universal( return reindexed -# ---------------------- scLong / SCFM 相关 ---------------------- +# ---------------------- scLong / SCFM utilities ---------------------- def load_sclong_model( gene2vec_path: str, @@ -81,10 +86,10 @@ def load_sclong_model( device: torch.device, ) -> DualEncoderSCFM: """ - 加载 scLong (DualEncoderSCFM) 模型。 + Load the scLong (DualEncoderSCFM) model. - - 如果提供了 scfm_ckpt_path,则加载预训练权重; - - 如果未提供或为 'none',则仅根据 hyper_params 初始化一个随机权重的模型。 + - Load pretrained weights when scfm_ckpt_path is supplied. + - If omitted or set to 'none', initialize random weights from hyper_params. """ with open(scfm_hyper_params_path, "rb") as f: scfm_hyper_params = pickle.load(f) @@ -114,27 +119,27 @@ def build_scfm_gene_mapping( expr_genes: List[str], ) -> Tuple[Dict[str, int], List[int]]: """ - 构建从 CCLE 表达矩阵的列 (基因名) 到 scFM 27k 基因的重排索引。 + Build reindexing positions from CCLE expression columns to the scFM 27k gene order. - 返回: - - input_mapping: dict[input_gene_id] -> 列索引 - - scfm_index_positions: 对于每个 scfm_genes_pad 中的基因,给出在 input_genes 中的列位置(找不到则指向末尾 pad)。 + Returns: + - input_mapping: dict[input_gene_id] -> column index + - scfm_index_positions: input column for each scfm_genes_pad entry; absent genes use the final padding column. """ - # 当前实现直接使用本项目中的 gene2ens.csv,保持与其它脚本一致 + # Use the gene2ens.csv mapping supplied with this project. gene2ens = pd.read_csv(symbol2ens_path, index_col=0) symbol2ens = gene2ens["Ensembl gene ID"].to_dict() - # 把 CCLE 表达矩阵的列名映射到 ENS,如果没有映射就保持原始名字 + # Map CCLE expression column names to Ensembl IDs; retain unmapped names. input_genes = [symbol2ens.get(sym, sym) for sym in expr_genes] input_gene_num = len(input_genes) input_mapping = {gid: i for i, gid in enumerate(input_genes)} - # 加载 scFM 预训练使用的 27k 基因 + # Load the 27k genes used for scFM pretraining. with open(scfm_genes_path, "r") as f: scfm_genes = [line.rstrip("\n") for line in f.readlines()] scfm_genes_pad = scfm_genes + ["PAD"] - # 对每个 scFM 基因,找到它在输入中的位置;若不存在则用 input_gene_num(pad) + # Locate each scFM gene in the input; use input_gene_num (padding) for missing genes. scfm_index_positions = [ input_mapping.get(gid, input_gene_num) for gid in scfm_genes_pad ] @@ -153,14 +158,14 @@ def build_cell_embeddings_with_sclong( device: torch.device, ) -> Tuple[Dict[str, np.ndarray], int]: """ - 用 scLong 为每个 cell line 构造一个 cell embedding。 + Build a scLong embedding for each cell line. - gexpr_feature: index 是 cell_line_id,columns 是基因名。 - 返回: + gexpr_feature: rows are cell_line_id values; columns are gene names. + Returns: - cell_emb_dict: {cell_line_id: embedding (np.ndarray, shape=(D,))} - - emb_dim: embedding 维度 D + - emb_dim: embedding dimension D """ - # 加载 SCFM 模型 + # Load the SCFM model. model = load_sclong_model( gene2vec_path=gene2vec_path, scfm_hyper_params_path=scfm_hyper_params_path, @@ -168,7 +173,7 @@ def build_cell_embeddings_with_sclong( device=device, ) - # 构建基因重排映射 + # Build the gene reindexing map. _, scfm_index_positions = build_scfm_gene_mapping( symbol2ens_path=symbol2ens_path, scfm_genes_path=scfm_genes_path, @@ -186,7 +191,7 @@ def build_cell_embeddings_with_sclong( ) # (1, seq_len=27k+1) out = model(x_scfm, return_encodings=True) - cell_emb = out["merged_decodings"][:, 0, :] # 只取第0位的 CLS 标记 + cell_emb = out["merged_decodings"][:, 0, :] # Select the CLS token at position 0. cell_emb_np = cell_emb.squeeze(0).cpu().numpy() cell_emb_dict[cell_line_id] = cell_emb_np if first_emb_dim < 0: @@ -198,7 +203,7 @@ def build_cell_embeddings_with_sclong( return cell_emb_dict, first_emb_dim -# ---------------------- DeepCDR 风格元数据 & 特征 ---------------------- +# ---------------------- DeepCDR-style metadata and features ---------------------- TCGA_LABEL_SET = [ "ALL", @@ -241,7 +246,7 @@ def calculate_graph_feat( adj_list: List[List[int]], max_atoms: int, ) -> Tuple[np.ndarray, np.ndarray]: - """和其它脚本一致的药物图特征预处理。""" + """Preprocess drug graph features as in the other training scripts.""" assert feat_mat.shape[0] == len(adj_list) feat = np.zeros((max_atoms, feat_mat.shape[-1]), dtype=np.float32) adj_mat = np.zeros((max_atoms, max_atoms), dtype=np.float32) @@ -267,8 +272,8 @@ def metadata_generate( methylation_file: str, ) -> Tuple[pd.DataFrame, dict, pd.DataFrame, pd.DataFrame, list]: """ - 与 `train_deepcdr_torch.py` / `train_tost_drug.py` 相同的元数据构造。 - 返回: + Construct metadata as in train_deepcdr_torch.py and the single-drug scLong scripts. + Returns: - mutation_feature - drug_feature - gexpr_feature @@ -366,7 +371,7 @@ def metadata_generate( def data_split(data_idx: list, ratio: float = 0.95): - """与 DeepCDR 相同的按 cancer type 分层划分。""" + """Stratify the split by cancer type, following DeepCDR.""" train_idx, test_idx = [], [] for each_type in TCGA_LABEL_SET: data_subtype_idx = [item for item in data_idx if item[-1] == each_type] @@ -388,15 +393,15 @@ def feature_extract_multi_source( max_atoms: int, ) -> Tuple[list, np.ndarray, np.ndarray, np.ndarray, list, list]: """ - 为每个样本构造完整的多源特征:药物图特征、基因突变和甲基化特征。 - - 返回: - - drug_data: 药物图特征列表,每个元素是 [feat, adj] - - mutation_data: 基因突变特征数组,形状为 (nb_instance, nb_mutation_feature) - - methylation_data: 甲基化特征数组,形状为 (nb_instance, nb_methylation_features) - - target: 目标值数组 (IC50) - - cancer_type_list: 癌症类型列表 - - data_idx: 原始数据索引列表 + Construct drug graph, mutation, and methylation features for each sample. + + Returns: + - drug_data: list of [feat, adj] drug graph features. + - mutation_data: array with shape (nb_instance, nb_mutation_feature). + - methylation_data: array with shape (nb_instance, nb_methylation_features). + - target: array of IC50 targets. + - cancer_type_list: cancer type for each sample. + - data_idx: original sample index list. """ nb_instance = len(data_idx) nb_mutation_feature = mutation_feature.shape[1] @@ -413,18 +418,18 @@ def feature_extract_multi_source( for idx in range(nb_instance): cell_line_id, pubchem_id, ln_ic50, cancer_type = data_idx[idx] - # 药物图特征 + # Drug graph features. feat_mat, adj_list, _ = drug_feature[str(pubchem_id)] feat, adj = calculate_graph_feat(feat_mat, adj_list, max_atoms) drug_data[idx] = [feat, adj] - # 基因突变特征 + # Mutation features. mutation_data[idx, :] = mutation_feature.loc[cell_line_id].values - # 甲基化特征 + # Methylation features. methylation_data[idx, :] = methylation_feature.loc[cell_line_id].values - # 目标值 + # Target value. target[idx] = ln_ic50 cancer_type_list.append([cancer_type, cell_line_id, pubchem_id]) @@ -439,19 +444,19 @@ def feature_extract_multi_source( ) -# ---------------------- Dataset & 模型定义 ---------------------- +# ---------------------- Datasets and models ---------------------- class MultiSourceSCFMDrugDataset(Dataset): """ - 多源 Dataset,返回一个 dict: + Dataset with multiple input modalities, returning a dictionary: - drug_feat : (N_atoms, F_d) - drug_adj : (N_atoms, N_atoms) - cell_emb : (D,) - mutation_data : (M_mut,) - methylation_data: (M_methy,) - - target : 标量 IC50 - DataLoader 默认的 collate 会自动把这些拼成 batched tensor。 + - target : scalar IC50 + The default DataLoader collation stacks these into batched tensors. """ def __init__( @@ -499,13 +504,13 @@ class MultiSourceSCFMDrugDataset(Dataset): class GraphConv(nn.Module): """ - 图卷积层,用于处理药物分子的图结构信息。 + Graph convolution layer for drug molecular graphs. - 计算方式:h_i = sum_j A_ji * (x_j W + b) - 其中: - - x: (B, N, F_in) 节点特征 - - A: (B, N, N) 邻接矩阵 - - out: (B, N, F_out) 输出特征 + Computation: h_i = sum_j A_ji * (x_j W + b). + Shapes: + - x: (B, N, F_in) node features + - A: (B, N, N) adjacency matrix + - out: (B, N, F_out) output features """ def __init__(self, in_features: int, out_features: int, bias: bool = True): @@ -526,8 +531,8 @@ class GraphConv(nn.Module): def forward(self, x: torch.Tensor, adj: torch.Tensor) -> torch.Tensor: """ - x: (B, N, F_in) 节点特征 - adj: (B, N, N) 邻接矩阵 + x: (B, N, F_in) node features + adj: (B, N, N) adjacency matrix """ # (B, N, F_out) = (B, N, F_in) @ (F_in, F_out) xw = torch.matmul(x, self.weight) @@ -540,17 +545,17 @@ class GraphConv(nn.Module): class MultiSourceSCFMDrugModel(nn.Module): """ - 多源数据融合的药物响应预测模型: - - 药物:GraphConv 处理图结构 - - 细胞:scLong embedding - - 细胞额外特征:基因突变 + 甲基化 - - 所有特征融合后预测 IC50 + Drug-response model with multiple input modalities: + - Drug: encode molecular graphs with GraphConv. + - Cell: scLong embeddings. + - Additional cell features: mutation and methylation. + - Fuse all features to predict IC50. - 约定输入形状: + Expected input shapes: drug_feat : (B, N, F_d) drug_adj : (B, N, N) cell_emb : (B, D) - mutation_feat : (B, M_mut) 或任意形状,模型内会展平 + mutation_feat : (B, M_mut), or another shape that the model will flatten methylation_feat: (B, M_methy) """ @@ -563,21 +568,21 @@ class MultiSourceSCFMDrugModel(nn.Module): hidden_dim: int = 256, ): super().__init__() - # 药物分支:两层 GraphConv + # Drug branch: two GraphConv layers. self.graph_conv1 = GraphConv(drug_feat_dim, 128) self.graph_conv2 = GraphConv(128, 128) self.relu = nn.ReLU() - # 细胞分支:scLong embedding -> 128 + # Cell branch: scLong embedding -> 128. self.cell_fc = nn.Linear(cell_emb_dim, 128) - # 基因突变分支:降到 64 维 + # Mutation branch: project to 64 dimensions. self.mut_fc = nn.Linear(mutation_dim, 64) - # 甲基化分支:降到 64 维 + # Methylation branch: project to 64 dimensions. self.methy_fc = nn.Linear(methylation_dim, 64) - # 融合层:128(drug) + 128(cell) + 64(mut) + 64(methy) = 384 + # Fusion: 128(drug) + 128(cell) + 64(mut) + 64(methy) = 384. self.fc1 = nn.Linear(128 + 128 + 64 + 64, hidden_dim) self.fc2 = nn.Linear(hidden_dim, 1) self.dropout = nn.Dropout(0.1) @@ -587,44 +592,44 @@ class MultiSourceSCFMDrugModel(nn.Module): drug_feat: torch.Tensor, # (B, N, F_d) drug_adj: torch.Tensor, # (B, N, N) cell_emb: torch.Tensor, # (B, D) - mutation_feat: torch.Tensor, # (B, M_mut) 或任意形状 + mutation_feat: torch.Tensor, # (B, M_mut), or another shape to flatten methylation_feat: torch.Tensor, # (B, M_methy) ) -> torch.Tensor: - # 药物分支 + # Drug branch. x_drug = self.relu(self.graph_conv1(drug_feat, drug_adj)) x_drug = self.relu(self.graph_conv2(x_drug, drug_adj)) x_drug = x_drug.mean(dim=1) # (B, 128) - # 细胞分支(scLong embedding) + # Cell branch (scLong embedding). x_cell = self.relu(self.cell_fc(cell_emb)) # (B, 128) feats = [x_drug, x_cell] - # 基因突变分支:先展平再降维 + # Mutation branch: flatten, then project. if mutation_feat is not None: mut_flat = mutation_feat.view(mutation_feat.size(0), -1) # (B, *) x_mut = self.relu(self.mut_fc(mut_flat)) # (B, 64) feats.append(x_mut) - # 甲基化分支 + # Methylation branch. if methylation_feat is not None: x_methy = self.relu(self.methy_fc(methylation_feat)) # (B, 64) feats.append(x_methy) - # 特征融合 + MLP 预测 - h = torch.cat(feats, dim=1) # (B, 384) 或更小(如果某些分支不用) + # Fuse features and predict with the MLP. + h = torch.cat(feats, dim=1) # (B, 384), or fewer columns when branches are disabled. h = self.relu(self.fc1(h)) h = self.dropout(h) out = self.fc2(h) # (B, 1) return out -# ---------------------- 训练与评估 ---------------------- +# ---------------------- Training and evaluation ---------------------- def train_epoch_multi_source(model, loader, optimizer, criterion, device): """ - 支持多源数据的训练函数 + Train with multiple input modalities. """ model.train() total_loss = 0.0 @@ -662,7 +667,7 @@ def train_epoch_multi_source(model, loader, optimizer, criterion, device): def eval_epoch_multi_source(model, loader, device): """ - 支持多源数据的评估函数 + Evaluate with multiple input modalities. """ model.eval() preds = [] @@ -709,37 +714,37 @@ def main(): ) ) parser.add_argument("--gpu_id", type=str, default="0") - parser.add_argument("--data_path", type=str, default="./data") + parser.add_argument("--data_path", type=str, default=str(SCRIPT_DIR / "data")) parser.add_argument("--epochs", type=int, default=200) parser.add_argument("--batch_size", type=int, default=64) parser.add_argument("--max_atoms", type=int, default=100) - # scLong / SCFM 相关路径(默认值可按需覆盖) + # scLong / SCFM paths; shared assets have bundled defaults. parser.add_argument( "--gene2vec_path", type=str, - default="../selected_gene2vec_27k.npy", + default=str(ASSET_DIR / "selected_gene2vec_27k.npy"), ) parser.add_argument( "--scfm_hyper_params_path", type=str, - default="../ckpts_1b/origin_400cell_5epoch.pkl", + required=True, help="Path to the matching legacy scLong hyperparameter file (not bundled).", ) parser.add_argument( "--scfm_ckpt_path", type=str, - default="../ckpts_1b/origin_400cell_5epoch_epoch9.pth", + required=True, help="Path to pretrained SCFM checkpoint .pth; use 'none' to initialize randomly.", ) parser.add_argument( "--symbol2ens_path", type=str, - default="../gene2ens.csv", + default=str(ASSET_DIR / "gene2ens.csv"), ) parser.add_argument( "--scfm_genes_path", type=str, - default="../selected_genes_27k.txt", + default=str(ASSET_DIR / "selected_genes_27k.txt"), ) args = parser.parse_args() @@ -762,12 +767,12 @@ def main(): f"{dpath}/CCLE/genomic_methylation_561celllines_808genes_demap_features.csv" ) - # 固定随机种子 + # Set random seeds. random.seed(0) np.random.seed(0) torch.manual_seed(0) - # 1) 读取 DeepCDR 风格的元数据 + # 1) Read DeepCDR-style metadata. ( mutation_feature, drug_feature, @@ -783,7 +788,7 @@ def main(): methylation_file, ) - # 2) scLong:为每个 cell line 预计算 embedding + # 2) Precompute a scLong embedding for each cell line. cell_emb_dict, cell_emb_dim = build_cell_embeddings_with_sclong( gexpr_feature=gexpr_feature, symbol2ens_path=args.symbol2ens_path, @@ -794,13 +799,13 @@ def main(): device=device, ) - # 3) 按 cancer type 分层划分 train/test + # 3) Split train/test samples by cancer type. train_idx, test_idx = data_split(data_idx) - # 4) 提取多源特征:drug + mutation + methylation + # 4) Extract drug, mutation, and methylation features. print_sys( - "[MultiSource] 使用 feature_extract_multi_source 提取 " - "drug + mutation + methylation 特征..." + "[MultiSource] Extract drug, mutation, and methylation features " + "with feature_extract_multi_source..." ) ( X_drug_train, @@ -832,7 +837,7 @@ def main(): args.max_atoms, ) - # 5) 构建 MultiSource 数据集 + # 5) Construct the MultiSource datasets. train_ds = MultiSourceSCFMDrugDataset( drug_data=X_drug_train, mutation_data=mut_train, @@ -850,7 +855,7 @@ def main(): cell_emb_dict=cell_emb_dict, ) - # 6) DataLoader & 维度信息 + # 6) Create DataLoaders and inspect feature dimensions. drug_dim = X_drug_train[0][0].shape[-1] mutation_dim = mutation_feature.shape[1] methylation_dim = methylation_feature.shape[1] @@ -871,9 +876,9 @@ def main(): pin_memory=torch.cuda.is_available(), ) - # 7) 模型:MultiSourceSCFMDrugModel + # 7) Construct MultiSourceSCFMDrugModel. print_sys( - f"[MultiSource] 使用 MultiSourceSCFMDrugModel 模型 " + f"[MultiSource] Use MultiSourceSCFMDrugModel " f"(drug_dim={drug_dim}, cell_emb_dim={cell_emb_dim}, " f"mutation_dim={mutation_dim}, methylation_dim={methylation_dim})" ) diff --git a/data/downstream/final_data/single_drug_response/README.md b/data/downstream/final_data/single_drug_response/README.md index 9c11a730824ba61b38291da41560d238bd6de00a..b7920f85a255b255724e7f4cf3279a4da2b55e63 100644 --- a/data/downstream/final_data/single_drug_response/README.md +++ b/data/downstream/final_data/single_drug_response/README.md @@ -1,45 +1,47 @@ -# Single-drug response data - -This folder is a cleaned data layout for single-drug response prediction. -The original collaborator package is kept unchanged at: - -```text -C:\Users\JXZ\Desktop\文章的相关材料\data\downstream\final_data\drug_response_pytorch\drug_response_pytorch -``` - -## Runnable data path - -Use this directory as `--data_path` for `drug_response/train_drug_performer_patch.py`: - -```text -C:\Users\JXZ\Desktop\文章的相关材料\data\downstream\final_data\single_drug_response\data -``` - -Expected layout: - -```text -data/ -|-- GDSC/ -| |-- 1.Drug_listMon Jun 24 09_00_55 2019.csv -| `-- drug_graph_feat/*.hkl -`-- CCLE/ - |-- Cell_lines_annotations_20181226.txt - |-- GDSC_IC50.csv - `-- genomic_expression_561celllines_697genes_demap_features.csv -``` - -## Checked statistics - -- Drug information table: (266, 8) -- Drug graph files: 223 -- IC50 matrix: (266, 969) -- Cell-line expression matrix: (561, 697) -- Cell-line annotation table: (1461, 33) - -## Optional reference files - -`optional_reference/` contains files from the original package that are not used by the current PerformerPatch single-drug response script, including mutation, copy-number, methylation, threshold and DeepCDR reference outputs. Keep them for provenance or future multi-omics experiments. - -## Files - -See `MANIFEST.csv` for file sizes and checksums. See `SUMMARY.json` for machine-readable shape checks. +# Single-drug response data + +This folder is a cleaned data layout for single-drug response prediction. +The bundled legacy collaborator reference package is located at this repository-relative path: + +```text +data/downstream/final_data/drug_response_pytorch/drug_response_pytorch +``` + +## Runnable data path + +From the repository root, use this directory as `--data_path` for `BioMod/drug_response/train_drug_performer_module.py`: + +```text +data/downstream/final_data/single_drug_response/data +``` + +Expected layout: + +```text +data/ +|-- GDSC/ +| |-- 1.Drug_listMon Jun 24 09_00_55 2019.csv +| `-- drug_graph_feat/*.hkl +`-- CCLE/ + |-- Cell_lines_annotations_20181226.txt + |-- GDSC_IC50.csv + `-- genomic_expression_561celllines_697genes_demap_features.csv +``` + +## Previously recorded statistics + +The figures below are retained from the existing dataset documentation; this path/comment audit did not recompute them. + +- Drug information table: (266, 8) +- Drug graph files: 223 +- IC50 matrix: (266, 969) +- Cell-line expression matrix: (561, 697) +- Cell-line annotation table: (1461, 33) + +## Optional reference files + +`optional_reference/` contains files from the original package that are not used by the current BioMod single-drug response script, including mutation, copy-number, methylation, threshold and DeepCDR reference outputs. Keep them for provenance or future multi-omics experiments. + +## Files + +See `MANIFEST.csv` for file sizes and checksums. See `SUMMARY.json` for machine-readable shape checks. diff --git a/model_dependencies/module_data_all/create_uncovered_modules.py b/model_dependencies/module_data_all/create_uncovered_modules.py index 084e68fae9c0c7c74ec9ca961f7f6f23e8a2d881..06b3139c99d2ffbac95360c171cd880df9a45cb8 100644 --- a/model_dependencies/module_data_all/create_uncovered_modules.py +++ b/model_dependencies/module_data_all/create_uncovered_modules.py @@ -1,146 +1,151 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" -为未被现有 module 覆盖的基因创建额外的 module -确保所有模型基因都能参与训练 - -使用方法: - python create_uncovered_modules.py \ - --merged_modules module_data_all/merged_modules.json \ - --output module_data_all/merged_modules_full.json \ - --module_size 50 -""" - -import json -import os -import csv -import argparse - -def create_uncovered_modules( - merged_modules_path, - gene_list_path, - gene_mapping_path, - output_path, - module_size=50, - verbose=True -): - """ - 为未被覆盖的基因创建额外的 module - - Args: - merged_modules_path: 已合并的 module 文件路径 - gene_list_path: 模型基因列表路径 - gene_mapping_path: 基因映射文件路径 - output_path: 输出文件路径 - module_size: 新 module 的大小 - verbose: 是否打印详细信息 - """ - # 加载模型基因列表 - with open(gene_list_path, 'r') as f: - model_genes = set(line.strip() for line in f) - - # 加载已有的 modules - with open(merged_modules_path, 'r') as f: - modules = json.load(f) - - # 加载基因映射 (symbol -> ensembl) - symbol_to_ens = {} - ens_to_symbol = {} - with open(gene_mapping_path, 'r') as f: - reader = csv.DictReader(f) - for row in reader: - symbol = row.get('Approved symbol', '').strip() - ens_id = row.get('Ensembl gene ID', '').strip() - if symbol and ens_id: - symbol_to_ens[symbol] = ens_id - ens_to_symbol[ens_id] = symbol - - # 找出已被覆盖的基因 - covered_genes_ens = set() - for module_name, genes in modules.items(): - for g in genes: - ens_id = symbol_to_ens.get(g) - if ens_id and ens_id in model_genes: - covered_genes_ens.add(ens_id) - elif g in model_genes: - covered_genes_ens.add(g) - - # 找出未被覆盖的基因 - uncovered_genes = model_genes - covered_genes_ens - uncovered_genes = sorted(list(uncovered_genes)) - - if verbose: - print(f"模型基因总数: {len(model_genes)}") - print(f"已被覆盖的基因: {len(covered_genes_ens)} ({len(covered_genes_ens)/len(model_genes)*100:.1f}%)") - print(f"未被覆盖的基因: {len(uncovered_genes)} ({len(uncovered_genes)/len(model_genes)*100:.1f}%)") - - # 为未覆盖的基因创建新的 modules - # 使用 Ensembl ID(因为模型使用的是 Ensembl ID) - new_modules = {} - num_new_modules = (len(uncovered_genes) + module_size - 1) // module_size - - for i in range(num_new_modules): - start_idx = i * module_size - end_idx = min((i + 1) * module_size, len(uncovered_genes)) - module_genes = uncovered_genes[start_idx:end_idx] - - # 尝试转换为 symbol(如果有映射的话),否则保持 Ensembl ID - module_genes_final = [] - for g in module_genes: - symbol = ens_to_symbol.get(g) - if symbol: - module_genes_final.append(symbol) - else: - module_genes_final.append(g) - - new_modules[f"uncovered_module_{i+1:04d}"] = module_genes_final - - if verbose: - print(f"\n创建了 {len(new_modules)} 个新 module (每个约 {module_size} 个基因)") - - # 合并原有 modules 和新 modules - merged_modules = {**modules, **new_modules} - - # 保存 - os.makedirs(os.path.dirname(output_path) if os.path.dirname(output_path) else '.', exist_ok=True) - with open(output_path, 'w', encoding='utf-8') as f: - json.dump(merged_modules, f, indent=2) - - if verbose: - print(f"\n最终结果:") - print(f" 原有 modules: {len(modules)}") - print(f" 新增 modules: {len(new_modules)}") - print(f" 总 modules: {len(merged_modules)}") - print(f" 保存到: {output_path}") - - return merged_modules - -def main(): - parser = argparse.ArgumentParser(description='Create modules for uncovered genes') - parser.add_argument('--merged_modules', type=str, - default='module_data_all/merged_modules.json', - help='Path to merged modules JSON file') - parser.add_argument('--gene_list', type=str, - default='../selected_genes_27k.txt', - help='Path to model gene list file') - parser.add_argument('--gene_mapping', type=str, - default='../gene2ens.csv', - help='Path to gene mapping CSV file') - parser.add_argument('--output', '-o', type=str, - default='module_data_all/merged_modules_full.json', - help='Output file path') - parser.add_argument('--module_size', type=int, default=50, - help='Size of new modules for uncovered genes') - - args = parser.parse_args() - - create_uncovered_modules( - merged_modules_path=args.merged_modules, - gene_list_path=args.gene_list, - gene_mapping_path=args.gene_mapping, - output_path=args.output, - module_size=args.module_size - ) - -if __name__ == '__main__': - main() +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +Create additional modules for genes not covered by existing modules +Ensure that all model genes can participate in training + +Run from the repository root: + python BioMod/module_data_all/create_uncovered_modules.py \ + --merged_modules model_dependencies/module_data_all/merged_modules.json \ + --output tutorial_runs/module_data/merged_modules_full.json \ + --module_size 50 +""" + +import json +import os +import csv +import argparse +from pathlib import Path + +PROJECT_ROOT = Path(__file__).resolve().parents[2] +ASSET_ROOT = PROJECT_ROOT / 'model_dependencies' +MODULE_DIR = Path(__file__).resolve().parent + +def create_uncovered_modules( + merged_modules_path, + gene_list_path, + gene_mapping_path, + output_path, + module_size=50, + verbose=True +): + """ + Create additional modules for uncovered genes + + Args: + merged_modules_path: merged module file path + gene_list_path: model gene list path + gene_mapping_path: Gene mapping file path + output_path: output file path + module_size: the size of the new module + verbose: whether to print detailed information + """ + # Load model gene list + with open(gene_list_path, 'r') as f: + model_genes = set(line.strip() for line in f) + + # Load existing modules + with open(merged_modules_path, 'r') as f: + modules = json.load(f) + + # Load gene map (symbol -> ensembl) + symbol_to_ens = {} + ens_to_symbol = {} + with open(gene_mapping_path, 'r') as f: + reader = csv.DictReader(f) + for row in reader: + symbol = row.get('Approved symbol', '').strip() + ens_id = row.get('Ensembl gene ID', '').strip() + if symbol and ens_id: + symbol_to_ens[symbol] = ens_id + ens_to_symbol[ens_id] = symbol + + # Find genes that are covered + covered_genes_ens = set() + for module_name, genes in modules.items(): + for g in genes: + ens_id = symbol_to_ens.get(g) + if ens_id and ens_id in model_genes: + covered_genes_ens.add(ens_id) + elif g in model_genes: + covered_genes_ens.add(g) + + # Find uncovered genes + uncovered_genes = model_genes - covered_genes_ens + uncovered_genes = sorted(list(uncovered_genes)) + + if verbose: + print(f"Total number of model genes: {len(model_genes)}") + print(f"Genes covered: {len(covered_genes_ens)} ({len(covered_genes_ens)/len(model_genes)*100:.1f}%)") + print(f"Genes not covered: {len(uncovered_genes)} ({len(uncovered_genes)/len(model_genes)*100:.1f}%)") + + # Create new modules for uncovered genes + # Use Ensembl ID (because the model uses Ensembl ID) + new_modules = {} + num_new_modules = (len(uncovered_genes) + module_size - 1) // module_size + + for i in range(num_new_modules): + start_idx = i * module_size + end_idx = min((i + 1) * module_size, len(uncovered_genes)) + module_genes = uncovered_genes[start_idx:end_idx] + + # Try converting to symbol if there is a mapping, otherwise keep the Ensembl ID + module_genes_final = [] + for g in module_genes: + symbol = ens_to_symbol.get(g) + if symbol: + module_genes_final.append(symbol) + else: + module_genes_final.append(g) + + new_modules[f"uncovered_module_{i+1:04d}"] = module_genes_final + + if verbose: + print(f"\nCreated {len(new_modules)} new modules (each approx. {module_size} genes)") + + # Merge existing modules and new modules + merged_modules = {**modules, **new_modules} + + # save + os.makedirs(os.path.dirname(output_path) if os.path.dirname(output_path) else '.', exist_ok=True) + with open(output_path, 'w', encoding='utf-8') as f: + json.dump(merged_modules, f, indent=2) + + if verbose: + print(f"\nFinal result:") + print(f" Original modules: {len(modules)}") + print(f" New modules: {len(new_modules)}") + print(f" Total modules: {len(merged_modules)}") + print(f" Save to: {output_path}") + + return merged_modules + +def main(): + parser = argparse.ArgumentParser(description='Create modules for uncovered genes') + parser.add_argument('--merged_modules', type=str, + default=str(MODULE_DIR / 'merged_modules.json'), + help='Path to merged modules JSON file') + parser.add_argument('--gene_list', type=str, + default=str(ASSET_ROOT / 'selected_genes_27k.txt'), + help='Path to model gene list file') + parser.add_argument('--gene_mapping', type=str, + default=str(ASSET_ROOT / 'gene2ens.csv'), + help='Path to gene mapping CSV file') + parser.add_argument('--output', '-o', type=str, + default=str(MODULE_DIR / 'merged_modules_full.json'), + help='Output file path') + parser.add_argument('--module_size', type=int, default=50, + help='Size of new modules for uncovered genes') + + args = parser.parse_args() + + create_uncovered_modules( + merged_modules_path=args.merged_modules, + gene_list_path=args.gene_list, + gene_mapping_path=args.gene_mapping, + output_path=args.output, + module_size=args.module_size + ) + +if __name__ == '__main__': + main() diff --git a/model_dependencies/module_data_all/merge_modules.py b/model_dependencies/module_data_all/merge_modules.py index d198062f433eb439efea493da130f885adb8027d..bae26048dcaf97610cb5613b019e8fca673c60e5 100644 --- a/model_dependencies/module_data_all/merge_modules.py +++ b/model_dependencies/module_data_all/merge_modules.py @@ -1,96 +1,96 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" -合并多个 module JSON 文件为一个文件 -用于将不同来源的 module 数据合并后进行训练 - -使用方法: - python merge_modules.py --output module_data_all/merged_modules.json -""" - -import json -import os -import argparse - -def merge_module_files(input_files, output_file, verbose=True): - """ - 合并多个 module JSON 文件 - - Args: - input_files: 输入文件路径列表 - output_file: 输出文件路径 - verbose: 是否打印详细信息 - """ - merged_modules = {} - total_modules = 0 - all_genes = set() - - for input_file in input_files: - if not os.path.exists(input_file): - print(f"Warning: File not found: {input_file}") - continue - - with open(input_file, 'r', encoding='utf-8') as f: - data = json.load(f) - - file_modules = 0 - file_genes = set() - - # 获取文件名作为前缀,避免 key 冲突 - basename = os.path.splitext(os.path.basename(input_file))[0] - - for key, genes in data.items(): - if isinstance(genes, (list, tuple)): - # 添加文件名前缀避免 key 冲突 - new_key = f"{basename}__{key}" - merged_modules[new_key] = list(genes) - file_modules += 1 - file_genes.update(genes) - - total_modules += file_modules - all_genes.update(file_genes) - - if verbose: - print(f"{os.path.basename(input_file)}:") - print(f" Modules: {file_modules}") - print(f" Unique genes: {len(file_genes)}") - - # 保存合并后的文件 - os.makedirs(os.path.dirname(output_file) if os.path.dirname(output_file) else '.', exist_ok=True) - with open(output_file, 'w', encoding='utf-8') as f: - json.dump(merged_modules, f, indent=2) - - if verbose: - print("=" * 50) - print(f"Total modules: {total_modules}") - print(f"Total unique genes: {len(all_genes)}") - print(f"Saved to: {output_file}") - - return merged_modules - -def main(): - parser = argparse.ArgumentParser(description='Merge multiple module JSON files') - parser.add_argument('--input', '-i', nargs='+', - default=[ - 'module_data/module1_hallmarks.json', - 'module_data/module2_reactome_2022_filtered.json', - 'module_data_2/module3_hotspot_260k.json', - # 添加外部文件时使用绝对路径 - ], - help='Input JSON files to merge') - parser.add_argument('--output', '-o', - default='module_data_all/merged_modules.json', - help='Output merged JSON file') - parser.add_argument('--add-external', type=str, default=None, - help='Additional external module file to include') - - args = parser.parse_args() - - input_files = args.input - if args.add_external: - input_files.append(args.add_external) - - merge_module_files(input_files, args.output) - -if __name__ == '__main__': - main() +#!/usr/bin/env python +# -*- coding: utf-8 -*- +"""Merge explicitly selected module JSON files. + +Run from the repository root, choosing the module sources for your experiment: + python BioMod/module_data_all/merge_modules.py --input first.json second.json \ + --output tutorial_runs/module_data/merged_modules.json + +The example input filenames are placeholders for your module definitions. +""" + +import json +import os +import argparse +from pathlib import Path + +PROJECT_ROOT = Path(__file__).resolve().parents[2] +ASSET_ROOT = PROJECT_ROOT / 'model_dependencies' +MODULE_DIR = Path(__file__).resolve().parent + +def merge_module_files(input_files, output_file, verbose=True): + """ + Merge multiple module JSON files + + Args: + input_files: list of input file paths + output_file: output file path + verbose: whether to print detailed information + """ + merged_modules = {} + total_modules = 0 + all_genes = set() + + for input_file in input_files: + if not os.path.exists(input_file): + print(f"Warning: File not found: {input_file}") + continue + + with open(input_file, 'r', encoding='utf-8') as f: + data = json.load(f) + + file_modules = 0 + file_genes = set() + + # Get the file name as a prefix to avoid key conflicts + basename = os.path.splitext(os.path.basename(input_file))[0] + + for key, genes in data.items(): + if isinstance(genes, (list, tuple)): + # Add file name prefix to avoid key conflicts + new_key = f"{basename}__{key}" + merged_modules[new_key] = list(genes) + file_modules += 1 + file_genes.update(genes) + + total_modules += file_modules + all_genes.update(file_genes) + + if verbose: + print(f"{os.path.basename(input_file)}:") + print(f" Modules: {file_modules}") + print(f" Unique genes: {len(file_genes)}") + + # Save the merged file + os.makedirs(os.path.dirname(output_file) if os.path.dirname(output_file) else '.', exist_ok=True) + with open(output_file, 'w', encoding='utf-8') as f: + json.dump(merged_modules, f, indent=2) + + if verbose: + print("=" * 50) + print(f"Total modules: {total_modules}") + print(f"Total unique genes: {len(all_genes)}") + print(f"Saved to: {output_file}") + + return merged_modules + +def main(): + parser = argparse.ArgumentParser(description='Merge multiple module JSON files') + parser.add_argument('--input', '-i', nargs='+', required=True, + help='Explicit input module JSON files; historical source collections are not all bundled.') + parser.add_argument('--output', '-o', + default=str(MODULE_DIR / 'merged_modules.json'), + help='Output merged JSON file') + parser.add_argument('--add-external', type=str, default=None, + help='Additional external module file to include') + + args = parser.parse_args() + + input_files = args.input + if args.add_external: + input_files.append(args.add_external) + + merge_module_files(input_files, args.output) + +if __name__ == '__main__': + main() diff --git a/model_dependencies/performer_pytorch_cont/__init__.py b/model_dependencies/performer_pytorch_cont/__init__.py index db6fd19e1a90e89dd5f16150d483da09aecb732e..97b4fa8dfe0614300aeb3b8ccc79a705a0a9e4e1 100644 --- a/model_dependencies/performer_pytorch_cont/__init__.py +++ b/model_dependencies/performer_pytorch_cont/__init__.py @@ -1,3 +1,2 @@ -# 明确指定使用 performer_pytorch.py(训练时使用的版本) -# 而不是 performer_pytorch_new.py(新版,使用 TSSA) +# Export the bundled Performer attention implementation. from performer_pytorch_cont.performer_pytorch import PerformerLM, Performer, FastAttention, SelfAttention \ No newline at end of file diff --git a/model_dependencies/performer_pytorch_cont/ding_models.py b/model_dependencies/performer_pytorch_cont/ding_models.py index 8dd3aff5ba622679fc216020170cd87a6c3ced0b..131015306fc61734b3fd994b0c6067f37fd25fe3 100644 --- a/model_dependencies/performer_pytorch_cont/ding_models.py +++ b/model_dependencies/performer_pytorch_cont/ding_models.py @@ -7,11 +7,10 @@ from tqdm import tqdm import networkx as nx import pickle import sys -sys.path.append('/mnt/c/Users/250010166/Desktop/single_cell/scLong/') class Gene2VecPositionalEmbedding_NoFreeze(nn.Module): - def __init__(self, dim, max_seq_len, gene2vec_file = 'data/gene2vec_16906.npy'): + def __init__(self, dim, max_seq_len, gene2vec_file = None): super().__init__() - gene2vec_weight = np.load(gene2vec_file) + gene2vec_weight = np.load(resolve_gene2vec_path(gene2vec_file)) gene2vec_weight = np.concatenate((gene2vec_weight, np.zeros((1, gene2vec_weight.shape[1]))), axis=0) gene2vec_weight = torch.from_numpy(gene2vec_weight) self.emb = nn.Embedding.from_pretrained(gene2vec_weight, freeze = False) @@ -23,8 +22,8 @@ class Gene2VecPositionalEmbedding_NoFreeze(nn.Module): class TwoLayerMLPEmb(nn.Module): def __init__(self, base_dim): super(TwoLayerMLPEmb, self).__init__() - self.fc1 = nn.Linear(1, 50) # Input size: 1, Output size: 100 - self.fc2 = nn.Linear(50, base_dim) # Input size: 100, Output size: 200 + self.fc1 = nn.Linear(1, 50) # Input size: 1; output size: 50. + self.fc2 = nn.Linear(50, base_dim) # Input size: 50; output size: base_dim. def forward(self, x): x = torch.relu(self.fc1(x)) @@ -38,10 +37,10 @@ class Gene_Ontology_GNN(nn.Module): num_layers = 1, device = 'cuda', use_gene2vec = True, - gene2vec_file = 'data/gene2vec_16906.npy'): + gene2vec_file = None): super().__init__() if use_gene2vec: - gene2vec_weight = np.load(gene2vec_file) + gene2vec_weight = np.load(resolve_gene2vec_path(gene2vec_file)) gene2vec_weight = np.concatenate((gene2vec_weight, np.zeros((1, gene2vec_weight.shape[1]))), axis=0) gene2vec_weight = torch.from_numpy(gene2vec_weight).float() self.emb = nn.Embedding.from_pretrained(gene2vec_weight, freeze = False) @@ -163,7 +162,7 @@ class PerformerLM_GO(nn.Module): go_num_layers = 1, device = 'cuda', go_use_gene2vec = True, - gene2vec_file = 'data/gene2vec_16906.npy' + gene2vec_file = None ): super().__init__() @@ -288,7 +287,7 @@ class DualEncoderSCFM(nn.Module): go_num_layers = 1, device = 'cuda', go_use_gene2vec = True, - gene2vec_file = 'data/gene2vec_16906.npy' + gene2vec_file = None ): super().__init__() diff --git a/model_dependencies/performer_pytorch_cont/performer_pytorch.py b/model_dependencies/performer_pytorch_cont/performer_pytorch.py index e78c32a3ab5a743c503319f5d26b250f26a001a7..6e71bf30fc28f0ed522fe1e0e5f9658e2ec74d07 100644 --- a/model_dependencies/performer_pytorch_cont/performer_pytorch.py +++ b/model_dependencies/performer_pytorch_cont/performer_pytorch.py @@ -1,4 +1,5 @@ import math +from pathlib import Path import numpy as np import torch import torch.nn.functional as F @@ -20,6 +21,19 @@ except: # helpers +def resolve_gene2vec_path(gene2vec_file): + """Validate an explicit embedding file for a legacy gene vocabulary.""" + if gene2vec_file is None: + raise ValueError( + 'Supply gene2vec_file with embeddings matched to this legacy model gene order. ' + 'The original gene2vec_16906.npy file is not bundled.' + ) + path = Path(gene2vec_file).expanduser().resolve() + if not path.is_file(): + raise FileNotFoundError(f'Gene2vec embeddings not found: {path}') + return str(path) + + def exists(val): return val is not None @@ -346,13 +360,13 @@ class CausalSelfAttention_TSSA(nn.Module): self.n_embd = dim self.dropout = dropout self.temp = nn.Parameter(torch.ones(heads, 1)) - # 使用可学习的标量偏置替代固定长度的矩阵 + # Use a learnable scaling factor rather than a fixed-length matrix. self.denom_bias = nn.Parameter(torch.zeros(heads, 1, 1)) def forward(self, x, pos_emb=None, context=None, mask=None, context_mask=None, output_attentions=False): B, T, C = x.size() - # 计算query, key, value + # Compute queries, keys, and values. w = self.to_qkv(x) w = w.view(B, T, self.n_head, C // self.n_head).transpose(1, 2) # (B, nh, T, hs) w_sq = w ** 2 @@ -365,11 +379,11 @@ class CausalSelfAttention_TSSA(nn.Module): attn = 1. / (1 + dots) attn = self.attn_dropout(attn) y = - torch.mul(w.mul(Pi.unsqueeze(-1)), attn) - y = y.transpose(1, 2).contiguous().view(B, T, C) # 重新组装所有头的输出 + y = y.transpose(1, 2).contiguous().view(B, T, C) # Concatenate the attention-head outputs. y = self.resid_dropout(self.to_out(y)) if output_attentions: - # 返回随机的注意力权重,因为TSSA不直接计算传统的注意力矩阵 + # Return placeholder weights; TSSA does not form a conventional attention matrix. attn_weights = torch.zeros(B, self.n_head, T, T, device=x.device) return y, attn_weights return y @@ -392,26 +406,26 @@ class SelfAttention(nn.Module): qkv_bias = False ): super().__init__() - # 使用TSSA替换原有的attention机制 + # Use TSSA for the attention operation. self.attn = CausalSelfAttention_TSSA( dim=dim, heads=heads, dropout=dropout, bias=qkv_bias ) - # 保存这些参数以便兼容原有接口 + # Retain these attributes for compatibility with the existing interface. self.heads = heads self.global_heads = heads - local_heads - self.local_attn = None # TSSA不需要local attention - self.to_out = self.attn.to_out # 共享输出投影层 + self.local_attn = None # TSSA does not use local attention. + self.to_out = self.attn.to_out # Reuse the TSSA output projection. def forward(self, x, pos_emb = None, context = None, mask = None, context_mask = None, output_attentions = False, **kwargs): - # 调用TSSA的forward方法 - # 注意:TSSA目前只支持自注意力模式,不支持交叉注意力 + # Run the TSSA forward pass. + # This TSSA implementation supports self-attention only. if exists(context) and context is not x: raise NotImplementedError("Cross attention is not implemented in TSSA") - # 调用TSSA attention + # Apply TSSA attention. return self.attn(x, pos_emb, context, mask, context_mask, output_attentions) if output_attentions: return self.dropout(out), attn_weights @@ -447,9 +461,9 @@ def apply_rotary_pos_emb(q, k, sinu_pos): # sinusoidal positional embeddings class Gene2VecPositionalEmbedding(nn.Module): - def __init__(self, dim, max_seq_len, gene2vec_file = 'data/gene2vec_16906.npy'): + def __init__(self, dim, max_seq_len, gene2vec_file = None): super().__init__() - gene2vec_weight = np.load(gene2vec_file) + gene2vec_weight = np.load(resolve_gene2vec_path(gene2vec_file)) gene2vec_weight = np.concatenate((gene2vec_weight, np.zeros((1, gene2vec_weight.shape[1]))), axis=0) gene2vec_weight = torch.from_numpy(gene2vec_weight) self.emb = nn.Embedding.from_pretrained(gene2vec_weight) @@ -593,7 +607,8 @@ class PerformerLM(nn.Module): tie_embed = False, # False: output is num of tokens, True: output is dim of tokens //multiply final embeddings with token weights for logits, like gpt decoder// g2v_position_emb = True, # priority: gene2vec, no embedding auto_check_redraw = True, - qkv_bias = False + qkv_bias = False, + gene2vec_file = None ): super().__init__() print('cast_tuple') @@ -604,7 +619,7 @@ class PerformerLM(nn.Module): self.token_emb = TwoLayerMLP() print('g2v') if g2v_position_emb: - self.pos_emb = Gene2VecPositionalEmbedding(dim, max_seq_len) + self.pos_emb = Gene2VecPositionalEmbedding(dim, max_seq_len, gene2vec_file=gene2vec_file) self.layer_pos_emb = Always(None) else: self.pos_emb = torch.zeros_like diff --git a/model_dependencies/performer_pytorch_cont/performer_pytorch_cont/__init__.py b/model_dependencies/performer_pytorch_cont/performer_pytorch_cont/__init__.py index db6fd19e1a90e89dd5f16150d483da09aecb732e..97b4fa8dfe0614300aeb3b8ccc79a705a0a9e4e1 100644 --- a/model_dependencies/performer_pytorch_cont/performer_pytorch_cont/__init__.py +++ b/model_dependencies/performer_pytorch_cont/performer_pytorch_cont/__init__.py @@ -1,3 +1,2 @@ -# 明确指定使用 performer_pytorch.py(训练时使用的版本) -# 而不是 performer_pytorch_new.py(新版,使用 TSSA) +# Export the bundled Performer attention implementation. from performer_pytorch_cont.performer_pytorch import PerformerLM, Performer, FastAttention, SelfAttention \ No newline at end of file diff --git a/model_dependencies/performer_pytorch_cont/performer_pytorch_cont/ding_models.py b/model_dependencies/performer_pytorch_cont/performer_pytorch_cont/ding_models.py index 8dd3aff5ba622679fc216020170cd87a6c3ced0b..131015306fc61734b3fd994b0c6067f37fd25fe3 100644 --- a/model_dependencies/performer_pytorch_cont/performer_pytorch_cont/ding_models.py +++ b/model_dependencies/performer_pytorch_cont/performer_pytorch_cont/ding_models.py @@ -7,11 +7,10 @@ from tqdm import tqdm import networkx as nx import pickle import sys -sys.path.append('/mnt/c/Users/250010166/Desktop/single_cell/scLong/') class Gene2VecPositionalEmbedding_NoFreeze(nn.Module): - def __init__(self, dim, max_seq_len, gene2vec_file = 'data/gene2vec_16906.npy'): + def __init__(self, dim, max_seq_len, gene2vec_file = None): super().__init__() - gene2vec_weight = np.load(gene2vec_file) + gene2vec_weight = np.load(resolve_gene2vec_path(gene2vec_file)) gene2vec_weight = np.concatenate((gene2vec_weight, np.zeros((1, gene2vec_weight.shape[1]))), axis=0) gene2vec_weight = torch.from_numpy(gene2vec_weight) self.emb = nn.Embedding.from_pretrained(gene2vec_weight, freeze = False) @@ -23,8 +22,8 @@ class Gene2VecPositionalEmbedding_NoFreeze(nn.Module): class TwoLayerMLPEmb(nn.Module): def __init__(self, base_dim): super(TwoLayerMLPEmb, self).__init__() - self.fc1 = nn.Linear(1, 50) # Input size: 1, Output size: 100 - self.fc2 = nn.Linear(50, base_dim) # Input size: 100, Output size: 200 + self.fc1 = nn.Linear(1, 50) # Input size: 1; output size: 50. + self.fc2 = nn.Linear(50, base_dim) # Input size: 50; output size: base_dim. def forward(self, x): x = torch.relu(self.fc1(x)) @@ -38,10 +37,10 @@ class Gene_Ontology_GNN(nn.Module): num_layers = 1, device = 'cuda', use_gene2vec = True, - gene2vec_file = 'data/gene2vec_16906.npy'): + gene2vec_file = None): super().__init__() if use_gene2vec: - gene2vec_weight = np.load(gene2vec_file) + gene2vec_weight = np.load(resolve_gene2vec_path(gene2vec_file)) gene2vec_weight = np.concatenate((gene2vec_weight, np.zeros((1, gene2vec_weight.shape[1]))), axis=0) gene2vec_weight = torch.from_numpy(gene2vec_weight).float() self.emb = nn.Embedding.from_pretrained(gene2vec_weight, freeze = False) @@ -163,7 +162,7 @@ class PerformerLM_GO(nn.Module): go_num_layers = 1, device = 'cuda', go_use_gene2vec = True, - gene2vec_file = 'data/gene2vec_16906.npy' + gene2vec_file = None ): super().__init__() @@ -288,7 +287,7 @@ class DualEncoderSCFM(nn.Module): go_num_layers = 1, device = 'cuda', go_use_gene2vec = True, - gene2vec_file = 'data/gene2vec_16906.npy' + gene2vec_file = None ): super().__init__() diff --git a/model_dependencies/performer_pytorch_cont/performer_pytorch_cont/performer_pytorch.py b/model_dependencies/performer_pytorch_cont/performer_pytorch_cont/performer_pytorch.py index e78c32a3ab5a743c503319f5d26b250f26a001a7..6e71bf30fc28f0ed522fe1e0e5f9658e2ec74d07 100644 --- a/model_dependencies/performer_pytorch_cont/performer_pytorch_cont/performer_pytorch.py +++ b/model_dependencies/performer_pytorch_cont/performer_pytorch_cont/performer_pytorch.py @@ -1,4 +1,5 @@ import math +from pathlib import Path import numpy as np import torch import torch.nn.functional as F @@ -20,6 +21,19 @@ except: # helpers +def resolve_gene2vec_path(gene2vec_file): + """Validate an explicit embedding file for a legacy gene vocabulary.""" + if gene2vec_file is None: + raise ValueError( + 'Supply gene2vec_file with embeddings matched to this legacy model gene order. ' + 'The original gene2vec_16906.npy file is not bundled.' + ) + path = Path(gene2vec_file).expanduser().resolve() + if not path.is_file(): + raise FileNotFoundError(f'Gene2vec embeddings not found: {path}') + return str(path) + + def exists(val): return val is not None @@ -346,13 +360,13 @@ class CausalSelfAttention_TSSA(nn.Module): self.n_embd = dim self.dropout = dropout self.temp = nn.Parameter(torch.ones(heads, 1)) - # 使用可学习的标量偏置替代固定长度的矩阵 + # Use a learnable scaling factor rather than a fixed-length matrix. self.denom_bias = nn.Parameter(torch.zeros(heads, 1, 1)) def forward(self, x, pos_emb=None, context=None, mask=None, context_mask=None, output_attentions=False): B, T, C = x.size() - # 计算query, key, value + # Compute queries, keys, and values. w = self.to_qkv(x) w = w.view(B, T, self.n_head, C // self.n_head).transpose(1, 2) # (B, nh, T, hs) w_sq = w ** 2 @@ -365,11 +379,11 @@ class CausalSelfAttention_TSSA(nn.Module): attn = 1. / (1 + dots) attn = self.attn_dropout(attn) y = - torch.mul(w.mul(Pi.unsqueeze(-1)), attn) - y = y.transpose(1, 2).contiguous().view(B, T, C) # 重新组装所有头的输出 + y = y.transpose(1, 2).contiguous().view(B, T, C) # Concatenate the attention-head outputs. y = self.resid_dropout(self.to_out(y)) if output_attentions: - # 返回随机的注意力权重,因为TSSA不直接计算传统的注意力矩阵 + # Return placeholder weights; TSSA does not form a conventional attention matrix. attn_weights = torch.zeros(B, self.n_head, T, T, device=x.device) return y, attn_weights return y @@ -392,26 +406,26 @@ class SelfAttention(nn.Module): qkv_bias = False ): super().__init__() - # 使用TSSA替换原有的attention机制 + # Use TSSA for the attention operation. self.attn = CausalSelfAttention_TSSA( dim=dim, heads=heads, dropout=dropout, bias=qkv_bias ) - # 保存这些参数以便兼容原有接口 + # Retain these attributes for compatibility with the existing interface. self.heads = heads self.global_heads = heads - local_heads - self.local_attn = None # TSSA不需要local attention - self.to_out = self.attn.to_out # 共享输出投影层 + self.local_attn = None # TSSA does not use local attention. + self.to_out = self.attn.to_out # Reuse the TSSA output projection. def forward(self, x, pos_emb = None, context = None, mask = None, context_mask = None, output_attentions = False, **kwargs): - # 调用TSSA的forward方法 - # 注意:TSSA目前只支持自注意力模式,不支持交叉注意力 + # Run the TSSA forward pass. + # This TSSA implementation supports self-attention only. if exists(context) and context is not x: raise NotImplementedError("Cross attention is not implemented in TSSA") - # 调用TSSA attention + # Apply TSSA attention. return self.attn(x, pos_emb, context, mask, context_mask, output_attentions) if output_attentions: return self.dropout(out), attn_weights @@ -447,9 +461,9 @@ def apply_rotary_pos_emb(q, k, sinu_pos): # sinusoidal positional embeddings class Gene2VecPositionalEmbedding(nn.Module): - def __init__(self, dim, max_seq_len, gene2vec_file = 'data/gene2vec_16906.npy'): + def __init__(self, dim, max_seq_len, gene2vec_file = None): super().__init__() - gene2vec_weight = np.load(gene2vec_file) + gene2vec_weight = np.load(resolve_gene2vec_path(gene2vec_file)) gene2vec_weight = np.concatenate((gene2vec_weight, np.zeros((1, gene2vec_weight.shape[1]))), axis=0) gene2vec_weight = torch.from_numpy(gene2vec_weight) self.emb = nn.Embedding.from_pretrained(gene2vec_weight) @@ -593,7 +607,8 @@ class PerformerLM(nn.Module): tie_embed = False, # False: output is num of tokens, True: output is dim of tokens //multiply final embeddings with token weights for logits, like gpt decoder// g2v_position_emb = True, # priority: gene2vec, no embedding auto_check_redraw = True, - qkv_bias = False + qkv_bias = False, + gene2vec_file = None ): super().__init__() print('cast_tuple') @@ -604,7 +619,7 @@ class PerformerLM(nn.Module): self.token_emb = TwoLayerMLP() print('g2v') if g2v_position_emb: - self.pos_emb = Gene2VecPositionalEmbedding(dim, max_seq_len) + self.pos_emb = Gene2VecPositionalEmbedding(dim, max_seq_len, gene2vec_file=gene2vec_file) self.layer_pos_emb = Always(None) else: self.pos_emb = torch.zeros_like diff --git a/model_dependencies/performer_pytorch_cont/performer_pytorch_cont/performer_pytorch_old.py b/model_dependencies/performer_pytorch_cont/performer_pytorch_cont/performer_pytorch_old.py index 629eee6cca2837530d644385f3f256c0de935c36..6b0ed3a9171aaff2a94b7a2331560461f043c613 100644 --- a/model_dependencies/performer_pytorch_cont/performer_pytorch_cont/performer_pytorch_old.py +++ b/model_dependencies/performer_pytorch_cont/performer_pytorch_cont/performer_pytorch_old.py @@ -1,4 +1,5 @@ import math +from pathlib import Path import numpy as np import torch import torch.nn.functional as F @@ -20,6 +21,19 @@ except: # helpers +def resolve_gene2vec_path(gene2vec_file): + """Validate an explicit embedding file for a legacy gene vocabulary.""" + if gene2vec_file is None: + raise ValueError( + 'Supply gene2vec_file with embeddings matched to this legacy model gene order. ' + 'The original gene2vec_16906.npy file is not bundled.' + ) + path = Path(gene2vec_file).expanduser().resolve() + if not path.is_file(): + raise FileNotFoundError(f'Gene2vec embeddings not found: {path}') + return str(path) + + def exists(val): return val is not None @@ -437,9 +451,9 @@ def apply_rotary_pos_emb(q, k, sinu_pos): # sinusoidal positional embeddings class Gene2VecPositionalEmbedding(nn.Module): - def __init__(self, dim, max_seq_len, gene2vec_file = 'data/gene2vec_16906.npy'): + def __init__(self, dim, max_seq_len, gene2vec_file = None): super().__init__() - gene2vec_weight = np.load(gene2vec_file) + gene2vec_weight = np.load(resolve_gene2vec_path(gene2vec_file)) gene2vec_weight = np.concatenate((gene2vec_weight, np.zeros((1, gene2vec_weight.shape[1]))), axis=0) gene2vec_weight = torch.from_numpy(gene2vec_weight) self.emb = nn.Embedding.from_pretrained(gene2vec_weight) @@ -583,7 +597,8 @@ class PerformerLM(nn.Module): tie_embed = False, # False: output is num of tokens, True: output is dim of tokens //multiply final embeddings with token weights for logits, like gpt decoder// g2v_position_emb = True, # priority: gene2vec, no embedding auto_check_redraw = True, - qkv_bias = False + qkv_bias = False, + gene2vec_file = None ): super().__init__() print('cast_tuple') @@ -594,7 +609,7 @@ class PerformerLM(nn.Module): self.token_emb = TwoLayerMLP() print('g2v') if g2v_position_emb: - self.pos_emb = Gene2VecPositionalEmbedding(dim, max_seq_len) + self.pos_emb = Gene2VecPositionalEmbedding(dim, max_seq_len, gene2vec_file=gene2vec_file) self.layer_pos_emb = Always(None) else: self.pos_emb = torch.zeros_like diff --git a/model_dependencies/performer_pytorch_cont/performer_pytorch_old.py b/model_dependencies/performer_pytorch_cont/performer_pytorch_old.py index 629eee6cca2837530d644385f3f256c0de935c36..6b0ed3a9171aaff2a94b7a2331560461f043c613 100644 --- a/model_dependencies/performer_pytorch_cont/performer_pytorch_old.py +++ b/model_dependencies/performer_pytorch_cont/performer_pytorch_old.py @@ -1,4 +1,5 @@ import math +from pathlib import Path import numpy as np import torch import torch.nn.functional as F @@ -20,6 +21,19 @@ except: # helpers +def resolve_gene2vec_path(gene2vec_file): + """Validate an explicit embedding file for a legacy gene vocabulary.""" + if gene2vec_file is None: + raise ValueError( + 'Supply gene2vec_file with embeddings matched to this legacy model gene order. ' + 'The original gene2vec_16906.npy file is not bundled.' + ) + path = Path(gene2vec_file).expanduser().resolve() + if not path.is_file(): + raise FileNotFoundError(f'Gene2vec embeddings not found: {path}') + return str(path) + + def exists(val): return val is not None @@ -437,9 +451,9 @@ def apply_rotary_pos_emb(q, k, sinu_pos): # sinusoidal positional embeddings class Gene2VecPositionalEmbedding(nn.Module): - def __init__(self, dim, max_seq_len, gene2vec_file = 'data/gene2vec_16906.npy'): + def __init__(self, dim, max_seq_len, gene2vec_file = None): super().__init__() - gene2vec_weight = np.load(gene2vec_file) + gene2vec_weight = np.load(resolve_gene2vec_path(gene2vec_file)) gene2vec_weight = np.concatenate((gene2vec_weight, np.zeros((1, gene2vec_weight.shape[1]))), axis=0) gene2vec_weight = torch.from_numpy(gene2vec_weight) self.emb = nn.Embedding.from_pretrained(gene2vec_weight) @@ -583,7 +597,8 @@ class PerformerLM(nn.Module): tie_embed = False, # False: output is num of tokens, True: output is dim of tokens //multiply final embeddings with token weights for logits, like gpt decoder// g2v_position_emb = True, # priority: gene2vec, no embedding auto_check_redraw = True, - qkv_bias = False + qkv_bias = False, + gene2vec_file = None ): super().__init__() print('cast_tuple') @@ -594,7 +609,7 @@ class PerformerLM(nn.Module): self.token_emb = TwoLayerMLP() print('g2v') if g2v_position_emb: - self.pos_emb = Gene2VecPositionalEmbedding(dim, max_seq_len) + self.pos_emb = Gene2VecPositionalEmbedding(dim, max_seq_len, gene2vec_file=gene2vec_file) self.layer_pos_emb = Always(None) else: self.pos_emb = torch.zeros_like