diff --git "a/BioMod/models_performer_module.py" "b/BioMod/models_performer_module.py" --- "a/BioMod/models_performer_module.py" +++ "b/BioMod/models_performer_module.py" @@ -1,1716 +1,1657 @@ -""" -Encoder-only Performer模型,使用module数据输入 -参考MAE的masking方式,但使用Performer作为encoder -只保留encoder,去掉decoder -""" - -import sys -import os -parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -if parent_dir not in sys.path: - sys.path.insert(0, parent_dir) - -# The bundled Performer implementation lives under model_dependencies rather -# than at the repository root. Add that directory explicitly so every task -# entry point can import the model without requiring a caller-managed -# PYTHONPATH. -model_dependencies_dir = os.path.join(parent_dir, "model_dependencies") -if os.path.isdir(model_dependencies_dir) and model_dependencies_dir not in sys.path: - sys.path.insert(0, model_dependencies_dir) - -try: - from performer_pytorch_cont.ding_models import ( - Performer, - Always, - cast_tuple, - ) -except ImportError: - try: - from performer_pytorch_cont.performer_pytorch import ( - Performer, - Always, - cast_tuple, - ) - except ImportError: - from performer_pytorch.performer_pytorch import ( - Performer, - Always, - cast_tuple, - ) -import torch -from torch import nn -import torch.nn.functional as F -from typing import Callable, Dict, List, Optional, Tuple, Union -import numpy as np - - -def _migrate_legacy_module_name(value: str) -> str: - """Translate serialized pre-rename names without changing their semantics.""" - replacements = ( - ("PATHCH", "MODULE"), - ("Pathch", "Module"), - ("pathch", "module"), - ("PATCHES", "MODULES"), - ("Patches", "Modules"), - ("patches", "modules"), - ("PATCH", "MODULE"), - ("Patch", "Module"), - ("patch", "module"), - ) - for old, new in replacements: - value = value.replace(old, new) - return value - - -def migrate_legacy_module_config(config: Dict[str, object]) -> Dict[str, object]: - """Return a module-named copy of an older serialized configuration.""" - def translate(value): - if isinstance(value, dict): - migrated = {} - for key, item in value.items(): - new_key = _migrate_legacy_module_name(key) if isinstance(key, str) else key - if new_key in migrated: - raise ValueError( - f"Legacy configuration migration would duplicate key {new_key!r}" - ) - migrated[new_key] = translate(item) - return migrated - if isinstance(value, list): - return [translate(item) for item in value] - if isinstance(value, tuple): - return tuple(translate(item) for item in value) - if isinstance(value, str): - return _migrate_legacy_module_name(value) - return value - - return translate(dict(config)) - - -def migrate_legacy_module_state_dict(state_dict): - """Translate old state-dict key names before PyTorch performs strict loading.""" - try: - migrated = state_dict.__class__() - except TypeError: - migrated = {} - - for key, value in state_dict.items(): - new_key = _migrate_legacy_module_name(key) if isinstance(key, str) else key - if new_key in migrated: - # If both spellings are supplied, prefer the explicit new spelling. - if key == new_key: - migrated[new_key] = value - continue - migrated[new_key] = value - - if hasattr(state_dict, "_metadata"): - migrated._metadata = { - _migrate_legacy_module_name(key): value - for key, value in state_dict._metadata.items() - } - return migrated - - -def normalize_fusion_method(fusion_method: str) -> str: - if fusion_method == "expression_only_mlp": - return "add" - if fusion_method in {"concatenate", "concat_v2"}: - raise ValueError( - f"fusion_method={fusion_method!r} has been removed. " - "Use one of: 'multiply', 'add', 'gated_residual'." - ) - if fusion_method not in {"multiply", "add", "gated_residual"}: - raise ValueError( - f"Unsupported fusion_method: {fusion_method}. " - "Use one of: 'multiply', 'add', 'gated_residual'." - ) - return fusion_method - - -REMOVED_PRETRAIN_CONFIG_KEYS = ( - "preprocess_gene2vec", - "extra_gene_mode", - "extra_gene_num", - "extra_gene_rank_path", - "extra_gene_module_size", - "extra_gene_module_strategy", - "module_embed_chunk_padded_gene_budget", - "cross_attn_pure_memory", -) - - -def sanitize_performer_module_config( - config: Dict[str, object], - *, - source: str = "config", - log_fn: Optional[Callable[[str], None]] = None, -) -> Dict[str, object]: - cleaned = migrate_legacy_module_config(config) - legacy_mode = str(cleaned.get("extra_gene_mode", "none")).lower() - legacy_num = int(cleaned.get("extra_gene_num", 0) or 0) - if legacy_mode != "none" or legacy_num > 0: - raise ValueError( - f"{source} requests removed extra-gene support " - f"(extra_gene_mode={legacy_mode}, extra_gene_num={legacy_num}). " - "Please use a checkpoint trained without extra genes." - ) - - removed = [key for key in REMOVED_PRETRAIN_CONFIG_KEYS if key in cleaned] - for key in removed: - cleaned.pop(key, None) - - if removed and log_fn is not None: - log_fn(f"{source}: dropped deprecated config keys: {', '.join(sorted(removed))}") - - return cleaned - -# 仅复用 ModuleToGeneDecoder,ModuleEmbedding 在本文件中重新实现以支持三种融合方式 -# from sclong_module.models_module import ModuleToGeneDecoder - - -class CrossAttentionDecoderBlock(nn.Module): - """ - Decoder block with masked-query -> visible-memory cross-attention only. - """ - - def __init__( - self, - dim: int, - heads: int, - ff_mult: int = 4, - attn_dropout: float = 0.0, - ff_dropout: float = 0.0, - ) -> None: - super().__init__() - if heads <= 0: - raise ValueError(f"CrossAttentionDecoderBlock requires heads > 0, got {heads}") - if dim % heads != 0: - raise ValueError( - f"CrossAttentionDecoderBlock requires dim divisible by heads, got dim={dim}, heads={heads}" - ) - - self.query_norm = nn.LayerNorm(dim) - self.memory_norm = nn.LayerNorm(dim) - self.cross_attn = nn.MultiheadAttention( - embed_dim=dim, - num_heads=heads, - dropout=attn_dropout, - batch_first=True, - ) - hidden_dim = int(dim * ff_mult) - self.ff_norm = nn.LayerNorm(dim) - self.ff = nn.Sequential( - nn.Linear(dim, hidden_dim), - nn.GELU(), - nn.Dropout(ff_dropout), - nn.Linear(hidden_dim, dim), - nn.Dropout(ff_dropout), - ) - - def forward(self, query: torch.Tensor, memory: torch.Tensor) -> torch.Tensor: - if query.shape[1] == 0: - return query - - if memory.shape[1] > 0: - attn_out, _ = self.cross_attn( - self.query_norm(query), - self.memory_norm(memory), - self.memory_norm(memory), - need_weights=False, - ) - query = query + attn_out - - query = query + self.ff(self.ff_norm(query)) - return query - - -class ModuleEmbedding(nn.Module): - """ - Gene-level embedding + MLP + attention pooling -> module token. - 支持三种融合方式: - - "multiply": 表达值 × gene2vec(原始方式) - - "add": 表达值经过独立 MLP,再与 gene embedding 相加 - - "gated_residual": 表达值先映射到 gene 空间,再由 gate 控制残差注入强度 - - 支持 use_gene2vec 参数控制是否使用预训练的 gene2vec: - - True: 使用预训练的 gene2vec 权重初始化(默认行为) - - False: 使用随机初始化的 embedding(用于消融实验) - """ - - @staticmethod - def _build_adaptive_buckets( - modules_by_length: Dict[int, List[int]], - bucket_count: int, - ) -> Dict[int, List[int]]: - """Minimize padded gene positions for a fixed number of buckets.""" - length_groups = sorted(modules_by_length.items()) - group_count = len(length_groups) - bucket_count = min(bucket_count, group_count) - if bucket_count <= 0: - raise ValueError("MODULE_EMBED_BUCKET_COUNT must be positive") - - prefix_counts = [0] - for _, module_ids in length_groups: - prefix_counts.append(prefix_counts[-1] + len(module_ids)) - - # dp[b][j] is the least padded element count for the first j distinct - # lengths split into b contiguous buckets. Contiguous length ranges - # preserve a single static padded shape per bucket. - infinity = float("inf") - previous = [infinity] * (group_count + 1) - previous[0] = 0 - split_points: List[List[int]] = [ - [-1] * (group_count + 1) for _ in range(bucket_count + 1) - ] - for bucket_idx in range(1, bucket_count + 1): - current = [infinity] * (group_count + 1) - for end in range(bucket_idx, group_count + 1): - bucket_length = length_groups[end - 1][0] - best_cost = infinity - best_start = -1 - for start in range(bucket_idx - 1, end): - cost = previous[start] + bucket_length * ( - prefix_counts[end] - prefix_counts[start] - ) - if cost < best_cost: - best_cost = cost - best_start = start - current[end] = best_cost - split_points[bucket_idx][end] = best_start - previous = current - - ranges = [] - end = group_count - for bucket_idx in range(bucket_count, 0, -1): - start = split_points[bucket_idx][end] - ranges.append((start, end)) - end = start - ranges.reverse() - - buckets: Dict[int, List[int]] = {} - for start, end in ranges: - bucket_length = length_groups[end - 1][0] - buckets[bucket_length] = [ - module_idx - for _, module_ids in length_groups[start:end] - for module_idx in module_ids - ] - return buckets - - def __init__( - self, - module_sets: List[List[int]], - num_genes: int, - gene_embed_dim: int, - embed_dim: int, - gene2vec_weight: Optional[torch.Tensor] = None, - fusion_method: str = "multiply", # "multiply" / "add" / "gated_residual" - use_gene2vec: bool = True, # 是否使用预训练的 gene2vec(False 则随机初始化) - ): - super().__init__() - self.num_modules = len(module_sets) - self.module_index_names: List[str] = [] - modules_by_length: Dict[int, List[int]] = {} - for module_idx, module_genes in enumerate(module_sets): - buffer_name = f"_module_gene_idx_{module_idx}" - # Module indices are static metadata. Non-persistent buffers follow - # module.to(device) without changing checkpoint state_dict keys. - self.register_buffer( - buffer_name, - torch.tensor(module_genes, dtype=torch.long), - persistent=False, - ) - self.module_index_names.append(buffer_name) - modules_by_length.setdefault(len(module_genes), []).append(module_idx) - - # Group modules with the same gene count so their identical operations - # can be issued as a single batched operation. All buffers are - # non-persistent: checkpoint keys remain byte-for-byte compatible. - self.module_batch_mode = os.environ.get("MODULE_EMBED_BATCH_MODE", "adaptive").lower() - if self.module_batch_mode not in {"legacy", "exact", "bucket", "adaptive"}: - raise ValueError( - "MODULE_EMBED_BATCH_MODE must be one of: " - "'legacy', 'exact', 'bucket', 'adaptive'; " - f"got {self.module_batch_mode!r}" - ) - self.module_groups: List[Tuple[int, Tuple[int, ...], int, int]] = [] - grouped_gene_indices: List[int] = [] - grouped_module_order: List[int] = [] - group_offset = 0 - for module_length in sorted(modules_by_length): - module_ids = tuple(modules_by_length[module_length]) - group_gene_indices = [ - gene_idx - for module_idx in module_ids - for gene_idx in module_sets[module_idx] - ] - group_end = group_offset + len(group_gene_indices) - self.module_groups.append( - (module_length, module_ids, group_offset, group_end) - ) - grouped_gene_indices.extend(group_gene_indices) - grouped_module_order.extend(module_ids) - group_offset = group_end - - restore_order = [0] * self.num_modules - for grouped_pos, module_idx in enumerate(grouped_module_order): - restore_order[module_idx] = grouped_pos - self.register_buffer( - "_grouped_gene_idx", - torch.tensor(grouped_gene_indices, dtype=torch.long), - persistent=False, - ) - self.register_buffer( - "_module_group_restore_idx", - torch.tensor(restore_order, dtype=torch.long), - persistent=False, - ) - - if self.module_batch_mode == "adaptive": - bucket_count = int(os.environ.get("MODULE_EMBED_BUCKET_COUNT", "16")) - modules_by_bucket = self._build_adaptive_buckets( - modules_by_length, - bucket_count, - ) - else: - modules_by_bucket: Dict[int, List[int]] = {} - for module_idx, module_genes in enumerate(module_sets): - bucket_length = 1 << (len(module_genes) - 1).bit_length() - modules_by_bucket.setdefault(bucket_length, []).append(module_idx) - self.module_buckets: List[ - Tuple[int, Tuple[int, ...], Tuple[int, ...], int, int, str] - ] = [] - bucketed_gene_indices: List[int] = [] - bucketed_module_order: List[int] = [] - bucket_offset = 0 - for bucket_idx, bucket_length in enumerate(sorted(modules_by_bucket)): - module_ids = tuple(modules_by_bucket[bucket_length]) - module_lengths = tuple(len(module_sets[idx]) for idx in module_ids) - group_gene_indices: List[int] = [] - valid_mask = torch.zeros(len(module_ids), bucket_length, dtype=torch.bool) - for group_pos, module_idx in enumerate(module_ids): - genes = module_sets[module_idx] - pad_count = bucket_length - len(genes) - group_gene_indices.extend(genes) - group_gene_indices.extend([0] * pad_count) - valid_mask[group_pos, :len(genes)] = True - group_end = bucket_offset + len(group_gene_indices) - mask_name = f"_module_bucket_mask_{bucket_idx}" - self.register_buffer(mask_name, valid_mask, persistent=False) - self.module_buckets.append( - ( - bucket_length, - module_ids, - module_lengths, - bucket_offset, - group_end, - mask_name, - ) - ) - bucketed_gene_indices.extend(group_gene_indices) - bucketed_module_order.extend(module_ids) - bucket_offset = group_end - - bucket_restore_order = [0] * self.num_modules - for grouped_pos, module_idx in enumerate(bucketed_module_order): - bucket_restore_order[module_idx] = grouped_pos - self.register_buffer( - "_bucketed_gene_idx", - torch.tensor(bucketed_gene_indices, dtype=torch.long), - persistent=False, - ) - self.register_buffer( - "_module_bucket_restore_idx", - torch.tensor(bucket_restore_order, dtype=torch.long), - persistent=False, - ) - # Deduplicate shared pointwise transformations, preserving module-local - # membership, order, multiplicity and attention normalization. - # Static maps add no checkpoint keys or trainable parameters. - unique_genes = sorted({gene for module in module_sets for gene in module}) - unique_position = {gene: pos for pos, gene in enumerate(unique_genes)} - bucket_unique_positions = [ - unique_position.get(gene, 0) for gene in bucketed_gene_indices - ] - self.register_buffer( - "_unique_gene_idx", torch.tensor(unique_genes, dtype=torch.long), - persistent=False, - ) - self.register_buffer( - "_bucket_unique_idx", - torch.tensor(bucket_unique_positions, dtype=torch.long), - persistent=False, - ) - self.gene_dedup_enabled = os.environ.get("MODULE_GENE_DEDUP", "1") == "1" - # The unlimited candidate was slower at batches 16/64/256 on H100. - # Default to the measured beneficial batch=1 path; 0 enables an explicit - # unlimited research comparison without changing checkpoint structure. - self.gene_dedup_max_batch_size = int( - os.environ.get("MODULE_GENE_DEDUP_MAX_BATCH_SIZE", "1") - ) - if self.gene_dedup_max_batch_size < 0: - raise ValueError("MODULE_GENE_DEDUP_MAX_BATCH_SIZE must be nonnegative") - self.fusion_method = normalize_fusion_method(fusion_method) - self.use_gene2vec = use_gene2vec - self.gene_embed_dim = gene_embed_dim - self.embed_dim = embed_dim - - # 根据 use_gene2vec 决定是否使用预训练权重 - if use_gene2vec and gene2vec_weight is not None: - self.gene_embed = nn.Embedding.from_pretrained(gene2vec_weight, freeze=False) - if self.gene_embed.weight.shape[0] < num_genes: - pad = torch.zeros(num_genes - self.gene_embed.weight.shape[0], self.gene_embed.weight.shape[1]) - self.gene_embed.weight.data = torch.cat([self.gene_embed.weight.data, pad], dim=0) - else: - # 随机初始化 embedding(消融实验:不使用 gene2vec 先验知识) - self.gene_embed = nn.Embedding(num_genes, gene_embed_dim) - # 使用 Xavier 初始化,与其他层保持一致 - nn.init.xavier_uniform_(self.gene_embed.weight) - - # 融合层(对齐MAE结构) - # 所有方式的proj都是: Dg → E → E(两层MLP) - self.expr_mlp = None - self.gated_expr_mlp = None - self.gated_residual_gate = None - - if self.fusion_method == "add": - # add: 表达值经过独立MLP,再与gene2vec相加 - hidden_dim = max(50, gene_embed_dim // 2) - self.expr_mlp = nn.Sequential( - nn.Linear(1, hidden_dim), - nn.GELU(), - nn.Linear(hidden_dim, gene_embed_dim), - ) - elif self.fusion_method == "gated_residual": - # gated_residual: gene2vec提供基底语义,表达值通过门控残差注入。 - hidden_dim = max(64, gene_embed_dim // 2) - self.gated_expr_mlp = nn.Sequential( - nn.Linear(1, hidden_dim), - nn.GELU(), - nn.Linear(hidden_dim, gene_embed_dim), - nn.LayerNorm(gene_embed_dim), - ) - self.gated_residual_gate = nn.Sequential( - nn.Linear(gene_embed_dim * 2, gene_embed_dim), - nn.GELU(), - nn.Linear(gene_embed_dim, gene_embed_dim), - nn.Sigmoid(), - ) - # else: multiply方式不需要额外的MLP - - # 统一的proj层: Dg → E → E - self.proj = nn.Sequential( - nn.Linear(gene_embed_dim, embed_dim), - nn.GELU(), - nn.Linear(embed_dim, embed_dim), - ) - self.attn_score = nn.Linear(embed_dim, 1) - self.pool_affine_enabled = os.environ.get("MODULE_POOL_AFFINE", "1") == "1" - # Paired H100 screening found this path helps batches 16/64/256 but - # regresses batch=1 against the dedup fast path. Preserve that fast path. - self.pool_affine_min_batch_size = int(os.environ.get("MODULE_POOL_AFFINE_MIN_BATCH_SIZE", "2")) - if self.pool_affine_min_batch_size < 1: - raise ValueError("MODULE_POOL_AFFINE_MIN_BATCH_SIZE must be positive") - # Derived model weights only; no cell-dependent features are cached. - # These buffers add no parameters or checkpoint entries. - self.register_buffer("_pool_affine_score_weight", torch.empty(0), persistent=False) - self.register_buffer("_pool_affine_score_bias", torch.empty(0), persistent=False) - self._pool_affine_cache_signature = None - - @property - def module_sets(self) -> List[torch.Tensor]: - """Expose module indices for existing interpretability callers.""" - return [getattr(self, name) for name in self.module_index_names] - - def _pool_affine_signature(self): - tensors = (self.proj[2].weight, self.proj[2].bias, - self.attn_score.weight, self.attn_score.bias) - return tuple((id(t), t.data_ptr(), t._version, t.device, t.dtype) for t in tensors) - - @torch.no_grad() - def prepare_pool_affine(self): - """Compose final affine and score AFTER checkpoint/device/dtype placement. - - Call again after changing weights, dtype or device. Forward rejects a - stale cache rather than silently using outdated model-derived weights. - """ - weight = self.proj[2].weight - compose_dtype = torch.float64 if weight.dtype == torch.float64 else torch.float32 - with torch.autocast(device_type=weight.device.type, enabled=False): - score_weight = self.attn_score.weight.to(dtype=compose_dtype) - self._pool_affine_score_weight = ( - score_weight @ weight.to(dtype=compose_dtype) - ).to(dtype=weight.dtype) - self._pool_affine_score_bias = ( - score_weight @ self.proj[2].bias.to(dtype=compose_dtype) - + self.attn_score.bias.to(dtype=compose_dtype) - ).to(dtype=weight.dtype) - self._pool_affine_cache_signature = self._pool_affine_signature() - - def _forward_pool_affine( - self, - expr: torch.Tensor, - return_module_targets: bool = True, - ) -> Tuple[torch.Tensor, List[torch.Tensor]]: - """Pool first-layer activations before the shared final affine. - - In real arithmetic, score(W2*h+b2)=(score_weight*W2)*h+const, - and sum(alpha*(W2*h+b2))=W2*sum(alpha*h)+b2*sum(alpha). - BF16 reassociation is validated separately; it is not bitwise equivalent. - """ - if self._pool_affine_cache_signature != self._pool_affine_signature(): - raise RuntimeError("Pool-affine cache is stale; call prepare_pool_affine() after loading/placing weights") - batch_size = expr.shape[0] - flat_expr = torch.index_select(expr, 1, self._bucketed_gene_idx) - flat_gene_emb = self.gene_embed(self._bucketed_gene_idx) - pooled_hidden = [] - attention_sums = [] - module_targets: List[Optional[torch.Tensor]] = ( - [None] * self.num_modules if return_module_targets else [] - ) - for ( - bucket_length, module_ids, module_lengths, start, end, mask_name - ) in self.module_buckets: - group_size = len(module_ids) - expr_group = flat_expr[:, start:end].reshape(batch_size, group_size, bucket_length) - gene_emb = flat_gene_emb[start:end].reshape(group_size, bucket_length, self.gene_embed_dim) - gene_emb = gene_emb.unsqueeze(0).expand(batch_size, -1, -1, -1) - fused = self._fuse_expr_and_gene_emb(expr_group, gene_emb) - hidden = self.proj[1](self.proj[0](fused)) - del fused - scores = F.linear(hidden, self._pool_affine_score_weight, self._pool_affine_score_bias).squeeze(-1) - valid_mask = getattr(self, mask_name) - scores = scores.masked_fill(~valid_mask.unsqueeze(0), float("-inf")) - attn = torch.softmax(scores, dim=2) - pooled_hidden.append(self._weighted_pool(hidden, attn)) - # Quantized softmax weights need not sum exactly to one. Preserve - # their sum in wider accumulation for the final bias contribution. - sum_dtype = torch.float64 if hidden.dtype == torch.float64 else torch.float32 - attention_sums.append(attn.sum(dim=2, dtype=sum_dtype)) - del hidden, scores, attn - if return_module_targets: - for group_pos, (module_idx, module_length) in enumerate(zip(module_ids, module_lengths)): - module_targets[module_idx] = expr_group[:, group_pos, :module_length] - pooled = torch.cat(pooled_hidden, dim=1) - sums = torch.cat(attention_sums, dim=1) - # One final affine over the module tokens, not over every gene position. - linear = F.linear(pooled, self.proj[2].weight, None) - with torch.autocast(device_type=expr.device.type, enabled=False): - wider = linear.to(dtype=sums.dtype) + sums.unsqueeze(-1) * self.proj[2].bias.to(dtype=sums.dtype) - tokens = wider.to(dtype=linear.dtype) - tokens = torch.index_select(tokens, 1, self._module_bucket_restore_idx) - return tokens, module_targets # type: ignore[return-value] - - def _fuse_expr_and_gene_emb(self, expr_slice: torch.Tensor, gene_emb: torch.Tensor) -> torch.Tensor: - """ - 将表达值与gene embedding融合到gene-token空间。 - 支持任意前缀维度,只要求最后一维是gene维。 - expr_slice: (..., G) / gene_emb: (..., G, Dg) -> (..., G, Dg) - """ - if self.fusion_method == "add": - expr_emb = self.expr_mlp(expr_slice.unsqueeze(-1)) # (B, G, Dg) - return expr_emb + gene_emb - if self.fusion_method == "gated_residual": - expr_emb = self.gated_expr_mlp(expr_slice.unsqueeze(-1)) # (B, G, Dg) - gate = self.gated_residual_gate(torch.cat([expr_emb, gene_emb], dim=-1)) # (B, G, Dg) - return gene_emb + gate * expr_emb - # 默认 multiply:表达值 × gene2vec - return expr_slice.unsqueeze(-1) * gene_emb # (B, G, Dg) - - @staticmethod - def _weighted_pool(gene_tok: torch.Tensor, attn: torch.Tensor) -> torch.Tensor: - """Pool without materializing the full attention-weighted tensor.""" - implementation = os.environ.get("MODULE_POOL_IMPL", "bmm") - if implementation == "bmm": - length, width = gene_tok.shape[-2:] - pooled = torch.bmm( - attn.reshape(-1, 1, length), - gene_tok.reshape(-1, length, width), - ) - return pooled.reshape(*gene_tok.shape[:-2], width) - if implementation != "mulsum": - raise ValueError(f"Unsupported MODULE_POOL_IMPL={implementation}") - return torch.sum(gene_tok * attn.unsqueeze(-1), dim=-2) - - def _forward_legacy( - self, - expr: torch.Tensor, - return_module_targets: bool = True, - ) -> Tuple[torch.Tensor, List[torch.Tensor]]: - """ - 逐module稳定实现。 - expr: (batch, num_genes) - returns: - module_tokens: (batch, num_modules, embed_dim) - module_targets: list of length num_modules, each (batch, genes_in_module) - """ - batch_size, _ = expr.shape - module_tokens = [] - module_targets: List[torch.Tensor] = [] - for index_name in self.module_index_names: - gene_idx = getattr(self, index_name) - expr_slice = torch.index_select(expr, dim=1, index=gene_idx) - if return_module_targets: - module_targets.append(expr_slice) - - gene_emb = self.gene_embed(gene_idx) - gene_emb = gene_emb.unsqueeze(0).expand(batch_size, -1, -1) - gene_tok = self._fuse_expr_and_gene_emb(expr_slice, gene_emb) - gene_tok = self.proj(gene_tok) - - attn = self.attn_score(gene_tok).squeeze(-1) - attn = torch.softmax(attn, dim=1) - pooled = self._weighted_pool(gene_tok, attn) - module_tokens.append(pooled) - - return torch.stack(module_tokens, dim=1), module_targets - - def _forward_grouped_exact( - self, - expr: torch.Tensor, - return_module_targets: bool = True, - ) -> Tuple[torch.Tensor, List[torch.Tensor]]: - """Batch modules of identical length without padding or approximation.""" - batch_size, _ = expr.shape - flat_expr = torch.index_select(expr, dim=1, index=self._grouped_gene_idx) - flat_gene_emb = self.gene_embed(self._grouped_gene_idx) - grouped_tokens = [] - module_targets: List[Optional[torch.Tensor]] = ( - [None] * self.num_modules if return_module_targets else [] - ) - - for module_length, module_ids, start, end in self.module_groups: - group_size = len(module_ids) - expr_group = flat_expr[:, start:end].reshape( - batch_size, group_size, module_length - ) - gene_emb = flat_gene_emb[start:end].reshape( - group_size, module_length, self.gene_embed_dim - ) - gene_emb = gene_emb.unsqueeze(0).expand(batch_size, -1, -1, -1) - gene_tok = self._fuse_expr_and_gene_emb(expr_group, gene_emb) - gene_tok = self.proj(gene_tok) - - attn = self.attn_score(gene_tok).squeeze(-1) - attn = torch.softmax(attn, dim=2) - pooled = self._weighted_pool(gene_tok, attn) - grouped_tokens.append(pooled) - - if return_module_targets: - for group_pos, module_idx in enumerate(module_ids): - module_targets[module_idx] = expr_group[:, group_pos, :] - - tokens = torch.cat(grouped_tokens, dim=1) - tokens = torch.index_select( - tokens, dim=1, index=self._module_group_restore_idx - ) - # Every module is assigned exactly once when groups are constructed. - return tokens, module_targets # type: ignore[return-value] - - def _forward_bucketed( - self, - expr: torch.Tensor, - return_module_targets: bool = True, - ) -> Tuple[torch.Tensor, List[torch.Tensor]]: - """Batch modules into fixed padded lengths and mask padded positions.""" - batch_size, _ = expr.shape - flat_expr = torch.index_select(expr, dim=1, index=self._bucketed_gene_idx) - flat_gene_emb = self.gene_embed(self._bucketed_gene_idx) - bucket_tokens = [] - module_targets: List[Optional[torch.Tensor]] = ( - [None] * self.num_modules if return_module_targets else [] - ) - - for ( - bucket_length, - module_ids, - module_lengths, - start, - end, - mask_name, - ) in self.module_buckets: - group_size = len(module_ids) - expr_group = flat_expr[:, start:end].reshape( - batch_size, group_size, bucket_length - ) - gene_emb = flat_gene_emb[start:end].reshape( - group_size, bucket_length, self.gene_embed_dim - ) - gene_emb = gene_emb.unsqueeze(0).expand(batch_size, -1, -1, -1) - gene_tok = self._fuse_expr_and_gene_emb(expr_group, gene_emb) - gene_tok = self.proj(gene_tok) - - valid_mask = getattr(self, mask_name) - attn = self.attn_score(gene_tok).squeeze(-1) - attn = attn.masked_fill(~valid_mask.unsqueeze(0), float("-inf")) - attn = torch.softmax(attn, dim=2) - pooled = self._weighted_pool(gene_tok, attn) - bucket_tokens.append(pooled) - - if return_module_targets: - for group_pos, (module_idx, module_length) in enumerate( - zip(module_ids, module_lengths) - ): - module_targets[module_idx] = expr_group[:, group_pos, :module_length] - - tokens = torch.cat(bucket_tokens, dim=1) - tokens = torch.index_select( - tokens, dim=1, index=self._module_bucket_restore_idx - ) - return tokens, module_targets # type: ignore[return-value] - - def _forward_unique_genes( - self, - expr: torch.Tensor, - return_module_targets: bool = True, - ) -> Tuple[torch.Tensor, List[torch.Tensor]]: - """Reuse gene features within this inference forward; pool per module. - - Gather one bucket at a time rather than the full repeated-gene tensor. - """ - batch_size = expr.shape[0] - unique_expr = torch.index_select(expr, 1, self._unique_gene_idx) - unique_emb = self.gene_embed(self._unique_gene_idx) - unique_emb = unique_emb.unsqueeze(0).expand(batch_size, -1, -1) - fused = self._fuse_expr_and_gene_emb(unique_expr, unique_emb) - unique_tokens = self.proj(fused) - del fused, unique_emb - unique_scores = self.attn_score(unique_tokens).squeeze(-1) - bucket_tokens = [] - module_targets: List[Optional[torch.Tensor]] = ( - [None] * self.num_modules if return_module_targets else [] - ) - for ( - bucket_length, module_ids, module_lengths, start, end, mask_name - ) in self.module_buckets: - group_size = len(module_ids) - indices = self._bucket_unique_idx[start:end] - scores = torch.index_select(unique_scores, 1, indices).reshape( - batch_size, group_size, bucket_length - ) - valid_mask = getattr(self, mask_name) - scores = scores.masked_fill(~valid_mask.unsqueeze(0), float("-inf")) - attn = torch.softmax(scores, dim=2) - gene_tok = torch.index_select(unique_tokens, 1, indices).reshape( - batch_size, group_size, bucket_length, self.embed_dim - ) - bucket_tokens.append(self._weighted_pool(gene_tok, attn)) - del gene_tok, attn, scores - if return_module_targets: - expr_group = torch.index_select(unique_expr, 1, indices).reshape( - batch_size, group_size, bucket_length - ) - for group_pos, (module_idx, module_length) in enumerate( - zip(module_ids, module_lengths) - ): - module_targets[module_idx] = expr_group[:, group_pos, :module_length] - tokens = torch.cat(bucket_tokens, dim=1) - tokens = torch.index_select(tokens, 1, self._module_bucket_restore_idx) - return tokens, module_targets # type: ignore[return-value] - - def forward( - self, - expr: torch.Tensor, - return_module_targets: bool = True, - ) -> Tuple[torch.Tensor, List[torch.Tensor]]: - # Keep original training/gradient behavior and gradient reduction order. - if ( - self.pool_affine_enabled and not self.training and not torch.is_grad_enabled() - and expr.shape[0] >= self.pool_affine_min_batch_size - ): - return self._forward_pool_affine(expr, return_module_targets) - if ( - self.gene_dedup_enabled and not self.training and not torch.is_grad_enabled() - and (self.gene_dedup_max_batch_size == 0 - or expr.shape[0] <= self.gene_dedup_max_batch_size) - ): - return self._forward_unique_genes(expr, return_module_targets) - if self.module_batch_mode == "legacy": - return self._forward_legacy(expr, return_module_targets) - if self.module_batch_mode == "exact": - return self._forward_grouped_exact(expr, return_module_targets) - return self._forward_bucketed(expr, return_module_targets) - - -class PerformerEncoderOnly_Module(nn.Module): - """ - Encoder-only Performer模型,使用module数据输入 - 参考MAE的masking方式,但使用Performer作为encoder - - 支持 use_gene2vec 参数控制是否使用预训练的 gene2vec: - - True: 使用预训练的 gene2vec 权重初始化(默认行为) - - False: 使用随机初始化的 embedding(用于消融实验) - """ - def __init__( - self, - module_sets: List[List[int]], # module数据 - num_genes: int, # 基因总数 - max_seq_len, # max length of sequence (module数量) - base_dim = 200, #基础embedding维度,所有token映射到这个维度 - enc_depth = 24, # encoder深度(参考MAE) - enc_heads = 16, # encoder头数(参考MAE) - enc_dim_head = 64, # encoder每个头的维度 - mask_ratio: float = 0.75, # masking比例(参考MAE) - # ==================== Performer特有参数 ==================== - local_attn_heads = 0,# 使用局部注意力的头数 - local_window_size = 256,# 局部注意力的窗口大小 - causal = False, # 是否使用因果(自回归)注意力 - ff_mult = 4,# 前馈网络(FFN)的扩展倍数 - nb_features = None,# Performer随机特征的数量 - feature_redraw_interval = 1000,# 重绘随机特征的步数间隔 - reversible = False,# 是否使用可逆层(节省显存) - ff_chunks = 1,# FFN的分块数量(用于内存优化) - ff_glu = False, # FFN是否使用GLU(门控线性单元)激活 - # ==================== Dropout参数 ==================== - emb_dropout = 0., - ff_dropout = 0., - attn_dropout = 0., - # ==================== Performer高级参数 ==================== - generalized_attention = False, - kernel_fn = nn.ReLU(), - use_scalenorm = False, - use_rezero = False, - cross_attend = False, - no_projection = False, - # ==================== 位置编码参数 ==================== - g2v_position_emb = False, # 对于modules,使用可学习的positional embedding - auto_check_redraw = True,# 是否自动检查并重绘projection矩阵 - qkv_bias = False, # QKV投影是否使用bias - # ==================== Gene2vec相关参数 ==================== - gene2vec_file = 'data/gene2vec_16906.npy', - gene_embed_dim: int = 200, # module embedding中基因embedding的维度 - fusion_method: str = "multiply", # "multiply" / "add" / "gated_residual" - use_cls_token: bool = False, # 是否使用CLS token(MAE默认不使用) - cls_loss_weight: float = 0.0, # CLS辅助loss权重;>0时直接训练CLS预测静态module均值 - cls_head_hidden_dim: int = 0, # CLS head隐藏层维度;0表示使用base_dim - use_gene2vec: bool = True, # 是否使用预训练的 gene2vec(False 则随机初始化,用于消融实验) - loss_mode: str = "gene_mean", # "gene_mean"(当前) / "module_mean"(MAE风格) - recon_target_level: str = "gene", # "gene"(逐基因重建) / "module_mean_value"(每module输出1个均值) - pretrain_arch: str = "mae", # "mae" / "encoder_only" / "encdec" - mask_style: str = "drop", # "drop" / "replace" - mask_token_mode: str = "learned", # 目前仅支持learned - pos_embed_mode: str = "module_id", # 目前仅支持module_id(learned module-id embedding) - decoder_type: str = "simple", # "none" / "simple" / "performer" / "cross_attn" - decoder_gene_agg: bool = False, # True: 先做module->gene聚合,再基因级重建 - dec_embed_dim: int = 512, - dec_depth: int = 4, - dec_heads: int = 8, - dec_dim_head: int = 64, - dec_ff_mult: int = 4, - dec_ff_dropout: float = 0.0, - dec_attn_dropout: float = 0.0, - cross_attn_pure_memory: Optional[bool] = None, - # ==================== Fine-tuning enhancements ==================== - use_family_emb: bool = False, - module_family_ids: Optional[List[int]] = None, - use_module_gate: bool = False, - ): - super().__init__() - local_attn_heads = cast_tuple(local_attn_heads) - self.max_seq_len = max_seq_len - self.base_dim = base_dim - self.num_genes = num_genes - self.module_sets = module_sets - self.num_modules = len(module_sets) - self.fusion_method = normalize_fusion_method(fusion_method) - self.mask_ratio = mask_ratio - self.zero_mask_identity_fastpath = ( - os.environ.get("ZERO_MASK_IDENTITY_FASTPATH", "1") == "1" - ) - self.use_cls = use_cls_token - self.cls_loss_weight = float(cls_loss_weight) if use_cls_token else 0.0 - self.cls_head_hidden_dim = int(cls_head_hidden_dim) - self.use_gene2vec = use_gene2vec - self.loss_mode = loss_mode - self.recon_target_level = recon_target_level - self.pretrain_arch = str(pretrain_arch).lower() - self.mask_style = str(mask_style).lower() - self.mask_token_mode = str(mask_token_mode).lower() - self.pos_embed_mode = str(pos_embed_mode).lower() - self.decoder_type = decoder_type - self.decoder_gene_agg = decoder_gene_agg - self.dec_embed_dim = dec_embed_dim - # Legacy knob retained only for backward-compatible kwargs/config loading. - # Current pretraining always uses pure visible memory in the cross-attn decoder. - self.cross_attn_pure_memory = True - self.base_module_gene_counts = [len(p) for p in module_sets] - self.module_gene_counts = list(self.base_module_gene_counts) - self.total_module_count = len(self.module_gene_counts) - if self.loss_mode not in {"gene_mean", "module_mean"}: - raise ValueError(f"Unsupported loss_mode: {self.loss_mode}. Use 'gene_mean' or 'module_mean'.") - if self.cls_loss_weight < 0: - raise ValueError(f"cls_loss_weight must be >= 0, got {self.cls_loss_weight}") - if self.cls_head_hidden_dim < 0: - raise ValueError(f"cls_head_hidden_dim must be >= 0, got {self.cls_head_hidden_dim}") - if self.recon_target_level not in {"gene", "module_mean_value"}: - raise ValueError( - f"Unsupported recon_target_level: {self.recon_target_level}. " - "Use 'gene' or 'module_mean_value'." - ) - if self.pretrain_arch not in {"mae", "encoder_only", "encdec"}: - raise ValueError( - f"Unsupported pretrain_arch: {self.pretrain_arch}. " - "Use 'mae', 'encoder_only' or 'encdec'." - ) - if self.mask_style not in {"drop", "replace"}: - raise ValueError( - f"Unsupported mask_style: {self.mask_style}. Use 'drop' or 'replace'." - ) - if self.mask_token_mode not in {"learned"}: - raise ValueError( - f"Unsupported mask_token_mode: {self.mask_token_mode}. " - "Only 'learned' is currently supported." - ) - if self.pos_embed_mode not in {"module_id"}: - raise ValueError( - f"Unsupported pos_embed_mode: {self.pos_embed_mode}. " - "Only 'module_id' is currently supported." - ) - if self.decoder_type not in {"none", "simple", "performer", "cross_attn"}: - raise ValueError( - f"Unsupported decoder_type: {self.decoder_type}. " - "Use 'none', 'simple', 'performer' or 'cross_attn'." - ) - if self.pretrain_arch == "mae": - if self.mask_style != "drop": - raise ValueError( - "pretrain_arch='mae' is fixed to mask_style='drop'. " - "Use pretrain_arch='encoder_only' or 'encdec' for mask_style='replace'." - ) - if self.decoder_type == "none": - raise ValueError("pretrain_arch='mae' requires decoder_type != 'none'.") - elif self.pretrain_arch == "encoder_only": - if self.mask_style != "replace": - raise ValueError("pretrain_arch='encoder_only' requires mask_style='replace'.") - if self.decoder_type != "none": - raise ValueError("pretrain_arch='encoder_only' requires decoder_type='none'.") - else: - if self.mask_style != "replace": - raise ValueError("pretrain_arch='encdec' requires mask_style='replace'.") - if self.decoder_type not in {"performer", "cross_attn"}: - raise ValueError( - "pretrain_arch='encdec' requires decoder_type in {'performer', 'cross_attn'}." - ) - if self.decoder_gene_agg and self.recon_target_level != "gene": - raise ValueError("decoder_gene_agg=True only supports recon_target_level='gene'.") - if self.dec_embed_dim <= 0: - raise ValueError(f"dec_embed_dim must be positive, got {self.dec_embed_dim}") - - # 加载gene2vec权重用于module embedding(仅�� use_gene2vec=True 时使用) - gene2vec_weight = None - if use_gene2vec and gene2vec_file and os.path.exists(gene2vec_file): - gene2vec_weight = torch.from_numpy(np.load(gene2vec_file)).float() - if gene2vec_weight.shape[0] < num_genes: - pad = torch.zeros(num_genes - gene2vec_weight.shape[0], gene2vec_weight.shape[1]) - gene2vec_weight = torch.cat([gene2vec_weight, pad], dim=0) - gene_embed_dim = gene2vec_weight.shape[1] - elif not use_gene2vec: - # 不使用 gene2vec 时,打印提示信息 - print(f"[INFO] use_gene2vec=False: Using randomly initialized gene embeddings (dim={gene_embed_dim})") - - # Module Embedding层 - self.module_embed = ModuleEmbedding( - module_sets=module_sets, - num_genes=num_genes, - gene_embed_dim=gene_embed_dim, - embed_dim=base_dim, - gene2vec_weight=gene2vec_weight, - fusion_method=self.fusion_method, - use_gene2vec=use_gene2vec, - ) - - # Positional embedding for modules - # 对pathway module来说更准确地说是 module-id embedding,而不是几何位置。 - pos_tokens = self.total_module_count + (1 if use_cls_token else 0) - self.pos_emb = nn.Parameter(torch.zeros(1, pos_tokens, base_dim)) - self.layer_pos_emb = Always(None) - - # Family-type embedding (2B): adds family identity to module tokens - self.use_family_emb = use_family_emb - if use_family_emb and module_family_ids is not None: - self.register_buffer( - 'module_family_ids', - torch.tensor(module_family_ids, dtype=torch.long), - ) - self.family_emb = nn.Parameter(torch.zeros(4, base_dim)) - else: - self.module_family_ids = None - self.family_emb = None - - # Sparse module gate (1B): learns per-module scalar gate - self.use_module_gate = use_module_gate - if use_module_gate: - self.module_gate = nn.Sequential( - nn.LayerNorm(base_dim), - nn.Linear(base_dim, 1), - nn.Sigmoid(), - ) - else: - self.module_gate = None - - # CLS token (可选) - if use_cls_token: - self.cls_token = nn.Parameter(torch.zeros(1, 1, base_dim)) - else: - self.cls_token = None - if self.use_cls and self.cls_loss_weight > 0: - cls_hidden_dim = self.cls_head_hidden_dim if self.cls_head_hidden_dim > 0 else base_dim - self.cls_head = nn.Sequential( - nn.LayerNorm(base_dim), - nn.Linear(base_dim, cls_hidden_dim), - nn.GELU(), - nn.Linear(cls_hidden_dim, self.num_modules), - ) - else: - self.cls_head = None - - self.dropout = nn.Dropout(emb_dropout) - - # Encoder (使用Performer) - self.encoder = Performer( - base_dim, enc_depth, enc_heads, enc_dim_head, - local_attn_heads, local_window_size, causal, ff_mult, nb_features, - feature_redraw_interval, reversible, ff_chunks, generalized_attention, - kernel_fn, use_scalenorm, use_rezero, ff_glu, ff_dropout, attn_dropout, - cross_attend, no_projection, auto_check_redraw, qkv_bias - ) - self.encoder_norm = nn.LayerNorm(base_dim) - - # Encoder -> Decoder 维度投影 - # - performer/cross_attn: 允许使用更小的 dec_embed_dim - # - simple/none: 不需要投影,直接使用 base_dim - if self.decoder_type in {"performer", "cross_attn"}: - self.enc_to_dec = nn.Identity() if self.dec_embed_dim == base_dim else nn.Linear(base_dim, self.dec_embed_dim) - self.decoder_dim = self.dec_embed_dim - else: - # simple/none: 不使用 enc_to_dec 投影,直接用 base_dim - self.enc_to_dec = nn.Identity() - self.decoder_dim = base_dim - - # Encoder-side mask token(replace-mask路径使用) - if self.mask_style == "replace": - self.encoder_mask_token = nn.Parameter(torch.zeros(1, 1, base_dim)) - else: - self.encoder_mask_token = None - - # Decoder-side mask token(仅MAE drop-mask路径需要) - if self.pretrain_arch == "mae": - self.mask_token = nn.Parameter(torch.zeros(1, 1, self.decoder_dim)) - else: - self.mask_token = None - - # 可选 decoder:用于让masked token与visible token发生信息交互(更接近MAE) - if self.decoder_type == "performer": - self.decoder_pos_emb = nn.Parameter(torch.zeros(1, pos_tokens, self.dec_embed_dim)) - self.decoder_dropout = nn.Dropout(emb_dropout) - self.decoder = Performer( - self.dec_embed_dim, dec_depth, dec_heads, dec_dim_head, - local_attn_heads=cast_tuple(0), local_window_size=256, causal=False, ff_mult=dec_ff_mult, - nb_features=None, feature_redraw_interval=feature_redraw_interval, - reversible=False, ff_chunks=1, generalized_attention=False, - kernel_fn=nn.ReLU(), use_scalenorm=False, use_rezero=False, ff_glu=False, - ff_dropout=dec_ff_dropout, attn_dropout=dec_attn_dropout, - cross_attend=False, no_projection=False, auto_check_redraw=auto_check_redraw, - qkv_bias=qkv_bias - ) - self.decoder_norm = nn.LayerNorm(self.dec_embed_dim) - elif self.decoder_type == "cross_attn": - self.decoder_pos_emb = nn.Parameter(torch.zeros(1, pos_tokens, self.dec_embed_dim)) - self.decoder_dropout = nn.Dropout(emb_dropout) - self.decoder = nn.ModuleList( - [ - CrossAttentionDecoderBlock( - dim=self.dec_embed_dim, - heads=dec_heads, - ff_mult=dec_ff_mult, - attn_dropout=dec_attn_dropout, - ff_dropout=dec_ff_dropout, - ) - for _ in range(dec_depth) - ] - ) - self.decoder_norm = nn.LayerNorm(self.dec_embed_dim) - else: - self.decoder_pos_emb = None - self.decoder_dropout = None - self.decoder = None - self.decoder_norm = None - - # 预计算 module<->gene 聚合索引(decoder_gene_agg 模式使用) - if self.decoder_gene_agg: - pg_module_idx = [] - pg_gene_idx = [] - for module_idx, module_genes in enumerate(self.module_sets): - for gene_idx in module_genes: - if 0 <= gene_idx < self.num_genes: - pg_module_idx.append(module_idx) - pg_gene_idx.append(gene_idx) - self.register_buffer("pg_module_idx", torch.tensor(pg_module_idx, dtype=torch.long)) - self.register_buffer("pg_gene_idx", torch.tensor(pg_gene_idx, dtype=torch.long)) - pg_gene_counts = torch.zeros(self.num_genes, dtype=torch.long) - if len(pg_gene_idx) > 0: - pg_gene_counts.scatter_add_(0, self.pg_gene_idx, torch.ones_like(self.pg_gene_idx)) - self.register_buffer("pg_gene_counts", pg_gene_counts) - else: - self.pg_module_idx = None - self.pg_gene_idx = None - self.pg_gene_counts = None - - # 重建头: - # - 默认: 每个module一个线性头 - # - decoder_gene_agg: module->gene后,用基因特异性头做重建 - # - module_mean_value: 每个module只输出一个均值标量 - if self.recon_target_level == "gene": - if self.decoder_gene_agg: - self.module_heads = None - self.gene_head = None - # 基因特异性重建头: 每个基因一套 (D->1) 参数 - self.gene_w = nn.Parameter(torch.rand(self.num_genes, self.decoder_dim, 1)) - self.gene_b = nn.Parameter(torch.rand(self.num_genes)) - nn.init.xavier_normal_(self.gene_w) - nn.init.xavier_normal_(self.gene_b.unsqueeze(-1)) - else: - self.module_heads = nn.ModuleList( - [nn.Linear(self.decoder_dim, g) for g in self.module_gene_counts] - ) - self.gene_head = None - self.gene_w = None - self.gene_b = None - self.module_scalar_heads = None - else: - self.module_heads = None - self.gene_head = None - self.gene_w = None - self.gene_b = None - self.module_scalar_heads = nn.ModuleList( - [nn.Linear(self.decoder_dim, 1) for _ in self.module_gene_counts] - ) - - self.last_loss_breakdown = { - "recon_loss": 0.0, - "cls_loss": 0.0, - "total_loss": 0.0, - } - - self._init_weights() - - def _init_weights(self) -> None: - """初始化权重""" - nn.init.normal_(self.pos_emb, std=0.02) - if self.encoder_mask_token is not None: - nn.init.normal_(self.encoder_mask_token, std=0.02) - if self.mask_token is not None: - nn.init.normal_(self.mask_token, std=0.02) - if self.decoder_pos_emb is not None: - nn.init.normal_(self.decoder_pos_emb, std=0.02) - if self.cls_token is not None: - nn.init.normal_(self.cls_token, std=0.02) - if self.family_emb is not None: - nn.init.normal_(self.family_emb, std=0.02) - for m in self.modules(): - if isinstance(m, nn.Linear): - nn.init.xavier_uniform_(m.weight) - if m.bias is not None: - nn.init.constant_(m.bias, 0) - elif isinstance(m, nn.LayerNorm): - nn.init.constant_(m.bias, 0) - nn.init.constant_(m.weight, 1.0) - - def _sample_module_mask( - self, - batch_size: int, - seq_len: int, - device: torch.device, - mask_ratio: float, - ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - if mask_ratio <= 0: - ids_restore = torch.arange(seq_len, device=device, dtype=torch.long).unsqueeze(0).expand(batch_size, -1) - mask = torch.zeros(batch_size, seq_len, device=device) - return mask, ids_restore, ids_restore - - len_keep = int(seq_len * (1 - mask_ratio)) - noise = torch.rand(batch_size, seq_len, device=device) - ids_shuffle = torch.argsort(noise, dim=1) - ids_restore = torch.argsort(ids_shuffle, dim=1) - mask = torch.ones([batch_size, seq_len], device=device) - mask[:, :len_keep] = 0 - mask = torch.gather(mask, dim=1, index=ids_restore) - return mask, ids_shuffle, ids_restore - - def random_masking(self, x: torch.Tensor, mask_ratio: float) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - """ - Per-sample random masking (参考MAE的实现) - x: (B, L, D) - module tokens - Returns: - x_masked: (B, L_visible, D) - 可见的module tokens - mask: (B, L) - mask矩阵,1表示masked,0表示visible - ids_restore: (B, L) - 用于恢复原始顺序的索引 - """ - """ - ┌─────────────────────────────────────────────────────────────────────────┐ - │ Random Masking 过程 │ - ├─────────────────────────────────────────────────────────────────────────┤ - │ │ - │ 原始顺序 (module 0-9): │ - │ [P0, P1, P2, P3, P4, P5, P6, P7, P8, P9] │ - │ │ - │ Step 1: 生成随机噪声 │ - │ noise = [0.3, 0.9, 0.1, 0.7, 0.2, 0.8, 0.4, 0.6, 0.5, 0.0] │ - │ │ - │ Step 2: 按噪声排序(随机打乱) │ - │ ids_shuffle = [9, 2, 4, 0, 6, 8, 7, 3, 5, 1] │ - │ # 噪声值: 0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9 │ - │ # 对应module: P9, P2, P4, P0, P6, P8, P7, P3, P5, P1 │ - │ │ - │ Step 3: 保留前25%(可见),mask后75% │ - │ ids_keep = [9, 2, 4] # 可见module的索引 │ - │ 可见tokens = [P9, P2, P4] # 打乱顺序后的可见module │ - │ │ - │ Step 4: ids_restore 用于恢复原始顺序 │ - │ ids_restore = [3, 9, 1, 7, 2, 8, 4, 6, 5, 0] │ - │ # 含义: │ - │ # ids_restore[0] = 3 → 位置0的module,打乱后在位置3 │ - │ # ids_restore[1] = 9 → 位置1的module,打乱后在位置9(被mask了) │ - │ # ids_restore[2] = 1 → 位置2的module,打乱后在位置1 │ - │ │ - └─────────────────────────────────────────────────────────────────────────┘ - """ - b, l, d = x.shape - mask, ids_shuffle, ids_restore = self._sample_module_mask( - batch_size=b, - seq_len=l, - device=x.device, - mask_ratio=mask_ratio, - ) - if mask_ratio <= 0 and self.zero_mask_identity_fastpath: - return x, mask, ids_restore - len_keep = int(l * (1 - mask_ratio)) - ids_keep = ids_shuffle[:, :len_keep] - x_masked = torch.gather(x, dim=1, index=ids_keep.unsqueeze(-1).repeat(1, 1, d)) - return x_masked, mask, ids_restore - - def replace_masking( - self, - x: torch.Tensor, - mask_ratio: float - ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - """ - BERT/scLong风格:保留完整module序列长度,仅将被mask的module替换为learned mask token。 - """ - if self.encoder_mask_token is None: - raise RuntimeError("replace_masking requires encoder_mask_token to be initialized.") - - b, l, _ = x.shape - mask, _, _ = self._sample_module_mask( - batch_size=b, - seq_len=l, - device=x.device, - mask_ratio=mask_ratio, - ) - ids_restore = torch.arange(l, device=x.device).unsqueeze(0).expand(b, -1) - mask_tokens = self.encoder_mask_token.expand(b, l, -1) - x_masked = torch.where(mask.bool().unsqueeze(-1), mask_tokens, x) - return x_masked, mask, ids_restore - - def _add_module_position_embedding(self, x: torch.Tensor) -> torch.Tensor: - pos = self.pos_emb[:, 1:, :] if self.use_cls else self.pos_emb - if pos.shape[1] != x.shape[1]: - raise RuntimeError( - f"pos_emb length mismatch: pos={pos.shape[1]}, tokens={x.shape[1]}, " - f"modules={self.num_modules}" - ) - x = x + pos - if self.use_family_emb and self.family_emb is not None: - x = x + self.family_emb[self.module_family_ids] - return x - - def forward_encoder( - self, - cells: torch.Tensor, - mask_ratio: Optional[float] = None, - return_module_targets: bool = True, - ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, List[torch.Tensor]]: - """ - 编码器前向传播 - cells: (B, num_genes) - 基因表达值 - Returns: - latent: (B, L_encoded+cls, D) - encoder输出 - mask: (B, L) - mask矩阵 - ids_restore: (B, L) - 恢复顺序的索引 - module_targets: List[Tensor] - 每个module的基因表达目标值 - """ - mask_ratio = self.mask_ratio if mask_ratio is None else mask_ratio - - # Step 1: 将基因表达转换为module tokens - x, module_targets = self.module_embed( - cells, - return_module_targets=return_module_targets, - ) # (B, L, D) Module Embedding层 - - # Step 2/3: 根据mask风格决定是在encoder前丢弃token,还是替换为learned mask token - if self.mask_style == "replace": - # 先替换内容token,再统一加module-id embedding,保留完整序列长度 - x, mask, ids_restore = self.replace_masking(x, mask_ratio) - x = self._add_module_position_embedding(x) - else: - # MAE路径:先加module-id embedding,再drop掉masked module - x = self._add_module_position_embedding(x) - x, mask, ids_restore = self.random_masking(x, mask_ratio) - - # Sparse module gating (1B): suppress unimportant modules - if self.use_module_gate and self.module_gate is not None: - gate = self.module_gate(x) # (B, L_visible, 1) - x = x * gate - - # Step 4: 添加CLS token(如果使用) - if self.use_cls and self.cls_token is not None: - cls_token = self.cls_token + self.pos_emb[:, :1, :] - cls_tokens = cls_token.expand(x.shape[0], -1, -1) - x = torch.cat((cls_tokens, x), dim=1) - - # Step 5: Encoder处理 - x_emb = self.dropout(x) - layer_pos_emb = self.layer_pos_emb(x_emb)# None - x = self.encoder(x_emb, pos_emb=layer_pos_emb) #输入输出 (B, L_visible, D)->(B, L_visible, D) 每个 token 从"单独的 module 表示"变成"全局上下文感知的表示" - x = self.encoder_norm(x) #layernorm - - return x, mask, ids_restore, module_targets - - def _build_full_tokens(self, x: torch.Tensor, ids_restore: torch.Tensor) -> torch.Tensor: - """ - 将encoder输出补齐mask token并恢复到原始module顺序。 - 返回: (B, L(+cls), D) - """ - # 先映射到decoder维度,再补mask token并恢复顺序 - x = self.enc_to_dec(x) - if self.use_cls: - cls_token = x[:, :1, :] - module_tokens = x[:, 1:, :] - else: - cls_token = None - module_tokens = x - - num_mask_tokens = ids_restore.shape[1] - module_tokens.shape[1] - if num_mask_tokens > 0: - if self.mask_token is None: - raise RuntimeError( - "Decoder-side mask_token is missing, but _build_full_tokens needs to restore masked positions." - ) - mask_tokens = self.mask_token.repeat(x.shape[0], num_mask_tokens, 1) - x_ = torch.cat([module_tokens, mask_tokens], dim=1) - else: - x_ = module_tokens - - x_ = torch.gather(x_, dim=1, index=ids_restore.unsqueeze(-1).repeat(1, 1, x.shape[2])) - - if self.use_cls and cls_token is not None: - return torch.cat([cls_token, x_], dim=1) - return x_ - - def _aggregate_module_to_gene( - self, - module_feats: torch.Tensor, - ) -> torch.Tensor: - """ - module_feats: (B, L, D) -> gene_feats: (B, G, D) - 每个基因特征=包含该基因的所有module特征均值。 - """ - bsz, _, dim = module_feats.shape - gene_feats = torch.zeros(bsz, self.num_genes, dim, device=module_feats.device, dtype=module_feats.dtype) - gene_counts = torch.zeros(bsz, self.num_genes, device=module_feats.device, dtype=module_feats.dtype) - - # 静态pathway module贡献 - if self.pg_module_idx is not None and self.pg_module_idx.numel() > 0: - pair_module_feats = module_feats[:, self.pg_module_idx, :] # (B, num_pairs, D) - gene_idx_expand = self.pg_gene_idx.view(1, -1, 1).expand(bsz, -1, dim) - gene_feats.scatter_add_(1, gene_idx_expand, pair_module_feats) - base_gene_counts = self.pg_gene_counts.to(dtype=module_feats.dtype, device=module_feats.device) - gene_counts = gene_counts + base_gene_counts.view(1, self.num_genes) - - gene_counts = gene_counts.clamp(min=1.0).unsqueeze(-1) # (B, G, 1) - return gene_feats / gene_counts - - def _predict_from_module_features( - self, - module_feats: torch.Tensor, - ) -> Union[List[torch.Tensor], torch.Tensor]: - """module_feats: (B, L, D)""" - if self.recon_target_level == "gene" and self.decoder_gene_agg: - gene_feats = self._aggregate_module_to_gene(module_feats) # (B, G, D) - assert self.gene_w is not None and self.gene_b is not None - # 基因特异性线性头:每个基因各自投影 - preds = torch.einsum("bgd,gdk->bgk", gene_feats, self.gene_w).squeeze(-1) - return preds + self.gene_b.unsqueeze(0) # (B, G) - - preds: List[torch.Tensor] = [] - if self.recon_target_level == "gene": - assert self.module_heads is not None - for i, head in enumerate(self.module_heads): - preds.append(head(module_feats[:, i, :])) - else: - assert self.module_scalar_heads is not None - for i, head in enumerate(self.module_scalar_heads): - preds.append(head(module_feats[:, i, :])) - return preds - - def _build_cls_targets(self, module_targets: List[torch.Tensor]) -> torch.Tensor: - """ - 使用静态module的均值表达作为CLS监督目标。 - 这样CLS需要从可见module中汇聚全局信息,预测整细胞的module-level profile。 - """ - static_module_targets = module_targets[: self.num_modules] - if len(static_module_targets) != self.num_modules: - raise RuntimeError( - f"cls target module count mismatch: expected {self.num_modules}, got {len(static_module_targets)}" - ) - module_means = [target.mean(dim=1, keepdim=True) for target in static_module_targets] - return torch.cat(module_means, dim=1) - - def compute_cls_loss( - self, - latent: torch.Tensor, - module_targets: List[torch.Tensor], - ) -> Tuple[torch.Tensor, torch.Tensor]: - if not self.use_cls or self.cls_head is None: - zero = torch.tensor(0.0, device=latent.device, dtype=latent.dtype) - return zero, zero - cls_encoding = latent[:, 0, :] - cls_preds = self.cls_head(cls_encoding) - cls_targets = self._build_cls_targets(module_targets).to(dtype=cls_preds.dtype) - cls_loss = F.mse_loss(cls_preds, cls_targets) - return cls_loss, cls_encoding - - def forward_decoder_simple( - self, - x: torch.Tensor, - ids_restore: torch.Tensor, - ) -> Union[List[torch.Tensor], torch.Tensor]: - """ - 简化的decoder:在encoder输出后,添加mask tokens,然后unshuffle,最后通过module heads重建 - x: (B, L_visible+cls, D) - encoder输出 - ids_restore: (B, L) - 恢复顺序的索引,L是总module数 - Returns: - - 默认模式: List[Tensor],每个module一个预测 - - decoder_gene_agg模式: Tensor(B, G),基因级预测 - """ - x_full = self._build_full_tokens(x, ids_restore) - module_feats = x_full[:, 1:, :] if self.use_cls else x_full - return self._predict_from_module_features(module_feats) - - def forward_decoder_performer( - self, - x: torch.Tensor, - ids_restore: torch.Tensor, - ) -> Union[List[torch.Tensor], torch.Tensor]: - """ - MAE风格decoder:补齐mask tokens并恢复顺序后,再经Performer做token mixing再重建。 - """ - x_full = self._build_full_tokens(x, ids_restore) - pos = self.decoder_pos_emb[:, :x_full.shape[1], :] - x_full = self.decoder_dropout(x_full + pos) - x_full = self.decoder(x_full, pos_emb=self.layer_pos_emb(x_full)) - x_full = self.decoder_norm(x_full) - module_feats = x_full[:, 1:, :] if self.use_cls else x_full - return self._predict_from_module_features(module_feats) - - def forward_decoder_cross_attn( - self, - x: torch.Tensor, - mask: torch.Tensor, - ids_restore: torch.Tensor, - ) -> Union[List[torch.Tensor], torch.Tensor]: - """ - Cross-attention-only decoder: - masked queries attend to visible encoder tokens, without query-query self-attention. - """ - if self.decoder is None or self.decoder_pos_emb is None or self.decoder_dropout is None or self.decoder_norm is None: - raise RuntimeError("cross_attn decoder requested but decoder modules are not initialized.") - - x_full = self._build_full_tokens(x, ids_restore) - if self.use_cls: - cls_token = x_full[:, :1, :] - module_tokens = x_full[:, 1:, :] - module_pos = self.decoder_pos_emb[:, 1:x_full.shape[1], :] - cls_memory = cls_token - else: - cls_token = None - module_tokens = x_full - module_pos = self.decoder_pos_emb[:, :x_full.shape[1], :] - cls_memory = None - - query_tokens = self.decoder_dropout(module_tokens + module_pos) - module_feats = module_tokens.clone() - - mask_bool = mask.bool() - visible_mask = ~mask_bool - batch_size, _, dim = module_feats.shape - visible_per_sample = int(visible_mask[0].sum().item()) if visible_mask.shape[1] > 0 else 0 - masked_per_sample = int(mask_bool[0].sum().item()) if mask_bool.shape[1] > 0 else 0 - - if masked_per_sample > 0: - visible_memory = module_feats[visible_mask].view(batch_size, visible_per_sample, dim) - if cls_memory is not None: - visible_memory = torch.cat([cls_memory, visible_memory], dim=1) - - masked_queries = query_tokens[mask_bool].view(batch_size, masked_per_sample, dim) - for block in self.decoder: - masked_queries = block(masked_queries, visible_memory) - # AMP may run attention/norm in fp32 for stability. Cast back before - # scattering into the decoder-state tensor, which may still be bf16. - masked_queries = self.decoder_norm(masked_queries).to(dtype=module_feats.dtype) - module_feats[mask_bool] = masked_queries.reshape(-1, dim) - - return self._predict_from_module_features(module_feats) - - def forward_predictor_from_encoder( - self, - x: torch.Tensor, - ) -> Union[List[torch.Tensor], torch.Tensor]: - module_feats = x[:, 1:, :] if self.use_cls else x - return self._predict_from_module_features(module_feats) - - def compute_loss( - self, - preds: Union[List[torch.Tensor], torch.Tensor], - targets: List[torch.Tensor], - mask: torch.Tensor, - ) -> torch.Tensor: # module层计算loss ablation study - """ - 计算重建损失(只对masked modules计算) - preds: List[Tensor] - 每个module的预测,每个 (B, genes_in_module) - targets: List[Tensor] - 每个module的目标,每个 (B, genes_in_module) - mask: (B, L) - mask矩阵,1表示masked,0表示visible - """ - if self.recon_target_level == "gene" and self.decoder_gene_agg: - assert isinstance(preds, torch.Tensor) - # 关键:严格按原module mask计算loss。 - # 对每个masked module,取该module涉及的基因在基因级预测中的子向量来计算误差。 - total_loss = torch.tensor(0.0, device=mask.device, dtype=torch.float32) - total_count = torch.tensor(0.0, device=mask.device, dtype=torch.float32) - for i, t in enumerate(targets): - module_mask = mask[:, i].float() # (B,) - module_gene_idx = torch.as_tensor(self.module_sets[i], dtype=torch.long, device=preds.device) - p = preds.index_select(dim=1, index=module_gene_idx) # (B, G_i) - diff = (p - t) ** 2 - if self.loss_mode == "gene_mean": - total_loss = total_loss + (diff * module_mask.unsqueeze(-1)).sum() - total_count = total_count + module_mask.sum() * t.shape[1] - else: - module_mse = diff.mean(dim=1) # (B,) - total_loss = total_loss + (module_mse * module_mask).sum() - total_count = total_count + module_mask.sum() - return total_loss / total_count.clamp(min=1.0) - - assert isinstance(preds, list) - if len(preds) != len(targets): - raise RuntimeError(f"pred/target module count mismatch: {len(preds)} vs {len(targets)}") - total_loss = torch.tensor(0.0, device=mask.device, dtype=torch.float32) - total_count = torch.tensor(0.0, device=mask.device, dtype=torch.float32) - - for i, (p, t) in enumerate(zip(preds, targets)): - module_mask = mask[:, i].float() # (B,) - if self.recon_target_level == "module_mean_value": - # decoder直接输出module均值,target为该module内基因均值 - # p: (B,1), t_mean: (B,1) - t_mean = t.mean(dim=1, keepdim=True) - diff = (p - t_mean) ** 2 - total_loss = total_loss + (diff.squeeze(-1) * module_mask).sum() - total_count = total_count + module_mask.sum() - else: - diff = (p - t) ** 2 # (B, G_i) - if self.loss_mode == "gene_mean": - # 当前逻辑:masked 区域内每个基因等权 - total_loss = total_loss + (diff * module_mask.unsqueeze(-1)).sum() - total_count = total_count + module_mask.sum() * t.shape[1] - else: - # MAE风格:先在module内部取均值,再在masked module上做均值(每个module等权) - module_mse = diff.mean(dim=1) # (B,) - total_loss = total_loss + (module_mse * module_mask).sum() - total_count = total_count + module_mask.sum() - - if total_count.item() == 0: - return torch.tensor(0.0, device=mask.device, dtype=torch.float32) - return total_loss / total_count - - def forward( - self, cells: torch.Tensor, mask_ratio: Optional[float] = None, - return_encodings: bool = False, return_module_targets: bool = True, - ): - """ - 前向传播 - cells: (B, num_genes) - 基因表达值 - mask_ratio: Optional[float] - masking比例,如果为None则使用self.mask_ratio - return_encodings: bool - 如果为True,返回编码而不是loss(用于zero-shot任务) - Returns: - 如果return_encodings=True: - dict with keys: 'module_encodings', 'mask', 'ids_restore', 'module_targets' - 否则: - loss: Tensor - 重建损失 - preds: - - 默认模式: List[Tensor](每个module一个预测) - - decoder_gene_agg模式: Tensor(B, G)(基因级预测) - mask: Tensor - mask矩阵 - """ - latent, mask, ids_restore, module_targets = self.forward_encoder( - cells, - mask_ratio=mask_ratio, - # Reconstruction always needs targets; representation callers can - # opt out explicitly while preserving the original API default. - return_module_targets=(return_module_targets or not return_encodings), - ) - - if return_encodings: - # 返回编码用于zero-shot任务 - # drop-mask路径返回可见token编码;replace-mask路径返回全module编码。 - if self.use_cls: - seq_encodings = latent[:, 1:, :] - else: - seq_encodings = latent - - out = { - 'module_encodings': seq_encodings, - 'mask': mask, - 'ids_restore': ids_restore, - 'module_targets': module_targets - } - if self.use_cls: - out['cls_encoding'] = latent[:, 0, :] - return out - - if self.decoder_type == "performer": - preds = self.forward_decoder_performer(latent, ids_restore) - elif self.decoder_type == "cross_attn": - preds = self.forward_decoder_cross_attn(latent, mask, ids_restore) - elif self.decoder_type == "simple": - preds = self.forward_decoder_simple(latent, ids_restore) - elif self.decoder_type == "none": - preds = self.forward_predictor_from_encoder(latent) - else: - raise RuntimeError(f"Unsupported decoder_type at forward time: {self.decoder_type}") - recon_loss = self.compute_loss(preds, module_targets, mask) - cls_loss, _ = self.compute_cls_loss(latent, module_targets) - total_loss = recon_loss + self.cls_loss_weight * cls_loss - self.last_loss_breakdown = { - "recon_loss": float(recon_loss.detach().cpu()), - "cls_loss": float(cls_loss.detach().cpu()), - "total_loss": float(total_loss.detach().cpu()), - } - return total_loss, preds, mask - - def get_last_loss_breakdown(self) -> dict: - return dict(self.last_loss_breakdown) - - def load_state_dict(self, state_dict, strict: bool = True, assign: bool = False): - """Load current or pre-rename weights under the module-named interface.""" - return super().load_state_dict( - migrate_legacy_module_state_dict(state_dict), - strict=strict, - assign=assign, - ) - - def check_redraw_projections(self): - """检查并重绘projection矩阵""" - self.encoder.check_redraw_projections() - - def fix_projection_matrices_(self): - """固定projection矩阵""" - self.encoder.fix_projection_matrices_() +"""Module-based Performer encoder with MAE-style masking and configurable +reconstruction decoders or direct prediction heads.""" + +import sys +import os +parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +if parent_dir not in sys.path: + sys.path.insert(0, parent_dir) + +# The bundled Performer implementation lives under model_dependencies rather +# than at the repository root. Add that directory explicitly so every task +# entry point can import the model without requiring a caller-managed +# PYTHONPATH. +model_dependencies_dir = os.path.join(parent_dir, "model_dependencies") +if os.path.isdir(model_dependencies_dir) and model_dependencies_dir not in sys.path: + sys.path.insert(0, model_dependencies_dir) + +try: + from performer_pytorch_cont.ding_models import ( + Performer, + Always, + cast_tuple, + ) +except ImportError: + try: + from performer_pytorch_cont.performer_pytorch import ( + Performer, + Always, + cast_tuple, + ) + except ImportError: + from performer_pytorch.performer_pytorch import ( + Performer, + Always, + cast_tuple, + ) +import torch +from torch import nn +import torch.nn.functional as F +from typing import Callable, Dict, List, Optional, Tuple, Union +import numpy as np + + +def _migrate_legacy_module_name(value: str) -> str: + """Translate serialized pre-rename names without changing their semantics.""" + replacements = ( + ("PATHCH", "MODULE"), + ("Pathch", "Module"), + ("pathch", "module"), + ("PATCHES", "MODULES"), + ("Patches", "Modules"), + ("patches", "modules"), + ("PATCH", "MODULE"), + ("Patch", "Module"), + ("patch", "module"), + ) + for old, new in replacements: + value = value.replace(old, new) + return value + + +def migrate_legacy_module_config(config: Dict[str, object]) -> Dict[str, object]: + """Return a module-named copy of an older serialized configuration.""" + def translate(value): + if isinstance(value, dict): + migrated = {} + for key, item in value.items(): + new_key = _migrate_legacy_module_name(key) if isinstance(key, str) else key + if new_key in migrated: + raise ValueError( + f"Legacy configuration migration would duplicate key {new_key!r}" + ) + migrated[new_key] = translate(item) + return migrated + if isinstance(value, list): + return [translate(item) for item in value] + if isinstance(value, tuple): + return tuple(translate(item) for item in value) + if isinstance(value, str): + return _migrate_legacy_module_name(value) + return value + + return translate(dict(config)) + + +def migrate_legacy_module_state_dict(state_dict): + """Translate old state-dict key names before PyTorch performs strict loading.""" + try: + migrated = state_dict.__class__() + except TypeError: + migrated = {} + + for key, value in state_dict.items(): + new_key = _migrate_legacy_module_name(key) if isinstance(key, str) else key + if new_key in migrated: + # If both spellings are supplied, prefer the explicit new spelling. + if key == new_key: + migrated[new_key] = value + continue + migrated[new_key] = value + + if hasattr(state_dict, "_metadata"): + migrated._metadata = { + _migrate_legacy_module_name(key): value + for key, value in state_dict._metadata.items() + } + return migrated + + +def normalize_fusion_method(fusion_method: str) -> str: + if fusion_method == "expression_only_mlp": + return "add" + if fusion_method in {"concatenate", "concat_v2"}: + raise ValueError( + f"fusion_method={fusion_method!r} has been removed. " + "Use one of: 'multiply', 'add', 'gated_residual'." + ) + if fusion_method not in {"multiply", "add", "gated_residual"}: + raise ValueError( + f"Unsupported fusion_method: {fusion_method}. " + "Use one of: 'multiply', 'add', 'gated_residual'." + ) + return fusion_method + + +REMOVED_PRETRAIN_CONFIG_KEYS = ( + "preprocess_gene2vec", + "extra_gene_mode", + "extra_gene_num", + "extra_gene_rank_path", + "extra_gene_module_size", + "extra_gene_module_strategy", + "module_embed_chunk_padded_gene_budget", + "cross_attn_pure_memory", +) + + +def sanitize_performer_module_config( + config: Dict[str, object], + *, + source: str = "config", + log_fn: Optional[Callable[[str], None]] = None, +) -> Dict[str, object]: + cleaned = migrate_legacy_module_config(config) + legacy_mode = str(cleaned.get("extra_gene_mode", "none")).lower() + legacy_num = int(cleaned.get("extra_gene_num", 0) or 0) + if legacy_mode != "none" or legacy_num > 0: + raise ValueError( + f"{source} requests removed extra-gene support " + f"(extra_gene_mode={legacy_mode}, extra_gene_num={legacy_num}). " + "Please use a checkpoint trained without extra genes." + ) + + removed = [key for key in REMOVED_PRETRAIN_CONFIG_KEYS if key in cleaned] + for key in removed: + cleaned.pop(key, None) + + if removed and log_fn is not None: + log_fn(f"{source}: dropped deprecated config keys: {', '.join(sorted(removed))}") + + return cleaned + +# ModuleEmbedding is implemented here to support three expression-fusion methods. +# from sclong_module.models_module import ModuleToGeneDecoder + + +class CrossAttentionDecoderBlock(nn.Module): + """ + Decoder block with masked-query -> visible-memory cross-attention only. + """ + + def __init__( + self, + dim: int, + heads: int, + ff_mult: int = 4, + attn_dropout: float = 0.0, + ff_dropout: float = 0.0, + ) -> None: + super().__init__() + if heads <= 0: + raise ValueError(f"CrossAttentionDecoderBlock requires heads > 0, got {heads}") + if dim % heads != 0: + raise ValueError( + f"CrossAttentionDecoderBlock requires dim divisible by heads, got dim={dim}, heads={heads}" + ) + + self.query_norm = nn.LayerNorm(dim) + self.memory_norm = nn.LayerNorm(dim) + self.cross_attn = nn.MultiheadAttention( + embed_dim=dim, + num_heads=heads, + dropout=attn_dropout, + batch_first=True, + ) + hidden_dim = int(dim * ff_mult) + self.ff_norm = nn.LayerNorm(dim) + self.ff = nn.Sequential( + nn.Linear(dim, hidden_dim), + nn.GELU(), + nn.Dropout(ff_dropout), + nn.Linear(hidden_dim, dim), + nn.Dropout(ff_dropout), + ) + + def forward(self, query: torch.Tensor, memory: torch.Tensor) -> torch.Tensor: + if query.shape[1] == 0: + return query + + if memory.shape[1] > 0: + attn_out, _ = self.cross_attn( + self.query_norm(query), + self.memory_norm(memory), + self.memory_norm(memory), + need_weights=False, + ) + query = query + attn_out + + query = query + self.ff(self.ff_norm(query)) + return query + + +class ModuleEmbedding(nn.Module): + """Gene embedding, MLP, and attention pooling produce a module token. + + Expression fusion supports three methods: + - multiply: multiply each expression value by its gene embedding. + - add: transform expression with a separate MLP and add the gene embedding. + - gated_residual: inject transformed expression through a learned gate. + + use_gene2vec=True initializes from supplied pretrained embeddings; + False uses random initialization for ablation experiments.""" + + @staticmethod + def _build_adaptive_buckets( + modules_by_length: Dict[int, List[int]], + bucket_count: int, + ) -> Dict[int, List[int]]: + """Minimize padded gene positions for a fixed number of buckets.""" + length_groups = sorted(modules_by_length.items()) + group_count = len(length_groups) + bucket_count = min(bucket_count, group_count) + if bucket_count <= 0: + raise ValueError("MODULE_EMBED_BUCKET_COUNT must be positive") + + prefix_counts = [0] + for _, module_ids in length_groups: + prefix_counts.append(prefix_counts[-1] + len(module_ids)) + + # dp[b][j] is the least padded element count for the first j distinct + # lengths split into b contiguous buckets. Contiguous length ranges + # preserve a single static padded shape per bucket. + infinity = float("inf") + previous = [infinity] * (group_count + 1) + previous[0] = 0 + split_points: List[List[int]] = [ + [-1] * (group_count + 1) for _ in range(bucket_count + 1) + ] + for bucket_idx in range(1, bucket_count + 1): + current = [infinity] * (group_count + 1) + for end in range(bucket_idx, group_count + 1): + bucket_length = length_groups[end - 1][0] + best_cost = infinity + best_start = -1 + for start in range(bucket_idx - 1, end): + cost = previous[start] + bucket_length * ( + prefix_counts[end] - prefix_counts[start] + ) + if cost < best_cost: + best_cost = cost + best_start = start + current[end] = best_cost + split_points[bucket_idx][end] = best_start + previous = current + + ranges = [] + end = group_count + for bucket_idx in range(bucket_count, 0, -1): + start = split_points[bucket_idx][end] + ranges.append((start, end)) + end = start + ranges.reverse() + + buckets: Dict[int, List[int]] = {} + for start, end in ranges: + bucket_length = length_groups[end - 1][0] + buckets[bucket_length] = [ + module_idx + for _, module_ids in length_groups[start:end] + for module_idx in module_ids + ] + return buckets + + def __init__( + self, + module_sets: List[List[int]], + num_genes: int, + gene_embed_dim: int, + embed_dim: int, + gene2vec_weight: Optional[torch.Tensor] = None, + fusion_method: str = "multiply", # "multiply" / "add" / "gated_residual" + use_gene2vec: bool = True, # Use pretrained gene2vec; False selects random initialization. + ): + super().__init__() + self.num_modules = len(module_sets) + self.module_index_names: List[str] = [] + modules_by_length: Dict[int, List[int]] = {} + for module_idx, module_genes in enumerate(module_sets): + buffer_name = f"_module_gene_idx_{module_idx}" + # Module indices are static metadata. Non-persistent buffers follow + # module.to(device) without changing checkpoint state_dict keys. + self.register_buffer( + buffer_name, + torch.tensor(module_genes, dtype=torch.long), + persistent=False, + ) + self.module_index_names.append(buffer_name) + modules_by_length.setdefault(len(module_genes), []).append(module_idx) + + # Group modules with the same gene count so their identical operations + # can be issued as a single batched operation. All buffers are + # non-persistent: checkpoint keys remain byte-for-byte compatible. + self.module_batch_mode = os.environ.get("MODULE_EMBED_BATCH_MODE", "adaptive").lower() + if self.module_batch_mode not in {"legacy", "exact", "bucket", "adaptive"}: + raise ValueError( + "MODULE_EMBED_BATCH_MODE must be one of: " + "'legacy', 'exact', 'bucket', 'adaptive'; " + f"got {self.module_batch_mode!r}" + ) + self.module_groups: List[Tuple[int, Tuple[int, ...], int, int]] = [] + grouped_gene_indices: List[int] = [] + grouped_module_order: List[int] = [] + group_offset = 0 + for module_length in sorted(modules_by_length): + module_ids = tuple(modules_by_length[module_length]) + group_gene_indices = [ + gene_idx + for module_idx in module_ids + for gene_idx in module_sets[module_idx] + ] + group_end = group_offset + len(group_gene_indices) + self.module_groups.append( + (module_length, module_ids, group_offset, group_end) + ) + grouped_gene_indices.extend(group_gene_indices) + grouped_module_order.extend(module_ids) + group_offset = group_end + + restore_order = [0] * self.num_modules + for grouped_pos, module_idx in enumerate(grouped_module_order): + restore_order[module_idx] = grouped_pos + self.register_buffer( + "_grouped_gene_idx", + torch.tensor(grouped_gene_indices, dtype=torch.long), + persistent=False, + ) + self.register_buffer( + "_module_group_restore_idx", + torch.tensor(restore_order, dtype=torch.long), + persistent=False, + ) + + if self.module_batch_mode == "adaptive": + bucket_count = int(os.environ.get("MODULE_EMBED_BUCKET_COUNT", "16")) + modules_by_bucket = self._build_adaptive_buckets( + modules_by_length, + bucket_count, + ) + else: + modules_by_bucket: Dict[int, List[int]] = {} + for module_idx, module_genes in enumerate(module_sets): + bucket_length = 1 << (len(module_genes) - 1).bit_length() + modules_by_bucket.setdefault(bucket_length, []).append(module_idx) + self.module_buckets: List[ + Tuple[int, Tuple[int, ...], Tuple[int, ...], int, int, str] + ] = [] + bucketed_gene_indices: List[int] = [] + bucketed_module_order: List[int] = [] + bucket_offset = 0 + for bucket_idx, bucket_length in enumerate(sorted(modules_by_bucket)): + module_ids = tuple(modules_by_bucket[bucket_length]) + module_lengths = tuple(len(module_sets[idx]) for idx in module_ids) + group_gene_indices: List[int] = [] + valid_mask = torch.zeros(len(module_ids), bucket_length, dtype=torch.bool) + for group_pos, module_idx in enumerate(module_ids): + genes = module_sets[module_idx] + pad_count = bucket_length - len(genes) + group_gene_indices.extend(genes) + group_gene_indices.extend([0] * pad_count) + valid_mask[group_pos, :len(genes)] = True + group_end = bucket_offset + len(group_gene_indices) + mask_name = f"_module_bucket_mask_{bucket_idx}" + self.register_buffer(mask_name, valid_mask, persistent=False) + self.module_buckets.append( + ( + bucket_length, + module_ids, + module_lengths, + bucket_offset, + group_end, + mask_name, + ) + ) + bucketed_gene_indices.extend(group_gene_indices) + bucketed_module_order.extend(module_ids) + bucket_offset = group_end + + bucket_restore_order = [0] * self.num_modules + for grouped_pos, module_idx in enumerate(bucketed_module_order): + bucket_restore_order[module_idx] = grouped_pos + self.register_buffer( + "_bucketed_gene_idx", + torch.tensor(bucketed_gene_indices, dtype=torch.long), + persistent=False, + ) + self.register_buffer( + "_module_bucket_restore_idx", + torch.tensor(bucket_restore_order, dtype=torch.long), + persistent=False, + ) + # Deduplicate shared pointwise transformations, preserving module-local + # membership, order, multiplicity and attention normalization. + # Static maps add no checkpoint keys or trainable parameters. + unique_genes = sorted({gene for module in module_sets for gene in module}) + unique_position = {gene: pos for pos, gene in enumerate(unique_genes)} + bucket_unique_positions = [ + unique_position.get(gene, 0) for gene in bucketed_gene_indices + ] + self.register_buffer( + "_unique_gene_idx", torch.tensor(unique_genes, dtype=torch.long), + persistent=False, + ) + self.register_buffer( + "_bucket_unique_idx", + torch.tensor(bucket_unique_positions, dtype=torch.long), + persistent=False, + ) + self.gene_dedup_enabled = os.environ.get("MODULE_GENE_DEDUP", "1") == "1" + # The unlimited candidate was slower at batches 16/64/256 on H100. + # Default to the measured beneficial batch=1 path; 0 enables an explicit + # unlimited research comparison without changing checkpoint structure. + self.gene_dedup_max_batch_size = int( + os.environ.get("MODULE_GENE_DEDUP_MAX_BATCH_SIZE", "1") + ) + if self.gene_dedup_max_batch_size < 0: + raise ValueError("MODULE_GENE_DEDUP_MAX_BATCH_SIZE must be nonnegative") + self.fusion_method = normalize_fusion_method(fusion_method) + self.use_gene2vec = use_gene2vec + self.gene_embed_dim = gene_embed_dim + self.embed_dim = embed_dim + + # Select pretrained or random initialization according to use_gene2vec. + if use_gene2vec and gene2vec_weight is not None: + self.gene_embed = nn.Embedding.from_pretrained(gene2vec_weight, freeze=False) + if self.gene_embed.weight.shape[0] < num_genes: + pad = torch.zeros(num_genes - self.gene_embed.weight.shape[0], self.gene_embed.weight.shape[1]) + self.gene_embed.weight.data = torch.cat([self.gene_embed.weight.data, pad], dim=0) + else: + # Random initialization for ablations without the gene2vec prior. + self.gene_embed = nn.Embedding(num_genes, gene_embed_dim) + # Use Xavier initialization, consistent with the other layers. + nn.init.xavier_uniform_(self.gene_embed.weight) + + # Expression-fusion layers. + # All fusion methods use a two-layer projection: Dg -> E -> E. + self.expr_mlp = None + self.gated_expr_mlp = None + self.gated_residual_gate = None + + if self.fusion_method == "add": + # add: transform expression with a separate MLP, then add gene2vec. + hidden_dim = max(50, gene_embed_dim // 2) + self.expr_mlp = nn.Sequential( + nn.Linear(1, hidden_dim), + nn.GELU(), + nn.Linear(hidden_dim, gene_embed_dim), + ) + elif self.fusion_method == "gated_residual": + # gated_residual: inject expression as a gated residual into the gene2vec representation. + hidden_dim = max(64, gene_embed_dim // 2) + self.gated_expr_mlp = nn.Sequential( + nn.Linear(1, hidden_dim), + nn.GELU(), + nn.Linear(hidden_dim, gene_embed_dim), + nn.LayerNorm(gene_embed_dim), + ) + self.gated_residual_gate = nn.Sequential( + nn.Linear(gene_embed_dim * 2, gene_embed_dim), + nn.GELU(), + nn.Linear(gene_embed_dim, gene_embed_dim), + nn.Sigmoid(), + ) + # multiply does not require an additional expression MLP. + + # Shared projection: Dg -> E -> E. + self.proj = nn.Sequential( + nn.Linear(gene_embed_dim, embed_dim), + nn.GELU(), + nn.Linear(embed_dim, embed_dim), + ) + self.attn_score = nn.Linear(embed_dim, 1) + self.pool_affine_enabled = os.environ.get("MODULE_POOL_AFFINE", "1") == "1" + # Paired H100 screening found this path helps batches 16/64/256 but + # regresses batch=1 against the dedup fast path. Preserve that fast path. + self.pool_affine_min_batch_size = int(os.environ.get("MODULE_POOL_AFFINE_MIN_BATCH_SIZE", "2")) + if self.pool_affine_min_batch_size < 1: + raise ValueError("MODULE_POOL_AFFINE_MIN_BATCH_SIZE must be positive") + # Derived model weights only; no cell-dependent features are cached. + # These buffers add no parameters or checkpoint entries. + self.register_buffer("_pool_affine_score_weight", torch.empty(0), persistent=False) + self.register_buffer("_pool_affine_score_bias", torch.empty(0), persistent=False) + self._pool_affine_cache_signature = None + + @property + def module_sets(self) -> List[torch.Tensor]: + """Expose module indices for existing interpretability callers.""" + return [getattr(self, name) for name in self.module_index_names] + + def _pool_affine_signature(self): + tensors = (self.proj[2].weight, self.proj[2].bias, + self.attn_score.weight, self.attn_score.bias) + return tuple((id(t), t.data_ptr(), t._version, t.device, t.dtype) for t in tensors) + + @torch.no_grad() + def prepare_pool_affine(self): + """Compose final affine and score AFTER checkpoint/device/dtype placement. + + Call again after changing weights, dtype or device. Forward rejects a + stale cache rather than silently using outdated model-derived weights. + """ + weight = self.proj[2].weight + compose_dtype = torch.float64 if weight.dtype == torch.float64 else torch.float32 + with torch.autocast(device_type=weight.device.type, enabled=False): + score_weight = self.attn_score.weight.to(dtype=compose_dtype) + self._pool_affine_score_weight = ( + score_weight @ weight.to(dtype=compose_dtype) + ).to(dtype=weight.dtype) + self._pool_affine_score_bias = ( + score_weight @ self.proj[2].bias.to(dtype=compose_dtype) + + self.attn_score.bias.to(dtype=compose_dtype) + ).to(dtype=weight.dtype) + self._pool_affine_cache_signature = self._pool_affine_signature() + + def _forward_pool_affine( + self, + expr: torch.Tensor, + return_module_targets: bool = True, + ) -> Tuple[torch.Tensor, List[torch.Tensor]]: + """Pool first-layer activations before the shared final affine. + + In real arithmetic, score(W2*h+b2)=(score_weight*W2)*h+const, + and sum(alpha*(W2*h+b2))=W2*sum(alpha*h)+b2*sum(alpha). + BF16 reassociation is validated separately; it is not bitwise equivalent. + """ + if self._pool_affine_cache_signature != self._pool_affine_signature(): + raise RuntimeError("Pool-affine cache is stale; call prepare_pool_affine() after loading/placing weights") + batch_size = expr.shape[0] + flat_expr = torch.index_select(expr, 1, self._bucketed_gene_idx) + flat_gene_emb = self.gene_embed(self._bucketed_gene_idx) + pooled_hidden = [] + attention_sums = [] + module_targets: List[Optional[torch.Tensor]] = ( + [None] * self.num_modules if return_module_targets else [] + ) + for ( + bucket_length, module_ids, module_lengths, start, end, mask_name + ) in self.module_buckets: + group_size = len(module_ids) + expr_group = flat_expr[:, start:end].reshape(batch_size, group_size, bucket_length) + gene_emb = flat_gene_emb[start:end].reshape(group_size, bucket_length, self.gene_embed_dim) + gene_emb = gene_emb.unsqueeze(0).expand(batch_size, -1, -1, -1) + fused = self._fuse_expr_and_gene_emb(expr_group, gene_emb) + hidden = self.proj[1](self.proj[0](fused)) + del fused + scores = F.linear(hidden, self._pool_affine_score_weight, self._pool_affine_score_bias).squeeze(-1) + valid_mask = getattr(self, mask_name) + scores = scores.masked_fill(~valid_mask.unsqueeze(0), float("-inf")) + attn = torch.softmax(scores, dim=2) + pooled_hidden.append(self._weighted_pool(hidden, attn)) + # Quantized softmax weights need not sum exactly to one. Preserve + # their sum in wider accumulation for the final bias contribution. + sum_dtype = torch.float64 if hidden.dtype == torch.float64 else torch.float32 + attention_sums.append(attn.sum(dim=2, dtype=sum_dtype)) + del hidden, scores, attn + if return_module_targets: + for group_pos, (module_idx, module_length) in enumerate(zip(module_ids, module_lengths)): + module_targets[module_idx] = expr_group[:, group_pos, :module_length] + pooled = torch.cat(pooled_hidden, dim=1) + sums = torch.cat(attention_sums, dim=1) + # One final affine over the module tokens, not over every gene position. + linear = F.linear(pooled, self.proj[2].weight, None) + with torch.autocast(device_type=expr.device.type, enabled=False): + wider = linear.to(dtype=sums.dtype) + sums.unsqueeze(-1) * self.proj[2].bias.to(dtype=sums.dtype) + tokens = wider.to(dtype=linear.dtype) + tokens = torch.index_select(tokens, 1, self._module_bucket_restore_idx) + return tokens, module_targets # type: ignore[return-value] + + def _fuse_expr_and_gene_emb(self, expr_slice: torch.Tensor, gene_emb: torch.Tensor) -> torch.Tensor: + """Fuse expression and gene embeddings into gene-token representations. + + Leading dimensions may vary; the final expression axis indexes genes. + expr_slice: (..., G), gene_emb: (..., G, Dg) -> (..., G, Dg).""" + if self.fusion_method == "add": + expr_emb = self.expr_mlp(expr_slice.unsqueeze(-1)) # (B, G, Dg) + return expr_emb + gene_emb + if self.fusion_method == "gated_residual": + expr_emb = self.gated_expr_mlp(expr_slice.unsqueeze(-1)) # (B, G, Dg) + gate = self.gated_residual_gate(torch.cat([expr_emb, gene_emb], dim=-1)) # (B, G, Dg) + return gene_emb + gate * expr_emb + # Default multiply fusion: expression value times gene2vec. + return expr_slice.unsqueeze(-1) * gene_emb # (B, G, Dg) + + @staticmethod + def _weighted_pool(gene_tok: torch.Tensor, attn: torch.Tensor) -> torch.Tensor: + """Pool without materializing the full attention-weighted tensor.""" + implementation = os.environ.get("MODULE_POOL_IMPL", "bmm") + if implementation == "bmm": + length, width = gene_tok.shape[-2:] + pooled = torch.bmm( + attn.reshape(-1, 1, length), + gene_tok.reshape(-1, length, width), + ) + return pooled.reshape(*gene_tok.shape[:-2], width) + if implementation != "mulsum": + raise ValueError(f"Unsupported MODULE_POOL_IMPL={implementation}") + return torch.sum(gene_tok * attn.unsqueeze(-1), dim=-2) + + def _forward_legacy( + self, + expr: torch.Tensor, + return_module_targets: bool = True, + ) -> Tuple[torch.Tensor, List[torch.Tensor]]: + """Reference implementation that processes one module at a time. + + expr: (batch, num_genes). + Returns module_tokens (batch, num_modules, embed_dim) and a list of + targets, each shaped (batch, genes_in_module).""" + batch_size, _ = expr.shape + module_tokens = [] + module_targets: List[torch.Tensor] = [] + for index_name in self.module_index_names: + gene_idx = getattr(self, index_name) + expr_slice = torch.index_select(expr, dim=1, index=gene_idx) + if return_module_targets: + module_targets.append(expr_slice) + + gene_emb = self.gene_embed(gene_idx) + gene_emb = gene_emb.unsqueeze(0).expand(batch_size, -1, -1) + gene_tok = self._fuse_expr_and_gene_emb(expr_slice, gene_emb) + gene_tok = self.proj(gene_tok) + + attn = self.attn_score(gene_tok).squeeze(-1) + attn = torch.softmax(attn, dim=1) + pooled = self._weighted_pool(gene_tok, attn) + module_tokens.append(pooled) + + return torch.stack(module_tokens, dim=1), module_targets + + def _forward_grouped_exact( + self, + expr: torch.Tensor, + return_module_targets: bool = True, + ) -> Tuple[torch.Tensor, List[torch.Tensor]]: + """Batch modules of identical length without padding or approximation.""" + batch_size, _ = expr.shape + flat_expr = torch.index_select(expr, dim=1, index=self._grouped_gene_idx) + flat_gene_emb = self.gene_embed(self._grouped_gene_idx) + grouped_tokens = [] + module_targets: List[Optional[torch.Tensor]] = ( + [None] * self.num_modules if return_module_targets else [] + ) + + for module_length, module_ids, start, end in self.module_groups: + group_size = len(module_ids) + expr_group = flat_expr[:, start:end].reshape( + batch_size, group_size, module_length + ) + gene_emb = flat_gene_emb[start:end].reshape( + group_size, module_length, self.gene_embed_dim + ) + gene_emb = gene_emb.unsqueeze(0).expand(batch_size, -1, -1, -1) + gene_tok = self._fuse_expr_and_gene_emb(expr_group, gene_emb) + gene_tok = self.proj(gene_tok) + + attn = self.attn_score(gene_tok).squeeze(-1) + attn = torch.softmax(attn, dim=2) + pooled = self._weighted_pool(gene_tok, attn) + grouped_tokens.append(pooled) + + if return_module_targets: + for group_pos, module_idx in enumerate(module_ids): + module_targets[module_idx] = expr_group[:, group_pos, :] + + tokens = torch.cat(grouped_tokens, dim=1) + tokens = torch.index_select( + tokens, dim=1, index=self._module_group_restore_idx + ) + # Every module is assigned exactly once when groups are constructed. + return tokens, module_targets # type: ignore[return-value] + + def _forward_bucketed( + self, + expr: torch.Tensor, + return_module_targets: bool = True, + ) -> Tuple[torch.Tensor, List[torch.Tensor]]: + """Batch modules into fixed padded lengths and mask padded positions.""" + batch_size, _ = expr.shape + flat_expr = torch.index_select(expr, dim=1, index=self._bucketed_gene_idx) + flat_gene_emb = self.gene_embed(self._bucketed_gene_idx) + bucket_tokens = [] + module_targets: List[Optional[torch.Tensor]] = ( + [None] * self.num_modules if return_module_targets else [] + ) + + for ( + bucket_length, + module_ids, + module_lengths, + start, + end, + mask_name, + ) in self.module_buckets: + group_size = len(module_ids) + expr_group = flat_expr[:, start:end].reshape( + batch_size, group_size, bucket_length + ) + gene_emb = flat_gene_emb[start:end].reshape( + group_size, bucket_length, self.gene_embed_dim + ) + gene_emb = gene_emb.unsqueeze(0).expand(batch_size, -1, -1, -1) + gene_tok = self._fuse_expr_and_gene_emb(expr_group, gene_emb) + gene_tok = self.proj(gene_tok) + + valid_mask = getattr(self, mask_name) + attn = self.attn_score(gene_tok).squeeze(-1) + attn = attn.masked_fill(~valid_mask.unsqueeze(0), float("-inf")) + attn = torch.softmax(attn, dim=2) + pooled = self._weighted_pool(gene_tok, attn) + bucket_tokens.append(pooled) + + if return_module_targets: + for group_pos, (module_idx, module_length) in enumerate( + zip(module_ids, module_lengths) + ): + module_targets[module_idx] = expr_group[:, group_pos, :module_length] + + tokens = torch.cat(bucket_tokens, dim=1) + tokens = torch.index_select( + tokens, dim=1, index=self._module_bucket_restore_idx + ) + return tokens, module_targets # type: ignore[return-value] + + def _forward_unique_genes( + self, + expr: torch.Tensor, + return_module_targets: bool = True, + ) -> Tuple[torch.Tensor, List[torch.Tensor]]: + """Reuse gene features within this inference forward; pool per module. + + Gather one bucket at a time rather than the full repeated-gene tensor. + """ + batch_size = expr.shape[0] + unique_expr = torch.index_select(expr, 1, self._unique_gene_idx) + unique_emb = self.gene_embed(self._unique_gene_idx) + unique_emb = unique_emb.unsqueeze(0).expand(batch_size, -1, -1) + fused = self._fuse_expr_and_gene_emb(unique_expr, unique_emb) + unique_tokens = self.proj(fused) + del fused, unique_emb + unique_scores = self.attn_score(unique_tokens).squeeze(-1) + bucket_tokens = [] + module_targets: List[Optional[torch.Tensor]] = ( + [None] * self.num_modules if return_module_targets else [] + ) + for ( + bucket_length, module_ids, module_lengths, start, end, mask_name + ) in self.module_buckets: + group_size = len(module_ids) + indices = self._bucket_unique_idx[start:end] + scores = torch.index_select(unique_scores, 1, indices).reshape( + batch_size, group_size, bucket_length + ) + valid_mask = getattr(self, mask_name) + scores = scores.masked_fill(~valid_mask.unsqueeze(0), float("-inf")) + attn = torch.softmax(scores, dim=2) + gene_tok = torch.index_select(unique_tokens, 1, indices).reshape( + batch_size, group_size, bucket_length, self.embed_dim + ) + bucket_tokens.append(self._weighted_pool(gene_tok, attn)) + del gene_tok, attn, scores + if return_module_targets: + expr_group = torch.index_select(unique_expr, 1, indices).reshape( + batch_size, group_size, bucket_length + ) + for group_pos, (module_idx, module_length) in enumerate( + zip(module_ids, module_lengths) + ): + module_targets[module_idx] = expr_group[:, group_pos, :module_length] + tokens = torch.cat(bucket_tokens, dim=1) + tokens = torch.index_select(tokens, 1, self._module_bucket_restore_idx) + return tokens, module_targets # type: ignore[return-value] + + def forward( + self, + expr: torch.Tensor, + return_module_targets: bool = True, + ) -> Tuple[torch.Tensor, List[torch.Tensor]]: + # Keep original training/gradient behavior and gradient reduction order. + if ( + self.pool_affine_enabled and not self.training and not torch.is_grad_enabled() + and expr.shape[0] >= self.pool_affine_min_batch_size + ): + return self._forward_pool_affine(expr, return_module_targets) + if ( + self.gene_dedup_enabled and not self.training and not torch.is_grad_enabled() + and (self.gene_dedup_max_batch_size == 0 + or expr.shape[0] <= self.gene_dedup_max_batch_size) + ): + return self._forward_unique_genes(expr, return_module_targets) + if self.module_batch_mode == "legacy": + return self._forward_legacy(expr, return_module_targets) + if self.module_batch_mode == "exact": + return self._forward_grouped_exact(expr, return_module_targets) + return self._forward_bucketed(expr, return_module_targets) + + +class PerformerEncoderOnly_Module(nn.Module): + """Performer encoder for module inputs with configurable masking and decoding. + + use_gene2vec=True initializes from supplied pretrained gene embeddings; + False uses random gene embeddings for ablation experiments.""" + def __init__( + self, + module_sets: List[List[int]], # Module definitions. + num_genes: int, # Total number of genes. + max_seq_len, # Maximum sequence length (number of modules). + base_dim = 200, # Base embedding dimension shared by all tokens. + enc_depth = 24, # Encoder depth. + enc_heads = 16, # Number of encoder attention heads. + enc_dim_head = 64, # Dimension of each encoder attention head. + mask_ratio: float = 0.75, # Fraction of modules to mask. + # ==================== Performer parameters ==================== + local_attn_heads = 0,# Number of local-attention heads. + local_window_size = 256,# Local-attention window size. + causal = False, # Enable causal (autoregressive) attention. + ff_mult = 4,# Feed-forward expansion factor. + nb_features = None,# Number of Performer random features. + feature_redraw_interval = 1000,# Steps between random-feature redraws. + reversible = False,# Use reversible layers to reduce memory use. + ff_chunks = 1,# Number of feed-forward chunks. + ff_glu = False, # Use gated linear units in the feed-forward network. + # ==================== Dropout parameters ==================== + emb_dropout = 0., + ff_dropout = 0., + attn_dropout = 0., + # ==================== Advanced Performer parameters ==================== + generalized_attention = False, + kernel_fn = nn.ReLU(), + use_scalenorm = False, + use_rezero = False, + cross_attend = False, + no_projection = False, + # ==================== Position embedding parameters ==================== + g2v_position_emb = False, # Use learned module position embeddings. + auto_check_redraw = True,# Automatically check and redraw projection matrices. + qkv_bias = False, # Include bias in query, key, and value projections. + # ==================== Gene2vec parameters ==================== + gene2vec_file = os.path.join(model_dependencies_dir, 'selected_gene2vec_27k.npy'), + gene_embed_dim: int = 200, # Gene embedding dimension used by the module embedder. + fusion_method: str = "multiply", # "multiply" / "add" / "gated_residual" + use_cls_token: bool = False, # Enable the optional CLS token. + cls_loss_weight: float = 0.0, # CLS auxiliary loss weight; positive values train CLS to predict static module means. + cls_head_hidden_dim: int = 0, # CLS-head hidden dimension; zero selects base_dim. + use_gene2vec: bool = True, # Use pretrained gene2vec; False selects random initialization for ablations. + loss_mode: str = "gene_mean", # gene_mean weights masked gene occurrences equally; module_mean weights modules equally. + recon_target_level: str = "gene", # gene reconstructs each gene; module_mean_value predicts one mean per module. + pretrain_arch: str = "mae", # "mae" / "encoder_only" / "encdec" + mask_style: str = "drop", # "drop" / "replace" + mask_token_mode: str = "learned", # Only learned mask tokens are supported. + pos_embed_mode: str = "module_id", # Only learned module-ID embeddings are supported. + decoder_type: str = "simple", # "none" / "simple" / "performer" / "cross_attn" + decoder_gene_agg: bool = False, # Aggregate module features into gene features before reconstruction. + dec_embed_dim: int = 512, + dec_depth: int = 4, + dec_heads: int = 8, + dec_dim_head: int = 64, + dec_ff_mult: int = 4, + dec_ff_dropout: float = 0.0, + dec_attn_dropout: float = 0.0, + cross_attn_pure_memory: Optional[bool] = None, + # ==================== Fine-tuning enhancements ==================== + use_family_emb: bool = False, + module_family_ids: Optional[List[int]] = None, + use_module_gate: bool = False, + ): + super().__init__() + local_attn_heads = cast_tuple(local_attn_heads) + self.max_seq_len = max_seq_len + self.base_dim = base_dim + self.num_genes = num_genes + self.module_sets = module_sets + self.num_modules = len(module_sets) + self.fusion_method = normalize_fusion_method(fusion_method) + self.mask_ratio = mask_ratio + self.zero_mask_identity_fastpath = ( + os.environ.get("ZERO_MASK_IDENTITY_FASTPATH", "1") == "1" + ) + self.use_cls = use_cls_token + self.cls_loss_weight = float(cls_loss_weight) if use_cls_token else 0.0 + self.cls_head_hidden_dim = int(cls_head_hidden_dim) + self.use_gene2vec = use_gene2vec + self.loss_mode = loss_mode + self.recon_target_level = recon_target_level + self.pretrain_arch = str(pretrain_arch).lower() + self.mask_style = str(mask_style).lower() + self.mask_token_mode = str(mask_token_mode).lower() + self.pos_embed_mode = str(pos_embed_mode).lower() + self.decoder_type = decoder_type + self.decoder_gene_agg = decoder_gene_agg + self.dec_embed_dim = dec_embed_dim + # Legacy knob retained only for backward-compatible kwargs/config loading. + # Current pretraining always uses pure visible memory in the cross-attn decoder. + self.cross_attn_pure_memory = True + self.base_module_gene_counts = [len(p) for p in module_sets] + self.module_gene_counts = list(self.base_module_gene_counts) + self.total_module_count = len(self.module_gene_counts) + if self.loss_mode not in {"gene_mean", "module_mean"}: + raise ValueError(f"Unsupported loss_mode: {self.loss_mode}. Use 'gene_mean' or 'module_mean'.") + if self.cls_loss_weight < 0: + raise ValueError(f"cls_loss_weight must be >= 0, got {self.cls_loss_weight}") + if self.cls_head_hidden_dim < 0: + raise ValueError(f"cls_head_hidden_dim must be >= 0, got {self.cls_head_hidden_dim}") + if self.recon_target_level not in {"gene", "module_mean_value"}: + raise ValueError( + f"Unsupported recon_target_level: {self.recon_target_level}. " + "Use 'gene' or 'module_mean_value'." + ) + if self.pretrain_arch not in {"mae", "encoder_only", "encdec"}: + raise ValueError( + f"Unsupported pretrain_arch: {self.pretrain_arch}. " + "Use 'mae', 'encoder_only' or 'encdec'." + ) + if self.mask_style not in {"drop", "replace"}: + raise ValueError( + f"Unsupported mask_style: {self.mask_style}. Use 'drop' or 'replace'." + ) + if self.mask_token_mode not in {"learned"}: + raise ValueError( + f"Unsupported mask_token_mode: {self.mask_token_mode}. " + "Only 'learned' is currently supported." + ) + if self.pos_embed_mode not in {"module_id"}: + raise ValueError( + f"Unsupported pos_embed_mode: {self.pos_embed_mode}. " + "Only 'module_id' is currently supported." + ) + if self.decoder_type not in {"none", "simple", "performer", "cross_attn"}: + raise ValueError( + f"Unsupported decoder_type: {self.decoder_type}. " + "Use 'none', 'simple', 'performer' or 'cross_attn'." + ) + if self.pretrain_arch == "mae": + if self.mask_style != "drop": + raise ValueError( + "pretrain_arch='mae' is fixed to mask_style='drop'. " + "Use pretrain_arch='encoder_only' or 'encdec' for mask_style='replace'." + ) + if self.decoder_type == "none": + raise ValueError("pretrain_arch='mae' requires decoder_type != 'none'.") + elif self.pretrain_arch == "encoder_only": + if self.mask_style != "replace": + raise ValueError("pretrain_arch='encoder_only' requires mask_style='replace'.") + if self.decoder_type != "none": + raise ValueError("pretrain_arch='encoder_only' requires decoder_type='none'.") + else: + if self.mask_style != "replace": + raise ValueError("pretrain_arch='encdec' requires mask_style='replace'.") + if self.decoder_type not in {"performer", "cross_attn"}: + raise ValueError( + "pretrain_arch='encdec' requires decoder_type in {'performer', 'cross_attn'}." + ) + if self.decoder_gene_agg and self.recon_target_level != "gene": + raise ValueError("decoder_gene_agg=True only supports recon_target_level='gene'.") + if self.dec_embed_dim <= 0: + raise ValueError(f"dec_embed_dim must be positive, got {self.dec_embed_dim}") + + # Load gene2vec for module embedding when use_gene2vec is enabled. + gene2vec_weight = None + if use_gene2vec and gene2vec_file and os.path.exists(gene2vec_file): + gene2vec_weight = torch.from_numpy(np.load(gene2vec_file)).float() + if gene2vec_weight.shape[0] < num_genes: + pad = torch.zeros(num_genes - gene2vec_weight.shape[0], gene2vec_weight.shape[1]) + gene2vec_weight = torch.cat([gene2vec_weight, pad], dim=0) + gene_embed_dim = gene2vec_weight.shape[1] + elif not use_gene2vec: + # Report random gene initialization when gene2vec is disabled. + print(f"[INFO] use_gene2vec=False: Using randomly initialized gene embeddings (dim={gene_embed_dim})") + + # Module embedding layer. + self.module_embed = ModuleEmbedding( + module_sets=module_sets, + num_genes=num_genes, + gene_embed_dim=gene_embed_dim, + embed_dim=base_dim, + gene2vec_weight=gene2vec_weight, + fusion_method=self.fusion_method, + use_gene2vec=use_gene2vec, + ) + + # Positional embedding for modules + # These are pathway module-ID embeddings, rather than geometric positions. + pos_tokens = self.total_module_count + (1 if use_cls_token else 0) + self.pos_emb = nn.Parameter(torch.zeros(1, pos_tokens, base_dim)) + self.layer_pos_emb = Always(None) + + # Family-type embedding (2B): adds family identity to module tokens + self.use_family_emb = use_family_emb + if use_family_emb and module_family_ids is not None: + self.register_buffer( + 'module_family_ids', + torch.tensor(module_family_ids, dtype=torch.long), + ) + self.family_emb = nn.Parameter(torch.zeros(4, base_dim)) + else: + self.module_family_ids = None + self.family_emb = None + + # Sparse module gate (1B): learns per-module scalar gate + self.use_module_gate = use_module_gate + if use_module_gate: + self.module_gate = nn.Sequential( + nn.LayerNorm(base_dim), + nn.Linear(base_dim, 1), + nn.Sigmoid(), + ) + else: + self.module_gate = None + + # Optional CLS token. + if use_cls_token: + self.cls_token = nn.Parameter(torch.zeros(1, 1, base_dim)) + else: + self.cls_token = None + if self.use_cls and self.cls_loss_weight > 0: + cls_hidden_dim = self.cls_head_hidden_dim if self.cls_head_hidden_dim > 0 else base_dim + self.cls_head = nn.Sequential( + nn.LayerNorm(base_dim), + nn.Linear(base_dim, cls_hidden_dim), + nn.GELU(), + nn.Linear(cls_hidden_dim, self.num_modules), + ) + else: + self.cls_head = None + + self.dropout = nn.Dropout(emb_dropout) + + # Performer encoder. + self.encoder = Performer( + base_dim, enc_depth, enc_heads, enc_dim_head, + local_attn_heads, local_window_size, causal, ff_mult, nb_features, + feature_redraw_interval, reversible, ff_chunks, generalized_attention, + kernel_fn, use_scalenorm, use_rezero, ff_glu, ff_dropout, attn_dropout, + cross_attend, no_projection, auto_check_redraw, qkv_bias + ) + self.encoder_norm = nn.LayerNorm(base_dim) + + # Encoder-to-decoder dimension projection. + # performer/cross_attn can use a smaller dec_embed_dim. + # simple/none use base_dim directly without a projection. + if self.decoder_type in {"performer", "cross_attn"}: + self.enc_to_dec = nn.Identity() if self.dec_embed_dim == base_dim else nn.Linear(base_dim, self.dec_embed_dim) + self.decoder_dim = self.dec_embed_dim + else: + # simple/none bypass enc_to_dec and use base_dim. + self.enc_to_dec = nn.Identity() + self.decoder_dim = base_dim + + # Encoder mask token used by the replace-mask path. + if self.mask_style == "replace": + self.encoder_mask_token = nn.Parameter(torch.zeros(1, 1, base_dim)) + else: + self.encoder_mask_token = None + + # Decoder mask token used by the MAE drop-mask path. + if self.pretrain_arch == "mae": + self.mask_token = nn.Parameter(torch.zeros(1, 1, self.decoder_dim)) + else: + self.mask_token = None + + # Optional decoder lets masked tokens exchange information with visible tokens. + if self.decoder_type == "performer": + self.decoder_pos_emb = nn.Parameter(torch.zeros(1, pos_tokens, self.dec_embed_dim)) + self.decoder_dropout = nn.Dropout(emb_dropout) + self.decoder = Performer( + self.dec_embed_dim, dec_depth, dec_heads, dec_dim_head, + local_attn_heads=cast_tuple(0), local_window_size=256, causal=False, ff_mult=dec_ff_mult, + nb_features=None, feature_redraw_interval=feature_redraw_interval, + reversible=False, ff_chunks=1, generalized_attention=False, + kernel_fn=nn.ReLU(), use_scalenorm=False, use_rezero=False, ff_glu=False, + ff_dropout=dec_ff_dropout, attn_dropout=dec_attn_dropout, + cross_attend=False, no_projection=False, auto_check_redraw=auto_check_redraw, + qkv_bias=qkv_bias + ) + self.decoder_norm = nn.LayerNorm(self.dec_embed_dim) + elif self.decoder_type == "cross_attn": + self.decoder_pos_emb = nn.Parameter(torch.zeros(1, pos_tokens, self.dec_embed_dim)) + self.decoder_dropout = nn.Dropout(emb_dropout) + self.decoder = nn.ModuleList( + [ + CrossAttentionDecoderBlock( + dim=self.dec_embed_dim, + heads=dec_heads, + ff_mult=dec_ff_mult, + attn_dropout=dec_attn_dropout, + ff_dropout=dec_ff_dropout, + ) + for _ in range(dec_depth) + ] + ) + self.decoder_norm = nn.LayerNorm(self.dec_embed_dim) + else: + self.decoder_pos_emb = None + self.decoder_dropout = None + self.decoder = None + self.decoder_norm = None + + # Precompute module-to-gene aggregation indices for decoder_gene_agg. + if self.decoder_gene_agg: + pg_module_idx = [] + pg_gene_idx = [] + for module_idx, module_genes in enumerate(self.module_sets): + for gene_idx in module_genes: + if 0 <= gene_idx < self.num_genes: + pg_module_idx.append(module_idx) + pg_gene_idx.append(gene_idx) + self.register_buffer("pg_module_idx", torch.tensor(pg_module_idx, dtype=torch.long)) + self.register_buffer("pg_gene_idx", torch.tensor(pg_gene_idx, dtype=torch.long)) + pg_gene_counts = torch.zeros(self.num_genes, dtype=torch.long) + if len(pg_gene_idx) > 0: + pg_gene_counts.scatter_add_(0, self.pg_gene_idx, torch.ones_like(self.pg_gene_idx)) + self.register_buffer("pg_gene_counts", pg_gene_counts) + else: + self.pg_module_idx = None + self.pg_gene_idx = None + self.pg_gene_counts = None + + # Reconstruction heads: + # Default: one linear head per module. + # decoder_gene_agg: aggregate to genes, then use gene-specific heads. + # module_mean_value: output one mean value per module. + if self.recon_target_level == "gene": + if self.decoder_gene_agg: + self.module_heads = None + self.gene_head = None + # Gene-specific reconstruction heads: separate D -> 1 parameters for each gene. + self.gene_w = nn.Parameter(torch.rand(self.num_genes, self.decoder_dim, 1)) + self.gene_b = nn.Parameter(torch.rand(self.num_genes)) + nn.init.xavier_normal_(self.gene_w) + nn.init.xavier_normal_(self.gene_b.unsqueeze(-1)) + else: + self.module_heads = nn.ModuleList( + [nn.Linear(self.decoder_dim, g) for g in self.module_gene_counts] + ) + self.gene_head = None + self.gene_w = None + self.gene_b = None + self.module_scalar_heads = None + else: + self.module_heads = None + self.gene_head = None + self.gene_w = None + self.gene_b = None + self.module_scalar_heads = nn.ModuleList( + [nn.Linear(self.decoder_dim, 1) for _ in self.module_gene_counts] + ) + + self.last_loss_breakdown = { + "recon_loss": 0.0, + "cls_loss": 0.0, + "total_loss": 0.0, + } + + self._init_weights() + + def _init_weights(self) -> None: + """Initialize model parameters.""" + nn.init.normal_(self.pos_emb, std=0.02) + if self.encoder_mask_token is not None: + nn.init.normal_(self.encoder_mask_token, std=0.02) + if self.mask_token is not None: + nn.init.normal_(self.mask_token, std=0.02) + if self.decoder_pos_emb is not None: + nn.init.normal_(self.decoder_pos_emb, std=0.02) + if self.cls_token is not None: + nn.init.normal_(self.cls_token, std=0.02) + if self.family_emb is not None: + nn.init.normal_(self.family_emb, std=0.02) + for m in self.modules(): + if isinstance(m, nn.Linear): + nn.init.xavier_uniform_(m.weight) + if m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.LayerNorm): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + + def _sample_module_mask( + self, + batch_size: int, + seq_len: int, + device: torch.device, + mask_ratio: float, + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + if mask_ratio <= 0: + ids_restore = torch.arange(seq_len, device=device, dtype=torch.long).unsqueeze(0).expand(batch_size, -1) + mask = torch.zeros(batch_size, seq_len, device=device) + return mask, ids_restore, ids_restore + + len_keep = int(seq_len * (1 - mask_ratio)) + noise = torch.rand(batch_size, seq_len, device=device) + ids_shuffle = torch.argsort(noise, dim=1) + ids_restore = torch.argsort(ids_shuffle, dim=1) + mask = torch.ones([batch_size, seq_len], device=device) + mask[:, :len_keep] = 0 + mask = torch.gather(mask, dim=1, index=ids_restore) + return mask, ids_shuffle, ids_restore + + def random_masking(self, x: torch.Tensor, mask_ratio: float) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Apply per-sample random masking following the MAE convention. + + x: (B, L, D), the module tokens. + Returns visible tokens (B, L_visible, D), a mask (B, L) with 1 for + masked positions, and restoration indices (B, L).""" + """Example with ten modules and a mask ratio of 0.75: + noise = [0.3, 0.9, 0.1, 0.7, 0.2, 0.8, 0.4, 0.6, 0.5, 0.0] + ids_shuffle = [9, 2, 4, 0, 6, 8, 7, 3, 5, 1] + len_keep = int(10 * (1 - 0.75)) = 2, so ids_keep = [9, 2]. + ids_restore = [3, 9, 1, 7, 2, 8, 4, 6, 5, 0] is the inverse permutation. + For example, original module 0 occupies position 3 after shuffling.""" + b, l, d = x.shape + mask, ids_shuffle, ids_restore = self._sample_module_mask( + batch_size=b, + seq_len=l, + device=x.device, + mask_ratio=mask_ratio, + ) + if mask_ratio <= 0 and self.zero_mask_identity_fastpath: + return x, mask, ids_restore + len_keep = int(l * (1 - mask_ratio)) + ids_keep = ids_shuffle[:, :len_keep] + x_masked = torch.gather(x, dim=1, index=ids_keep.unsqueeze(-1).repeat(1, 1, d)) + return x_masked, mask, ids_restore + + def replace_masking( + self, + x: torch.Tensor, + mask_ratio: float + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Retain all module positions and replace masked contents with a learned token.""" + if self.encoder_mask_token is None: + raise RuntimeError("replace_masking requires encoder_mask_token to be initialized.") + + b, l, _ = x.shape + mask, _, _ = self._sample_module_mask( + batch_size=b, + seq_len=l, + device=x.device, + mask_ratio=mask_ratio, + ) + ids_restore = torch.arange(l, device=x.device).unsqueeze(0).expand(b, -1) + mask_tokens = self.encoder_mask_token.expand(b, l, -1) + x_masked = torch.where(mask.bool().unsqueeze(-1), mask_tokens, x) + return x_masked, mask, ids_restore + + def _add_module_position_embedding(self, x: torch.Tensor) -> torch.Tensor: + pos = self.pos_emb[:, 1:, :] if self.use_cls else self.pos_emb + if pos.shape[1] != x.shape[1]: + raise RuntimeError( + f"pos_emb length mismatch: pos={pos.shape[1]}, tokens={x.shape[1]}, " + f"modules={self.num_modules}" + ) + x = x + pos + if self.use_family_emb and self.family_emb is not None: + x = x + self.family_emb[self.module_family_ids] + return x + + def forward_encoder( + self, + cells: torch.Tensor, + mask_ratio: Optional[float] = None, + return_module_targets: bool = True, + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, List[torch.Tensor]]: + """Encode gene expression into contextual module representations. + + cells: (B, num_genes). + Returns latent (B, L_encoded + optional_cls, D), mask (B, L), + ids_restore (B, L), and gene-expression targets for each module.""" + mask_ratio = self.mask_ratio if mask_ratio is None else mask_ratio + + # Step 1: convert gene expression into module tokens. + x, module_targets = self.module_embed( + cells, + return_module_targets=return_module_targets, + ) # Module embeddings with shape (B, L, D). + + # Steps 2/3: drop masked tokens or replace their contents before encoding. + if self.mask_style == "replace": + # Replace token contents, then add module-ID embeddings while retaining all positions. + x, mask, ids_restore = self.replace_masking(x, mask_ratio) + x = self._add_module_position_embedding(x) + else: + # MAE path: add module-ID embeddings, then drop masked modules. + x = self._add_module_position_embedding(x) + x, mask, ids_restore = self.random_masking(x, mask_ratio) + + # Sparse module gating (1B): suppress unimportant modules + if self.use_module_gate and self.module_gate is not None: + gate = self.module_gate(x) # (B, L_visible, 1) + x = x * gate + + # Step 4: prepend the CLS token when enabled. + if self.use_cls and self.cls_token is not None: + cls_token = self.cls_token + self.pos_emb[:, :1, :] + cls_tokens = cls_token.expand(x.shape[0], -1, -1) + x = torch.cat((cls_tokens, x), dim=1) + + # Step 5: run the encoder. + x_emb = self.dropout(x) + layer_pos_emb = self.layer_pos_emb(x_emb)# None + x = self.encoder(x_emb, pos_emb=layer_pos_emb) # Shape remains (B, L_visible, D); each token now incorporates global context. + x = self.encoder_norm(x) #layernorm + + return x, mask, ids_restore, module_targets + + def _build_full_tokens(self, x: torch.Tensor, ids_restore: torch.Tensor) -> torch.Tensor: + """Insert decoder mask tokens and restore the original module order. + + Returns (B, L + optional_cls, D).""" + # Project to decoder width, insert mask tokens, and restore module order. + x = self.enc_to_dec(x) + if self.use_cls: + cls_token = x[:, :1, :] + module_tokens = x[:, 1:, :] + else: + cls_token = None + module_tokens = x + + num_mask_tokens = ids_restore.shape[1] - module_tokens.shape[1] + if num_mask_tokens > 0: + if self.mask_token is None: + raise RuntimeError( + "Decoder-side mask_token is missing, but _build_full_tokens needs to restore masked positions." + ) + mask_tokens = self.mask_token.repeat(x.shape[0], num_mask_tokens, 1) + x_ = torch.cat([module_tokens, mask_tokens], dim=1) + else: + x_ = module_tokens + + x_ = torch.gather(x_, dim=1, index=ids_restore.unsqueeze(-1).repeat(1, 1, x.shape[2])) + + if self.use_cls and cls_token is not None: + return torch.cat([cls_token, x_], dim=1) + return x_ + + def _aggregate_module_to_gene( + self, + module_feats: torch.Tensor, + ) -> torch.Tensor: + """Map module_feats (B, L, D) to gene_feats (B, G, D). + + Each gene receives the mean features of all modules containing it.""" + bsz, _, dim = module_feats.shape + gene_feats = torch.zeros(bsz, self.num_genes, dim, device=module_feats.device, dtype=module_feats.dtype) + gene_counts = torch.zeros(bsz, self.num_genes, device=module_feats.device, dtype=module_feats.dtype) + + # Contributions from the fixed pathway module definitions. + if self.pg_module_idx is not None and self.pg_module_idx.numel() > 0: + pair_module_feats = module_feats[:, self.pg_module_idx, :] # (B, num_pairs, D) + gene_idx_expand = self.pg_gene_idx.view(1, -1, 1).expand(bsz, -1, dim) + gene_feats.scatter_add_(1, gene_idx_expand, pair_module_feats) + base_gene_counts = self.pg_gene_counts.to(dtype=module_feats.dtype, device=module_feats.device) + gene_counts = gene_counts + base_gene_counts.view(1, self.num_genes) + + gene_counts = gene_counts.clamp(min=1.0).unsqueeze(-1) # (B, G, 1) + return gene_feats / gene_counts + + def _predict_from_module_features( + self, + module_feats: torch.Tensor, + ) -> Union[List[torch.Tensor], torch.Tensor]: + """module_feats: (B, L, D)""" + if self.recon_target_level == "gene" and self.decoder_gene_agg: + gene_feats = self._aggregate_module_to_gene(module_feats) # (B, G, D) + assert self.gene_w is not None and self.gene_b is not None + # Apply a separate linear projection to each gene. + preds = torch.einsum("bgd,gdk->bgk", gene_feats, self.gene_w).squeeze(-1) + return preds + self.gene_b.unsqueeze(0) # (B, G) + + preds: List[torch.Tensor] = [] + if self.recon_target_level == "gene": + assert self.module_heads is not None + for i, head in enumerate(self.module_heads): + preds.append(head(module_feats[:, i, :])) + else: + assert self.module_scalar_heads is not None + for i, head in enumerate(self.module_scalar_heads): + preds.append(head(module_feats[:, i, :])) + return preds + + def _build_cls_targets(self, module_targets: List[torch.Tensor]) -> torch.Tensor: + """Supervise CLS with mean expression values from the static module sets. + + CLS aggregates visible-module information to predict the cell's + module-level expression profile.""" + static_module_targets = module_targets[: self.num_modules] + if len(static_module_targets) != self.num_modules: + raise RuntimeError( + f"cls target module count mismatch: expected {self.num_modules}, got {len(static_module_targets)}" + ) + module_means = [target.mean(dim=1, keepdim=True) for target in static_module_targets] + return torch.cat(module_means, dim=1) + + def compute_cls_loss( + self, + latent: torch.Tensor, + module_targets: List[torch.Tensor], + ) -> Tuple[torch.Tensor, torch.Tensor]: + if not self.use_cls or self.cls_head is None: + zero = torch.tensor(0.0, device=latent.device, dtype=latent.dtype) + return zero, zero + cls_encoding = latent[:, 0, :] + cls_preds = self.cls_head(cls_encoding) + cls_targets = self._build_cls_targets(module_targets).to(dtype=cls_preds.dtype) + cls_loss = F.mse_loss(cls_preds, cls_targets) + return cls_loss, cls_encoding + + def forward_decoder_simple( + self, + x: torch.Tensor, + ids_restore: torch.Tensor, + ) -> Union[List[torch.Tensor], torch.Tensor]: + """Insert mask tokens, restore module order, and apply reconstruction heads. + + x: (B, L_visible + optional_cls, D). + ids_restore: (B, L), where L is the total module count. + Returns per-module predictions, or a (B, G) tensor for decoder_gene_agg.""" + x_full = self._build_full_tokens(x, ids_restore) + module_feats = x_full[:, 1:, :] if self.use_cls else x_full + return self._predict_from_module_features(module_feats) + + def forward_decoder_performer( + self, + x: torch.Tensor, + ids_restore: torch.Tensor, + ) -> Union[List[torch.Tensor], torch.Tensor]: + """Restore masked positions, mix tokens with a Performer decoder, and reconstruct.""" + x_full = self._build_full_tokens(x, ids_restore) + pos = self.decoder_pos_emb[:, :x_full.shape[1], :] + x_full = self.decoder_dropout(x_full + pos) + x_full = self.decoder(x_full, pos_emb=self.layer_pos_emb(x_full)) + x_full = self.decoder_norm(x_full) + module_feats = x_full[:, 1:, :] if self.use_cls else x_full + return self._predict_from_module_features(module_feats) + + def forward_decoder_cross_attn( + self, + x: torch.Tensor, + mask: torch.Tensor, + ids_restore: torch.Tensor, + ) -> Union[List[torch.Tensor], torch.Tensor]: + """ + Cross-attention-only decoder: + masked queries attend to visible encoder tokens, without query-query self-attention. + """ + if self.decoder is None or self.decoder_pos_emb is None or self.decoder_dropout is None or self.decoder_norm is None: + raise RuntimeError("cross_attn decoder requested but decoder modules are not initialized.") + + x_full = self._build_full_tokens(x, ids_restore) + if self.use_cls: + cls_token = x_full[:, :1, :] + module_tokens = x_full[:, 1:, :] + module_pos = self.decoder_pos_emb[:, 1:x_full.shape[1], :] + cls_memory = cls_token + else: + cls_token = None + module_tokens = x_full + module_pos = self.decoder_pos_emb[:, :x_full.shape[1], :] + cls_memory = None + + query_tokens = self.decoder_dropout(module_tokens + module_pos) + module_feats = module_tokens.clone() + + mask_bool = mask.bool() + visible_mask = ~mask_bool + batch_size, _, dim = module_feats.shape + visible_per_sample = int(visible_mask[0].sum().item()) if visible_mask.shape[1] > 0 else 0 + masked_per_sample = int(mask_bool[0].sum().item()) if mask_bool.shape[1] > 0 else 0 + + if masked_per_sample > 0: + visible_memory = module_feats[visible_mask].view(batch_size, visible_per_sample, dim) + if cls_memory is not None: + visible_memory = torch.cat([cls_memory, visible_memory], dim=1) + + masked_queries = query_tokens[mask_bool].view(batch_size, masked_per_sample, dim) + for block in self.decoder: + masked_queries = block(masked_queries, visible_memory) + # AMP may run attention/norm in fp32 for stability. Cast back before + # scattering into the decoder-state tensor, which may still be bf16. + masked_queries = self.decoder_norm(masked_queries).to(dtype=module_feats.dtype) + module_feats[mask_bool] = masked_queries.reshape(-1, dim) + + return self._predict_from_module_features(module_feats) + + def forward_predictor_from_encoder( + self, + x: torch.Tensor, + ) -> Union[List[torch.Tensor], torch.Tensor]: + module_feats = x[:, 1:, :] if self.use_cls else x + return self._predict_from_module_features(module_feats) + + def compute_loss( + self, + preds: Union[List[torch.Tensor], torch.Tensor], + targets: List[torch.Tensor], + mask: torch.Tensor, + ) -> torch.Tensor: # Module-level reconstruction loss and aggregation ablations. + """Compute reconstruction loss over masked modules. + + preds and targets contain module predictions and expression targets. + mask: (B, L), with 1 for masked modules and 0 for visible modules. + The decoder_gene_agg path accepts gene-level prediction tensors.""" + if self.recon_target_level == "gene" and self.decoder_gene_agg: + assert isinstance(preds, torch.Tensor) + # Use the original module mask to determine the reconstruction loss. + # For each masked module, select its genes from the gene-level predictions. + total_loss = torch.tensor(0.0, device=mask.device, dtype=torch.float32) + total_count = torch.tensor(0.0, device=mask.device, dtype=torch.float32) + for i, t in enumerate(targets): + module_mask = mask[:, i].float() # (B,) + module_gene_idx = torch.as_tensor(self.module_sets[i], dtype=torch.long, device=preds.device) + p = preds.index_select(dim=1, index=module_gene_idx) # (B, G_i) + diff = (p - t) ** 2 + if self.loss_mode == "gene_mean": + total_loss = total_loss + (diff * module_mask.unsqueeze(-1)).sum() + total_count = total_count + module_mask.sum() * t.shape[1] + else: + module_mse = diff.mean(dim=1) # (B,) + total_loss = total_loss + (module_mse * module_mask).sum() + total_count = total_count + module_mask.sum() + return total_loss / total_count.clamp(min=1.0) + + assert isinstance(preds, list) + if len(preds) != len(targets): + raise RuntimeError(f"pred/target module count mismatch: {len(preds)} vs {len(targets)}") + total_loss = torch.tensor(0.0, device=mask.device, dtype=torch.float32) + total_count = torch.tensor(0.0, device=mask.device, dtype=torch.float32) + + for i, (p, t) in enumerate(zip(preds, targets)): + module_mask = mask[:, i].float() # (B,) + if self.recon_target_level == "module_mean_value": + # Predict a module mean and compare it with the mean of its target genes. + # p: (B,1), t_mean: (B,1) + t_mean = t.mean(dim=1, keepdim=True) + diff = (p - t_mean) ** 2 + total_loss = total_loss + (diff.squeeze(-1) * module_mask).sum() + total_count = total_count + module_mask.sum() + else: + diff = (p - t) ** 2 # (B, G_i) + if self.loss_mode == "gene_mean": + # Weight each gene occurrence in masked modules equally. + total_loss = total_loss + (diff * module_mask.unsqueeze(-1)).sum() + total_count = total_count + module_mask.sum() * t.shape[1] + else: + # Average within each module, then across masked modules with equal module weights. + module_mse = diff.mean(dim=1) # (B,) + total_loss = total_loss + (module_mse * module_mask).sum() + total_count = total_count + module_mask.sum() + + if total_count.item() == 0: + return torch.tensor(0.0, device=mask.device, dtype=torch.float32) + return total_loss / total_count + + def forward( + self, cells: torch.Tensor, mask_ratio: Optional[float] = None, + return_encodings: bool = False, return_module_targets: bool = True, + ): + """Run representation extraction or reconstruction. + + cells: (B, num_genes). + mask_ratio=None uses the configured ratio. + When return_encodings=True, return module_encodings, mask, ids_restore, + optional module_targets, and cls_encoding when CLS is enabled. + Otherwise return total loss (reconstruction plus weighted CLS loss), + predictions (per-module tensors or a gene-level tensor), and the mask.""" + latent, mask, ids_restore, module_targets = self.forward_encoder( + cells, + mask_ratio=mask_ratio, + # Reconstruction always needs targets; representation callers can + # opt out explicitly while preserving the original API default. + return_module_targets=(return_module_targets or not return_encodings), + ) + + if return_encodings: + # Return representations for zero-shot tasks. + # drop-mask returns visible tokens; replace-mask returns every module token. + if self.use_cls: + seq_encodings = latent[:, 1:, :] + else: + seq_encodings = latent + + out = { + 'module_encodings': seq_encodings, + 'mask': mask, + 'ids_restore': ids_restore, + 'module_targets': module_targets + } + if self.use_cls: + out['cls_encoding'] = latent[:, 0, :] + return out + + if self.decoder_type == "performer": + preds = self.forward_decoder_performer(latent, ids_restore) + elif self.decoder_type == "cross_attn": + preds = self.forward_decoder_cross_attn(latent, mask, ids_restore) + elif self.decoder_type == "simple": + preds = self.forward_decoder_simple(latent, ids_restore) + elif self.decoder_type == "none": + preds = self.forward_predictor_from_encoder(latent) + else: + raise RuntimeError(f"Unsupported decoder_type at forward time: {self.decoder_type}") + recon_loss = self.compute_loss(preds, module_targets, mask) + cls_loss, _ = self.compute_cls_loss(latent, module_targets) + total_loss = recon_loss + self.cls_loss_weight * cls_loss + self.last_loss_breakdown = { + "recon_loss": float(recon_loss.detach().cpu()), + "cls_loss": float(cls_loss.detach().cpu()), + "total_loss": float(total_loss.detach().cpu()), + } + return total_loss, preds, mask + + def get_last_loss_breakdown(self) -> dict: + return dict(self.last_loss_breakdown) + + def load_state_dict(self, state_dict, strict: bool = True, assign: bool = False): + """Load current or pre-rename weights under the module-named interface.""" + return super().load_state_dict( + migrate_legacy_module_state_dict(state_dict), + strict=strict, + assign=assign, + ) + + def check_redraw_projections(self): + """Check whether projection matrices need redrawing and redraw them.""" + self.encoder.check_redraw_projections() + + def fix_projection_matrices_(self): + """Disable automatic projection-matrix redraws.""" + self.encoder.fix_projection_matrices_()