| """ | |
| Shared pytest fixtures and helpers for attention module testing. | |
| Auto‑discovers all registered attention variants and provides | |
| standard test inputs (hidden states, masks, configs). | |
| """ | |
| from typing import List | |
| import pytest | |
| import torch | |
| from transformers import AutoModel, DistilBertConfig | |
| import os | |
| import sys | |
| _FRAMEWORK_PARENT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) | |
| if _FRAMEWORK_PARENT not in sys.path: | |
| sys.path.insert(0, _FRAMEWORK_PARENT) | |
| # ---------------------------------------------------------------------- | |
| # Ensure the framework registries are populated before tests run. | |
| # Import the top‑level framework so that all @ATTENTION_REGISTRY.register | |
| # decorators are executed. | |
| # ---------------------------------------------------------------------- | |
| import framework | |
| import framework.attentions | |
| from framework.core.registry import ATTENTION_REGISTRY | |
| # ---------------------------------------------------------------------- | |
| # Fixture: standard DistilBERT config | |
| # ---------------------------------------------------------------------- | |
| def model_config(): | |
| """Return a standard DistilBertConfig (768 hidden, 12 heads).""" | |
| return DistilBertConfig( | |
| hidden_size=768, | |
| num_attention_heads=12, | |
| dropout=0.1, | |
| attention_probs_dropout_prob=0.1, | |
| max_position_embeddings=512, | |
| ) | |
| # ---------------------------------------------------------------------- | |
| # Fixture: batch of random hidden states | |
| # ---------------------------------------------------------------------- | |
| def hidden_states(): | |
| """Return a random (B=2, L=32, D=768) tensor, reproducible.""" | |
| torch.manual_seed(42) | |
| return torch.randn(2, 32, 768) | |
| # ---------------------------------------------------------------------- | |
| # Fixture: standard 2D binary attention mask | |
| # ---------------------------------------------------------------------- | |
| def attention_mask(): | |
| """Return a 2D binary mask (B=2, L=32). First 24 tokens valid, last 8 padding.""" | |
| mask = torch.ones(2, 32) | |
| mask[:, 24:] = 0 | |
| return mask | |
| # ---------------------------------------------------------------------- | |
| # Fixture: small hidden states for quick tests | |
| # ---------------------------------------------------------------------- | |
| def small_hidden_states(): | |
| """Return a small (B=2, L=8, D=768) tensor.""" | |
| torch.manual_seed(123) | |
| return torch.randn(2, 8, 768) | |
| # ---------------------------------------------------------------------- | |
| # Fixture: small attention mask | |
| # ---------------------------------------------------------------------- | |
| def small_attention_mask(): | |
| """Return a small 2D mask (B=2, L=8). Last 2 tokens padding.""" | |
| mask = torch.ones(2, 8) | |
| mask[:, 6:] = 0 | |
| return mask | |
| # ---------------------------------------------------------------------- | |
| # Fixture: list of all registered attention keys | |
| # ---------------------------------------------------------------------- | |
| def registered_attention_keys() -> List[str]: | |
| """Return all keys currently in ATTENTION_REGISTRY.""" | |
| if ATTENTION_REGISTRY is None: | |
| return [] | |
| return sorted(ATTENTION_REGISTRY.keys()) | |
| # ---------------------------------------------------------------------- | |
| # Fixture: original DistilBERT attention module (for zero‑shot tests) | |
| # ---------------------------------------------------------------------- | |
| def distilbert_attention(model_config): | |
| """ | |
| Load a real DistilBERT model or initialized DistilBertModel and return its first layer's attention module. | |
| This gives us the q_lin, k_lin, v_lin, out_lin weights for zero-shot testing. | |
| """ | |
| from transformers import DistilBertModel | |
| model = DistilBertModel(model_config) | |
| return model.transformer.layer[0].attention | |
| # ---------------------------------------------------------------------- | |
| # Utility: instantiate an attention module by registry key | |
| # ---------------------------------------------------------------------- | |
| def build_attention(key: str, config, pretrained_attn=None): | |
| """ | |
| Build a single attention instance from the registry. | |
| Handles modules that require or don't require pretrained_attn. | |
| """ | |
| cls = ATTENTION_REGISTRY.get(key) | |
| if cls is None: | |
| raise ValueError(f"Unknown attention type: {key}") | |
| # Try with pretrained_attn first, then without | |
| try: | |
| return cls(config=config, rank=32, pretrained_attn=pretrained_attn) | |
| except TypeError: | |
| pass | |
| try: | |
| return cls(config=config, rank=32) | |
| except TypeError: | |
| pass | |
| try: | |
| return cls(config) | |
| except TypeError: | |
| pass | |
| # Last resort: just the config | |
| return cls(config) | |
| # ---------------------------------------------------------------------- | |
| # Utility: collect actual attention types that can be instantiated | |
| # ---------------------------------------------------------------------- | |
| def get_testable_attention_keys(config, pretrained_attn=None): | |
| """Return the subset of registry keys whose modules can be instantiated.""" | |
| testable = [] | |
| for key in ATTENTION_REGISTRY.keys(): | |
| try: | |
| build_attention(key, config, pretrained_attn) | |
| testable.append(key) | |
| except Exception: | |
| continue | |
| return testable | |
Xet Storage Details
- Size:
- 5.49 kB
- Xet hash:
- 7e5e2b9ba5b315182719f4f8b0c8bbeaa538c8f50ee51d4d26117270485e5564
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.