tahamajs/RR / benchmark /core /model_factory.py
tahamajs's picture
download
raw
6.85 kB
"""Model instantiation and dynamic attention layer replacement factory."""
from __future__ import annotations
import sys
import traceback
from pathlib import Path
from typing import Tuple, Optional, Any, Dict
import torch
import torch.nn as nn
from transformers import AutoModelForSequenceClassification, AutoConfig
PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent
CODES_DIR = PROJECT_ROOT / "Codes"
if str(CODES_DIR) not in sys.path:
sys.path.insert(0, str(CODES_DIR))
from .registry_utils import get_inject_function, has_inject_function
try:
from framework.core.registry import ATTENTION_REGISTRY
import framework.attentions
except ImportError:
ATTENTION_REGISTRY = {}
def get_transformer_layers(model: nn.Module) -> Optional[nn.ModuleList]:
"""Extract the sequential list of transformer layer blocks from standard HuggingFace models."""
if hasattr(model, "distilbert") and hasattr(model.distilbert, "transformer"):
return model.distilbert.transformer.layer
if hasattr(model, "bert") and hasattr(model.bert, "encoder"):
return model.bert.encoder.layer
if hasattr(model, "roberta") and hasattr(model.roberta, "encoder"):
return model.roberta.encoder.layer
if hasattr(model, "deberta") and hasattr(model.deberta, "encoder"):
return model.deberta.encoder.layer
if hasattr(model, "electra") and hasattr(model.electra, "encoder"):
return model.electra.encoder.layer
if hasattr(model, "albert") and hasattr(model.albert, "encoder"):
return model.albert.encoder.albert_layer_groups[0].albert_layers
if hasattr(model, "transformer") and hasattr(model.transformer, "layer"):
return model.transformer.layer
if hasattr(model, "transformer") and hasattr(model.transformer, "h"):
return model.transformer.h
if hasattr(model, "model") and hasattr(model.model, "layers"):
return model.model.layers
return None
def build_model_with_attention(
model_name: str,
base_model_name: str = "distilbert-base-uncased",
num_labels: int = 2,
pretrained: bool = True,
**kwargs: Any,
) -> Tuple[Optional[nn.Module], str]:
"""
Construct a sequence classification model with standard attention layers replaced
by the specified custom attention module from ATTENTION_REGISTRY.
Args:
model_name: Identifier registered in ATTENTION_REGISTRY (or 'full' for baseline).
base_model_name: Base HuggingFace backbone identifier.
num_labels: Target number of classification classes or 1 for regression.
pretrained: Whether to load pre-trained backbone checkpoint weights.
**kwargs: Additional parameters passed to attention constructors.
Returns:
Tuple of (instantiated_model, status_message).
"""
model_key = model_name.strip().lower()
try:
config = AutoConfig.from_pretrained(base_model_name, num_labels=num_labels)
if num_labels == 1:
config.problem_type = "regression"
if pretrained:
model = AutoModelForSequenceClassification.from_pretrained(
base_model_name,
config=config,
)
else:
model = AutoModelForSequenceClassification.from_config(config)
if model_key == "full":
return model, "OK"
layers = get_transformer_layers(model)
if layers is None:
return None, f"FAIL_BUILD: Unsupported backbone architecture '{base_model_name}'."
total_layers = len(layers)
inject_fn = get_inject_function(model_key)
for i, layer in enumerate(layers):
old_attn = getattr(layer, "attention", None)
if inject_fn is not None:
new_attn = inject_fn(
layer=layer,
model_config=config,
layer_idx=i,
total_layers=total_layers,
old_attn=old_attn,
**kwargs,
)
layer.attention = new_attn
else:
candidate_keys = [model_key, f"{model_key}_attention", f"{model_key}_layer", f"{model_key}_module"]
attn_cls = None
for ck in candidate_keys:
candidate = ATTENTION_REGISTRY.get(ck)
if candidate is not None and isinstance(candidate, type):
attn_cls = candidate
break
if attn_cls is None:
return None, f"FAIL_BUILD: Attention model '{model_name}' not found in ATTENTION_REGISTRY."
try:
new_attn = attn_cls(config, **kwargs)
except TypeError:
try:
new_attn = attn_cls(d_model=config.hidden_size, n_heads=config.num_attention_heads, **kwargs)
except Exception:
new_attn = attn_cls(**kwargs)
layer.attention = new_attn
return model, "OK"
except Exception as e:
err = traceback.format_exc()
return None, f"FAIL_BUILD: {str(e)}\n{err}"
def extract_attention_outputs(
model: nn.Module,
inputs: Dict[str, torch.Tensor],
output_attentions: bool = True,
) -> Optional[List[torch.Tensor]]:
"""
Execute forward pass with attention inspection enabled, returning layer-wise attention matrices.
"""
model.eval()
try:
device = next(model.parameters()).device
dev_inputs = {k: v.to(device) for k, v in inputs.items() if isinstance(v, torch.Tensor)}
except Exception:
dev_inputs = inputs
with torch.no_grad():
try:
outputs = model(**dev_inputs, output_attentions=output_attentions)
if hasattr(outputs, "attentions") and outputs.attentions is not None:
return list(outputs.attentions)
except Exception:
try:
outputs = model(**dev_inputs)
except Exception:
pass
# Check for custom attention weight hooks or layer attributes
custom_attns = []
layers = get_transformer_layers(model)
if layers is not None:
for layer in layers:
attn = getattr(layer, "attention", None)
if attn is not None:
for attr_name in ("last_attn_weights", "attn_weights", "attention_matrix", "scores"):
if hasattr(attn, attr_name) and getattr(attn, attr_name) is not None:
val = getattr(attn, attr_name)
if isinstance(val, torch.Tensor):
custom_attns.append(val)
break
if custom_attns:
return custom_attns
return None

Xet Storage Details

Size:
6.85 kB
·
Xet hash:
b026e0f0e0b670cf64e14a0d62f6f47175dec3f9f702cbf06d8d45d37e4448f9

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.