N4YuC4's picture
Upload eranex-embed-v1.0-base — MLM, STS, Bitext evaluation included
0a775fc verified
Raw History Blame Contribute Delete
16.4 kB
"""EranexEmbed model mimarisi.
Mamba SSM tabanlı çift yönlü çok katmanlı cümle embedding modeli.
HuggingFace PreTrainedModel ve HF Viewer tam uyumludur.
"""
import torch
import torch.nn as nn
import torch.nn.functional as F
from mamba_ssm import Mamba
from transformers import AutoConfig, AutoModel, PretrainedConfig, PreTrainedModel
class EranexEmbedConfig(PretrainedConfig):
"""EranexEmbed modeli için yapılandırma.
Attributes:
model_name: Model adı (ör. "eranex-embed").
model_version: Semantik versiyon (ör. "1.0").
model_variation: Aktif varyasyon (base | search | reranker).
vocab_size: Kelime dağarcığı boyutu.
embedding_dim: Embedding vektör boyutu.
output_dim: Hedef çıktı embedding boyutu.
num_layers: Mamba katman sayısı.
dropout_rate: Embedding ve hidden katmanlar için dropout oranı.
mamba_d_state: Mamba SSM durum boyutu.
mamba_d_conv: Mamba konvolüsyon çekirdek boyutu.
mamba_expand: Mamba iç genişleme faktörü.
"""
model_type = "eranex_embed"
def __init__(
self,
model_name: str = "eranex-embed",
model_version: str = "1.0",
model_variation: str = "base",
vocab_size: int = 32000,
embedding_dim: int = 384,
output_dim: int = 128,
num_layers: int = 8,
gradient_checkpointing: bool = False,
dropout_rate: float = 0.1,
mamba_d_state: int = 64,
mamba_d_conv: int = 4,
mamba_expand: int = 2,
mlm_mask_prob: float = 0.15,
mlm_replace_mask_pct: float = 0.85,
mlm_random_token_pct: float = 0.10,
mlm_num_special_tokens: int = 5,
mlm_mask_token_id: int = 4,
tie_word_embeddings: bool = True,
**kwargs,
):
self.model_name = model_name
self.model_version = model_version
self.model_variation = model_variation
self.vocab_size = vocab_size
self.embedding_dim = embedding_dim
self.output_dim = output_dim
self.num_layers = num_layers
self.gradient_checkpointing = gradient_checkpointing
self.dropout_rate = dropout_rate
self.mamba_d_state = mamba_d_state
self.mamba_d_conv = mamba_d_conv
self.mamba_expand = mamba_expand
self.mlm_mask_prob = mlm_mask_prob
self.mlm_replace_mask_pct = mlm_replace_mask_pct
self.mlm_random_token_pct = mlm_random_token_pct
self.mlm_num_special_tokens = mlm_num_special_tokens
self.mlm_mask_token_id = mlm_mask_token_id
super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs)
@property
def formatted_version(self) -> str:
"""Versiyon dizesini 'vX.Y' standart formatına dönüştürür."""
v = str(self.model_version)
return v if v.startswith("v") else f"v{v}"
@property
def model_id(self) -> str:
"""Canonical model kimliği: name-vX.Y-variation formatında."""
if hasattr(self, "model_variation") and self.model_variation:
return f"{self.model_name}-{self.formatted_version}-{self.model_variation}"
return f"{self.model_name}-{self.formatted_version}"
# AutoConfig kaydı
AutoConfig.register("eranex_embed", EranexEmbedConfig)
EranexEmbedConfig.register_for_auto_class("AutoConfig")
class SequenceReverser(nn.Module):
"""HF Viewer ve Mamba için padding duyarlı sekans ters çevirme matrisini hesaplar."""
def forward(self, attention_mask: torch.Tensor, seq_len: int, dim: int) -> torch.Tensor:
idx = torch.arange(seq_len, device=attention_mask.device).unsqueeze(0).expand(attention_mask.size(0), -1)
lengths = attention_mask.sum(dim=1)
rev_idx = lengths.unsqueeze(1) - 1 - idx
rev_idx = torch.where(idx < lengths.unsqueeze(1), rev_idx, idx)
return rev_idx.unsqueeze(-1).expand(-1, -1, dim)
class EranexEmbedBlock(nn.Module):
"""Bidirectional Mamba block combining forward and backward SSM streams."""
def __init__(
self,
dim: int,
d_state: int = 64,
d_conv: int = 4,
expand: int = 2,
):
super().__init__()
self.norm_fwd = nn.LayerNorm(dim)
self.mamba_fwd = Mamba(
d_model=dim,
d_state=d_state,
d_conv=d_conv,
expand=expand,
)
self.norm_bwd = nn.LayerNorm(dim)
self.mamba_bwd = Mamba(
d_model=dim,
d_state=d_state,
d_conv=d_conv,
expand=expand,
)
def __getitem__(self, key: str):
return getattr(self, key)
def __setitem__(self, key: str, value):
setattr(self, key, value)
def __contains__(self, key: str) -> bool:
return hasattr(self, key)
def keys(self):
return ["norm_fwd", "mamba_fwd", "norm_bwd", "mamba_bwd"]
def forward(
self,
x: torch.Tensor,
rev_idx_3d: torch.Tensor,
dropout: nn.Dropout | None = None,
) -> torch.Tensor:
# 1. İleri (Forward) yön
x_fwd = self.norm_fwd(x)
y_fwd = self.mamba_fwd(x_fwd)
# 2. Geri (Backward) yön (Padding sabit tutularak geçerli tokenlar ters çevrilir)
x_bwd = torch.gather(self.norm_bwd(x), 1, rev_idx_3d)
y_bwd = self.mamba_bwd(x_bwd)
y_bwd_rev = torch.gather(y_bwd, 1, rev_idx_3d)
# 3. Residual bağlantı ve dropout
residual = y_fwd + y_bwd_rev
if dropout is not None:
residual = dropout(residual)
return x + residual
class AttentionPooling(nn.Module):
"""Cümle gömmeleri (sentence embeddings) için Attention Pooling mekanizması."""
def __init__(self, dim: int):
super().__init__()
self.attn = nn.Sequential(
nn.Linear(dim, dim),
nn.Tanh(),
nn.Linear(dim, 1, bias=False)
)
nn.init.zeros_(self.attn[-1].weight)
def forward(self, x: torch.Tensor, mask: torch.Tensor) -> torch.Tensor:
scores = self.attn(x).squeeze(-1)
scores = scores.masked_fill(mask == 0, -1e4)
weights = torch.softmax(scores, dim=-1).unsqueeze(-1)
mask_3d = mask.unsqueeze(-1).to(x.dtype)
return (x * weights * mask_3d).sum(dim=1)
class EmbeddingProjection(nn.Linear):
"""Zengin iç temsil boyutunu hedef kompakt vektör boyutuna izdüşüren projeksiyon katmanı."""
pass
class Normalize(nn.Module):
"""L2 normalizasyon modülü (vektörleri birim hiper-küreye izdüşürür)."""
def __init__(self, p: float = 2.0, dim: int = -1, eps: float = 1e-12):
super().__init__()
self.p = p
self.dim = dim
self.eps = eps
def forward(self, x: torch.Tensor) -> torch.Tensor:
return F.normalize(x, p=self.p, dim=self.dim, eps=self.eps)
class EranexEmbedModel(PreTrainedModel):
"""Çok katmanlı Çift Yönlü Mamba SSM tabanlı cümle embedding modeli.
Masked Language Modeling (MLM) tabanlı olarak eğitilir.
Inference sırasında normalize edilmiş cümle embedding vektörü döndürülür.
"""
config_class = EranexEmbedConfig
_tied_weights_keys = {"mlm_head.3.weight": "token_embeddings.weight"}
def __init__(self, config: EranexEmbedConfig):
super().__init__(config)
self.token_embeddings = nn.Embedding(config.vocab_size, config.embedding_dim)
# Düzenlileştirme (Regularization) için dropout katmanları
self.embed_dropout = nn.Dropout(config.dropout_rate)
self.hidden_dropout = nn.Dropout(config.dropout_rate)
# Dizi ters çevirme operatörü (HF Viewer'da temiz SequenceReverser düğümü olarak görünür)
self.sequence_reverser = SequenceReverser()
# Çok Katmanlı Bidirectional Mamba yapısı
self.num_layers = config.num_layers
self.layers = nn.ModuleList([
EranexEmbedBlock(
dim=config.embedding_dim,
d_state=config.mamba_d_state,
d_conv=config.mamba_d_conv,
expand=config.mamba_expand,
)
for _ in range(self.num_layers)
])
self.encoder_norm = nn.LayerNorm(config.embedding_dim)
# MLM head: masked token prediction. BERT tarzı Linear→GELU→LN→Linear yapısı.
self.mlm_head = nn.Sequential(
nn.Linear(config.embedding_dim, config.embedding_dim),
nn.GELU(),
nn.LayerNorm(config.embedding_dim),
nn.Linear(config.embedding_dim, config.vocab_size),
)
# Attention Pooling katmanı
self.attention_pooling = AttentionPooling(config.embedding_dim)
# Projeksiyon ve L2 Normalizasyon katmanları (HF Viewer'da anlamlı özel bloklar olarak görünür)
self.projection = EmbeddingProjection(config.embedding_dim, config.output_dim)
self.normalize = Normalize(p=2.0, dim=-1)
# Cross-Encoder relevans skor başlığı: Birleşik dizi için skaler logit üretir.
self.cross_score_head = nn.Linear(config.embedding_dim, 1)
self.post_init()
def get_input_embeddings(self):
return self.token_embeddings
def set_input_embeddings(self, value):
self.token_embeddings = value
def get_output_embeddings(self):
"""MLM head'in çıkış katmanı, weight tying için kullanılır."""
return self.mlm_head[-1]
def set_output_embeddings(self, value):
self.mlm_head[-1] = value
def _forward_encoder(self, x: torch.Tensor, attention_mask: torch.Tensor) -> torch.Tensor:
"""Çok katmanlı çift yönlü Mamba bloklarını çalıştırır."""
attention_mask = attention_mask.long()
seq_len = x.size(1)
rev_idx_3d = self.sequence_reverser(attention_mask, seq_len, x.size(2))
for layer in self.layers:
x = layer(x, rev_idx_3d, self.hidden_dropout)
return x
def forward(
self,
input_ids: torch.Tensor,
attention_mask: torch.Tensor | None = None,
mode: str = "embedding",
output_hidden_states: bool = False,
**kwargs,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
"""Forward pass:
- mode='embedding': token embedding → Mamba encoder → Attention Pooling → Projection(128) → Normalize.
- mode='cross': [CLS] query [SEP] passage [SEP] → Mamba encoder → Attention Pooling → Linear(384, 1) logit.
"""
x = self.embed_dropout(self.token_embeddings(input_ids))
if attention_mask is None:
attention_mask = (input_ids != 0).long()
if self.config.gradient_checkpointing and self.training:
x = torch.utils.checkpoint.checkpoint(
self._forward_encoder, x, attention_mask, use_reentrant=False
)
else:
x = self._forward_encoder(x, attention_mask)
x = self.encoder_norm(x)
if mode == "cross":
# Cross-Encoder Rerank Modu: Attention Pooling sonrası skaler logit
pooled = self.attention_pooling(x, attention_mask)
logits = self.cross_score_head(pooled)
return logits
# Bi-Encoder Embedding Modu
token_output = x
sentence_vector = self.attention_pooling(x, attention_mask)
sentence_vector = self.projection(sentence_vector)
sentence_vector = self.normalize(sentence_vector)
if output_hidden_states:
return sentence_vector, token_output
return sentence_vector
if output_hidden_states:
return sentence_vector, token_output
return sentence_vector
def score_pairs(
self,
queries: list[str],
passages: list[str],
tokenizer,
batch_size: int = 32,
max_len: int | None = None,
) -> torch.Tensor:
"""Cross-Encoder modu ile (query, passage) çiftlerinin ikili relevans skorlarını (0.0 - 1.0) hesaplar."""
target_len = max_len or getattr(self.config, "max_len", 1024)
scores = []
device = next(self.parameters()).device
was_training = self.training
self.eval()
with torch.no_grad():
for i in range(0, len(queries), batch_size):
q_batch = queries[i : i + batch_size]
p_batch = passages[i : i + batch_size]
enc = tokenizer(
q_batch,
p_batch,
padding=True,
truncation=True,
max_length=target_len,
return_tensors="pt",
)
input_ids = enc["input_ids"].to(device)
attention_mask = enc["attention_mask"].to(device)
logits = self.forward(input_ids, attention_mask=attention_mask, mode="cross")
probs = torch.sigmoid(logits).squeeze(-1)
scores.append(probs.cpu())
if was_training:
self.train()
return torch.cat(scores, dim=0) if scores else torch.empty(0)
def apply_mlm_mask(
batch: torch.Tensor,
vocab_size: int,
device: torch.device | str,
mask_prob: float = 0.15,
replace_mask_pct: float = 0.85,
random_token_pct: float = 0.10,
num_special_tokens: int = 5,
mask_token_id: int = 4,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Token batch'ine Standart / BERT tarzı MLM maskeleme uygular.
Args:
batch: Token ID tensorü (batch_size, seq_len).
vocab_size: Kelime dağarcığı boyutu.
device: Hesaplama cihazı.
mask_prob: Maskeleme olasılığı (varsayılan: 0.15).
replace_mask_pct: Maskelenenlerin [MASK] ile değiştirilme oranı (varsayılan: 0.80).
random_token_pct: Maskelenenlerin rastgele token ile değiştirilme oranı (varsayılan: 0.10).
num_special_tokens: Özel token sayısı (0..N-1 maskelenmez).
mask_token_id: [MASK] token ID'si.
Returns:
tuple: (mlm_batch, mlm_labels)
"""
mlm_batch = batch.clone()
replace_prob = mask_prob * replace_mask_pct
random_prob = replace_prob + (mask_prob * random_token_pct)
rand_tensor = torch.rand(mlm_batch.shape, device=device)
rand_tensor.masked_fill_(batch < num_special_tokens, 1.0)
masked_indices = rand_tensor < mask_prob
mlm_labels = mlm_batch.clone()
mlm_labels[~masked_indices] = -100
indices_replaced = masked_indices & (rand_tensor < replace_prob)
mlm_batch[indices_replaced] = mask_token_id
indices_random = masked_indices & (rand_tensor >= replace_prob) & (rand_tensor < random_prob)
random_words = torch.randint(
num_special_tokens, vocab_size, mlm_batch.shape,
dtype=torch.long, device=device,
)
mlm_batch[indices_random] = random_words[indices_random]
return mlm_batch, mlm_labels
def compute_mlm_metrics_chunked(
mlm_head: nn.Module,
masked_hidden: torch.Tensor,
labels: torch.Tensor,
chunk_size: int = 512,
) -> tuple[float, int, int]:
"""Hesaplama sırasında VRAM patlamasını önlemek için micro-chunking ile MLM loss ve accuracy değerlerini hesaplar.
Args:
mlm_head: MLM tahmin kafa katmanı.
masked_hidden: Sadece maskeli tokenların gizli temsil vektörleri (N, hidden_dim).
labels: Maskeli tokenların gerçek ID etiketleri (N,).
chunk_size: Tek seferde işlenecek mikro parçalama boyutu (varsayılan: 512).
Returns:
tuple: (toplam_sum_loss, dogru_tahmin_sayisi, toplam_maskeli_token_sayisi)
"""
total_masked = masked_hidden.size(0)
if total_masked == 0:
return 0.0, 0, 0
total_loss = 0.0
correct_count = 0
for c_idx in range(0, total_masked, chunk_size):
sub_hidden = masked_hidden[c_idx : c_idx + chunk_size]
sub_labels = labels[c_idx : c_idx + chunk_size]
logits = mlm_head(sub_hidden)
total_loss += F.cross_entropy(logits, sub_labels, reduction="sum").item()
preds = logits.argmax(dim=-1)
correct_count += (preds == sub_labels).sum().item()
return total_loss, correct_count, total_masked
# AutoModel kaydı
AutoModel.register(EranexEmbedConfig, EranexEmbedModel)
EranexEmbedModel.register_for_auto_class("AutoModel")