Feature Extraction
Transformers
Safetensors
eranex_embed
sentence-embeddings
mamba
state-space-model
multilingual
custom_code
Instructions to use Eranex-Tech/eranex-embed-v1.0-base with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use Eranex-Tech/eranex-embed-v1.0-base with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("feature-extraction", model="Eranex-Tech/eranex-embed-v1.0-base", trust_remote_code=True)# pip install -U transformers accelerate # Load model directly from transformers import AutoModel model = AutoModel.from_pretrained("Eranex-Tech/eranex-embed-v1.0-base", trust_remote_code=True, device_map="auto") - Notebooks
- Google Colab
- Kaggle
Download model.py from Eranex-Tech/eranex-embed-v1.0-base: direct link, hf CLI and curl.
- Browser
- Download file 16.4 kB
-
https://huggingface.co/Eranex-Tech/eranex-embed-v1.0-base/resolve/main/model.py
- Command line
-
hf download hf://Eranex-Tech/eranex-embed-v1.0-base/model.py
-
curl -L -o model.py https://huggingface.co/Eranex-Tech/eranex-embed-v1.0-base/resolve/main/model.py
16.4 kB
| """EranexEmbed model mimarisi. | |
| Mamba SSM tabanlı çift yönlü çok katmanlı cümle embedding modeli. | |
| HuggingFace PreTrainedModel ve HF Viewer tam uyumludur. | |
| """ | |
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| from mamba_ssm import Mamba | |
| from transformers import AutoConfig, AutoModel, PretrainedConfig, PreTrainedModel | |
| class EranexEmbedConfig(PretrainedConfig): | |
| """EranexEmbed modeli için yapılandırma. | |
| Attributes: | |
| model_name: Model adı (ör. "eranex-embed"). | |
| model_version: Semantik versiyon (ör. "1.0"). | |
| model_variation: Aktif varyasyon (base | search | reranker). | |
| vocab_size: Kelime dağarcığı boyutu. | |
| embedding_dim: Embedding vektör boyutu. | |
| output_dim: Hedef çıktı embedding boyutu. | |
| num_layers: Mamba katman sayısı. | |
| dropout_rate: Embedding ve hidden katmanlar için dropout oranı. | |
| mamba_d_state: Mamba SSM durum boyutu. | |
| mamba_d_conv: Mamba konvolüsyon çekirdek boyutu. | |
| mamba_expand: Mamba iç genişleme faktörü. | |
| """ | |
| model_type = "eranex_embed" | |
| def __init__( | |
| self, | |
| model_name: str = "eranex-embed", | |
| model_version: str = "1.0", | |
| model_variation: str = "base", | |
| vocab_size: int = 32000, | |
| embedding_dim: int = 384, | |
| output_dim: int = 128, | |
| num_layers: int = 8, | |
| gradient_checkpointing: bool = False, | |
| dropout_rate: float = 0.1, | |
| mamba_d_state: int = 64, | |
| mamba_d_conv: int = 4, | |
| mamba_expand: int = 2, | |
| mlm_mask_prob: float = 0.15, | |
| mlm_replace_mask_pct: float = 0.85, | |
| mlm_random_token_pct: float = 0.10, | |
| mlm_num_special_tokens: int = 5, | |
| mlm_mask_token_id: int = 4, | |
| tie_word_embeddings: bool = True, | |
| **kwargs, | |
| ): | |
| self.model_name = model_name | |
| self.model_version = model_version | |
| self.model_variation = model_variation | |
| self.vocab_size = vocab_size | |
| self.embedding_dim = embedding_dim | |
| self.output_dim = output_dim | |
| self.num_layers = num_layers | |
| self.gradient_checkpointing = gradient_checkpointing | |
| self.dropout_rate = dropout_rate | |
| self.mamba_d_state = mamba_d_state | |
| self.mamba_d_conv = mamba_d_conv | |
| self.mamba_expand = mamba_expand | |
| self.mlm_mask_prob = mlm_mask_prob | |
| self.mlm_replace_mask_pct = mlm_replace_mask_pct | |
| self.mlm_random_token_pct = mlm_random_token_pct | |
| self.mlm_num_special_tokens = mlm_num_special_tokens | |
| self.mlm_mask_token_id = mlm_mask_token_id | |
| super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs) | |
| def formatted_version(self) -> str: | |
| """Versiyon dizesini 'vX.Y' standart formatına dönüştürür.""" | |
| v = str(self.model_version) | |
| return v if v.startswith("v") else f"v{v}" | |
| def model_id(self) -> str: | |
| """Canonical model kimliği: name-vX.Y-variation formatında.""" | |
| if hasattr(self, "model_variation") and self.model_variation: | |
| return f"{self.model_name}-{self.formatted_version}-{self.model_variation}" | |
| return f"{self.model_name}-{self.formatted_version}" | |
| # AutoConfig kaydı | |
| AutoConfig.register("eranex_embed", EranexEmbedConfig) | |
| EranexEmbedConfig.register_for_auto_class("AutoConfig") | |
| class SequenceReverser(nn.Module): | |
| """HF Viewer ve Mamba için padding duyarlı sekans ters çevirme matrisini hesaplar.""" | |
| def forward(self, attention_mask: torch.Tensor, seq_len: int, dim: int) -> torch.Tensor: | |
| idx = torch.arange(seq_len, device=attention_mask.device).unsqueeze(0).expand(attention_mask.size(0), -1) | |
| lengths = attention_mask.sum(dim=1) | |
| rev_idx = lengths.unsqueeze(1) - 1 - idx | |
| rev_idx = torch.where(idx < lengths.unsqueeze(1), rev_idx, idx) | |
| return rev_idx.unsqueeze(-1).expand(-1, -1, dim) | |
| class EranexEmbedBlock(nn.Module): | |
| """Bidirectional Mamba block combining forward and backward SSM streams.""" | |
| def __init__( | |
| self, | |
| dim: int, | |
| d_state: int = 64, | |
| d_conv: int = 4, | |
| expand: int = 2, | |
| ): | |
| super().__init__() | |
| self.norm_fwd = nn.LayerNorm(dim) | |
| self.mamba_fwd = Mamba( | |
| d_model=dim, | |
| d_state=d_state, | |
| d_conv=d_conv, | |
| expand=expand, | |
| ) | |
| self.norm_bwd = nn.LayerNorm(dim) | |
| self.mamba_bwd = Mamba( | |
| d_model=dim, | |
| d_state=d_state, | |
| d_conv=d_conv, | |
| expand=expand, | |
| ) | |
| def __getitem__(self, key: str): | |
| return getattr(self, key) | |
| def __setitem__(self, key: str, value): | |
| setattr(self, key, value) | |
| def __contains__(self, key: str) -> bool: | |
| return hasattr(self, key) | |
| def keys(self): | |
| return ["norm_fwd", "mamba_fwd", "norm_bwd", "mamba_bwd"] | |
| def forward( | |
| self, | |
| x: torch.Tensor, | |
| rev_idx_3d: torch.Tensor, | |
| dropout: nn.Dropout | None = None, | |
| ) -> torch.Tensor: | |
| # 1. İleri (Forward) yön | |
| x_fwd = self.norm_fwd(x) | |
| y_fwd = self.mamba_fwd(x_fwd) | |
| # 2. Geri (Backward) yön (Padding sabit tutularak geçerli tokenlar ters çevrilir) | |
| x_bwd = torch.gather(self.norm_bwd(x), 1, rev_idx_3d) | |
| y_bwd = self.mamba_bwd(x_bwd) | |
| y_bwd_rev = torch.gather(y_bwd, 1, rev_idx_3d) | |
| # 3. Residual bağlantı ve dropout | |
| residual = y_fwd + y_bwd_rev | |
| if dropout is not None: | |
| residual = dropout(residual) | |
| return x + residual | |
| class AttentionPooling(nn.Module): | |
| """Cümle gömmeleri (sentence embeddings) için Attention Pooling mekanizması.""" | |
| def __init__(self, dim: int): | |
| super().__init__() | |
| self.attn = nn.Sequential( | |
| nn.Linear(dim, dim), | |
| nn.Tanh(), | |
| nn.Linear(dim, 1, bias=False) | |
| ) | |
| nn.init.zeros_(self.attn[-1].weight) | |
| def forward(self, x: torch.Tensor, mask: torch.Tensor) -> torch.Tensor: | |
| scores = self.attn(x).squeeze(-1) | |
| scores = scores.masked_fill(mask == 0, -1e4) | |
| weights = torch.softmax(scores, dim=-1).unsqueeze(-1) | |
| mask_3d = mask.unsqueeze(-1).to(x.dtype) | |
| return (x * weights * mask_3d).sum(dim=1) | |
| class EmbeddingProjection(nn.Linear): | |
| """Zengin iç temsil boyutunu hedef kompakt vektör boyutuna izdüşüren projeksiyon katmanı.""" | |
| pass | |
| class Normalize(nn.Module): | |
| """L2 normalizasyon modülü (vektörleri birim hiper-küreye izdüşürür).""" | |
| def __init__(self, p: float = 2.0, dim: int = -1, eps: float = 1e-12): | |
| super().__init__() | |
| self.p = p | |
| self.dim = dim | |
| self.eps = eps | |
| def forward(self, x: torch.Tensor) -> torch.Tensor: | |
| return F.normalize(x, p=self.p, dim=self.dim, eps=self.eps) | |
| class EranexEmbedModel(PreTrainedModel): | |
| """Çok katmanlı Çift Yönlü Mamba SSM tabanlı cümle embedding modeli. | |
| Masked Language Modeling (MLM) tabanlı olarak eğitilir. | |
| Inference sırasında normalize edilmiş cümle embedding vektörü döndürülür. | |
| """ | |
| config_class = EranexEmbedConfig | |
| _tied_weights_keys = {"mlm_head.3.weight": "token_embeddings.weight"} | |
| def __init__(self, config: EranexEmbedConfig): | |
| super().__init__(config) | |
| self.token_embeddings = nn.Embedding(config.vocab_size, config.embedding_dim) | |
| # Düzenlileştirme (Regularization) için dropout katmanları | |
| self.embed_dropout = nn.Dropout(config.dropout_rate) | |
| self.hidden_dropout = nn.Dropout(config.dropout_rate) | |
| # Dizi ters çevirme operatörü (HF Viewer'da temiz SequenceReverser düğümü olarak görünür) | |
| self.sequence_reverser = SequenceReverser() | |
| # Çok Katmanlı Bidirectional Mamba yapısı | |
| self.num_layers = config.num_layers | |
| self.layers = nn.ModuleList([ | |
| EranexEmbedBlock( | |
| dim=config.embedding_dim, | |
| d_state=config.mamba_d_state, | |
| d_conv=config.mamba_d_conv, | |
| expand=config.mamba_expand, | |
| ) | |
| for _ in range(self.num_layers) | |
| ]) | |
| self.encoder_norm = nn.LayerNorm(config.embedding_dim) | |
| # MLM head: masked token prediction. BERT tarzı Linear→GELU→LN→Linear yapısı. | |
| self.mlm_head = nn.Sequential( | |
| nn.Linear(config.embedding_dim, config.embedding_dim), | |
| nn.GELU(), | |
| nn.LayerNorm(config.embedding_dim), | |
| nn.Linear(config.embedding_dim, config.vocab_size), | |
| ) | |
| # Attention Pooling katmanı | |
| self.attention_pooling = AttentionPooling(config.embedding_dim) | |
| # Projeksiyon ve L2 Normalizasyon katmanları (HF Viewer'da anlamlı özel bloklar olarak görünür) | |
| self.projection = EmbeddingProjection(config.embedding_dim, config.output_dim) | |
| self.normalize = Normalize(p=2.0, dim=-1) | |
| # Cross-Encoder relevans skor başlığı: Birleşik dizi için skaler logit üretir. | |
| self.cross_score_head = nn.Linear(config.embedding_dim, 1) | |
| self.post_init() | |
| def get_input_embeddings(self): | |
| return self.token_embeddings | |
| def set_input_embeddings(self, value): | |
| self.token_embeddings = value | |
| def get_output_embeddings(self): | |
| """MLM head'in çıkış katmanı, weight tying için kullanılır.""" | |
| return self.mlm_head[-1] | |
| def set_output_embeddings(self, value): | |
| self.mlm_head[-1] = value | |
| def _forward_encoder(self, x: torch.Tensor, attention_mask: torch.Tensor) -> torch.Tensor: | |
| """Çok katmanlı çift yönlü Mamba bloklarını çalıştırır.""" | |
| attention_mask = attention_mask.long() | |
| seq_len = x.size(1) | |
| rev_idx_3d = self.sequence_reverser(attention_mask, seq_len, x.size(2)) | |
| for layer in self.layers: | |
| x = layer(x, rev_idx_3d, self.hidden_dropout) | |
| return x | |
| def forward( | |
| self, | |
| input_ids: torch.Tensor, | |
| attention_mask: torch.Tensor | None = None, | |
| mode: str = "embedding", | |
| output_hidden_states: bool = False, | |
| **kwargs, | |
| ) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: | |
| """Forward pass: | |
| - mode='embedding': token embedding → Mamba encoder → Attention Pooling → Projection(128) → Normalize. | |
| - mode='cross': [CLS] query [SEP] passage [SEP] → Mamba encoder → Attention Pooling → Linear(384, 1) logit. | |
| """ | |
| x = self.embed_dropout(self.token_embeddings(input_ids)) | |
| if attention_mask is None: | |
| attention_mask = (input_ids != 0).long() | |
| if self.config.gradient_checkpointing and self.training: | |
| x = torch.utils.checkpoint.checkpoint( | |
| self._forward_encoder, x, attention_mask, use_reentrant=False | |
| ) | |
| else: | |
| x = self._forward_encoder(x, attention_mask) | |
| x = self.encoder_norm(x) | |
| if mode == "cross": | |
| # Cross-Encoder Rerank Modu: Attention Pooling sonrası skaler logit | |
| pooled = self.attention_pooling(x, attention_mask) | |
| logits = self.cross_score_head(pooled) | |
| return logits | |
| # Bi-Encoder Embedding Modu | |
| token_output = x | |
| sentence_vector = self.attention_pooling(x, attention_mask) | |
| sentence_vector = self.projection(sentence_vector) | |
| sentence_vector = self.normalize(sentence_vector) | |
| if output_hidden_states: | |
| return sentence_vector, token_output | |
| return sentence_vector | |
| if output_hidden_states: | |
| return sentence_vector, token_output | |
| return sentence_vector | |
| def score_pairs( | |
| self, | |
| queries: list[str], | |
| passages: list[str], | |
| tokenizer, | |
| batch_size: int = 32, | |
| max_len: int | None = None, | |
| ) -> torch.Tensor: | |
| """Cross-Encoder modu ile (query, passage) çiftlerinin ikili relevans skorlarını (0.0 - 1.0) hesaplar.""" | |
| target_len = max_len or getattr(self.config, "max_len", 1024) | |
| scores = [] | |
| device = next(self.parameters()).device | |
| was_training = self.training | |
| self.eval() | |
| with torch.no_grad(): | |
| for i in range(0, len(queries), batch_size): | |
| q_batch = queries[i : i + batch_size] | |
| p_batch = passages[i : i + batch_size] | |
| enc = tokenizer( | |
| q_batch, | |
| p_batch, | |
| padding=True, | |
| truncation=True, | |
| max_length=target_len, | |
| return_tensors="pt", | |
| ) | |
| input_ids = enc["input_ids"].to(device) | |
| attention_mask = enc["attention_mask"].to(device) | |
| logits = self.forward(input_ids, attention_mask=attention_mask, mode="cross") | |
| probs = torch.sigmoid(logits).squeeze(-1) | |
| scores.append(probs.cpu()) | |
| if was_training: | |
| self.train() | |
| return torch.cat(scores, dim=0) if scores else torch.empty(0) | |
| def apply_mlm_mask( | |
| batch: torch.Tensor, | |
| vocab_size: int, | |
| device: torch.device | str, | |
| mask_prob: float = 0.15, | |
| replace_mask_pct: float = 0.85, | |
| random_token_pct: float = 0.10, | |
| num_special_tokens: int = 5, | |
| mask_token_id: int = 4, | |
| ) -> tuple[torch.Tensor, torch.Tensor]: | |
| """Token batch'ine Standart / BERT tarzı MLM maskeleme uygular. | |
| Args: | |
| batch: Token ID tensorü (batch_size, seq_len). | |
| vocab_size: Kelime dağarcığı boyutu. | |
| device: Hesaplama cihazı. | |
| mask_prob: Maskeleme olasılığı (varsayılan: 0.15). | |
| replace_mask_pct: Maskelenenlerin [MASK] ile değiştirilme oranı (varsayılan: 0.80). | |
| random_token_pct: Maskelenenlerin rastgele token ile değiştirilme oranı (varsayılan: 0.10). | |
| num_special_tokens: Özel token sayısı (0..N-1 maskelenmez). | |
| mask_token_id: [MASK] token ID'si. | |
| Returns: | |
| tuple: (mlm_batch, mlm_labels) | |
| """ | |
| mlm_batch = batch.clone() | |
| replace_prob = mask_prob * replace_mask_pct | |
| random_prob = replace_prob + (mask_prob * random_token_pct) | |
| rand_tensor = torch.rand(mlm_batch.shape, device=device) | |
| rand_tensor.masked_fill_(batch < num_special_tokens, 1.0) | |
| masked_indices = rand_tensor < mask_prob | |
| mlm_labels = mlm_batch.clone() | |
| mlm_labels[~masked_indices] = -100 | |
| indices_replaced = masked_indices & (rand_tensor < replace_prob) | |
| mlm_batch[indices_replaced] = mask_token_id | |
| indices_random = masked_indices & (rand_tensor >= replace_prob) & (rand_tensor < random_prob) | |
| random_words = torch.randint( | |
| num_special_tokens, vocab_size, mlm_batch.shape, | |
| dtype=torch.long, device=device, | |
| ) | |
| mlm_batch[indices_random] = random_words[indices_random] | |
| return mlm_batch, mlm_labels | |
| def compute_mlm_metrics_chunked( | |
| mlm_head: nn.Module, | |
| masked_hidden: torch.Tensor, | |
| labels: torch.Tensor, | |
| chunk_size: int = 512, | |
| ) -> tuple[float, int, int]: | |
| """Hesaplama sırasında VRAM patlamasını önlemek için micro-chunking ile MLM loss ve accuracy değerlerini hesaplar. | |
| Args: | |
| mlm_head: MLM tahmin kafa katmanı. | |
| masked_hidden: Sadece maskeli tokenların gizli temsil vektörleri (N, hidden_dim). | |
| labels: Maskeli tokenların gerçek ID etiketleri (N,). | |
| chunk_size: Tek seferde işlenecek mikro parçalama boyutu (varsayılan: 512). | |
| Returns: | |
| tuple: (toplam_sum_loss, dogru_tahmin_sayisi, toplam_maskeli_token_sayisi) | |
| """ | |
| total_masked = masked_hidden.size(0) | |
| if total_masked == 0: | |
| return 0.0, 0, 0 | |
| total_loss = 0.0 | |
| correct_count = 0 | |
| for c_idx in range(0, total_masked, chunk_size): | |
| sub_hidden = masked_hidden[c_idx : c_idx + chunk_size] | |
| sub_labels = labels[c_idx : c_idx + chunk_size] | |
| logits = mlm_head(sub_hidden) | |
| total_loss += F.cross_entropy(logits, sub_labels, reduction="sum").item() | |
| preds = logits.argmax(dim=-1) | |
| correct_count += (preds == sub_labels).sum().item() | |
| return total_loss, correct_count, total_masked | |
| # AutoModel kaydı | |
| AutoModel.register(EranexEmbedConfig, EranexEmbedModel) | |
| EranexEmbedModel.register_for_auto_class("AutoModel") | |