Fill-Mask
Transformers
Safetensors
pakmosaic
pakistan
multilingual
encoder
research-preview
sota-target
custom_code
Instructions to use ProximaAI/PakMosaic-Small with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use ProximaAI/PakMosaic-Small with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("fill-mask", model="ProximaAI/PakMosaic-Small", trust_remote_code=True)# pip install -U transformers accelerate # Load model directly from transformers import AutoModelForMaskedLM model = AutoModelForMaskedLM.from_pretrained("ProximaAI/PakMosaic-Small", trust_remote_code=True, device_map="auto") - Notebooks
- Google Colab
- Kaggle
Add Transformers AutoModelForMaskedLM remote code
Browse files- configuration_pakmosaic.py +71 -0
configuration_pakmosaic.py
ADDED
|
@@ -0,0 +1,71 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Hugging Face config for PakMosaic. Custom architecture, not BERT."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
from transformers import PretrainedConfig
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
class PakMosaicConfig(PretrainedConfig):
|
| 9 |
+
model_type = "pakmosaic"
|
| 10 |
+
|
| 11 |
+
def __init__(
|
| 12 |
+
self,
|
| 13 |
+
vocab_size: int = 32000,
|
| 14 |
+
hidden_size: int = 512,
|
| 15 |
+
num_hidden_layers: int = 12,
|
| 16 |
+
num_attention_heads: int = 8,
|
| 17 |
+
intermediate_size: int = 2048,
|
| 18 |
+
max_position_embeddings: int = 512,
|
| 19 |
+
rope_theta: float = 10000.0,
|
| 20 |
+
attention_dropout: float = 0.0,
|
| 21 |
+
hidden_dropout: float = 0.0,
|
| 22 |
+
ffn_activation: str = "geglu",
|
| 23 |
+
attention_kind: str = "full",
|
| 24 |
+
local_window: int = 128,
|
| 25 |
+
global_every: int = 3,
|
| 26 |
+
use_bias: bool = False,
|
| 27 |
+
tie_word_embeddings: bool = True,
|
| 28 |
+
pad_token_id: int = 0,
|
| 29 |
+
bos_token_id: int = 1,
|
| 30 |
+
eos_token_id: int = 2,
|
| 31 |
+
unk_token_id: int = 3,
|
| 32 |
+
mask_token_id: int = 4,
|
| 33 |
+
layer_norm_eps: float = 1e-5,
|
| 34 |
+
initializer_range: float = 0.02,
|
| 35 |
+
script_signal: str = "none",
|
| 36 |
+
**kwargs,
|
| 37 |
+
) -> None:
|
| 38 |
+
kwargs.pop("algebraic_parameters", None)
|
| 39 |
+
super().__init__(
|
| 40 |
+
pad_token_id=pad_token_id,
|
| 41 |
+
bos_token_id=bos_token_id,
|
| 42 |
+
eos_token_id=eos_token_id,
|
| 43 |
+
**kwargs,
|
| 44 |
+
)
|
| 45 |
+
self.vocab_size = vocab_size
|
| 46 |
+
self.hidden_size = hidden_size
|
| 47 |
+
self.num_hidden_layers = num_hidden_layers
|
| 48 |
+
self.num_attention_heads = num_attention_heads
|
| 49 |
+
self.intermediate_size = intermediate_size
|
| 50 |
+
self.max_position_embeddings = max_position_embeddings
|
| 51 |
+
self.rope_theta = rope_theta
|
| 52 |
+
self.attention_dropout = attention_dropout
|
| 53 |
+
self.hidden_dropout = hidden_dropout
|
| 54 |
+
self.ffn_activation = ffn_activation
|
| 55 |
+
self.attention_kind = attention_kind
|
| 56 |
+
self.local_window = local_window
|
| 57 |
+
self.global_every = global_every
|
| 58 |
+
self.use_bias = use_bias
|
| 59 |
+
self.tie_word_embeddings = tie_word_embeddings
|
| 60 |
+
self.unk_token_id = unk_token_id
|
| 61 |
+
self.mask_token_id = mask_token_id
|
| 62 |
+
self.layer_norm_eps = layer_norm_eps
|
| 63 |
+
self.initializer_range = initializer_range
|
| 64 |
+
self.script_signal = script_signal
|
| 65 |
+
|
| 66 |
+
@property
|
| 67 |
+
def head_dim(self) -> int:
|
| 68 |
+
return self.hidden_size // self.num_attention_heads
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
PakMosaicConfig.register_for_auto_class()
|