Initial BashkirRoBERTa release

README.md

@@ -0,0 +1,92 @@
+---
+language:
+- ba
+license: other
+library_name: transformers
+pipeline_tag: fill-mask
+tags:
+- bashkir
+- masked-language-modeling
+- roberta
+- sentencepiece
+- custom-code
+---
+
+# BashkirRoBERTa
+
+BashkirRoBERTa is a 50M-parameter masked language model for Bashkir.
+It predicts a missing token from both sides of its context. The release includes
+the original 16,384-piece SentencePiece BPE tokenizer and an exact Pre-LayerNorm
+Transformer implementation.
+
+It is intended for fill-mask experiments and further fine-tuning. It is not a
+chat or text-generation model.
+
+## Quick start
+
+```python
+from transformers import AutoModelForMaskedLM, AutoTokenizer
+
+repo_id = "failed09/bashkir-roberta"
+tokenizer = AutoTokenizer.from_pretrained(repo_id, trust_remote_code=True)
+model = AutoModelForMaskedLM.from_pretrained(repo_id, trust_remote_code=True)
+
+inputs = tokenizer("Мин башҡорт телен [MASK].", return_tensors="pt")
+logits = model(**inputs).logits
+mask_index = inputs["input_ids"][0].tolist().index(tokenizer.mask_token_id)
+top_id = logits[0, mask_index].argmax().item()
+print(tokenizer.decode([top_id]))  # яратам
+```
+
+`trust_remote_code=True` is required because this checkpoint uses a custom
+Pre-LayerNorm architecture, not the post-LayerNorm implementation in the stock
+`RobertaForMaskedLM` class.
+
+## Architecture
+
+| Property | Value |
+|---|---:|
+| Transformer blocks | 8, Pre-LayerNorm |
+| Hidden size / attention heads | 640 / 10 |
+| Feed-forward size | 2,560 |
+| Context window | 256 subword tokens |
+| Vocabulary | SentencePiece BPE, 16,384 tokens |
+| Parameters | 50.04M |
+| MLM head | tied to input embeddings |
+
+Special IDs are fixed: `<pad>` 0, `<unk>` 1, `<s>` 2, `</s>` 3, `[CLS]` 4,
+`[SEP]` 5, and `[MASK]` 6.
+
+## Training and evaluation
+
+The model was pretrained with dynamic masked-language modelling on a Bashkir
+text collection assembled from encyclopedic, periodical, and literary sources.
+The source texts are not distributed in this repository. Public Bashkir
+Wikipedia is one component; the remaining source materials may carry separate
+terms. This release contains only model weights, code, and tokenizer assets.
+
+On a held-out Bashkir encyclopedic evaluation set, the project reports 24.7%
+top-1 and 54.0% top-5 accuracy for masked subword prediction. These results are
+diagnostic MLM measurements, not a general-purpose language understanding score.
+
+## Limitations
+
+- A mask corresponds to one SentencePiece token, which may be a full word or a
+  subword fragment.
+- The model can reflect errors, source bias, and rare memorized sequences from
+  its training material. Do not use it to reconstruct non-public source text.
+- It has not been contrastively trained for semantic retrieval. Do not treat raw
+  MLM outputs as sentence embeddings.
+
+## License and provenance
+
+The checkpoint is released under the repository's **custom terms** (`other` on
+the Hub) while the source-rights audit is completed. No training texts are
+redistributed. Please contact the maintainer through the Hub for provenance or
+removal requests before commercial redistribution.
+
+## Reproducibility
+
+The original checkpoint was converted to `model.safetensors` without
+retraining. The included exporter verifies that the converted model produces
+exactly the same logits as the original checkpoint for a fixed inference input.

config.json

@@ -0,0 +1,45 @@
+{
+  "transformers_version": "5.15.1",
+  "architectures": [
+    "BashkirRobertaForMaskedLM"
+  ],
+  "output_hidden_states": false,
+  "return_dict": true,
+  "dtype": "float32",
+  "chunk_size_feed_forward": 0,
+  "is_encoder_decoder": false,
+  "id2label": {
+    "0": "LABEL_0",
+    "1": "LABEL_1"
+  },
+  "label2id": {
+    "LABEL_0": 0,
+    "LABEL_1": 1
+  },
+  "problem_type": null,
+  "_name_or_path": "",
+  "pad_token_id": 0,
+  "bos_token_id": 2,
+  "eos_token_id": 3,
+  "cls_token_id": 4,
+  "sep_token_id": 5,
+  "mask_token_id": 6,
+  "tie_word_embeddings": true,
+  "vocab_size": 16384,
+  "hidden_size": 640,
+  "num_hidden_layers": 8,
+  "num_attention_heads": 10,
+  "intermediate_size": 2560,
+  "max_position_embeddings": 256,
+  "hidden_dropout_prob": 0.1,
+  "model_type": "bashkir-roberta-preln",
+  "output_attentions": false,
+  "auto_map": {
+    "AutoConfig": "configuration_bashkir_roberta.BashkirRobertaConfig",
+    "AutoModelForMaskedLM": "modeling_bashkir_roberta.BashkirRobertaForMaskedLM",
+    "AutoTokenizer": [
+      "tokenization_bashkir_roberta.BashkirRobertaTokenizer",
+      null
+    ]
+  }
+}

configuration_bashkir_roberta.py

@@ -0,0 +1,50 @@
+"""Configuration for the BashkirRoBERTa Pre-LN masked-language model."""
+
+from transformers import PretrainedConfig
+
+
+class BashkirRobertaConfig(PretrainedConfig):
+    """Keeps the exact architecture of the project's flagship checkpoint.
+
+    Despite the familiar name, this is not Hugging Face's post-LayerNorm
+    ``RobertaConfig``: the encoder blocks here use pre-LayerNorm.
+    """
+
+    model_type = "bashkir-roberta-preln"
+
+    def __init__(
+        self,
+        vocab_size=16_384,
+        hidden_size=640,
+        num_hidden_layers=8,
+        num_attention_heads=10,
+        intermediate_size=2_560,
+        max_position_embeddings=256,
+        hidden_dropout_prob=0.1,
+        pad_token_id=0,
+        bos_token_id=2,
+        eos_token_id=3,
+        cls_token_id=4,
+        sep_token_id=5,
+        mask_token_id=6,
+        tie_word_embeddings=True,
+        **kwargs,
+    ):
+        super().__init__(
+            pad_token_id=pad_token_id,
+            bos_token_id=bos_token_id,
+            eos_token_id=eos_token_id,
+            cls_token_id=cls_token_id,
+            sep_token_id=sep_token_id,
+            mask_token_id=mask_token_id,
+            tie_word_embeddings=tie_word_embeddings,
+            **kwargs,
+        )
+        self.vocab_size = vocab_size
+        self.hidden_size = hidden_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+        self.intermediate_size = intermediate_size
+        self.max_position_embeddings = max_position_embeddings
+        self.hidden_dropout_prob = hidden_dropout_prob
+

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:48e4d4e7b8af675d222e58e14776abac24b7e83a937d7884bd9d735ea9dd2173
+size 200171016

modeling_bashkir_roberta.py

@@ -0,0 +1,114 @@
+"""Exact Hugging Face-compatible implementation of BashkirRoBERTa Pre-LN."""
+
+import torch
+from torch import nn
+from transformers import PreTrainedModel
+from transformers.modeling_outputs import MaskedLMOutput
+
+from configuration_bashkir_roberta import BashkirRobertaConfig
+
+
+class BashkirRobertaEmbeddings(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.word_embeddings = nn.Embedding(
+            config.vocab_size, config.hidden_size, padding_idx=config.pad_token_id
+        )
+        self.position_embeddings = nn.Embedding(
+            config.max_position_embeddings, config.hidden_size
+        )
+        self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=1e-12)
+        self.dropout = nn.Dropout(config.hidden_dropout_prob)
+
+    def forward(self, input_ids):
+        sequence_length = input_ids.shape[1]
+        positions = torch.arange(
+            sequence_length, dtype=torch.long, device=input_ids.device
+        ).unsqueeze(0)
+        hidden_states = self.word_embeddings(input_ids) + self.position_embeddings(positions)
+        return self.dropout(self.LayerNorm(hidden_states))
+
+
+class BashkirRobertaLayer(nn.Module):
+    """Pre-LayerNorm transformer block, matching ``train_flagship.py`` exactly."""
+
+    def __init__(self, config):
+        super().__init__()
+        self.self_attn = nn.MultiheadAttention(
+            config.hidden_size,
+            config.num_attention_heads,
+            dropout=config.hidden_dropout_prob,
+            batch_first=True,
+        )
+        self.norm1 = nn.LayerNorm(config.hidden_size, eps=1e-12)
+        self.ffn = nn.Sequential(
+            nn.Linear(config.hidden_size, config.intermediate_size),
+            nn.GELU(),
+            nn.Dropout(config.hidden_dropout_prob),
+            nn.Linear(config.intermediate_size, config.hidden_size),
+            nn.Dropout(config.hidden_dropout_prob),
+        )
+        self.norm2 = nn.LayerNorm(config.hidden_size, eps=1e-12)
+
+    def forward(self, hidden_states, padding_mask=None):
+        normalized = self.norm1(hidden_states)
+        attention, _ = self.self_attn(
+            normalized, normalized, normalized, key_padding_mask=padding_mask
+        )
+        hidden_states = hidden_states + attention
+        return hidden_states + self.ffn(self.norm2(hidden_states))
+
+
+class BashkirRobertaForMaskedLM(PreTrainedModel):
+    config_class = BashkirRobertaConfig
+    base_model_prefix = "bashkir_roberta"
+    main_input_name = "input_ids"
+    _tied_weights_keys = {"lm_head.weight": "embeddings.word_embeddings.weight"}
+
+    def __init__(self, config):
+        super().__init__(config)
+        self.embeddings = BashkirRobertaEmbeddings(config)
+        self.layers = nn.ModuleList(
+            [BashkirRobertaLayer(config) for _ in range(config.num_hidden_layers)]
+        )
+        self.norm_final = nn.LayerNorm(config.hidden_size, eps=1e-12)
+        self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
+        self.post_init()
+
+    def get_input_embeddings(self):
+        return self.embeddings.word_embeddings
+
+    def set_input_embeddings(self, value):
+        self.embeddings.word_embeddings = value
+
+    def get_output_embeddings(self):
+        return self.lm_head
+
+    def set_output_embeddings(self, new_embeddings):
+        self.lm_head = new_embeddings
+
+    def tie_weights(self, **kwargs):
+        if self.config.tie_word_embeddings:
+            self.lm_head.weight = self.embeddings.word_embeddings.weight
+
+    def forward(self, input_ids, attention_mask=None, labels=None, return_dict=True, **kwargs):
+        if input_ids.shape[1] > self.config.max_position_embeddings:
+            raise ValueError(
+                f"Sequence length {input_ids.shape[1]} exceeds "
+                f"max_position_embeddings={self.config.max_position_embeddings}."
+            )
+        padding_mask = input_ids.eq(self.config.pad_token_id) if attention_mask is None else attention_mask.eq(0)
+        hidden_states = self.embeddings(input_ids)
+        for layer in self.layers:
+            hidden_states = layer(hidden_states, padding_mask=padding_mask)
+        hidden_states = self.norm_final(hidden_states)
+        logits = self.lm_head(hidden_states)
+
+        loss = None
+        if labels is not None:
+            loss = nn.functional.cross_entropy(
+                logits.view(-1, self.config.vocab_size), labels.view(-1), ignore_index=-100
+            )
+        if not return_dict:
+            return (loss, logits) if loss is not None else (logits,)
+        return MaskedLMOutput(loss=loss, logits=logits)

spm_bashkir_bert_16k.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4e77b9cfc9ab8c200406ee46c2d20fc46734da9ddcf01dff0f0e370f6c4ac244
+size 586970

tokenization_bashkir_roberta.py

@@ -0,0 +1,61 @@
+"""SentencePiece tokenizer preserving BashkirRoBERTa's original token IDs."""
+
+from pathlib import Path
+
+import sentencepiece as spm
+from transformers import PreTrainedTokenizer
+
+
+class BashkirRobertaTokenizer(PreTrainedTokenizer):
+    vocab_files_names = {"vocab_file": "spm_bashkir_bert_16k.model"}
+    model_input_names = ["input_ids", "attention_mask"]
+
+    def __init__(self, vocab_file, **kwargs):
+        self.vocab_file = vocab_file
+        self.sp_model = spm.SentencePieceProcessor(model_file=vocab_file)
+        defaults = {
+            "bos_token": "<s>", "eos_token": "</s>", "unk_token": "<unk>",
+            "pad_token": "<pad>", "cls_token": "[CLS]", "sep_token": "[SEP]",
+            "mask_token": "[MASK]",
+        }
+        for key, value in defaults.items():
+            kwargs.setdefault(key, value)
+        super().__init__(**kwargs)
+
+    @property
+    def vocab_size(self):
+        return self.sp_model.get_piece_size()
+
+    def get_vocab(self):
+        return {self.sp_model.id_to_piece(i): i for i in range(self.vocab_size)}
+
+    def _tokenize(self, text):
+        return self.sp_model.encode(text, out_type=str)
+
+    def _convert_token_to_id(self, token):
+        return self.sp_model.piece_to_id(token)
+
+    def _convert_id_to_token(self, index):
+        return self.sp_model.id_to_piece(index)
+
+    def convert_tokens_to_string(self, tokens):
+        return self.sp_model.decode(tokens)
+
+    def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
+        if token_ids_1 is not None:
+            return [self.bos_token_id] + token_ids_0 + [self.sep_token_id] + token_ids_1 + [self.eos_token_id]
+        return [self.bos_token_id] + token_ids_0 + [self.eos_token_id]
+
+    def get_special_tokens_mask(self, token_ids_0, token_ids_1=None, already_has_special_tokens=False):
+        if already_has_special_tokens:
+            return super().get_special_tokens_mask(token_ids_0, token_ids_1, True)
+        if token_ids_1 is None:
+            return [1] + [0] * len(token_ids_0) + [1]
+        return [1] + [0] * len(token_ids_0) + [1] + [0] * len(token_ids_1) + [1]
+
+    def save_vocabulary(self, save_directory, filename_prefix=None):
+        source = Path(self.vocab_file)
+        name = ((filename_prefix + "-") if filename_prefix else "") + self.vocab_files_names["vocab_file"]
+        target = Path(save_directory) / name
+        target.write_bytes(source.read_bytes())
+        return (str(target),)

tokenizer_config.json

@@ -0,0 +1,76 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "[CLS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "5": {
+      "content": "[SEP]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "6": {
+      "content": "[MASK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "backend": "custom",
+  "bos_token": "<s>",
+  "cls_token": "[CLS]",
+  "eos_token": "</s>",
+  "mask_token": "[MASK]",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "sep_token": "[SEP]",
+  "tokenizer_class": "BashkirRobertaTokenizer",
+  "unk_token": "<unk>",
+  "auto_map": {
+    "AutoTokenizer": [
+      "tokenization_bashkir_roberta.BashkirRobertaTokenizer",
+      null
+    ]
+  }
+}