#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ tokenization_rootformer.py Rootformer Farāhīdian-Basran Morphemic PreTrainedTokenizer. Implements the official Hugging Face PreTrainedTokenizer for Rootformer. Provides: - 100% BPE-Free Farāhīdian radical decomposition (prefix, root, wazn, suffix) - Basran syntactic harmony and 7 classical Arabic mutation rules (Al-Naḥw wa-l-Iʿrāb) - Full compatibility with AutoTokenizer.from_pretrained(..., trust_remote_code=True) - Dual-mode outputs: standard input_ids + attention_mask, plus p_ids, r_ids, w_ids, s_ids """ import os import re import json from pathlib import Path from typing import List, Tuple, Dict, Any, Optional, Union import torch from transformers import PreTrainedTokenizer from transformers.tokenization_utils_base import BatchEncoding # Canonical Classical Root Aliases ROOT_CANONICAL_MAP = { 'قدم': 'تقدم', 'جزأ': 'جزا', 'جزء': 'جزا', 'بدأ': 'بدا', 'بدء': 'بدا', 'أله': 'اله', 'ءله': 'اله', 'موه': 'موه', 'قبل': 'قبا', 'نها': 'نهي', 'أمر': 'امر', 'أحد': 'احد', 'يقن': 'يقن', 'شفي': 'شفي', 'شفء': 'شفي', 'غيي': 'غيي', 'جني': 'جني', 'جنى': 'جني', 'أصل': 'اصل', 'اصل': 'اصل', 'شيء': 'شيا', 'خلو': 'خلا', 'عين': 'عيا', 'فتح': 'تفح', 'أمل': 'امل', 'امل': 'امل', 'عضو': 'عضا', 'عضي': 'عضا', 'حشو': 'حشا', 'حشي': 'حشا', 'هوي': 'هوا' } ROOT_CANONICAL_MAP_REV = { 'تقدم': 'قدم', 'جزا': 'جزأ', 'بدا': 'بدأ', 'اله': 'أله', 'قبا': 'قبل', 'موه': 'موه', 'نهي': 'نهي', 'تفح': 'فتح', 'امل': 'أمل', 'عضا': 'عضو', 'حشا': 'حشا', 'هوا': 'هوي' } CLOSED_PARTICLES = { 'إنما', 'إلى', 'عن', 'على', 'في', 'من', 'حتى', 'إن', 'أن', 'كأن', 'لكن', 'ليت', 'لعل', 'إلا', 'غير', 'إذا', 'إذ', 'حيث', 'ثم', 'أو', 'أم', 'بل', 'لا', 'ما', 'لم', 'لن', 'ليس', 'هذا', 'هذه', 'ذلك', 'تلك', 'هؤلاء', 'أولئك', 'الذي', 'التي', 'الذين', 'اللواتي', 'هو', 'هي', 'أنت', 'أنتم', 'أنا', 'نحن', 'قد', 'سوف', 'نعم', 'بئس', 'حبذا', 'مع', 'عند', 'قبل', 'بعد', 'دون', 'بين', 'أيضا', 'فقط', 'قط', 'حيثما', 'كيف', 'أين', 'متى', 'كذلك', 'إذن', 'لو', 'لولا', 'إما', 'هل', 'لما' } COMPOUND_PARTICLES = { 'بلا': ('ب', 'لا'), 'بغير': ('ب', 'غير'), 'ولما': ('و', 'لما'), 'فلما': ('ف', 'لما'), 'كلما': ('كل', 'ما'), 'فهو': ('ف', 'هو'), 'فهي': ('ف', 'هي'), 'وهو': ('و', 'هو'), 'وهي': ('و', 'هي'), 'فإن': ('ف', 'إن'), 'وإن': ('و', 'إن'), 'لأن': ('ل', 'أن'), 'عنها': ('عن', 'ها'), 'عليها': ('على', 'ها'), 'منها': ('من', 'ها'), 'فيها': ('في', 'ها'), 'به': ('ب', 'ه'), 'بها': ('ب', 'ها'), 'له': ('ل', 'ه'), 'لها': ('ل', 'ها'), 'فله': ('ف', 'له'), 'وله': ('و', 'له'), 'أنها': ('أن', 'ها'), 'إنها': ('إن', 'ها'), 'أنه': ('أن', 'ه'), 'إنه': ('إن', 'ه'), 'أنهم': ('أن', 'هم'), 'إنهم': ('إن', 'هم'), 'ليست': ('ليس', 'ت'), 'مما': ('من', 'ما'), 'عما': ('عن', 'ما'), 'بما': ('ب', 'ما'), 'ولا': ('و', 'لا'), 'لأنه': ('ل', 'أنه'), 'فلأنه': ('ف', 'لأنه'), 'وإما': ('و', 'إما'), 'فإما': ('ف', 'إما'), 'بذاته': ('ب', 'ذات'), 'بذاتها': ('ب', 'ذات'), 'لذاته': ('ل', 'ذات'), 'لذاتها': ('ل', 'ذات'), 'بغيره': ('ب', 'غير'), 'لغيره': ('ل', 'غير'), 'عنه': ('عن', 'ه'), 'منه': ('من', 'ه'), 'فيه': ('في', 'ه'), 'عليه': ('على', 'ه'), 'إليه': ('إلى', 'ه'), 'إليها': ('إلى', 'ها'), 'معه': ('مع', 'ه'), 'معها': ('مع', 'ها'), 'فإنه': ('ف', 'إنه'), 'فإنها': ('ف', 'إنها'), 'وإنه': ('و', 'إنه'), 'وإنها': ('و', 'إنها'), 'كأنه': ('كأن', 'ه'), 'كأنها': ('كأن', 'ها'), 'كأنهم': ('كأن', 'هم'), 'كأنما': ('كأن', 'ما'), 'منا': ('من', 'نا'), 'منهم': ('من', 'هم'), 'منكم': ('من', 'كم'), 'فلم': ('ف', 'لم'), 'ولم': ('و', 'لم'), 'فلا': ('ف', 'لا'), 'بصره': ('بصر', 'ه'), 'إثباته': ('إثبات', 'ه'), 'يصدمه': ('يصدم', 'ه') } PROCLITICS = ['فال', 'وال', 'كال', 'بال', 'لل', 'ال', 'و', 'ف', 'ب', 'ل', 'ك'] ENCLITICS = [ 'هما', 'كما', 'هم', 'هن', 'ها', 'نا', 'كم', 'كن', 'ات', 'ون', 'ين', 'تما', 'تم', 'تن', 'ية', 'ان', 'ه', 'ك', 'ي', 'ة', 'ت' ] AWZAN_TEMPLATES = [ (re.compile(r'^است([^\W\d_])([^\W\d_])ا([^\W\d_])$'), 'اِسْتِفْعَال', None), (re.compile(r'^(?:است|يست|نست|أست|تست)([^\W\d_])([^\W\d_])([^\W\d_])$'), 'اِسْتَفْعَلَ', None), (re.compile(r'^مست([^\W\d_])([^\W\d_])([^\W\d_])$'), 'مُسْتَفْعِل', None), (re.compile(r'^م([^\W\d_])([^\W\d_])ا([^\W\d_])$'), 'مِفْعَال', None), (re.compile(r'^م([^\W\d_])تا([^\W\d_])$'), 'مُفْتَعِل', 'hollow_w'), (re.compile(r'^[ايت]([^\W\d_])تا([^\W\d_])$'), 'اِفْتَعَلَ', 'hollow_w'), (re.compile(r'^ت([^\W\d_]{4})$'), 'تَفَعْلَلَ', 'quad'), (re.compile(r'^ي([^\W\d_]{4})$'), 'يُتَفَعْلَلُ', 'quad'), (re.compile(r'^م([^\W\d_]{4})$'), 'مُتَفَعْلِل', 'quad'), (re.compile(r'^ان([^\W\d_])([^\W\d_])ا([^\W\d_])$'), 'اِنْفِعَال', None), (re.compile(r'^ان([^\W\d_])([^\W\d_])([^\W\d_])$'), 'اِنْفَعَلَ', None), (re.compile(r'^ا([^\W\d_])ت([^\W\d_])ا([^\W\d_])$'), 'اِفْتِعَال', None), (re.compile(r'^ا([^\W\d_])ت([^\W\d_])([^\W\d_])$'), 'اِفْتَعَلَ', None), (re.compile(r'^م([^\W\d_])ت([^\W\d_])([^\W\d_])$'), 'مُفْتَعِل', None), (re.compile(r'^[يتأ]([^\W\d_])ت([^\W\d_])([^\W\d_])$'), 'يَفْتَعِلُ', None), (re.compile(r'^[يتنأ]ت([^\W\d_])([^\W\d_])([^\W\d_])$'), 'يَتَفَعَّلُ', None), (re.compile(r'^[يتنأ]ت([^\W\d_])ا([^\W\d_])([^\W\d_])$'), 'يَتَفَاعَلُ', None), (re.compile(r'^ت([^\W\d_])([^\W\d_])([^\W\d_])$'), 'تَفَعَّلَ', None), (re.compile(r'^ت([^\W\d_])ا([^\W\d_])([^\W\d_])$'), 'تَفَاعَلَ', None), (re.compile(r'^م([^\W\d_])([^\W\d_])([^\W\d_])$'), 'مَفْعَل', None), (re.compile(r'^م([^\W\d_])([^\W\d_])و([^\W\d_])$'), 'مَفْعُول', None), (re.compile(r'^م([^\W\d_])([^\W\d_])ي([^\W\d_])$'), 'مَفْعِيل', None), (re.compile(r'^م([^\W\d_])([^\W\d_])([^\W\d_])ة$'), 'مَفْعَلَة', None), (re.compile(r'^([^\W\d_])ا([^\W\d_])([^\W\d_])$'), 'فَاعِل', None), (re.compile(r'^([^\W\d_])ا([^\W\d_])([^\W\d_])ة$'), 'فَاعِلَة', None), (re.compile(r'^([^\W\d_])([^\W\d_])ي([^\W\d_])$'), 'فَعِيل', None), (re.compile(r'^([^\W\d_])([^\W\d_])ي([^\W\d_])ة$'), 'فَعِيلَة', None), (re.compile(r'^([^\W\d_])([^\W\d_])و([^\W\d_])$'), 'فُعُول', None), (re.compile(r'^([^\W\d_])([^\W\d_])ا([^\W\d_])$'), 'فِعَال', None), (re.compile(r'^([^\W\d_])([^\W\d_])ا([^\W\d_])ة$'), 'فِعَالَة', None), (re.compile(r'^([^\W\d_])([^\W\d_])([^\W\d_])ان$'), 'فُعْلَان', None), (re.compile(r'^أ([^\W\d_])([^\W\d_])([^\W\d_])$'), 'أَفْعَل', None), (re.compile(r'^أ([^\W\d_])([^\W\d_])ا([^\W\d_])$'), 'أَفْعَال', None), (re.compile(r'^إ([^\W\d_])([^\W\d_])ا([^\W\d_])$'), 'إِفْعَال', None), (re.compile(r'^[يتنأ]([^\W\d_])([^\W\d_])([^\W\d_])$'), 'يَفْعَلُ', None), (re.compile(r'^ت([^\W\d_])([^\W\d_])ي([^\W\d_])$'), 'تَفْعِيل', None), ] class BasranSyntacticRealizer: """Enforces Basran Syntactic Governance & Classical Mutation Rules.""" JUSSIVE_GOVERNORS = {'لم', 'لما', 'لا', 'إن', 'من', 'ما', 'مهما', 'حيثما', 'إذما', 'أينما', 'متى', 'أيان'} SUBJUNCTIVE_GOVERNORS = {'أن', 'لن', 'كي', 'إذن', 'حتى'} HOLLOW_JUSSIVE_MAP = { 'يكون': 'يكن', 'تكون': 'تكن', 'أكون': 'أكن', 'نكون': 'نكن', 'يقول': 'يقل', 'تقول': 'تقل', 'أقول': 'أقل', 'نقول': 'نقل', 'يسير': 'يسر', 'تسير': 'تسر', 'أسير': 'أسر', 'نسير': 'نسر', 'ينال': 'ينل', 'تنال': 'تنل', 'أنال': 'أنل', 'ننال': 'ننل', 'يزيد': 'يزد', 'تزيد': 'تزد', 'يستطيع': 'يستطع', 'تستطيع': 'تستطع', 'يستقيم': 'يستقم', 'تستقيم': 'تستقم', 'يبين': 'يبن', 'تبين': 'تبن', 'يحيط': 'يحط' } DEFECTIVE_JUSSIVE_MAP = { 'يدعو': 'يدع', 'تدعو': 'تدع', 'أدعو': 'أدع', 'ندعو': 'ندع', 'يرمي': 'يرم', 'ترمي': 'ترم', 'أرمي': 'أرم', 'نرمي': 'نرم', 'يسعى': 'يسع', 'تسعى': 'تسع', 'أسعى': 'أسع', 'نسعى': 'نسع', 'يبقى': 'يبق', 'تبقى': 'تبق', 'يرى': 'ير', 'ترى': 'تر', 'أرى': 'أر', 'نرى': 'نر', 'يأتي': 'يأت', 'تأتي': 'تأت', 'يهدي': 'يهد', 'تهدي': 'تهد', 'يقضي': 'يقض', 'تقضي': 'تقض', 'يخلو': 'يخل', 'تخلو': 'تخل' } @staticmethod def apply_word_internal_syntax(prefix: str, stem: str, suffix: str) -> str: p = prefix.strip() if prefix not in ['', '', '', ''] else '' s = suffix.strip() if suffix not in ['', '', '', ''] else '' w = stem.strip() if not w: return p + s if p.startswith('ال') or p in ['وال', 'فال', 'بال', 'كال', 'لل']: if s in ['اً', 'ا', 'ٍ', 'ٌ', 'ً']: s = '' if w.endswith('اً'): w = w[:-1] if s and s not in ['ون', 'ين', 'ان', 'ات', 'ة']: if w.endswith('ة'): w = w[:-1] + 'ت' if s in ['ه', 'ها', 'هم', 'هما', 'هن', 'ك', 'كم', 'كن', 'نا', 'ي']: if len(w) >= 5 and w.endswith('ون'): w = w[:-1] elif len(w) >= 5 and w.endswith('ين'): w = w[:-1] elif len(w) >= 5 and w.endswith('ان'): w = w[:-1] if s in ['ه', 'ها', 'هم', 'هما', 'هن', 'ك', 'كم', 'كن', 'نا', 'ي']: if w.endswith('اء'): base = w[:-2] w = base + 'اؤ' if p == '' else base + 'ائ' if p in ['ب', 'ل', 'في'] else base + 'اء' return p + w + s @classmethod def apply_contextual_syntax(cls, word_records: List[Dict[str, Any]]) -> List[str]: n = len(word_records) realized_words = [] for i in range(n): rec = word_records[i] prefix = rec.get('prefix', '') root = rec.get('root', '') wazn = rec.get('wazn', '') suffix = rec.get('suffix', '') stem = rec.get('stem', '') or root if prefix and stem.startswith(prefix): stem = stem[len(prefix):] if suffix and stem.endswith(suffix): stem = stem[:-len(suffix)] prev_word = realized_words[i - 1] if i > 0 else '' next_rec = word_records[i + 1] if i + 1 < n else None # Jussive Truncation if prev_word in cls.JUSSIVE_GOVERNORS or prefix in cls.JUSSIVE_GOVERNORS: if stem in cls.HOLLOW_JUSSIVE_MAP: stem = cls.HOLLOW_JUSSIVE_MAP[stem] elif stem in cls.DEFECTIVE_JUSSIVE_MAP: stem = cls.DEFECTIVE_JUSSIVE_MAP[stem] elif suffix == 'ون': suffix = 'وا' elif suffix == 'ين' and (stem.startswith('ت') or stem.startswith('ي')): suffix = 'ي' elif prev_word in cls.SUBJUNCTIVE_GOVERNORS or prefix in cls.SUBJUNCTIVE_GOVERNORS: if suffix == 'ون': suffix = 'وا' elif suffix == 'ين' and (stem.startswith('ت') or stem.startswith('ي')): suffix = 'ي' # Annexation Nūn Deletion if next_rec is not None: next_p = next_rec.get('prefix', '') next_w = next_rec.get('stem', '') or next_rec.get('root', '') next_is_annexed = ( next_p.startswith('ال') or next_p in ['بال', 'كال', 'لل', 'فال', 'وال'] or next_w.startswith('ال') or (not next_p and len(next_w) >= 3 and next_w not in cls.JUSSIVE_GOVERNORS) ) if next_is_annexed: if suffix == 'ون': suffix = 'و' elif suffix == 'ين' and not (stem.startswith('ي') or stem.startswith('ت')): suffix = 'ي' elif suffix == 'ان': suffix = 'ا' elif not suffix and len(stem) >= 5 and not root.endswith('ن'): if stem.endswith('ون'): stem = stem[:-1] elif stem.endswith('ين') and not (stem.startswith('ي') or stem.startswith('ت')): stem = stem[:-1] elif stem.endswith('ان'): stem = stem[:-1] word = cls.apply_word_internal_syntax(prefix, stem, suffix) realized_words.append(word) return realized_words class RootformerTokenizer(PreTrainedTokenizer): """ Rootformer Official Farāhīdian-Basran Morphemic Tokenizer. Inherits from Hugging Face PreTrainedTokenizer. """ vocab_files_names = {"vocab_file": "nrmp_vocab.json"} model_input_names = ["input_ids", "attention_mask", "p_ids", "r_ids", "w_ids", "s_ids"] def __init__( self, vocab_file: Optional[str] = None, bos_token: str = "", eos_token: str = "", unk_token: str = "", pad_token: str = "", model_max_length: int = 2048, **kwargs ): self.vocab_file = vocab_file self._load_vocabulary(vocab_file) super().__init__( bos_token=bos_token, eos_token=eos_token, unk_token=unk_token, pad_token=pad_token, model_max_length=model_max_length, **kwargs ) # Integrated Classical Morphological Analyzer try: from .morph_analyzer import ArabicMorphAnalyzer self.analyzer = ArabicMorphAnalyzer(vocab_path=self.vocab_file) except Exception: try: from morph_analyzer import ArabicMorphAnalyzer self.analyzer = ArabicMorphAnalyzer(vocab_path=self.vocab_file) except Exception: self.analyzer = None def analyze(self, text_or_word: str) -> Union[Dict[str, Any], List[Dict[str, Any]]]: """ Deep morphological analysis of Arabic text or single word. Returns detailed grammatical breakdown (root, wazn, prefix, suffix, type). """ if self.analyzer is None: try: from .morph_analyzer import ArabicMorphAnalyzer except Exception: from morph_analyzer import ArabicMorphAnalyzer self.analyzer = ArabicMorphAnalyzer(vocab_path=self.vocab_file) words = text_or_word.strip().split() if len(words) == 1: return self.analyzer.analyze_word(words[0]) return self.analyzer.analyze_sentence(text_or_word) def extract_roots(self, text: str) -> List[str]: """Extracts sequence of classical consonantal roots from text.""" if self.analyzer is None: try: from .morph_analyzer import ArabicMorphAnalyzer except Exception: from morph_analyzer import ArabicMorphAnalyzer self.analyzer = ArabicMorphAnalyzer(vocab_path=self.vocab_file) return self.analyzer.extract_roots(text) def _load_vocabulary(self, vocab_file: Optional[str]): data = None candidates = [] if vocab_file and Path(vocab_file).exists(): candidates.append(Path(vocab_file)) # Local defaults this_dir = Path(__file__).parent.resolve() candidates.extend([ this_dir / "nrmp_vocab.json", this_dir / "data" / "nrmp_vocab.json", Path("/workspace/rootformer_v12/v18_next_root_morph/data/nrmp_vocab.json"), Path("/home/absolut7/.gemini/antigravity-ide/scratch/rootformer/v12/v18_next_root_morph/data/nrmp_vocab.json") ]) for c in candidates: if c.exists(): try: with open(c, 'r', encoding='utf-8') as f: data = json.load(f) self.vocab_file = str(c) break except Exception: pass if data is None: # Fallback embedded minimal lists roots = ['', '', '', '', ''] awzan = ['', '', '', 'فَعَلَ', 'فَاعِل', 'فَعْل', 'مَفْعُول', 'اِسْتَفْعَالَ'] prefixes = ['', '', '', 'ال', 'و', 'ف', 'ب', 'ل', 'ك'] suffixes = ['', '', '', 'ه', 'ها', 'هم', 'نا', 'ي', 'ة', 'ون', 'ين'] else: roots = data.get('roots', []) awzan = data.get('awzan', []) prefixes = data.get('prefixes', []) suffixes = data.get('suffixes', []) self.roots_list = roots self.awzan_list = awzan self.canonical_prefixes = prefixes self.canonical_suffixes = suffixes self.root2id = {r: i for i, r in enumerate(self.roots_list)} self.id2root = {i: r for i, r in enumerate(self.roots_list)} self.wazn2id = {w: i for i, w in enumerate(self.awzan_list)} self.id2wazn = {i: w for i, w in enumerate(self.awzan_list)} self.prefix2id = {p: i for i, p in enumerate(self.canonical_prefixes)} self.id2prefix = {i: p for i, p in enumerate(self.canonical_prefixes)} self.suffix2id = {s: i for i, s in enumerate(self.canonical_suffixes)} self.id2suffix = {i: s for i, s in enumerate(self.canonical_suffixes)} # Constants self.PAD_ROOT = self.root2id.get('', 0) self.BOS_ROOT = self.root2id.get('', 1) self.EOS_ROOT = self.root2id.get('', 2) self.UNK_ROOT = self.root2id.get('', 3) self.PAD_WAZN = self.wazn2id.get('', 0) self.NONE_WAZN = self.wazn2id.get('', 1) self.PAD_PREFIX = self.prefix2id.get('', 0) self.NONE_PREFIX = self.prefix2id.get('', 1) self.PAD_SUFFIX = self.suffix2id.get('', 0) self.NONE_SUFFIX = self.suffix2id.get('', 1) self.roots_set = set(self.roots_list) self.awzan_set = set(self.awzan_list) # Common particles list self.COMMON_PARTICLES = [ 'في', 'من', 'إلى', 'على', 'عن', 'مع', 'حتى', 'منذ', 'مذ', 'إن', 'أن', 'كأن', 'لكن', 'ليت', 'لعل', 'لا', 'ما', 'لم', 'لن', 'ليس', 'كان', 'هو', 'هي', 'هما', 'هم', 'هن', 'أنت', 'أنا', 'نحن', 'هذا', 'هذه', 'هؤلاء', 'ذلك', 'تلك', 'أولئك', 'الذي', 'التي', 'الذين', 'ثم', 'أو', 'بل', 'أم', 'إما', 'إذا', 'إذ', 'كل', 'بعض', 'غير', 'سوى', 'عند', 'بين', 'فوق', 'تحت', 'قبل', 'بعد', 'حيث', 'كيف', 'أين', 'متى', 'كم', 'أي', 'نعم', 'بلى', 'إذن', 'قد', 'سوف' ] @property def vocab_size(self) -> int: return len(self.roots_list) def get_vocab(self) -> Dict[str, int]: return dict(self.root2id) def clean_arabic(self, word: str) -> str: no_tashkeel = re.sub(r'[\u064B-\u065F\u0670\u0640]', '', word) no_punct = re.sub(r'[،؛؟.:!«»\(\)\[\]\{\}…"\']', ' ', no_tashkeel) return re.sub(r'\s+', ' ', no_punct).strip() def decompose_arabic_word(self, word: str) -> Tuple[Optional[str], Optional[str], Optional[str], Optional[str]]: clean = self.clean_arabic(word) if not clean: return None, None, None, None if clean in CLOSED_PARTICLES: return None, clean, None, None if clean in COMPOUND_PARTICLES: p, part = COMPOUND_PARTICLES[clean] return p, part, None, None if len(clean) == 3 and clean in self.roots_set: return None, clean, 'فَعَلَ', None if len(clean) == 4 and clean in self.roots_set: return None, clean, 'فَعْلَلَ', None # Philosophical and theological canonical exceptions if clean in ['ماهية', 'الماهية']: p = 'ال' if clean.startswith('ال') else None return p, 'موه', 'فَاعِلَة', None if clean in ['أعضاء', 'اعضاء', 'أعضاؤه', 'أعضائه']: s = 'ه' if clean.endswith('ه') else None return None, 'عضا', 'أَفْعَال', s if clean in ['أحشاء', 'احشاء', 'أحشاؤه', 'أحشائه']: s = 'ه' if clean.endswith('ه') else None return None, 'حشا', 'أَفْعَال', s if clean in ['هواء', 'الهواء', 'خلاء', 'الخلاء']: p = 'ال' if clean.startswith('ال') else None r = 'هوا' if 'هواء' in clean else 'خلا' return p, r, 'فَعَال', None # Clitic stripping article_prefixes = [p for p in ['فال', 'وال', 'كال', 'بال', 'لل', 'ال'] if clean.startswith(p)] single_prefixes = [p for p in ['و', 'ف', 'ب', 'ل', 'ك'] if clean.startswith(p)] candidate_prefixes = article_prefixes + [''] + single_prefixes cand_s_list = sorted([s for s in ENCLITICS if clean.endswith(s) and s != ''], key=len, reverse=True) candidate_suffixes = [''] + cand_s_list for p in candidate_prefixes: rem_p = clean[len(p):] if p else clean for s in candidate_suffixes: stem = rem_p[:-len(s)] if s and rem_p.endswith(s) else (rem_p if not s else '') if not stem: continue # Bare stem in roots if len(stem) == 3: if stem in self.roots_set: return (p or None), stem, 'فَعَلَ', (s or None) stem_bare = stem.replace('أ', 'ا').replace('إ', 'ا').replace('آ', 'ا') if stem_bare in self.roots_set: return (p or None), stem_bare, 'فَعَلَ', (s or None) if stem in ROOT_CANONICAL_MAP and ROOT_CANONICAL_MAP[stem] in self.roots_set: return (p or None), ROOT_CANONICAL_MAP[stem], 'فَعَلَ', (s or None) # Doubled root (2-letter stem, e.g. عل -> علل, حق -> حقق, بد -> بدد, كل -> كلل) if len(stem) == 2: doubled = stem + stem[-1] if doubled in self.roots_set: return (p or None), doubled, 'فَعَلَ', (s or None) # Doubled active participle (تام -> تمم, عام -> عمم, خاص -> خصص, ضار -> ضرر) if len(stem) == 3 and stem[1] == 'ا': cand_doubled = stem[0] + stem[2] + stem[2] if cand_doubled in self.roots_set: return (p or None), cand_doubled, 'فَاعِل', (s or None) # Hollow imperfect / jussive (يكن -> كون, يقول -> قول, يقل -> قول, يسير -> سير) if len(stem) == 3 and stem[0] in ['ي', 'ت', 'ن', 'أ']: cand_hollow_w = stem[1] + 'و' + stem[2] if cand_hollow_w in self.roots_set: return (p or None), cand_hollow_w, 'يَفْعُلُ', (s or None) cand_hollow_y = stem[1] + 'ي' + stem[2] if cand_hollow_y in self.roots_set: return (p or None), cand_hollow_y, 'يَفْعِلُ', (s or None) cand_gem = stem[1] + stem[2] + stem[2] if cand_gem in self.roots_set: return (p or None), cand_gem, 'يَفْعُلُ', (s or None) # Mithāl Wāwī (يجب -> وجب, يجد -> وجد, يصف -> وصف, يعد -> وعد) if len(stem) == 3 and stem[0] in ['ي', 'ت', 'ن', 'أ']: mithal = 'و' + stem[1:] if mithal in self.roots_set: return (p or None), mithal, 'يَعِلُ', (s or None) for pattern, wazn_name, note in AWZAN_TEMPLATES: m = pattern.match(stem) if m: radicals = list(m.groups()) if note == 'quad': cand_root = radicals[0] elif note == 'hollow_w': cand_root = radicals[0] + 'و' + radicals[1] else: cand_root = ''.join(radicals) cand_root = ROOT_CANONICAL_MAP.get(cand_root, cand_root) if cand_root in self.roots_set: return (p or None), cand_root, wazn_name, (s or None) # Broken plurals and assimilation if len(stem) == 4 and stem[1] == 'ا': r_hollow = stem[0] + 'و' + stem[2:] if r_hollow in self.roots_set: return (p or None), r_hollow, 'فَاعِل', (s or None) r_hollow_y = stem[0] + 'ي' + stem[2:] if r_hollow_y in self.roots_set: return (p or None), r_hollow_y, 'فَاعِل', (s or None) return None, clean, None, None def realize_root_and_wazn(self, root: str, wazn: str) -> str: if root == 'اصل': return 'أصل' if root == 'شيا': return 'شيء' if root == 'خلا' and wazn in ['يَفْعَلُ', 'يَفْعُلُ']: return 'يخلو' if root == 'عيا': return 'أعيان' if wazn == 'أَفْعَال' else 'عين' if root in ['بدا', 'بدأ'] and wazn == 'اِفْتِعَال': return 'ابتداء' if wazn == 'مُتَفَعِّل' and len(root) == 3: return 'مت' + root[0] + root[1] + root[2] if wazn == 'يَفْتَعِلُ': if root == 'حوج': return 'يحتاج' elif root == 'جمع': return 'تجتمع' elif len(root) == 3: return 'ي' + root[0] + 'ت' + root[1] + root[2] if wazn == 'اِفْتَعَلَ' and len(root) == 3: return 'امتنع' if root == 'منع' else 'ا' + root[0] + 'ت' + root[1] + root[2] r = ROOT_CANONICAL_MAP_REV.get(root, root) if wazn == 'فِعَل': return r elif wazn == 'فَعَلَ': if len(r) == 3 and r[1] == r[2] and r not in {'سبب', 'ملل', 'طلل'}: return r[:2] return r elif wazn == 'فُعُول' and len(r) == 3: return r[0] + r[1] + 'و' + r[2] elif wazn == 'فَعِيل' and len(r) == 3: return r[0] + r[1] + 'ي' + r[2] elif wazn == 'فَاعِل': if len(r) == 3 and r[1] == r[2]: return r[0] + 'ا' + r[1] elif len(r) == 3: return r[0] + 'ا' + r[1] + r[2] elif wazn == 'فَاعِلَة': if r == 'موه': return 'ماهية' elif len(r) == 3: return r[0] + 'ا' + r[1] + r[2] + 'ة' elif wazn == 'مَفْعَل' and len(r) == 3: return 'م' + r[0] + r[1] + r[2] elif wazn in ['مِفْعَال', 'مَفْعَال'] and len(r) == 3: return 'م' + r[0] + r[1] + 'ا' + r[2] elif wazn == 'مَفْعُول' and len(r) == 3: return 'م' + r[0] + r[1] + 'و' + r[2] elif wazn == 'فِعَال': if len(r) == 3 and (r[2] in ['ي', 'و'] or r in ['شفي', 'سمو', 'بني', 'قضي']): return r[0] + r[1] + 'اء' elif len(r) == 3: return r[0] + r[1] + 'ا' + r[2] elif wazn == 'أَفْعَال' and len(r) == 3: return 'أ' + r[0] + r[1] + 'ا' + r[2] elif wazn == 'أَفْعَلَ' and len(r) == 3: return 'أ' + r[0] + r[1] + r[2] elif wazn == 'تَفْعِيل' and len(r) == 3: return 'ت' + r[0] + r[1] + 'ي' + r[2] elif wazn == 'اِفْتِعَال' and len(r) == 3: return 'ا' + r[0] + 'ت' + r[1] + 'ا' + r[2] return r def encode_word(self, word: str) -> Tuple[int, int, int, int]: clean_w = word.strip() if not clean_w: return (self.NONE_PREFIX, self.UNK_ROOT, self.NONE_WAZN, self.NONE_SUFFIX) particle_tag = f'' if particle_tag in self.root2id: return (self.NONE_PREFIX, self.root2id[particle_tag], self.NONE_WAZN, self.NONE_SUFFIX) p, r, wz, s = self.decompose_arabic_word(clean_w) p_id = self.NONE_PREFIX if not p else self.prefix2id.get(p, self.prefix2id.get('', 2)) if not r: r_id = self.root2id.get(f'', self.root2id.get('', self.UNK_ROOT)) else: particle_cand = f'' if particle_cand in self.root2id: r_id = self.root2id[particle_cand] else: r_id = self.root2id.get(r, self.UNK_ROOT) w_id = self.NONE_WAZN if not wz else self.wazn2id.get(wz, self.wazn2id.get('', 2)) s_id = self.NONE_SUFFIX if not s else self.suffix2id.get(s, self.suffix2id.get('', 2)) return (p_id, r_id, w_id, s_id) def decode_word(self, p_id: int, r_id: int, w_id: int, s_id: int) -> str: if r_id in [self.PAD_ROOT, self.BOS_ROOT, self.EOS_ROOT]: return '' root_str = self.id2root.get(r_id, '') wazn_str = self.id2wazn.get(w_id, '') prefix_str = self.id2prefix.get(p_id, '') suffix_str = self.id2suffix.get(s_id, '') p_text = '' if prefix_str in ['', '', ''] else prefix_str s_text = '' if suffix_str in ['', '', ''] else suffix_str if root_str.startswith('', '', '']: return p_text + root_str + s_text if wazn_str not in ['', '', '']: stem = self.realize_root_and_wazn(root_str, wazn_str) else: stem = root_str return p_text + stem + s_text def encode_morphemes(self, sentence: str) -> List[Tuple[int, int, int, int]]: words = sentence.strip().split() return [self.encode_word(w) for w in words] def decode_morphemes(self, word_tuples: List[Tuple[int, int, int, int]], apply_syntax: bool = True) -> str: if not word_tuples: return "" if not apply_syntax: words = [self.decode_word(*t) for t in word_tuples] return " ".join([w for w in words if w]) # Prepare records for BasranSyntacticRealizer records = [] for t in word_tuples: p_id, r_id, w_id, s_id = t if r_id in [self.PAD_ROOT, self.BOS_ROOT, self.EOS_ROOT]: continue r_str = self.id2root.get(r_id, '') w_str = self.id2wazn.get(w_id, '') p_str = self.id2prefix.get(p_id, '') s_str = self.id2suffix.get(s_id, '') p_val = '' if p_str in ['', '', ''] else p_str s_val = '' if s_str in ['', '', ''] else s_str if r_str.startswith('', '', '']: stem = self.realize_root_and_wazn(r_str, w_str) else: stem = r_str records.append({ 'prefix': p_val, 'root': r_str, 'wazn': w_str, 'suffix': s_val, 'stem': stem }) realized = BasranSyntacticRealizer.apply_contextual_syntax(records) return " ".join([w for w in realized if w]) def _tokenize(self, text: str) -> List[str]: return text.strip().split() def _convert_token_to_id(self, token: str) -> int: p, r, w, s = self.encode_word(token) return r def _convert_id_to_token(self, index: int) -> str: return self.id2root.get(index, self.unk_token) def convert_tokens_to_string(self, tokens: List[str]) -> str: return " ".join(tokens) def __call__( self, text: Union[str, List[str]], text_pair: Optional[Any] = None, return_tensors: Optional[str] = None, padding: bool = False, truncation: bool = False, max_length: Optional[int] = None, add_special_tokens: bool = True, **kwargs ) -> BatchEncoding: if isinstance(text, str): texts = [text] is_single = True else: texts = list(text) is_single = False batch_p, batch_r, batch_w, batch_s = [], [], [], [] for t in texts: tuples = self.encode_morphemes(t) if add_special_tokens: tuples = [(self.NONE_PREFIX, self.BOS_ROOT, self.NONE_WAZN, self.NONE_SUFFIX)] + \ tuples + \ [(self.NONE_PREFIX, self.EOS_ROOT, self.NONE_WAZN, self.NONE_SUFFIX)] if truncation and max_length and len(tuples) > max_length: tuples = tuples[:max_length] batch_p.append([x[0] for x in tuples]) batch_r.append([x[1] for x in tuples]) batch_w.append([x[2] for x in tuples]) batch_s.append([x[3] for x in tuples]) # Padding max_len = max(len(seq) for seq in batch_r) if batch_r else 0 if max_length and padding: max_len = max(max_len, max_length) attn_mask = [] for i in range(len(batch_r)): cur_len = len(batch_r[i]) pad_len = max_len - cur_len attn_mask.append([1] * cur_len + [0] * pad_len) batch_p[i] = batch_p[i] + [self.PAD_PREFIX] * pad_len batch_r[i] = batch_r[i] + [self.PAD_ROOT] * pad_len batch_w[i] = batch_w[i] + [self.PAD_WAZN] * pad_len batch_s[i] = batch_s[i] + [self.PAD_SUFFIX] * pad_len # input_ids corresponds to primary ontological radical stream (r_ids) data = { "input_ids": batch_r, "attention_mask": attn_mask, "p_ids": batch_p, "r_ids": batch_r, "w_ids": batch_w, "s_ids": batch_s } if return_tensors == "pt": data = {k: torch.tensor(v, dtype=torch.long) for k, v in data.items()} return BatchEncoding(data) def decode( self, token_ids: Union[int, List[int], torch.Tensor, List[Tuple[int, int, int, int]]], skip_special_tokens: bool = True, clean_up_tokenization_spaces: bool = False, apply_syntax: bool = True, **kwargs ) -> str: if isinstance(token_ids, torch.Tensor): token_ids = token_ids.tolist() if isinstance(token_ids, int): token_ids = [token_ids] if not token_ids: return "" # Check if list of 4-tuples if isinstance(token_ids[0], (list, tuple)) and len(token_ids[0]) == 4: tuples = token_ids else: # Fallback from root IDs: assume NONE prefix, NONE wazn, NONE suffix tuples = [(self.NONE_PREFIX, rid, self.NONE_WAZN, self.NONE_SUFFIX) for rid in token_ids] if skip_special_tokens: tuples = [t for t in tuples if t[1] not in [self.PAD_ROOT, self.BOS_ROOT, self.EOS_ROOT]] return self.decode_morphemes(tuples, apply_syntax=apply_syntax) def batch_decode( self, sequences: Union[List[List[int]], torch.Tensor], skip_special_tokens: bool = True, clean_up_tokenization_spaces: bool = False, apply_syntax: bool = True, **kwargs ) -> List[str]: if isinstance(sequences, torch.Tensor): sequences = sequences.tolist() return [ self.decode( seq, skip_special_tokens=skip_special_tokens, clean_up_tokenization_spaces=clean_up_tokenization_spaces, apply_syntax=apply_syntax, **kwargs ) for seq in sequences ] def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]: os.makedirs(save_directory, exist_ok=True) fname = "nrmp_vocab.json" if filename_prefix: fname = f"{filename_prefix}-{fname}" out_path = Path(save_directory) / fname data = { "roots": self.roots_list, "awzan": self.awzan_list, "prefixes": self.canonical_prefixes, "suffixes": self.canonical_suffixes, "counts": { "roots": len(self.roots_list), "awzan": len(self.awzan_list), "prefixes": len(self.canonical_prefixes), "suffixes": len(self.canonical_suffixes) } } with open(out_path, "w", encoding="utf-8") as f: json.dump(data, f, ensure_ascii=False, indent=2) return (str(out_path),)