rootformer-v18-basran-transmute / tokenization_rootformer.py
enver's picture
Add classical Arabic morphological analyzer integration (tokenization_rootformer.py)
a952d1d verified
Raw History Blame Contribute Delete
39 kB
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
tokenization_rootformer.py
Rootformer Farฤhฤซdian-Basran Morphemic PreTrainedTokenizer.
Implements the official Hugging Face PreTrainedTokenizer for Rootformer.
Provides:
- 100% BPE-Free Farฤhฤซdian radical decomposition (prefix, root, wazn, suffix)
- Basran syntactic harmony and 7 classical Arabic mutation rules (Al-Naแธฅw wa-l-Iสฟrฤb)
- Full compatibility with AutoTokenizer.from_pretrained(..., trust_remote_code=True)
- Dual-mode outputs: standard input_ids + attention_mask, plus p_ids, r_ids, w_ids, s_ids
"""
import os
import re
import json
from pathlib import Path
from typing import List, Tuple, Dict, Any, Optional, Union
import torch
from transformers import PreTrainedTokenizer
from transformers.tokenization_utils_base import BatchEncoding
# Canonical Classical Root Aliases
ROOT_CANONICAL_MAP = {
'ู‚ุฏู…': 'ุชู‚ุฏู…', 'ุฌุฒุฃ': 'ุฌุฒุง', 'ุฌุฒุก': 'ุฌุฒุง', 'ุจุฏุฃ': 'ุจุฏุง', 'ุจุฏุก': 'ุจุฏุง',
'ุฃู„ู‡': 'ุงู„ู‡', 'ุกู„ู‡': 'ุงู„ู‡', 'ู…ูˆู‡': 'ู…ูˆู‡', 'ู‚ุจู„': 'ู‚ุจุง', 'ู†ู‡ุง': 'ู†ู‡ูŠ',
'ุฃู…ุฑ': 'ุงู…ุฑ', 'ุฃุญุฏ': 'ุงุญุฏ', 'ูŠู‚ู†': 'ูŠู‚ู†', 'ุดููŠ': 'ุดููŠ', 'ุดูุก': 'ุดููŠ',
'ุบูŠูŠ': 'ุบูŠูŠ', 'ุฌู†ูŠ': 'ุฌู†ูŠ', 'ุฌู†ู‰': 'ุฌู†ูŠ', 'ุฃุตู„': 'ุงุตู„', 'ุงุตู„': 'ุงุตู„',
'ุดูŠุก': 'ุดูŠุง', 'ุฎู„ูˆ': 'ุฎู„ุง', 'ุนูŠู†': 'ุนูŠุง', 'ูุชุญ': 'ุชูุญ', 'ุฃู…ู„': 'ุงู…ู„',
'ุงู…ู„': 'ุงู…ู„', 'ุนุถูˆ': 'ุนุถุง', 'ุนุถูŠ': 'ุนุถุง', 'ุญุดูˆ': 'ุญุดุง', 'ุญุดูŠ': 'ุญุดุง',
'ู‡ูˆูŠ': 'ู‡ูˆุง'
}
ROOT_CANONICAL_MAP_REV = {
'ุชู‚ุฏู…': 'ู‚ุฏู…', 'ุฌุฒุง': 'ุฌุฒุฃ', 'ุจุฏุง': 'ุจุฏุฃ', 'ุงู„ู‡': 'ุฃู„ู‡', 'ู‚ุจุง': 'ู‚ุจู„',
'ู…ูˆู‡': 'ู…ูˆู‡', 'ู†ู‡ูŠ': 'ู†ู‡ูŠ', 'ุชูุญ': 'ูุชุญ', 'ุงู…ู„': 'ุฃู…ู„', 'ุนุถุง': 'ุนุถูˆ',
'ุญุดุง': 'ุญุดุง', 'ู‡ูˆุง': 'ู‡ูˆูŠ'
}
CLOSED_PARTICLES = {
'ุฅู†ู…ุง', 'ุฅู„ู‰', 'ุนู†', 'ุนู„ู‰', 'ููŠ', 'ู…ู†', 'ุญุชู‰', 'ุฅู†', 'ุฃู†', 'ูƒุฃู†', 'ู„ูƒู†', 'ู„ูŠุช', 'ู„ุนู„',
'ุฅู„ุง', 'ุบูŠุฑ', 'ุฅุฐุง', 'ุฅุฐ', 'ุญูŠุซ', 'ุซู…', 'ุฃูˆ', 'ุฃู…', 'ุจู„', 'ู„ุง', 'ู…ุง', 'ู„ู…', 'ู„ู†', 'ู„ูŠุณ',
'ู‡ุฐุง', 'ู‡ุฐู‡', 'ุฐู„ูƒ', 'ุชู„ูƒ', 'ู‡ุคู„ุงุก', 'ุฃูˆู„ุฆูƒ', 'ุงู„ุฐูŠ', 'ุงู„ุชูŠ', 'ุงู„ุฐูŠู†', 'ุงู„ู„ูˆุงุชูŠ',
'ู‡ูˆ', 'ู‡ูŠ', 'ุฃู†ุช', 'ุฃู†ุชู…', 'ุฃู†ุง', 'ู†ุญู†', 'ู‚ุฏ', 'ุณูˆู', 'ู†ุนู…', 'ุจุฆุณ', 'ุญุจุฐุง',
'ู…ุน', 'ุนู†ุฏ', 'ู‚ุจู„', 'ุจุนุฏ', 'ุฏูˆู†', 'ุจูŠู†', 'ุฃูŠุถุง', 'ูู‚ุท', 'ู‚ุท', 'ุญูŠุซู…ุง', 'ูƒูŠู', 'ุฃูŠู†', 'ู…ุชู‰',
'ูƒุฐู„ูƒ', 'ุฅุฐู†', 'ู„ูˆ', 'ู„ูˆู„ุง', 'ุฅู…ุง', 'ู‡ู„', 'ู„ู…ุง'
}
COMPOUND_PARTICLES = {
'ุจู„ุง': ('ุจ', 'ู„ุง'), 'ุจุบูŠุฑ': ('ุจ', 'ุบูŠุฑ'), 'ูˆู„ู…ุง': ('ูˆ', 'ู„ู…ุง'), 'ูู„ู…ุง': ('ู', 'ู„ู…ุง'),
'ูƒู„ู…ุง': ('ูƒู„', 'ู…ุง'), 'ูู‡ูˆ': ('ู', 'ู‡ูˆ'), 'ูู‡ูŠ': ('ู', 'ู‡ูŠ'), 'ูˆู‡ูˆ': ('ูˆ', 'ู‡ูˆ'),
'ูˆู‡ูŠ': ('ูˆ', 'ู‡ูŠ'), 'ูุฅู†': ('ู', 'ุฅู†'), 'ูˆุฅู†': ('ูˆ', 'ุฅู†'), 'ู„ุฃู†': ('ู„', 'ุฃู†'),
'ุนู†ู‡ุง': ('ุนู†', 'ู‡ุง'), 'ุนู„ูŠู‡ุง': ('ุนู„ู‰', 'ู‡ุง'), 'ู…ู†ู‡ุง': ('ู…ู†', 'ู‡ุง'), 'ููŠู‡ุง': ('ููŠ', 'ู‡ุง'),
'ุจู‡': ('ุจ', 'ู‡'), 'ุจู‡ุง': ('ุจ', 'ู‡ุง'), 'ู„ู‡': ('ู„', 'ู‡'), 'ู„ู‡ุง': ('ู„', 'ู‡ุง'),
'ูู„ู‡': ('ู', 'ู„ู‡'), 'ูˆู„ู‡': ('ูˆ', 'ู„ู‡'), 'ุฃู†ู‡ุง': ('ุฃู†', 'ู‡ุง'), 'ุฅู†ู‡ุง': ('ุฅู†', 'ู‡ุง'),
'ุฃู†ู‡': ('ุฃู†', 'ู‡'), 'ุฅู†ู‡': ('ุฅู†', 'ู‡'), 'ุฃู†ู‡ู…': ('ุฃู†', 'ู‡ู…'), 'ุฅู†ู‡ู…': ('ุฅู†', 'ู‡ู…'),
'ู„ูŠุณุช': ('ู„ูŠุณ', 'ุช'), 'ู…ู…ุง': ('ู…ู†', 'ู…ุง'), 'ุนู…ุง': ('ุนู†', 'ู…ุง'), 'ุจู…ุง': ('ุจ', 'ู…ุง'),
'ูˆู„ุง': ('ูˆ', 'ู„ุง'), 'ู„ุฃู†ู‡': ('ู„', 'ุฃู†ู‡'), 'ูู„ุฃู†ู‡': ('ู', 'ู„ุฃู†ู‡'), 'ูˆุฅู…ุง': ('ูˆ', 'ุฅู…ุง'),
'ูุฅู…ุง': ('ู', 'ุฅู…ุง'), 'ุจุฐุงุชู‡': ('ุจ', 'ุฐุงุช'), 'ุจุฐุงุชู‡ุง': ('ุจ', 'ุฐุงุช'), 'ู„ุฐุงุชู‡': ('ู„', 'ุฐุงุช'),
'ู„ุฐุงุชู‡ุง': ('ู„', 'ุฐุงุช'), 'ุจุบูŠุฑู‡': ('ุจ', 'ุบูŠุฑ'), 'ู„ุบูŠุฑู‡': ('ู„', 'ุบูŠุฑ'), 'ุนู†ู‡': ('ุนู†', 'ู‡'),
'ู…ู†ู‡': ('ู…ู†', 'ู‡'), 'ููŠู‡': ('ููŠ', 'ู‡'), 'ุนู„ูŠู‡': ('ุนู„ู‰', 'ู‡'), 'ุฅู„ูŠู‡': ('ุฅู„ู‰', 'ู‡'),
'ุฅู„ูŠู‡ุง': ('ุฅู„ู‰', 'ู‡ุง'), 'ู…ุนู‡': ('ู…ุน', 'ู‡'), 'ู…ุนู‡ุง': ('ู…ุน', 'ู‡ุง'), 'ูุฅู†ู‡': ('ู', 'ุฅู†ู‡'),
'ูุฅู†ู‡ุง': ('ู', 'ุฅู†ู‡ุง'), 'ูˆุฅู†ู‡': ('ูˆ', 'ุฅู†ู‡'), 'ูˆุฅู†ู‡ุง': ('ูˆ', 'ุฅู†ู‡ุง'), 'ูƒุฃู†ู‡': ('ูƒุฃู†', 'ู‡'),
'ูƒุฃู†ู‡ุง': ('ูƒุฃู†', 'ู‡ุง'), 'ูƒุฃู†ู‡ู…': ('ูƒุฃู†', 'ู‡ู…'), 'ูƒุฃู†ู…ุง': ('ูƒุฃู†', 'ู…ุง'), 'ู…ู†ุง': ('ู…ู†', 'ู†ุง'),
'ู…ู†ู‡ู…': ('ู…ู†', 'ู‡ู…'), 'ู…ู†ูƒู…': ('ู…ู†', 'ูƒู…'), 'ูู„ู…': ('ู', 'ู„ู…'), 'ูˆู„ู…': ('ูˆ', 'ู„ู…'),
'ูู„ุง': ('ู', 'ู„ุง'), 'ุจุตุฑู‡': ('ุจุตุฑ', 'ู‡'), 'ุฅุซุจุงุชู‡': ('ุฅุซุจุงุช', 'ู‡'), 'ูŠุตุฏู…ู‡': ('ูŠุตุฏู…', 'ู‡')
}
PROCLITICS = ['ูุงู„', 'ูˆุงู„', 'ูƒุงู„', 'ุจุงู„', 'ู„ู„', 'ุงู„', 'ูˆ', 'ู', 'ุจ', 'ู„', 'ูƒ']
ENCLITICS = [
'ู‡ู…ุง', 'ูƒู…ุง', 'ู‡ู…', 'ู‡ู†', 'ู‡ุง', 'ู†ุง', 'ูƒู…', 'ูƒู†', 'ุงุช', 'ูˆู†', 'ูŠู†', 'ุชู…ุง', 'ุชู…', 'ุชู†',
'ูŠุฉ', 'ุงู†', 'ู‡', 'ูƒ', 'ูŠ', 'ุฉ', 'ุช'
]
AWZAN_TEMPLATES = [
(re.compile(r'^ุงุณุช([^\W\d_])([^\W\d_])ุง([^\W\d_])$'), 'ุงูุณู’ุชููู’ุนูŽุงู„', None),
(re.compile(r'^(?:ุงุณุช|ูŠุณุช|ู†ุณุช|ุฃุณุช|ุชุณุช)([^\W\d_])([^\W\d_])([^\W\d_])$'), 'ุงูุณู’ุชูŽูู’ุนูŽู„ูŽ', None),
(re.compile(r'^ู…ุณุช([^\W\d_])([^\W\d_])([^\W\d_])$'), 'ู…ูุณู’ุชูŽูู’ุนูู„', None),
(re.compile(r'^ู…([^\W\d_])([^\W\d_])ุง([^\W\d_])$'), 'ู…ููู’ุนูŽุงู„', None),
(re.compile(r'^ู…([^\W\d_])ุชุง([^\W\d_])$'), 'ู…ููู’ุชูŽุนูู„', 'hollow_w'),
(re.compile(r'^[ุงูŠุช]([^\W\d_])ุชุง([^\W\d_])$'), 'ุงููู’ุชูŽุนูŽู„ูŽ', 'hollow_w'),
(re.compile(r'^ุช([^\W\d_]{4})$'), 'ุชูŽููŽุนู’ู„ูŽู„ูŽ', 'quad'),
(re.compile(r'^ูŠ([^\W\d_]{4})$'), 'ูŠูุชูŽููŽุนู’ู„ูŽู„ู', 'quad'),
(re.compile(r'^ู…([^\W\d_]{4})$'), 'ู…ูุชูŽููŽุนู’ู„ูู„', 'quad'),
(re.compile(r'^ุงู†([^\W\d_])([^\W\d_])ุง([^\W\d_])$'), 'ุงูู†ู’ููุนูŽุงู„', None),
(re.compile(r'^ุงู†([^\W\d_])([^\W\d_])([^\W\d_])$'), 'ุงูู†ู’ููŽุนูŽู„ูŽ', None),
(re.compile(r'^ุง([^\W\d_])ุช([^\W\d_])ุง([^\W\d_])$'), 'ุงููู’ุชูุนูŽุงู„', None),
(re.compile(r'^ุง([^\W\d_])ุช([^\W\d_])([^\W\d_])$'), 'ุงููู’ุชูŽุนูŽู„ูŽ', None),
(re.compile(r'^ู…([^\W\d_])ุช([^\W\d_])([^\W\d_])$'), 'ู…ููู’ุชูŽุนูู„', None),
(re.compile(r'^[ูŠุชุฃ]([^\W\d_])ุช([^\W\d_])([^\W\d_])$'), 'ูŠูŽูู’ุชูŽุนูู„ู', None),
(re.compile(r'^[ูŠุชู†ุฃ]ุช([^\W\d_])([^\W\d_])([^\W\d_])$'), 'ูŠูŽุชูŽููŽุนูŽู‘ู„ู', None),
(re.compile(r'^[ูŠุชู†ุฃ]ุช([^\W\d_])ุง([^\W\d_])([^\W\d_])$'), 'ูŠูŽุชูŽููŽุงุนูŽู„ู', None),
(re.compile(r'^ุช([^\W\d_])([^\W\d_])([^\W\d_])$'), 'ุชูŽููŽุนูŽู‘ู„ูŽ', None),
(re.compile(r'^ุช([^\W\d_])ุง([^\W\d_])([^\W\d_])$'), 'ุชูŽููŽุงุนูŽู„ูŽ', None),
(re.compile(r'^ู…([^\W\d_])([^\W\d_])([^\W\d_])$'), 'ู…ูŽูู’ุนูŽู„', None),
(re.compile(r'^ู…([^\W\d_])([^\W\d_])ูˆ([^\W\d_])$'), 'ู…ูŽูู’ุนููˆู„', None),
(re.compile(r'^ู…([^\W\d_])([^\W\d_])ูŠ([^\W\d_])$'), 'ู…ูŽูู’ุนููŠู„', None),
(re.compile(r'^ู…([^\W\d_])([^\W\d_])([^\W\d_])ุฉ$'), 'ู…ูŽูู’ุนูŽู„ูŽุฉ', None),
(re.compile(r'^([^\W\d_])ุง([^\W\d_])([^\W\d_])$'), 'ููŽุงุนูู„', None),
(re.compile(r'^([^\W\d_])ุง([^\W\d_])([^\W\d_])ุฉ$'), 'ููŽุงุนูู„ูŽุฉ', None),
(re.compile(r'^([^\W\d_])([^\W\d_])ูŠ([^\W\d_])$'), 'ููŽุนููŠู„', None),
(re.compile(r'^([^\W\d_])([^\W\d_])ูŠ([^\W\d_])ุฉ$'), 'ููŽุนููŠู„ูŽุฉ', None),
(re.compile(r'^([^\W\d_])([^\W\d_])ูˆ([^\W\d_])$'), 'ููุนููˆู„', None),
(re.compile(r'^([^\W\d_])([^\W\d_])ุง([^\W\d_])$'), 'ููุนูŽุงู„', None),
(re.compile(r'^([^\W\d_])([^\W\d_])ุง([^\W\d_])ุฉ$'), 'ููุนูŽุงู„ูŽุฉ', None),
(re.compile(r'^([^\W\d_])([^\W\d_])([^\W\d_])ุงู†$'), 'ููุนู’ู„ูŽุงู†', None),
(re.compile(r'^ุฃ([^\W\d_])([^\W\d_])([^\W\d_])$'), 'ุฃูŽูู’ุนูŽู„', None),
(re.compile(r'^ุฃ([^\W\d_])([^\W\d_])ุง([^\W\d_])$'), 'ุฃูŽูู’ุนูŽุงู„', None),
(re.compile(r'^ุฅ([^\W\d_])([^\W\d_])ุง([^\W\d_])$'), 'ุฅููู’ุนูŽุงู„', None),
(re.compile(r'^[ูŠุชู†ุฃ]([^\W\d_])([^\W\d_])([^\W\d_])$'), 'ูŠูŽูู’ุนูŽู„ู', None),
(re.compile(r'^ุช([^\W\d_])([^\W\d_])ูŠ([^\W\d_])$'), 'ุชูŽูู’ุนููŠู„', None),
]
class BasranSyntacticRealizer:
"""Enforces Basran Syntactic Governance & Classical Mutation Rules."""
JUSSIVE_GOVERNORS = {'ู„ู…', 'ู„ู…ุง', 'ู„ุง', 'ุฅู†', 'ู…ู†', 'ู…ุง', 'ู…ู‡ู…ุง', 'ุญูŠุซู…ุง', 'ุฅุฐู…ุง', 'ุฃูŠู†ู…ุง', 'ู…ุชู‰', 'ุฃูŠุงู†'}
SUBJUNCTIVE_GOVERNORS = {'ุฃู†', 'ู„ู†', 'ูƒูŠ', 'ุฅุฐู†', 'ุญุชู‰'}
HOLLOW_JUSSIVE_MAP = {
'ูŠูƒูˆู†': 'ูŠูƒู†', 'ุชูƒูˆู†': 'ุชูƒู†', 'ุฃูƒูˆู†': 'ุฃูƒู†', 'ู†ูƒูˆู†': 'ู†ูƒู†',
'ูŠู‚ูˆู„': 'ูŠู‚ู„', 'ุชู‚ูˆู„': 'ุชู‚ู„', 'ุฃู‚ูˆู„': 'ุฃู‚ู„', 'ู†ู‚ูˆู„': 'ู†ู‚ู„',
'ูŠุณูŠุฑ': 'ูŠุณุฑ', 'ุชุณูŠุฑ': 'ุชุณุฑ', 'ุฃุณูŠุฑ': 'ุฃุณุฑ', 'ู†ุณูŠุฑ': 'ู†ุณุฑ',
'ูŠู†ุงู„': 'ูŠู†ู„', 'ุชู†ุงู„': 'ุชู†ู„', 'ุฃู†ุงู„': 'ุฃู†ู„', 'ู†ู†ุงู„': 'ู†ู†ู„',
'ูŠุฒูŠุฏ': 'ูŠุฒุฏ', 'ุชุฒูŠุฏ': 'ุชุฒุฏ', 'ูŠุณุชุทูŠุน': 'ูŠุณุชุทุน', 'ุชุณุชุทูŠุน': 'ุชุณุชุทุน',
'ูŠุณุชู‚ูŠู…': 'ูŠุณุชู‚ู…', 'ุชุณุชู‚ูŠู…': 'ุชุณุชู‚ู…', 'ูŠุจูŠู†': 'ูŠุจู†', 'ุชุจูŠู†': 'ุชุจู†',
'ูŠุญูŠุท': 'ูŠุญุท'
}
DEFECTIVE_JUSSIVE_MAP = {
'ูŠุฏุนูˆ': 'ูŠุฏุน', 'ุชุฏุนูˆ': 'ุชุฏุน', 'ุฃุฏุนูˆ': 'ุฃุฏุน', 'ู†ุฏุนูˆ': 'ู†ุฏุน',
'ูŠุฑู…ูŠ': 'ูŠุฑู…', 'ุชุฑู…ูŠ': 'ุชุฑู…', 'ุฃุฑู…ูŠ': 'ุฃุฑู…', 'ู†ุฑู…ูŠ': 'ู†ุฑู…',
'ูŠุณุนู‰': 'ูŠุณุน', 'ุชุณุนู‰': 'ุชุณุน', 'ุฃุณุนู‰': 'ุฃุณุน', 'ู†ุณุนู‰': 'ู†ุณุน',
'ูŠุจู‚ู‰': 'ูŠุจู‚', 'ุชุจู‚ู‰': 'ุชุจู‚', 'ูŠุฑู‰': 'ูŠุฑ', 'ุชุฑู‰': 'ุชุฑ',
'ุฃุฑู‰': 'ุฃุฑ', 'ู†ุฑู‰': 'ู†ุฑ', 'ูŠุฃุชูŠ': 'ูŠุฃุช', 'ุชุฃุชูŠ': 'ุชุฃุช',
'ูŠู‡ุฏูŠ': 'ูŠู‡ุฏ', 'ุชู‡ุฏูŠ': 'ุชู‡ุฏ', 'ูŠู‚ุถูŠ': 'ูŠู‚ุถ', 'ุชู‚ุถูŠ': 'ุชู‚ุถ',
'ูŠุฎู„ูˆ': 'ูŠุฎู„', 'ุชุฎู„ูˆ': 'ุชุฎู„'
}
@staticmethod
def apply_word_internal_syntax(prefix: str, stem: str, suffix: str) -> str:
p = prefix.strip() if prefix not in ['<NONE>', '<PAD>', '<UNK>', ''] else ''
s = suffix.strip() if suffix not in ['<NONE>', '<PAD>', '<UNK>', ''] else ''
w = stem.strip()
if not w:
return p + s
if p.startswith('ุงู„') or p in ['ูˆุงู„', 'ูุงู„', 'ุจุงู„', 'ูƒุงู„', 'ู„ู„']:
if s in ['ุงู‹', 'ุง', 'ู', 'ูŒ', 'ู‹']:
s = ''
if w.endswith('ุงู‹'):
w = w[:-1]
if s and s not in ['ูˆู†', 'ูŠู†', 'ุงู†', 'ุงุช', 'ุฉ']:
if w.endswith('ุฉ'):
w = w[:-1] + 'ุช'
if s in ['ู‡', 'ู‡ุง', 'ู‡ู…', 'ู‡ู…ุง', 'ู‡ู†', 'ูƒ', 'ูƒู…', 'ูƒู†', 'ู†ุง', 'ูŠ']:
if len(w) >= 5 and w.endswith('ูˆู†'):
w = w[:-1]
elif len(w) >= 5 and w.endswith('ูŠู†'):
w = w[:-1]
elif len(w) >= 5 and w.endswith('ุงู†'):
w = w[:-1]
if s in ['ู‡', 'ู‡ุง', 'ู‡ู…', 'ู‡ู…ุง', 'ู‡ู†', 'ูƒ', 'ูƒู…', 'ูƒู†', 'ู†ุง', 'ูŠ']:
if w.endswith('ุงุก'):
base = w[:-2]
w = base + 'ุงุค' if p == '' else base + 'ุงุฆ' if p in ['ุจ', 'ู„', 'ููŠ'] else base + 'ุงุก'
return p + w + s
@classmethod
def apply_contextual_syntax(cls, word_records: List[Dict[str, Any]]) -> List[str]:
n = len(word_records)
realized_words = []
for i in range(n):
rec = word_records[i]
prefix = rec.get('prefix', '')
root = rec.get('root', '')
wazn = rec.get('wazn', '')
suffix = rec.get('suffix', '')
stem = rec.get('stem', '') or root
if prefix and stem.startswith(prefix):
stem = stem[len(prefix):]
if suffix and stem.endswith(suffix):
stem = stem[:-len(suffix)]
prev_word = realized_words[i - 1] if i > 0 else ''
next_rec = word_records[i + 1] if i + 1 < n else None
# Jussive Truncation
if prev_word in cls.JUSSIVE_GOVERNORS or prefix in cls.JUSSIVE_GOVERNORS:
if stem in cls.HOLLOW_JUSSIVE_MAP:
stem = cls.HOLLOW_JUSSIVE_MAP[stem]
elif stem in cls.DEFECTIVE_JUSSIVE_MAP:
stem = cls.DEFECTIVE_JUSSIVE_MAP[stem]
elif suffix == 'ูˆู†':
suffix = 'ูˆุง'
elif suffix == 'ูŠู†' and (stem.startswith('ุช') or stem.startswith('ูŠ')):
suffix = 'ูŠ'
elif prev_word in cls.SUBJUNCTIVE_GOVERNORS or prefix in cls.SUBJUNCTIVE_GOVERNORS:
if suffix == 'ูˆู†':
suffix = 'ูˆุง'
elif suffix == 'ูŠู†' and (stem.startswith('ุช') or stem.startswith('ูŠ')):
suffix = 'ูŠ'
# Annexation Nลซn Deletion
if next_rec is not None:
next_p = next_rec.get('prefix', '')
next_w = next_rec.get('stem', '') or next_rec.get('root', '')
next_is_annexed = (
next_p.startswith('ุงู„') or next_p in ['ุจุงู„', 'ูƒุงู„', 'ู„ู„', 'ูุงู„', 'ูˆุงู„'] or
next_w.startswith('ุงู„') or
(not next_p and len(next_w) >= 3 and next_w not in cls.JUSSIVE_GOVERNORS)
)
if next_is_annexed:
if suffix == 'ูˆู†':
suffix = 'ูˆ'
elif suffix == 'ูŠู†' and not (stem.startswith('ูŠ') or stem.startswith('ุช')):
suffix = 'ูŠ'
elif suffix == 'ุงู†':
suffix = 'ุง'
elif not suffix and len(stem) >= 5 and not root.endswith('ู†'):
if stem.endswith('ูˆู†'):
stem = stem[:-1]
elif stem.endswith('ูŠู†') and not (stem.startswith('ูŠ') or stem.startswith('ุช')):
stem = stem[:-1]
elif stem.endswith('ุงู†'):
stem = stem[:-1]
word = cls.apply_word_internal_syntax(prefix, stem, suffix)
realized_words.append(word)
return realized_words
class RootformerTokenizer(PreTrainedTokenizer):
"""
Rootformer Official Farฤhฤซdian-Basran Morphemic Tokenizer.
Inherits from Hugging Face PreTrainedTokenizer.
"""
vocab_files_names = {"vocab_file": "nrmp_vocab.json"}
model_input_names = ["input_ids", "attention_mask", "p_ids", "r_ids", "w_ids", "s_ids"]
def __init__(
self,
vocab_file: Optional[str] = None,
bos_token: str = "<BOS>",
eos_token: str = "<EOS>",
unk_token: str = "<UNK>",
pad_token: str = "<PAD>",
model_max_length: int = 2048,
**kwargs
):
self.vocab_file = vocab_file
self._load_vocabulary(vocab_file)
super().__init__(
bos_token=bos_token,
eos_token=eos_token,
unk_token=unk_token,
pad_token=pad_token,
model_max_length=model_max_length,
**kwargs
)
# Integrated Classical Morphological Analyzer
try:
from .morph_analyzer import ArabicMorphAnalyzer
self.analyzer = ArabicMorphAnalyzer(vocab_path=self.vocab_file)
except Exception:
try:
from morph_analyzer import ArabicMorphAnalyzer
self.analyzer = ArabicMorphAnalyzer(vocab_path=self.vocab_file)
except Exception:
self.analyzer = None
def analyze(self, text_or_word: str) -> Union[Dict[str, Any], List[Dict[str, Any]]]:
"""
Deep morphological analysis of Arabic text or single word.
Returns detailed grammatical breakdown (root, wazn, prefix, suffix, type).
"""
if self.analyzer is None:
try:
from .morph_analyzer import ArabicMorphAnalyzer
except Exception:
from morph_analyzer import ArabicMorphAnalyzer
self.analyzer = ArabicMorphAnalyzer(vocab_path=self.vocab_file)
words = text_or_word.strip().split()
if len(words) == 1:
return self.analyzer.analyze_word(words[0])
return self.analyzer.analyze_sentence(text_or_word)
def extract_roots(self, text: str) -> List[str]:
"""Extracts sequence of classical consonantal roots from text."""
if self.analyzer is None:
try:
from .morph_analyzer import ArabicMorphAnalyzer
except Exception:
from morph_analyzer import ArabicMorphAnalyzer
self.analyzer = ArabicMorphAnalyzer(vocab_path=self.vocab_file)
return self.analyzer.extract_roots(text)
def _load_vocabulary(self, vocab_file: Optional[str]):
data = None
candidates = []
if vocab_file and Path(vocab_file).exists():
candidates.append(Path(vocab_file))
# Local defaults
this_dir = Path(__file__).parent.resolve()
candidates.extend([
this_dir / "nrmp_vocab.json",
this_dir / "data" / "nrmp_vocab.json",
Path("/workspace/rootformer_v12/v18_next_root_morph/data/nrmp_vocab.json"),
Path("/home/absolut7/.gemini/antigravity-ide/scratch/rootformer/v12/v18_next_root_morph/data/nrmp_vocab.json")
])
for c in candidates:
if c.exists():
try:
with open(c, 'r', encoding='utf-8') as f:
data = json.load(f)
self.vocab_file = str(c)
break
except Exception:
pass
if data is None:
# Fallback embedded minimal lists
roots = ['<PAD>', '<BOS>', '<EOS>', '<UNK>', '<PARTICLE>']
awzan = ['<PAD>', '<NONE>', '<UNK>', 'ููŽุนูŽู„ูŽ', 'ููŽุงุนูู„', 'ููŽุนู’ู„', 'ู…ูŽูู’ุนููˆู„', 'ุงูุณู’ุชูŽูู’ุนูŽุงู„ูŽ']
prefixes = ['<PAD>', '<NONE>', '<UNK>', 'ุงู„', 'ูˆ', 'ู', 'ุจ', 'ู„', 'ูƒ']
suffixes = ['<PAD>', '<NONE>', '<UNK>', 'ู‡', 'ู‡ุง', 'ู‡ู…', 'ู†ุง', 'ูŠ', 'ุฉ', 'ูˆู†', 'ูŠู†']
else:
roots = data.get('roots', [])
awzan = data.get('awzan', [])
prefixes = data.get('prefixes', [])
suffixes = data.get('suffixes', [])
self.roots_list = roots
self.awzan_list = awzan
self.canonical_prefixes = prefixes
self.canonical_suffixes = suffixes
self.root2id = {r: i for i, r in enumerate(self.roots_list)}
self.id2root = {i: r for i, r in enumerate(self.roots_list)}
self.wazn2id = {w: i for i, w in enumerate(self.awzan_list)}
self.id2wazn = {i: w for i, w in enumerate(self.awzan_list)}
self.prefix2id = {p: i for i, p in enumerate(self.canonical_prefixes)}
self.id2prefix = {i: p for i, p in enumerate(self.canonical_prefixes)}
self.suffix2id = {s: i for i, s in enumerate(self.canonical_suffixes)}
self.id2suffix = {i: s for i, s in enumerate(self.canonical_suffixes)}
# Constants
self.PAD_ROOT = self.root2id.get('<PAD>', 0)
self.BOS_ROOT = self.root2id.get('<BOS>', 1)
self.EOS_ROOT = self.root2id.get('<EOS>', 2)
self.UNK_ROOT = self.root2id.get('<UNK>', 3)
self.PAD_WAZN = self.wazn2id.get('<PAD>', 0)
self.NONE_WAZN = self.wazn2id.get('<NONE>', 1)
self.PAD_PREFIX = self.prefix2id.get('<PAD>', 0)
self.NONE_PREFIX = self.prefix2id.get('<NONE>', 1)
self.PAD_SUFFIX = self.suffix2id.get('<PAD>', 0)
self.NONE_SUFFIX = self.suffix2id.get('<NONE>', 1)
self.roots_set = set(self.roots_list)
self.awzan_set = set(self.awzan_list)
# Common particles list
self.COMMON_PARTICLES = [
'ููŠ', 'ู…ู†', 'ุฅู„ู‰', 'ุนู„ู‰', 'ุนู†', 'ู…ุน', 'ุญุชู‰', 'ู…ู†ุฐ', 'ู…ุฐ',
'ุฅู†', 'ุฃู†', 'ูƒุฃู†', 'ู„ูƒู†', 'ู„ูŠุช', 'ู„ุนู„', 'ู„ุง', 'ู…ุง', 'ู„ู…', 'ู„ู†',
'ู„ูŠุณ', 'ูƒุงู†', 'ู‡ูˆ', 'ู‡ูŠ', 'ู‡ู…ุง', 'ู‡ู…', 'ู‡ู†', 'ุฃู†ุช', 'ุฃู†ุง', 'ู†ุญู†',
'ู‡ุฐุง', 'ู‡ุฐู‡', 'ู‡ุคู„ุงุก', 'ุฐู„ูƒ', 'ุชู„ูƒ', 'ุฃูˆู„ุฆูƒ', 'ุงู„ุฐูŠ', 'ุงู„ุชูŠ', 'ุงู„ุฐูŠู†',
'ุซู…', 'ุฃูˆ', 'ุจู„', 'ุฃู…', 'ุฅู…ุง', 'ุฅุฐุง', 'ุฅุฐ', 'ูƒู„', 'ุจุนุถ', 'ุบูŠุฑ',
'ุณูˆู‰', 'ุนู†ุฏ', 'ุจูŠู†', 'ููˆู‚', 'ุชุญุช', 'ู‚ุจู„', 'ุจุนุฏ', 'ุญูŠุซ', 'ูƒูŠู',
'ุฃูŠู†', 'ู…ุชู‰', 'ูƒู…', 'ุฃูŠ', 'ู†ุนู…', 'ุจู„ู‰', 'ุฅุฐู†', 'ู‚ุฏ', 'ุณูˆู'
]
@property
def vocab_size(self) -> int:
return len(self.roots_list)
def get_vocab(self) -> Dict[str, int]:
return dict(self.root2id)
def clean_arabic(self, word: str) -> str:
no_tashkeel = re.sub(r'[\u064B-\u065F\u0670\u0640]', '', word)
no_punct = re.sub(r'[ุŒุ›ุŸ.:!ยซยป\(\)\[\]\{\}โ€ฆ"\']', ' ', no_tashkeel)
return re.sub(r'\s+', ' ', no_punct).strip()
def decompose_arabic_word(self, word: str) -> Tuple[Optional[str], Optional[str], Optional[str], Optional[str]]:
clean = self.clean_arabic(word)
if not clean:
return None, None, None, None
if clean in CLOSED_PARTICLES:
return None, clean, None, None
if clean in COMPOUND_PARTICLES:
p, part = COMPOUND_PARTICLES[clean]
return p, part, None, None
if len(clean) == 3 and clean in self.roots_set:
return None, clean, 'ููŽุนูŽู„ูŽ', None
if len(clean) == 4 and clean in self.roots_set:
return None, clean, 'ููŽุนู’ู„ูŽู„ูŽ', None
# Philosophical and theological canonical exceptions
if clean in ['ู…ุงู‡ูŠุฉ', 'ุงู„ู…ุงู‡ูŠุฉ']:
p = 'ุงู„' if clean.startswith('ุงู„') else None
return p, 'ู…ูˆู‡', 'ููŽุงุนูู„ูŽุฉ', None
if clean in ['ุฃุนุถุงุก', 'ุงุนุถุงุก', 'ุฃุนุถุงุคู‡', 'ุฃุนุถุงุฆู‡']:
s = 'ู‡' if clean.endswith('ู‡') else None
return None, 'ุนุถุง', 'ุฃูŽูู’ุนูŽุงู„', s
if clean in ['ุฃุญุดุงุก', 'ุงุญุดุงุก', 'ุฃุญุดุงุคู‡', 'ุฃุญุดุงุฆู‡']:
s = 'ู‡' if clean.endswith('ู‡') else None
return None, 'ุญุดุง', 'ุฃูŽูู’ุนูŽุงู„', s
if clean in ['ู‡ูˆุงุก', 'ุงู„ู‡ูˆุงุก', 'ุฎู„ุงุก', 'ุงู„ุฎู„ุงุก']:
p = 'ุงู„' if clean.startswith('ุงู„') else None
r = 'ู‡ูˆุง' if 'ู‡ูˆุงุก' in clean else 'ุฎู„ุง'
return p, r, 'ููŽุนูŽุงู„', None
# Clitic stripping
article_prefixes = [p for p in ['ูุงู„', 'ูˆุงู„', 'ูƒุงู„', 'ุจุงู„', 'ู„ู„', 'ุงู„'] if clean.startswith(p)]
single_prefixes = [p for p in ['ูˆ', 'ู', 'ุจ', 'ู„', 'ูƒ'] if clean.startswith(p)]
candidate_prefixes = article_prefixes + [''] + single_prefixes
cand_s_list = sorted([s for s in ENCLITICS if clean.endswith(s) and s != ''], key=len, reverse=True)
candidate_suffixes = [''] + cand_s_list
for p in candidate_prefixes:
rem_p = clean[len(p):] if p else clean
for s in candidate_suffixes:
stem = rem_p[:-len(s)] if s and rem_p.endswith(s) else (rem_p if not s else '')
if not stem:
continue
# Bare stem in roots
if len(stem) == 3:
if stem in self.roots_set:
return (p or None), stem, 'ููŽุนูŽู„ูŽ', (s or None)
stem_bare = stem.replace('ุฃ', 'ุง').replace('ุฅ', 'ุง').replace('ุข', 'ุง')
if stem_bare in self.roots_set:
return (p or None), stem_bare, 'ููŽุนูŽู„ูŽ', (s or None)
if stem in ROOT_CANONICAL_MAP and ROOT_CANONICAL_MAP[stem] in self.roots_set:
return (p or None), ROOT_CANONICAL_MAP[stem], 'ููŽุนูŽู„ูŽ', (s or None)
# Doubled root (2-letter stem, e.g. ุนู„ -> ุนู„ู„, ุญู‚ -> ุญู‚ู‚, ุจุฏ -> ุจุฏุฏ, ูƒู„ -> ูƒู„ู„)
if len(stem) == 2:
doubled = stem + stem[-1]
if doubled in self.roots_set:
return (p or None), doubled, 'ููŽุนูŽู„ูŽ', (s or None)
# Doubled active participle (ุชุงู… -> ุชู…ู…, ุนุงู… -> ุนู…ู…, ุฎุงุต -> ุฎุตุต, ุถุงุฑ -> ุถุฑุฑ)
if len(stem) == 3 and stem[1] == 'ุง':
cand_doubled = stem[0] + stem[2] + stem[2]
if cand_doubled in self.roots_set:
return (p or None), cand_doubled, 'ููŽุงุนูู„', (s or None)
# Hollow imperfect / jussive (ูŠูƒู† -> ูƒูˆู†, ูŠู‚ูˆู„ -> ู‚ูˆู„, ูŠู‚ู„ -> ู‚ูˆู„, ูŠุณูŠุฑ -> ุณูŠุฑ)
if len(stem) == 3 and stem[0] in ['ูŠ', 'ุช', 'ู†', 'ุฃ']:
cand_hollow_w = stem[1] + 'ูˆ' + stem[2]
if cand_hollow_w in self.roots_set:
return (p or None), cand_hollow_w, 'ูŠูŽูู’ุนูู„ู', (s or None)
cand_hollow_y = stem[1] + 'ูŠ' + stem[2]
if cand_hollow_y in self.roots_set:
return (p or None), cand_hollow_y, 'ูŠูŽูู’ุนูู„ู', (s or None)
cand_gem = stem[1] + stem[2] + stem[2]
if cand_gem in self.roots_set:
return (p or None), cand_gem, 'ูŠูŽูู’ุนูู„ู', (s or None)
# Mithฤl Wฤwฤซ (ูŠุฌุจ -> ูˆุฌุจ, ูŠุฌุฏ -> ูˆุฌุฏ, ูŠุตู -> ูˆุตู, ูŠุนุฏ -> ูˆุนุฏ)
if len(stem) == 3 and stem[0] in ['ูŠ', 'ุช', 'ู†', 'ุฃ']:
mithal = 'ูˆ' + stem[1:]
if mithal in self.roots_set:
return (p or None), mithal, 'ูŠูŽุนูู„ู', (s or None)
for pattern, wazn_name, note in AWZAN_TEMPLATES:
m = pattern.match(stem)
if m:
radicals = list(m.groups())
if note == 'quad':
cand_root = radicals[0]
elif note == 'hollow_w':
cand_root = radicals[0] + 'ูˆ' + radicals[1]
else:
cand_root = ''.join(radicals)
cand_root = ROOT_CANONICAL_MAP.get(cand_root, cand_root)
if cand_root in self.roots_set:
return (p or None), cand_root, wazn_name, (s or None)
# Broken plurals and assimilation
if len(stem) == 4 and stem[1] == 'ุง':
r_hollow = stem[0] + 'ูˆ' + stem[2:]
if r_hollow in self.roots_set:
return (p or None), r_hollow, 'ููŽุงุนูู„', (s or None)
r_hollow_y = stem[0] + 'ูŠ' + stem[2:]
if r_hollow_y in self.roots_set:
return (p or None), r_hollow_y, 'ููŽุงุนูู„', (s or None)
return None, clean, None, None
def realize_root_and_wazn(self, root: str, wazn: str) -> str:
if root == 'ุงุตู„':
return 'ุฃุตู„'
if root == 'ุดูŠุง':
return 'ุดูŠุก'
if root == 'ุฎู„ุง' and wazn in ['ูŠูŽูู’ุนูŽู„ู', 'ูŠูŽูู’ุนูู„ู']:
return 'ูŠุฎู„ูˆ'
if root == 'ุนูŠุง':
return 'ุฃุนูŠุงู†' if wazn == 'ุฃูŽูู’ุนูŽุงู„' else 'ุนูŠู†'
if root in ['ุจุฏุง', 'ุจุฏุฃ'] and wazn == 'ุงููู’ุชูุนูŽุงู„':
return 'ุงุจุชุฏุงุก'
if wazn == 'ู…ูุชูŽููŽุนูู‘ู„' and len(root) == 3:
return 'ู…ุช' + root[0] + root[1] + root[2]
if wazn == 'ูŠูŽูู’ุชูŽุนูู„ู':
if root == 'ุญูˆุฌ':
return 'ูŠุญุชุงุฌ'
elif root == 'ุฌู…ุน':
return 'ุชุฌุชู…ุน'
elif len(root) == 3:
return 'ูŠ' + root[0] + 'ุช' + root[1] + root[2]
if wazn == 'ุงููู’ุชูŽุนูŽู„ูŽ' and len(root) == 3:
return 'ุงู…ุชู†ุน' if root == 'ู…ู†ุน' else 'ุง' + root[0] + 'ุช' + root[1] + root[2]
r = ROOT_CANONICAL_MAP_REV.get(root, root)
if wazn == 'ููุนูŽู„':
return r
elif wazn == 'ููŽุนูŽู„ูŽ':
if len(r) == 3 and r[1] == r[2] and r not in {'ุณุจุจ', 'ู…ู„ู„', 'ุทู„ู„'}:
return r[:2]
return r
elif wazn == 'ููุนููˆู„' and len(r) == 3:
return r[0] + r[1] + 'ูˆ' + r[2]
elif wazn == 'ููŽุนููŠู„' and len(r) == 3:
return r[0] + r[1] + 'ูŠ' + r[2]
elif wazn == 'ููŽุงุนูู„':
if len(r) == 3 and r[1] == r[2]:
return r[0] + 'ุง' + r[1]
elif len(r) == 3:
return r[0] + 'ุง' + r[1] + r[2]
elif wazn == 'ููŽุงุนูู„ูŽุฉ':
if r == 'ู…ูˆู‡':
return 'ู…ุงู‡ูŠุฉ'
elif len(r) == 3:
return r[0] + 'ุง' + r[1] + r[2] + 'ุฉ'
elif wazn == 'ู…ูŽูู’ุนูŽู„' and len(r) == 3:
return 'ู…' + r[0] + r[1] + r[2]
elif wazn in ['ู…ููู’ุนูŽุงู„', 'ู…ูŽูู’ุนูŽุงู„'] and len(r) == 3:
return 'ู…' + r[0] + r[1] + 'ุง' + r[2]
elif wazn == 'ู…ูŽูู’ุนููˆู„' and len(r) == 3:
return 'ู…' + r[0] + r[1] + 'ูˆ' + r[2]
elif wazn == 'ููุนูŽุงู„':
if len(r) == 3 and (r[2] in ['ูŠ', 'ูˆ'] or r in ['ุดููŠ', 'ุณู…ูˆ', 'ุจู†ูŠ', 'ู‚ุถูŠ']):
return r[0] + r[1] + 'ุงุก'
elif len(r) == 3:
return r[0] + r[1] + 'ุง' + r[2]
elif wazn == 'ุฃูŽูู’ุนูŽุงู„' and len(r) == 3:
return 'ุฃ' + r[0] + r[1] + 'ุง' + r[2]
elif wazn == 'ุฃูŽูู’ุนูŽู„ูŽ' and len(r) == 3:
return 'ุฃ' + r[0] + r[1] + r[2]
elif wazn == 'ุชูŽูู’ุนููŠู„' and len(r) == 3:
return 'ุช' + r[0] + r[1] + 'ูŠ' + r[2]
elif wazn == 'ุงููู’ุชูุนูŽุงู„' and len(r) == 3:
return 'ุง' + r[0] + 'ุช' + r[1] + 'ุง' + r[2]
return r
def encode_word(self, word: str) -> Tuple[int, int, int, int]:
clean_w = word.strip()
if not clean_w:
return (self.NONE_PREFIX, self.UNK_ROOT, self.NONE_WAZN, self.NONE_SUFFIX)
particle_tag = f'<P:{clean_w}>'
if particle_tag in self.root2id:
return (self.NONE_PREFIX, self.root2id[particle_tag], self.NONE_WAZN, self.NONE_SUFFIX)
p, r, wz, s = self.decompose_arabic_word(clean_w)
p_id = self.NONE_PREFIX if not p else self.prefix2id.get(p, self.prefix2id.get('<UNK>', 2))
if not r:
r_id = self.root2id.get(f'<P:{clean_w}>', self.root2id.get('<PARTICLE>', self.UNK_ROOT))
else:
particle_cand = f'<P:{r}>'
if particle_cand in self.root2id:
r_id = self.root2id[particle_cand]
else:
r_id = self.root2id.get(r, self.UNK_ROOT)
w_id = self.NONE_WAZN if not wz else self.wazn2id.get(wz, self.wazn2id.get('<UNK>', 2))
s_id = self.NONE_SUFFIX if not s else self.suffix2id.get(s, self.suffix2id.get('<UNK>', 2))
return (p_id, r_id, w_id, s_id)
def decode_word(self, p_id: int, r_id: int, w_id: int, s_id: int) -> str:
if r_id in [self.PAD_ROOT, self.BOS_ROOT, self.EOS_ROOT]:
return ''
root_str = self.id2root.get(r_id, '<UNK>')
wazn_str = self.id2wazn.get(w_id, '<NONE>')
prefix_str = self.id2prefix.get(p_id, '<NONE>')
suffix_str = self.id2suffix.get(s_id, '<NONE>')
p_text = '' if prefix_str in ['<NONE>', '<PAD>', '<UNK>'] else prefix_str
s_text = '' if suffix_str in ['<NONE>', '<PAD>', '<UNK>'] else suffix_str
if root_str.startswith('<P:'):
return p_text + root_str[3:-1] + s_text
if root_str in ['<UNK>', '<PARTICLE>', '<PAD>']:
return p_text + root_str + s_text
if wazn_str not in ['<NONE>', '<PAD>', '<UNK>']:
stem = self.realize_root_and_wazn(root_str, wazn_str)
else:
stem = root_str
return p_text + stem + s_text
def encode_morphemes(self, sentence: str) -> List[Tuple[int, int, int, int]]:
words = sentence.strip().split()
return [self.encode_word(w) for w in words]
def decode_morphemes(self, word_tuples: List[Tuple[int, int, int, int]], apply_syntax: bool = True) -> str:
if not word_tuples:
return ""
if not apply_syntax:
words = [self.decode_word(*t) for t in word_tuples]
return " ".join([w for w in words if w])
# Prepare records for BasranSyntacticRealizer
records = []
for t in word_tuples:
p_id, r_id, w_id, s_id = t
if r_id in [self.PAD_ROOT, self.BOS_ROOT, self.EOS_ROOT]:
continue
r_str = self.id2root.get(r_id, '')
w_str = self.id2wazn.get(w_id, '<NONE>')
p_str = self.id2prefix.get(p_id, '')
s_str = self.id2suffix.get(s_id, '')
p_val = '' if p_str in ['<NONE>', '<PAD>', '<UNK>'] else p_str
s_val = '' if s_str in ['<NONE>', '<PAD>', '<UNK>'] else s_str
if r_str.startswith('<P:'):
stem = r_str[3:-1]
elif w_str not in ['<NONE>', '<PAD>', '<UNK>']:
stem = self.realize_root_and_wazn(r_str, w_str)
else:
stem = r_str
records.append({
'prefix': p_val,
'root': r_str,
'wazn': w_str,
'suffix': s_val,
'stem': stem
})
realized = BasranSyntacticRealizer.apply_contextual_syntax(records)
return " ".join([w for w in realized if w])
def _tokenize(self, text: str) -> List[str]:
return text.strip().split()
def _convert_token_to_id(self, token: str) -> int:
p, r, w, s = self.encode_word(token)
return r
def _convert_id_to_token(self, index: int) -> str:
return self.id2root.get(index, self.unk_token)
def convert_tokens_to_string(self, tokens: List[str]) -> str:
return " ".join(tokens)
def __call__(
self,
text: Union[str, List[str]],
text_pair: Optional[Any] = None,
return_tensors: Optional[str] = None,
padding: bool = False,
truncation: bool = False,
max_length: Optional[int] = None,
add_special_tokens: bool = True,
**kwargs
) -> BatchEncoding:
if isinstance(text, str):
texts = [text]
is_single = True
else:
texts = list(text)
is_single = False
batch_p, batch_r, batch_w, batch_s = [], [], [], []
for t in texts:
tuples = self.encode_morphemes(t)
if add_special_tokens:
tuples = [(self.NONE_PREFIX, self.BOS_ROOT, self.NONE_WAZN, self.NONE_SUFFIX)] + \
tuples + \
[(self.NONE_PREFIX, self.EOS_ROOT, self.NONE_WAZN, self.NONE_SUFFIX)]
if truncation and max_length and len(tuples) > max_length:
tuples = tuples[:max_length]
batch_p.append([x[0] for x in tuples])
batch_r.append([x[1] for x in tuples])
batch_w.append([x[2] for x in tuples])
batch_s.append([x[3] for x in tuples])
# Padding
max_len = max(len(seq) for seq in batch_r) if batch_r else 0
if max_length and padding:
max_len = max(max_len, max_length)
attn_mask = []
for i in range(len(batch_r)):
cur_len = len(batch_r[i])
pad_len = max_len - cur_len
attn_mask.append([1] * cur_len + [0] * pad_len)
batch_p[i] = batch_p[i] + [self.PAD_PREFIX] * pad_len
batch_r[i] = batch_r[i] + [self.PAD_ROOT] * pad_len
batch_w[i] = batch_w[i] + [self.PAD_WAZN] * pad_len
batch_s[i] = batch_s[i] + [self.PAD_SUFFIX] * pad_len
# input_ids corresponds to primary ontological radical stream (r_ids)
data = {
"input_ids": batch_r,
"attention_mask": attn_mask,
"p_ids": batch_p,
"r_ids": batch_r,
"w_ids": batch_w,
"s_ids": batch_s
}
if return_tensors == "pt":
data = {k: torch.tensor(v, dtype=torch.long) for k, v in data.items()}
return BatchEncoding(data)
def decode(
self,
token_ids: Union[int, List[int], torch.Tensor, List[Tuple[int, int, int, int]]],
skip_special_tokens: bool = True,
clean_up_tokenization_spaces: bool = False,
apply_syntax: bool = True,
**kwargs
) -> str:
if isinstance(token_ids, torch.Tensor):
token_ids = token_ids.tolist()
if isinstance(token_ids, int):
token_ids = [token_ids]
if not token_ids:
return ""
# Check if list of 4-tuples
if isinstance(token_ids[0], (list, tuple)) and len(token_ids[0]) == 4:
tuples = token_ids
else:
# Fallback from root IDs: assume NONE prefix, NONE wazn, NONE suffix
tuples = [(self.NONE_PREFIX, rid, self.NONE_WAZN, self.NONE_SUFFIX) for rid in token_ids]
if skip_special_tokens:
tuples = [t for t in tuples if t[1] not in [self.PAD_ROOT, self.BOS_ROOT, self.EOS_ROOT]]
return self.decode_morphemes(tuples, apply_syntax=apply_syntax)
def batch_decode(
self,
sequences: Union[List[List[int]], torch.Tensor],
skip_special_tokens: bool = True,
clean_up_tokenization_spaces: bool = False,
apply_syntax: bool = True,
**kwargs
) -> List[str]:
if isinstance(sequences, torch.Tensor):
sequences = sequences.tolist()
return [
self.decode(
seq,
skip_special_tokens=skip_special_tokens,
clean_up_tokenization_spaces=clean_up_tokenization_spaces,
apply_syntax=apply_syntax,
**kwargs
)
for seq in sequences
]
def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
os.makedirs(save_directory, exist_ok=True)
fname = "nrmp_vocab.json"
if filename_prefix:
fname = f"{filename_prefix}-{fname}"
out_path = Path(save_directory) / fname
data = {
"roots": self.roots_list,
"awzan": self.awzan_list,
"prefixes": self.canonical_prefixes,
"suffixes": self.canonical_suffixes,
"counts": {
"roots": len(self.roots_list),
"awzan": len(self.awzan_list),
"prefixes": len(self.canonical_prefixes),
"suffixes": len(self.canonical_suffixes)
}
}
with open(out_path, "w", encoding="utf-8") as f:
json.dump(data, f, ensure_ascii=False, indent=2)
return (str(out_path),)