File size: 14,830 Bytes
f9bc0c1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 | """Convert canonical-new and Sikkim Herald legacy Kirat Rai text to Unicode.
SIL publishes a TECkit mapping for the canonical 2021 ``kirat rai font new``
encoding. Sikkim Herald PDFs use a different, older layout hidden behind per-PDF
font subsets and ASCII extraction values. Four initial subsets plus the two
2024/2025 subsets audited in 2026 share one stable old-to-new remap derived by
exact glyph-outline command and advance-width identity. The Herald converter
applies that premap before the SIL rules.
That map is expressed with TECkit ``ByteClass``/``UniClass`` declarations plus a
handful of explicit multi-byte ligature rules. A ``[class] > [class]`` rule maps
the byte at position *n* of the byte class to the codepoint at position *n* of the
paired Unicode class. This module parses that map natively (no ``teckit_compile``
dependency, which is unavailable on this machine) and applies the forward
Legacy->Unicode pass.
The two layouts must not be auto-detected from text: even Herald strings without
the formerly conspicuous ``f R x F I L`` bytes are globally permuted. Callers must
select the canonical-new or Herald converter explicitly. The later pair adds one
exact match, extracted ``T`` -> canonical ``Z`` -> U+16D6C KIRAT RAI VOWEL SIGN
EU. It also establishes that extracted ``C`` has an empty outline and that the
custom glyph stored under ``:`` is a visible double quotation mark. These
edition-specific findings are pinned by PDF/font hashes in the converter-layout
audit; they are not inferred from ASCII character names.
"""
from __future__ import annotations
import re
import unicodedata
from dataclasses import dataclass
from pathlib import Path
from typing import Any
from .errors import DataValidationError
DEFAULT_KIRATRAI_LEGACY_MAP = Path(__file__).with_name("maps") / "kiratraifontnew.map"
# Bytes absent from SIL's canonical-new map. They remain a compatibility constant;
# in Herald PDFs these ASCII values belong to the separate permuted layout below.
UNMAPPED_LEGACY_BYTES: frozenset[str] = frozenset("fRxFIL\\")
# Sikkim Herald extracted ASCII -> canonical ``kiratraifontnew`` byte. Derived by
# exact RecordingPen outline-command and advance-width identity against the font
# embedded in Unicode proposal L2/22-043R. Shared by four PDF subsets.
KIRATRAI_HERALD_PREMAP: dict[str, str] = {
"D": "q",
"F": "g",
"G": "P",
"H": "j",
"I": "W",
"J": "J",
"K": "Q",
"L": "$",
"O": "O",
"R": "w",
"S": "G",
"T": "Z",
"U": "o",
"a": "k",
"b": "A",
"c": "D",
"d": "K",
"e": "m",
"f": "N",
"g": "s",
"h": "a",
"i": "v",
"j": "b",
"k": "r",
"l": "i",
"m": "c",
"n": "p",
"o": "n",
"p": "t",
"q": "d",
"r": "e",
"s": "C",
"t": "h",
"u": "u",
"v": "B",
"w": "l",
"x": "T",
"y": "y",
"z": "z",
}
# Confirmed non-ink glyphs in the audited Herald layout. The two 2024/2025 font
# subsets give C an empty RecordingPen command stream; backslash was already proven
# blank in the earlier four-subset corpus. They normalize to an ordinary space.
KIRATRAI_HERALD_BLANK_GLYPHS: frozenset[str] = frozenset("\\C")
# The old font stores a visible opening/closing double-quotation glyph under the
# extracted ASCII colon code. Original-resolution review of the 2024-09-26 PDF,
# page 3, includes bboxes (75.239998, 566.760010, 81.175995, 584.615967) and
# (211.019241, 566.760010, 216.955246, 584.615967). This is a punctuation
# correction, not a script-letter inference.
KIRATRAI_HERALD_PUNCTUATION_MAP: dict[str, str] = {":": '"'}
# Confirmed literal values in the audited Herald PDFs.
KIRATRAI_HERALD_PASSTHROUGH: frozenset[str] = frozenset(" \t\r\n0123456789(),-/.;")
_KIRATRAI_CODEPOINT_RE = re.compile(r"[\U00016D40-\U00016D7F]")
_BYTE_TOKEN_RE = re.compile(r"0x([0-9A-Fa-f]{2})")
_UNI_TOKEN_RE = re.compile(r"U\+([0-9A-Fa-f]{4,6})")
_BYTECLASS_RE = re.compile(r"^ByteClass\s*\[([^\]]+)\]\s*=\s*\((.*)\)\s*$")
_UNICLASS_RE = re.compile(r"^UniClass\s*\[([^\]]+)\]\s*=\s*\((.*)\)\s*$")
_CLASS_RULE_RE = re.compile(r"^\[([^\]]+)\]\s*<?>\s*\[([^\]]+)\]\s*$")
def _expand_byte_tokens(body: str) -> tuple[int, ...]:
"""Expand a TECkit byte-class body into an ordered list of byte values.
Supports both explicit ``0xNN`` tokens and ``0xNN .. 0xMM`` ranges.
"""
values: list[int] = []
tokens = re.split(r"\s+", body.strip())
index = 0
while index < len(tokens):
token = tokens[index]
if not token:
index += 1
continue
if index + 2 < len(tokens) and tokens[index + 1] == "..":
start = int(token, 16)
end = int(tokens[index + 2], 16)
if end < start:
raise DataValidationError(f"invalid byte range in map: {token}..{tokens[index + 2]}")
values.extend(range(start, end + 1))
index += 3
continue
values.append(int(token, 16))
index += 1
return tuple(values)
def _expand_uni_tokens(body: str) -> tuple[int, ...]:
"""Expand a TECkit Unicode-class body into an ordered list of codepoints."""
values: list[int] = []
tokens = re.split(r"\s+", body.strip())
index = 0
while index < len(tokens):
token = tokens[index]
if not token:
index += 1
continue
if index + 2 < len(tokens) and tokens[index + 1] == "..":
start = int(_UNI_TOKEN_RE.match(token).group(1), 16)
end = int(_UNI_TOKEN_RE.match(tokens[index + 2]).group(1), 16)
if end < start:
raise DataValidationError(f"invalid unicode range in map: {token}..{tokens[index + 2]}")
values.extend(range(start, end + 1))
index += 3
continue
match = _UNI_TOKEN_RE.match(token)
if match is None:
raise DataValidationError(f"unparseable unicode token in map: {token!r}")
values.append(int(match.group(1), 16))
index += 1
return tuple(values)
@dataclass(frozen=True, slots=True)
class KiratRaiLegacyConversion:
legacy_text: str
unicode_text: str
kiratrai_char_count: int
replacement_count: int
unmapped_codepoints: list[str]
def to_dict(self) -> dict[str, Any]:
return {
"legacy_text": self.legacy_text,
"unicode_text": self.unicode_text,
"kiratrai_char_count": self.kiratrai_char_count,
"replacement_count": self.replacement_count,
"unmapped_codepoints": self.unmapped_codepoints,
}
class KiratRaiLegacyConverter:
"""Native reader/applier for SIL's ``kiratraifontnew.map`` TECkit mapping.
Rules are flattened to ``(source_bytes, target_codepoints)`` pairs and applied
longest-match-first so that explicit multi-byte ligature rules (e.g. ``//`` ->
double danda) take precedence over the single-byte class rules.
"""
def __init__(self, rules: list[tuple[tuple[int, ...], tuple[int, ...]]]) -> None:
if not rules:
raise DataValidationError("Kirat Rai legacy converter requires at least one mapping rule")
# De-duplicate while preserving the longest-source-first ordering.
seen: set[tuple[int, ...]] = set()
ordered: list[tuple[tuple[int, ...], tuple[int, ...]]] = []
for source, target in sorted(rules, key=lambda item: len(item[0]), reverse=True):
if source in seen:
continue
seen.add(source)
ordered.append((source, target))
self._rules = ordered
@classmethod
def from_map_file(cls, path: str | Path = DEFAULT_KIRATRAI_LEGACY_MAP) -> "KiratRaiLegacyConverter":
map_path = Path(path)
if not map_path.is_file():
raise DataValidationError(f"Kirat Rai legacy map does not exist: {map_path}")
byte_classes: dict[str, tuple[int, ...]] = {}
uni_classes: dict[str, tuple[int, ...]] = {}
rules: list[tuple[tuple[int, ...], tuple[int, ...]]] = []
in_byte_pass = False
# First parse class declarations; collect rule lines for a second pass so
# class rules can reference classes regardless of declaration order.
rule_lines: list[str] = []
for raw_line in map_path.read_text(encoding="utf-8-sig").splitlines():
line = raw_line.split(";", 1)[0].strip()
if not line:
continue
lowered = line.lower()
if lowered.startswith("pass("):
in_byte_pass = lowered == "pass(byte_unicode)"
continue
if not in_byte_pass:
continue
byte_match = _BYTECLASS_RE.match(line)
if byte_match:
byte_classes[byte_match.group(1).strip()] = _expand_byte_tokens(byte_match.group(2))
continue
uni_match = _UNICLASS_RE.match(line)
if uni_match:
uni_classes[uni_match.group(1).strip()] = _expand_uni_tokens(uni_match.group(2))
continue
rule_lines.append(line)
for line in rule_lines:
class_rule = _CLASS_RULE_RE.match(line)
if class_rule:
left_name = class_rule.group(1).strip()
right_name = class_rule.group(2).strip()
left = byte_classes.get(left_name)
right = uni_classes.get(right_name)
if left is None:
raise DataValidationError(f"class rule references unknown byte class: {left_name!r}")
if right is None:
raise DataValidationError(f"class rule references unknown unicode class: {right_name!r}")
if len(left) != len(right):
raise DataValidationError(
f"class rule length mismatch for [{left_name}]>[{right_name}]: {len(left)} bytes vs {len(right)} codepoints"
)
for byte_value, codepoint in zip(left, right):
rules.append(((byte_value,), (codepoint,)))
continue
if "<>" in line or ">" in line:
left_text, right_text = line.split("<>", 1) if "<>" in line else line.split(">", 1)
source = _BYTE_TOKEN_RE.findall(left_text)
target = _UNI_TOKEN_RE.findall(right_text)
if source and target:
rules.append(
(
tuple(int(value, 16) for value in source),
tuple(int(value, 16) for value in target),
)
)
return cls(rules)
def convert(self, text: str) -> KiratRaiLegacyConversion:
output: list[str] = []
unmapped: list[str] = []
replacements = 0
index = 0
length = len(text)
while index < length:
matched = False
for source, target in self._rules:
if self._matches(text, index, source):
output.extend(chr(value) for value in target)
replacements += 1
index += len(source)
matched = True
break
if matched:
continue
char = text[index]
output.append(char)
code = ord(char)
if not (0x16D40 <= code <= 0x16D7F):
unmapped.append(f"U+{code:04X}")
index += 1
converted = unicodedata.normalize("NFC", "".join(output))
return KiratRaiLegacyConversion(
legacy_text=text,
unicode_text=converted,
kiratrai_char_count=len(_KIRATRAI_CODEPOINT_RE.findall(converted)),
replacement_count=replacements,
unmapped_codepoints=sorted(set(unmapped)),
)
@staticmethod
def _matches(text: str, index: int, source: tuple[int, ...]) -> bool:
if index + len(source) > len(text):
return False
return all(ord(text[index + offset]) == code for offset, code in enumerate(source))
class KiratRaiHeraldConverter:
"""Convert the permuted Sikkim Herald layout through the canonical SIL map."""
def __init__(self, canonical: KiratRaiLegacyConverter) -> None:
self._canonical = canonical
@classmethod
def from_map_file(cls, path: str | Path = DEFAULT_KIRATRAI_LEGACY_MAP) -> "KiratRaiHeraldConverter":
return cls(KiratRaiLegacyConverter.from_map_file(path))
def convert(self, text: str) -> KiratRaiLegacyConversion:
output: list[str] = []
canonical_run: list[str] = []
unmapped: set[str] = set()
replacements = 0
def flush() -> None:
nonlocal replacements
if not canonical_run:
return
result = self._canonical.convert("".join(canonical_run))
output.append(result.unicode_text)
replacements += result.replacement_count
unmapped.update(result.unmapped_codepoints)
canonical_run.clear()
for char in text:
remapped = KIRATRAI_HERALD_PREMAP.get(char)
if remapped is not None:
canonical_run.append(remapped)
continue
punctuation = KIRATRAI_HERALD_PUNCTUATION_MAP.get(char)
if punctuation is not None:
canonical_run.append(punctuation)
continue
if char in KIRATRAI_HERALD_BLANK_GLYPHS:
canonical_run.append(" ")
continue
if char in KIRATRAI_HERALD_PASSTHROUGH:
canonical_run.append(char)
continue
if 0x16D40 <= ord(char) <= 0x16D7F:
flush()
output.append(char)
continue
flush()
output.append(char)
unmapped.add(f"U+{ord(char):04X}")
flush()
converted = unicodedata.normalize("NFC", "".join(output))
return KiratRaiLegacyConversion(
legacy_text=text,
unicode_text=converted,
kiratrai_char_count=len(_KIRATRAI_CODEPOINT_RE.findall(converted)),
replacement_count=replacements,
unmapped_codepoints=sorted(unmapped),
)
def convert_kiratrai_legacy_text(
text: str, map_path: str | Path = DEFAULT_KIRATRAI_LEGACY_MAP
) -> KiratRaiLegacyConversion:
return KiratRaiLegacyConverter.from_map_file(map_path).convert(text)
def convert_kiratrai_herald_text(
text: str, map_path: str | Path = DEFAULT_KIRATRAI_LEGACY_MAP
) -> KiratRaiLegacyConversion:
"""Convert Sikkim Herald's older/permuted Kirat Rai layout to Unicode."""
return KiratRaiHeraldConverter.from_map_file(map_path).convert(text)
|