| """Literal NFC character codec for the experimental compact TrOCR decoder. |
| |
| Tokenizer BOS0 is serialization only; training targets exclude it. Generation |
| starts with EOS2 and suppresses BOS/PAD/UNK. Unknown outputs remain visible as |
| replacement characters when decoded, never silently removed. |
| """ |
| import unicodedata |
|
|
|
|
| class CharacterCodec: |
| def __init__(self, characters): |
| if len(characters) != len(set(characters)) or any(len(c) != 1 for c in characters): |
| raise ValueError('Require ordered, unique Unicode codepoints') |
| self.characters = list(characters) |
| self.native = set(characters) |
| self.character_ids = {c: i + 4 for i, c in enumerate(characters)} |
| self.id_characters = {i: c for c, i in self.character_ids.items()} |
| self.classes = len(characters) + 4 |
|
|
| def encode(self, text): |
| if not text or text != unicodedata.normalize('NFC', text).strip(): |
| raise ValueError('Require nonempty, stripped NFC text') |
| if any(c not in self.native for c in text): |
| raise ValueError('Text outside the fixed OCR alphabet') |
| return [0, *[self.character_ids[c] for c in text], 2], 'characters' |
|
|
| def decode(self, ids): |
| pieces = [] |
| for index in ids: |
| if index in [0, 1, 2]: |
| continue |
| if index == 3: |
| pieces.append('\ufffd') |
| elif index in self.id_characters: |
| pieces.append(self.id_characters[index]) |
| else: |
| raise ValueError('Unassigned character output ID') |
| return unicodedata.normalize('NFC', ''.join(pieces)) |
|
|
| def manifest(self): |
| return {'version': 1, 'kind': 'literal_NFC_characters', 'classes': self.classes, |
| 'special_ids': {'BOS_serialization': 0, 'PAD': 1, 'EOS_and_decoder_start': 2, 'UNK': 3}, |
| 'character_ids': self.character_ids, 'generation_suppressed_ids': [0, 1, 3], |
| 'training_target': 'Drop serialization BOS0 only; retain every text codepoint and EOS2', |
| 'unknown_output': 'Decode3 as U+FFFD, never silently remove it', |
| 'scope': 'Experimental codec, not recognition quality or Paddle qualification.'} |
|
|
