File size: 2,230 Bytes
5f70891 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 | """Literal NFC character codec for the experimental compact TrOCR decoder.
Tokenizer BOS0 is serialization only; training targets exclude it. Generation
starts with EOS2 and suppresses BOS/PAD/UNK. Unknown outputs remain visible as
replacement characters when decoded, never silently removed.
"""
import unicodedata
class CharacterCodec:
def __init__(self, characters):
if len(characters) != len(set(characters)) or any(len(c) != 1 for c in characters):
raise ValueError('Require ordered, unique Unicode codepoints')
self.characters = list(characters)
self.native = set(characters)
self.character_ids = {c: i + 4 for i, c in enumerate(characters)}
self.id_characters = {i: c for c, i in self.character_ids.items()}
self.classes = len(characters) + 4
def encode(self, text):
if not text or text != unicodedata.normalize('NFC', text).strip():
raise ValueError('Require nonempty, stripped NFC text')
if any(c not in self.native for c in text):
raise ValueError('Text outside the fixed OCR alphabet')
return [0, *[self.character_ids[c] for c in text], 2], 'characters'
def decode(self, ids):
pieces = []
for index in ids:
if index in [0, 1, 2]:
continue
if index == 3:
pieces.append('\ufffd')
elif index in self.id_characters:
pieces.append(self.id_characters[index])
else:
raise ValueError('Unassigned character output ID')
return unicodedata.normalize('NFC', ''.join(pieces))
def manifest(self):
return {'version': 1, 'kind': 'literal_NFC_characters', 'classes': self.classes,
'special_ids': {'BOS_serialization': 0, 'PAD': 1, 'EOS_and_decoder_start': 2, 'UNK': 3},
'character_ids': self.character_ids, 'generation_suppressed_ids': [0, 1, 3],
'training_target': 'Drop serialization BOS0 only; retain every text codepoint and EOS2',
'unknown_output': 'Decode3 as U+FFFD, never silently remove it',
'scope': 'Experimental codec, not recognition quality or Paddle qualification.'}
|