File size: 6,881 Bytes
81cffe7 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 | """Prototype segmented (incremental-input) items from Standard v1 development messages.
A real IME user converts a message phrase by phrase, and each conversion sees the text
already committed. This prototype cuts each v1 development message into consecutive
conversion segments at word boundaries (the Standard extraction rule: UniDic/Sudachi
boundary, not inside a compound, not starting with a function or auxiliary-like word),
4-64 kana each, readings agreed by both dictionaries standalone and in context. Text
outside Japanese runs (digits, Latin, emoji) is committed literally.
Every segment item carries the gold committed prefix as left context (teacher-forced).
Candidate pools are exported once; free-running evaluation then replaces the context
by the model's own earlier choices, which is valid only if the n-best pool does not
depend on left context (checked by the blank-context development export).
Labels are dictionary-derived drafts on already exposed development messages: a
protocol prototype, never a capability claim.
"""
import argparse
import collections
import json
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT / 'src'))
sys.path.insert(0, str(ROOT / 'scripts/benchmarks'))
from vimeml.training.data import write_json
from prepare_standard_ime import RUN
from prepare_standard_ime_v2 import read, fresh
from standard_ime import track_files
VERSION = 'ime-incremental-ja-proto1'
SOURCE = ROOT / 'artifacts/benchmarks/ime-standard-ja-v1-ai-expert-r1/development/evaluation_items.json'
TARGET = 8
FUNCTION = ('助詞', '助動詞', '接尾辞', '補助記号')
def cuts(reader, run):
"""Allowed segment starts inside one Japanese run."""
words = list(reader.mecab(run))
sudachi = {w.begin() for w in reader.sudachi.tokenize(run, reader.mode)}
allowed, offset = set(), 0
for i, word in enumerate(words):
f, previous = word.feature, words[i - 1].feature if i else None
compound = previous is not None and (previous.pos1 == '接頭辞' or previous.pos1 == '名詞' and f.pos1 == '名詞')
after_mark = previous is not None and previous.pos1 == '補助記号'
if offset in sudachi and (after_mark or not compound and f.pos1 not in FUNCTION and f.pos2 != '非自立可能'):
allowed.add(offset)
offset += len(word.surface)
return sorted(allowed - {0})
def segment_run(reader, run):
"""Greedy segments of about TARGET characters; None when a reading is not agreed."""
bounds, start = [], 0
for cut in cuts(reader, run):
if cut - start >= TARGET:
bounds.append((start, cut))
start = cut
bounds.append((start, len(run)))
readings = []
for i, (a, b) in enumerate(bounds):
standalone, contextual = reader.analyse(run[a:b]), reader.analyse(run[:b], a)
if (standalone['unknown'] or contextual['unknown'] or not standalone['agreement'] or not standalone['kana_only']
or standalone['unidic'] != contextual['unidic'] or standalone['sudachi'] != contextual['sudachi']):
return None
readings.append(standalone['unidic'])
while len(bounds) > 1 and len(readings[-1]) < 4:
(a, _), (_, b) = bounds[-2], bounds[-1]
merged = reader.analyse(run[:b], a)
if merged['unknown'] or merged['unidic'] != readings[-2] + readings[-1]:
return None
bounds[-2:], readings[-2:] = [(a, b)], [merged['unidic']]
if any(not 4 <= len(r) <= 64 for r in readings):
return None
return list(zip(bounds, readings))
def prepare(args):
from vimeml.benchmarks.readings import DualReading
reader = DualReading()
fresh(args.output)
items, rejects, messages = [], collections.Counter(), 0
for row in read(SOURCE):
p = row['provenance']
text = p['cleaned_message']
segments = []
for run in RUN.finditer(text):
if len(run.group()) < 4:
continue
parts = segment_run(reader, run.group())
if parts is None:
segments = None
break
segments += [(run.start() + a, run.start() + b, reading) for (a, b), reading in parts]
if not segments:
rejects['unsegmentable_message' if segments is None else 'no_japanese_run'] += 1
continue
messages += 1
for n, (start, end, reading) in enumerate(segments, 1):
items.append({'index': f'{VERSION}-{messages:04}-{n:02}', 'context_text': text[:start], 'input': reading,
'expected_output': [text[start:end]], 'category': row['category'],
'provenance': {**{k: p[k] for k in ('source', 'author_id', 'group_id', 'document_id')},
'parent_case': row['index'], 'message': messages, 'segment': n,
'segments': len(segments), 'span_start': start, 'span_end': end,
'cleaned_message': text},
'review': {'status': 'dictionary_draft', 'human_native_review': False, 'not_formal_gold': True,
'acceptable_forms_exhaustive': False}})
track_files(args.output / 'segmented', items)
stats = read(args.output / 'segmented/stats.json')
lengths = collections.Counter(min(i['provenance']['segments'], 6) for i in items if i['provenance']['segment'] == 1)
stats.update(messages=messages, segments_per_message={('6+' if k == 6 else str(k)): v for k, v in sorted(lengths.items())},
rejections=dict(rejects))
write_json(args.output / 'segmented/stats.json', stats)
write_json(args.output / 'manifest.json', {
'version': VERSION, 'status': 'prototype_awaiting_candidate_export', 'parent': str(SOURCE.relative_to(ROOT)),
'splits': {'segmented': len(items)}, 'messages': messages, 'labels_formal_gold': False,
'label_quality': 'dictionary_draft', 'exposure': 'Standard v1 development messages (exposed)',
'segmentation': f'Greedy word-boundary cuts after >= {TARGET} characters; final segments under 4 kana merged',
'evaluation_modes': {'teacher_forced': 'gold committed prefix as context (as exported)',
'free_running': "model's earlier choices replace the gold prefix; requires context-independent pools"}})
print(json.dumps({'messages': messages, 'items': len(items), 'rejections': rejects, 'stats': stats}, ensure_ascii=False, indent=1))
def main():
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument('--output', type=Path, default=ROOT / f'artifacts/benchmarks/{VERSION}')
prepare(parser.parse_args())
if __name__ == '__main__':
main()
|