"""Prototype segmented (incremental-input) items from Standard v1 development messages. A real IME user converts a message phrase by phrase, and each conversion sees the text already committed. This prototype cuts each v1 development message into consecutive conversion segments at word boundaries (the Standard extraction rule: UniDic/Sudachi boundary, not inside a compound, not starting with a function or auxiliary-like word), 4-64 kana each, readings agreed by both dictionaries standalone and in context. Text outside Japanese runs (digits, Latin, emoji) is committed literally. Every segment item carries the gold committed prefix as left context (teacher-forced). Candidate pools are exported once; free-running evaluation then replaces the context by the model's own earlier choices, which is valid only if the n-best pool does not depend on left context (checked by the blank-context development export). Labels are dictionary-derived drafts on already exposed development messages: a protocol prototype, never a capability claim. """ import argparse import collections import json import sys from pathlib import Path ROOT = Path(__file__).resolve().parents[2] sys.path.insert(0, str(ROOT / 'src')) sys.path.insert(0, str(ROOT / 'scripts/benchmarks')) from vimeml.training.data import write_json from prepare_standard_ime import RUN from prepare_standard_ime_v2 import read, fresh from standard_ime import track_files VERSION = 'ime-incremental-ja-proto1' SOURCE = ROOT / 'artifacts/benchmarks/ime-standard-ja-v1-ai-expert-r1/development/evaluation_items.json' TARGET = 8 FUNCTION = ('助詞', '助動詞', '接尾辞', '補助記号') def cuts(reader, run): """Allowed segment starts inside one Japanese run.""" words = list(reader.mecab(run)) sudachi = {w.begin() for w in reader.sudachi.tokenize(run, reader.mode)} allowed, offset = set(), 0 for i, word in enumerate(words): f, previous = word.feature, words[i - 1].feature if i else None compound = previous is not None and (previous.pos1 == '接頭辞' or previous.pos1 == '名詞' and f.pos1 == '名詞') after_mark = previous is not None and previous.pos1 == '補助記号' if offset in sudachi and (after_mark or not compound and f.pos1 not in FUNCTION and f.pos2 != '非自立可能'): allowed.add(offset) offset += len(word.surface) return sorted(allowed - {0}) def segment_run(reader, run): """Greedy segments of about TARGET characters; None when a reading is not agreed.""" bounds, start = [], 0 for cut in cuts(reader, run): if cut - start >= TARGET: bounds.append((start, cut)) start = cut bounds.append((start, len(run))) readings = [] for i, (a, b) in enumerate(bounds): standalone, contextual = reader.analyse(run[a:b]), reader.analyse(run[:b], a) if (standalone['unknown'] or contextual['unknown'] or not standalone['agreement'] or not standalone['kana_only'] or standalone['unidic'] != contextual['unidic'] or standalone['sudachi'] != contextual['sudachi']): return None readings.append(standalone['unidic']) while len(bounds) > 1 and len(readings[-1]) < 4: (a, _), (_, b) = bounds[-2], bounds[-1] merged = reader.analyse(run[:b], a) if merged['unknown'] or merged['unidic'] != readings[-2] + readings[-1]: return None bounds[-2:], readings[-2:] = [(a, b)], [merged['unidic']] if any(not 4 <= len(r) <= 64 for r in readings): return None return list(zip(bounds, readings)) def prepare(args): from vimeml.benchmarks.readings import DualReading reader = DualReading() fresh(args.output) items, rejects, messages = [], collections.Counter(), 0 for row in read(SOURCE): p = row['provenance'] text = p['cleaned_message'] segments = [] for run in RUN.finditer(text): if len(run.group()) < 4: continue parts = segment_run(reader, run.group()) if parts is None: segments = None break segments += [(run.start() + a, run.start() + b, reading) for (a, b), reading in parts] if not segments: rejects['unsegmentable_message' if segments is None else 'no_japanese_run'] += 1 continue messages += 1 for n, (start, end, reading) in enumerate(segments, 1): items.append({'index': f'{VERSION}-{messages:04}-{n:02}', 'context_text': text[:start], 'input': reading, 'expected_output': [text[start:end]], 'category': row['category'], 'provenance': {**{k: p[k] for k in ('source', 'author_id', 'group_id', 'document_id')}, 'parent_case': row['index'], 'message': messages, 'segment': n, 'segments': len(segments), 'span_start': start, 'span_end': end, 'cleaned_message': text}, 'review': {'status': 'dictionary_draft', 'human_native_review': False, 'not_formal_gold': True, 'acceptable_forms_exhaustive': False}}) track_files(args.output / 'segmented', items) stats = read(args.output / 'segmented/stats.json') lengths = collections.Counter(min(i['provenance']['segments'], 6) for i in items if i['provenance']['segment'] == 1) stats.update(messages=messages, segments_per_message={('6+' if k == 6 else str(k)): v for k, v in sorted(lengths.items())}, rejections=dict(rejects)) write_json(args.output / 'segmented/stats.json', stats) write_json(args.output / 'manifest.json', { 'version': VERSION, 'status': 'prototype_awaiting_candidate_export', 'parent': str(SOURCE.relative_to(ROOT)), 'splits': {'segmented': len(items)}, 'messages': messages, 'labels_formal_gold': False, 'label_quality': 'dictionary_draft', 'exposure': 'Standard v1 development messages (exposed)', 'segmentation': f'Greedy word-boundary cuts after >= {TARGET} characters; final segments under 4 kana merged', 'evaluation_modes': {'teacher_forced': 'gold committed prefix as context (as exported)', 'free_running': "model's earlier choices replace the gold prefix; requires context-independent pools"}}) print(json.dumps({'messages': messages, 'items': len(items), 'rejections': rejects, 'stats': stats}, ensure_ascii=False, indent=1)) def main(): parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument('--output', type=Path, default=ROOT / f'artifacts/benchmarks/{VERSION}') prepare(parser.parse_args()) if __name__ == '__main__': main()