vimeml-tiny-zh-v1 / source /scripts /benchmarks /prepare_homophone_ime.py
Voltline's picture
Release vimeml-tiny-zh-v1 (step 844730): FP32 bundle, INT8 Core ML packages, source snapshot
81cffe7 verified
Raw History Blame Contribute Delete
21.6 kB
"""Written-text homophone challenge track from one pinned Japanese Wikipedia shard.
Standard IME targets are conversational, and the error analysis left few semantic
errors there; the AJIMEE gap is encyclopedic homophones. This track tests that gap
directly, with no model involved in selection:
- lexicon: UniDic lemmas (common nouns, verbs, adjectives) sharing reading, part of
speech and conjugation type, each lemma >= 2 characters and >= MIN_COUNT tokens in
the shard. Equal UniDic lemma means a notation variant (聞く/聴く), so groups contain
only different words, consistent with vimeml.benchmarks.notation;
- spans: a 6-20 character conversion span containing one group word at word
boundaries, with paragraph left context, readings agreed by UniDic and Sudachi
standalone and in context (the Standard extraction rule);
- pair design: for each sampled group, one item whose reference is the most frequent
lemma and one whose reference is the second, from different articles, so always
choosing the frequent word scores about half;
- screens: sentence containment/near match against the local training text (Standard
v1 audit) and any 12-character overlap with AJIMEE.
Commands: download (pinned shard), prepare (unscored draft), review-pack (AI review
sheet), freeze (AI expert labels, never native gold).
"""
import argparse
import collections
import copy
import json
import random
import re
import sys
import uuid
from datetime import datetime, timezone
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT / 'src'))
sys.path.insert(0, str(ROOT / 'scripts/benchmarks'))
from vimeml.training.data import write_json
from prepare_standard_ime import RUN, KANJI, normalized, overlap_audit
from prepare_standard_ime_v2 import PAIRS, read, fresh
from standard_ime import track_files
VERSION = 'ime-homophone-ja-v1'
SEED = 20261021
REPO, REVISION = 'wikimedia/wikipedia', 'b04c8d1ceb2f5cd4588862100d08de323dccfbaa'
SHARD = '20231101.ja/train-00014-of-00015.parquet'
SOURCE = ROOT / f'datasets/{VERSION}/jawiki'
GROUPS = 320
MIN_COUNT = 30
PER_LEMMA = 3
CONTEXT = 200
CONTENT = ('名詞', '動詞', '形容詞')
FUNCTION = ('助詞', '助動詞', '接尾辞', '補助記号')
LATIN = re.compile(r'[A-Za-z0-9A-Za-z0-9]')
def download(args):
from huggingface_hub import hf_hub_download
path = Path(hf_hub_download(REPO, SHARD, repo_type='dataset', revision=REVISION, local_dir=SOURCE))
write_json(SOURCE / 'download-manifest.json', {
'repository': f'https://huggingface.co/datasets/{REPO}', 'revision': REVISION, 'file': SHARD,
'bytes': path.stat().st_size, 'license': 'CC-BY-SA-4.0 (Wikipedia text); GFDL dual-licensed upstream',
'purpose': 'Evaluation-only; never a training input',
'shard_choice': 'Last shard of the 20231101 dump (latest article IDs)'})
print(path)
def lines():
import pyarrow.parquet as pq
table = pq.read_table(SOURCE / SHARD, columns=['id', 'url', 'title', 'text'])
for row in table.to_pylist():
for number, line in enumerate(row['text'].split('\n')):
if len(line) >= 30 and not line[0].isspace() and len(LATIN.findall(line)) <= .2 * len(line):
yield row, number, line
def homophone_key(word):
"""(reading, POS, conjugation type), lemma for a content word that can head a group."""
f = word.feature
if (word.is_unk or f.pos1 not in CONTENT or f.pos2 in ('固有名詞', '数詞', '非自立可能') or not f.lemma
or '-' in f.lemma or len(f.lemma) < 2 or not KANJI.search(f.lemma) or not KANJI.search(word.surface)
or not f.lForm or len(f.lForm) < 2 or f.pos1 == '名詞' and f.kana != f.lForm):
return None, None
return (f.lForm, f.pos1, f.cType), f.lemma
def lexicon(reader):
counts = collections.Counter()
for _, _, line in lines():
for word in reader.mecab(line):
key, lemma = homophone_key(word)
if key:
counts[key, lemma] += 1
groups = collections.defaultdict(dict)
for (key, lemma), n in counts.items():
if n >= MIN_COUNT:
groups[key][lemma] = n
return {'|'.join(key): dict(sorted(members.items(), key=lambda x: -x[1]))
for key, members in sorted(groups.items()) if len(members) >= 2}
def boundaries(reader, line, words):
sudachi = {w.begin() for w in reader.sudachi.tokenize(line, reader.mode)} | {len(line)}
starts, offset = [], 0
for i, word in enumerate(words):
previous = words[i - 1].feature if i else None
compound = previous is not None and (previous.pos1 == '接頭辞' or previous.pos1 == '名詞' and word.feature.pos1 == '名詞')
starts.append((offset, offset in sudachi and not compound and word.feature.pos1 not in FUNCTION
and word.feature.pos2 != '非自立可能'))
offset += len(word.surface)
return starts, sudachi
def spans(reader, line, words, h, rng):
"""Seeded word-boundary span [start, end) of 6-20 characters around word h."""
starts, sudachi = boundaries(reader, line, words)
ends = []
for j in range(h, len(words)):
if words[j].feature.pos1 == '補助記号':
break
end = starts[j][0] + len(words[j].surface)
if j + 1 == len(words) or words[j + 1].feature.pos1 == '補助記号' or starts[j + 1][1]:
ends.append(end)
options = []
for s in range(max(0, h - 3), h + 1):
if not starts[s][1] or any(w.feature.pos1 == '補助記号' for w in words[s:h]):
continue
options += [(starts[s][0], e) for e in ends if 6 <= e - starts[s][0] <= 20]
rng.shuffle(options)
return options
def sentence(line, start, end):
left = line.rfind('。', 0, start) + 1
right = line.find('。', end)
return line[left:len(line) if right < 0 else right + 1]
def candidates(reader, groups, rng):
"""Up to PER_LEMMA seeded reservoir occurrences per (group, lemma), then extraction checks."""
members = {(key, lemma) for key, m in groups.items() for lemma in m}
seen, reservoir = collections.Counter(), collections.defaultdict(list)
for row, number, line in lines():
for h, word in enumerate(reader.mecab(line)):
key, lemma = homophone_key(word)
if key is None or ('|'.join(key), lemma) not in members:
continue
slot = '|'.join(key), lemma
seen[slot] += 1
sample = (row['id'], number, h)
if len(reservoir[slot]) < 4 * PER_LEMMA:
reservoir[slot].append(sample)
else:
k = rng.randrange(seen[slot])
if k < 4 * PER_LEMMA:
reservoir[slot][k] = sample
wanted = collections.defaultdict(list)
for slot, samples in reservoir.items():
for a, n, h in samples:
wanted[a, n].append((slot, h))
found, rejects = collections.defaultdict(list), collections.Counter()
for row, number, line in lines():
if (row['id'], number) not in wanted:
continue
words = list(reader.mecab(line))
for slot, h in wanted[row['id'], number]:
if len(found[slot]) >= PER_LEMMA:
continue
case, reason = extract(reader, row, number, line, words, h, slot, groups, rng)
if case is None:
rejects[reason] += 1
else:
found[slot].append(case)
return found, rejects
def extract(reader, row, number, line, words, h, slot, groups, rng):
key, lemma = slot
reason = 'no_agreed_span'
for start, end in spans(reader, line, words, h, rng)[:8]:
target = line[start:end]
context = line[:start]
if not RUN.fullmatch(target) or not KANJI.search(target):
continue
if len(context) < 10:
reason = 'short_context'
continue
if len(context) > CONTEXT:
cut = context.find('。', len(context) - CONTEXT)
context = context[cut + 1:] if 0 <= cut < len(context) - 10 else context[-CONTEXT:]
standalone = reader.analyse(target)
contextual = reader.analyse(line[:end], start)
if (standalone['unknown'] or contextual['unknown'] or not standalone['agreement'] or not standalone['kana_only']
or standalone['unidic'] != contextual['unidic'] or standalone['sudachi'] != contextual['sudachi']):
continue
reading, pos, ctype = key.split('|')
return {'index': 'pending', 'context_text': context, 'input': standalone['unidic'], 'expected_output': [target],
'category': 'jawiki-homophone',
'provenance': {'source': 'jawiki', 'source_doc_id': row['id'], 'title': row['title'], 'url': row['url'],
'line': number, 'span_start': start, 'span_end': end,
'cleaned_message': sentence(line, start, end),
'homophone': {'surface': words[h].surface, 'lemma': lemma, 'reading': reading, 'pos': pos,
'conjugation': ctype, 'competitors': [m for m in groups[key] if m != lemma],
'group_counts': groups[key]},
'dataset': f'{REPO}@{REVISION}/{SHARD}', 'license': 'CC-BY-SA-4.0'},
'review': {'status': 'pending', 'human_native_review': False, 'not_formal_gold': True,
'reading_evidence': 'Two orthographic dictionaries agree; review required',
'acceptable_forms_exhaustive': False}}, None
return None, reason
def ajimee_screen(pool):
"""Pool positions sharing any normalized 12-character substring with an AJIMEE sentence."""
def grams(text):
return {text[i:i + 12] for i in range(max(0, len(text) - 11))}
seen = set()
for row in read(ROOT / 'artifacts/benchmarks/ajimee-jwtd-v2-v1/evaluation_items.json'):
seen |= grams(normalized(row['original_text']))
return {i for i, r in enumerate(pool) if grams(normalized(r['provenance']['cleaned_message'])) & seen}
def prepare(args):
from vimeml.benchmarks.readings import DualReading
out = args.output
fresh(out)
reader = DualReading()
rng = random.Random(SEED)
groups = lexicon(reader)
write_json(out / 'lexicon.json', groups)
print(f'{len(groups)} homophone groups', flush=True)
found, rejects = candidates(reader, groups, rng)
pool = [case for slot in sorted(found) for case in found[slot]]
for case in pool:
case['provenance']['document_id'] = case['provenance']['source_doc_id']
training = overlap_audit(pool, out)
ajimee = ajimee_screen(pool)
reasons = collections.Counter(extraction=sum(rejects.values()), training_overlap=len(training), ajimee_overlap=len(ajimee))
by_slot = collections.defaultdict(list)
for i, case in enumerate(pool):
if i not in training and i not in ajimee:
h = case['provenance']['homophone']
by_slot['|'.join((h['reading'], h['pos'], h['conjugation'])), h['lemma']].append(case)
order = sorted(groups)
rng.shuffle(order)
selected, articles, used_groups = [], set(), 0
for key in order:
if used_groups == GROUPS:
break
pair = list(groups[key])[:2]
picks = []
for lemma in pair:
pick = next((c for c in by_slot[key, lemma] if c['provenance']['source_doc_id'] not in articles
and all(c['provenance']['source_doc_id'] != p['provenance']['source_doc_id'] for p in picks)), None)
if pick is None:
break
picks.append(pick)
if len(picks) < 2:
reasons['group_without_both_lemmas'] += 1
continue
used_groups += 1
for rank, case in enumerate(picks):
case = copy.deepcopy(case)
case['index'] = f'{VERSION}-{len(selected) + 1:04}'
case['provenance'].update(group_rank=rank, group_id=f'homophone:{key}', release=VERSION)
selected.append(case)
articles.add(case['provenance']['source_doc_id'])
if used_groups != GROUPS:
raise ValueError(f'Only {used_groups} complete groups; no padding or silent quota change.')
track_files(out / 'challenge', selected)
stats = read(out / 'challenge/stats.json')
stats.update(groups=used_groups, pos=dict(collections.Counter(r['provenance']['homophone']['pos'] for r in selected)),
target_length=dict(collections.Counter('6-10' if len(r['expected_output'][0]) <= 10 else '11-20' for r in selected)))
write_json(out / 'challenge/stats.json', stats)
write_json(out / 'manifest.json', {
'format': 'vimeml_standard_ime_preparation_v1', 'version': f'{VERSION}-draft',
'status': 'awaiting_label_review_and_candidate_export', 'created_utc': datetime.now(timezone.utc).isoformat(),
'seed': SEED, 'source': {'dataset': f'https://huggingface.co/datasets/{REPO}', 'revision': REVISION, 'file': SHARD,
'license': 'CC-BY-SA-4.0'},
'splits': {'challenge': len(selected)}, 'labels_formal_gold': False, 'models_scored': False,
'candidate_generation_uses_model_scores': False, 'native_review_completed': False,
'lexicon_policy': f'UniDic lemma groups by (lForm, POS, conjugation); lemmas >= 2 characters, >= {MIN_COUNT} tokens in the shard; '
'no proper nouns, numerals or auxiliary-like verbs; nouns read without rendaku',
'selection_policy': f'Seed {SEED}; {GROUPS} uniformly sampled groups x two most frequent lemmas; one item per article; '
f'{PER_LEMMA} extracted candidates per lemma from a seeded reservoir',
'exclusions': dict(reasons), 'extraction_rejections': dict(rejects),
'contamination': 'Wikipedia is not a direct training source, but web crawls may contain copies; '
'screened only against the local training text, not the V4.3 expansion shards'})
print(json.dumps({'selected': len(selected), 'pool': len(pool), 'exclusions': reasons, 'stats': stats}, ensure_ascii=False, indent=1))
def review_pack(args):
"""AI review sheet in the Standard format plus the homophone group; no model output or candidates."""
from vimeml.benchmarks.readings import DualReading
reader = DualReading()
rows = read(args.prepared / 'challenge/evaluation_items.json')
if args.output.exists():
raise ValueError(f'Preserve existing review sheet: {args.output}')
sheet = []
for row in rows:
target = row['expected_output'][0]
forms = {target}
for group in PAIRS:
for current in group:
if current in target:
forms.update(target.replace(current, alt) for alt in group)
suggestions = [f for f in sorted(forms - {target}) if (lambda c: not c['unknown'] and c['unidic'] == row['input']
and c['sudachi'] == row['input'])(reader.analyse(f))]
h = row['provenance']['homophone']
sheet.append({'id': row['index'], 'source': 'jawiki', 'left_context': row['context_text'], 'target': target,
'proposed_reading': row['input'], 'accepted_answers': [target],
'original_message': row['provenance']['cleaned_message'], 'title': row['provenance']['title'],
'homophone': {k: h[k] for k in ('surface', 'lemma', 'reading', 'competitors')},
'spelling_suggestions': suggestions, 'decision': 'pending', 'reviewer': '',
'review_type': 'ai_expert', 'native_japanese_review': False, 'notes': ''})
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(''.join(json.dumps(r, ensure_ascii=False) + '\n' for r in sheet), encoding='utf-8')
def notice(output):
(output / 'NOTICE.md').write_text("""# Source attribution and use scope
Local research benchmark. Conversion spans, readings and left contexts are
derived from Japanese Wikipedia (dump 20231101, via the Hugging Face dataset
wikimedia/wikipedia at a pinned revision; see manifest.json). Wikipedia text is
licensed CC BY-SA 4.0; adapted material keeps that license. Each case records the
source article title and URL in provenance for attribution.
Selection used dictionaries and corpus counts only: no LM scores, candidate pools
or model outputs. Labels are AI expert reviewed, not human native gold.
""", encoding='utf-8')
def freeze(args):
fresh(args.output)
manifest = read(args.prepared / 'manifest.json')
original = read(args.prepared / 'challenge/evaluation_items.json')
reviews = [json.loads(line) for line in args.reviews.read_text(encoding='utf-8').splitlines() if line.strip()]
by_id = {r['id']: r for r in reviews}
if len(by_id) != len(reviews) or set(by_id) != {r['index'] for r in original}:
raise ValueError('Review IDs do not match the prepared draft.')
selected, quarantine = [], []
for row in original:
review = by_id[row['index']]
if review['left_context'] != row['context_text'] or review['target'] != row['expected_output'][0]:
raise ValueError('Review context/target changed; prepare a new version.')
if not review['reviewer'].strip() or not review['notes'].strip():
raise ValueError(f'Review not complete: {row["index"]}')
if review['review_type'] != 'ai_expert' or review['native_japanese_review'] is not False:
raise ValueError('AI expert review must be identified explicitly, without a native-human claim.')
if review['decision'] == 'exclude':
quarantine.append({'case': row, 'review': review})
continue
if review['decision'] != 'accept':
raise ValueError(f'Pending review: {row["index"]}')
answers = review['accepted_answers']
if not answers or len(set(answers)) != len(answers) or any(not isinstance(a, str) or not a.strip() for a in answers):
raise ValueError('Accepted forms must be a nonempty unique string list.')
if not re.fullmatch(r'[ァ-ヺー。、!?「」『』()・:,,.!?]{4,64}', review['proposed_reading']):
raise ValueError('Reviewed input must follow the 4-64-character orthographic katakana protocol.')
frozen = copy.deepcopy(row)
frozen['input'] = review['proposed_reading']
frozen['expected_output'] = answers
frozen['review'] = {'status': 'ai_expert_reviewed', 'human_native_review': False, 'not_formal_gold': True,
'reviewer': review['reviewer'], 'notes': review['notes'], 'acceptable_forms_exhaustive': False}
selected.append(frozen)
track_files(args.output / 'challenge', selected)
changed = sum(by_id[r['index']]['proposed_reading'] != r['input'] for r in original)
multiple = sum(len(r['expected_output']) > 1 for r in selected)
complete = collections.Counter(r['provenance']['group_id'] for r in selected)
manifest.update(version=VERSION, release_id=uuid.uuid4().hex, status='labels_frozen_awaiting_candidates',
frozen_utc=datetime.now(timezone.utc).isoformat(), splits={'challenge': len(selected)},
labels_frozen=True, review_completed=True, label_quality='ai_expert_reviewed',
review_policy='Frozen pre-score AI expert references; not human native gold',
label_change_policy='Immutable release; corrections require a new release ID and an explicit changelog',
complete_pairs=sum(n == 2 for n in complete.values()),
change_note=f'{len(original)} cases reviewed before any candidate export or scoring; {len(quarantine)} excluded, '
f'{changed} readings corrected, {multiple} with multiple acceptable forms')
write_json(args.output / 'manifest.json', manifest)
write_json(args.output / 'review-decisions.json', reviews)
write_json(args.output / 'quarantine.json', quarantine)
write_json(args.output / 'overlap-audit.json', read(args.prepared / 'overlap-audit.json'))
notice(args.output)
print(manifest['change_note'])
def main():
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
subs = parser.add_subparsers(dest='command', required=True)
subs.add_parser('download')
sub = subs.add_parser('prepare')
sub.add_argument('--output', type=Path, default=ROOT / f'artifacts/benchmarks/{VERSION}-draft')
sub = subs.add_parser('review-pack')
sub.add_argument('--prepared', type=Path, default=ROOT / f'artifacts/benchmarks/{VERSION}-draft')
sub.add_argument('--output', type=Path, default=ROOT / f'artifacts/benchmarks/{VERSION}-review/challenge-review.jsonl')
sub = subs.add_parser('freeze')
sub.add_argument('--prepared', type=Path, default=ROOT / f'artifacts/benchmarks/{VERSION}-draft')
sub.add_argument('--reviews', type=Path, required=True)
sub.add_argument('--output', type=Path, default=ROOT / f'artifacts/benchmarks/{VERSION}-ai-expert')
args = parser.parse_args()
{'download': download, 'prepare': prepare, 'review-pack': review_pack, 'freeze': freeze}[args.command](args)
if __name__ == '__main__':
main()