Download source/scripts/benchmarks/prepare_homophone_ime.py from Voltline/vimeml-tiny-zh-v1: direct link, hf CLI and curl.
- Browser
- Download file 21.6 kB
-
https://huggingface.co/Voltline/vimeml-tiny-zh-v1/resolve/main/source/scripts/benchmarks/prepare_homophone_ime.py
- Command line
-
hf download hf://Voltline/vimeml-tiny-zh-v1/source/scripts/benchmarks/prepare_homophone_ime.py
-
curl -L -o prepare_homophone_ime.py https://huggingface.co/Voltline/vimeml-tiny-zh-v1/resolve/main/source/scripts/benchmarks/prepare_homophone_ime.py
21.6 kB
| """Written-text homophone challenge track from one pinned Japanese Wikipedia shard. | |
| Standard IME targets are conversational, and the error analysis left few semantic | |
| errors there; the AJIMEE gap is encyclopedic homophones. This track tests that gap | |
| directly, with no model involved in selection: | |
| - lexicon: UniDic lemmas (common nouns, verbs, adjectives) sharing reading, part of | |
| speech and conjugation type, each lemma >= 2 characters and >= MIN_COUNT tokens in | |
| the shard. Equal UniDic lemma means a notation variant (聞く/聴く), so groups contain | |
| only different words, consistent with vimeml.benchmarks.notation; | |
| - spans: a 6-20 character conversion span containing one group word at word | |
| boundaries, with paragraph left context, readings agreed by UniDic and Sudachi | |
| standalone and in context (the Standard extraction rule); | |
| - pair design: for each sampled group, one item whose reference is the most frequent | |
| lemma and one whose reference is the second, from different articles, so always | |
| choosing the frequent word scores about half; | |
| - screens: sentence containment/near match against the local training text (Standard | |
| v1 audit) and any 12-character overlap with AJIMEE. | |
| Commands: download (pinned shard), prepare (unscored draft), review-pack (AI review | |
| sheet), freeze (AI expert labels, never native gold). | |
| """ | |
| import argparse | |
| import collections | |
| import copy | |
| import json | |
| import random | |
| import re | |
| import sys | |
| import uuid | |
| from datetime import datetime, timezone | |
| from pathlib import Path | |
| ROOT = Path(__file__).resolve().parents[2] | |
| sys.path.insert(0, str(ROOT / 'src')) | |
| sys.path.insert(0, str(ROOT / 'scripts/benchmarks')) | |
| from vimeml.training.data import write_json | |
| from prepare_standard_ime import RUN, KANJI, normalized, overlap_audit | |
| from prepare_standard_ime_v2 import PAIRS, read, fresh | |
| from standard_ime import track_files | |
| VERSION = 'ime-homophone-ja-v1' | |
| SEED = 20261021 | |
| REPO, REVISION = 'wikimedia/wikipedia', 'b04c8d1ceb2f5cd4588862100d08de323dccfbaa' | |
| SHARD = '20231101.ja/train-00014-of-00015.parquet' | |
| SOURCE = ROOT / f'datasets/{VERSION}/jawiki' | |
| GROUPS = 320 | |
| MIN_COUNT = 30 | |
| PER_LEMMA = 3 | |
| CONTEXT = 200 | |
| CONTENT = ('名詞', '動詞', '形容詞') | |
| FUNCTION = ('助詞', '助動詞', '接尾辞', '補助記号') | |
| LATIN = re.compile(r'[A-Za-z0-9A-Za-z0-9]') | |
| def download(args): | |
| from huggingface_hub import hf_hub_download | |
| path = Path(hf_hub_download(REPO, SHARD, repo_type='dataset', revision=REVISION, local_dir=SOURCE)) | |
| write_json(SOURCE / 'download-manifest.json', { | |
| 'repository': f'https://huggingface.co/datasets/{REPO}', 'revision': REVISION, 'file': SHARD, | |
| 'bytes': path.stat().st_size, 'license': 'CC-BY-SA-4.0 (Wikipedia text); GFDL dual-licensed upstream', | |
| 'purpose': 'Evaluation-only; never a training input', | |
| 'shard_choice': 'Last shard of the 20231101 dump (latest article IDs)'}) | |
| print(path) | |
| def lines(): | |
| import pyarrow.parquet as pq | |
| table = pq.read_table(SOURCE / SHARD, columns=['id', 'url', 'title', 'text']) | |
| for row in table.to_pylist(): | |
| for number, line in enumerate(row['text'].split('\n')): | |
| if len(line) >= 30 and not line[0].isspace() and len(LATIN.findall(line)) <= .2 * len(line): | |
| yield row, number, line | |
| def homophone_key(word): | |
| """(reading, POS, conjugation type), lemma for a content word that can head a group.""" | |
| f = word.feature | |
| if (word.is_unk or f.pos1 not in CONTENT or f.pos2 in ('固有名詞', '数詞', '非自立可能') or not f.lemma | |
| or '-' in f.lemma or len(f.lemma) < 2 or not KANJI.search(f.lemma) or not KANJI.search(word.surface) | |
| or not f.lForm or len(f.lForm) < 2 or f.pos1 == '名詞' and f.kana != f.lForm): | |
| return None, None | |
| return (f.lForm, f.pos1, f.cType), f.lemma | |
| def lexicon(reader): | |
| counts = collections.Counter() | |
| for _, _, line in lines(): | |
| for word in reader.mecab(line): | |
| key, lemma = homophone_key(word) | |
| if key: | |
| counts[key, lemma] += 1 | |
| groups = collections.defaultdict(dict) | |
| for (key, lemma), n in counts.items(): | |
| if n >= MIN_COUNT: | |
| groups[key][lemma] = n | |
| return {'|'.join(key): dict(sorted(members.items(), key=lambda x: -x[1])) | |
| for key, members in sorted(groups.items()) if len(members) >= 2} | |
| def boundaries(reader, line, words): | |
| sudachi = {w.begin() for w in reader.sudachi.tokenize(line, reader.mode)} | {len(line)} | |
| starts, offset = [], 0 | |
| for i, word in enumerate(words): | |
| previous = words[i - 1].feature if i else None | |
| compound = previous is not None and (previous.pos1 == '接頭辞' or previous.pos1 == '名詞' and word.feature.pos1 == '名詞') | |
| starts.append((offset, offset in sudachi and not compound and word.feature.pos1 not in FUNCTION | |
| and word.feature.pos2 != '非自立可能')) | |
| offset += len(word.surface) | |
| return starts, sudachi | |
| def spans(reader, line, words, h, rng): | |
| """Seeded word-boundary span [start, end) of 6-20 characters around word h.""" | |
| starts, sudachi = boundaries(reader, line, words) | |
| ends = [] | |
| for j in range(h, len(words)): | |
| if words[j].feature.pos1 == '補助記号': | |
| break | |
| end = starts[j][0] + len(words[j].surface) | |
| if j + 1 == len(words) or words[j + 1].feature.pos1 == '補助記号' or starts[j + 1][1]: | |
| ends.append(end) | |
| options = [] | |
| for s in range(max(0, h - 3), h + 1): | |
| if not starts[s][1] or any(w.feature.pos1 == '補助記号' for w in words[s:h]): | |
| continue | |
| options += [(starts[s][0], e) for e in ends if 6 <= e - starts[s][0] <= 20] | |
| rng.shuffle(options) | |
| return options | |
| def sentence(line, start, end): | |
| left = line.rfind('。', 0, start) + 1 | |
| right = line.find('。', end) | |
| return line[left:len(line) if right < 0 else right + 1] | |
| def candidates(reader, groups, rng): | |
| """Up to PER_LEMMA seeded reservoir occurrences per (group, lemma), then extraction checks.""" | |
| members = {(key, lemma) for key, m in groups.items() for lemma in m} | |
| seen, reservoir = collections.Counter(), collections.defaultdict(list) | |
| for row, number, line in lines(): | |
| for h, word in enumerate(reader.mecab(line)): | |
| key, lemma = homophone_key(word) | |
| if key is None or ('|'.join(key), lemma) not in members: | |
| continue | |
| slot = '|'.join(key), lemma | |
| seen[slot] += 1 | |
| sample = (row['id'], number, h) | |
| if len(reservoir[slot]) < 4 * PER_LEMMA: | |
| reservoir[slot].append(sample) | |
| else: | |
| k = rng.randrange(seen[slot]) | |
| if k < 4 * PER_LEMMA: | |
| reservoir[slot][k] = sample | |
| wanted = collections.defaultdict(list) | |
| for slot, samples in reservoir.items(): | |
| for a, n, h in samples: | |
| wanted[a, n].append((slot, h)) | |
| found, rejects = collections.defaultdict(list), collections.Counter() | |
| for row, number, line in lines(): | |
| if (row['id'], number) not in wanted: | |
| continue | |
| words = list(reader.mecab(line)) | |
| for slot, h in wanted[row['id'], number]: | |
| if len(found[slot]) >= PER_LEMMA: | |
| continue | |
| case, reason = extract(reader, row, number, line, words, h, slot, groups, rng) | |
| if case is None: | |
| rejects[reason] += 1 | |
| else: | |
| found[slot].append(case) | |
| return found, rejects | |
| def extract(reader, row, number, line, words, h, slot, groups, rng): | |
| key, lemma = slot | |
| reason = 'no_agreed_span' | |
| for start, end in spans(reader, line, words, h, rng)[:8]: | |
| target = line[start:end] | |
| context = line[:start] | |
| if not RUN.fullmatch(target) or not KANJI.search(target): | |
| continue | |
| if len(context) < 10: | |
| reason = 'short_context' | |
| continue | |
| if len(context) > CONTEXT: | |
| cut = context.find('。', len(context) - CONTEXT) | |
| context = context[cut + 1:] if 0 <= cut < len(context) - 10 else context[-CONTEXT:] | |
| standalone = reader.analyse(target) | |
| contextual = reader.analyse(line[:end], start) | |
| if (standalone['unknown'] or contextual['unknown'] or not standalone['agreement'] or not standalone['kana_only'] | |
| or standalone['unidic'] != contextual['unidic'] or standalone['sudachi'] != contextual['sudachi']): | |
| continue | |
| reading, pos, ctype = key.split('|') | |
| return {'index': 'pending', 'context_text': context, 'input': standalone['unidic'], 'expected_output': [target], | |
| 'category': 'jawiki-homophone', | |
| 'provenance': {'source': 'jawiki', 'source_doc_id': row['id'], 'title': row['title'], 'url': row['url'], | |
| 'line': number, 'span_start': start, 'span_end': end, | |
| 'cleaned_message': sentence(line, start, end), | |
| 'homophone': {'surface': words[h].surface, 'lemma': lemma, 'reading': reading, 'pos': pos, | |
| 'conjugation': ctype, 'competitors': [m for m in groups[key] if m != lemma], | |
| 'group_counts': groups[key]}, | |
| 'dataset': f'{REPO}@{REVISION}/{SHARD}', 'license': 'CC-BY-SA-4.0'}, | |
| 'review': {'status': 'pending', 'human_native_review': False, 'not_formal_gold': True, | |
| 'reading_evidence': 'Two orthographic dictionaries agree; review required', | |
| 'acceptable_forms_exhaustive': False}}, None | |
| return None, reason | |
| def ajimee_screen(pool): | |
| """Pool positions sharing any normalized 12-character substring with an AJIMEE sentence.""" | |
| def grams(text): | |
| return {text[i:i + 12] for i in range(max(0, len(text) - 11))} | |
| seen = set() | |
| for row in read(ROOT / 'artifacts/benchmarks/ajimee-jwtd-v2-v1/evaluation_items.json'): | |
| seen |= grams(normalized(row['original_text'])) | |
| return {i for i, r in enumerate(pool) if grams(normalized(r['provenance']['cleaned_message'])) & seen} | |
| def prepare(args): | |
| from vimeml.benchmarks.readings import DualReading | |
| out = args.output | |
| fresh(out) | |
| reader = DualReading() | |
| rng = random.Random(SEED) | |
| groups = lexicon(reader) | |
| write_json(out / 'lexicon.json', groups) | |
| print(f'{len(groups)} homophone groups', flush=True) | |
| found, rejects = candidates(reader, groups, rng) | |
| pool = [case for slot in sorted(found) for case in found[slot]] | |
| for case in pool: | |
| case['provenance']['document_id'] = case['provenance']['source_doc_id'] | |
| training = overlap_audit(pool, out) | |
| ajimee = ajimee_screen(pool) | |
| reasons = collections.Counter(extraction=sum(rejects.values()), training_overlap=len(training), ajimee_overlap=len(ajimee)) | |
| by_slot = collections.defaultdict(list) | |
| for i, case in enumerate(pool): | |
| if i not in training and i not in ajimee: | |
| h = case['provenance']['homophone'] | |
| by_slot['|'.join((h['reading'], h['pos'], h['conjugation'])), h['lemma']].append(case) | |
| order = sorted(groups) | |
| rng.shuffle(order) | |
| selected, articles, used_groups = [], set(), 0 | |
| for key in order: | |
| if used_groups == GROUPS: | |
| break | |
| pair = list(groups[key])[:2] | |
| picks = [] | |
| for lemma in pair: | |
| pick = next((c for c in by_slot[key, lemma] if c['provenance']['source_doc_id'] not in articles | |
| and all(c['provenance']['source_doc_id'] != p['provenance']['source_doc_id'] for p in picks)), None) | |
| if pick is None: | |
| break | |
| picks.append(pick) | |
| if len(picks) < 2: | |
| reasons['group_without_both_lemmas'] += 1 | |
| continue | |
| used_groups += 1 | |
| for rank, case in enumerate(picks): | |
| case = copy.deepcopy(case) | |
| case['index'] = f'{VERSION}-{len(selected) + 1:04}' | |
| case['provenance'].update(group_rank=rank, group_id=f'homophone:{key}', release=VERSION) | |
| selected.append(case) | |
| articles.add(case['provenance']['source_doc_id']) | |
| if used_groups != GROUPS: | |
| raise ValueError(f'Only {used_groups} complete groups; no padding or silent quota change.') | |
| track_files(out / 'challenge', selected) | |
| stats = read(out / 'challenge/stats.json') | |
| stats.update(groups=used_groups, pos=dict(collections.Counter(r['provenance']['homophone']['pos'] for r in selected)), | |
| target_length=dict(collections.Counter('6-10' if len(r['expected_output'][0]) <= 10 else '11-20' for r in selected))) | |
| write_json(out / 'challenge/stats.json', stats) | |
| write_json(out / 'manifest.json', { | |
| 'format': 'vimeml_standard_ime_preparation_v1', 'version': f'{VERSION}-draft', | |
| 'status': 'awaiting_label_review_and_candidate_export', 'created_utc': datetime.now(timezone.utc).isoformat(), | |
| 'seed': SEED, 'source': {'dataset': f'https://huggingface.co/datasets/{REPO}', 'revision': REVISION, 'file': SHARD, | |
| 'license': 'CC-BY-SA-4.0'}, | |
| 'splits': {'challenge': len(selected)}, 'labels_formal_gold': False, 'models_scored': False, | |
| 'candidate_generation_uses_model_scores': False, 'native_review_completed': False, | |
| 'lexicon_policy': f'UniDic lemma groups by (lForm, POS, conjugation); lemmas >= 2 characters, >= {MIN_COUNT} tokens in the shard; ' | |
| 'no proper nouns, numerals or auxiliary-like verbs; nouns read without rendaku', | |
| 'selection_policy': f'Seed {SEED}; {GROUPS} uniformly sampled groups x two most frequent lemmas; one item per article; ' | |
| f'{PER_LEMMA} extracted candidates per lemma from a seeded reservoir', | |
| 'exclusions': dict(reasons), 'extraction_rejections': dict(rejects), | |
| 'contamination': 'Wikipedia is not a direct training source, but web crawls may contain copies; ' | |
| 'screened only against the local training text, not the V4.3 expansion shards'}) | |
| print(json.dumps({'selected': len(selected), 'pool': len(pool), 'exclusions': reasons, 'stats': stats}, ensure_ascii=False, indent=1)) | |
| def review_pack(args): | |
| """AI review sheet in the Standard format plus the homophone group; no model output or candidates.""" | |
| from vimeml.benchmarks.readings import DualReading | |
| reader = DualReading() | |
| rows = read(args.prepared / 'challenge/evaluation_items.json') | |
| if args.output.exists(): | |
| raise ValueError(f'Preserve existing review sheet: {args.output}') | |
| sheet = [] | |
| for row in rows: | |
| target = row['expected_output'][0] | |
| forms = {target} | |
| for group in PAIRS: | |
| for current in group: | |
| if current in target: | |
| forms.update(target.replace(current, alt) for alt in group) | |
| suggestions = [f for f in sorted(forms - {target}) if (lambda c: not c['unknown'] and c['unidic'] == row['input'] | |
| and c['sudachi'] == row['input'])(reader.analyse(f))] | |
| h = row['provenance']['homophone'] | |
| sheet.append({'id': row['index'], 'source': 'jawiki', 'left_context': row['context_text'], 'target': target, | |
| 'proposed_reading': row['input'], 'accepted_answers': [target], | |
| 'original_message': row['provenance']['cleaned_message'], 'title': row['provenance']['title'], | |
| 'homophone': {k: h[k] for k in ('surface', 'lemma', 'reading', 'competitors')}, | |
| 'spelling_suggestions': suggestions, 'decision': 'pending', 'reviewer': '', | |
| 'review_type': 'ai_expert', 'native_japanese_review': False, 'notes': ''}) | |
| args.output.parent.mkdir(parents=True, exist_ok=True) | |
| args.output.write_text(''.join(json.dumps(r, ensure_ascii=False) + '\n' for r in sheet), encoding='utf-8') | |
| def notice(output): | |
| (output / 'NOTICE.md').write_text("""# Source attribution and use scope | |
| Local research benchmark. Conversion spans, readings and left contexts are | |
| derived from Japanese Wikipedia (dump 20231101, via the Hugging Face dataset | |
| wikimedia/wikipedia at a pinned revision; see manifest.json). Wikipedia text is | |
| licensed CC BY-SA 4.0; adapted material keeps that license. Each case records the | |
| source article title and URL in provenance for attribution. | |
| Selection used dictionaries and corpus counts only: no LM scores, candidate pools | |
| or model outputs. Labels are AI expert reviewed, not human native gold. | |
| """, encoding='utf-8') | |
| def freeze(args): | |
| fresh(args.output) | |
| manifest = read(args.prepared / 'manifest.json') | |
| original = read(args.prepared / 'challenge/evaluation_items.json') | |
| reviews = [json.loads(line) for line in args.reviews.read_text(encoding='utf-8').splitlines() if line.strip()] | |
| by_id = {r['id']: r for r in reviews} | |
| if len(by_id) != len(reviews) or set(by_id) != {r['index'] for r in original}: | |
| raise ValueError('Review IDs do not match the prepared draft.') | |
| selected, quarantine = [], [] | |
| for row in original: | |
| review = by_id[row['index']] | |
| if review['left_context'] != row['context_text'] or review['target'] != row['expected_output'][0]: | |
| raise ValueError('Review context/target changed; prepare a new version.') | |
| if not review['reviewer'].strip() or not review['notes'].strip(): | |
| raise ValueError(f'Review not complete: {row["index"]}') | |
| if review['review_type'] != 'ai_expert' or review['native_japanese_review'] is not False: | |
| raise ValueError('AI expert review must be identified explicitly, without a native-human claim.') | |
| if review['decision'] == 'exclude': | |
| quarantine.append({'case': row, 'review': review}) | |
| continue | |
| if review['decision'] != 'accept': | |
| raise ValueError(f'Pending review: {row["index"]}') | |
| answers = review['accepted_answers'] | |
| if not answers or len(set(answers)) != len(answers) or any(not isinstance(a, str) or not a.strip() for a in answers): | |
| raise ValueError('Accepted forms must be a nonempty unique string list.') | |
| if not re.fullmatch(r'[ァ-ヺー。、!?「」『』()・:,,.!?]{4,64}', review['proposed_reading']): | |
| raise ValueError('Reviewed input must follow the 4-64-character orthographic katakana protocol.') | |
| frozen = copy.deepcopy(row) | |
| frozen['input'] = review['proposed_reading'] | |
| frozen['expected_output'] = answers | |
| frozen['review'] = {'status': 'ai_expert_reviewed', 'human_native_review': False, 'not_formal_gold': True, | |
| 'reviewer': review['reviewer'], 'notes': review['notes'], 'acceptable_forms_exhaustive': False} | |
| selected.append(frozen) | |
| track_files(args.output / 'challenge', selected) | |
| changed = sum(by_id[r['index']]['proposed_reading'] != r['input'] for r in original) | |
| multiple = sum(len(r['expected_output']) > 1 for r in selected) | |
| complete = collections.Counter(r['provenance']['group_id'] for r in selected) | |
| manifest.update(version=VERSION, release_id=uuid.uuid4().hex, status='labels_frozen_awaiting_candidates', | |
| frozen_utc=datetime.now(timezone.utc).isoformat(), splits={'challenge': len(selected)}, | |
| labels_frozen=True, review_completed=True, label_quality='ai_expert_reviewed', | |
| review_policy='Frozen pre-score AI expert references; not human native gold', | |
| label_change_policy='Immutable release; corrections require a new release ID and an explicit changelog', | |
| complete_pairs=sum(n == 2 for n in complete.values()), | |
| change_note=f'{len(original)} cases reviewed before any candidate export or scoring; {len(quarantine)} excluded, ' | |
| f'{changed} readings corrected, {multiple} with multiple acceptable forms') | |
| write_json(args.output / 'manifest.json', manifest) | |
| write_json(args.output / 'review-decisions.json', reviews) | |
| write_json(args.output / 'quarantine.json', quarantine) | |
| write_json(args.output / 'overlap-audit.json', read(args.prepared / 'overlap-audit.json')) | |
| notice(args.output) | |
| print(manifest['change_note']) | |
| def main(): | |
| parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) | |
| subs = parser.add_subparsers(dest='command', required=True) | |
| subs.add_parser('download') | |
| sub = subs.add_parser('prepare') | |
| sub.add_argument('--output', type=Path, default=ROOT / f'artifacts/benchmarks/{VERSION}-draft') | |
| sub = subs.add_parser('review-pack') | |
| sub.add_argument('--prepared', type=Path, default=ROOT / f'artifacts/benchmarks/{VERSION}-draft') | |
| sub.add_argument('--output', type=Path, default=ROOT / f'artifacts/benchmarks/{VERSION}-review/challenge-review.jsonl') | |
| sub = subs.add_parser('freeze') | |
| sub.add_argument('--prepared', type=Path, default=ROOT / f'artifacts/benchmarks/{VERSION}-draft') | |
| sub.add_argument('--reviews', type=Path, required=True) | |
| sub.add_argument('--output', type=Path, default=ROOT / f'artifacts/benchmarks/{VERSION}-ai-expert') | |
| args = parser.parse_args() | |
| {'download': download, 'prepare': prepare, 'review-pack': review_pack, 'freeze': freeze}[args.command](args) | |
| if __name__ == '__main__': | |
| main() | |