File size: 7,179 Bytes
81cffe7 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 | """Second Mac candidate handoff: new frozen tracks at n-best 20 and 50, plus pool checks.
Tracks and why they are exported:
- standard-v2-blind, homophone-v1: new frozen releases (n20 for comparability with the
registered protocol, n50 for the next benchmark version);
- standard-v1-development: n50 measures how many pool misses a larger n-best recovers;
n20 repeats the original export to check determinism;
- standard-v1-development-nocontext (n20): the same inputs with empty left context,
to test whether AzooKey pools depend on context with Zenzai off (free-running
incremental evaluation is valid only if they do not);
- incremental-proto1: segmented development prototype.
No language model runs during export. Commands: package, import.
"""
import argparse
import copy
import hashlib
import json
import shutil
import sys
import zipfile
from datetime import datetime, timezone
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT / 'src'))
sys.path.insert(0, str(ROOT / 'scripts/benchmarks'))
from vimeml.benchmarks.expanded_ime import CONVERTER
from vimeml.benchmarks.standard_ime import FORMAT, load_standard_export
from vimeml.training.data import write_json
from prepare_standard_ime_v2 import read, fresh
from standard_ime import track_files
BENCH = ROOT / 'artifacts/benchmarks'
VERSION = 'ime-candidates-v2'
TRACKS = {
'standard-v2-blind': (BENCH / 'ime-standard-ja-v2-blind-ai-expert', 'blind', (20, 50)),
'homophone-v1': (BENCH / 'ime-homophone-ja-v1-ai-expert', 'challenge', (20, 50)),
'standard-v1-development': (BENCH / 'ime-standard-ja-v1-ai-expert-r1', 'development', (20, 50)),
'standard-v1-development-nocontext': (BENCH / 'ime-standard-ja-v1-ai-expert-r1', 'development', (20,)),
'incremental-proto1': (BENCH / 'ime-incremental-ja-proto1', 'segmented', (20, 50)),
}
FILES = ('ajimee-input.json', 'case-map.json', 'azookey-candidates.json', 'converter-version.txt',
'dictionary-versions.txt', 'swift-version.txt', 'export-flags.txt', 'completed.txt', 'export.log')
def items(track):
release, split, _ = TRACKS[track]
rows = read(release / split / 'evaluation_items.json')
if track.endswith('-nocontext'):
rows = copy.deepcopy(rows)
for row in rows:
row['context_text'] = ''
return rows
def package(args):
out = args.output
fresh(out)
tracks = {}
for track, (release, split, n_best) in TRACKS.items():
track_files(out / track, items(track))
(out / track / 'evaluation_items.json').unlink()
(out / track / 'stats.json').unlink()
manifest = read(release / 'manifest.json')
tracks[track] = {'release': release.name, 'release_id': manifest.get('release_id'), 'split': split,
'n_best': list(n_best), 'cases': len(read(out / track / 'case-map.json')),
'input_sha256': hashlib.sha256((out / track / 'ajimee-input.json').read_bytes()).hexdigest()}
# LF: the Mac shell reads this line by line (CRLF made n_best "20\r").
(out / 'tracks.tsv').write_text(''.join(f'{t}\t{n}\n' for t, v in tracks.items() for n in v['n_best']),
encoding='utf-8', newline='\n')
write_json(out / 'manifest.json', {'version': VERSION, 'created_utc': datetime.now(timezone.utc).isoformat(),
'converter': CONVERTER, 'flags': 'typo off, Zenzai disabled, stable off',
'tracks': tracks, 'lm_scoring': False})
for name in ('export_azookey_tracks.sh', 'run_candidates_mac.sh'):
shutil.copyfile(ROOT / 'scripts/benchmarks' / name, out / name)
shutil.copyfile(ROOT / 'templates/benchmarks/candidates-v2-mac.md', out / 'README.md')
for source in ('wrime', 'jmultiwoz'):
shutil.copyfile(ROOT / f'datasets/ime-standard-ja-v1/{source}/LICENSE', out / f'LICENSE-{source}.txt')
(out / 'NOTICE.md').write_text(
(BENCH / 'ime-standard-ja-v2-blind-ai-expert/NOTICE.md').read_text(encoding='utf-8') + '\n' +
(BENCH / 'ime-homophone-ja-v1-ai-expert/NOTICE.md').read_text(encoding='utf-8'), encoding='utf-8', newline='\n')
with zipfile.ZipFile(out.with_suffix('.zip'), 'x', zipfile.ZIP_DEFLATED) as archive:
for path in sorted(out.rglob('*')):
if path.is_file():
archive.write(path, str(path.relative_to(out.parent)))
print(json.dumps(tracks, indent=1))
def import_results(args):
"""Each (track, n-best) becomes a loadable standard export; inputs must match the package byte for byte."""
handoff = read(args.package / 'manifest.json')
fresh(args.output)
with zipfile.ZipFile(args.archive) as archive:
names = archive.namelist()
for track, info in handoff['tracks'].items():
for n in info['n_best']:
payload = {}
for filename in FILES:
suffix = f'azookey-results/{track}/n{n}/{filename}'
matches = [name for name in names if name == suffix or name.endswith('/' + suffix)]
if len(matches) != 1:
raise ValueError(f'Expected exactly one {suffix}.')
payload[filename] = archive.read(matches[0])
for filename in ('ajimee-input.json', 'case-map.json'):
if payload[filename] != (args.package / track / filename).read_bytes():
raise ValueError(f'{track}/n{n}: returned {filename} differs from the package.')
dest = args.output / f'{track}-n{n}'
rows = items(track)
track_files(dest, rows)
(dest / 'ajimee-results').mkdir()
for filename, content in payload.items():
(dest / 'ajimee-results' / filename).write_bytes(content)
release, split, _ = TRACKS[track]
manifest = {**read(release / 'manifest.json'), 'format': FORMAT, 'status': 'complete', 'split': split,
'track': track, 'n_best': n, 'stats': read(dest / 'stats.json'),
'candidate_provenance': f'{VERSION} Mac export', 'handoff_created_utc': handoff['created_utc']}
write_json(dest / 'manifest.json', manifest)
_, _, loaded = load_standard_export(dest)
print(f'{track}-n{n}: {len(loaded)} cases loaded', flush=True)
def main():
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
subs = parser.add_subparsers(dest='command', required=True)
sub = subs.add_parser('package')
sub.add_argument('--output', type=Path, default=BENCH / f'{VERSION}-handoff')
sub = subs.add_parser('import')
sub.add_argument('--archive', type=Path, required=True)
sub.add_argument('--package', type=Path, default=BENCH / f'{VERSION}-handoff')
sub.add_argument('--output', type=Path, default=BENCH / f'{VERSION}-imported')
args = parser.parse_args()
{'package': package, 'import': import_results}[args.command](args)
if __name__ == '__main__':
main()
|