| { | |
| "format": "vimeml_public_evaluation_zh_v1", | |
| "model_version": "zh-v1-step844730", | |
| "checkpoint_sha256": "44c423a1e7cb0e32161afc94d6167cabb439a3d25fd9de6d1fbb150a927542d4", | |
| "training": { | |
| "arm": "12k-fresh-s1", | |
| "epochs": 1, | |
| "updates": 844730, | |
| "seed": 1591996171, | |
| "supervised_targets": 14519248976, | |
| "web_characters": 21280000000, | |
| "chat_share_of_supervised_tokens": 0.05, | |
| "validation_bpc_fp32": { | |
| "web": 4.354450029201291, | |
| "chat": 5.690730282652201 | |
| }, | |
| "selection": "final checkpoint of the release arm, chosen by the preregistered eight-arm rule before blind use", | |
| "blind_used_for_selection": false | |
| }, | |
| "ime_standard_zh_v1_r1": { | |
| "labels": "AI-reviewed, not native-speaker gold", | |
| "development": { | |
| "full": { | |
| "items": 1456, | |
| "top1": 1363 | |
| }, | |
| "abbrev": { | |
| "items": 990, | |
| "top1": 337 | |
| } | |
| }, | |
| "blind_confirmation": { | |
| "full": { | |
| "items": 1445, | |
| "top1": 1355, | |
| "top1_rate": 0.938, | |
| "top3_rate": 0.972, | |
| "mrr": 0.954 | |
| }, | |
| "abbrev": { | |
| "items": 964, | |
| "top1_rate": 0.35 | |
| }, | |
| "rime_ice_first_candidate_top1_rate": 0.723, | |
| "vs_rime": "+329/-19, McNemar p=3.6e-74", | |
| "passed": true | |
| }, | |
| "baselines_blind_full_top1_rate": { | |
| "vimeml-tiny-zh-v1": 0.938, | |
| "Qwen2.5-0.5B": 0.918, | |
| "Qwen3-0.6B-Base": 0.892, | |
| "rime-ice + wanxiang grammar": 0.822, | |
| "rime-ice": 0.722, | |
| "luna_pinyin_simp": 0.689 | |
| } | |
| }, | |
| "coreml": { | |
| "development_rankings": { | |
| "cuda_fp32": { | |
| "full": { | |
| "items": 1456, | |
| "top1": 1363 | |
| }, | |
| "abbrev": { | |
| "items": 990, | |
| "top1": 337 | |
| } | |
| }, | |
| "coreml_fp32": { | |
| "full": { | |
| "items": 1456, | |
| "top1": 1363, | |
| "top3": 1416 | |
| }, | |
| "abbrev": { | |
| "items": 990, | |
| "top1": 337, | |
| "top3": 389 | |
| } | |
| }, | |
| "coreml_int8": { | |
| "full": { | |
| "items": 1456, | |
| "top1": 1364, | |
| "top3": 1416 | |
| }, | |
| "abbrev": { | |
| "items": 990, | |
| "top1": 335, | |
| "top3": 389 | |
| } | |
| }, | |
| "kvchunk_fp32": { | |
| "full": { | |
| "items": 1456, | |
| "top1": 1363, | |
| "top3": 1416 | |
| }, | |
| "abbrev": { | |
| "items": 990, | |
| "top1": 337, | |
| "top3": 389 | |
| } | |
| }, | |
| "kvchunk_int8": { | |
| "full": { | |
| "items": 1456, | |
| "top1": 1364, | |
| "top3": 1416 | |
| }, | |
| "abbrev": { | |
| "items": 990, | |
| "top1": 335, | |
| "top3": 389 | |
| } | |
| } | |
| }, | |
| "int8_vs_fp32_top1_changes": { | |
| "full": "+2/-1", | |
| "abbrev": "+0/-2" | |
| }, | |
| "fp32_alignment_max_abs": { | |
| "stateless": 3.91e-05, | |
| "kvchunk": 4.63e-05 | |
| }, | |
| "int8_strict_alignment": { | |
| "passed": false, | |
| "max_abs": 0.392, | |
| "note": "expected lossy quantization drift" | |
| }, | |
| "kvchunk_int8_vs_stateless_int8_max_abs": 4.15e-05, | |
| "package_bytes": { | |
| "fp32": 80367198, | |
| "int8": 22792958, | |
| "kvchunk_int8": 22830199 | |
| } | |
| }, | |
| "measurements": { | |
| "mac": { | |
| "source_sha256": "713bb2d455235bf3348828591ae60b6ffcd7a2e637c718b27f416e2edcde6465", | |
| "hardware": "Apple M3, macOS 27.3, CPU_ONLY, Python host process", | |
| "zh_v1_int8": { | |
| "package_bytes": 22792958, | |
| "load_ms_p50": 28.388874998199753, | |
| "warm_p50_of_process_p50_ms": { | |
| "T1": 1.652375000048778, | |
| "T8": 2.4276039985124953, | |
| "T32": 3.84631300039473, | |
| "T64": 5.521499999304069, | |
| "T128": 8.48720900103217 | |
| }, | |
| "warm_p50_of_process_p95_ms": { | |
| "T1": 2.1141144019566127, | |
| "T8": 2.8757605994542246, | |
| "T32": 4.280841699255689, | |
| "T64": 5.88047934925271, | |
| "T128": 8.882592001282319 | |
| } | |
| }, | |
| "models": [ | |
| "v42-fp32", | |
| "v42-int8", | |
| "v42-kvchunk-fp32", | |
| "v42-kvchunk-int8", | |
| "zhv1-fp32", | |
| "zhv1-int8", | |
| "zhv1-kvchunk-fp32", | |
| "zhv1-kvchunk-int8" | |
| ] | |
| }, | |
| "iphone": null, | |
| "keyboard_extension": null | |
| }, | |
| "deployment_status": "Release for iOS deployment in Vime. Core ML packages pass the Mac and simulator functional gates; iPhone and keyboard-extension measurements have not been run yet.", | |
| "scope": "Aggregate metrics only. No corpus, benchmark text, candidate pools, per-case scores, case identifiers, user inputs or raw device traces are distributed." | |
| } | |