| { | |
| "schema_version": 1, | |
| "protocol": "Single-message public ONNX runtime; strict lexical accuracy excludes neutral and ambiguous tokens; exact cases accept allowed alternative labels.", | |
| "annotation_status": "AI-authored provisional rubric, user-supplied annotations and AI contextual review; not independent expert gold", | |
| "stress_suites": { | |
| "conversation_regression": { | |
| "suite_sha256": "de879b6b7c1c4c28a44bef7882da6396ddd2b11a08ec1db1d90435bf67b8eae3", | |
| "fp32": { | |
| "cases": 100, | |
| "strict_lexical_tokens": 640, | |
| "ambiguous_tokens": 13, | |
| "strict_lexical_accuracy": 0.9234375, | |
| "exact_case_accuracy": 0.73, | |
| "high_confidence_errors": 14, | |
| "per_class": { | |
| "O": { | |
| "precision": 0.9938837920489296, | |
| "recall": 1.0, | |
| "f1": 0.9969325153374233, | |
| "support": 325 | |
| }, | |
| "BA": { | |
| "precision": 0.9377162629757786, | |
| "recall": 0.9124579124579124, | |
| "f1": 0.924914675767918, | |
| "support": 297 | |
| }, | |
| "RU": { | |
| "precision": 0.9788732394366197, | |
| "recall": 0.9423728813559322, | |
| "f1": 0.9602763385146805, | |
| "support": 295 | |
| }, | |
| "TAT": { | |
| "precision": 0.5142857142857142, | |
| "recall": 0.75, | |
| "f1": 0.6101694915254237, | |
| "support": 24 | |
| }, | |
| "OTHER": { | |
| "precision": 0.8, | |
| "recall": 1.0, | |
| "f1": 0.888888888888889, | |
| "support": 24 | |
| } | |
| } | |
| }, | |
| "int8": { | |
| "cases": 100, | |
| "strict_lexical_tokens": 640, | |
| "ambiguous_tokens": 13, | |
| "strict_lexical_accuracy": 0.925, | |
| "exact_case_accuracy": 0.73, | |
| "high_confidence_errors": 15, | |
| "per_class": { | |
| "O": { | |
| "precision": 0.9938837920489296, | |
| "recall": 1.0, | |
| "f1": 0.9969325153374233, | |
| "support": 325 | |
| }, | |
| "BA": { | |
| "precision": 0.9319727891156463, | |
| "recall": 0.9225589225589226, | |
| "f1": 0.9272419627749577, | |
| "support": 297 | |
| }, | |
| "RU": { | |
| "precision": 0.9822064056939501, | |
| "recall": 0.9355932203389831, | |
| "f1": 0.9583333333333334, | |
| "support": 295 | |
| }, | |
| "TAT": { | |
| "precision": 0.5294117647058824, | |
| "recall": 0.75, | |
| "f1": 0.6206896551724139, | |
| "support": 24 | |
| }, | |
| "OTHER": { | |
| "precision": 0.8275862068965517, | |
| "recall": 1.0, | |
| "f1": 0.9056603773584906, | |
| "support": 24 | |
| } | |
| } | |
| } | |
| }, | |
| "borderline_conversations": { | |
| "suite_sha256": "c9c5439627b1e4f74fcf481b30c3f291e9d2f3a763bdd370a1491cd924094ce1", | |
| "fp32": { | |
| "cases": 40, | |
| "strict_lexical_tokens": 263, | |
| "ambiguous_tokens": 0, | |
| "strict_lexical_accuracy": 0.9239543726235742, | |
| "exact_case_accuracy": 0.675, | |
| "high_confidence_errors": 7, | |
| "per_class": { | |
| "O": { | |
| "precision": 1.0, | |
| "recall": 1.0, | |
| "f1": 1.0, | |
| "support": 115 | |
| }, | |
| "BA": { | |
| "precision": 0.9357798165137615, | |
| "recall": 0.9026548672566371, | |
| "f1": 0.918918918918919, | |
| "support": 113 | |
| }, | |
| "RU": { | |
| "precision": 0.95, | |
| "recall": 0.9743589743589743, | |
| "f1": 0.9620253164556962, | |
| "support": 117 | |
| }, | |
| "TAT": { | |
| "precision": 0.75, | |
| "recall": 0.8181818181818182, | |
| "f1": 0.7826086956521738, | |
| "support": 22 | |
| }, | |
| "OTHER": { | |
| "precision": 0.9, | |
| "recall": 0.8181818181818182, | |
| "f1": 0.8571428571428572, | |
| "support": 11 | |
| } | |
| } | |
| }, | |
| "int8": { | |
| "cases": 40, | |
| "strict_lexical_tokens": 263, | |
| "ambiguous_tokens": 0, | |
| "strict_lexical_accuracy": 0.9315589353612167, | |
| "exact_case_accuracy": 0.725, | |
| "high_confidence_errors": 9, | |
| "per_class": { | |
| "O": { | |
| "precision": 1.0, | |
| "recall": 1.0, | |
| "f1": 1.0, | |
| "support": 115 | |
| }, | |
| "BA": { | |
| "precision": 0.9217391304347826, | |
| "recall": 0.9380530973451328, | |
| "f1": 0.9298245614035088, | |
| "support": 113 | |
| }, | |
| "RU": { | |
| "precision": 0.9655172413793104, | |
| "recall": 0.9572649572649573, | |
| "f1": 0.9613733905579399, | |
| "support": 117 | |
| }, | |
| "TAT": { | |
| "precision": 0.8181818181818182, | |
| "recall": 0.8181818181818182, | |
| "f1": 0.8181818181818182, | |
| "support": 22 | |
| }, | |
| "OTHER": { | |
| "precision": 0.9, | |
| "recall": 0.8181818181818182, | |
| "f1": 0.8571428571428572, | |
| "support": 11 | |
| } | |
| } | |
| } | |
| }, | |
| "new_household_conversations": { | |
| "suite_sha256": "da216b8d4b411a595fe842b11421d6b793fedd38c85c89963c3adf72fcd3f282", | |
| "fp32": { | |
| "cases": 40, | |
| "strict_lexical_tokens": 344, | |
| "ambiguous_tokens": 0, | |
| "strict_lexical_accuracy": 0.936046511627907, | |
| "exact_case_accuracy": 0.75, | |
| "high_confidence_errors": 16, | |
| "per_class": { | |
| "O": { | |
| "precision": 1.0, | |
| "recall": 1.0, | |
| "f1": 1.0, | |
| "support": 104 | |
| }, | |
| "BA": { | |
| "precision": 0.925, | |
| "recall": 0.9098360655737705, | |
| "f1": 0.9173553719008265, | |
| "support": 122 | |
| }, | |
| "RU": { | |
| "precision": 0.9736842105263158, | |
| "recall": 0.9946236559139785, | |
| "f1": 0.9840425531914894, | |
| "support": 186 | |
| }, | |
| "TAT": { | |
| "precision": 0.8125, | |
| "recall": 0.7222222222222222, | |
| "f1": 0.7647058823529411, | |
| "support": 36 | |
| }, | |
| "OTHER": { | |
| "precision": 0.0, | |
| "recall": 0, | |
| "f1": 0, | |
| "support": 0 | |
| } | |
| } | |
| }, | |
| "int8": { | |
| "cases": 40, | |
| "strict_lexical_tokens": 344, | |
| "ambiguous_tokens": 0, | |
| "strict_lexical_accuracy": 0.9476744186046512, | |
| "exact_case_accuracy": 0.825, | |
| "high_confidence_errors": 16, | |
| "per_class": { | |
| "O": { | |
| "precision": 1.0, | |
| "recall": 1.0, | |
| "f1": 1.0, | |
| "support": 104 | |
| }, | |
| "BA": { | |
| "precision": 0.9333333333333333, | |
| "recall": 0.9180327868852459, | |
| "f1": 0.9256198347107439, | |
| "support": 122 | |
| }, | |
| "RU": { | |
| "precision": 0.9789473684210527, | |
| "recall": 1.0, | |
| "f1": 0.9893617021276596, | |
| "support": 186 | |
| }, | |
| "TAT": { | |
| "precision": 0.8235294117647058, | |
| "recall": 0.7777777777777778, | |
| "f1": 0.7999999999999999, | |
| "support": 36 | |
| }, | |
| "OTHER": { | |
| "precision": 0, | |
| "recall": 0, | |
| "f1": 0, | |
| "support": 0 | |
| } | |
| } | |
| }, | |
| "fp32_pytorch_parity": { | |
| "cases": 40, | |
| "max_probability_delta": 5.066394805908203e-06, | |
| "disagreement_tokens": 0, | |
| "passed": true | |
| } | |
| } | |
| }, | |
| "selection_validation": { | |
| "earlier_annotations": { | |
| "rows": 139, | |
| "masked_tokens": 13, | |
| "lexical_accuracy": 0.9244, | |
| "used_for_epoch_selection": true | |
| }, | |
| "borderline_annotations": { | |
| "rows": 77, | |
| "masked_tokens": 56, | |
| "lexical_accuracy": 0.9376000000000001, | |
| "used_for_epoch_selection": true | |
| }, | |
| "new_household_annotations": { | |
| "rows": 61, | |
| "masked_tokens": 33, | |
| "lexical_accuracy": 0.9043000000000001, | |
| "used_for_epoch_selection": true | |
| } | |
| }, | |
| "limitations": [ | |
| "Authored stress tests do not measure natural traffic accuracy.", | |
| "Previously inspected suites are regression tests; new household suite has two context variants per independent core.", | |
| "No-diacritic BA/TAT text can be intrinsically ambiguous.", | |
| "Shared words, Russian-root inflections and short switches can be mislabeled with high confidence." | |
| ] | |
| } | |