WIlfLin commited on
Commit
782bc12
·
verified ·
1 Parent(s): 2737bcb

Clarify token metrics and model-only formatting behavior

Browse files
Files changed (3) hide show
  1. README.md +17 -1
  2. evaluation-summary.json +12 -11
  3. token-metrics.json +86 -0
README.md CHANGED
@@ -59,7 +59,23 @@ Chinese-data continued full fine-tuning of **S1-mini** by **Superwhisper**, for
59
 
60
  词法指标删除标点,保留大小写、数字、否定和词序。它反映与参考文本的一致程度,不等于人工语义正确率。英文集侧重复杂改口,不能代表所有日常英文口述。中文清理改善伴随不修改参考上的退化;不能假定所有编辑均正确。
61
 
62
- 外部后处理服务的保护规则会改变最终结果。部署默认使用较宽松的 `balanced` 校验,允许更多删词、重复清理和英文语序整理;保留数字、否定、代码、引用与明确改口最终内容检查。`strict` 是更保守的可选策略。**这些校验不是模型权重的一部分**;本仓库的推理示例直接输出模型结果。原始模型分数不能直接当作整条部署流水线的成绩。完整数值与范围见 [`evaluation-summary.json`](evaluation-summary.json)。
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
63
 
64
  ## 推理示例
65
 
 
59
 
60
  词法指标删除标点,保留大小写、数字、否定和词序。它反映与参考文本的一致程度,不等于人工语义正确率。英文集侧重复杂改口,不能代表所有日常英文口述。中文清理改善伴随不修改参考上的退化;不能假定所有编辑均正确。
61
 
62
+ 外部后处理服务的保护规则会改变最终结果。部署默认使用较宽松的 `balanced` 校验,允许更多删词、重复清理和英文语序整理;保留数字、否定、代码、引用与明确改口最终内容检查。`strict` 是更保守的可选策略。**这些校验不是模型权重的一部分**;本仓库的推理示例直接输出模型结果。原始模型分数不能直接当作整条部署流水线的成绩。校验拒绝时直接返回原始转写,不做额外规则改写、CT 标点或规则排版。完整数值与范围见 [`evaluation-summary.json`](evaluation-summary.json)。
63
+
64
+
65
+ ### Token accuracy 的单独核对
66
+
67
+ 上游公开报告的 94.8% 来自其 7519 条英文测试和 Q4_K_M GGUF。公开模型卡/文章未给出 token 定义、对齐、聚合方式或评分脚本,因此不能把下面结果称作官方同口径复现。
68
+
69
+ 在相同 100 条英文 Disfl-QA、相同 tokenizer 和提示格式下,我们另外计算了:
70
+
71
+ | 明确定义的本地指标 | 原版 S1-mini | S1-mini-Chinese |
72
+ | --- | ---: | ---: |
73
+ | 教师强制目标 token 准确率,不含 EOS | 90.25% | 94.63% |
74
+ | 自由生成 token 编辑准确率 | 61.15% | 86.77% |
75
+
76
+ 教师强制指标在每步提供真实的先前目标 token,只统计目标文字,不计 system/user/control 提示词;按所有目标 token 微平均。我们的计数是 1252/1323。它不是自由生成整句正确率。
77
+
78
+ 自由生成指标为 `1 - sum(token Levenshtein distance) / sum(reference token count)`,使用完整输出的同一 tokenizer ID,保留标点和大小写,不含 EOS。它也不是官方已确认的公式。完整定义和计数见 [`token-metrics.json`](token-metrics.json)。
79
 
80
  ## 推理示例
81
 
evaluation-summary.json CHANGED
@@ -59,19 +59,26 @@
59
  "truncated": 0
60
  }
61
  },
62
- "balanced_postprocessor_offline": {
 
 
 
 
 
 
 
63
  "all": {
64
  "n": 400,
65
  "lexical_exact": 208,
66
  "exact": 86,
67
- "cer": 0.214497528830313,
68
  "truncated": 0
69
  },
70
  "test_en": {
71
  "n": 100,
72
  "lexical_exact": 34,
73
  "exact": 32,
74
- "cer": 0.28697487051601767,
75
  "truncated": 0
76
  },
77
  "test_identity": {
@@ -85,15 +92,9 @@
85
  "n": 200,
86
  "lexical_exact": 80,
87
  "exact": 14,
88
- "cer": 0.1737271667314419,
89
  "truncated": 0
90
  }
91
  },
92
- "scope": "400 frozen public text rows: Chinese cleanup 200, Chinese identity 100, English Disfl-QA 100; lexical CER strips punctuation, not semantic accuracy. Postprocessor is separately calibrated and not an independent blind evaluation.",
93
- "test_manifest": {
94
- "rows": 400,
95
- "seed": 20261107,
96
- "sha256": "55d21ed3b91bd19093f6e4620d76565ec36afd9ad3c32a05c95f1e43a785fd88",
97
- "selection": "200 Chinese cleanup + 100 disjoint Chinese target-to-target identity + 100 English cleanup, official test splits only. No subsequent tuning on this test."
98
- }
99
  }
 
59
  "truncated": 0
60
  }
61
  },
62
+ "scope": "400 frozen public text rows: Chinese cleanup 200, Chinese identity 100, English Disfl-QA 100; lexical CER strips punctuation, not semantic accuracy. Postprocessor is separately calibrated and not an independent blind evaluation.",
63
+ "test_manifest": {
64
+ "rows": 400,
65
+ "seed": 20261107,
66
+ "sha256": "55d21ed3b91bd19093f6e4620d76565ec36afd9ad3c32a05c95f1e43a785fd88",
67
+ "selection": "200 Chinese cleanup + 100 disjoint Chinese target-to-target identity + 100 English cleanup, official test splits only. No subsequent tuning on this test."
68
+ },
69
+ "balanced_guard_raw_fallback_offline": {
70
  "all": {
71
  "n": 400,
72
  "lexical_exact": 208,
73
  "exact": 86,
74
+ "cer": 0.2158154859967051,
75
  "truncated": 0
76
  },
77
  "test_en": {
78
  "n": 100,
79
  "lexical_exact": 34,
80
  "exact": 32,
81
+ "cer": 0.28850949549203914,
82
  "truncated": 0
83
  },
84
  "test_identity": {
 
92
  "n": 200,
93
  "lexical_exact": 80,
94
  "exact": 14,
95
+ "cer": 0.17528177225029148,
96
  "truncated": 0
97
  }
98
  },
99
+ "token_metric_definitions": "token-metrics.json; official vendor formula and dataset unavailable, not a strict reproduction."
 
 
 
 
 
 
100
  }
token-metrics.json ADDED
@@ -0,0 +1,86 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "official_comparability": "Not established: official 7519-case dataset, tokenization, alignment, aggregation and scoring script are not published in the cited model card/blog. These are explicitly defined local metrics.",
3
+ "data": "Same frozen 400 public text cases used for both models; categories English Disfl-QA 100, Chinese cleanup 200, Chinese identity 100.",
4
+ "definitions": {
5
+ "teacher_forced": "Correct argmax next-token predictions at target positions, conditioned on gold preceding target tokens; excludes system/user/control tokens. Both including and excluding target EOS are reported. Micro aggregation.",
6
+ "generated_token_edit_accuracy": "1 - total Levenshtein edits / total reference token count, using shared Qwen3/S1 tokenizer IDs on full generated/reference strings; preserves case and punctuation; includes no EOS. This is not sentence accuracy or a verified definition of vendor token accuracy."
7
+ },
8
+ "models": {
9
+ "upstream_s1_mini": {
10
+ "test_zh": {
11
+ "reference_generated_tokens": 1576,
12
+ "generated_edit_distance": 633,
13
+ "cases": 200,
14
+ "target_tokens_including_eos": 1776,
15
+ "correct_including_eos": 1311,
16
+ "target_tokens_excluding_eos": 1576,
17
+ "correct_excluding_eos": 1234,
18
+ "teacher_forced_accuracy_including_eos": 0.7381756756756757,
19
+ "teacher_forced_accuracy_excluding_eos": 0.7829949238578681,
20
+ "generated_token_edit_accuracy": 0.5983502538071066
21
+ },
22
+ "test_identity": {
23
+ "reference_generated_tokens": 780,
24
+ "generated_edit_distance": 72,
25
+ "cases": 100,
26
+ "target_tokens_including_eos": 880,
27
+ "correct_including_eos": 811,
28
+ "target_tokens_excluding_eos": 780,
29
+ "correct_excluding_eos": 770,
30
+ "teacher_forced_accuracy_including_eos": 0.9215909090909091,
31
+ "teacher_forced_accuracy_excluding_eos": 0.9871794871794872,
32
+ "generated_token_edit_accuracy": 0.9076923076923077
33
+ },
34
+ "test_en": {
35
+ "reference_generated_tokens": 1323,
36
+ "generated_edit_distance": 514,
37
+ "cases": 100,
38
+ "target_tokens_including_eos": 1423,
39
+ "correct_including_eos": 1290,
40
+ "target_tokens_excluding_eos": 1323,
41
+ "correct_excluding_eos": 1194,
42
+ "teacher_forced_accuracy_including_eos": 0.9065354884047786,
43
+ "teacher_forced_accuracy_excluding_eos": 0.9024943310657596,
44
+ "generated_token_edit_accuracy": 0.6114890400604687
45
+ }
46
+ },
47
+ "s1_mini_chinese": {
48
+ "test_zh": {
49
+ "reference_generated_tokens": 1576,
50
+ "generated_edit_distance": 505,
51
+ "cases": 200,
52
+ "target_tokens_including_eos": 1776,
53
+ "correct_including_eos": 1362,
54
+ "target_tokens_excluding_eos": 1576,
55
+ "correct_excluding_eos": 1286,
56
+ "teacher_forced_accuracy_including_eos": 0.7668918918918919,
57
+ "teacher_forced_accuracy_excluding_eos": 0.815989847715736,
58
+ "generated_token_edit_accuracy": 0.6795685279187818
59
+ },
60
+ "test_identity": {
61
+ "reference_generated_tokens": 780,
62
+ "generated_edit_distance": 73,
63
+ "cases": 100,
64
+ "target_tokens_including_eos": 880,
65
+ "correct_including_eos": 809,
66
+ "target_tokens_excluding_eos": 780,
67
+ "correct_excluding_eos": 771,
68
+ "teacher_forced_accuracy_including_eos": 0.9193181818181818,
69
+ "teacher_forced_accuracy_excluding_eos": 0.9884615384615385,
70
+ "generated_token_edit_accuracy": 0.9064102564102564
71
+ },
72
+ "test_en": {
73
+ "reference_generated_tokens": 1323,
74
+ "generated_edit_distance": 175,
75
+ "cases": 100,
76
+ "target_tokens_including_eos": 1423,
77
+ "correct_including_eos": 1348,
78
+ "target_tokens_excluding_eos": 1323,
79
+ "correct_excluding_eos": 1252,
80
+ "teacher_forced_accuracy_including_eos": 0.9472944483485594,
81
+ "teacher_forced_accuracy_excluding_eos": 0.9463340891912321,
82
+ "generated_token_edit_accuracy": 0.8677248677248677
83
+ }
84
+ }
85
+ }
86
+ }