kem-gov commited on
Commit
eccaaa6
·
verified ·
1 Parent(s): 3760150

Sync model repo (text/metadata)

Browse files
benchmarks/whisper-tiny-vivo-x300-fp32.yaml CHANGED
@@ -5,32 +5,11 @@
5
  # (7.558% / 3.111%), NOT baseline_metadata.yaml — whose accuracy field wrongly
6
  # copies the INT8 (optimized) value (see docs/marvik/upstream.md). WER/CER as PERCENT.
7
  version: 1.0.0
8
- task: speech-asr
 
9
  created_at: '2026-07-13T00:00:00Z'
10
  context:
11
- model:
12
- id: Arm/whisper-tiny-int8-xnnpack-executorch
13
- filename: whisper-tiny-fp32-executorch.pte
14
- base_model_id: openai/whisper-tiny
15
- profile: Baseline
16
- weight_dtype: fp32
17
- model_size_mb: 220.259
18
- parameter_count: 37760640
19
- format: pte
20
- target:
21
- name: Vivo X300
22
- hardware_class: Premium smartphone
23
- cpu_architecture: arm64
24
- cpu_model: C1-Ultra, C1-Premium, C1-Pro
25
- cpu_core_count: 8
26
- arch_feature_flags:
27
- - armv9.2-a
28
- - sve2
29
- - i8mm
30
- - bf16
31
- system_memory_gb: 16
32
- os: android
33
- os_version: Android 16 / OriginOS 6
34
  runtime:
35
  name: executorch
36
  execution_backend: cpu
 
5
  # (7.558% / 3.111%), NOT baseline_metadata.yaml — whose accuracy field wrongly
6
  # copies the INT8 (optimized) value (see docs/marvik/upstream.md). WER/CER as PERCENT.
7
  version: 1.0.0
8
+ report_type: speech-asr
9
+ profile: Baseline
10
  created_at: '2026-07-13T00:00:00Z'
11
  context:
12
+ target_ref: vivo-x300
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
13
  runtime:
14
  name: executorch
15
  execution_backend: cpu
benchmarks/whisper-tiny-vivo-x300-int8.yaml CHANGED
@@ -8,42 +8,11 @@
8
  # (7000 ms / 1169.5 ms p50 = 5.985); the earlier 10 s value was not.
9
  # - accuracy is the ExecuTorch INT8 (optimized) WER/CER, as PERCENT (repo convention).
10
  version: 1.0.0
11
- task: speech-asr
 
12
  created_at: '2026-07-13T00:00:00Z'
13
  context:
14
- model:
15
- id: Arm/whisper-tiny-int8-xnnpack-executorch
16
- filename: whisper-tiny-int8-executorch.pte
17
- base_model_id: openai/whisper-tiny
18
- profile: Arm-Optimized
19
- weight_dtype: int8
20
- quantization:
21
- method: PTQ-dynamic
22
- weight_bits: 8
23
- activation_bits: 8
24
- symmetric: true
25
- mode: dynamic
26
- weight_granularity: per-channel
27
- variant: 8da8w + embed_tokens weight-only int8
28
- fp32_layers:
29
- - embed_positions
30
- model_size_mb: 116.578
31
- parameter_count: 37760640
32
- format: pte
33
- target:
34
- name: Vivo X300
35
- hardware_class: Premium smartphone
36
- cpu_architecture: arm64
37
- cpu_model: C1-Ultra, C1-Premium, C1-Pro
38
- cpu_core_count: 8
39
- arch_feature_flags:
40
- - armv9.2-a
41
- - sve2
42
- - i8mm
43
- - bf16
44
- system_memory_gb: 16
45
- os: android
46
- os_version: Android 16 / OriginOS 6
47
  runtime:
48
  name: executorch
49
  execution_backend: cpu
 
8
  # (7000 ms / 1169.5 ms p50 = 5.985); the earlier 10 s value was not.
9
  # - accuracy is the ExecuTorch INT8 (optimized) WER/CER, as PERCENT (repo convention).
10
  version: 1.0.0
11
+ report_type: speech-asr
12
+ profile: Arm-Optimized
13
  created_at: '2026-07-13T00:00:00Z'
14
  context:
15
+ target_ref: vivo-x300
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
16
  runtime:
17
  name: executorch
18
  execution_backend: cpu
metadata.yaml CHANGED
@@ -1,59 +1,22 @@
1
- version: 1.0.0
2
- task: speech-asr
3
- created_at: '2026-06-25T01:54:42Z'
4
- context:
5
- model:
6
- id: Arm/whisper-tiny-int8-xnnpack-executorch
7
- base_model_id: openai/whisper-tiny
8
- profile: Arm-Optimized
9
- weight_dtype: int8
10
- quantization:
11
- method: PTQ-dynamic
12
- weight_bits: 8
13
- activation_bits: 8
14
- symmetric: true
15
- mode: dynamic
16
- weight_granularity: per-channel
17
- model_size_mb: 116.578
18
- parameter_count: 37760640
19
- filename: whisper-tiny-int8-executorch.pte
20
- format: pte
21
- target:
22
- name: Vivo X300
23
- hardware_class: Premium smartphone
24
- cpu_architecture: arm64
25
- cpu_model: C1-Ultra, C1-Premium, C1-Pro
26
- cpu_core_count: 8
27
- system_memory_gb: 16
28
- os: android
29
- os_version: Android 16 / OriginOS 6
30
- runtime:
31
- name: executorch
32
- execution_backend: cpu
33
- config:
34
- optimisations:
35
- - XNNPACK
36
- - KleidiAI
37
- version: 1.1.0
38
- dataset:
39
- name: librispeech_asr
40
- sample_count: 2620
41
- benchmark:
42
- batch_size: 1
43
- audio_length_s: 7.0
44
- num_runs: 50
45
- warmup_runs: 10
46
- mode: offline
47
- sample_rate_hz: 16000
48
- performance:
49
- end_to_end_latency_ms:
50
- p50: 1169.5
51
- p90: 1200.0
52
- p99: 1203.0
53
- peak_memory_mb: 3940.71
54
- rtfx: 5.985
55
- model_load_time_ms: 346.75
56
- accuracy:
57
- normalised_wer: 0.075716
58
- normalisation: Whisper-style
59
- cer: 0.031322
 
1
+ schema_version: 1.0.0
2
+ report_type: speech-asr
3
+ task_type: automatic-speech-recognition
4
+ title: Whisper Tiny INT8 — ExecuTorch + XNNPACK
5
+ id: Arm/whisper-tiny-int8-xnnpack-executorch
6
+ filename: whisper-tiny-int8-executorch.pte
7
+ base_model_id: openai/whisper-tiny
8
+ profile: Arm-Optimized
9
+ weight_dtype: int8
10
+ quantization:
11
+ method: PTQ-dynamic
12
+ weight_bits: 8
13
+ activation_bits: 8
14
+ symmetric: true
15
+ mode: dynamic
16
+ weight_granularity: per-channel
17
+ variant: 8da8w + embed_tokens weight-only int8
18
+ fp32_layers:
19
+ - embed_positions
20
+ model_size_mb: 116.578
21
+ parameter_count: 37760640
22
+ format: pte