Add AIST-87M throughput benchmark

README.md

@@ -136,10 +136,22 @@ Exact-gate tradeoff against the same dual-audio local baseline:
 | SALT audio-text R@1 avg | 0.008 | 0.007 | flat |
 | SALT image-audio R@1 avg | 0.138 | 0.148 | -0.010 |
 
-These are footprint and local exact-gate measurements, not a universal latency
-benchmark. Wall-clock speed still depends on runtime, device, batching, and
-audio preprocessing, but the deployed model removes one audio encoder pass and
-shrinks the audio projection path.
+Measured PyTorch audio-stack throughput on an NVIDIA L4, using synthetic 10s
+32 kHz CPU waveforms passed through waveform -> audio encoder -> projection ->
+normalized embedding. Median wall time is over 50 timed iterations after 20
+warmup iterations. This excludes audio file decode, dataset download, and MTEB
+result serialization.
+
+| Batch | AIST-87M median ms | AIST-87M throughput | AIST-95M median ms | AIST-95M throughput | Speedup |
+|---:|---:|---:|---:|---:|---:|
+| 1 | 5.36 | 186.7 clips/s; 1,867 audio-s/s | 10.50 | 95.2 clips/s; 952 audio-s/s | 1.96x |
+| 8 | 16.46 | 486.0 clips/s; 4,860 audio-s/s | 60.29 | 132.7 clips/s; 1,327 audio-s/s | 3.66x |
+| 16 | 41.19 | 388.5 clips/s; 3,885 audio-s/s | 133.95 | 119.4 clips/s; 1,194 audio-s/s | 3.25x |
+
+Projection-only throughput at feature batch 2048 is also higher for the
+single-audio path: 314k features/s for `AIST-87M` vs 282k features/s for the
+dual-audio tower. Raw benchmark output is included as
+`aist87m_vs_dual_audio_throughput_l4_20260504.json`.
 
 ## Architecture
 
@@ -175,6 +187,7 @@ The audio encoder in this artifact is the merged native checkpoint:
 | `parameter_breakdown.json` | Exact parameter accounting |
 | `aist87m_memory_slice_release_report.md` | Human-memory slice report |
 | `aist87m_memory_slice_release_report.json` | Machine-readable evaluation summary |
+| `aist87m_vs_dual_audio_throughput_l4_20260504.json` | L4 throughput benchmark vs dual-audio tower |
 
 ## Caveats
 

aist87m_vs_dual_audio_throughput_l4_20260504.json

@@ -0,0 +1,320 @@
+{
+  "aist87": {
+    "audio_adapter_params": 0,
+    "audio_checkpoint": "triembed/checkpoints/mn20_native_merged_aistmix_audioheavy100k175k175k_continue_from_balanced_20260426T143137Z/latest_model.pt",
+    "audio_head_input_dim": 1280,
+    "audio_path_params": 32144280,
+    "audio_projection_params": 12306560,
+    "audio_sample_rate": 32000,
+    "audio_seconds": 10.0,
+    "checkpoint": "triembed/checkpoints/aist81m_raw1280_mn20_lora_teacher_20260502T001744Z/best_model.pt",
+    "config_run_name": "aist81m_raw1280_mn20_lora_teacher_20260502T001744Z",
+    "encoder_params": 19837720,
+    "name": "AIST-87M",
+    "projection_only": {
+      "batch_size": 2048,
+      "cuda_events": {
+        "max_ms": 6.870016098022461,
+        "mean_ms": 6.54273796081543,
+        "median_ms": 6.502351999282837,
+        "min_ms": 6.4389119148254395,
+        "p95_ms": 6.860799789428711
+      },
+      "features_per_sec_cuda": 313018.8022606908,
+      "features_per_sec_cuda_median": 314962.9549778111,
+      "features_per_sec_wall": 312183.5609238512,
+      "features_per_sec_wall_median": 314158.8298674527,
+      "wall": {
+        "max_ms": 6.898713065311313,
+        "mean_ms": 6.5602429350838065,
+        "median_ms": 6.51899550575763,
+        "min_ms": 6.456336006522179,
+        "p95_ms": 6.8806440103799105
+      }
+    },
+    "waveform_to_embedding": {
+      "1": {
+        "audio_seconds_per_sec_cuda": 1767.600123842052,
+        "audio_seconds_per_sec_cuda_median": 1872.479629331263,
+        "audio_seconds_per_sec_wall": 1762.1861633836697,
+        "audio_seconds_per_sec_wall_median": 1866.9151048991507,
+        "cuda_events": {
+          "max_ms": 6.4194560050964355,
+          "mean_ms": 5.657388153076172,
+          "median_ms": 5.340512037277222,
+          "min_ms": 5.30841588973999,
+          "p95_ms": 6.403071880340576
+        },
+        "items_per_sec_cuda": 176.7600123842052,
+        "items_per_sec_cuda_median": 187.2479629331263,
+        "items_per_sec_wall": 176.21861633836699,
+        "items_per_sec_wall_median": 186.69151048991506,
+        "wall": {
+          "max_ms": 6.437776144593954,
+          "mean_ms": 5.674769333563745,
+          "median_ms": 5.3564299596473575,
+          "min_ms": 5.324810044839978,
+          "p95_ms": 6.428366061300039
+        }
+      },
+      "16": {
+        "audio_seconds_per_sec_cuda": 3615.280314396054,
+        "audio_seconds_per_sec_cuda_median": 3886.776599612583,
+        "audio_seconds_per_sec_wall": 3613.4877550602714,
+        "audio_seconds_per_sec_wall_median": 3884.636076443369,
+        "cuda_events": {
+          "max_ms": 92.48665618896484,
+          "mean_ms": 44.25659591674805,
+          "median_ms": 41.16521644592285,
+          "min_ms": 40.92313766479492,
+          "p95_ms": 68.5506591796875
+        },
+        "items_per_sec_cuda": 361.5280314396054,
+        "items_per_sec_cuda_median": 388.6776599612583,
+        "items_per_sec_wall": 361.34877550602715,
+        "items_per_sec_wall_median": 388.4636076443369,
+        "wall": {
+          "max_ms": 92.50957006588578,
+          "mean_ms": 44.278550487942994,
+          "median_ms": 41.187899420037866,
+          "min_ms": 40.94505496323109,
+          "p95_ms": 68.57325998134911
+        }
+      },
+      "8": {
+        "audio_seconds_per_sec_cuda": 4831.796135966214,
+        "audio_seconds_per_sec_cuda_median": 4866.393306722875,
+        "audio_seconds_per_sec_wall": 4825.5348259899165,
+        "audio_seconds_per_sec_wall_median": 4859.857786409293,
+        "cuda_events": {
+          "max_ms": 18.59174346923828,
+          "mean_ms": 16.556989936828614,
+          "median_ms": 16.439279556274414,
+          "min_ms": 16.22015953063965,
+          "p95_ms": 16.830463409423828
+        },
+        "items_per_sec_cuda": 483.17961359662144,
+        "items_per_sec_cuda_median": 486.6393306722875,
+        "items_per_sec_wall": 482.5534825989917,
+        "items_per_sec_wall_median": 485.9857786409293,
+        "wall": {
+          "max_ms": 18.62559001892805,
+          "mean_ms": 16.57847324386239,
+          "median_ms": 16.46138704381883,
+          "min_ms": 16.24572789296508,
+          "p95_ms": 16.852295957505703
+        }
+      }
+    }
+  },
+  "comparison_by_batch_size": {
+    "1": {
+      "aist87_audio_seconds_per_sec_wall": 1762.1861633836697,
+      "aist87_audio_seconds_per_sec_wall_median": 1866.9151048991507,
+      "aist87_items_per_sec_cuda": 176.7600123842052,
+      "aist87_items_per_sec_cuda_median": 187.2479629331263,
+      "aist87_items_per_sec_wall": 176.21861633836699,
+      "aist87_items_per_sec_wall_median": 186.69151048991506,
+      "aist87_mean_cuda_ms": 5.657388153076172,
+      "aist87_mean_wall_ms": 5.674769333563745,
+      "aist87_median_cuda_ms": 5.340512037277222,
+      "aist87_median_wall_ms": 5.3564299596473575,
+      "dual_audio_seconds_per_sec_wall": 841.2845368070198,
+      "dual_audio_seconds_per_sec_wall_median": 952.2081950454711,
+      "dual_items_per_sec_cuda": 84.26473443749266,
+      "dual_items_per_sec_cuda_median": 95.38999079009756,
+      "dual_items_per_sec_wall": 84.12845368070198,
+      "dual_items_per_sec_wall_median": 95.22081950454712,
+      "dual_mean_cuda_ms": 11.867360725402833,
+      "dual_mean_wall_ms": 11.8865848146379,
+      "dual_median_cuda_ms": 10.483280181884766,
+      "dual_median_wall_ms": 10.501904995180666,
+      "speedup_audio_seconds_per_sec_wall": 2.094637529024135,
+      "speedup_audio_seconds_per_sec_wall_median": 1.960616506571863,
+      "speedup_items_per_sec_cuda": 2.0976748287900353,
+      "speedup_items_per_sec_cuda_median": 1.962972858915136,
+      "speedup_items_per_sec_wall": 2.094637529024135,
+      "speedup_items_per_sec_wall_median": 1.9606165065718626
+    },
+    "16": {
+      "aist87_audio_seconds_per_sec_wall": 3613.4877550602714,
+      "aist87_audio_seconds_per_sec_wall_median": 3884.636076443369,
+      "aist87_items_per_sec_cuda": 361.5280314396054,
+      "aist87_items_per_sec_cuda_median": 388.6776599612583,
+      "aist87_items_per_sec_wall": 361.34877550602715,
+      "aist87_items_per_sec_wall_median": 388.4636076443369,
+      "aist87_mean_cuda_ms": 44.25659591674805,
+      "aist87_mean_wall_ms": 44.278550487942994,
+      "aist87_median_cuda_ms": 41.16521644592285,
+      "aist87_median_wall_ms": 41.187899420037866,
+      "dual_audio_seconds_per_sec_wall": 1173.9429007331944,
+      "dual_audio_seconds_per_sec_wall_median": 1194.4542069368163,
+      "dual_items_per_sec_cuda": 117.41469647216593,
+      "dual_items_per_sec_cuda_median": 119.4681483517097,
+      "dual_items_per_sec_wall": 117.39429007331943,
+      "dual_items_per_sec_wall_median": 119.44542069368163,
+      "dual_mean_cuda_ms": 136.26914245605468,
+      "dual_mean_wall_ms": 136.29282983019948,
+      "dual_median_cuda_ms": 133.92691040039062,
+      "dual_median_wall_ms": 133.95239354576916,
+      "speedup_audio_seconds_per_sec_wall": 3.078077948087119,
+      "speedup_audio_seconds_per_sec_wall_median": 3.252226878086467,
+      "speedup_items_per_sec_cuda": 3.079069676131288,
+      "speedup_items_per_sec_cuda_median": 3.2533998837665585,
+      "speedup_items_per_sec_wall": 3.0780779480871194,
+      "speedup_items_per_sec_wall_median": 3.252226878086467
+    },
+    "8": {
+      "aist87_audio_seconds_per_sec_wall": 4825.5348259899165,
+      "aist87_audio_seconds_per_sec_wall_median": 4859.857786409293,
+      "aist87_items_per_sec_cuda": 483.17961359662144,
+      "aist87_items_per_sec_cuda_median": 486.6393306722875,
+      "aist87_items_per_sec_wall": 482.5534825989917,
+      "aist87_items_per_sec_wall_median": 485.9857786409293,
+      "aist87_mean_cuda_ms": 16.556989936828614,
+      "aist87_mean_wall_ms": 16.57847324386239,
+      "aist87_median_cuda_ms": 16.439279556274414,
+      "aist87_median_wall_ms": 16.46138704381883,
+      "dual_audio_seconds_per_sec_wall": 1304.9761567848066,
+      "dual_audio_seconds_per_sec_wall_median": 1326.985212046842,
+      "dual_items_per_sec_cuda": 130.54346670818657,
+      "dual_items_per_sec_cuda_median": 132.74486741223745,
+      "dual_items_per_sec_wall": 130.49761567848066,
+      "dual_items_per_sec_wall_median": 132.6985212046842,
+      "dual_mean_cuda_ms": 61.28227020263672,
+      "dual_mean_wall_ms": 61.30380205344409,
+      "dual_median_cuda_ms": 60.26598358154297,
+      "dual_median_wall_ms": 60.28703204356134,
+      "speedup_audio_seconds_per_sec_wall": 3.6977953971810824,
+      "speedup_audio_seconds_per_sec_wall_median": 3.6623300262051015,
+      "speedup_items_per_sec_cuda": 3.7012929546042197,
+      "speedup_items_per_sec_cuda_median": 3.6659747390536417,
+      "speedup_items_per_sec_wall": 3.697795397181083,
+      "speedup_items_per_sec_wall_median": 3.6623300262051015
+    }
+  },
+  "dual_audio_tower": {
+    "audio_adapter_params": 0,
+    "audio_head_input_dim": 2304,
+    "audio_path_params": 40390311,
+    "audio_projection_params": 14272640,
+    "audio_sample_rate": 32000,
+    "audio_seconds": 10.0,
+    "checkpoint": "triembed/checkpoints/wordnet_validaudio_real_mn20_whisper_wavcaps_speech150k_d2_retrieval_20260418T004617Z/best_model.pt",
+    "config_run_name": "wordnet_validaudio_real_mn20_whisper_wavcaps_speech150k_d2_retrieval_20260418T004617Z",
+    "encoder_params": 26117671,
+    "name": "AIST-95M dual-audio tower",
+    "projection_only": {
+      "batch_size": 2048,
+      "cuda_events": {
+        "max_ms": 7.5786237716674805,
+        "mean_ms": 7.27663553237915,
+        "median_ms": 7.247359991073608,
+        "min_ms": 7.027616024017334,
+        "p95_ms": 7.532544136047363
+      },
+      "features_per_sec_cuda": 281448.75346400525,
+      "features_per_sec_cuda_median": 282585.6591258707,
+      "features_per_sec_wall": 280732.89089042274,
+      "features_per_sec_wall_median": 281735.96789576364,
+      "wall": {
+        "max_ms": 7.597371004521847,
+        "mean_ms": 7.295190789736807,
+        "median_ms": 7.26921739988029,
+        "min_ms": 7.046802900731564,
+        "p95_ms": 7.553461939096451
+      }
+    },
+    "waveform_to_embedding": {
+      "1": {
+        "audio_seconds_per_sec_cuda": 842.6473443749267,
+        "audio_seconds_per_sec_cuda_median": 953.8999079009756,
+        "audio_seconds_per_sec_wall": 841.2845368070198,
+        "audio_seconds_per_sec_wall_median": 952.2081950454711,
+        "cuda_events": {
+          "max_ms": 53.797889709472656,
+          "mean_ms": 11.867360725402833,
+          "median_ms": 10.483280181884766,
+          "min_ms": 10.369024276733398,
+          "p95_ms": 10.95680046081543
+        },
+        "items_per_sec_cuda": 84.26473443749266,
+        "items_per_sec_cuda_median": 95.38999079009756,
+        "items_per_sec_wall": 84.12845368070198,
+        "items_per_sec_wall_median": 95.22081950454712,
+        "wall": {
+          "max_ms": 53.818747168406844,
+          "mean_ms": 11.8865848146379,
+          "median_ms": 10.501904995180666,
+          "min_ms": 10.387809947133064,
+          "p95_ms": 10.97815902903676
+        }
+      },
+      "16": {
+        "audio_seconds_per_sec_cuda": 1174.1469647216593,
+        "audio_seconds_per_sec_cuda_median": 1194.6814835170971,
+        "audio_seconds_per_sec_wall": 1173.9429007331944,
+        "audio_seconds_per_sec_wall_median": 1194.4542069368163,
+        "cuda_events": {
+          "max_ms": 172.75180053710938,
+          "mean_ms": 136.26914245605468,
+          "median_ms": 133.92691040039062,
+          "min_ms": 131.3126678466797,
+          "p95_ms": 163.50003051757812
+        },
+        "items_per_sec_cuda": 117.41469647216593,
+        "items_per_sec_cuda_median": 119.4681483517097,
+        "items_per_sec_wall": 117.39429007331943,
+        "items_per_sec_wall_median": 119.44542069368163,
+        "wall": {
+          "max_ms": 172.77729604393244,
+          "mean_ms": 136.29282983019948,
+          "median_ms": 133.95239354576916,
+          "min_ms": 131.3318640459329,
+          "p95_ms": 163.52250101044774
+        }
+      },
+      "8": {
+        "audio_seconds_per_sec_cuda": 1305.4346670818657,
+        "audio_seconds_per_sec_cuda_median": 1327.4486741223745,
+        "audio_seconds_per_sec_wall": 1304.9761567848066,
+        "audio_seconds_per_sec_wall_median": 1326.985212046842,
+        "cuda_events": {
+          "max_ms": 109.47174072265625,
+          "mean_ms": 61.28227020263672,
+          "median_ms": 60.26598358154297,
+          "min_ms": 58.52467346191406,
+          "p95_ms": 61.911041259765625
+        },
+        "items_per_sec_cuda": 130.54346670818657,
+        "items_per_sec_cuda_median": 132.74486741223745,
+        "items_per_sec_wall": 130.49761567848066,
+        "items_per_sec_wall_median": 132.6985212046842,
+        "wall": {
+          "max_ms": 109.49674621224403,
+          "mean_ms": 61.30380205344409,
+          "median_ms": 60.28703204356134,
+          "min_ms": 58.54333913885057,
+          "p95_ms": 61.929865973070264
+        }
+      }
+    }
+  },
+  "environment": {
+    "batch_sizes": [
+      1,
+      8,
+      16
+    ],
+    "cuda_available": true,
+    "cuda_device_name": "NVIDIA L4",
+    "device": "cuda:0",
+    "iters": 50,
+    "note": "Synthetic random waveforms; excludes file decode, dataset download, and MTEB result serialization.",
+    "platform": "Linux-6.8.12-9-pve-x86_64-with-glibc2.39",
+    "python": "3.12.3",
+    "seed": 1234,
+    "torch": "2.10.0+cu126",
+    "warmup": 20
+  }
+}

manifest.json

@@ -4,9 +4,20 @@
   "release_type": "safetensors",
   "license": "apache-2.0",
   "parameter_count": 87118774,
-  "modalities": ["audio", "image", "speech", "text"],
+  "modalities": [
+    "audio",
+    "image",
+    "speech",
+    "text"
+  ],
   "output_dim": 1280,
-  "matryoshka_slices": [1280, 768, 512, 256, 128],
+  "matryoshka_slices": [
+    1280,
+    768,
+    512,
+    256,
+    128
+  ],
   "audio_encoder": {
     "name": "efficientat_mn20_as_native_merged",
     "checkpoint": "triembed/checkpoints/mn20_native_merged_aistmix_audioheavy100k175k175k_continue_from_balanced_20260426T143137Z/latest_model.pt",
@@ -26,7 +37,8 @@
       "768": 0,
       "512": 0
     },
-    "report": "aist87m_memory_slice_release_report.json"
+    "report": "aist87m_memory_slice_release_report.json",
+    "throughput_report": "aist87m_vs_dual_audio_throughput_l4_20260504.json"
   },
   "files": [
     {
@@ -53,6 +65,11 @@
       "path": "aist87m_memory_slice_release_report.json",
       "size_bytes": 40579,
       "sha256": "6b5894bd3b8de9e8c99ccd236587fc6f4652399ac95b343ba301478fcdb7c77a"
+    },
+    {
+      "path": "aist87m_vs_dual_audio_throughput_l4_20260504.json",
+      "size_bytes": 13261,
+      "sha256": "58bdf1f02d4f5a497f582f4a65978d9a9b6dc37f5ca3905250b0dcdd8ea13817"
     }
   ]
 }