| { |
| "bootstrap": { |
| "n": 2000, |
| "seed": 0, |
| "unit": "recording" |
| }, |
| "code_dirty": [ |
| "hf-release-v0.1/config.json", |
| "training/hf_convert.py", |
| "training/hq5a_cough_eval.sbatch", |
| "training/hq5a_evalsets.sbatch", |
| "training/release_gate.py", |
| "hf-release-v0.1/release-manifest.json", |
| "training/hq5a_event_calib.sbatch" |
| ], |
| "code_revision": "e4db7af66b151e230c6075e28253302b9b48ff6e", |
| "family": "laughter-v0.1", |
| "members": [ |
| "laughter-v0.1-seed0", |
| "laughter-v0.1-seed1", |
| "laughter-v0.1-seed10", |
| "laughter-v0.1-seed11", |
| "laughter-v0.1-seed12", |
| "laughter-v0.1-seed13", |
| "laughter-v0.1-seed14", |
| "laughter-v0.1-seed2", |
| "laughter-v0.1-seed3", |
| "laughter-v0.1-seed4", |
| "laughter-v0.1-seed5", |
| "laughter-v0.1-seed6", |
| "laughter-v0.1-seed7", |
| "laughter-v0.1-seed8", |
| "laughter-v0.1-seed9" |
| ], |
| "reproduce": "For each public surface: AP of each probs/<surface>__laughter-v0.1-seedN.npy against target>=0.5 in <surface>.clips.tsv, mean over the 15 seeds = family_mean; margin = family_mean - reference AP; CI by paired recording (speaker on VocalSound) bootstrap, 2000 resamples, seed 0.", |
| "rule": "internal-and-any-public", |
| "set_digests": { |
| "ami-eval-v1": "1ab59b11012d18d9", |
| "annotation-eval-v5": "2dcdba95c052bbde", |
| "podcastfillers-eval": "e4deca38c6380c35", |
| "public-vocalsound": "c4557271c7f68f70", |
| "fsd50k-panel": "b55cd8789c24fda8" |
| }, |
| "table": { |
| "ami-eval-v1": { |
| "ci_hi": 0.03942520109963582, |
| "ci_lo": 0.012154065156311903, |
| "family_mean": 0.9483739882117678, |
| "floor_ci_upper": 0.0039598598827272615, |
| "floor_sigma": 0.002556051282820607, |
| "label": "AMI", |
| "margin": 0.02538138623377517, |
| "member_ap": [ |
| 0.9507902186015077, |
| 0.9500519992061698, |
| 0.9452210675451327, |
| 0.9503056961627866, |
| 0.9508894430969833, |
| 0.9470331036343588, |
| 0.9450138166749027, |
| 0.9465480039960334, |
| 0.9513989118555961, |
| 0.9483455133944823, |
| 0.9432426745514125, |
| 0.9511137070448046, |
| 0.9483085823703894, |
| 0.9485920621237138, |
| 0.9487550229182443 |
| ], |
| "n": 15, |
| "n_clips": 3082, |
| "n_recordings": 24, |
| "reference_ap": 0.9229926019779926, |
| "reference_model": "eat" |
| }, |
| "annotation-eval-v5": { |
| "ci_hi": 0.023578607063105828, |
| "ci_lo": -0.014470678820449834, |
| "family_mean": 0.8996943593326668, |
| "floor_ci_upper": 0.004720526387522345, |
| "floor_sigma": 0.003047054159932776, |
| "label": "internal", |
| "margin": 0.0028852774625475197, |
| "member_ap": [ |
| 0.8975475415661825, |
| 0.9039947028281456, |
| 0.8983932783475506, |
| 0.8993346068750109, |
| 0.9005183386530095, |
| 0.9016603462847035, |
| 0.9028680945100267, |
| 0.8955774413407963, |
| 0.9002877424905875, |
| 0.9031921793787044, |
| 0.8952398854096171, |
| 0.9011704571931498, |
| 0.8995806005577214, |
| 0.8942340859359875, |
| 0.9018160886188078 |
| ], |
| "n": 15, |
| "n_clips": 2297, |
| "n_recordings": 21, |
| "reference_ap": 0.8968090818701193, |
| "reference_model": "ced-base", |
| "_per_clip": "not shipped (internal audio)" |
| }, |
| "podcastfillers-eval": { |
| "ci_hi": 0.050521526766975876, |
| "ci_lo": -0.02075922278013193, |
| "family_mean": 0.8333527146573345, |
| "floor_ci_upper": 0.01673321979650875, |
| "floor_sigma": 0.010801131654468506, |
| "label": "PF", |
| "margin": 0.014778337531855201, |
| "member_ap": [ |
| 0.8344361951641851, |
| 0.841598928384278, |
| 0.8411140826012734, |
| 0.8366294402860901, |
| 0.8337691285784827, |
| 0.8382277485746962, |
| 0.8403275294236768, |
| 0.831845088772049, |
| 0.8383355800012807, |
| 0.8350686755602265, |
| 0.7975593280141196, |
| 0.8376021852481, |
| 0.8343551388336643, |
| 0.8279070269767151, |
| 0.8315146434411776 |
| ], |
| "n": 15, |
| "n_clips": 10398, |
| "n_recordings": 20, |
| "reference_ap": 0.8185743771254793, |
| "reference_model": "gillick" |
| }, |
| "public-vocalsound": { |
| "ci_hi": -0.005603668917087099, |
| "ci_lo": -0.02523479335876062, |
| "family_mean": 0.9550608658482184, |
| "floor_ci_upper": 0.007326952674220729, |
| "floor_sigma": 0.004729477137259021, |
| "label": "VS", |
| "margin": -0.015235673719981468, |
| "member_ap": [ |
| 0.9520376579742674, |
| 0.9552183790146299, |
| 0.9540970381377161, |
| 0.9509495673970976, |
| 0.9519229996151687, |
| 0.9537347287207015, |
| 0.9557549955025102, |
| 0.9619265200931599, |
| 0.9544285936941963, |
| 0.9517720709172127, |
| 0.9652754551778516, |
| 0.9524594157733894, |
| 0.9488156226153927, |
| 0.9624520243219058, |
| 0.9550679187680756 |
| ], |
| "n": 15, |
| "n_clips": 3591, |
| "n_recordings": 574, |
| "reference_ap": 0.9702965395681998, |
| "reference_model": "ced-base" |
| } |
| }, |
| "tolerance": { |
| "ap": 1e-12, |
| "ci": 0.005 |
| }, |
| "verdict": "FAIL: internal", |
| "z_whole_ci": { |
| "AMI": 6.409667762358887, |
| "PF": 0.8831735739787857, |
| "VS": -2.0794011367900485, |
| "internal": 0.6112194331068893 |
| }, |
| "curation": { |
| "by": "release packaging", |
| "date": "2026-09-05", |
| "note": "Public bundle, curated from the internal eval bundle: the private internal surface's per-clip data is excluded (its digest and aggregate row are kept so the card's internal row stays auditable), run names are mapped to release naming (laughter-v0.1-seed0..14, order preserved against the member_ap arrays), the fifteen per-seed configs ship once as configs/laughter-v0.1.json (runs are seeds 0-14 of it), and operational fields are removed from that config. eval/podcastfillers-relabel.tsv records the 100 PodcastFillers non-laughter clips both model families scored above 0.9 and the listening verdict on each (97 contain audible laughter); the table above uses the published labels. 2026-09-09: the FSD50K NSV panel added (fsd50k-panel.clips.tsv, probs for the 15 seeds and ced-base); its column is defined under fsd50k_panel. 2026-09-10: the two no-teacher arms retrained on seeds 0-14 so every trained row shares the same seeds; card_table holds every cell." |
| }, |
| "fsd50k_panel": { |
| "label": "FSD50K NSV", |
| "what": "Share of FSD50K non-laughter vocalisation clips (any of cough, sneeze, breathing, gasp, sigh, crying, burping; no Laughter label) scored as laughter, at the threshold where the model recovers 90% of the panel's Laughter clips. Each clip's score is the max over its 4 s windows (hop 4 s). Lower is better.", |
| "n_laughter": 933, |
| "n_nsv": 1136, |
| "n_clips": 4269, |
| "recall": 0.9, |
| "family_mean": 0.1459507042253521, |
| "family_sd": 0.013560746885563212, |
| "member_nsv_fp": [ |
| 0.14964788732394366, |
| 0.15316901408450703, |
| 0.1267605633802817, |
| 0.13556338028169015, |
| 0.1637323943661972, |
| 0.13468309859154928, |
| 0.13556338028169015, |
| 0.1619718309859155, |
| 0.12147887323943662, |
| 0.13908450704225353, |
| 0.16549295774647887, |
| 0.14964788732394366, |
| 0.15933098591549297, |
| 0.14788732394366197, |
| 0.14524647887323944 |
| ], |
| "reference_model": "ced-base", |
| "reference_nsv_fp": 0.03609154929577465, |
| "others": { |
| "eat-lora-no-teacher": { |
| "nsv_fp": 0.1931338028169014, |
| "sd": 0.01694497340594801, |
| "n": 15 |
| }, |
| "eat-full-finetune-teacher": { |
| "nsv_fp": 0.17951877934272303, |
| "sd": 0.04972447014311274, |
| "n": 15 |
| }, |
| "eat-full-finetune-no-teacher": { |
| "nsv_fp": 0.26085680751173707, |
| "sd": 0.030258187344898973, |
| "n": 15 |
| }, |
| "eat": { |
| "nsv_fp": 0.0801056338028169, |
| "n": 1 |
| }, |
| "ced-tiny": { |
| "nsv_fp": 0.07130281690140845, |
| "n": 1 |
| }, |
| "beats": { |
| "nsv_fp": 0.14524647887323944, |
| "n": 1 |
| }, |
| "gillick": { |
| "nsv_fp": 0.48327464788732394, |
| "n": 1 |
| } |
| }, |
| "reproduce": "s = probs/fsd50k-panel__<model>.npy aligned with fsd50k-panel.clips.tsv; thr = the 840th highest score among laughter==1 rows (ceil(0.9*933)); nsv_fp = mean(s[nsv==1] >= thr). Clips are FSD50K dev audio (train and val folds) that carry panel labels; none of it is training data." |
| }, |
| "card_table": { |
| "note": "Every cell of the card's results table. Trained rows: mean and sd over seeds 0-14. Off-the-shelf rows: value and 95% bootstrap half-width (2000 draws, seed 0) over recordings (speakers on VocalSound, clips on the FSD50K panel). Built by training.card_results.", |
| "columns": [ |
| "annotation-eval-v5", |
| "podcastfillers-eval", |
| "ami-eval-v1", |
| "public-vocalsound", |
| "fsd50k-panel" |
| ], |
| "panel_recall": 0.9, |
| "bootstrap": 2000, |
| "n_laughter": 933, |
| "n_nsv": 1136, |
| "rows": { |
| "**this release**: EAT + LoRA, teacher-trained": { |
| "annotation-eval-v5": { |
| "value": 0.8996943593326668, |
| "spread": 0.0029931705939927142, |
| "spread_kind": "seed sd", |
| "n": 15 |
| }, |
| "podcastfillers-eval": { |
| "value": 0.8333527146573345, |
| "spread": 0.010610126355847991, |
| "spread_kind": "seed sd", |
| "n": 15 |
| }, |
| "ami-eval-v1": { |
| "value": 0.9483739882117678, |
| "spread": 0.002510850524772026, |
| "spread_kind": "seed sd", |
| "n": 15 |
| }, |
| "public-vocalsound": { |
| "value": 0.9550608658482184, |
| "spread": 0.0046458419014504355, |
| "spread_kind": "seed sd", |
| "n": 15 |
| }, |
| "fsd50k-panel": { |
| "value": 0.14595070422535214, |
| "spread": 0.013560746885563212, |
| "spread_kind": "seed sd", |
| "n": 15 |
| } |
| }, |
| "EAT + LoRA, no teacher": { |
| "annotation-eval-v5": { |
| "value": 0.9022181395955502, |
| "spread": 0.0028596226591157932, |
| "spread_kind": "seed sd", |
| "n": 15 |
| }, |
| "podcastfillers-eval": { |
| "value": 0.846728733967997, |
| "spread": 0.014180319446455675, |
| "spread_kind": "seed sd", |
| "n": 15 |
| }, |
| "ami-eval-v1": { |
| "value": 0.9524466643323456, |
| "spread": 0.003276968221102732, |
| "spread_kind": "seed sd", |
| "n": 15 |
| }, |
| "public-vocalsound": { |
| "value": 0.9420809875492873, |
| "spread": 0.006998231467778743, |
| "spread_kind": "seed sd", |
| "n": 15 |
| }, |
| "fsd50k-panel": { |
| "value": 0.1931338028169014, |
| "spread": 0.01694497340594801, |
| "spread_kind": "seed sd", |
| "n": 15 |
| } |
| }, |
| "EAT full fine-tune, teacher-trained": { |
| "annotation-eval-v5": { |
| "value": 0.9166429554840164, |
| "spread": 0.005625359041947865, |
| "spread_kind": "seed sd", |
| "n": 15 |
| }, |
| "podcastfillers-eval": { |
| "value": 0.8750703685982714, |
| "spread": 0.011921685111299707, |
| "spread_kind": "seed sd", |
| "n": 15 |
| }, |
| "ami-eval-v1": { |
| "value": 0.9446209554473263, |
| "spread": 0.005212745805492586, |
| "spread_kind": "seed sd", |
| "n": 15 |
| }, |
| "public-vocalsound": { |
| "value": 0.9437988023703398, |
| "spread": 0.01700459450721973, |
| "spread_kind": "seed sd", |
| "n": 15 |
| }, |
| "fsd50k-panel": { |
| "value": 0.17951877934272303, |
| "spread": 0.04972447014311274, |
| "spread_kind": "seed sd", |
| "n": 15 |
| } |
| }, |
| "EAT full fine-tune, no teacher": { |
| "annotation-eval-v5": { |
| "value": 0.918610648846479, |
| "spread": 0.006374031205233863, |
| "spread_kind": "seed sd", |
| "n": 15 |
| }, |
| "podcastfillers-eval": { |
| "value": 0.8808029276378652, |
| "spread": 0.008482409285809667, |
| "spread_kind": "seed sd", |
| "n": 15 |
| }, |
| "ami-eval-v1": { |
| "value": 0.9523479177219377, |
| "spread": 0.004569827830633127, |
| "spread_kind": "seed sd", |
| "n": 15 |
| }, |
| "public-vocalsound": { |
| "value": 0.9089073852987579, |
| "spread": 0.059372105425155815, |
| "spread_kind": "seed sd", |
| "n": 15 |
| }, |
| "fsd50k-panel": { |
| "value": 0.26085680751173707, |
| "spread": 0.030258187344898973, |
| "spread_kind": "seed sd", |
| "n": 15 |
| } |
| }, |
| "CED-base, AudioSet head": { |
| "annotation-eval-v5": { |
| "value": 0.8968090818701193, |
| "spread": 0.0228192185597908, |
| "spread_kind": "bootstrap 95% half-width", |
| "n": 1 |
| }, |
| "podcastfillers-eval": { |
| "value": 0.8105831097976058, |
| "spread": 0.06036189372814932, |
| "spread_kind": "bootstrap 95% half-width", |
| "n": 1 |
| }, |
| "ami-eval-v1": { |
| "value": 0.8541544974880243, |
| "spread": 0.039408864655789466, |
| "spread_kind": "bootstrap 95% half-width", |
| "n": 1 |
| }, |
| "public-vocalsound": { |
| "value": 0.9702965395681998, |
| "spread": 0.009631731710918856, |
| "spread_kind": "bootstrap 95% half-width", |
| "n": 1 |
| }, |
| "fsd50k-panel": { |
| "value": 0.03609154929577465, |
| "spread": 0.016285211267605633, |
| "spread_kind": "bootstrap 95% half-width", |
| "n": 1 |
| } |
| }, |
| "EAT, AudioSet head (the base checkpoint, zero-shot)": { |
| "annotation-eval-v5": { |
| "value": 0.8829045193537908, |
| "spread": 0.02558446961170535, |
| "spread_kind": "bootstrap 95% half-width", |
| "n": 1 |
| }, |
| "podcastfillers-eval": { |
| "value": 0.7844658804345496, |
| "spread": 0.04947444610463847, |
| "spread_kind": "bootstrap 95% half-width", |
| "n": 1 |
| }, |
| "ami-eval-v1": { |
| "value": 0.9229926019779926, |
| "spread": 0.024309727861856867, |
| "spread_kind": "bootstrap 95% half-width", |
| "n": 1 |
| }, |
| "public-vocalsound": { |
| "value": 0.9571953064111093, |
| "spread": 0.013309956523687383, |
| "spread_kind": "bootstrap 95% half-width", |
| "n": 1 |
| }, |
| "fsd50k-panel": { |
| "value": 0.0801056338028169, |
| "spread": 0.027739876760563317, |
| "spread_kind": "bootstrap 95% half-width", |
| "n": 1 |
| } |
| }, |
| "CED-tiny, AudioSet head": { |
| "annotation-eval-v5": { |
| "value": 0.8857709447136373, |
| "spread": 0.02259637836585593, |
| "spread_kind": "bootstrap 95% half-width", |
| "n": 1 |
| }, |
| "podcastfillers-eval": { |
| "value": 0.7980219459500315, |
| "spread": 0.06555965664476748, |
| "spread_kind": "bootstrap 95% half-width", |
| "n": 1 |
| }, |
| "ami-eval-v1": { |
| "value": 0.8564672033882755, |
| "spread": 0.03207273515356468, |
| "spread_kind": "bootstrap 95% half-width", |
| "n": 1 |
| }, |
| "public-vocalsound": { |
| "value": 0.9558670210236383, |
| "spread": 0.012283620217255986, |
| "spread_kind": "bootstrap 95% half-width", |
| "n": 1 |
| }, |
| "fsd50k-panel": { |
| "value": 0.07130281690140845, |
| "spread": 0.02112676056338028, |
| "spread_kind": "bootstrap 95% half-width", |
| "n": 1 |
| } |
| }, |
| "BEATs, AudioSet head": { |
| "annotation-eval-v5": { |
| "value": 0.8383485927709583, |
| "spread": 0.031978569016706926, |
| "spread_kind": "bootstrap 95% half-width", |
| "n": 1 |
| }, |
| "podcastfillers-eval": { |
| "value": 0.7475556060918509, |
| "spread": 0.06449430520655708, |
| "spread_kind": "bootstrap 95% half-width", |
| "n": 1 |
| }, |
| "ami-eval-v1": { |
| "value": 0.8419949943494651, |
| "spread": 0.029730828291308575, |
| "spread_kind": "bootstrap 95% half-width", |
| "n": 1 |
| }, |
| "public-vocalsound": { |
| "value": 0.9273063305843975, |
| "spread": 0.016602341784197117, |
| "spread_kind": "bootstrap 95% half-width", |
| "n": 1 |
| }, |
| "fsd50k-panel": { |
| "value": 0.14524647887323944, |
| "spread": 0.03962367957746479, |
| "spread_kind": "bootstrap 95% half-width", |
| "n": 1 |
| } |
| }, |
| "Gillick et al.": { |
| "annotation-eval-v5": { |
| "value": 0.8590035695842937, |
| "spread": 0.022847874353034203, |
| "spread_kind": "bootstrap 95% half-width", |
| "n": 1 |
| }, |
| "podcastfillers-eval": { |
| "value": 0.8185743771254793, |
| "spread": 0.04195457050713697, |
| "spread_kind": "bootstrap 95% half-width", |
| "n": 1 |
| }, |
| "ami-eval-v1": { |
| "value": 0.8525958378855036, |
| "spread": 0.030944159938566085, |
| "spread_kind": "bootstrap 95% half-width", |
| "n": 1 |
| }, |
| "public-vocalsound": { |
| "value": 0.7171790828625292, |
| "spread": 0.0312806324260928, |
| "spread_kind": "bootstrap 95% half-width", |
| "n": 1 |
| }, |
| "fsd50k-panel": { |
| "value": 0.48327464788732394, |
| "spread": 0.053268045774647804, |
| "spread_kind": "bootstrap 95% half-width", |
| "n": 1 |
| } |
| } |
| } |
| } |
| } |
