Spaces:
Running
Running
File size: 1,965 Bytes
cd19b51 a1e7884 b083097 72b06db b083097 cd19b51 dad6879 cd19b51 b083097 72b06db cd19b51 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 | {
"headline": "34 real docs bugs in 11 of 12 Python libraries. 3 in 4 flagged findings were real (45 of 60; these 45 are 34 distinct bugs); 30 bugs reproduce a real API error when you run the docs",
"lead": "DocDrift gives Beam an entire Python library (docs + code, up to ~400k tokens) in one call. Beam checks every suspicious doc example against the whole codebase, then writes a script that runs it. Across 12 popular libraries (bugs in 11 of 12 libraries; pydantic-settings had none) it found 34 real docs bugs; 30 crash exactly as the docs are written.",
"lead_short": "Beam read 12 whole Python libraries and ran their docs: 34 real bugs, 30 reproduce.",
"tiles": [
[
34,
"real docs bugs · 11 of 12 libraries"
],
[
30,
"crash when you run the docs"
],
[
"2",
"fixes filed upstream (starlette #3746, nox #1216)"
]
],
"true_library_count": 11,
"libraries_without_true": [
"pydantic-settings"
],
"hybrid_true": 34,
"repo_count": 12,
"hybrid_findings": 60,
"hybrid_true_findings": 45,
"hybrid_precision": 0.75,
"repro_true": 30,
"repro_findings": 41,
"repro_true_findings": 38,
"repro_precision": 0.927,
"rejected_true": 0,
"true_candidates_judged": 32,
"removed_false": 15,
"kept_false": 9,
"universe_size": 45,
"symbol_true": 24,
"whole_true": 11,
"additions_total": 13,
"additions_true": 9,
"additions_new": 7,
"false_repro_badges": 3,
"whole_tokens": 4424575,
"triage_tokens": 4739392,
"repro_tokens": 138138,
"end_to_end_tokens": 9302105,
"judged": 56,
"max_probe_tokens": 947446,
"label_note": "Labels: checked against the source by an AI reviewer, not a human. Recall is measured against the 45 drifts any method found, so it flatters the methods.",
"end_to_end_tokens_text": "~9.3M tokens (two whole-repo passes plus repro scripts)",
"non_bug_badges": "2 false + 1 unclear (werkzeug S38, S39 false; S41 unclear)"
}
|