Spaces:
Running
Running
| { | |
| "headline": "34 real docs bugs in 11 of 12 Python libraries. 3 in 4 flagged findings were real (45 of 60; these 45 are 34 distinct bugs); 30 bugs reproduce a real API error when you run the docs", | |
| "lead": "DocDrift gives Beam an entire Python library (docs + code, up to ~400k tokens) in one call. Beam checks every suspicious doc example against the whole codebase, then writes a script that runs it. Across 12 popular libraries (bugs in 11 of 12 libraries; pydantic-settings had none) it found 34 real docs bugs; 30 crash exactly as the docs are written.", | |
| "lead_short": "Beam read 12 whole Python libraries and ran their docs: 34 real bugs, 30 reproduce.", | |
| "tiles": [ | |
| [ | |
| 34, | |
| "real docs bugs · 11 of 12 libraries" | |
| ], | |
| [ | |
| 30, | |
| "crash when you run the docs" | |
| ], | |
| [ | |
| "2", | |
| "fixes filed upstream (starlette #3746, nox #1216)" | |
| ] | |
| ], | |
| "true_library_count": 11, | |
| "libraries_without_true": [ | |
| "pydantic-settings" | |
| ], | |
| "hybrid_true": 34, | |
| "repo_count": 12, | |
| "hybrid_findings": 60, | |
| "hybrid_true_findings": 45, | |
| "hybrid_precision": 0.75, | |
| "repro_true": 30, | |
| "repro_findings": 41, | |
| "repro_true_findings": 38, | |
| "repro_precision": 0.927, | |
| "rejected_true": 0, | |
| "true_candidates_judged": 32, | |
| "removed_false": 15, | |
| "kept_false": 9, | |
| "universe_size": 45, | |
| "symbol_true": 24, | |
| "whole_true": 11, | |
| "additions_total": 13, | |
| "additions_true": 9, | |
| "additions_new": 7, | |
| "false_repro_badges": 3, | |
| "whole_tokens": 4424575, | |
| "triage_tokens": 4739392, | |
| "repro_tokens": 138138, | |
| "end_to_end_tokens": 9302105, | |
| "judged": 56, | |
| "max_probe_tokens": 947446, | |
| "label_note": "Labels: checked against the source by an AI reviewer, not a human. Recall is measured against the 45 drifts any method found, so it flatters the methods.", | |
| "end_to_end_tokens_text": "~9.3M tokens (two whole-repo passes plus repro scripts)", | |
| "non_bug_badges": "2 false + 1 unclear (werkzeug S38, S39 false; S41 unclear)" | |
| } | |