{
  "headline": "34 real docs bugs in 11 of 12 Python libraries. 3 in 4 flagged findings were real (45 of 60; these 45 are 34 distinct bugs); 30 bugs reproduce a real API error when you run the docs",
  "lead": "DocDrift gives Beam an entire Python library (docs + code, up to ~400k tokens) in one call. Beam checks every suspicious doc example against the whole codebase, then writes a script that runs it. Across 12 popular libraries (bugs in 11 of 12 libraries; pydantic-settings had none) it found 34 real docs bugs; 30 crash exactly as the docs are written.",
  "lead_short": "Beam read 12 whole Python libraries and ran their docs: 34 real bugs, 30 reproduce.",
  "tiles": [
    [
      34,
      "real docs bugs · 11 of 12 libraries"
    ],
    [
      30,
      "crash when you run the docs"
    ],
    [
      "2",
      "fixes filed upstream (starlette #3746, nox #1216)"
    ]
  ],
  "true_library_count": 11,
  "libraries_without_true": [
    "pydantic-settings"
  ],
  "hybrid_true": 34,
  "repo_count": 12,
  "hybrid_findings": 60,
  "hybrid_true_findings": 45,
  "hybrid_precision": 0.75,
  "repro_true": 30,
  "repro_findings": 41,
  "repro_true_findings": 38,
  "repro_precision": 0.927,
  "rejected_true": 0,
  "true_candidates_judged": 32,
  "removed_false": 15,
  "kept_false": 9,
  "universe_size": 45,
  "symbol_true": 24,
  "whole_true": 11,
  "additions_total": 13,
  "additions_true": 9,
  "additions_new": 7,
  "false_repro_badges": 3,
  "whole_tokens": 4424575,
  "triage_tokens": 4739392,
  "repro_tokens": 138138,
  "end_to_end_tokens": 9302105,
  "judged": 56,
  "max_probe_tokens": 947446,
  "label_note": "Labels: checked against the source by an AI reviewer, not a human. Recall is measured against the 45 drifts any method found, so it flatters the methods.",
  "end_to_end_tokens_text": "~9.3M tokens (two whole-repo passes plus repro scripts)",
  "non_bug_badges": "2 false + 1 unclear (werkzeug S38, S39 false; S41 unclear)"
}