Buckets:
AnonymousSpottedToad/candidate_bucket / codebase /python /Consiliency_treesitter-chunker /scripts /regenerate_language_coverage.py
| #!/usr/bin/env python3 | |
| """Regenerate the committed language-coverage report (JSON + Markdown). | |
| This is the ONE sanctioned way the committed ``docs/language-coverage.json`` and | |
| ``docs/language-coverage.md`` change. The smoke-tier gate | |
| (``tests/test_language_smoke.py``) recomputes the sweep live and diffs it against | |
| the committed JSON, so any *unintended* coverage change -- a language that stops | |
| loading, or one that gains/loses an extraction surface -- fails loudly in CI. | |
| It MUST be run on the pinned, ABI-paired stack (tree_sitter 0.25 / | |
| tree-sitter-language-pack 0.9.0). ``assert_grammar_runtime_pins`` fails closed | |
| before anything is written if the stack drifted off-pin. Running it twice | |
| produces no git diff: the JSON is byte-stable (``sort_keys`` + fixed indent) and | |
| the volatile ``pins`` block is normalized to placeholders so the report does not | |
| churn on a patch-version bump within the pinned range. | |
| """ | |
| from __future__ import annotations | |
| import json | |
| import sys | |
| from pathlib import Path | |
| REPO_ROOT = Path(__file__).resolve().parents[1] | |
| sys.path.insert(0, str(REPO_ROOT)) | |
| from tests.boundary_ir_conformance import assert_grammar_runtime_pins # noqa: E402 | |
| from tests.language_smoke import ( # noqa: E402 | |
| EMPTY, | |
| EXTRACTION_GAP, | |
| LOAD_ONLY, | |
| RICH, | |
| SPARSE, | |
| compute_coverage, | |
| ) | |
| JSON_PATH = REPO_ROOT / "docs" / "language-coverage.json" | |
| MD_PATH = REPO_ROOT / "docs" / "language-coverage.md" | |
| # Languages whose parsed-but-empty extraction is the *expected* result -- they | |
| # are data/markup formats with no function/type surface, so 0 boundaries is not | |
| # a gap. Used only to annotate the Markdown narrative; it does not affect the | |
| # JSON oracle or any assertion. | |
| DATA_MARKUP_LANGUAGES = frozenset( | |
| { | |
| "css", | |
| "html", | |
| "json", | |
| "markdown", | |
| "latex", | |
| "yaml", | |
| "xml", | |
| "toml", | |
| } | |
| ) | |
| def _normalize_for_commit(coverage: dict) -> dict: | |
| """Replace volatile pin patch-versions with the pinned major.minor ranges. | |
| Keeps the committed JSON byte-stable across patch bumps inside the pin so the | |
| diff-against-committed gate tracks coverage, not dependency patch churn. | |
| """ | |
| normalized = json.loads(json.dumps(coverage)) # deep copy | |
| normalized["pins"] = { | |
| "tree_sitter": ">=0.25,<0.26", | |
| "tree_sitter_language_pack": "==0.9.0", | |
| } | |
| return normalized | |
| def _json_text(coverage: dict) -> str: | |
| return json.dumps(coverage, indent=2, sort_keys=True) + "\n" | |
| def _md_table_rows(languages: dict) -> list[str]: | |
| rows = [] | |
| for name in sorted(languages): | |
| rec = languages[name] | |
| load = "yes" if rec["load"] == "loads" else "**NO**" | |
| extraction = rec["extraction"] | |
| kinds = rec.get("kinds") | |
| kinds_text = ", ".join(f"`{k}`" for k in kinds) if kinds else "-" | |
| count = rec.get("node_count", "-") | |
| rows.append(f"| `{name}` | {load} | {extraction} | {count} | {kinds_text} |") | |
| return rows | |
| def _md_named_list(languages: dict, classification: str) -> list[str]: | |
| return sorted( | |
| name | |
| for name, rec in languages.items() | |
| if rec.get("extraction") == classification | |
| ) | |
| def _markdown(coverage: dict) -> str: | |
| summary = coverage["summary"] | |
| languages = coverage["languages"] | |
| ex = summary["extraction"] | |
| rich = _md_named_list(languages, RICH) | |
| sparse = _md_named_list(languages, SPARSE) | |
| empty = _md_named_list(languages, EMPTY) | |
| gaps = _md_named_list(languages, EXTRACTION_GAP) | |
| load_only = ex.get(LOAD_ONLY, 0) | |
| empty_gap = [lng for lng in empty if lng not in DATA_MARKUP_LANGUAGES] | |
| empty_expected = [lng for lng in empty if lng in DATA_MARKUP_LANGUAGES] | |
| lines: list[str] = [] | |
| lines.append("# Language coverage") | |
| lines.append("") | |
| lines.append( | |
| "Honest, machine-verified per-language coverage across the **entire** " | |
| "[tree-sitter-language-pack](https://github.com/Goldziher/" | |
| "tree-sitter-language-pack), generated by " | |
| "`scripts/regenerate_language_coverage.py` and continuously enforced by " | |
| "the smoke-tier gate (`tests/test_language_smoke.py`)." | |
| ) | |
| lines.append("") | |
| lines.append( | |
| "Pinned stack: `tree_sitter " | |
| f"{coverage['pins']['tree_sitter']}` + `tree-sitter-language-pack " | |
| f"{coverage['pins']['tree_sitter_language_pack']}` -- the " | |
| "verified-byte-stable pairing. Do not float the pack to >=1.x." | |
| ) | |
| lines.append("") | |
| lines.append("## Headline") | |
| lines.append("") | |
| lines.append( | |
| f"- **{summary['loads']} / {summary['pack_total']}** pack languages " | |
| "**LOAD** (grammar loads + parses a trivial input under the pinned " | |
| f"runtime); **{summary['fails_to_load']}** fail to load." | |
| ) | |
| lines.append( | |
| f"- **{ex.get(RICH, 0)} RICH** (>= 2 distinct boundary kinds) + " | |
| f"**{ex.get(SPARSE, 0)} SPARSE** (exactly 1 kind) extraction-verified " | |
| "on a curated valid sample." | |
| ) | |
| lines.append( | |
| f"- **{ex.get(EMPTY, 0)} EMPTY** (parse clean, 0 boundaries) and " | |
| f"**{ex.get(EXTRACTION_GAP, 0)} EXTRACTION-GAP** (chunker advertises the " | |
| "extension but no parser resolves -- a real, fixable bug surfaced here)." | |
| ) | |
| lines.append( | |
| f"- **{load_only} LOAD-ONLY** -- the long tail with no curated sample, " | |
| "honestly *not* extraction-tested rather than guessed." | |
| ) | |
| lines.append("") | |
| lines.append("## Honest tiers") | |
| lines.append("") | |
| lines.append( | |
| "1. **12 deep golden** -- full byte-level golden conformance + " | |
| "determinism gate (`tests/test_boundary_ir_golden_snapshots.py`): " | |
| + ", ".join( | |
| f"`{lng}`" | |
| for lng in ( | |
| "c", | |
| "cpp", | |
| "csharp", | |
| "go", | |
| "java", | |
| "javascript", | |
| "kotlin", | |
| "php", | |
| "python", | |
| "ruby", | |
| "swift", | |
| "typescript", | |
| ) | |
| ) | |
| + "." | |
| ) | |
| lines.append( | |
| "2. **Extraction-verified** -- RICH + SPARSE above: a real boundary " | |
| "surface on a valid sample, kind-set asserted by the smoke gate." | |
| ) | |
| lines.append( | |
| f"3. **{summary['loads']} load-verified** -- every pack grammar loads " | |
| "under the pin; forward ABI drift fails the smoke gate loudly." | |
| ) | |
| lines.append("") | |
| lines.append("## Named broken / sparse / gap languages (no inflation)") | |
| lines.append("") | |
| if gaps: | |
| lines.append( | |
| "**EXTRACTION-GAP (advertised extension, no backing parser -- " | |
| "broken):** " | |
| + ", ".join(f"`{lng}`" for lng in gaps) | |
| + ". chunker's `EXTENSION_MAP` routes these file types to a language " | |
| "name the parser layer cannot resolve; the IR records " | |
| "`boundary.parse_error` (\"Language 'X' not found\") and emits 0 " | |
| "nodes. `asm`/`rst` exist in the pack under different names " | |
| "(name-mapping gap); `cobol`/`fsharp`/`raku` have no pack grammar at " | |
| "all. Surfaced here, not patched (this PR does not change extraction " | |
| "behavior)." | |
| ) | |
| lines.append("") | |
| if empty_gap: | |
| lines.append( | |
| "**EMPTY -- genuine extraction gaps (code that parses but yields no " | |
| "boundaries):** " | |
| + ", ".join(f"`{lng}`" for lng in empty_gap) | |
| + ". These are real source languages whose grammar loads and parses " | |
| "cleanly, but chunker emits no boundary nodes for them today." | |
| ) | |
| lines.append("") | |
| if empty_expected: | |
| lines.append( | |
| "**EMPTY -- expected (data/markup, no function/type surface):** " | |
| + ", ".join(f"`{lng}`" for lng in empty_expected) | |
| + ". 0 boundaries is correct for these formats." | |
| ) | |
| lines.append("") | |
| if sparse: | |
| lines.append( | |
| "**SPARSE (1 kind -- thin surface):** " | |
| + ", ".join(f"`{lng}`" for lng in sparse) | |
| + ". Mostly single-construct (function-oriented) languages where 1 " | |
| "kind is the honest ceiling." | |
| ) | |
| lines.append("") | |
| lines.append("## Full per-language table") | |
| lines.append("") | |
| lines.append("| language | loads | extraction | nodes | kinds |") | |
| lines.append("| --- | --- | --- | --- | --- |") | |
| lines.extend(_md_table_rows(languages)) | |
| lines.append("") | |
| return "\n".join(lines) | |
| def main() -> int: | |
| assert_grammar_runtime_pins() | |
| coverage = _normalize_for_commit(compute_coverage()) | |
| JSON_PATH.parent.mkdir(parents=True, exist_ok=True) | |
| json_text = _json_text(coverage) | |
| md_text = _markdown(coverage) | |
| json_existing = ( | |
| JSON_PATH.read_text(encoding="utf-8") if JSON_PATH.exists() else None | |
| ) | |
| md_existing = MD_PATH.read_text(encoding="utf-8") if MD_PATH.exists() else None | |
| JSON_PATH.write_text(json_text, encoding="utf-8") | |
| MD_PATH.write_text(md_text, encoding="utf-8") | |
| summary = coverage["summary"] | |
| print("Regenerated language coverage (pinned stack):") | |
| print( | |
| f" pack_total={summary['pack_total']} loads={summary['loads']} " | |
| f"fails_to_load={summary['fails_to_load']}" | |
| ) | |
| print(f" extraction={summary['extraction']}") | |
| print( | |
| f" {JSON_PATH.name}: " | |
| f"{'unchanged' if json_existing == json_text else 'updated'}" | |
| ) | |
| print(f" {MD_PATH.name}: {'unchanged' if md_existing == md_text else 'updated'}") | |
| return 0 | |
| if __name__ == "__main__": | |
| raise SystemExit(main()) | |
Xet Storage Details
- Size:
- 9.63 kB
- Xet hash:
- 95fb421b7bbaa49a72fb09e09c755b398186308f32ca7696b0a176768139d960
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.