"""Build the retrieval corpus and the dense FAISS index from the pinned Mathlib environment. Stage 1 (Phase 1.1): run scripts/extract_corpus.lean with `lake env lean` (allowed for offline data-building scripts, CLAUDE.md ยง7) to dump data/index/corpus_raw.jsonl, then post-process it into data/index/corpus.jsonl and data/index/corpus_meta.json. The full extraction takes ~15 min on a laptop (single process). Stage 2 (Phase 1.3): the full corpus.jsonl is embedded once per model (key from config/models.yaml โ†’ embeddings.candidates), normally on a Kaggle GPU via scripts/kaggle_job.py + scripts/kaggle_embed.py (CPU takes 3โ€“27 h per model). From those vectors `--from-vectors DIR` builds two exact cosine indexes (`IndexFlatIP` over โ„“2-normalised vectors) per model: data/index/full// the system's index (whole corpus, used by the Retriever) data/index/eval// evaluation index = corpus minus the queries of Q (the declarations of corpus_eval.jsonl), with the query vectors `--embed KEY` does the same on the local CPU. `--bm25` saves the BM25 index of the full corpus and its lemma table (names, signatures) to data/index/full/bm25/, so the Retriever does not rebuild it at every start. `--extra-names` runs scripts/extract_names.lean and saves the theorem names of Init/Std/Batteries (data/index/full/names_extra.json): they feed only the `unknown identifier` suggestions and are never added to the search index. Usage: python scripts/build_index.py [--module-prefix Mathlib] [--skip-extract] python scripts/build_index.py --from-vectors data/index/kaggle/output python scripts/build_index.py --embed bge-small python scripts/build_index.py --bm25 python scripts/build_index.py --extra-names """ from __future__ import annotations import argparse import hashlib import json import os import shutil import subprocess import sys from datetime import datetime, timezone from pathlib import Path ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "src")) from formal_mas.retrieval.corpus import ( # noqa: E402 SourceIndex, build_corpus, corpus_stats, load_jsonl, read_raw, write_jsonl, ) from formal_mas.retrieval.retriever import ( # noqa: E402 BM25_DIR, EXTRA_NAMES_FILE, build_extra_names, build_lexical_index, ) EXTRACT_SCRIPT = ROOT / "scripts" / "extract_corpus.lean" NAMES_SCRIPT = ROOT / "scripts" / "extract_names.lean" def mathlib_commit(lean_project: Path) -> str | None: manifest = json.loads((lean_project / "lake-manifest.json").read_text(encoding="utf-8")) for pkg in manifest.get("packages", []): if pkg.get("name") == "mathlib": return pkg.get("rev") return None def run_extraction(lean_project: Path, raw_path: Path, module_prefix: str) -> None: """Dump the environment with Lean. Raises CalledProcessError if Lean fails.""" run_lean_dump(lean_project, EXTRACT_SCRIPT, { "FORMAL_MAS_CORPUS_OUT": str(raw_path.resolve()), "FORMAL_MAS_MODULE_PREFIX": module_prefix, }, raw_path) def run_lean_dump(lean_project: Path, script: Path, env_vars: dict[str, str], out_path: Path) -> None: out_path.parent.mkdir(parents=True, exist_ok=True) env = {**os.environ, **env_vars} proc = subprocess.run( ["lake", "env", "lean", str(script)], cwd=lean_project, env=env, capture_output=True, text=True, encoding="utf-8", errors="replace", ) if proc.returncode != 0: raise subprocess.CalledProcessError(proc.returncode, proc.args, proc.stdout, proc.stderr) QUERY_FILES = ("queries.json", "queries_no_instr.npy", "queries_instr.npy") def names_sha(names: list[str]) -> str: return hashlib.sha256("\n".join(names).encode("utf-8")).hexdigest() def write_index(out: Path, vectors, names: list[str], meta: dict) -> None: import faiss out.mkdir(parents=True, exist_ok=True) index = faiss.IndexFlatIP(vectors.shape[1]) index.add(vectors) faiss.write_index(index, str(out / "index.faiss")) (out / "names.json").write_text(json.dumps(names), encoding="utf-8") (out / "meta.json").write_text(json.dumps({**meta, "size": len(names)}, indent=2), encoding="utf-8") def build_from_vectors(vec_dir: Path, index_root: Path) -> list[Path]: """Turn embedded full-corpus folders (Kaggle output layout) into the two indexes per model: full/ (system index, whole corpus) and eval/ (corpus minus the queries of Q, i.e. exactly the declarations of corpus_eval.jsonl, plus the precomputed query vectors).""" import numpy as np names = [r["name"] for r in load_jsonl(index_root / "corpus.jsonl")] eval_names = {r["name"] for r in load_jsonl(index_root / "corpus_eval.jsonl")} corpus_meta = json.loads((index_root / "corpus_meta.json").read_text(encoding="utf-8")) built = [] for folder in sorted(p for p in vec_dir.iterdir() if (p / "meta.json").exists()): meta = json.loads((folder / "meta.json").read_text(encoding="utf-8")) if meta.get("limit"): print(f"skip {folder.name}: speed-test sample (limit={meta['limit']})") continue if meta["corpus_names_sha256"] != names_sha(names): raise ValueError(f"{folder.name}: vectors were embedded from a different corpus.jsonl") vectors = np.load(folder / "corpus.npy").astype(np.float32) vectors /= np.linalg.norm(vectors, axis=1, keepdims=True).clip(min=1e-12) # undo fp16 rounding meta = {**meta, "index": "faiss.IndexFlatIP (l2-normalised vectors = cosine)", "mathlib_commit": corpus_meta.get("mathlib_commit"), "lean_toolchain": corpus_meta.get("lean_toolchain"), "created_utc": datetime.now(timezone.utc).isoformat(timespec="seconds")} key = meta["embedding_key"] write_index(index_root / "full" / key, vectors, names, {**meta, "corpus": "corpus.jsonl"}) keep = np.array([n in eval_names for n in names]) eval_dir = index_root / "eval" / key write_index(eval_dir, vectors[keep], [n for n, k in zip(names, keep) if k], {**meta, "corpus": "corpus_eval.jsonl (corpus.jsonl minus the queries of Q)"}) for f in QUERY_FILES: if (folder / f).exists(): shutil.copy(folder / f, eval_dir / f) built.append(eval_dir) print(f"{key}: full {len(names)}, eval {int(keep.sum())}") return built def embed_local(key: str, index_root: Path, vec_dir: Path, chunk: int, batch_size: int) -> Path: """CPU fallback producing the same layout as scripts/kaggle_embed.py (resumable chunks).""" import numpy as np from formal_mas.retrieval.encoders import Encoder, load_specs, model_revision spec = load_specs()[key] records = load_jsonl(index_root / "corpus.jsonl") out = vec_dir / key parts = out / "chunks" parts.mkdir(parents=True, exist_ok=True) encoder = Encoder(spec, batch_size=batch_size) for c, s in enumerate(range(0, len(records), chunk)): part = parts / f"{c:04d}.npy" if not part.exists(): np.save(part, encoder.encode([spec.doc_text(r) for r in records[s:s + chunk]]).astype(np.float16)) print(f"chunk {c}: {s + chunk}/{len(records)}", flush=True) np.save(out / "corpus.npy", np.concatenate([np.load(p) for p in sorted(parts.glob("*.npy"))])) queries = [r for split in ("dev", "test") for r in load_jsonl(index_root.parent / "retrieval_eval" / f"q_{split}.jsonl")] qtexts = [r["query"] for r in queries] np.save(out / "queries_no_instr.npy", encoder.encode([spec.query_text(t, False) for t in qtexts])) if spec.query_instruction: np.save(out / "queries_instr.npy", encoder.encode([spec.query_text(t, True) for t in qtexts])) (out / "queries.json").write_text(json.dumps([r["name"] for r in queries]), encoding="utf-8") meta = { "embedding_key": key, "hf_id": spec.hf_id, "hf_revision": model_revision(spec.hf_id), "backend": spec.backend, "max_length": spec.max_length, "doc_prefix": spec.doc_prefix, "doc_template": spec.doc_template, "query_instruction": spec.query_instruction, "dim": encoder.dim, "precision": "fp32", "storage": {"corpus": "float16", "queries": "float32"}, "n_corpus": len(records), "corpus_names_sha256": names_sha([r["name"] for r in records]), "limit": None, "n_queries": len(queries), "device": "cpu", } (out / "meta.json").write_text(json.dumps(meta, indent=2), encoding="utf-8") return out def main(argv: list[str] | None = None) -> None: parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) parser.add_argument("--lean-project", type=Path, default=ROOT / "lean_project") parser.add_argument("--out", type=Path, default=ROOT / "data" / "index") parser.add_argument("--module-prefix", default="Mathlib", help="only modules under this prefix") parser.add_argument("--skip-extract", action="store_true", help="reuse an existing corpus_raw.jsonl") parser.add_argument("--from-vectors", type=Path, metavar="DIR", help="stage 2 only: build full/ and eval/ indexes from embedded folders in DIR") parser.add_argument("--embed", metavar="KEY", help="stage 2 only: embed on CPU, then build the indexes") parser.add_argument("--bm25", action="store_true", help="stage 2 only: build the persisted BM25 index of the full corpus (full/bm25)") parser.add_argument("--extra-names", action="store_true", help="stage 2 only: dump Init/Std/Batteries theorem names for suggest_names " "(full/names_extra.json; never added to the search index)") parser.add_argument("--chunk", type=int, default=10_000, help="--embed: texts per resumable chunk") parser.add_argument("--batch-size", type=int, default=32) args = parser.parse_args(argv) if args.embed: vec_dir = args.out / "vectors" embed_local(args.embed, args.out, vec_dir, args.chunk, args.batch_size) args.from_vectors = vec_dir if args.from_vectors: build_from_vectors(args.from_vectors, args.out) if args.bm25: build_lexical_index(args.out / "corpus.jsonl", args.out / "full" / BM25_DIR) print(f"wrote BM25 index to {args.out / 'full' / BM25_DIR}") if args.extra_names: raw = args.out / "names_extra_raw.jsonl" run_lean_dump(args.lean_project, NAMES_SCRIPT, {"FORMAL_MAS_NAMES_OUT": str(raw.resolve())}, raw) lean_meta = {"toolchain": (args.lean_project / "lean-toolchain").read_text(encoding="utf-8").strip(), "mathlib_commit": mathlib_commit(args.lean_project)} n = build_extra_names(raw, args.out / "full" / EXTRA_NAMES_FILE, lean_meta) print(f"wrote {n} Init/Std/Batteries theorem names to {args.out / 'full' / EXTRA_NAMES_FILE}") if args.from_vectors or args.bm25 or args.extra_names: return raw_path = args.out / "corpus_raw.jsonl" if not args.skip_extract: run_extraction(args.lean_project, raw_path, args.module_prefix) lean_meta, entries = read_raw(raw_path) sources = SourceIndex(args.lean_project / ".lake" / "packages" / "mathlib") decls, excluded = build_corpus(entries, sources) corpus_path = args.out / "corpus.jsonl" n = write_jsonl(decls, corpus_path) stats = corpus_stats(load_jsonl(corpus_path)) meta = { "mathlib_commit": mathlib_commit(args.lean_project), "lean_toolchain": (args.lean_project / "lean-toolchain").read_text(encoding="utf-8").strip(), "created_utc": datetime.now(timezone.utc).isoformat(timespec="seconds"), "extraction": lean_meta, "postprocess": "whitespace collapsed; leading name and universe lists `.{...}` removed", "excluded": dict(excluded), "corpus": stats, } (args.out / "corpus_meta.json").write_text(json.dumps(meta, indent=2), encoding="utf-8") print(f"wrote {n} declarations to {corpus_path}") print(json.dumps({"excluded": dict(excluded), **stats}, indent=2)) if __name__ == "__main__": main()