formal-mas / scripts /build_index.py
NicolasT08's picture
Formal-MAS source code
dfe1826 verified
Raw History Blame Contribute Delete
12.3 kB
"""Build the retrieval corpus and the dense FAISS index from the pinned Mathlib environment.
Stage 1 (Phase 1.1): run scripts/extract_corpus.lean with `lake env lean` (allowed for
offline data-building scripts, CLAUDE.md §7) to dump data/index/corpus_raw.jsonl, then
post-process it into data/index/corpus.jsonl and data/index/corpus_meta.json.
The full extraction takes ~15 min on a laptop (single process).
Stage 2 (Phase 1.3): the full corpus.jsonl is embedded once per model (key from
config/models.yaml → embeddings.candidates), normally on a Kaggle GPU via
scripts/kaggle_job.py + scripts/kaggle_embed.py (CPU takes 3–27 h per model). From those
vectors `--from-vectors DIR` builds two exact cosine indexes (`IndexFlatIP` over
ℓ2-normalised vectors) per model:
data/index/full/<key>/ the system's index (whole corpus, used by the Retriever)
data/index/eval/<key>/ evaluation index = corpus minus the queries of Q (the
declarations of corpus_eval.jsonl), with the query vectors
`--embed KEY` does the same on the local CPU. `--bm25` saves the BM25 index of the full
corpus and its lemma table (names, signatures) to data/index/full/bm25/, so the Retriever
does not rebuild it at every start. `--extra-names` runs scripts/extract_names.lean and
saves the theorem names of Init/Std/Batteries (data/index/full/names_extra.json): they feed
only the `unknown identifier` suggestions and are never added to the search index.
Usage:
python scripts/build_index.py [--module-prefix Mathlib] [--skip-extract]
python scripts/build_index.py --from-vectors data/index/kaggle/output
python scripts/build_index.py --embed bge-small
python scripts/build_index.py --bm25
python scripts/build_index.py --extra-names
"""
from __future__ import annotations
import argparse
import hashlib
import json
import os
import shutil
import subprocess
import sys
from datetime import datetime, timezone
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "src"))
from formal_mas.retrieval.corpus import ( # noqa: E402
SourceIndex,
build_corpus,
corpus_stats,
load_jsonl,
read_raw,
write_jsonl,
)
from formal_mas.retrieval.retriever import ( # noqa: E402
BM25_DIR,
EXTRA_NAMES_FILE,
build_extra_names,
build_lexical_index,
)
EXTRACT_SCRIPT = ROOT / "scripts" / "extract_corpus.lean"
NAMES_SCRIPT = ROOT / "scripts" / "extract_names.lean"
def mathlib_commit(lean_project: Path) -> str | None:
manifest = json.loads((lean_project / "lake-manifest.json").read_text(encoding="utf-8"))
for pkg in manifest.get("packages", []):
if pkg.get("name") == "mathlib":
return pkg.get("rev")
return None
def run_extraction(lean_project: Path, raw_path: Path, module_prefix: str) -> None:
"""Dump the environment with Lean. Raises CalledProcessError if Lean fails."""
run_lean_dump(lean_project, EXTRACT_SCRIPT, {
"FORMAL_MAS_CORPUS_OUT": str(raw_path.resolve()),
"FORMAL_MAS_MODULE_PREFIX": module_prefix,
}, raw_path)
def run_lean_dump(lean_project: Path, script: Path, env_vars: dict[str, str], out_path: Path) -> None:
out_path.parent.mkdir(parents=True, exist_ok=True)
env = {**os.environ, **env_vars}
proc = subprocess.run(
["lake", "env", "lean", str(script)],
cwd=lean_project,
env=env,
capture_output=True,
text=True,
encoding="utf-8",
errors="replace",
)
if proc.returncode != 0:
raise subprocess.CalledProcessError(proc.returncode, proc.args, proc.stdout, proc.stderr)
QUERY_FILES = ("queries.json", "queries_no_instr.npy", "queries_instr.npy")
def names_sha(names: list[str]) -> str:
return hashlib.sha256("\n".join(names).encode("utf-8")).hexdigest()
def write_index(out: Path, vectors, names: list[str], meta: dict) -> None:
import faiss
out.mkdir(parents=True, exist_ok=True)
index = faiss.IndexFlatIP(vectors.shape[1])
index.add(vectors)
faiss.write_index(index, str(out / "index.faiss"))
(out / "names.json").write_text(json.dumps(names), encoding="utf-8")
(out / "meta.json").write_text(json.dumps({**meta, "size": len(names)}, indent=2), encoding="utf-8")
def build_from_vectors(vec_dir: Path, index_root: Path) -> list[Path]:
"""Turn embedded full-corpus folders (Kaggle output layout) into the two indexes per model:
full/<key> (system index, whole corpus) and eval/<key> (corpus minus the queries of Q,
i.e. exactly the declarations of corpus_eval.jsonl, plus the precomputed query vectors)."""
import numpy as np
names = [r["name"] for r in load_jsonl(index_root / "corpus.jsonl")]
eval_names = {r["name"] for r in load_jsonl(index_root / "corpus_eval.jsonl")}
corpus_meta = json.loads((index_root / "corpus_meta.json").read_text(encoding="utf-8"))
built = []
for folder in sorted(p for p in vec_dir.iterdir() if (p / "meta.json").exists()):
meta = json.loads((folder / "meta.json").read_text(encoding="utf-8"))
if meta.get("limit"):
print(f"skip {folder.name}: speed-test sample (limit={meta['limit']})")
continue
if meta["corpus_names_sha256"] != names_sha(names):
raise ValueError(f"{folder.name}: vectors were embedded from a different corpus.jsonl")
vectors = np.load(folder / "corpus.npy").astype(np.float32)
vectors /= np.linalg.norm(vectors, axis=1, keepdims=True).clip(min=1e-12) # undo fp16 rounding
meta = {**meta, "index": "faiss.IndexFlatIP (l2-normalised vectors = cosine)",
"mathlib_commit": corpus_meta.get("mathlib_commit"),
"lean_toolchain": corpus_meta.get("lean_toolchain"),
"created_utc": datetime.now(timezone.utc).isoformat(timespec="seconds")}
key = meta["embedding_key"]
write_index(index_root / "full" / key, vectors, names, {**meta, "corpus": "corpus.jsonl"})
keep = np.array([n in eval_names for n in names])
eval_dir = index_root / "eval" / key
write_index(eval_dir, vectors[keep], [n for n, k in zip(names, keep) if k],
{**meta, "corpus": "corpus_eval.jsonl (corpus.jsonl minus the queries of Q)"})
for f in QUERY_FILES:
if (folder / f).exists():
shutil.copy(folder / f, eval_dir / f)
built.append(eval_dir)
print(f"{key}: full {len(names)}, eval {int(keep.sum())}")
return built
def embed_local(key: str, index_root: Path, vec_dir: Path, chunk: int, batch_size: int) -> Path:
"""CPU fallback producing the same layout as scripts/kaggle_embed.py (resumable chunks)."""
import numpy as np
from formal_mas.retrieval.encoders import Encoder, load_specs, model_revision
spec = load_specs()[key]
records = load_jsonl(index_root / "corpus.jsonl")
out = vec_dir / key
parts = out / "chunks"
parts.mkdir(parents=True, exist_ok=True)
encoder = Encoder(spec, batch_size=batch_size)
for c, s in enumerate(range(0, len(records), chunk)):
part = parts / f"{c:04d}.npy"
if not part.exists():
np.save(part, encoder.encode([spec.doc_text(r) for r in records[s:s + chunk]]).astype(np.float16))
print(f"chunk {c}: {s + chunk}/{len(records)}", flush=True)
np.save(out / "corpus.npy", np.concatenate([np.load(p) for p in sorted(parts.glob("*.npy"))]))
queries = [r for split in ("dev", "test")
for r in load_jsonl(index_root.parent / "retrieval_eval" / f"q_{split}.jsonl")]
qtexts = [r["query"] for r in queries]
np.save(out / "queries_no_instr.npy", encoder.encode([spec.query_text(t, False) for t in qtexts]))
if spec.query_instruction:
np.save(out / "queries_instr.npy", encoder.encode([spec.query_text(t, True) for t in qtexts]))
(out / "queries.json").write_text(json.dumps([r["name"] for r in queries]), encoding="utf-8")
meta = {
"embedding_key": key, "hf_id": spec.hf_id, "hf_revision": model_revision(spec.hf_id),
"backend": spec.backend, "max_length": spec.max_length, "doc_prefix": spec.doc_prefix,
"doc_template": spec.doc_template, "query_instruction": spec.query_instruction,
"dim": encoder.dim, "precision": "fp32", "storage": {"corpus": "float16", "queries": "float32"},
"n_corpus": len(records), "corpus_names_sha256": names_sha([r["name"] for r in records]),
"limit": None, "n_queries": len(queries), "device": "cpu",
}
(out / "meta.json").write_text(json.dumps(meta, indent=2), encoding="utf-8")
return out
def main(argv: list[str] | None = None) -> None:
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
parser.add_argument("--lean-project", type=Path, default=ROOT / "lean_project")
parser.add_argument("--out", type=Path, default=ROOT / "data" / "index")
parser.add_argument("--module-prefix", default="Mathlib", help="only modules under this prefix")
parser.add_argument("--skip-extract", action="store_true", help="reuse an existing corpus_raw.jsonl")
parser.add_argument("--from-vectors", type=Path, metavar="DIR",
help="stage 2 only: build full/ and eval/ indexes from embedded folders in DIR")
parser.add_argument("--embed", metavar="KEY", help="stage 2 only: embed on CPU, then build the indexes")
parser.add_argument("--bm25", action="store_true",
help="stage 2 only: build the persisted BM25 index of the full corpus (full/bm25)")
parser.add_argument("--extra-names", action="store_true",
help="stage 2 only: dump Init/Std/Batteries theorem names for suggest_names "
"(full/names_extra.json; never added to the search index)")
parser.add_argument("--chunk", type=int, default=10_000, help="--embed: texts per resumable chunk")
parser.add_argument("--batch-size", type=int, default=32)
args = parser.parse_args(argv)
if args.embed:
vec_dir = args.out / "vectors"
embed_local(args.embed, args.out, vec_dir, args.chunk, args.batch_size)
args.from_vectors = vec_dir
if args.from_vectors:
build_from_vectors(args.from_vectors, args.out)
if args.bm25:
build_lexical_index(args.out / "corpus.jsonl", args.out / "full" / BM25_DIR)
print(f"wrote BM25 index to {args.out / 'full' / BM25_DIR}")
if args.extra_names:
raw = args.out / "names_extra_raw.jsonl"
run_lean_dump(args.lean_project, NAMES_SCRIPT, {"FORMAL_MAS_NAMES_OUT": str(raw.resolve())}, raw)
lean_meta = {"toolchain": (args.lean_project / "lean-toolchain").read_text(encoding="utf-8").strip(),
"mathlib_commit": mathlib_commit(args.lean_project)}
n = build_extra_names(raw, args.out / "full" / EXTRA_NAMES_FILE, lean_meta)
print(f"wrote {n} Init/Std/Batteries theorem names to {args.out / 'full' / EXTRA_NAMES_FILE}")
if args.from_vectors or args.bm25 or args.extra_names:
return
raw_path = args.out / "corpus_raw.jsonl"
if not args.skip_extract:
run_extraction(args.lean_project, raw_path, args.module_prefix)
lean_meta, entries = read_raw(raw_path)
sources = SourceIndex(args.lean_project / ".lake" / "packages" / "mathlib")
decls, excluded = build_corpus(entries, sources)
corpus_path = args.out / "corpus.jsonl"
n = write_jsonl(decls, corpus_path)
stats = corpus_stats(load_jsonl(corpus_path))
meta = {
"mathlib_commit": mathlib_commit(args.lean_project),
"lean_toolchain": (args.lean_project / "lean-toolchain").read_text(encoding="utf-8").strip(),
"created_utc": datetime.now(timezone.utc).isoformat(timespec="seconds"),
"extraction": lean_meta,
"postprocess": "whitespace collapsed; leading name and universe lists `.{...}` removed",
"excluded": dict(excluded),
"corpus": stats,
}
(args.out / "corpus_meta.json").write_text(json.dumps(meta, indent=2), encoding="utf-8")
print(f"wrote {n} declarations to {corpus_path}")
print(json.dumps({"excluded": dict(excluded), **stats}, indent=2))
if __name__ == "__main__":
main()