Spaces:
Running on Zero
Running on Zero
File size: 2,798 Bytes
e8c5cc2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 | """Generate natural-language evaluation queries from papers, via Groq.
python scripts/generate_eval_queries.py --config configs/ml.yaml --n 300
Resumable: re-running tops up an existing file rather than starting over.
Spot-check the output before trusting any metric computed from it - if the
questions read like paper titles rather than something you would type, the
benchmark is worthless and you want to know now.
"""
from __future__ import annotations
import argparse
import random
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
if hasattr(sys.stdout, "reconfigure"):
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
from src.api.llm import is_configured
from src.config import load_config, seed_everything
from src.data.arxiv_source import read_papers
from src.data.chunking import read_chunks
from src.data.query_gen import generate_queries
BODY_SECTIONS = {"method", "methods", "approach", "experiments", "model", "architecture"}
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--config", default="configs/ml.yaml")
parser.add_argument("--n", type=int, default=300)
parser.add_argument("--out", default="data/generated_eval.jsonl")
parser.add_argument("--workers", type=int, default=6)
parser.add_argument("--model", default=None, help="override answer.model")
args = parser.parse_args()
if not is_configured():
raise SystemExit("GROQ_API_KEY is not set. Put it in .env (UTF-8, not UTF-16).")
cfg = load_config(args.config)
seed = seed_everything(cfg["seed"])
rng = random.Random(seed)
model = args.model or cfg["answer"]["model"]
papers = read_papers(cfg["paths"]["papers"])
with_text = {p.paper_id for p in papers if p.full_text}
# One method-ish chunk per paper gives the model something concrete to work
# from beyond the abstract's marketing.
context: dict[str, str] = {}
for chunk in read_chunks(cfg["paths"]["chunks"]):
if chunk.paper_id in with_text and chunk.section in BODY_SECTIONS:
context.setdefault(chunk.paper_id, chunk.text)
eligible = [p for p in papers if p.paper_id in context]
print(f"[qgen] {len(eligible)} eligible papers | model={model} | target={args.n}")
generated = generate_queries(
papers=eligible,
context_by_paper=context,
n_queries=args.n,
model=model,
out_path=args.out,
workers=args.workers,
rng=rng,
)
print("\n[qgen] spot-check - would you type these?\n")
for item in rng.sample(generated, min(8, len(generated))):
print(f' Q: "{item.query}"')
print(f" -> {item.gold_title[:70]}\n")
if __name__ == "__main__":
main()
|