Spaces:
Running
Running
Download experiments/baselines.py from parsa-hke/negative-space-reader: direct link, hf CLI and curl.
- Browser
- Download file 10.7 kB
-
https://huggingface.co/spaces/parsa-hke/negative-space-reader/resolve/main/experiments/baselines.py
- Command line
-
hf download hf://spaces/parsa-hke/negative-space-reader/experiments/baselines.py
-
curl -L -o baselines.py https://huggingface.co/spaces/parsa-hke/negative-space-reader/resolve/main/experiments/baselines.py
10.7 kB
| """Frontier-model baselines (d)-(f) (plan §0.2 #17; §7.3, §9.3). | |
| Baseline (e), candidate list (7.3, `baselines.stage: candidate`): a seeded sample of `llm_gap_instances` | |
| OmitBench-synthetic instances per genre (`baselines.regimes` of the run `baselines.synth_run`), each edited document | |
| written as plain Markdown into prompts/llm_gap_candidate.md with the genre's frozen catalogue; the model (the | |
| `models.teacher` route: configs/llm_gap_anthropic.yaml for Claude Opus 5.5 by Message Batches, configs/openrouter.yaml | |
| for the open teacher) returns at most `top_k` absent type ids, most conspicuous first. Scored like the system | |
| (bench/metrics.py) and paired, instance by instance, with the level-1 and genre-uniform rankings recorded in | |
| `baselines.compare_to` (bench_eval_v2): MRR is cut at top_k for every system, since the model ranks only top_k. | |
| python -m experiments.run --config configs/baselines.yaml --overlay llm_gap_anthropic --name llm_gap_dry \ | |
| --set baselines.dry_run=true --set 'run.seeds=[0]' | |
| dry_run renders every prompt, writes <run>/<genre>/requests.jsonl and an estimate of tokens and dollars, and calls | |
| no model. Baselines (d) and (f): Phase 9. | |
| """ | |
| from __future__ import annotations | |
| import csv | |
| import json | |
| import random | |
| from pathlib import Path | |
| from typing import Any, Literal | |
| def plain_markdown(title: str | None, sections: list[dict[str, Any]]) -> str: | |
| """An edited document as the model reads it: title, then each section under its heading (no markers).""" | |
| out = [f"# {title}"] if title else [] | |
| for s in sections: | |
| out.append((f"## {s['heading']}\n" if s.get("heading") else "") + s["text"].strip()) | |
| return "\n\n".join(out) | |
| def sample_instances(folder: Path, n: int, regimes: list[str], seed: int) -> list[dict[str, Any]]: | |
| rows = [json.loads(x) for x in open(folder / "instances.jsonl", encoding="utf-8")] | |
| rows = [r for r in rows if r["regime"] in regimes] | |
| rows.sort(key=lambda r: (r["regime"], r["doc_id"])) | |
| return random.Random(seed).sample(rows, min(n, len(rows))) | |
| def answer_model(types: list[str]) -> Any: | |
| from pydantic import BaseModel, create_model | |
| return create_model("AbsentTypes", __base__=BaseModel, absent_types=(list[Literal[tuple(types)]], ...)) # type: ignore[valid-type] | |
| def _route(teacher: Any, cfg: dict[str, Any]) -> str: | |
| tcfg = cfg["models"]["teacher"] | |
| return getattr(teacher, "route", None) or f"{tcfg.get('backend')}:{tcfg.get('id')}" | |
| def _mean_ci(x: list[float], n_boot: int, seed: int) -> tuple[float, float, float]: | |
| from bench.metrics import bootstrap_mean | |
| return bootstrap_mean(x, n_boot=n_boot, seed=seed) | |
| def _answer(teacher: Any, prompts: list[str], model: Any, max_tokens: int) -> list[Any]: | |
| """All answers in one call; if some item fails (e.g. a prompt beyond the route's context), each prompt alone, the | |
| failed ones None (scored as an empty ranking and counted as unanswered; prompts are never truncated). Answers | |
| already received come back from the teacher's cache.""" | |
| from models.teacher import StructuredOutputError | |
| try: | |
| return teacher.generate_structured_batch(prompts, model, max_tokens=max_tokens) | |
| except StructuredOutputError as e: | |
| print(f"[baselines] {e}; answering one prompt at a time", flush=True) | |
| out = [] | |
| for p in prompts: | |
| try: | |
| out.append(teacher.generate_structured_batch([p], model, max_tokens=max_tokens)[0]) | |
| except StructuredOutputError: | |
| out.append(None) | |
| return out | |
| def run_candidate(cfg: dict[str, Any], run_dir: Path, *, teacher: Any = None) -> dict[str, Any]: | |
| import yaml | |
| from bench.metrics import instance_metrics, paired_bootstrap | |
| from models.teacher import PROMPTS, load_teacher | |
| from packs.builder import _path | |
| b = cfg["baselines"] | |
| genres = [cfg["pack"]] if cfg.get("pack") else list(cfg["genres"]) | |
| top_k, n, dry = int(b.get("top_k", 10)), int(b["llm_gap_instances"]), bool(b.get("dry_run", False)) | |
| regimes, seed = list(b.get("regimes") or ["single_type"]), int(b.get("sample_seed", 2026)) | |
| n_boot = cfg["bench"].get("bootstrap_samples", 1000) | |
| prompt = PROMPTS.get("llm_gap_candidate") | |
| run_dir = Path(run_dir) | |
| m: dict[str, Any] = {"dry_run": dry, "prompt": prompt.tag} | |
| if not dry and teacher is None: | |
| teacher = load_teacher(cfg["models"]["teacher"]) | |
| system = None if dry else _route(teacher, cfg) | |
| for g in genres: | |
| out = run_dir / g | |
| out.mkdir(parents=True, exist_ok=True) | |
| folder = Path(str(b["synth_run"]).format(genre=g)) | |
| insts = sample_instances(folder, n, regimes, seed) | |
| want = {f"{r['regime']}:{r['doc_id']}" for r in insts} | |
| edited = {} | |
| for line in open(folder / "edited.jsonl", encoding="utf-8"): | |
| r = json.loads(line) | |
| if r["instance_id"] in want: | |
| edited[r["instance_id"]] = r | |
| schema = yaml.safe_load((_path(cfg["paths"]["packs"]) / g / "schema.yaml").read_text(encoding="utf-8")) | |
| types = [t["id"] for t in schema["types"]] | |
| info = cfg["genre_info"][g] | |
| catalogue = "\n".join(f"{t['id']}: {t.get('definition', '').strip()}" for t in schema["types"]) | |
| ids = [f"{r['regime']}:{r['doc_id']}" for r in insts] | |
| prompts = [prompt.render(document_kind=info["document_kind"], author_role=info["author_role"], | |
| type_catalogue=catalogue, top_k=top_k, | |
| document=plain_markdown(edited[i].get("title"), edited[i]["sections"])) for i in ids] | |
| chars = sum(len(p) for p in prompts) | |
| est = b.get("estimate") or {} | |
| tok_in = chars / float(est.get("chars_per_token", 3.5)) | |
| tok_out = len(prompts) * float(est.get("output_tokens", 2500)) | |
| usd = (tok_in * est.get("usd_per_mtok_input", 4.0) + tok_out * est.get("usd_per_mtok_output", 20.0)) / 1e6 | |
| usd *= float(est.get("batch_discount", 0.5)) | |
| m.update({f"{g}/instances": len(ids), f"{g}/prompt_chars": chars, f"{g}/estimated_input_tokens": round(tok_in), | |
| f"{g}/estimated_output_tokens": round(tok_out), f"{g}/estimated_usd": round(usd, 2)}) | |
| if dry: | |
| with open(out / "requests.jsonl", "w", encoding="utf-8") as f: | |
| for i, p in zip(ids, prompts): | |
| f.write(json.dumps({"instance_id": i, "prompt": p}, ensure_ascii=False) + "\n") | |
| continue | |
| answers = _answer(teacher, prompts, answer_model(types), int(b.get("max_tokens", 16000))) | |
| gt = {f"{r['regime']}:{r['doc_id']}": set(r["ground_truth"]) for r in insts} | |
| rows, invalid = [], 0 | |
| for i, a in zip(ids, answers): | |
| ranked: list[str] = [] | |
| for t in (a.absent_types if a is not None else []): | |
| if t not in types: | |
| invalid += 1 | |
| elif t not in ranked: | |
| ranked.append(t) | |
| ranked = ranked[:top_k] | |
| im = instance_metrics(ranked, gt[i], ks=(1, 5, 10)) | |
| rows.append({"instance_id": i, "regime": i.split(":", 1)[0], "system": system, "ground_truth": len(gt[i]), | |
| "answered": a is not None, **{k: im[k] for k in ("recall@1", "recall@5", "recall@10", "ndcg@10")}, | |
| "mrr@10": im["mrr"], "rank_first": im["rank_first"] or "", "ranked": "|".join(ranked)}) | |
| with open(out / "per_instance.csv", "w", newline="", encoding="utf-8") as f: | |
| w = csv.DictWriter(f, fieldnames=list(rows[0])) | |
| w.writeheader() | |
| w.writerows(rows) | |
| m[f"{g}/invalid_ids"] = invalid | |
| m[f"{g}/unanswered"] = sum(not r["answered"] for r in rows) | |
| # the recorded systems on the same instances, MRR cut at top_k like the model's | |
| rec: dict[str, dict[str, dict[str, float]]] = {} | |
| for r in csv.DictReader(open(Path(b["compare_to"]) / g / "per_instance.csv", encoding="utf-8")): | |
| if r["instance_id"] in gt: | |
| rf = int(r["rank_first"]) if r["rank_first"] else None | |
| rec.setdefault(r["system"], {})[r["instance_id"]] = { | |
| **{k: float(r[k]) for k in ("recall@1", "recall@5", "recall@10", "ndcg@10")}, | |
| "mrr@10": 1 / rf if rf and rf <= top_k else 0.0} | |
| metrics = ("recall@1", "recall@5", "recall@10", "mrr@10", "ndcg@10") | |
| llm = {r["instance_id"]: {k: float(r[k]) for k in metrics} for r in rows} | |
| table = [] | |
| for name, vals in [(system, llm)] + [(s, rec[s]) for s in ("level1", "genre_uniform") if s in rec]: | |
| row: dict[str, Any] = {"genre": g, "system": name, "n": len(ids)} | |
| for k in metrics: | |
| mean, lo, hi = _mean_ci([vals[i][k] for i in ids], n_boot, seed) | |
| row.update({k: round(mean, 3), f"{k}_ci": f"[{lo:.3f}, {hi:.3f}]"}) | |
| table.append(row) | |
| m[f"{g}/{name}/recall@5"] = row["recall@5"] | |
| for s in ("level1", "genre_uniform"): | |
| if s in rec: | |
| row = {"genre": g, "system": f"{system} - {s}", "n": len(ids)} | |
| for k in metrics: | |
| d = paired_bootstrap([llm[i][k] for i in ids], [rec[s][i][k] for i in ids], n_boot=n_boot, seed=seed) | |
| row.update({k: round(d["diff"], 3), f"{k}_ci": f"[{d['ci_low']:.3f}, {d['ci_high']:.3f}]"}) | |
| table.append(row) | |
| for folder_out in [out] + ([_path(b["tables_dir"])] if b.get("tables_dir") else []): | |
| folder_out.mkdir(parents=True, exist_ok=True) | |
| path = folder_out / f"llm_gap_candidate_{g}.csv" | |
| rows_out = list(table) | |
| if path.exists(): # keep the other models' rows (one recorded run per model); this run replaces its own | |
| ours = {r["system"] for r in table} | |
| kept = [r for r in csv.DictReader(open(path, encoding="utf-8")) if r["system"] not in ours] | |
| rows_out = [r for r in kept if " - " not in r["system"]] + [r for r in table if " - " not in r["system"]] \ | |
| + [r for r in kept if " - " in r["system"]] + [r for r in table if " - " in r["system"]] | |
| with open(path, "w", newline="", encoding="utf-8") as f: | |
| w = csv.DictWriter(f, fieldnames=list(table[0])) | |
| w.writeheader() | |
| w.writerows(rows_out) | |
| return m | |
| def main(cfg: dict[str, Any], run_dir: Path) -> dict[str, Any]: | |
| stage = (cfg.get("baselines") or {}).get("stage", "candidate") | |
| if stage != "candidate": | |
| raise ValueError(f"baselines.stage {stage!r}: only candidate (baseline (e), plan 7.3) is implemented") | |
| return run_candidate(cfg, run_dir) | |