File size: 6,284 Bytes
7845694 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 | """Rebuild Every's 204 inference rows and 154 author-labeled rows without executing its archive."""
from __future__ import annotations
import argparse
import json
from pathlib import Path
import re
import zipfile
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--archive", type=Path, required=True)
parser.add_argument("--experiments", type=Path, required=True)
parser.add_argument("--selection", type=Path, required=True)
parser.add_argument("--output-dir", type=Path, required=True)
args = parser.parse_args()
if args.output_dir.exists():
parser.error("Output directory must be new")
args.output_dir.mkdir(parents=True)
data = json.loads(args.experiments.read_text())
wanted = {
row["id"]
for line in args.selection.read_text().splitlines()
if line.strip()
for row in [json.loads(line)]
if row["source"] == "every"
}
original = []
with zipfile.ZipFile(args.archive) as archive:
script = archive.read("scripts/run-experiments.mjs").decode()
policy = json.loads(re.search(r'const SUPPORT_POLICY = ("[^\n]+");', script).group(1))
for experiment in data["experiments"]:
name = experiment["id"]
if name == "judge-grid":
cells = {(cell["output"], cell["question"]): cell for cell in experiment["typesafe"]["cells"]}
for document in experiment["documents"]:
for question_id, question in experiment["questions"].items():
original.append(
{
"id": f"{name}/{document['id']}/{question_id}",
"experiment": name,
"source_item": document["id"],
"question_id": question_id,
"document": {"policy": policy, "candidate_response": document["response"]},
"question": question,
"expected": bool(document["expected"][question_id]),
"recorded_typesafe": cells[document["id"], question_id],
}
)
if name in {"code-rag", "company-brain"}:
folder = "code-repo" if name == "code-rag" else "company-brain"
for query in experiment["grid"]:
for cell in query["cells"]:
original.append(
{
"id": f"{name}/{cell['document']}/{query['id']}",
"experiment": name,
"source_item": cell["document"],
"question_id": query["id"],
"document": archive.read(f"fixtures/{folder}/{cell['document']}").decode(),
"question": query["question"],
"expected": cell["relevant"],
"recorded_typesafe": cell,
}
)
options = [{"id": "yes", "description": "Yes"}, {"id": "no", "description": "No"}]
inference, gold = [], []
for row in original:
item = {
"id": "every/" + row["id"],
"group_id": f"every/{row['experiment']}/{row['source_item']}",
"family": "every_" + row["experiment"],
"split": "external_test",
"state": row["document"],
"question": row["question"],
"options": options,
"provenance": {
"source": "Every Parallel Judgment Lab",
"experiment": row["experiment"],
"question_id": row["question_id"],
"source_item": row["source_item"],
"label_source": "Original author label",
"adaptation": "Native Noul replaced with binary Yes/No Choice; state/question unchanged",
},
}
inference.append(item)
gold.append({**item, "label": 0 if row["expected"] else 1})
firewall = next(experiment for experiment in data["experiments"] if experiment["id"] == "action-firewall")
for item in firewall["rows"]:
for question_id, question in firewall["questions"].items():
inference.append(
{
"id": f"every/action-firewall/{item['id']}/{question_id}",
"group_id": f"every/action-firewall/{item['id']}",
"family": "every_action-firewall",
"split": "external_test",
"state": item["text"],
"question": question,
"options": options,
"provenance": {
"source": "Every Parallel Judgment Lab",
"experiment": "action-firewall",
"question_id": question_id,
"source_item": item["id"],
"label_source": "No per-signal labels; evaluate composed action only",
"adaptation": "Native Noul replaced with binary Yes/No Choice; state/question unchanged",
},
}
)
if len(inference) != 204 or len(gold) != 154 or {row["id"] for row in inference} != wanted:
raise ValueError("Rebuilt Every rows differ from the frozen 204-row selection")
(args.output_dir / "inference204.jsonl").write_text(
"".join(json.dumps(row, ensure_ascii=False) + "\n" for row in inference)
)
(args.output_dir / "gold154.jsonl").write_text(
"".join(json.dumps(row, ensure_ascii=False) + "\n" for row in gold)
)
(args.output_dir / "firewall-actions.json").write_text(
json.dumps(
{
"expected_actions": {row["id"]: row["expected"] for row in firewall["rows"]},
"label_source": "Original author intended actions; no per-signal truth labels",
},
indent=2,
)
+ "\n"
)
print("wrote 204 inference rows, 154 labeled rows, and firewall actions")
if __name__ == "__main__":
main()
|