File size: 5,426 Bytes
a7c56ef | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 | """Head-to-head benchmark: vanilla Qwen2.5-7B-Instruct vs Prometheus active_model.
ADJUST PATHS
Runs the same 28 probes against both models, captures every prompt/response pair,
and produces a shareable JSON report with no implementation details exposed.
Usage:
python3 benchmark_comparison.py [--device cuda:0]
"""
import gc
import json
import torch
from datetime import datetime
from pathlib import Path
from transformers import AutoModelForCausalLM, AutoTokenizer
from self_eval import ALL_PROBES, generate, evaluate_probe
VANILLA_DIR = Path("/path/to/models/Qwen2.5-7B-Instruct")
PROMETHEUS_DIR = Path("/path/to/models/tryingforagi/active_model")
OUTPUT_FILE = Path("/path/to/models/prometheus/prometheus_benchmark.json")
def benchmark_model(model_dir, device, label):
print(f"\n{'='*60}")
print(f" Benchmarking: {label}")
print(f" Model: {model_dir}")
print(f"{'='*60}\n")
model = AutoModelForCausalLM.from_pretrained(
str(model_dir), device_map={"": device}, torch_dtype="auto",
low_cpu_mem_usage=True)
tokenizer = AutoTokenizer.from_pretrained(str(model_dir))
if tokenizer.pad_token is None:
tokenizer.pad_token = tokenizer.eos_token
model.eval()
results = []
by_category = {}
for i, probe in enumerate(ALL_PROBES):
response = generate(model, tokenizer, probe["prompt"])
score, details = evaluate_probe(model, tokenizer, probe)
entry = {
"probe": probe["prompt"],
"category": probe.get("category", "unknown"),
"response": response,
"score": round(score, 3),
}
results.append(entry)
cat = entry["category"]
if cat not in by_category:
by_category[cat] = {"scores": [], "probes": []}
by_category[cat]["scores"].append(score)
by_category[cat]["probes"].append(entry)
status = "PASS" if score >= 0.5 else "FAIL"
print(f" [{i+1:2d}/28] {status} {score:.3f} {cat:<15s} {probe['prompt'][:60]}")
category_scores = {}
for cat, data in by_category.items():
avg = sum(data["scores"]) / len(data["scores"])
category_scores[cat] = round(avg, 3)
overall = sum(e["score"] for e in results) / len(results)
del model, tokenizer
gc.collect()
torch.cuda.empty_cache()
return {
"label": label,
"overall_score": round(overall, 3),
"category_scores": dict(sorted(category_scores.items(), key=lambda x: x[1])),
"probes": results,
}
def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument("--device", default="cuda:0")
args = parser.parse_args()
report = {
"title": "Project Prometheus: Before vs After Benchmark",
"timestamp": datetime.now().isoformat(),
"description": (
"Head-to-head comparison of vanilla Qwen2.5-7B-Instruct against "
"the same model after one autonomous self-improvement cycle. "
"The model evaluated itself, identified its weakest area, selected "
"a modification strategy, and validated the result -- all without "
"human intervention. This benchmark re-runs the same 28 probes "
"against both models to independently verify the improvement."
),
"base_model": "Qwen/Qwen2.5-7B-Instruct",
"probes_count": len(ALL_PROBES),
"categories": sorted(set(p.get("category", "unknown") for p in ALL_PROBES)),
}
print("Phase 1: Benchmarking vanilla model...")
report["vanilla"] = benchmark_model(VANILLA_DIR, args.device, "Vanilla Qwen2.5-7B-Instruct")
print("\nPhase 2: Benchmarking Prometheus model...")
report["prometheus"] = benchmark_model(PROMETHEUS_DIR, args.device, "Prometheus (1 self-improvement cycle)")
v_score = report["vanilla"]["overall_score"]
p_score = report["prometheus"]["overall_score"]
delta = round(p_score - v_score, 3)
print(f"\n{'='*60}")
print(f" RESULTS")
print(f"{'='*60}")
print(f" Vanilla: {v_score}")
print(f" Prometheus: {p_score}")
print(f" Delta: {'+' if delta >= 0 else ''}{delta}")
print(f"\n Category breakdown:")
all_cats = sorted(set(
list(report["vanilla"]["category_scores"].keys()) +
list(report["prometheus"]["category_scores"].keys())
))
for cat in all_cats:
vs = report["vanilla"]["category_scores"].get(cat, 0)
ps = report["prometheus"]["category_scores"].get(cat, 0)
d = round(ps - vs, 3)
marker = " <--" if d != 0 else ""
print(f" {cat:<15s} {vs:.3f} -> {ps:.3f} ({'+' if d >= 0 else ''}{d}){marker}")
report["summary"] = {
"vanilla_overall": v_score,
"prometheus_overall": p_score,
"delta": delta,
"improved_categories": [
cat for cat in all_cats
if report["prometheus"]["category_scores"].get(cat, 0) >
report["vanilla"]["category_scores"].get(cat, 0)
],
"degraded_categories": [
cat for cat in all_cats
if report["prometheus"]["category_scores"].get(cat, 0) <
report["vanilla"]["category_scores"].get(cat, 0)
],
}
with open(OUTPUT_FILE, "w") as f:
json.dump(report, f, indent=2)
print(f"\n Full report saved to {OUTPUT_FILE}")
if __name__ == "__main__":
main()
|