File size: 5,252 Bytes
23ea6bd | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 | """
Nova 1.0 — Humanity's Last Exam (HLE) Automated Evaluation Benchmark Suite
Evaluates Nova 1.0 across 5 core expert academic domains:
1. Advanced Mathematics & Calculus
2. Physics & General Relativity
3. Computer Science & Data Structures
4. Chemistry & Molecular Geometry
5. Formal Logic & Transitive Inferences
"""
import os
import sys
import argparse
import time
import torch
# Add project root to sys.path
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
from config.model_config import Nova1Config
from src.tokenizer.bpe_tokenizer import Nova1Tokenizer
from src.model.nova1_hrm import Nova1HRM
from src.inference.generate import Nova1Generator
HLE_BENCHMARK_QUESTIONS = [
{
"category": "Mathematics & Calculus",
"prompt": "Evaluate the definite integral int_0^1 x^7 dx.",
"expected_keywords": ["1/8", "1 / 8", "0.125", "power rule"],
},
{
"category": "Physics & Quantum Mechanics",
"prompt": "What is the energy formula of a photon with frequency nu?",
"expected_keywords": ["h", "nu", "planck", "E = h"],
},
{
"category": "Physics & General Relativity",
"prompt": "Calculate the Schwarzschild radius of a mass M.",
"expected_keywords": ["r_s", "2 * G * M", "2GM", "c^2", "c2", "radius"],
},
{
"category": "Computer Science & Algorithms",
"prompt": "What is the time complexity of building a heap from an array of N elements?",
"expected_keywords": ["O(N)", "O(n)", "linear", "floyd"],
},
{
"category": "Chemistry & Molecular Geometry",
"prompt": "What is the molecular geometry and hybridization of sulfur hexafluoride (SF6)?",
"expected_keywords": ["octahedral", "sp3d2", "sp^3d^2"],
},
{
"category": "Formal Logic & Syllogisms",
"prompt": "If all A are B, and all B are C, are all A also C?",
"expected_keywords": ["yes", "transitive", "logically", "follows"],
},
]
def evaluate_hle_benchmark():
parser = argparse.ArgumentParser(description="Evaluate Nova 1.0 on HLE Benchmark")
parser.add_argument("--checkpoint", type=str, default="checkpoints/nova1_final.pt", help="Path to checkpoint")
parser.add_argument("--tokenizer_path", type=str, default="checkpoints/nova1_gemini_tokenizer.json", help="Tokenizer JSON")
args = parser.parse_args()
tokenizer = Nova1Tokenizer.load(args.tokenizer_path)
checkpoint = torch.load(args.checkpoint, map_location="cpu", weights_only=False)
config: Nova1Config = checkpoint.get("config", Nova1Config(vocab_size=tokenizer.vocab_size))
config.device = "cuda" if torch.cuda.is_available() else "cpu"
config.dtype = "bfloat16"
model = Nova1HRM(config)
state_dict = checkpoint["model_state"] if "model_state" in checkpoint else checkpoint["model_state_dict"]
model.load_state_dict(state_dict)
model.to(config.device)
model.eval()
generator = Nova1Generator(model, tokenizer, config)
print(f"\n=======================================================")
print(f"🏛️ Running Humanity's Last Exam (HLE) Evaluation Benchmark")
print(f"Model: Nova 1.0 ({sum(p.numel() for p in model.parameters()) / 1e6:.1f}M params) | Device: {config.device.upper()}")
print(f"=======================================================\n", flush=True)
passed_count = 0
total_count = len(HLE_BENCHMARK_QUESTIONS)
results_table = []
for idx, test in enumerate(HLE_BENCHMARK_QUESTIONS, 1):
prompt = test["prompt"]
category = test["category"]
expected = test["expected_keywords"]
start_time = time.time()
output = generator.generate(
prompt=prompt,
max_new_tokens=90,
temperature=0.4,
top_k=30,
repetition_penalty=1.05
)
latency = time.time() - start_time
# Check if output contains expected keywords
output_lower = output.lower()
matched = any(kw.lower() in output_lower for kw in expected)
if matched:
passed_count += 1
status = "PASSED ✅"
else:
status = "FAILED ❌"
results_table.append({
"id": idx,
"category": category,
"status": status,
"latency": latency,
"output": output
})
print(f"[{idx}/{total_count}] Category: {category} -> {status} ({latency:.2f}s)", flush=True)
acc_pct = (passed_count / total_count) * 100.0
print(f"\n=======================================================")
print(f"📊 HUMANITY'S LAST EXAM (HLE) BENCHMARK SCORECARD")
print(f"=======================================================")
print(f"Total Questions Evaluated: {total_count}")
print(f"Questions Passed: {passed_count}")
print(f"Final HLE Benchmark Score: {acc_pct:.1f}%")
print(f"=======================================================\n", flush=True)
for r in results_table:
print(f"\n--- [Question {r['id']}] {r['category']} ({r['status']}) ---")
print(f"{r['output'].strip()}")
print("-" * 50)
if __name__ == "__main__":
evaluate_hle_benchmark()
|