tahamajs/RR / Codes /tests /test_benchmark_datasets.py
tahamajs's picture
download
raw
2.35 kB
"""
Unit tests for BenchmarkDatasetRegistry and load_benchmark_dataset.
Tests offline synthetic fallback, registered benchmarks, and data formats.
"""
import pytest
from framework.data.benchmark_datasets import (
BenchmarkDatasetInfo,
BenchmarkDatasetRegistry,
load_benchmark_dataset,
)
def test_registry_contains_standard_benchmarks():
benchmarks = BenchmarkDatasetRegistry.list_benchmarks()
required = [
"wikitext2",
"wikitext103",
"c4",
"ptb",
"pg19",
"gsm8k",
"humaneval",
"mmlu",
"pile",
"arc",
"arc_challenge",
"hellaswag",
"lambada",
"glue_sst2",
"glue_mrpc",
"glue_qqp",
"glue_stsb",
"glue_mnli",
"glue_qnli",
"glue_rte",
"glue_wnli",
"glue_cola",
]
for req in required:
assert req in benchmarks, f"Missing benchmark '{req}' in registry."
def test_synthetic_loading_all_benchmarks():
benchmarks = [
"wikitext2",
"c4",
"ptb",
"pg19",
"gsm8k",
"humaneval",
"mmlu",
"pile",
"arc",
"arc_challenge",
"hellaswag",
"lambada",
"glue/sst2",
"glue/mrpc",
]
for b in benchmarks:
samples = load_benchmark_dataset(
dataset_name=b,
use_synthetic=True,
max_samples=10,
return_dataloader=False,
seed=123,
)
assert isinstance(samples, list)
assert len(samples) == 10, f"Expected 10 samples for {b}, got {len(samples)}"
assert all(isinstance(s, dict) for s in samples)
def test_custom_registration():
info = BenchmarkDatasetInfo(
name="custom_math",
aliases=["my_math"],
category="reasoning",
default_hf_path="custom/math",
synthetic_generator=lambda n, s: [{"text": f"Math prob #{i}", "label": i} for i in range(n)],
)
BenchmarkDatasetRegistry.register(info)
assert BenchmarkDatasetRegistry.is_registered("custom_math")
assert BenchmarkDatasetRegistry.is_registered("my_math")
res = load_benchmark_dataset("my_math", use_synthetic=True, max_samples=4, return_dataloader=False)
assert len(res) == 4
assert res[0]["text"] == "Math prob #0"

Xet Storage Details

Size:
2.35 kB
·
Xet hash:
93cb312c4a18dd42a0f35c63196b13a1d94bee7ea54882a55773d5228fde44ef

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.