File size: 1,876 Bytes
c95c7b0 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 | import os
import random
import pytest
import lm_eval
import lm_eval.tasks as tasks
import lm_eval.api.model as model
import lm_eval.models as models
import lm_eval.evaluator as evaluator
from lm_eval.api.utils import DEFAULT_SEED, set_seed
# TODO: More fine grained unit tests rather than this big honking integration
# test once we break evaluator into smaller, more manageable pieces
def _ll_fn(requests):
for ctx, cont in requests:
if len(ctx) == 0:
continue
# Check text-target-separator default spacing convention.
# ctx + (' ' + cont)
assert ctx[-1] != " "
assert cont[0] == " "
res = []
random.seed(DEFAULT_SEED)
for _ in requests:
res.append((-random.random(), False))
return res
def _ll_perp_fn(requests):
for (string,) in requests:
assert isinstance(string, str)
res = []
random.seed(DEFAULT_SEED)
for _ in requests:
res.append(-random.random())
return res
@pytest.mark.parametrize("task_name", lm_eval.list_tasks())
def test_evaluator(task_name):
set_seed()
template_names = tasks.list_templates(task_name)
# Only choose 1 promptsource template.
template_name = template_names[0] if template_names else None
task = tasks.get_task(task_name, template_name)
os.system("rm test_cache.db")
lm = model.CachingLM(models.get_model("dummy"), "test_cache.db")
lm.loglikelihood = _ll_fn
lm.loglikelihood_rolling = _ll_perp_fn
limit = 5
e1 = evaluator.evaluate(
model=lm,
tasks=[task],
num_fewshot=0,
bootstrap_iters=10,
limit=limit,
)["results"]
e2 = evaluator.evaluate(
model=lm,
tasks=[task],
num_fewshot=0,
bootstrap_iters=10,
limit=limit,
)["results"]
# Check that caching is working
assert e1 == e2
|