File size: 713 Bytes
c95c7b0 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 | from lm_eval import tasks
from pytablewriter import MarkdownTableWriter
writer = MarkdownTableWriter()
writer.headers = ["Task Name", "Train", "Val", "Test", "Val/Test Docs", "Metrics"]
values = []
def chk(tf):
if tf:
return "✓"
else:
return " "
for tname, Task in tasks.TASK_REGISTRY.items():
task = Task()
v = [
tname,
chk(task.has_training_docs()),
chk(task.has_validation_docs()),
chk(task.has_test_docs()),
len(list(task.test_docs() if task.has_test_docs() else task.validation_docs())),
", ".join(task.aggregation().keys()),
]
print(v)
values.append(v)
writer.value_matrix = values
print(writer.dumps())
|