File size: 1,127 Bytes
76d1db9 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 | # Chat-aware HumanEval for LLaDA-8B-Instruct. Identical to lm-eval's stock
# humaneval.yaml EXCEPT the two things a chat model breaks (see utils.py):
# until: <|eot_id|> instead of Python-keyword stops, which decapitate a
# fenced answer the moment the model writes "\ndef".
# filter_fn: build_predictions_chat, which pulls the program out of the
# markdown fence instead of concatenating prose onto the signature.
# Filter NAME stays "create_test" so analysis/lmeval_to_items.py keeps selecting it.
task: humaneval_instruct
dataset_path: openai/openai_humaneval
unsafe_code: true
output_type: generate_until
test_split: test
doc_to_text: "{{prompt}}"
doc_to_target: "{{test}}\ncheck({{entry_point}})"
metric_list:
- metric: !function utils.pass_at_k
aggregation: mean
higher_is_better: true
k: [1]
generation_kwargs:
until:
- "<|eot_id|>"
max_gen_toks: 1024
do_sample: false
repeats: 1
num_fewshot: 0
filter_list:
- name: "create_test"
filter:
- function: "custom"
filter_fn: !function utils.build_predictions_chat
metadata:
version: 1.0
|