# Chat-aware HumanEval for LLaDA-8B-Instruct. Identical to lm-eval's stock
# humaneval.yaml EXCEPT the two things a chat model breaks (see utils.py):
#   until:       <|eot_id|> instead of Python-keyword stops, which decapitate a
#                fenced answer the moment the model writes "\ndef".
#   filter_fn:   build_predictions_chat, which pulls the program out of the
#                markdown fence instead of concatenating prose onto the signature.
# Filter NAME stays "create_test" so analysis/lmeval_to_items.py keeps selecting it.
task: humaneval_instruct
dataset_path: openai/openai_humaneval
unsafe_code: true
output_type: generate_until
test_split: test
doc_to_text: "{{prompt}}"
doc_to_target: "{{test}}\ncheck({{entry_point}})"
metric_list:
  - metric: !function utils.pass_at_k
    aggregation: mean
    higher_is_better: true
    k: [1]
generation_kwargs:
  until:
    - "<|eot_id|>"
  max_gen_toks: 1024
  do_sample: false
repeats: 1
num_fewshot: 0
filter_list:
  - name: "create_test"
    filter:
      - function: "custom"
        filter_fn: !function utils.build_predictions_chat
metadata:
  version: 1.0