| { |
| "results": { |
| "humaneval": { |
| "name": "humaneval", |
| "alias": "humaneval", |
| "sample_len": 164, |
| "pass@1,create_test": 0.0, |
| "pass@1_stderr,create_test": 0.0 |
| } |
| }, |
| "group_subtasks": {}, |
| "configs": { |
| "humaneval": { |
| "task": "humaneval", |
| "dataset_path": "openai/openai_humaneval", |
| "test_split": "test", |
| "doc_to_text": "{{prompt}}", |
| "doc_to_target": "{{test}}\ncheck({{entry_point}})", |
| "unsafe_code": true, |
| "description": "", |
| "target_delimiter": " ", |
| "fewshot_delimiter": "\n\n", |
| "fewshot_config": { |
| "sampler": "default", |
| "split": null, |
| "process_docs": null, |
| "fewshot_indices": null, |
| "samples": null, |
| "doc_to_text": "{{prompt}}", |
| "doc_to_choice": null, |
| "doc_to_target": "{{test}}\ncheck({{entry_point}})", |
| "gen_prefix": null, |
| "fewshot_delimiter": "\n\n", |
| "target_delimiter": " " |
| }, |
| "num_fewshot": 0, |
| "metric_list": [ |
| { |
| "metric": "def pass_at_k(references: list[str], predictions: list[list[str]], k: list[int] = None):\n global compute_\n assert k is not None\n if isinstance(k, int):\n k = [k]\n res = compute_.compute(\n references=references,\n predictions=predictions,\n k=k,\n )\n return res[0]\n", |
| "aggregation": "mean", |
| "higher_is_better": true, |
| "k": [ |
| 1 |
| ] |
| } |
| ], |
| "output_type": "generate_until", |
| "generation_kwargs": { |
| "until": [ |
| "\nclass", |
| "\ndef", |
| "\n#", |
| "\nif", |
| "\nprint" |
| ], |
| "max_gen_toks": 1024, |
| "do_sample": false |
| }, |
| "repeats": 1, |
| "filter_list": [ |
| { |
| "name": "create_test", |
| "filter": [ |
| { |
| "function": "custom", |
| "filter_fn": "<function build_predictions at 0x7835491a2f20>" |
| } |
| ] |
| } |
| ], |
| "should_decontaminate": false, |
| "metadata": { |
| "version": 1.0, |
| "model": "dsv4-hybrid", |
| "base_url": "http://localhost:PORT/v1/chat/completions", |
| "num_concurrent": 8, |
| "max_retries": 3, |
| "tokenized_requests": false, |
| "tokenizer": "<model_path>", |
| "tokenizer_backend": "huggingface", |
| "config_source": "<lm_eval>/tasks/humaneval/humaneval.yaml" |
| } |
| } |
| }, |
| "versions": { |
| "humaneval": 1.0 |
| }, |
| "n-shot": { |
| "humaneval": 0 |
| }, |
| "higher_is_better": { |
| "humaneval": { |
| "pass_at_k": true, |
| "pass@1": true |
| } |
| }, |
| "n-samples": { |
| "humaneval": { |
| "original": 164, |
| "effective": 164 |
| } |
| }, |
| "config": { |
| "model": "local-chat-completions", |
| "model_args": { |
| "model": "dsv4-hybrid", |
| "base_url": "http://localhost:PORT/v1/chat/completions", |
| "num_concurrent": 8, |
| "max_retries": 3, |
| "tokenized_requests": false, |
| "tokenizer": "<model_path>", |
| "tokenizer_backend": "huggingface" |
| }, |
| "batch_size": 1, |
| "batch_sizes": [], |
| "device": "cuda:0", |
| "use_cache": null, |
| "limit": 164.0, |
| "bootstrap_iters": 100000, |
| "gen_kwargs": {}, |
| "random_seed": 0, |
| "numpy_seed": 1234, |
| "torch_seed": 1234, |
| "fewshot_seed": 1234 |
| }, |
| "git_hash": null, |
| "date": 1786210286.7190146, |
| "transformers_version": "5.14.1", |
| "lm_eval_version": "0.4.12", |
| "upper_git_hash": null, |
| "tokenizer_pad_token": [ |
| "<｜end▁of▁sentence｜>", |
| "1" |
| ], |
| "tokenizer_eos_token": [ |
| "<｜end▁of▁sentence｜>", |
| "1" |
| ], |
| "tokenizer_bos_token": [ |
| "<｜begin▁of▁sentence｜>", |
| "0" |
| ], |
| "eot_token_id": 1, |
| "max_length": 2047, |
| "task_hashes": {}, |
| "model_source": "local-chat-completions", |
| "model_name": "dsv4-hybrid", |
| "model_name_sanitized": "dsv4-hybrid", |
| "system_instruction": null, |
| "system_instruction_sha": null, |
| "fewshot_as_multiturn": true, |
| "chat_template": null, |
| "chat_template_sha": null, |
| "total_evaluation_time_seconds": "34.3056967179873", |
| "hardware_note": "2x NVIDIA RTX PRO 6000 Blackwell (96GB, SM120), TP=2. Detailed host environment omitted." |
| } |
