| { | |
| "model_families": { | |
| "qwen3": { | |
| "tokenizer": "Qwen/Qwen3-0.6B", | |
| "training": { | |
| "train": { | |
| "rows": 13654, | |
| "max_prompt_tokens": 520, | |
| "max_total_tokens": 604, | |
| "max_response_tokens": 113 | |
| }, | |
| "eval": { | |
| "rows": 3414, | |
| "max_prompt_tokens": 515, | |
| "max_total_tokens": 608, | |
| "max_response_tokens": 109 | |
| } | |
| }, | |
| "inference": { | |
| "cypherbench": { | |
| "selector": { | |
| "rows": 35281, | |
| "max_prompt_tokens": 281, | |
| "required_context": 297 | |
| }, | |
| "generator_worst_case": { | |
| "rows": 2348, | |
| "max_prompt_tokens": 875, | |
| "required_context": 1131 | |
| }, | |
| "generator_gold_subschema": { | |
| "rows": 2348, | |
| "max_prompt_tokens": 498 | |
| } | |
| }, | |
| "mind_the_query": { | |
| "selector": { | |
| "rows": 153805, | |
| "max_prompt_tokens": 298, | |
| "required_context": 314 | |
| }, | |
| "generator_worst_case": { | |
| "rows": 7059, | |
| "max_prompt_tokens": 1859, | |
| "required_context": 2115 | |
| }, | |
| "generator_gold_subschema": { | |
| "rows": 7045, | |
| "max_prompt_tokens": 753 | |
| } | |
| }, | |
| "neo4j_text2cypher": { | |
| "selector": { | |
| "rows": 45281, | |
| "max_prompt_tokens": 371, | |
| "required_context": 387 | |
| }, | |
| "generator_worst_case": { | |
| "rows": 2470, | |
| "max_prompt_tokens": 2129, | |
| "required_context": 2385 | |
| }, | |
| "generator_gold_subschema": { | |
| "rows": 2394, | |
| "max_prompt_tokens": 1270 | |
| } | |
| } | |
| }, | |
| "summary": { | |
| "required_cutoff_len": 608, | |
| "required_inference_context": 2385 | |
| } | |
| }, | |
| "qwen2.5_coder": { | |
| "tokenizer": "Qwen/Qwen2.5-Coder-3B-Instruct", | |
| "training": { | |
| "train": { | |
| "rows": 13654, | |
| "max_prompt_tokens": 520, | |
| "max_total_tokens": 604, | |
| "max_response_tokens": 113 | |
| }, | |
| "eval": { | |
| "rows": 3414, | |
| "max_prompt_tokens": 515, | |
| "max_total_tokens": 608, | |
| "max_response_tokens": 109 | |
| } | |
| }, | |
| "inference": { | |
| "cypherbench": { | |
| "selector": { | |
| "rows": 35281, | |
| "max_prompt_tokens": 281, | |
| "required_context": 297 | |
| }, | |
| "generator_worst_case": { | |
| "rows": 2348, | |
| "max_prompt_tokens": 875, | |
| "required_context": 1131 | |
| }, | |
| "generator_gold_subschema": { | |
| "rows": 2348, | |
| "max_prompt_tokens": 498 | |
| } | |
| }, | |
| "mind_the_query": { | |
| "selector": { | |
| "rows": 153805, | |
| "max_prompt_tokens": 298, | |
| "required_context": 314 | |
| }, | |
| "generator_worst_case": { | |
| "rows": 7059, | |
| "max_prompt_tokens": 1859, | |
| "required_context": 2115 | |
| }, | |
| "generator_gold_subschema": { | |
| "rows": 7045, | |
| "max_prompt_tokens": 753 | |
| } | |
| }, | |
| "neo4j_text2cypher": { | |
| "selector": { | |
| "rows": 45281, | |
| "max_prompt_tokens": 371, | |
| "required_context": 387 | |
| }, | |
| "generator_worst_case": { | |
| "rows": 2470, | |
| "max_prompt_tokens": 2129, | |
| "required_context": 2385 | |
| }, | |
| "generator_gold_subschema": { | |
| "rows": 2394, | |
| "max_prompt_tokens": 1270 | |
| } | |
| } | |
| }, | |
| "summary": { | |
| "required_cutoff_len": 608, | |
| "required_inference_context": 2385 | |
| } | |
| }, | |
| "llama3": { | |
| "tokenizer": "meta-llama/Llama-3.2-1B-Instruct", | |
| "training": { | |
| "train": { | |
| "rows": 13654, | |
| "max_prompt_tokens": 521, | |
| "max_total_tokens": 605, | |
| "max_response_tokens": 110 | |
| }, | |
| "eval": { | |
| "rows": 3414, | |
| "max_prompt_tokens": 516, | |
| "max_total_tokens": 609, | |
| "max_response_tokens": 108 | |
| } | |
| }, | |
| "inference": { | |
| "cypherbench": { | |
| "selector": { | |
| "rows": 35281, | |
| "max_prompt_tokens": 282, | |
| "required_context": 298 | |
| }, | |
| "generator_worst_case": { | |
| "rows": 2348, | |
| "max_prompt_tokens": 877, | |
| "required_context": 1133 | |
| }, | |
| "generator_gold_subschema": { | |
| "rows": 2348, | |
| "max_prompt_tokens": 500 | |
| } | |
| }, | |
| "mind_the_query": { | |
| "selector": { | |
| "rows": 153805, | |
| "max_prompt_tokens": 300, | |
| "required_context": 316 | |
| }, | |
| "generator_worst_case": { | |
| "rows": 7059, | |
| "max_prompt_tokens": 1861, | |
| "required_context": 2117 | |
| }, | |
| "generator_gold_subschema": { | |
| "rows": 7045, | |
| "max_prompt_tokens": 755 | |
| } | |
| }, | |
| "neo4j_text2cypher": { | |
| "selector": { | |
| "rows": 45281, | |
| "max_prompt_tokens": 367, | |
| "required_context": 383 | |
| }, | |
| "generator_worst_case": { | |
| "rows": 2470, | |
| "max_prompt_tokens": 2131, | |
| "required_context": 2387 | |
| }, | |
| "generator_gold_subschema": { | |
| "rows": 2394, | |
| "max_prompt_tokens": 1270 | |
| } | |
| } | |
| }, | |
| "summary": { | |
| "required_cutoff_len": 609, | |
| "required_inference_context": 2387 | |
| } | |
| } | |
| } | |
| } | |