{
  "examples": 160,
  "seconds": 2.227508544921875,
  "decoding": "raw greedy, no repair, no teacher fallback",
  "checkpoint_sha256": "9489b5cde69c11f64b3e7031182e8248aa83790f1ad4aeefbf99a4e5e152e692",
  "sql_metric": "Compile against the supplied schema, then compare results on two generated SQLite fixtures after dialect adaptation; not native MySQL/PostgreSQL execution.",
  "metrics": {
    "json_valid": {
      "correct": 160,
      "total": 160,
      "rate": 1.0
    },
    "schema_valid": {
      "correct": 159,
      "total": 160,
      "rate": 0.99375
    },
    "action_correct": {
      "correct": 159,
      "total": 160,
      "rate": 0.99375
    },
    "tool_correct": {
      "correct": 146,
      "total": 152,
      "rate": 0.9605263157894737
    },
    "arguments_exact": {
      "correct": 124,
      "total": 152,
      "rate": 0.8157894736842105
    },
    "sql_equivalent": {
      "correct": 94,
      "total": 112,
      "rate": 0.8392857142857143
    },
    "success": {
      "correct": 132,
      "total": 160,
      "rate": 0.825
    }
  },
  "groups": {
    "language:noisy_en": {
      "json_valid": {
        "correct": 40,
        "total": 40,
        "rate": 1.0
      },
      "schema_valid": {
        "correct": 40,
        "total": 40,
        "rate": 1.0
      },
      "action_correct": {
        "correct": 39,
        "total": 40,
        "rate": 0.975
      },
      "tool_correct": {
        "correct": 35,
        "total": 38,
        "rate": 0.9210526315789473
      },
      "arguments_exact": {
        "correct": 31,
        "total": 38,
        "rate": 0.8157894736842105
      },
      "sql_equivalent": {
        "correct": 23,
        "total": 28,
        "rate": 0.8214285714285714
      },
      "success": {
        "correct": 33,
        "total": 40,
        "rate": 0.825
      }
    },
    "backend:supabase": {
      "json_valid": {
        "correct": 80,
        "total": 80,
        "rate": 1.0
      },
      "schema_valid": {
        "correct": 80,
        "total": 80,
        "rate": 1.0
      },
      "action_correct": {
        "correct": 79,
        "total": 80,
        "rate": 0.9875
      },
      "tool_correct": {
        "correct": 73,
        "total": 76,
        "rate": 0.9605263157894737
      },
      "arguments_exact": {
        "correct": 59,
        "total": 76,
        "rate": 0.7763157894736842
      },
      "sql_equivalent": {
        "correct": 46,
        "total": 56,
        "rate": 0.8214285714285714
      },
      "success": {
        "correct": 63,
        "total": 80,
        "rate": 0.7875
      }
    },
    "operation:all": {
      "json_valid": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "schema_valid": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "action_correct": {
        "correct": 7,
        "total": 8,
        "rate": 0.875
      },
      "tool_correct": {
        "correct": 7,
        "total": 8,
        "rate": 0.875
      },
      "arguments_exact": {
        "correct": 5,
        "total": 8,
        "rate": 0.625
      },
      "sql_equivalent": {
        "correct": 5,
        "total": 8,
        "rate": 0.625
      },
      "success": {
        "correct": 5,
        "total": 8,
        "rate": 0.625
      }
    },
    "operation:null": {
      "json_valid": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "schema_valid": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "action_correct": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "tool_correct": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "arguments_exact": {
        "correct": 2,
        "total": 8,
        "rate": 0.25
      },
      "sql_equivalent": {
        "correct": 2,
        "total": 8,
        "rate": 0.25
      },
      "success": {
        "correct": 2,
        "total": 8,
        "rate": 0.25
      }
    },
    "operation:gte": {
      "json_valid": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "schema_valid": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "action_correct": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "tool_correct": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "arguments_exact": {
        "correct": 7,
        "total": 8,
        "rate": 0.875
      },
      "sql_equivalent": {
        "correct": 7,
        "total": 8,
        "rate": 0.875
      },
      "success": {
        "correct": 7,
        "total": 8,
        "rate": 0.875
      }
    },
    "language:en": {
      "json_valid": {
        "correct": 40,
        "total": 40,
        "rate": 1.0
      },
      "schema_valid": {
        "correct": 39,
        "total": 40,
        "rate": 0.975
      },
      "action_correct": {
        "correct": 40,
        "total": 40,
        "rate": 1.0
      },
      "tool_correct": {
        "correct": 37,
        "total": 38,
        "rate": 0.9736842105263158
      },
      "arguments_exact": {
        "correct": 25,
        "total": 38,
        "rate": 0.6578947368421053
      },
      "sql_equivalent": {
        "correct": 20,
        "total": 28,
        "rate": 0.7142857142857143
      },
      "success": {
        "correct": 27,
        "total": 40,
        "rate": 0.675
      }
    },
    "operation:project": {
      "json_valid": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "schema_valid": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "action_correct": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "tool_correct": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "arguments_exact": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "sql_equivalent": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "success": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      }
    },
    "language:hinglish": {
      "json_valid": {
        "correct": 40,
        "total": 40,
        "rate": 1.0
      },
      "schema_valid": {
        "correct": 40,
        "total": 40,
        "rate": 1.0
      },
      "action_correct": {
        "correct": 40,
        "total": 40,
        "rate": 1.0
      },
      "tool_correct": {
        "correct": 38,
        "total": 38,
        "rate": 1.0
      },
      "arguments_exact": {
        "correct": 34,
        "total": 38,
        "rate": 0.8947368421052632
      },
      "sql_equivalent": {
        "correct": 25,
        "total": 28,
        "rate": 0.8928571428571429
      },
      "success": {
        "correct": 36,
        "total": 40,
        "rate": 0.9
      }
    },
    "operation:sum": {
      "json_valid": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "schema_valid": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "action_correct": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "tool_correct": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "arguments_exact": {
        "correct": 6,
        "total": 8,
        "rate": 0.75
      },
      "sql_equivalent": {
        "correct": 6,
        "total": 8,
        "rate": 0.75
      },
      "success": {
        "correct": 6,
        "total": 8,
        "rate": 0.75
      }
    },
    "operation:distinct": {
      "json_valid": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "schema_valid": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "action_correct": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "tool_correct": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "arguments_exact": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "sql_equivalent": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "success": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      }
    },
    "operation:project_eq": {
      "json_valid": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "schema_valid": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "action_correct": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "tool_correct": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "arguments_exact": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "sql_equivalent": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "success": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      }
    },
    "operation:lte": {
      "json_valid": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "schema_valid": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "action_correct": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "tool_correct": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "arguments_exact": {
        "correct": 4,
        "total": 8,
        "rate": 0.5
      },
      "sql_equivalent": {
        "correct": 4,
        "total": 8,
        "rate": 0.5
      },
      "success": {
        "correct": 4,
        "total": 8,
        "rate": 0.5
      }
    },
    "language:hi": {
      "json_valid": {
        "correct": 40,
        "total": 40,
        "rate": 1.0
      },
      "schema_valid": {
        "correct": 40,
        "total": 40,
        "rate": 1.0
      },
      "action_correct": {
        "correct": 40,
        "total": 40,
        "rate": 1.0
      },
      "tool_correct": {
        "correct": 36,
        "total": 38,
        "rate": 0.9473684210526315
      },
      "arguments_exact": {
        "correct": 34,
        "total": 38,
        "rate": 0.8947368421052632
      },
      "sql_equivalent": {
        "correct": 26,
        "total": 28,
        "rate": 0.9285714285714286
      },
      "success": {
        "correct": 36,
        "total": 40,
        "rate": 0.9
      }
    },
    "operation:list_tables": {
      "json_valid": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "schema_valid": {
        "correct": 7,
        "total": 8,
        "rate": 0.875
      },
      "action_correct": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "tool_correct": {
        "correct": 3,
        "total": 8,
        "rate": 0.375
      },
      "arguments_exact": {
        "correct": 1,
        "total": 8,
        "rate": 0.125
      },
      "sql_equivalent": {
        "correct": 0,
        "total": 0,
        "rate": 0.0
      },
      "success": {
        "correct": 1,
        "total": 8,
        "rate": 0.125
      }
    },
    "operation:ambiguous": {
      "json_valid": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "schema_valid": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "action_correct": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "tool_correct": {
        "correct": 0,
        "total": 0,
        "rate": 0.0
      },
      "arguments_exact": {
        "correct": 0,
        "total": 0,
        "rate": 0.0
      },
      "sql_equivalent": {
        "correct": 0,
        "total": 0,
        "rate": 0.0
      },
      "success": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      }
    },
    "operation:lt": {
      "json_valid": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "schema_valid": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "action_correct": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "tool_correct": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "arguments_exact": {
        "correct": 7,
        "total": 8,
        "rate": 0.875
      },
      "sql_equivalent": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "success": {
        "correct": 7,
        "total": 8,
        "rate": 0.875
      }
    },
    "operation:gt": {
      "json_valid": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "schema_valid": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "action_correct": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "tool_correct": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "arguments_exact": {
        "correct": 7,
        "total": 8,
        "rate": 0.875
      },
      "sql_equivalent": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "success": {
        "correct": 7,
        "total": 8,
        "rate": 0.875
      }
    },
    "operation:avg": {
      "json_valid": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "schema_valid": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "action_correct": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "tool_correct": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "arguments_exact": {
        "correct": 7,
        "total": 8,
        "rate": 0.875
      },
      "sql_equivalent": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "success": {
        "correct": 7,
        "total": 8,
        "rate": 0.875
      }
    },
    "operation:count": {
      "json_valid": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "schema_valid": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "action_correct": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "tool_correct": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "arguments_exact": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "sql_equivalent": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "success": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      }
    },
    "operation:group_count": {
      "json_valid": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "schema_valid": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "action_correct": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "tool_correct": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "arguments_exact": {
        "correct": 6,
        "total": 8,
        "rate": 0.75
      },
      "sql_equivalent": {
        "correct": 6,
        "total": 8,
        "rate": 0.75
      },
      "success": {
        "correct": 6,
        "total": 8,
        "rate": 0.75
      }
    },
    "operation:top": {
      "json_valid": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "schema_valid": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "action_correct": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "tool_correct": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "arguments_exact": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "sql_equivalent": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "success": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      }
    },
    "backend:mysql": {
      "json_valid": {
        "correct": 80,
        "total": 80,
        "rate": 1.0
      },
      "schema_valid": {
        "correct": 79,
        "total": 80,
        "rate": 0.9875
      },
      "action_correct": {
        "correct": 80,
        "total": 80,
        "rate": 1.0
      },
      "tool_correct": {
        "correct": 73,
        "total": 76,
        "rate": 0.9605263157894737
      },
      "arguments_exact": {
        "correct": 65,
        "total": 76,
        "rate": 0.8552631578947368
      },
      "sql_equivalent": {
        "correct": 48,
        "total": 56,
        "rate": 0.8571428571428571
      },
      "success": {
        "correct": 69,
        "total": 80,
        "rate": 0.8625
      }
    },
    "operation:weather": {
      "json_valid": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "schema_valid": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "action_correct": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "tool_correct": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "arguments_exact": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "sql_equivalent": {
        "correct": 0,
        "total": 0,
        "rate": 0.0
      },
      "success": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      }
    },
    "operation:search": {
      "json_valid": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "schema_valid": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "action_correct": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "tool_correct": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "arguments_exact": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "sql_equivalent": {
        "correct": 0,
        "total": 0,
        "rate": 0.0
      },
      "success": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      }
    },
    "operation:ticket": {
      "json_valid": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "schema_valid": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "action_correct": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "tool_correct": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "arguments_exact": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "sql_equivalent": {
        "correct": 0,
        "total": 0,
        "rate": 0.0
      },
      "success": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      }
    },
    "operation:read_file": {
      "json_valid": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "schema_valid": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "action_correct": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "tool_correct": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "arguments_exact": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      },
      "sql_equivalent": {
        "correct": 0,
        "total": 0,
        "rate": 0.0
      },
      "success": {
        "correct": 8,
        "total": 8,
        "rate": 1.0
      }
    }
  }
}