| { |
| "seconds": 1.1846249103546143, |
| "cases": 112, |
| "gold_executes": 107, |
| "prediction_equivalent": 91, |
| "engine_versions": { |
| "mysql": "8.0.46", |
| "postgresql": "16.15" |
| }, |
| "results": [ |
| { |
| "id": "manual_adff4eabe2639c0c128c_en", |
| "backend": "mysql", |
| "operation": "all", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_adff4eabe2639c0c128c_noisy_en", |
| "backend": "mysql", |
| "operation": "all", |
| "gold_executes": true, |
| "prediction_equivalent": false |
| }, |
| { |
| "id": "manual_adff4eabe2639c0c128c_hi", |
| "backend": "mysql", |
| "operation": "all", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_adff4eabe2639c0c128c_hinglish", |
| "backend": "mysql", |
| "operation": "all", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_efee65268e6dc2bb069e_en", |
| "backend": "supabase", |
| "operation": "project", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_efee65268e6dc2bb069e_noisy_en", |
| "backend": "supabase", |
| "operation": "project", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_efee65268e6dc2bb069e_hi", |
| "backend": "supabase", |
| "operation": "project", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_efee65268e6dc2bb069e_hinglish", |
| "backend": "supabase", |
| "operation": "project", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_2c31add7d7b45843dec1_en", |
| "backend": "mysql", |
| "operation": "project_eq", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_2c31add7d7b45843dec1_noisy_en", |
| "backend": "mysql", |
| "operation": "project_eq", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_2c31add7d7b45843dec1_hi", |
| "backend": "mysql", |
| "operation": "project_eq", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_2c31add7d7b45843dec1_hinglish", |
| "backend": "mysql", |
| "operation": "project_eq", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_82c104614df1ce6e2762_en", |
| "backend": "supabase", |
| "operation": "gt", |
| "gold_executes": false, |
| "prediction_equivalent": false, |
| "error": "Wrong non-SQL arguments" |
| }, |
| { |
| "id": "manual_82c104614df1ce6e2762_noisy_en", |
| "backend": "supabase", |
| "operation": "gt", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_82c104614df1ce6e2762_hi", |
| "backend": "supabase", |
| "operation": "gt", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_82c104614df1ce6e2762_hinglish", |
| "backend": "supabase", |
| "operation": "gt", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_35406fa024e66bcb5dd7_en", |
| "backend": "supabase", |
| "operation": "lt", |
| "gold_executes": false, |
| "prediction_equivalent": false, |
| "error": "Wrong non-SQL arguments" |
| }, |
| { |
| "id": "manual_35406fa024e66bcb5dd7_noisy_en", |
| "backend": "supabase", |
| "operation": "lt", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_35406fa024e66bcb5dd7_hi", |
| "backend": "supabase", |
| "operation": "lt", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_35406fa024e66bcb5dd7_hinglish", |
| "backend": "supabase", |
| "operation": "lt", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_0b8db53d711baed4f4ed_en", |
| "backend": "supabase", |
| "operation": "gte", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_0b8db53d711baed4f4ed_noisy_en", |
| "backend": "supabase", |
| "operation": "gte", |
| "gold_executes": true, |
| "prediction_equivalent": false |
| }, |
| { |
| "id": "manual_0b8db53d711baed4f4ed_hi", |
| "backend": "supabase", |
| "operation": "gte", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_0b8db53d711baed4f4ed_hinglish", |
| "backend": "supabase", |
| "operation": "gte", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_b02f421828127e2eb96b_en", |
| "backend": "mysql", |
| "operation": "lte", |
| "gold_executes": true, |
| "prediction_equivalent": false |
| }, |
| { |
| "id": "manual_b02f421828127e2eb96b_noisy_en", |
| "backend": "mysql", |
| "operation": "lte", |
| "gold_executes": true, |
| "prediction_equivalent": false |
| }, |
| { |
| "id": "manual_b02f421828127e2eb96b_hi", |
| "backend": "mysql", |
| "operation": "lte", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_b02f421828127e2eb96b_hinglish", |
| "backend": "mysql", |
| "operation": "lte", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_2e2ac3c5dcf95429daaa_en", |
| "backend": "supabase", |
| "operation": "null", |
| "gold_executes": true, |
| "prediction_equivalent": false |
| }, |
| { |
| "id": "manual_2e2ac3c5dcf95429daaa_noisy_en", |
| "backend": "supabase", |
| "operation": "null", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_2e2ac3c5dcf95429daaa_hi", |
| "backend": "supabase", |
| "operation": "null", |
| "gold_executes": true, |
| "prediction_equivalent": false |
| }, |
| { |
| "id": "manual_2e2ac3c5dcf95429daaa_hinglish", |
| "backend": "supabase", |
| "operation": "null", |
| "gold_executes": true, |
| "prediction_equivalent": false |
| }, |
| { |
| "id": "manual_40464b00088c9159e3d0_en", |
| "backend": "supabase", |
| "operation": "count", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_40464b00088c9159e3d0_noisy_en", |
| "backend": "supabase", |
| "operation": "count", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_40464b00088c9159e3d0_hi", |
| "backend": "supabase", |
| "operation": "count", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_40464b00088c9159e3d0_hinglish", |
| "backend": "supabase", |
| "operation": "count", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_d735c1ac1979ff1e8620_en", |
| "backend": "supabase", |
| "operation": "sum", |
| "gold_executes": true, |
| "prediction_equivalent": false |
| }, |
| { |
| "id": "manual_d735c1ac1979ff1e8620_noisy_en", |
| "backend": "supabase", |
| "operation": "sum", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_d735c1ac1979ff1e8620_hi", |
| "backend": "supabase", |
| "operation": "sum", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_d735c1ac1979ff1e8620_hinglish", |
| "backend": "supabase", |
| "operation": "sum", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_8efcf517aee64807a8e5_en", |
| "backend": "mysql", |
| "operation": "avg", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_8efcf517aee64807a8e5_noisy_en", |
| "backend": "mysql", |
| "operation": "avg", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_8efcf517aee64807a8e5_hi", |
| "backend": "mysql", |
| "operation": "avg", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_8efcf517aee64807a8e5_hinglish", |
| "backend": "mysql", |
| "operation": "avg", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_cab55a7429427fbc5093_en", |
| "backend": "mysql", |
| "operation": "distinct", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_cab55a7429427fbc5093_noisy_en", |
| "backend": "mysql", |
| "operation": "distinct", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_cab55a7429427fbc5093_hi", |
| "backend": "mysql", |
| "operation": "distinct", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_cab55a7429427fbc5093_hinglish", |
| "backend": "mysql", |
| "operation": "distinct", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_b9c176c07809f42564a0_en", |
| "backend": "mysql", |
| "operation": "top", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_b9c176c07809f42564a0_noisy_en", |
| "backend": "mysql", |
| "operation": "top", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_b9c176c07809f42564a0_hi", |
| "backend": "mysql", |
| "operation": "top", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_b9c176c07809f42564a0_hinglish", |
| "backend": "mysql", |
| "operation": "top", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_475287613764159ee5eb_en", |
| "backend": "supabase", |
| "operation": "group_count", |
| "gold_executes": true, |
| "prediction_equivalent": false |
| }, |
| { |
| "id": "manual_475287613764159ee5eb_noisy_en", |
| "backend": "supabase", |
| "operation": "group_count", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_475287613764159ee5eb_hi", |
| "backend": "supabase", |
| "operation": "group_count", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_475287613764159ee5eb_hinglish", |
| "backend": "supabase", |
| "operation": "group_count", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_2ad3e251c936135e2005_en", |
| "backend": "supabase", |
| "operation": "all", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_2ad3e251c936135e2005_noisy_en", |
| "backend": "supabase", |
| "operation": "all", |
| "gold_executes": false, |
| "prediction_equivalent": false, |
| "error": "Wrong tool" |
| }, |
| { |
| "id": "manual_2ad3e251c936135e2005_hi", |
| "backend": "supabase", |
| "operation": "all", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_2ad3e251c936135e2005_hinglish", |
| "backend": "supabase", |
| "operation": "all", |
| "gold_executes": true, |
| "prediction_equivalent": false |
| }, |
| { |
| "id": "manual_a060620c638d2dc3759e_en", |
| "backend": "mysql", |
| "operation": "project", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_a060620c638d2dc3759e_noisy_en", |
| "backend": "mysql", |
| "operation": "project", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_a060620c638d2dc3759e_hi", |
| "backend": "mysql", |
| "operation": "project", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_a060620c638d2dc3759e_hinglish", |
| "backend": "mysql", |
| "operation": "project", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_eb982cf071094adaca4f_en", |
| "backend": "supabase", |
| "operation": "project_eq", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_eb982cf071094adaca4f_noisy_en", |
| "backend": "supabase", |
| "operation": "project_eq", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_eb982cf071094adaca4f_hi", |
| "backend": "supabase", |
| "operation": "project_eq", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_eb982cf071094adaca4f_hinglish", |
| "backend": "supabase", |
| "operation": "project_eq", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_c023a5b6f95173ae856d_en", |
| "backend": "mysql", |
| "operation": "gt", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_c023a5b6f95173ae856d_noisy_en", |
| "backend": "mysql", |
| "operation": "gt", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_c023a5b6f95173ae856d_hi", |
| "backend": "mysql", |
| "operation": "gt", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_c023a5b6f95173ae856d_hinglish", |
| "backend": "mysql", |
| "operation": "gt", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_affa5e5e14444cd82bb4_en", |
| "backend": "mysql", |
| "operation": "lt", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_affa5e5e14444cd82bb4_noisy_en", |
| "backend": "mysql", |
| "operation": "lt", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_affa5e5e14444cd82bb4_hi", |
| "backend": "mysql", |
| "operation": "lt", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_affa5e5e14444cd82bb4_hinglish", |
| "backend": "mysql", |
| "operation": "lt", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_c4f25575abfd24e3cd63_en", |
| "backend": "supabase", |
| "operation": "lte", |
| "gold_executes": true, |
| "prediction_equivalent": false |
| }, |
| { |
| "id": "manual_c4f25575abfd24e3cd63_noisy_en", |
| "backend": "supabase", |
| "operation": "lte", |
| "gold_executes": true, |
| "prediction_equivalent": false |
| }, |
| { |
| "id": "manual_c4f25575abfd24e3cd63_hi", |
| "backend": "supabase", |
| "operation": "lte", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_c4f25575abfd24e3cd63_hinglish", |
| "backend": "supabase", |
| "operation": "lte", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_23c0aa3a4596f995ad2a_en", |
| "backend": "mysql", |
| "operation": "count", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_23c0aa3a4596f995ad2a_noisy_en", |
| "backend": "mysql", |
| "operation": "count", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_23c0aa3a4596f995ad2a_hi", |
| "backend": "mysql", |
| "operation": "count", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_23c0aa3a4596f995ad2a_hinglish", |
| "backend": "mysql", |
| "operation": "count", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_b564e1a7b1e7fc6d2ab4_en", |
| "backend": "supabase", |
| "operation": "avg", |
| "gold_executes": false, |
| "prediction_equivalent": false, |
| "error": "Wrong non-SQL arguments" |
| }, |
| { |
| "id": "manual_b564e1a7b1e7fc6d2ab4_noisy_en", |
| "backend": "supabase", |
| "operation": "avg", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_b564e1a7b1e7fc6d2ab4_hi", |
| "backend": "supabase", |
| "operation": "avg", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_b564e1a7b1e7fc6d2ab4_hinglish", |
| "backend": "supabase", |
| "operation": "avg", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_2164b376f732a1c371ab_en", |
| "backend": "mysql", |
| "operation": "group_count", |
| "gold_executes": true, |
| "prediction_equivalent": false |
| }, |
| { |
| "id": "manual_2164b376f732a1c371ab_noisy_en", |
| "backend": "mysql", |
| "operation": "group_count", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_2164b376f732a1c371ab_hi", |
| "backend": "mysql", |
| "operation": "group_count", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_2164b376f732a1c371ab_hinglish", |
| "backend": "mysql", |
| "operation": "group_count", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_e8f90532d34c12209a83_en", |
| "backend": "mysql", |
| "operation": "gte", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_e8f90532d34c12209a83_noisy_en", |
| "backend": "mysql", |
| "operation": "gte", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_e8f90532d34c12209a83_hi", |
| "backend": "mysql", |
| "operation": "gte", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_e8f90532d34c12209a83_hinglish", |
| "backend": "mysql", |
| "operation": "gte", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_04b2e5886e750df09a51_en", |
| "backend": "mysql", |
| "operation": "null", |
| "gold_executes": true, |
| "prediction_equivalent": false |
| }, |
| { |
| "id": "manual_04b2e5886e750df09a51_noisy_en", |
| "backend": "mysql", |
| "operation": "null", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_04b2e5886e750df09a51_hi", |
| "backend": "mysql", |
| "operation": "null", |
| "gold_executes": true, |
| "prediction_equivalent": false |
| }, |
| { |
| "id": "manual_04b2e5886e750df09a51_hinglish", |
| "backend": "mysql", |
| "operation": "null", |
| "gold_executes": true, |
| "prediction_equivalent": false |
| }, |
| { |
| "id": "manual_d30264bbc3b08aedbc51_en", |
| "backend": "mysql", |
| "operation": "sum", |
| "gold_executes": false, |
| "prediction_equivalent": false, |
| "error": "(1054, \"Unknown column 'name' in 'field list'\")" |
| }, |
| { |
| "id": "manual_d30264bbc3b08aedbc51_noisy_en", |
| "backend": "mysql", |
| "operation": "sum", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_d30264bbc3b08aedbc51_hi", |
| "backend": "mysql", |
| "operation": "sum", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_d30264bbc3b08aedbc51_hinglish", |
| "backend": "mysql", |
| "operation": "sum", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_7dd163238936d96d0bd4_en", |
| "backend": "supabase", |
| "operation": "distinct", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_7dd163238936d96d0bd4_noisy_en", |
| "backend": "supabase", |
| "operation": "distinct", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_7dd163238936d96d0bd4_hi", |
| "backend": "supabase", |
| "operation": "distinct", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_7dd163238936d96d0bd4_hinglish", |
| "backend": "supabase", |
| "operation": "distinct", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_fd0bab1cc08d7298f131_en", |
| "backend": "supabase", |
| "operation": "top", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_fd0bab1cc08d7298f131_noisy_en", |
| "backend": "supabase", |
| "operation": "top", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_fd0bab1cc08d7298f131_hi", |
| "backend": "supabase", |
| "operation": "top", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| }, |
| { |
| "id": "manual_fd0bab1cc08d7298f131_hinglish", |
| "backend": "supabase", |
| "operation": "top", |
| "gold_executes": true, |
| "prediction_equivalent": true |
| } |
| ], |
| "metric_note": "prediction_equivalent includes correct tool/schema/project checks and two native SQL fixtures. gold_executes is a legacy label for completion of the combined validation path, not an independent gold correctness score; prediction errors can leave it false." |
| } |
