Berca Akbayır
Claude Opus 4.8 (1M context)
Step-by-step gateway traces, 100-tool catalog, Run button & per-panel metrics (#3)
3812402 unverified Download data/examples.json from ratel-ai/Ratel: direct link, hf CLI and curl.
- Browser
- Download file 41.9 kB
-
https://huggingface.co/spaces/ratel-ai/Ratel/resolve/main/data/examples.json
- Command line
-
hf download hf://spaces/ratel-ai/Ratel/data/examples.json
-
curl -L -o examples.json https://huggingface.co/spaces/ratel-ai/Ratel/resolve/main/data/examples.json
41.9 kB
[
{
"query": "create a task for this branch",
"goldTop5": [
"create_task_for_branch"
],
"source": "kestral-live",
"results": {
"bm25": [
{
"id": "create_task_for_branch",
"score": 8.582813
},
{
"id": "find_task_by_branch",
"score": 5.919037
},
{
"id": "register_branch_on_task",
"score": 5.742068
},
{
"id": "claim_task_and_branch",
"score": 5.503339
},
{
"id": "update_task",
"score": 4.36724
},
{
"id": "create_tasks_batch",
"score": 4.257123
},
{
"id": "search_tasks_by_time",
"score": 4.231214
},
{
"id": "check_task_duplicates",
"score": 4.085538
},
{
"id": "search_tasks_by_creator",
"score": 4.031451
},
{
"id": "create_task",
"score": 3.719965
}
],
"semantic": [
{
"id": "create_task_for_branch",
"score": 0.85397
},
{
"id": "find_task_by_branch",
"score": 0.761279
},
{
"id": "claim_task_and_branch",
"score": 0.757177
},
{
"id": "register_branch_on_task",
"score": 0.748798
},
{
"id": "create_task",
"score": 0.728596
},
{
"id": "create_project",
"score": 0.703093
},
{
"id": "update_task",
"score": 0.695863
},
{
"id": "create_sprint",
"score": 0.681412
},
{
"id": "search_tasks_with_comments",
"score": 0.669281
},
{
"id": "create_tasks_batch",
"score": 0.668715
}
],
"hybrid": [
{
"id": "create_task_for_branch",
"score": 0.033333
},
{
"id": "find_task_by_branch",
"score": 0.032787
},
{
"id": "claim_task_and_branch",
"score": 0.032002
},
{
"id": "register_branch_on_task",
"score": 0.032002
},
{
"id": "update_task",
"score": 0.030777
},
{
"id": "create_task",
"score": 0.030118
},
{
"id": "create_tasks_batch",
"score": 0.029877
},
{
"id": "search_tasks_by_time",
"score": 0.028485
},
{
"id": "create_project",
"score": 0.028205
},
{
"id": "create_sprint",
"score": 0.027912
}
]
},
"retrievedTokens": {
"bm25": 285,
"semantic": 302,
"hybrid": 302
}
},
{
"query": "find a task by its linked branch name",
"goldTop5": [
"find_task_by_branch"
],
"source": "kestral-live",
"results": {
"bm25": [
{
"id": "find_task_by_branch",
"score": 13.422369
},
{
"id": "create_task_for_branch",
"score": 11.877792
},
{
"id": "register_branch_on_task",
"score": 8.169369
},
{
"id": "claim_task_and_branch",
"score": 7.61269
},
{
"id": "list_repository_branches",
"score": 6.077036
},
{
"id": "update_task",
"score": 5.685644
},
{
"id": "link_pr_to_task",
"score": 5.248134
},
{
"id": "get_ci_build_status",
"score": 5.19636
},
{
"id": "search_projects",
"score": 5.08555
},
{
"id": "find_customer_mentions",
"score": 4.345972
}
],
"semantic": [
{
"id": "find_task_by_branch",
"score": 0.882817
},
{
"id": "create_task_for_branch",
"score": 0.772516
},
{
"id": "register_branch_on_task",
"score": 0.730849
},
{
"id": "claim_task_and_branch",
"score": 0.716964
},
{
"id": "search_projects",
"score": 0.701384
},
{
"id": "search_tasks",
"score": 0.693268
},
{
"id": "search_tasks_with_comments",
"score": 0.685869
},
{
"id": "search_my_tasks_by_keyword",
"score": 0.677011
},
{
"id": "link_pr_to_task",
"score": 0.676214
},
{
"id": "check_task_duplicates",
"score": 0.654522
}
],
"hybrid": [
{
"id": "find_task_by_branch",
"score": 0.033333
},
{
"id": "create_task_for_branch",
"score": 0.032787
},
{
"id": "register_branch_on_task",
"score": 0.032258
},
{
"id": "claim_task_and_branch",
"score": 0.031746
},
{
"id": "search_projects",
"score": 0.030331
},
{
"id": "link_pr_to_task",
"score": 0.029857
},
{
"id": "search_tasks",
"score": 0.029274
},
{
"id": "update_task",
"score": 0.029083
},
{
"id": "search_tasks_with_comments",
"score": 0.028665
},
{
"id": "list_repository_branches",
"score": 0.028446
}
]
},
"retrievedTokens": {
"bm25": 285,
"semantic": 285,
"hybrid": 285
}
},
{
"query": "set up a new project and add three initial tasks to it",
"goldTop5": [
"create_project",
"create_tasks_batch"
],
"source": "kestral-live",
"results": {
"bm25": [
{
"id": "create_tasks_batch",
"score": 11.64656
},
{
"id": "create_project",
"score": 8.406992
},
{
"id": "create_task",
"score": 7.687551
},
{
"id": "add_task_comment",
"score": 5.994229
},
{
"id": "set_member_role",
"score": 5.74906
},
{
"id": "update_task",
"score": 5.106228
},
{
"id": "invite_member",
"score": 5.100462
},
{
"id": "set_calendar_reminder",
"score": 4.691238
},
{
"id": "edit_task_comment",
"score": 4.133376
},
{
"id": "claim_task_and_branch",
"score": 4.129135
}
],
"semantic": [
{
"id": "create_project",
"score": 0.726703
},
{
"id": "create_tasks_batch",
"score": 0.725056
},
{
"id": "create_task",
"score": 0.723463
},
{
"id": "setup_workflow",
"score": 0.66783
},
{
"id": "create_milestone",
"score": 0.658306
},
{
"id": "update_task",
"score": 0.647122
},
{
"id": "create_task_for_branch",
"score": 0.644604
},
{
"id": "manage_project",
"score": 0.64306
},
{
"id": "add_task_comment",
"score": 0.638271
},
{
"id": "check_task_duplicates",
"score": 0.636886
}
],
"hybrid": [
{
"id": "create_project",
"score": 0.03306
},
{
"id": "create_tasks_batch",
"score": 0.03306
},
{
"id": "create_task",
"score": 0.032258
},
{
"id": "update_task",
"score": 0.030769
},
{
"id": "add_task_comment",
"score": 0.030579
},
{
"id": "create_task_for_branch",
"score": 0.02885
},
{
"id": "claim_task_and_branch",
"score": 0.028778
},
{
"id": "search_projects",
"score": 0.027973
},
{
"id": "check_task_duplicates",
"score": 0.027826
},
{
"id": "manage_project",
"score": 0.027425
}
]
},
"retrievedTokens": {
"bm25": 403,
"semantic": 403,
"hybrid": 403
}
},
{
"query": "plan my day and prioritize what to work on",
"goldTop5": [
"plan_day_workflow",
"get_daily_planning_context"
],
"source": "kestral-live",
"results": {
"bm25": [
{
"id": "plan_day_workflow",
"score": 14.549468
},
{
"id": "get_daily_planning_context",
"score": 13.414358
},
{
"id": "create_sprint",
"score": 5.727487
},
{
"id": "list_stale_tasks",
"score": 3.786772
},
{
"id": "end_day_review_workflow",
"score": 3.745481
},
{
"id": "claim_task_and_branch",
"score": 2.937606
},
{
"id": "list_calendar_events",
"score": 2.661258
},
{
"id": "get_daily_closeout_context",
"score": 2.562011
},
{
"id": "search_tasks_recently_updated",
"score": 2.543044
},
{
"id": "get_workload_distribution",
"score": 2.496693
}
],
"semantic": [
{
"id": "plan_day_workflow",
"score": 0.728498
},
{
"id": "get_daily_planning_context",
"score": 0.673375
},
{
"id": "sort_tasks",
"score": 0.658833
},
{
"id": "update_task",
"score": 0.64029
},
{
"id": "create_task",
"score": 0.630744
},
{
"id": "create_sprint",
"score": 0.629696
},
{
"id": "list_my_active_tasks",
"score": 0.629065
},
{
"id": "end_day_review_workflow",
"score": 0.627313
},
{
"id": "set_calendar_reminder",
"score": 0.614109
},
{
"id": "list_tasks_by_priority",
"score": 0.612222
}
],
"hybrid": [
{
"id": "plan_day_workflow",
"score": 0.033333
},
{
"id": "get_daily_planning_context",
"score": 0.032787
},
{
"id": "create_sprint",
"score": 0.031514
},
{
"id": "end_day_review_workflow",
"score": 0.03055
},
{
"id": "list_my_active_tasks",
"score": 0.029437
},
{
"id": "claim_task_and_branch",
"score": 0.028043
},
{
"id": "search_tasks_recently_updated",
"score": 0.027526
},
{
"id": "list_stale_tasks",
"score": 0.026862
},
{
"id": "get_daily_closeout_context",
"score": 0.026036
},
{
"id": "list_calendar_events",
"score": 0.026021
}
]
},
"retrievedTokens": {
"bm25": 210,
"semantic": 230,
"hybrid": 210
}
},
{
"query": "mark a task as done",
"goldTop5": [
"update_task"
],
"source": "kestral-live",
"results": {
"bm25": [
{
"id": "update_task",
"score": 6.984727
},
{
"id": "mark_notification_read",
"score": 5.047772
},
{
"id": "get_daily_closeout_context",
"score": 4.363218
},
{
"id": "end_day_review_workflow",
"score": 4.320774
},
{
"id": "create_tasks_batch",
"score": 1.668265
},
{
"id": "search_my_tasks_by_keyword",
"score": 1.638963
},
{
"id": "link_pr_to_task",
"score": 1.634014
},
{
"id": "add_task_comment",
"score": 1.631011
},
{
"id": "check_task_duplicates",
"score": 1.615335
},
{
"id": "register_branch_on_task",
"score": 1.615335
}
],
"semantic": [
{
"id": "add_task_comment",
"score": 0.719891
},
{
"id": "edit_task_comment",
"score": 0.701684
},
{
"id": "update_task",
"score": 0.683156
},
{
"id": "complete_task_with_review",
"score": 0.676587
},
{
"id": "create_task",
"score": 0.668378
},
{
"id": "search_tasks_by_time",
"score": 0.654669
},
{
"id": "mark_notification_read",
"score": 0.644681
},
{
"id": "check_task_duplicates",
"score": 0.643997
},
{
"id": "search_tasks_with_comments",
"score": 0.640929
},
{
"id": "list_my_active_tasks",
"score": 0.639023
}
],
"hybrid": [
{
"id": "update_task",
"score": 0.032796
},
{
"id": "add_task_comment",
"score": 0.031592
},
{
"id": "mark_notification_read",
"score": 0.031545
},
{
"id": "end_day_review_workflow",
"score": 0.029958
},
{
"id": "check_task_duplicates",
"score": 0.029631
},
{
"id": "complete_task_with_review",
"score": 0.029572
},
{
"id": "create_tasks_batch",
"score": 0.028958
},
{
"id": "search_tasks_with_comments",
"score": 0.02879
},
{
"id": "search_my_tasks_by_keyword",
"score": 0.028372
},
{
"id": "claim_task_and_branch",
"score": 0.027588
}
]
},
"retrievedTokens": {
"bm25": 310,
"semantic": 341,
"hybrid": 322
}
},
{
"query": "leave a progress update comment on the task for my current branch",
"goldTop5": [
"add_task_comment",
"find_task_by_branch"
],
"source": "kestral-live",
"results": {
"bm25": [
{
"id": "add_task_comment",
"score": 15.242415
},
{
"id": "find_task_by_branch",
"score": 8.339669
},
{
"id": "register_branch_on_task",
"score": 8.110106
},
{
"id": "claim_task_and_branch",
"score": 7.904187
},
{
"id": "list_my_active_tasks",
"score": 7.638011
},
{
"id": "update_task",
"score": 7.152776
},
{
"id": "create_task_for_branch",
"score": 5.69091
},
{
"id": "delete_task_comment",
"score": 5.51934
},
{
"id": "search_tasks_with_comments",
"score": 5.45519
},
{
"id": "edit_task_comment",
"score": 5.404408
}
],
"semantic": [
{
"id": "add_task_comment",
"score": 0.776397
},
{
"id": "claim_task_and_branch",
"score": 0.75717
},
{
"id": "update_task",
"score": 0.742931
},
{
"id": "link_pr_to_task",
"score": 0.715575
},
{
"id": "edit_task_comment",
"score": 0.707744
},
{
"id": "delete_task_comment",
"score": 0.703722
},
{
"id": "create_task_for_branch",
"score": 0.693377
},
{
"id": "register_branch_on_task",
"score": 0.688841
},
{
"id": "complete_task_with_review",
"score": 0.681186
},
{
"id": "find_task_by_branch",
"score": 0.678148
}
],
"hybrid": [
{
"id": "add_task_comment",
"score": 0.033333
},
{
"id": "claim_task_and_branch",
"score": 0.032266
},
{
"id": "update_task",
"score": 0.031514
},
{
"id": "register_branch_on_task",
"score": 0.031054
},
{
"id": "find_task_by_branch",
"score": 0.030886
},
{
"id": "delete_task_comment",
"score": 0.03031
},
{
"id": "create_task_for_branch",
"score": 0.030303
},
{
"id": "link_pr_to_task",
"score": 0.030159
},
{
"id": "edit_task_comment",
"score": 0.030118
},
{
"id": "list_my_active_tasks",
"score": 0.029139
}
]
},
"retrievedTokens": {
"bm25": 238,
"semantic": 372,
"hybrid": 372
}
},
{
"query": "link this pull request to its task",
"goldTop5": [
"link_pr_to_task"
],
"source": "kestral-live",
"results": {
"bm25": [
{
"id": "link_pr_to_task",
"score": 12.730273
},
{
"id": "get_pull_request_status",
"score": 8.122603
},
{
"id": "list_open_pull_requests",
"score": 8.081908
},
{
"id": "find_task_by_branch",
"score": 4.108042
},
{
"id": "complete_task_with_review",
"score": 3.956365
},
{
"id": "create_task_for_branch",
"score": 3.867998
},
{
"id": "search_tasks",
"score": 3.867998
},
{
"id": "link_external_document",
"score": 3.341391
},
{
"id": "count_feedback",
"score": 2.87453
},
{
"id": "search_feedback",
"score": 2.807718
}
],
"semantic": [
{
"id": "link_pr_to_task",
"score": 0.775116
},
{
"id": "link_external_document",
"score": 0.712594
},
{
"id": "link_document_to_project",
"score": 0.696661
},
{
"id": "complete_task_with_review",
"score": 0.684236
},
{
"id": "list_open_pull_requests",
"score": 0.680026
},
{
"id": "get_pull_request_status",
"score": 0.677963
},
{
"id": "create_task_for_branch",
"score": 0.671398
},
{
"id": "search_tasks_with_comments",
"score": 0.667271
},
{
"id": "add_task_comment",
"score": 0.666917
},
{
"id": "find_task_by_branch",
"score": 0.665737
}
],
"hybrid": [
{
"id": "link_pr_to_task",
"score": 0.033333
},
{
"id": "get_pull_request_status",
"score": 0.031778
},
{
"id": "list_open_pull_requests",
"score": 0.031754
},
{
"id": "complete_task_with_review",
"score": 0.031498
},
{
"id": "link_external_document",
"score": 0.031319
},
{
"id": "create_task_for_branch",
"score": 0.030536
},
{
"id": "find_task_by_branch",
"score": 0.030366
},
{
"id": "link_document_to_project",
"score": 0.030214
},
{
"id": "search_tasks",
"score": 0.02904
},
{
"id": "add_task_comment",
"score": 0.028219
}
]
},
"retrievedTokens": {
"bm25": 260,
"semantic": 261,
"hybrid": 260
}
},
{
"query": "complete the task for my branch",
"goldTop5": [
"update_task",
"complete_task_with_review",
"find_task_by_branch"
],
"source": "kestral-live",
"results": {
"bm25": [
{
"id": "update_task",
"score": 6.38694
},
{
"id": "find_task_by_branch",
"score": 5.919037
},
{
"id": "register_branch_on_task",
"score": 5.742068
},
{
"id": "create_task_for_branch",
"score": 5.69091
},
{
"id": "complete_task_with_review",
"score": 5.647087
},
{
"id": "claim_task_and_branch",
"score": 5.503339
},
{
"id": "search_tasks_by_time",
"score": 5.17397
},
{
"id": "list_repository_branches",
"score": 3.530267
},
{
"id": "get_ci_build_status",
"score": 3.459164
},
{
"id": "get_sprint_summary",
"score": 2.790999
}
],
"semantic": [
{
"id": "create_task_for_branch",
"score": 0.744962
},
{
"id": "claim_task_and_branch",
"score": 0.736477
},
{
"id": "find_task_by_branch",
"score": 0.730112
},
{
"id": "complete_task_with_review",
"score": 0.713954
},
{
"id": "register_branch_on_task",
"score": 0.707054
},
{
"id": "update_task",
"score": 0.673621
},
{
"id": "link_pr_to_task",
"score": 0.627147
},
{
"id": "end_day_review_workflow",
"score": 0.623937
},
{
"id": "list_my_active_tasks",
"score": 0.618193
},
{
"id": "add_task_comment",
"score": 0.614251
}
],
"hybrid": [
{
"id": "create_task_for_branch",
"score": 0.03254
},
{
"id": "find_task_by_branch",
"score": 0.032522
},
{
"id": "update_task",
"score": 0.032051
},
{
"id": "claim_task_and_branch",
"score": 0.031778
},
{
"id": "register_branch_on_task",
"score": 0.031754
},
{
"id": "complete_task_with_review",
"score": 0.031498
},
{
"id": "get_sprint_summary",
"score": 0.028577
},
{
"id": "link_pr_to_task",
"score": 0.028485
},
{
"id": "search_tasks_by_time",
"score": 0.028139
},
{
"id": "add_task_comment",
"score": 0.027651
}
]
},
"retrievedTokens": {
"bm25": 346,
"semantic": 302,
"hybrid": 389
}
},
{
"query": "search my tasks by keyword",
"goldTop5": [
"search_my_tasks_by_keyword"
],
"source": "kestral-live",
"results": {
"bm25": [
{
"id": "search_my_tasks_by_keyword",
"score": 10.961438
},
{
"id": "search_tasks",
"score": 4.214272
},
{
"id": "search_tasks_with_comments",
"score": 4.192751
},
{
"id": "search_tasks_by_creator",
"score": 3.429143
},
{
"id": "search_tasks_recently_updated",
"score": 3.369742
},
{
"id": "search_tasks_by_time",
"score": 3.312568
},
{
"id": "search_projects",
"score": 2.893722
},
{
"id": "search_feedback",
"score": 2.582146
},
{
"id": "search_customers",
"score": 2.569209
},
{
"id": "search_document_chunks",
"score": 2.470203
}
],
"semantic": [
{
"id": "search_my_tasks_by_keyword",
"score": 0.873639
},
{
"id": "search_tasks",
"score": 0.76778
},
{
"id": "search_tasks_by_time",
"score": 0.753032
},
{
"id": "search_tasks_with_comments",
"score": 0.752168
},
{
"id": "search_tasks_recently_updated",
"score": 0.727744
},
{
"id": "list_my_active_tasks",
"score": 0.723047
},
{
"id": "search_projects",
"score": 0.71409
},
{
"id": "search_tasks_by_creator",
"score": 0.699643
},
{
"id": "list_tasks_by_tag",
"score": 0.687532
},
{
"id": "sort_tasks",
"score": 0.681137
}
],
"hybrid": [
{
"id": "search_my_tasks_by_keyword",
"score": 0.033333
},
{
"id": "search_tasks",
"score": 0.032787
},
{
"id": "search_tasks_with_comments",
"score": 0.032002
},
{
"id": "search_tasks_by_time",
"score": 0.031514
},
{
"id": "search_tasks_recently_updated",
"score": 0.03125
},
{
"id": "search_tasks_by_creator",
"score": 0.030798
},
{
"id": "search_projects",
"score": 0.030303
},
{
"id": "find_documents",
"score": 0.027588
},
{
"id": "search_document_chunks",
"score": 0.027151
},
{
"id": "list_my_active_tasks",
"score": 0.027149
}
]
},
"retrievedTokens": {
"bm25": 247,
"semantic": 288,
"hybrid": 247
}
},
{
"query": "what am I working on right now",
"goldTop5": [
"list_my_active_tasks"
],
"source": "kestral-live",
"results": {
"bm25": [
{
"id": "list_my_active_tasks",
"score": 6.887891
},
{
"id": "claim_task_and_branch",
"score": 2.937606
},
{
"id": "get_workload_distribution",
"score": 2.496693
},
{
"id": "generate_burndown_chart",
"score": 2.457397
},
{
"id": "get_cycle_time_stats",
"score": 2.457397
},
{
"id": "create_sprint",
"score": 2.364366
},
{
"id": "get_daily_planning_context",
"score": 2.329096
},
{
"id": "plan_day_workflow",
"score": 2.311852
},
{
"id": "complete_task_with_review",
"score": 2.152494
}
],
"semantic": [
{
"id": "list_my_active_tasks",
"score": 0.65723
},
{
"id": "search_tasks_recently_updated",
"score": 0.630614
},
{
"id": "create_project",
"score": 0.628721
},
{
"id": "update_task",
"score": 0.622613
},
{
"id": "search_tasks_by_time",
"score": 0.602394
},
{
"id": "get_project_activity_feed",
"score": 0.59763
},
{
"id": "get_ci_build_status",
"score": 0.597276
},
{
"id": "get_project_pending_changes",
"score": 0.591788
},
{
"id": "create_task",
"score": 0.587362
},
{
"id": "get_project_brain",
"score": 0.586405
}
],
"hybrid": [
{
"id": "list_my_active_tasks",
"score": 0.033333
},
{
"id": "claim_task_and_branch",
"score": 0.029551
},
{
"id": "plan_day_workflow",
"score": 0.02901
},
{
"id": "create_sprint",
"score": 0.02773
},
{
"id": "complete_task_with_review",
"score": 0.027206
},
{
"id": "get_daily_planning_context",
"score": 0.026779
},
{
"id": "generate_burndown_chart",
"score": 0.025488
},
{
"id": "get_workload_distribution",
"score": 0.023763
},
{
"id": "get_cycle_time_stats",
"score": 0.023377
},
{
"id": "search_tasks_recently_updated",
"score": 0.016393
}
]
},
"retrievedTokens": {
"bm25": 208,
"semantic": 216,
"hybrid": 227
}
},
{
"query": "end of day review, what did I finish today",
"goldTop5": [
"end_day_review_workflow",
"get_daily_closeout_context"
],
"source": "kestral-live",
"results": {
"bm25": [
{
"id": "end_day_review_workflow",
"score": 17.563652
},
{
"id": "get_daily_closeout_context",
"score": 16.391945
},
{
"id": "complete_task_with_review",
"score": 6.644857
},
{
"id": "get_daily_planning_context",
"score": 5.627342
},
{
"id": "list_calendar_events",
"score": 5.322516
},
{
"id": "create_sprint",
"score": 3.800738
},
{
"id": "list_stale_tasks",
"score": 3.786772
},
{
"id": "plan_day_workflow",
"score": 3.759144
},
{
"id": "get_daily_brief",
"score": 3.184076
},
{
"id": "get_pull_request_status",
"score": 2.978694
}
],
"semantic": [
{
"id": "end_day_review_workflow",
"score": 0.767961
},
{
"id": "get_daily_closeout_context",
"score": 0.718509
},
{
"id": "complete_task_with_review",
"score": 0.668029
},
{
"id": "get_daily_brief",
"score": 0.652707
},
{
"id": "get_daily_planning_context",
"score": 0.616047
},
{
"id": "plan_day_workflow",
"score": 0.593682
},
{
"id": "search_tasks_by_time",
"score": 0.574053
},
{
"id": "search_tasks_recently_updated",
"score": 0.569633
},
{
"id": "list_overdue_tasks",
"score": 0.566135
},
{
"id": "create_sprint",
"score": 0.56253
}
],
"hybrid": [
{
"id": "end_day_review_workflow",
"score": 0.033333
},
{
"id": "get_daily_closeout_context",
"score": 0.032787
},
{
"id": "complete_task_with_review",
"score": 0.032258
},
{
"id": "get_daily_planning_context",
"score": 0.031498
},
{
"id": "get_daily_brief",
"score": 0.030579
},
{
"id": "plan_day_workflow",
"score": 0.03031
},
{
"id": "create_sprint",
"score": 0.029877
},
{
"id": "search_tasks_recently_updated",
"score": 0.028624
},
{
"id": "list_stale_tasks",
"score": 0.028139
},
{
"id": "get_pull_request_status",
"score": 0.027151
}
]
},
"retrievedTokens": {
"bm25": 242,
"semantic": 242,
"hybrid": 242
}
},
{
"query": "assign this task to me and start working on it",
"goldTop5": [
"claim_task_and_branch",
"update_task"
],
"source": "kestral-live",
"results": {
"bm25": [
{
"id": "claim_task_and_branch",
"score": 12.364825
},
{
"id": "create_sprint",
"score": 6.165104
},
{
"id": "get_cycle_time_stats",
"score": 5.160541
},
{
"id": "link_pr_to_task",
"score": 4.674805
},
{
"id": "update_task",
"score": 4.111688
},
{
"id": "list_my_active_tasks",
"score": 3.998288
},
{
"id": "complete_task_with_review",
"score": 3.74111
},
{
"id": "schedule_meeting",
"score": 3.731916
},
{
"id": "get_daily_planning_context",
"score": 3.331949
},
{
"id": "plan_day_workflow",
"score": 3.307281
}
],
"semantic": [
{
"id": "update_task",
"score": 0.694133
},
{
"id": "create_task",
"score": 0.691895
},
{
"id": "claim_task_and_branch",
"score": 0.688046
},
{
"id": "list_my_active_tasks",
"score": 0.659452
},
{
"id": "add_task_comment",
"score": 0.649138
},
{
"id": "setup_workflow",
"score": 0.648945
},
{
"id": "link_pr_to_task",
"score": 0.646606
},
{
"id": "search_my_tasks_by_keyword",
"score": 0.63947
},
{
"id": "create_tasks_batch",
"score": 0.639073
},
{
"id": "create_task_for_branch",
"score": 0.638434
}
],
"hybrid": [
{
"id": "claim_task_and_branch",
"score": 0.032796
},
{
"id": "update_task",
"score": 0.032292
},
{
"id": "list_my_active_tasks",
"score": 0.031258
},
{
"id": "link_pr_to_task",
"score": 0.031025
},
{
"id": "create_sprint",
"score": 0.030679
},
{
"id": "setup_workflow",
"score": 0.029274
},
{
"id": "add_task_comment",
"score": 0.028612
},
{
"id": "plan_day_workflow",
"score": 0.028191
},
{
"id": "search_my_tasks_by_keyword",
"score": 0.028083
},
{
"id": "create_tasks_batch",
"score": 0.028039
}
]
},
"retrievedTokens": {
"bm25": 243,
"semantic": 446,
"hybrid": 346
}
},
{
"query": "change the due date of a task",
"goldTop5": [
"update_task"
],
"source": "kestral-live",
"results": {
"bm25": [
{
"id": "update_task",
"score": 9.971552
},
{
"id": "search_tasks_by_time",
"score": 8.69059
},
{
"id": "sort_tasks",
"score": 7.676823
},
{
"id": "create_task",
"score": 7.549388
},
{
"id": "create_milestone",
"score": 7.185852
},
{
"id": "list_overdue_tasks",
"score": 6.767224
},
{
"id": "get_project_pending_changes",
"score": 4.261696
},
{
"id": "search_tasks_recently_updated",
"score": 3.892081
},
{
"id": "reject_pending_change",
"score": 3.658265
},
{
"id": "approve_pending_change",
"score": 3.644672
}
],
"semantic": [
{
"id": "update_task",
"score": 0.734993
},
{
"id": "list_overdue_tasks",
"score": 0.726096
},
{
"id": "search_tasks_by_time",
"score": 0.725798
},
{
"id": "create_milestone",
"score": 0.6956
},
{
"id": "create_task",
"score": 0.683216
},
{
"id": "search_tasks_recently_updated",
"score": 0.662366
},
{
"id": "sort_tasks",
"score": 0.661868
},
{
"id": "create_sprint",
"score": 0.660779
},
{
"id": "list_pending_change_details",
"score": 0.641593
},
{
"id": "edit_task_comment",
"score": 0.638713
}
],
"hybrid": [
{
"id": "update_task",
"score": 0.033333
},
{
"id": "search_tasks_by_time",
"score": 0.032522
},
{
"id": "list_overdue_tasks",
"score": 0.031778
},
{
"id": "create_milestone",
"score": 0.031498
},
{
"id": "create_task",
"score": 0.031498
},
{
"id": "sort_tasks",
"score": 0.031281
},
{
"id": "search_tasks_recently_updated",
"score": 0.03031
},
{
"id": "create_sprint",
"score": 0.02901
},
{
"id": "approve_pending_change",
"score": 0.028778
},
{
"id": "get_project_pending_changes",
"score": 0.028665
}
]
},
"retrievedTokens": {
"bm25": 393,
"semantic": 343,
"hybrid": 343
}
},
{
"query": "show overdue tasks",
"goldTop5": [
"list_overdue_tasks"
],
"source": "kestral-live",
"results": {
"bm25": [
{
"id": "list_overdue_tasks",
"score": 9.433943
},
{
"id": "get_daily_planning_context",
"score": 4.315806
},
{
"id": "plan_day_workflow",
"score": 4.283854
},
{
"id": "mark_notification_read",
"score": 3.31239
},
{
"id": "get_daily_brief",
"score": 3.184076
},
{
"id": "get_document_content",
"score": 3.088366
},
{
"id": "create_tasks_batch",
"score": 1.668265
},
{
"id": "search_my_tasks_by_keyword",
"score": 1.638963
},
{
"id": "link_pr_to_task",
"score": 1.634014
},
{
"id": "add_task_comment",
"score": 1.631011
}
],
"semantic": [
{
"id": "list_overdue_tasks",
"score": 0.856753
},
{
"id": "search_tasks_by_time",
"score": 0.714079
},
{
"id": "search_tasks_recently_updated",
"score": 0.692586
},
{
"id": "sort_tasks",
"score": 0.68577
},
{
"id": "list_my_active_tasks",
"score": 0.67862
},
{
"id": "create_task",
"score": 0.67397
},
{
"id": "list_stale_tasks",
"score": 0.666958
},
{
"id": "update_task",
"score": 0.657325
},
{
"id": "get_project_activity_feed",
"score": 0.655829
},
{
"id": "complete_task_with_review",
"score": 0.654767
}
],
"hybrid": [
{
"id": "list_overdue_tasks",
"score": 0.033333
},
{
"id": "list_my_active_tasks",
"score": 0.028446
},
{
"id": "get_daily_planning_context",
"score": 0.028298
},
{
"id": "add_task_comment",
"score": 0.028191
},
{
"id": "mark_notification_read",
"score": 0.027921
},
{
"id": "plan_day_workflow",
"score": 0.027894
},
{
"id": "complete_task_with_review",
"score": 0.027826
},
{
"id": "check_task_duplicates",
"score": 0.027799
},
{
"id": "search_tasks_recently_updated",
"score": 0.027623
},
{
"id": "search_tasks_by_time",
"score": 0.027505
}
]
},
"retrievedTokens": {
"bm25": 180,
"semantic": 232,
"hybrid": 165
}
}
]
[
{
"query": "create a task for this branch",
"goldTop5": [
"create_task_for_branch"
],
"source": "kestral-live",
"results": {
"bm25": [
{
"id": "create_task_for_branch",
"score": 8.582813
},
{
"id": "find_task_by_branch",
"score": 5.919037
},
{
"id": "register_branch_on_task",
"score": 5.742068
},
{
"id": "claim_task_and_branch",
"score": 5.503339
},
{
"id": "update_task",
"score": 4.36724
},
{
"id": "create_tasks_batch",
"score": 4.257123
},
{
"id": "search_tasks_by_time",
"score": 4.231214
},
{
"id": "check_task_duplicates",
"score": 4.085538
},
{
"id": "search_tasks_by_creator",
"score": 4.031451
},
{
"id": "create_task",
"score": 3.719965
}
],
"semantic": [
{
"id": "create_task_for_branch",
"score": 0.85397
},
{
"id": "find_task_by_branch",
"score": 0.761279
},
{
"id": "claim_task_and_branch",
"score": 0.757177
},
{
"id": "register_branch_on_task",
"score": 0.748798
},
{
"id": "create_task",
"score": 0.728596
},
{
"id": "create_project",
"score": 0.703093
},
{
"id": "update_task",
"score": 0.695863
},
{
"id": "create_sprint",
"score": 0.681412
},
{
"id": "search_tasks_with_comments",
"score": 0.669281
},
{
"id": "create_tasks_batch",
"score": 0.668715
}
],
"hybrid": [
{
"id": "create_task_for_branch",
"score": 0.033333
},
{
"id": "find_task_by_branch",
"score": 0.032787