[ { "query": "create a task for this branch", "goldTop5": [ "create_task_for_branch" ], "source": "kestral-live", "results": { "bm25": [ { "id": "create_task_for_branch", "score": 8.582813 }, { "id": "find_task_by_branch", "score": 5.919037 }, { "id": "register_branch_on_task", "score": 5.742068 }, { "id": "claim_task_and_branch", "score": 5.503339 }, { "id": "update_task", "score": 4.36724 }, { "id": "create_tasks_batch", "score": 4.257123 }, { "id": "search_tasks_by_time", "score": 4.231214 }, { "id": "check_task_duplicates", "score": 4.085538 }, { "id": "search_tasks_by_creator", "score": 4.031451 }, { "id": "create_task", "score": 3.719965 } ], "semantic": [ { "id": "create_task_for_branch", "score": 0.85397 }, { "id": "find_task_by_branch", "score": 0.761279 }, { "id": "claim_task_and_branch", "score": 0.757177 }, { "id": "register_branch_on_task", "score": 0.748798 }, { "id": "create_task", "score": 0.728596 }, { "id": "create_project", "score": 0.703093 }, { "id": "update_task", "score": 0.695863 }, { "id": "create_sprint", "score": 0.681412 }, { "id": "search_tasks_with_comments", "score": 0.669281 }, { "id": "create_tasks_batch", "score": 0.668715 } ], "hybrid": [ { "id": "create_task_for_branch", "score": 0.033333 }, { "id": "find_task_by_branch", "score": 0.032787 }, { "id": "claim_task_and_branch", "score": 0.032002 }, { "id": "register_branch_on_task", "score": 0.032002 }, { "id": "update_task", "score": 0.030777 }, { "id": "create_task", "score": 0.030118 }, { "id": "create_tasks_batch", "score": 0.029877 }, { "id": "search_tasks_by_time", "score": 0.028485 }, { "id": "create_project", "score": 0.028205 }, { "id": "create_sprint", "score": 0.027912 } ] }, "retrievedTokens": { "bm25": 285, "semantic": 302, "hybrid": 302 } }, { "query": "find a task by its linked branch name", "goldTop5": [ "find_task_by_branch" ], "source": "kestral-live", "results": { "bm25": [ { "id": "find_task_by_branch", "score": 13.422369 }, { "id": "create_task_for_branch", "score": 11.877792 }, { "id": "register_branch_on_task", "score": 8.169369 }, { "id": "claim_task_and_branch", "score": 7.61269 }, { "id": "list_repository_branches", "score": 6.077036 }, { "id": "update_task", "score": 5.685644 }, { "id": "link_pr_to_task", "score": 5.248134 }, { "id": "get_ci_build_status", "score": 5.19636 }, { "id": "search_projects", "score": 5.08555 }, { "id": "find_customer_mentions", "score": 4.345972 } ], "semantic": [ { "id": "find_task_by_branch", "score": 0.882817 }, { "id": "create_task_for_branch", "score": 0.772516 }, { "id": "register_branch_on_task", "score": 0.730849 }, { "id": "claim_task_and_branch", "score": 0.716964 }, { "id": "search_projects", "score": 0.701384 }, { "id": "search_tasks", "score": 0.693268 }, { "id": "search_tasks_with_comments", "score": 0.685869 }, { "id": "search_my_tasks_by_keyword", "score": 0.677011 }, { "id": "link_pr_to_task", "score": 0.676214 }, { "id": "check_task_duplicates", "score": 0.654522 } ], "hybrid": [ { "id": "find_task_by_branch", "score": 0.033333 }, { "id": "create_task_for_branch", "score": 0.032787 }, { "id": "register_branch_on_task", "score": 0.032258 }, { "id": "claim_task_and_branch", "score": 0.031746 }, { "id": "search_projects", "score": 0.030331 }, { "id": "link_pr_to_task", "score": 0.029857 }, { "id": "search_tasks", "score": 0.029274 }, { "id": "update_task", "score": 0.029083 }, { "id": "search_tasks_with_comments", "score": 0.028665 }, { "id": "list_repository_branches", "score": 0.028446 } ] }, "retrievedTokens": { "bm25": 285, "semantic": 285, "hybrid": 285 } }, { "query": "set up a new project and add three initial tasks to it", "goldTop5": [ "create_project", "create_tasks_batch" ], "source": "kestral-live", "results": { "bm25": [ { "id": "create_tasks_batch", "score": 11.64656 }, { "id": "create_project", "score": 8.406992 }, { "id": "create_task", "score": 7.687551 }, { "id": "add_task_comment", "score": 5.994229 }, { "id": "set_member_role", "score": 5.74906 }, { "id": "update_task", "score": 5.106228 }, { "id": "invite_member", "score": 5.100462 }, { "id": "set_calendar_reminder", "score": 4.691238 }, { "id": "edit_task_comment", "score": 4.133376 }, { "id": "claim_task_and_branch", "score": 4.129135 } ], "semantic": [ { "id": "create_project", "score": 0.726703 }, { "id": "create_tasks_batch", "score": 0.725056 }, { "id": "create_task", "score": 0.723463 }, { "id": "setup_workflow", "score": 0.66783 }, { "id": "create_milestone", "score": 0.658306 }, { "id": "update_task", "score": 0.647122 }, { "id": "create_task_for_branch", "score": 0.644604 }, { "id": "manage_project", "score": 0.64306 }, { "id": "add_task_comment", "score": 0.638271 }, { "id": "check_task_duplicates", "score": 0.636886 } ], "hybrid": [ { "id": "create_project", "score": 0.03306 }, { "id": "create_tasks_batch", "score": 0.03306 }, { "id": "create_task", "score": 0.032258 }, { "id": "update_task", "score": 0.030769 }, { "id": "add_task_comment", "score": 0.030579 }, { "id": "create_task_for_branch", "score": 0.02885 }, { "id": "claim_task_and_branch", "score": 0.028778 }, { "id": "search_projects", "score": 0.027973 }, { "id": "check_task_duplicates", "score": 0.027826 }, { "id": "manage_project", "score": 0.027425 } ] }, "retrievedTokens": { "bm25": 403, "semantic": 403, "hybrid": 403 } }, { "query": "plan my day and prioritize what to work on", "goldTop5": [ "plan_day_workflow", "get_daily_planning_context" ], "source": "kestral-live", "results": { "bm25": [ { "id": "plan_day_workflow", "score": 14.549468 }, { "id": "get_daily_planning_context", "score": 13.414358 }, { "id": "create_sprint", "score": 5.727487 }, { "id": "list_stale_tasks", "score": 3.786772 }, { "id": "end_day_review_workflow", "score": 3.745481 }, { "id": "claim_task_and_branch", "score": 2.937606 }, { "id": "list_calendar_events", "score": 2.661258 }, { "id": "get_daily_closeout_context", "score": 2.562011 }, { "id": "search_tasks_recently_updated", "score": 2.543044 }, { "id": "get_workload_distribution", "score": 2.496693 } ], "semantic": [ { "id": "plan_day_workflow", "score": 0.728498 }, { "id": "get_daily_planning_context", "score": 0.673375 }, { "id": "sort_tasks", "score": 0.658833 }, { "id": "update_task", "score": 0.64029 }, { "id": "create_task", "score": 0.630744 }, { "id": "create_sprint", "score": 0.629696 }, { "id": "list_my_active_tasks", "score": 0.629065 }, { "id": "end_day_review_workflow", "score": 0.627313 }, { "id": "set_calendar_reminder", "score": 0.614109 }, { "id": "list_tasks_by_priority", "score": 0.612222 } ], "hybrid": [ { "id": "plan_day_workflow", "score": 0.033333 }, { "id": "get_daily_planning_context", "score": 0.032787 }, { "id": "create_sprint", "score": 0.031514 }, { "id": "end_day_review_workflow", "score": 0.03055 }, { "id": "list_my_active_tasks", "score": 0.029437 }, { "id": "claim_task_and_branch", "score": 0.028043 }, { "id": "search_tasks_recently_updated", "score": 0.027526 }, { "id": "list_stale_tasks", "score": 0.026862 }, { "id": "get_daily_closeout_context", "score": 0.026036 }, { "id": "list_calendar_events", "score": 0.026021 } ] }, "retrievedTokens": { "bm25": 210, "semantic": 230, "hybrid": 210 } }, { "query": "mark a task as done", "goldTop5": [ "update_task" ], "source": "kestral-live", "results": { "bm25": [ { "id": "update_task", "score": 6.984727 }, { "id": "mark_notification_read", "score": 5.047772 }, { "id": "get_daily_closeout_context", "score": 4.363218 }, { "id": "end_day_review_workflow", "score": 4.320774 }, { "id": "create_tasks_batch", "score": 1.668265 }, { "id": "search_my_tasks_by_keyword", "score": 1.638963 }, { "id": "link_pr_to_task", "score": 1.634014 }, { "id": "add_task_comment", "score": 1.631011 }, { "id": "check_task_duplicates", "score": 1.615335 }, { "id": "register_branch_on_task", "score": 1.615335 } ], "semantic": [ { "id": "add_task_comment", "score": 0.719891 }, { "id": "edit_task_comment", "score": 0.701684 }, { "id": "update_task", "score": 0.683156 }, { "id": "complete_task_with_review", "score": 0.676587 }, { "id": "create_task", "score": 0.668378 }, { "id": "search_tasks_by_time", "score": 0.654669 }, { "id": "mark_notification_read", "score": 0.644681 }, { "id": "check_task_duplicates", "score": 0.643997 }, { "id": "search_tasks_with_comments", "score": 0.640929 }, { "id": "list_my_active_tasks", "score": 0.639023 } ], "hybrid": [ { "id": "update_task", "score": 0.032796 }, { "id": "add_task_comment", "score": 0.031592 }, { "id": "mark_notification_read", "score": 0.031545 }, { "id": "end_day_review_workflow", "score": 0.029958 }, { "id": "check_task_duplicates", "score": 0.029631 }, { "id": "complete_task_with_review", "score": 0.029572 }, { "id": "create_tasks_batch", "score": 0.028958 }, { "id": "search_tasks_with_comments", "score": 0.02879 }, { "id": "search_my_tasks_by_keyword", "score": 0.028372 }, { "id": "claim_task_and_branch", "score": 0.027588 } ] }, "retrievedTokens": { "bm25": 310, "semantic": 341, "hybrid": 322 } }, { "query": "leave a progress update comment on the task for my current branch", "goldTop5": [ "add_task_comment", "find_task_by_branch" ], "source": "kestral-live", "results": { "bm25": [ { "id": "add_task_comment", "score": 15.242415 }, { "id": "find_task_by_branch", "score": 8.339669 }, { "id": "register_branch_on_task", "score": 8.110106 }, { "id": "claim_task_and_branch", "score": 7.904187 }, { "id": "list_my_active_tasks", "score": 7.638011 }, { "id": "update_task", "score": 7.152776 }, { "id": "create_task_for_branch", "score": 5.69091 }, { "id": "delete_task_comment", "score": 5.51934 }, { "id": "search_tasks_with_comments", "score": 5.45519 }, { "id": "edit_task_comment", "score": 5.404408 } ], "semantic": [ { "id": "add_task_comment", "score": 0.776397 }, { "id": "claim_task_and_branch", "score": 0.75717 }, { "id": "update_task", "score": 0.742931 }, { "id": "link_pr_to_task", "score": 0.715575 }, { "id": "edit_task_comment", "score": 0.707744 }, { "id": "delete_task_comment", "score": 0.703722 }, { "id": "create_task_for_branch", "score": 0.693377 }, { "id": "register_branch_on_task", "score": 0.688841 }, { "id": "complete_task_with_review", "score": 0.681186 }, { "id": "find_task_by_branch", "score": 0.678148 } ], "hybrid": [ { "id": "add_task_comment", "score": 0.033333 }, { "id": "claim_task_and_branch", "score": 0.032266 }, { "id": "update_task", "score": 0.031514 }, { "id": "register_branch_on_task", "score": 0.031054 }, { "id": "find_task_by_branch", "score": 0.030886 }, { "id": "delete_task_comment", "score": 0.03031 }, { "id": "create_task_for_branch", "score": 0.030303 }, { "id": "link_pr_to_task", "score": 0.030159 }, { "id": "edit_task_comment", "score": 0.030118 }, { "id": "list_my_active_tasks", "score": 0.029139 } ] }, "retrievedTokens": { "bm25": 238, "semantic": 372, "hybrid": 372 } }, { "query": "link this pull request to its task", "goldTop5": [ "link_pr_to_task" ], "source": "kestral-live", "results": { "bm25": [ { "id": "link_pr_to_task", "score": 12.730273 }, { "id": "get_pull_request_status", "score": 8.122603 }, { "id": "list_open_pull_requests", "score": 8.081908 }, { "id": "find_task_by_branch", "score": 4.108042 }, { "id": "complete_task_with_review", "score": 3.956365 }, { "id": "create_task_for_branch", "score": 3.867998 }, { "id": "search_tasks", "score": 3.867998 }, { "id": "link_external_document", "score": 3.341391 }, { "id": "count_feedback", "score": 2.87453 }, { "id": "search_feedback", "score": 2.807718 } ], "semantic": [ { "id": "link_pr_to_task", "score": 0.775116 }, { "id": "link_external_document", "score": 0.712594 }, { "id": "link_document_to_project", "score": 0.696661 }, { "id": "complete_task_with_review", "score": 0.684236 }, { "id": "list_open_pull_requests", "score": 0.680026 }, { "id": "get_pull_request_status", "score": 0.677963 }, { "id": "create_task_for_branch", "score": 0.671398 }, { "id": "search_tasks_with_comments", "score": 0.667271 }, { "id": "add_task_comment", "score": 0.666917 }, { "id": "find_task_by_branch", "score": 0.665737 } ], "hybrid": [ { "id": "link_pr_to_task", "score": 0.033333 }, { "id": "get_pull_request_status", "score": 0.031778 }, { "id": "list_open_pull_requests", "score": 0.031754 }, { "id": "complete_task_with_review", "score": 0.031498 }, { "id": "link_external_document", "score": 0.031319 }, { "id": "create_task_for_branch", "score": 0.030536 }, { "id": "find_task_by_branch", "score": 0.030366 }, { "id": "link_document_to_project", "score": 0.030214 }, { "id": "search_tasks", "score": 0.02904 }, { "id": "add_task_comment", "score": 0.028219 } ] }, "retrievedTokens": { "bm25": 260, "semantic": 261, "hybrid": 260 } }, { "query": "complete the task for my branch", "goldTop5": [ "update_task", "complete_task_with_review", "find_task_by_branch" ], "source": "kestral-live", "results": { "bm25": [ { "id": "update_task", "score": 6.38694 }, { "id": "find_task_by_branch", "score": 5.919037 }, { "id": "register_branch_on_task", "score": 5.742068 }, { "id": "create_task_for_branch", "score": 5.69091 }, { "id": "complete_task_with_review", "score": 5.647087 }, { "id": "claim_task_and_branch", "score": 5.503339 }, { "id": "search_tasks_by_time", "score": 5.17397 }, { "id": "list_repository_branches", "score": 3.530267 }, { "id": "get_ci_build_status", "score": 3.459164 }, { "id": "get_sprint_summary", "score": 2.790999 } ], "semantic": [ { "id": "create_task_for_branch", "score": 0.744962 }, { "id": "claim_task_and_branch", "score": 0.736477 }, { "id": "find_task_by_branch", "score": 0.730112 }, { "id": "complete_task_with_review", "score": 0.713954 }, { "id": "register_branch_on_task", "score": 0.707054 }, { "id": "update_task", "score": 0.673621 }, { "id": "link_pr_to_task", "score": 0.627147 }, { "id": "end_day_review_workflow", "score": 0.623937 }, { "id": "list_my_active_tasks", "score": 0.618193 }, { "id": "add_task_comment", "score": 0.614251 } ], "hybrid": [ { "id": "create_task_for_branch", "score": 0.03254 }, { "id": "find_task_by_branch", "score": 0.032522 }, { "id": "update_task", "score": 0.032051 }, { "id": "claim_task_and_branch", "score": 0.031778 }, { "id": "register_branch_on_task", "score": 0.031754 }, { "id": "complete_task_with_review", "score": 0.031498 }, { "id": "get_sprint_summary", "score": 0.028577 }, { "id": "link_pr_to_task", "score": 0.028485 }, { "id": "search_tasks_by_time", "score": 0.028139 }, { "id": "add_task_comment", "score": 0.027651 } ] }, "retrievedTokens": { "bm25": 346, "semantic": 302, "hybrid": 389 } }, { "query": "search my tasks by keyword", "goldTop5": [ "search_my_tasks_by_keyword" ], "source": "kestral-live", "results": { "bm25": [ { "id": "search_my_tasks_by_keyword", "score": 10.961438 }, { "id": "search_tasks", "score": 4.214272 }, { "id": "search_tasks_with_comments", "score": 4.192751 }, { "id": "search_tasks_by_creator", "score": 3.429143 }, { "id": "search_tasks_recently_updated", "score": 3.369742 }, { "id": "search_tasks_by_time", "score": 3.312568 }, { "id": "search_projects", "score": 2.893722 }, { "id": "search_feedback", "score": 2.582146 }, { "id": "search_customers", "score": 2.569209 }, { "id": "search_document_chunks", "score": 2.470203 } ], "semantic": [ { "id": "search_my_tasks_by_keyword", "score": 0.873639 }, { "id": "search_tasks", "score": 0.76778 }, { "id": "search_tasks_by_time", "score": 0.753032 }, { "id": "search_tasks_with_comments", "score": 0.752168 }, { "id": "search_tasks_recently_updated", "score": 0.727744 }, { "id": "list_my_active_tasks", "score": 0.723047 }, { "id": "search_projects", "score": 0.71409 }, { "id": "search_tasks_by_creator", "score": 0.699643 }, { "id": "list_tasks_by_tag", "score": 0.687532 }, { "id": "sort_tasks", "score": 0.681137 } ], "hybrid": [ { "id": "search_my_tasks_by_keyword", "score": 0.033333 }, { "id": "search_tasks", "score": 0.032787 }, { "id": "search_tasks_with_comments", "score": 0.032002 }, { "id": "search_tasks_by_time", "score": 0.031514 }, { "id": "search_tasks_recently_updated", "score": 0.03125 }, { "id": "search_tasks_by_creator", "score": 0.030798 }, { "id": "search_projects", "score": 0.030303 }, { "id": "find_documents", "score": 0.027588 }, { "id": "search_document_chunks", "score": 0.027151 }, { "id": "list_my_active_tasks", "score": 0.027149 } ] }, "retrievedTokens": { "bm25": 247, "semantic": 288, "hybrid": 247 } }, { "query": "what am I working on right now", "goldTop5": [ "list_my_active_tasks" ], "source": "kestral-live", "results": { "bm25": [ { "id": "list_my_active_tasks", "score": 6.887891 }, { "id": "claim_task_and_branch", "score": 2.937606 }, { "id": "get_workload_distribution", "score": 2.496693 }, { "id": "generate_burndown_chart", "score": 2.457397 }, { "id": "get_cycle_time_stats", "score": 2.457397 }, { "id": "create_sprint", "score": 2.364366 }, { "id": "get_daily_planning_context", "score": 2.329096 }, { "id": "plan_day_workflow", "score": 2.311852 }, { "id": "complete_task_with_review", "score": 2.152494 } ], "semantic": [ { "id": "list_my_active_tasks", "score": 0.65723 }, { "id": "search_tasks_recently_updated", "score": 0.630614 }, { "id": "create_project", "score": 0.628721 }, { "id": "update_task", "score": 0.622613 }, { "id": "search_tasks_by_time", "score": 0.602394 }, { "id": "get_project_activity_feed", "score": 0.59763 }, { "id": "get_ci_build_status", "score": 0.597276 }, { "id": "get_project_pending_changes", "score": 0.591788 }, { "id": "create_task", "score": 0.587362 }, { "id": "get_project_brain", "score": 0.586405 } ], "hybrid": [ { "id": "list_my_active_tasks", "score": 0.033333 }, { "id": "claim_task_and_branch", "score": 0.029551 }, { "id": "plan_day_workflow", "score": 0.02901 }, { "id": "create_sprint", "score": 0.02773 }, { "id": "complete_task_with_review", "score": 0.027206 }, { "id": "get_daily_planning_context", "score": 0.026779 }, { "id": "generate_burndown_chart", "score": 0.025488 }, { "id": "get_workload_distribution", "score": 0.023763 }, { "id": "get_cycle_time_stats", "score": 0.023377 }, { "id": "search_tasks_recently_updated", "score": 0.016393 } ] }, "retrievedTokens": { "bm25": 208, "semantic": 216, "hybrid": 227 } }, { "query": "end of day review, what did I finish today", "goldTop5": [ "end_day_review_workflow", "get_daily_closeout_context" ], "source": "kestral-live", "results": { "bm25": [ { "id": "end_day_review_workflow", "score": 17.563652 }, { "id": "get_daily_closeout_context", "score": 16.391945 }, { "id": "complete_task_with_review", "score": 6.644857 }, { "id": "get_daily_planning_context", "score": 5.627342 }, { "id": "list_calendar_events", "score": 5.322516 }, { "id": "create_sprint", "score": 3.800738 }, { "id": "list_stale_tasks", "score": 3.786772 }, { "id": "plan_day_workflow", "score": 3.759144 }, { "id": "get_daily_brief", "score": 3.184076 }, { "id": "get_pull_request_status", "score": 2.978694 } ], "semantic": [ { "id": "end_day_review_workflow", "score": 0.767961 }, { "id": "get_daily_closeout_context", "score": 0.718509 }, { "id": "complete_task_with_review", "score": 0.668029 }, { "id": "get_daily_brief", "score": 0.652707 }, { "id": "get_daily_planning_context", "score": 0.616047 }, { "id": "plan_day_workflow", "score": 0.593682 }, { "id": "search_tasks_by_time", "score": 0.574053 }, { "id": "search_tasks_recently_updated", "score": 0.569633 }, { "id": "list_overdue_tasks", "score": 0.566135 }, { "id": "create_sprint", "score": 0.56253 } ], "hybrid": [ { "id": "end_day_review_workflow", "score": 0.033333 }, { "id": "get_daily_closeout_context", "score": 0.032787 }, { "id": "complete_task_with_review", "score": 0.032258 }, { "id": "get_daily_planning_context", "score": 0.031498 }, { "id": "get_daily_brief", "score": 0.030579 }, { "id": "plan_day_workflow", "score": 0.03031 }, { "id": "create_sprint", "score": 0.029877 }, { "id": "search_tasks_recently_updated", "score": 0.028624 }, { "id": "list_stale_tasks", "score": 0.028139 }, { "id": "get_pull_request_status", "score": 0.027151 } ] }, "retrievedTokens": { "bm25": 242, "semantic": 242, "hybrid": 242 } }, { "query": "assign this task to me and start working on it", "goldTop5": [ "claim_task_and_branch", "update_task" ], "source": "kestral-live", "results": { "bm25": [ { "id": "claim_task_and_branch", "score": 12.364825 }, { "id": "create_sprint", "score": 6.165104 }, { "id": "get_cycle_time_stats", "score": 5.160541 }, { "id": "link_pr_to_task", "score": 4.674805 }, { "id": "update_task", "score": 4.111688 }, { "id": "list_my_active_tasks", "score": 3.998288 }, { "id": "complete_task_with_review", "score": 3.74111 }, { "id": "schedule_meeting", "score": 3.731916 }, { "id": "get_daily_planning_context", "score": 3.331949 }, { "id": "plan_day_workflow", "score": 3.307281 } ], "semantic": [ { "id": "update_task", "score": 0.694133 }, { "id": "create_task", "score": 0.691895 }, { "id": "claim_task_and_branch", "score": 0.688046 }, { "id": "list_my_active_tasks", "score": 0.659452 }, { "id": "add_task_comment", "score": 0.649138 }, { "id": "setup_workflow", "score": 0.648945 }, { "id": "link_pr_to_task", "score": 0.646606 }, { "id": "search_my_tasks_by_keyword", "score": 0.63947 }, { "id": "create_tasks_batch", "score": 0.639073 }, { "id": "create_task_for_branch", "score": 0.638434 } ], "hybrid": [ { "id": "claim_task_and_branch", "score": 0.032796 }, { "id": "update_task", "score": 0.032292 }, { "id": "list_my_active_tasks", "score": 0.031258 }, { "id": "link_pr_to_task", "score": 0.031025 }, { "id": "create_sprint", "score": 0.030679 }, { "id": "setup_workflow", "score": 0.029274 }, { "id": "add_task_comment", "score": 0.028612 }, { "id": "plan_day_workflow", "score": 0.028191 }, { "id": "search_my_tasks_by_keyword", "score": 0.028083 }, { "id": "create_tasks_batch", "score": 0.028039 } ] }, "retrievedTokens": { "bm25": 243, "semantic": 446, "hybrid": 346 } }, { "query": "change the due date of a task", "goldTop5": [ "update_task" ], "source": "kestral-live", "results": { "bm25": [ { "id": "update_task", "score": 9.971552 }, { "id": "search_tasks_by_time", "score": 8.69059 }, { "id": "sort_tasks", "score": 7.676823 }, { "id": "create_task", "score": 7.549388 }, { "id": "create_milestone", "score": 7.185852 }, { "id": "list_overdue_tasks", "score": 6.767224 }, { "id": "get_project_pending_changes", "score": 4.261696 }, { "id": "search_tasks_recently_updated", "score": 3.892081 }, { "id": "reject_pending_change", "score": 3.658265 }, { "id": "approve_pending_change", "score": 3.644672 } ], "semantic": [ { "id": "update_task", "score": 0.734993 }, { "id": "list_overdue_tasks", "score": 0.726096 }, { "id": "search_tasks_by_time", "score": 0.725798 }, { "id": "create_milestone", "score": 0.6956 }, { "id": "create_task", "score": 0.683216 }, { "id": "search_tasks_recently_updated", "score": 0.662366 }, { "id": "sort_tasks", "score": 0.661868 }, { "id": "create_sprint", "score": 0.660779 }, { "id": "list_pending_change_details", "score": 0.641593 }, { "id": "edit_task_comment", "score": 0.638713 } ], "hybrid": [ { "id": "update_task", "score": 0.033333 }, { "id": "search_tasks_by_time", "score": 0.032522 }, { "id": "list_overdue_tasks", "score": 0.031778 }, { "id": "create_milestone", "score": 0.031498 }, { "id": "create_task", "score": 0.031498 }, { "id": "sort_tasks", "score": 0.031281 }, { "id": "search_tasks_recently_updated", "score": 0.03031 }, { "id": "create_sprint", "score": 0.02901 }, { "id": "approve_pending_change", "score": 0.028778 }, { "id": "get_project_pending_changes", "score": 0.028665 } ] }, "retrievedTokens": { "bm25": 393, "semantic": 343, "hybrid": 343 } }, { "query": "show overdue tasks", "goldTop5": [ "list_overdue_tasks" ], "source": "kestral-live", "results": { "bm25": [ { "id": "list_overdue_tasks", "score": 9.433943 }, { "id": "get_daily_planning_context", "score": 4.315806 }, { "id": "plan_day_workflow", "score": 4.283854 }, { "id": "mark_notification_read", "score": 3.31239 }, { "id": "get_daily_brief", "score": 3.184076 }, { "id": "get_document_content", "score": 3.088366 }, { "id": "create_tasks_batch", "score": 1.668265 }, { "id": "search_my_tasks_by_keyword", "score": 1.638963 }, { "id": "link_pr_to_task", "score": 1.634014 }, { "id": "add_task_comment", "score": 1.631011 } ], "semantic": [ { "id": "list_overdue_tasks", "score": 0.856753 }, { "id": "search_tasks_by_time", "score": 0.714079 }, { "id": "search_tasks_recently_updated", "score": 0.692586 }, { "id": "sort_tasks", "score": 0.68577 }, { "id": "list_my_active_tasks", "score": 0.67862 }, { "id": "create_task", "score": 0.67397 }, { "id": "list_stale_tasks", "score": 0.666958 }, { "id": "update_task", "score": 0.657325 }, { "id": "get_project_activity_feed", "score": 0.655829 }, { "id": "complete_task_with_review", "score": 0.654767 } ], "hybrid": [ { "id": "list_overdue_tasks", "score": 0.033333 }, { "id": "list_my_active_tasks", "score": 0.028446 }, { "id": "get_daily_planning_context", "score": 0.028298 }, { "id": "add_task_comment", "score": 0.028191 }, { "id": "mark_notification_read", "score": 0.027921 }, { "id": "plan_day_workflow", "score": 0.027894 }, { "id": "complete_task_with_review", "score": 0.027826 }, { "id": "check_task_duplicates", "score": 0.027799 }, { "id": "search_tasks_recently_updated", "score": 0.027623 }, { "id": "search_tasks_by_time", "score": 0.027505 } ] }, "retrievedTokens": { "bm25": 180, "semantic": 232, "hybrid": 165 } } ]