Spaces:
Running
Running
Devdan Schretlen
Claude Opus 5 (1M context)
Expand the calendar corpus to every faculty, and rewrite the UI copy
fcdba1d | { | |
| "description": "Hand-curated, corpus-grounded test set (authored directly from indexed chunks; regenerate candidates with eval/make_testset.py once a local LLM backend is available). Categories: code-lookup (hinges on a course code, BM25's job), multi-hop (spans chunks, the agent/reranker's job), collision (edition- or cohort-qualified, the corpus's trap), policy. gold_keys are the durable gold reference and survive a re-ingest; gold_ids are a positional cache that eval/retrieval_metrics.py:resolve_gold_ids overwrites at load time.", | |
| "items": [ | |
| { | |
| "id": "q001", | |
| "category": "code-lookup", | |
| "question": "What are the prerequisites for CPSC 221?", | |
| "reference_answer": "One of CPSC 210 or CPEN 221, and either one of CPSC 121, MATH 220 (or MATH_O 220), or a score of 68% or higher in MATH 226.", | |
| "gold_ids": [ | |
| 10, | |
| 182 | |
| ], | |
| "gold_keys": [ | |
| "live-2627:course-descriptions/subject/cpscv#cpsc-v-221-4-basic-algorithms-and-data-structures:1", | |
| "archive-2526:course-descriptions/subject/cpscv#cpsc-v-221-4-basic-algorithms-and-data-structures:1" | |
| ], | |
| "source_chunk_id": 10 | |
| }, | |
| { | |
| "id": "q002", | |
| "category": "code-lookup", | |
| "question": "What are the prerequisites for CPSC 320?", | |
| "reference_answer": "CPSC 221 (or DSCI 221) plus at least 3 credits of MATH or STAT at the 200 level or above (the 2025/26 edition also accepted COMM 291 or BIOL 300 for the second part).", | |
| "gold_ids": [ | |
| 24, | |
| 197 | |
| ], | |
| "gold_keys": [ | |
| "live-2627:course-descriptions/subject/cpscv#cpsc-v-320-3-intermediate-algorithm-design-and-analysis:1", | |
| "archive-2526:course-descriptions/subject/cpscv#cpsc-v-320-3-intermediate-algorithm-design-and-analysis:1" | |
| ], | |
| "source_chunk_id": 24 | |
| }, | |
| { | |
| "id": "q003", | |
| "category": "code-lookup", | |
| "question": "What are the prerequisites for CPSC 313?", | |
| "reference_answer": "CPSC 213, and either CPSC 221 or DSCI 221.", | |
| "gold_ids": [ | |
| 20, | |
| 193 | |
| ], | |
| "gold_keys": [ | |
| "live-2627:course-descriptions/subject/cpscv#cpsc-v-313-3-computer-hardware-and-operating-systems:1", | |
| "archive-2526:course-descriptions/subject/cpscv#cpsc-v-313-3-computer-hardware-and-operating-systems:1" | |
| ], | |
| "source_chunk_id": 20 | |
| }, | |
| { | |
| "id": "q004", | |
| "category": "code-lookup", | |
| "question": "Which courses satisfy the prerequisite for MATH 200, Calculus III?", | |
| "reference_answer": "One of MATH 101, MATH 103, MATH 105, MATH 121, SCIE 001, MATH_O 101, or MATH_O 103.", | |
| "gold_ids": [ | |
| 580, | |
| 2319 | |
| ], | |
| "gold_keys": [ | |
| "live-2627:course-descriptions/subject/mathv#math-v-200-3-calculus-iii:1", | |
| "archive-2526:course-descriptions/subject/mathv#math-v-200-3-calculus-iii:1" | |
| ], | |
| "source_chunk_id": 580 | |
| }, | |
| { | |
| "id": "q005", | |
| "category": "code-lookup", | |
| "question": "How many credits is CPSC 110 worth and what does it cover?", | |
| "reference_answer": "CPSC 110 (Computation, Programs, and Programming) is worth 4 credits and covers fundamental program and computation structures.", | |
| "gold_ids": [ | |
| 5, | |
| 177 | |
| ], | |
| "gold_keys": [ | |
| "live-2627:course-descriptions/subject/cpscv#cpsc-v-110-4-computation-programs-and-programming:1", | |
| "archive-2526:course-descriptions/subject/cpscv#cpsc-v-110-4-computation-programs-and-programming:1" | |
| ], | |
| "source_chunk_id": 5 | |
| }, | |
| { | |
| "id": "q006", | |
| "category": "multi-hop", | |
| "question": "Compare the B.A. degree requirements for students who entered the program in 2023/24 with those who entered in 2024/25 or later.", | |
| "reference_answer": "Students entering 2023/24 or earlier must complete seven requirements: Writing and Research, Language, Science, Literature, Outside, Upper-level, and Arts Credit Minimum. Students entering 2024/25 or later complete five: Writing and Research, Ways of Knowing Breadth, Outside, Upper-level, and Arts Credit Minimum — the separate Language, Science, and Literature requirements are folded into the Ways of Knowing Breadth requirement.", | |
| "gold_ids": [ | |
| 114, | |
| 128 | |
| ], | |
| "gold_keys": [ | |
| "live-2627:faculties-colleges-and-schools/faculty-arts/bachelor-arts/degree-requirements-students-who-enter-program-202324-or-earlier#body:1", | |
| "live-2627:faculties-colleges-and-schools/faculty-arts/bachelor-arts/degree-requirements-students-who-enter-program-202425-or-later#body:1" | |
| ], | |
| "source_chunk_id": 114 | |
| }, | |
| { | |
| "id": "q007", | |
| "category": "multi-hop", | |
| "question": "How does admission to the Bachelor of Computer Science program differ from B.Sc. admission from secondary school?", | |
| "reference_answer": "Bachelor of Computer Science admission is a competitive selection process evaluating applicants on academic and other criteria (not every qualified applicant is admitted); B.Sc. admission from secondary school requires starting first year in September of the Winter session of admission (January starts are not permitted).", | |
| "gold_ids": [ | |
| 1769, | |
| 1791 | |
| ], | |
| "gold_keys": [ | |
| "live-2627:faculties-colleges-and-schools/faculty-science/bachelor-computer-science#admission:1", | |
| "live-2627:faculties-colleges-and-schools/faculty-science/bachelor-science/admission-and-transfer#admission-from-secondary-school:1" | |
| ], | |
| "source_chunk_id": 1769 | |
| }, | |
| { | |
| "id": "q008", | |
| "category": "multi-hop", | |
| "question": "I have finished CPSC 210. What else do I need to complete before I can take CPSC 313?", | |
| "reference_answer": "CPSC 313 requires CPSC 213 and either CPSC 221 or DSCI 221. CPSC 221 in turn requires (beyond CPSC 210) one of CPSC 121, MATH 220, MATH_O 220, or a 68%+ score in MATH 226. So after CPSC 210 you still need CPSC 213 and CPSC 221 (with its discrete-math prerequisite).", | |
| "gold_ids": [ | |
| 20, | |
| 10 | |
| ], | |
| "gold_keys": [ | |
| "live-2627:course-descriptions/subject/cpscv#cpsc-v-313-3-computer-hardware-and-operating-systems:1", | |
| "live-2627:course-descriptions/subject/cpscv#cpsc-v-221-4-basic-algorithms-and-data-structures:1" | |
| ], | |
| "source_chunk_id": 20 | |
| }, | |
| { | |
| "id": "q009", | |
| "category": "collision", | |
| "question": "In the 2025/26 calendar, what is the minimum number of credits required for a B.Sc. degree?", | |
| "reference_answer": "A minimum of 120 credits (a major, double major, or General Science option requires at least 120 credits but may require more).", | |
| "gold_ids": [ | |
| 3508 | |
| ], | |
| "gold_keys": [ | |
| "archive-2526:faculties-colleges-and-schools/faculty-science/bachelor-science/general-degree-requirements#body:1" | |
| ], | |
| "source_chunk_id": 3508 | |
| }, | |
| { | |
| "id": "q010", | |
| "category": "collision", | |
| "question": "According to the current 2026/27 calendar, what is the minimum number of credits required for a B.Sc. degree?", | |
| "reference_answer": "A minimum of 120 credits (a major, double major, or General Science option requires at least 120 credits but may require more).", | |
| "gold_ids": [ | |
| 2023 | |
| ], | |
| "gold_keys": [ | |
| "live-2627:faculties-colleges-and-schools/faculty-science/bachelor-science/general-degree-requirements#body:1" | |
| ], | |
| "source_chunk_id": 2023 | |
| }, | |
| { | |
| "id": "q011", | |
| "category": "collision", | |
| "question": "Does a B.A. student who entered the program in 2024/25 or later have to complete a language requirement?", | |
| "reference_answer": "No. Their five degree requirements are Writing and Research, Ways of Knowing Breadth, Outside, Upper-level, and Arts Credit Minimum — there is no separate language requirement (that applies to students who entered in 2023/24 or earlier).", | |
| "gold_ids": [ | |
| 128 | |
| ], | |
| "gold_keys": [ | |
| "live-2627:faculties-colleges-and-schools/faculty-arts/bachelor-arts/degree-requirements-students-who-enter-program-202425-or-later#body:1" | |
| ], | |
| "source_chunk_id": 128 | |
| }, | |
| { | |
| "id": "q012", | |
| "category": "policy", | |
| "question": "What are the three levels of academic standing at UBC?", | |
| "reference_answer": "In Good Standing; On Academic Probation; and Failed, Required to Withdraw. All students are In Good Standing on initial entry to the University.", | |
| "gold_ids": [ | |
| 0 | |
| ], | |
| "gold_keys": [ | |
| "live-2627:campus-wide-policies-and-regulations/academic-standing#body:1" | |
| ], | |
| "source_chunk_id": 0 | |
| } | |
| ] | |
| } | |