Numo-1-E2B / task-contracts.json
alphintom's picture
Numo-1-E2B v1 (numo-1-e2b-v1)
a2c0567 verified
Raw History Blame Contribute Delete
5.71 kB
{
"tasks": {
"consensus4": {
"candidate_order": "fixed_ascending_semantic_order",
"candidates": {
"0": "Mean support code in [0.00, 0.25)",
"1": "Mean support code in [0.25, 0.50)",
"2": "Mean support code in [0.50, 0.75)",
"3": "Mean support code in [0.75, 1.00]"
},
"default_candidate_ids": [
"0",
"1",
"2",
"3"
],
"input_scope": "English evidence and hypothesis",
"kind": "score",
"question": "Estimate the mean human support code for this evidence and hypothesis: no support = 0, partial support = 0.5, full support = 1. Choose the stated interval. This is coded annotator consensus.",
"semantic_values": {
"0": 0,
"1": 1,
"2": 2,
"3": 3
}
},
"consensus5": {
"candidate_order": "fixed_ascending_semantic_order",
"candidates": {
"0": "Mean support code in [0.00, 0.20)",
"1": "Mean support code in [0.20, 0.40)",
"2": "Mean support code in [0.40, 0.60)",
"3": "Mean support code in [0.60, 0.80)",
"4": "Mean support code in [0.80, 1.00]"
},
"default_candidate_ids": [
"0",
"1",
"2",
"3",
"4"
],
"input_scope": "English evidence and hypothesis",
"kind": "score",
"question": "Estimate the mean human support code for this evidence and hypothesis: no support = 0, partial support = 0.5, full support = 1. Choose the stated interval. This is coded annotator consensus.",
"semantic_values": {
"0": 0,
"1": 1,
"2": 2,
"3": 3,
"4": 4
}
},
"scientific_field": {
"candidate_order": "caller_supplied_subset_2_to_8",
"candidates": {
"cs": "Computer science",
"econ": "Economics",
"eess": "Electrical engineering and systems science",
"math": "Mathematics",
"physics": "Physics and astronomy",
"q-bio": "Quantitative biology",
"q-fin": "Quantitative finance",
"stat": "Statistics"
},
"default_candidate_ids": [
"cs",
"econ",
"eess",
"math",
"physics",
"q-bio",
"q-fin",
"stat"
],
"input_scope": "English scientific title and abstract",
"kind": "choice",
"question": "Which listed broad scientific field best matches this paper?"
},
"support_boolean": {
"candidate_order": "fixed_ascending_semantic_order",
"candidates": {
"0": "The evidence does not fully support the hypothesis.",
"1": "The evidence fully supports the hypothesis."
},
"default_candidate_ids": [
"0",
"1"
],
"input_scope": "English evidence and hypothesis",
"kind": "noul",
"question": "Does the supplied evidence fully support the hypothesis? Partial support is not full support.",
"semantic_values": {
"0": false,
"1": true
}
},
"support_grade3": {
"candidate_order": "fixed_ascending_semantic_order",
"candidates": {
"0": "None: the evidence provides no support.",
"1": "Partial: the evidence supports part but not all of the hypothesis.",
"2": "Full: the evidence supports the complete hypothesis."
},
"default_candidate_ids": [
"0",
"1",
"2"
],
"input_scope": "English evidence and hypothesis",
"kind": "score",
"question": "Grade how much the supplied evidence supports the hypothesis.",
"semantic_values": {
"0": 0,
"1": 1,
"2": 2
}
},
"research_area": {
"kind": "choice",
"question": "Which research area best describes the main scientific question or contribution? Choose the primary area, not a technique merely used. Patient health, clinical care and public health belong to Medicine; biological mechanisms, organisms and ecology belong to Biology; physical phenomena and astronomy belong to Physics; general algorithms, software, hardware and computing systems belong to Computer science. If none of these four areas describes the primary question, choose Other research area outside these four. Do not force a neighboring area merely because the paper uses its tools, studies its practitioners, or mentions it as context.",
"candidates": {
"0": "Medicine and health",
"1": "Biology",
"2": "Physics and astronomy",
"3": "Computer science",
"4": "Other research area outside these four"
},
"default_candidate_ids": [
"0",
"1",
"2",
"3",
"4"
]
},
"study_approach": {
"kind": "choice",
"question": "Classify the main completed approach, not background. Human experiments assign a treatment, task or behavioral condition to living people. Observational studies analyze observed clinical, field or astronomical data. Non-human experiments manipulate animals, cells or physical systems. Choose computational/theoretical for a main model, algorithm, simulation or proof, even when tested on real data. Reviews synthesize prior work; protocols have no completed results.",
"candidates": {
"0": "Human experiment or intervention",
"1": "Observational empirical study",
"2": "Non-human controlled experiment",
"3": "Computational, mathematical or theoretical study",
"4": "Evidence synthesis or review",
"5": "Protocol, proposal or unclear study approach"
},
"default_candidate_ids": [
"0",
"1",
"2",
"3",
"4",
"5"
]
},
"biomedical_relevance": {
"kind": "score",
"candidate_order": "fixed_ascending_semantic_order",
"question": "How relevant is the candidate paper to the seed paper?",
"candidates": {
"0": "Irrelevant",
"1": "Partially relevant",
"2": "Relevant"
},
"default_candidate_ids": [
"0",
"1",
"2"
],
"semantic_values": {
"0": 0,
"1": 1,
"2": 2
},
"input_scope": "English 'Seed paper:' and 'Candidate paper:' titles and abstracts"
}
}
}