File size: 13,048 Bytes
20e20c3 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 | {
"architecture": "Source1ForScoring",
"harness_version": "0.1.0",
"pooling": "mean",
"max_length": 8192,
"input_format": "classifier",
"layout": [
[
"format",
"label_single",
10
],
[
"topic",
"label_single",
15
],
[
"content_type",
"label_single",
4
],
[
"educational_value",
"score",
6
],
[
"reasoning_depth",
"score",
6
],
[
"writing_quality",
"score",
6
],
[
"information_density",
"score",
6
],
[
"reliability",
"score",
6
],
[
"spam_seo",
"score",
6
],
[
"boilerplate",
"score",
6
],
[
"toxicity",
"score",
6
],
[
"code_quality",
"score",
6
],
[
"math_quality",
"score",
6
]
],
"schema_fingerprint": "7c8ff0235cbc7416",
"layout_fingerprint": "3df647d7f4bd5ee0",
"schema": {
"name": "source-1",
"version": 1,
"labels": {
"format": {
"short": "fmt",
"kind": "single",
"description": "What kind of document this is.",
"values": {
"tutorial": "Step-by-step instruction, how-to guides, lessons, worked walkthroughs.",
"reference": "Documentation, manuals, API references, specifications, encyclopedia or wiki entries.",
"news": "News reporting, press releases, announcements of current events.",
"forum_qa": "Forum threads, Q&A pages, comment sections, mailing-list discussions.",
"academic": "Research papers, theses, textbooks, lecture notes, scholarly writing.",
"fiction": "Stories, novels, poetry, screenplays, fan fiction and other creative writing.",
"code_file": "A source-code file or notebook - code with or without comments.",
"product_page": "Product listings, shopping pages, marketing or landing pages, ads.",
"blog_opinion": "Personal blogs, essays, opinion pieces, newsletters, reviews.",
"other": "Anything else - legal text, transcripts, lists, records, fragments."
}
},
"topic": {
"short": "top",
"kind": "single",
"description": "The main subject of the document.",
"values": {
"science": "Natural sciences - physics, chemistry, biology, earth and space science.",
"technology": "Engineering, hardware, electronics, IT operations, applied technology.",
"programming": "Software development, computer science, code and developer tooling.",
"math": "Mathematics, statistics and formal logic.",
"health": "Medicine, health, nutrition, fitness and mental health.",
"finance": "Finance, economics, business, investing, accounting and careers.",
"history": "History, archaeology and historical biography.",
"politics_law": "Politics, government, law, public policy and current affairs.",
"society": "Social science, education, culture, relationships and psychology.",
"philosophy_religion": "Philosophy, ethics, religion and spirituality.",
"arts_entertainment": "Art, music, film, television, games, celebrities and pop culture.",
"literature": "Literature, language, linguistics, and writing about books and writing.",
"sports": "Sports and athletics.",
"lifestyle": "Food, travel, home, fashion, hobbies, parenting and pets.",
"other": "Anything else, or no clear subject."
}
},
"content_type": {
"short": "ctype",
"kind": "single",
"description": "What the text is made of.",
"values": {
"plain_text": "Prose or other natural-language text with no significant code or math.",
"text_with_code": "Natural-language text that includes code snippets or code blocks.",
"code_only": "Source code with, at most, comments and docstrings.",
"math_heavy": "Text dominated by equations, formulas, proofs or worked calculations."
}
}
},
"quality": {
"educational_value": {
"short": "edu",
"weight": 0.3,
"description": "Does it teach something useful?",
"rubric": {
"0": "Teaches nothing - spam, ads, navigation, gibberish, or text with no informational purpose.",
"1": "Almost nothing to learn - a few incidental facts buried in promotional, personal or trivial text.",
"2": "Some useful information, but superficial, fragmentary, or mixed with a lot of irrelevant material.",
"3": "Useful and coherent - conveys real knowledge or skills, though without much depth or completeness.",
"4": "Clearly educational - explains concepts, methods or knowledge well enough to learn from, with minor gaps.",
"5": "Outstanding teaching material - deep, clear and well structured, with examples or worked solutions; comparable to an excellent textbook or expert tutorial."
}
},
"reasoning_depth": {
"short": "rsn",
"weight": 0.2,
"description": "Does it explain why and walk through steps, or just state facts?",
"rubric": {
"0": "No reasoning at all - fragments, lists, boilerplate or non-content.",
"1": "Bare assertions or opinions with no explanation.",
"2": "Occasional explanation, but mostly unsupported statements; steps are skipped.",
"3": "Explains the why behind key points, with some step-by-step structure or argument.",
"4": "Consistent, explicit reasoning - derivations, cause and effect, justified arguments, worked examples.",
"5": "Rigorous multi-step reasoning throughout - proofs, careful derivations, or thorough analysis that motivates every step and weighs alternatives."
}
},
"writing_quality": {
"short": "wrt",
"weight": 0.15,
"description": "Is it clear, well organized and coherent? For code, judge naming, structure and comments.",
"rubric": {
"0": "Unreadable - garbled, machine-generated nonsense, broken encoding, keyword soup.",
"1": "Very poor - frequent errors, incoherent structure, fragments; hard to follow.",
"2": "Below average - understandable but disorganized, repetitive or error-prone.",
"3": "Adequate - clear and coherent with minor issues.",
"4": "Good - well organized, fluent, precise and sensibly structured.",
"5": "Excellent - exemplary, publication-quality writing."
}
},
"information_density": {
"short": "dns",
"weight": 0.2,
"description": "How much real content per word, versus padding and filler?",
"rubric": {
"0": "No real content - filler, repetition, boilerplate.",
"1": "Mostly padding - long intros, fluff, repeated phrases or SEO filler around a little content.",
"2": "Noticeable padding or digressions; it could be far shorter without losing anything.",
"3": "Reasonable - mostly on point, with some filler.",
"4": "Dense - little wasted text; most sentences carry information.",
"5": "Very dense yet readable - every sentence adds substance."
}
},
"reliability": {
"short": "rel",
"weight": 0.15,
"description": "Does it look careful and trustworthy, or sloppy and made up? For fiction and other creative writing, judge care and internal consistency; invented events are not a reliability problem.",
"rubric": {
"0": "Fabricated, nonsensical or deceptive - scams, fake news, conspiracy content, generated nonsense.",
"1": "Largely unreliable - many errors, sensational or unsupported claims, misleading framing.",
"2": "Questionable - some errors or unsupported claims; careless.",
"3": "Generally plausible and consistent; no obvious errors, but informal or unverifiable.",
"4": "Careful and accurate - precise, consistent claims; shows its work or cites sources.",
"5": "Authoritative - expert-level accuracy, well sourced, carefully qualified claims."
}
}
},
"red_flags": {
"spam_seo": {
"short": "spam",
"description": "Keyword stuffing, clickbait, affiliate filler, and other text written to rank or sell rather than to inform.",
"penalty": {
"threshold": 1,
"weight": 0.5
},
"rubric": {
"0": "None.",
"1": "Minor promotion - a call to action or a brief ad in otherwise genuine content.",
"2": "Noticeable promotion - repeated calls to action, affiliate links, marketing tone.",
"3": "Substantial - the text mainly exists to promote, sell or rank; visible keyword repetition.",
"4": "Mostly spam - keyword stuffing, clickbait, thin affiliate or doorway content.",
"5": "Pure spam - auto-generated SEO text, scams, keyword lists."
}
},
"boilerplate": {
"short": "boil",
"description": "Templates, auto-generated pages, lists of links, cookie banners, menus and other non-content text.",
"penalty": {
"threshold": 1,
"weight": 0.4
},
"rubric": {
"0": "None - all of the text is real content.",
"1": "A little - a stray header, footer or copyright line.",
"2": "Noticeable - navigation, cookie or legal text, or share-button text mixed in (roughly 10-25% of the text).",
"3": "Substantial - roughly 25-50% templates, menus, link lists or legal notices.",
"4": "Mostly boilerplate - over half of the text is templated or navigational.",
"5": "Entirely boilerplate - auto-generated pages, link lists, cookie banners, error or index pages."
}
},
"toxicity": {
"short": "tox",
"aggregate": "max",
"description": "Hate, harassment and explicit content. Rate the text's own toxicity, not its subject: scholarly, journalistic or educational discussion of hate, violence or sexuality is not toxic.",
"penalty": {
"threshold": 1,
"weight": 0.8
},
"rubric": {
"0": "None.",
"1": "Mild - occasional profanity or rudeness, not aimed at anyone.",
"2": "Moderate - insults, crude humor, mild sexual references, heated hostility.",
"3": "Significant - harassment, demeaning stereotypes, graphic violence, or partly explicit sexual content.",
"4": "Severe - hate speech, targeted harassment, or sexually explicit material as the main content.",
"5": "Extreme - violent extremism, dehumanizing hate, incitement, or sexual content involving minors."
}
}
},
"gated": {
"code_quality": {
"short": "code",
"description": "Is the code readable, correct-looking and documented, and written by a person rather than generated?",
"applies_when": {
"content_type": [
"text_with_code",
"code_only"
],
"format": [
"code_file"
]
},
"rubric": {
"0": "Not usable code - garbled, minified, obfuscated or broken beyond repair.",
"1": "Very poor - likely non-functional fragments, no structure, or auto-generated boilerplate.",
"2": "Poor - may work but messy; unclear names, no comments, hard-coded values.",
"3": "Acceptable - readable and plausibly correct, with some structure and minimal documentation.",
"4": "Good - clean, idiomatic, well named and documented; plausibly correct, with edge cases handled.",
"5": "Excellent - exemplary, production-quality, well-documented and instructive code."
}
},
"math_quality": {
"short": "math",
"description": "Is the notation correct, are the steps shown, and do the solutions follow logically?",
"applies_when": {
"content_type": [
"math_heavy"
],
"topic": [
"math"
]
},
"rubric": {
"0": "Garbled math - broken notation or nonsense.",
"1": "Mostly wrong or incoherent; results asserted without work.",
"2": "Some correct math, but errors, sloppy notation, or skipped steps that break the logic.",
"3": "Generally correct, with standard notation and the key steps shown.",
"4": "Correct, clean notation, complete steps and a clear logical flow.",
"5": "Rigorous and elegant - precise notation, every step justified, solutions easy to verify."
}
}
},
"composite": {
"gated_weight": 0.2,
"hard_filters": [
"toxicity >= 4",
"spam_seo >= 4",
"boilerplate >= 4.5"
]
}
},
"text_normalize": "collapse_spaces"
} |