| { |
| "architecture": "Source1ForScoring", |
| "harness_version": "0.1.0", |
| "pooling": "mean", |
| "max_length": 8192, |
| "input_format": "classifier", |
| "layout": [ |
| [ |
| "format", |
| "label_single", |
| 10 |
| ], |
| [ |
| "topic", |
| "label_single", |
| 15 |
| ], |
| [ |
| "content_type", |
| "label_single", |
| 4 |
| ], |
| [ |
| "educational_value", |
| "score", |
| 6 |
| ], |
| [ |
| "reasoning_depth", |
| "score", |
| 6 |
| ], |
| [ |
| "writing_quality", |
| "score", |
| 6 |
| ], |
| [ |
| "information_density", |
| "score", |
| 6 |
| ], |
| [ |
| "reliability", |
| "score", |
| 6 |
| ], |
| [ |
| "spam_seo", |
| "score", |
| 6 |
| ], |
| [ |
| "boilerplate", |
| "score", |
| 6 |
| ], |
| [ |
| "toxicity", |
| "score", |
| 6 |
| ], |
| [ |
| "code_quality", |
| "score", |
| 6 |
| ], |
| [ |
| "math_quality", |
| "score", |
| 6 |
| ] |
| ], |
| "schema_fingerprint": "7c8ff0235cbc7416", |
| "layout_fingerprint": "3df647d7f4bd5ee0", |
| "schema": { |
| "name": "source-1", |
| "version": 1, |
| "labels": { |
| "format": { |
| "short": "fmt", |
| "kind": "single", |
| "description": "What kind of document this is.", |
| "values": { |
| "tutorial": "Step-by-step instruction, how-to guides, lessons, worked walkthroughs.", |
| "reference": "Documentation, manuals, API references, specifications, encyclopedia or wiki entries.", |
| "news": "News reporting, press releases, announcements of current events.", |
| "forum_qa": "Forum threads, Q&A pages, comment sections, mailing-list discussions.", |
| "academic": "Research papers, theses, textbooks, lecture notes, scholarly writing.", |
| "fiction": "Stories, novels, poetry, screenplays, fan fiction and other creative writing.", |
| "code_file": "A source-code file or notebook - code with or without comments.", |
| "product_page": "Product listings, shopping pages, marketing or landing pages, ads.", |
| "blog_opinion": "Personal blogs, essays, opinion pieces, newsletters, reviews.", |
| "other": "Anything else - legal text, transcripts, lists, records, fragments." |
| } |
| }, |
| "topic": { |
| "short": "top", |
| "kind": "single", |
| "description": "The main subject of the document.", |
| "values": { |
| "science": "Natural sciences - physics, chemistry, biology, earth and space science.", |
| "technology": "Engineering, hardware, electronics, IT operations, applied technology.", |
| "programming": "Software development, computer science, code and developer tooling.", |
| "math": "Mathematics, statistics and formal logic.", |
| "health": "Medicine, health, nutrition, fitness and mental health.", |
| "finance": "Finance, economics, business, investing, accounting and careers.", |
| "history": "History, archaeology and historical biography.", |
| "politics_law": "Politics, government, law, public policy and current affairs.", |
| "society": "Social science, education, culture, relationships and psychology.", |
| "philosophy_religion": "Philosophy, ethics, religion and spirituality.", |
| "arts_entertainment": "Art, music, film, television, games, celebrities and pop culture.", |
| "literature": "Literature, language, linguistics, and writing about books and writing.", |
| "sports": "Sports and athletics.", |
| "lifestyle": "Food, travel, home, fashion, hobbies, parenting and pets.", |
| "other": "Anything else, or no clear subject." |
| } |
| }, |
| "content_type": { |
| "short": "ctype", |
| "kind": "single", |
| "description": "What the text is made of.", |
| "values": { |
| "plain_text": "Prose or other natural-language text with no significant code or math.", |
| "text_with_code": "Natural-language text that includes code snippets or code blocks.", |
| "code_only": "Source code with, at most, comments and docstrings.", |
| "math_heavy": "Text dominated by equations, formulas, proofs or worked calculations." |
| } |
| } |
| }, |
| "quality": { |
| "educational_value": { |
| "short": "edu", |
| "weight": 0.3, |
| "description": "Does it teach something useful?", |
| "rubric": { |
| "0": "Teaches nothing - spam, ads, navigation, gibberish, or text with no informational purpose.", |
| "1": "Almost nothing to learn - a few incidental facts buried in promotional, personal or trivial text.", |
| "2": "Some useful information, but superficial, fragmentary, or mixed with a lot of irrelevant material.", |
| "3": "Useful and coherent - conveys real knowledge or skills, though without much depth or completeness.", |
| "4": "Clearly educational - explains concepts, methods or knowledge well enough to learn from, with minor gaps.", |
| "5": "Outstanding teaching material - deep, clear and well structured, with examples or worked solutions; comparable to an excellent textbook or expert tutorial." |
| } |
| }, |
| "reasoning_depth": { |
| "short": "rsn", |
| "weight": 0.2, |
| "description": "Does it explain why and walk through steps, or just state facts?", |
| "rubric": { |
| "0": "No reasoning at all - fragments, lists, boilerplate or non-content.", |
| "1": "Bare assertions or opinions with no explanation.", |
| "2": "Occasional explanation, but mostly unsupported statements; steps are skipped.", |
| "3": "Explains the why behind key points, with some step-by-step structure or argument.", |
| "4": "Consistent, explicit reasoning - derivations, cause and effect, justified arguments, worked examples.", |
| "5": "Rigorous multi-step reasoning throughout - proofs, careful derivations, or thorough analysis that motivates every step and weighs alternatives." |
| } |
| }, |
| "writing_quality": { |
| "short": "wrt", |
| "weight": 0.15, |
| "description": "Is it clear, well organized and coherent? For code, judge naming, structure and comments.", |
| "rubric": { |
| "0": "Unreadable - garbled, machine-generated nonsense, broken encoding, keyword soup.", |
| "1": "Very poor - frequent errors, incoherent structure, fragments; hard to follow.", |
| "2": "Below average - understandable but disorganized, repetitive or error-prone.", |
| "3": "Adequate - clear and coherent with minor issues.", |
| "4": "Good - well organized, fluent, precise and sensibly structured.", |
| "5": "Excellent - exemplary, publication-quality writing." |
| } |
| }, |
| "information_density": { |
| "short": "dns", |
| "weight": 0.2, |
| "description": "How much real content per word, versus padding and filler?", |
| "rubric": { |
| "0": "No real content - filler, repetition, boilerplate.", |
| "1": "Mostly padding - long intros, fluff, repeated phrases or SEO filler around a little content.", |
| "2": "Noticeable padding or digressions; it could be far shorter without losing anything.", |
| "3": "Reasonable - mostly on point, with some filler.", |
| "4": "Dense - little wasted text; most sentences carry information.", |
| "5": "Very dense yet readable - every sentence adds substance." |
| } |
| }, |
| "reliability": { |
| "short": "rel", |
| "weight": 0.15, |
| "description": "Does it look careful and trustworthy, or sloppy and made up? For fiction and other creative writing, judge care and internal consistency; invented events are not a reliability problem.", |
| "rubric": { |
| "0": "Fabricated, nonsensical or deceptive - scams, fake news, conspiracy content, generated nonsense.", |
| "1": "Largely unreliable - many errors, sensational or unsupported claims, misleading framing.", |
| "2": "Questionable - some errors or unsupported claims; careless.", |
| "3": "Generally plausible and consistent; no obvious errors, but informal or unverifiable.", |
| "4": "Careful and accurate - precise, consistent claims; shows its work or cites sources.", |
| "5": "Authoritative - expert-level accuracy, well sourced, carefully qualified claims." |
| } |
| } |
| }, |
| "red_flags": { |
| "spam_seo": { |
| "short": "spam", |
| "description": "Keyword stuffing, clickbait, affiliate filler, and other text written to rank or sell rather than to inform.", |
| "penalty": { |
| "threshold": 1, |
| "weight": 0.5 |
| }, |
| "rubric": { |
| "0": "None.", |
| "1": "Minor promotion - a call to action or a brief ad in otherwise genuine content.", |
| "2": "Noticeable promotion - repeated calls to action, affiliate links, marketing tone.", |
| "3": "Substantial - the text mainly exists to promote, sell or rank; visible keyword repetition.", |
| "4": "Mostly spam - keyword stuffing, clickbait, thin affiliate or doorway content.", |
| "5": "Pure spam - auto-generated SEO text, scams, keyword lists." |
| } |
| }, |
| "boilerplate": { |
| "short": "boil", |
| "description": "Templates, auto-generated pages, lists of links, cookie banners, menus and other non-content text.", |
| "penalty": { |
| "threshold": 1, |
| "weight": 0.4 |
| }, |
| "rubric": { |
| "0": "None - all of the text is real content.", |
| "1": "A little - a stray header, footer or copyright line.", |
| "2": "Noticeable - navigation, cookie or legal text, or share-button text mixed in (roughly 10-25% of the text).", |
| "3": "Substantial - roughly 25-50% templates, menus, link lists or legal notices.", |
| "4": "Mostly boilerplate - over half of the text is templated or navigational.", |
| "5": "Entirely boilerplate - auto-generated pages, link lists, cookie banners, error or index pages." |
| } |
| }, |
| "toxicity": { |
| "short": "tox", |
| "aggregate": "max", |
| "description": "Hate, harassment and explicit content. Rate the text's own toxicity, not its subject: scholarly, journalistic or educational discussion of hate, violence or sexuality is not toxic.", |
| "penalty": { |
| "threshold": 1, |
| "weight": 0.8 |
| }, |
| "rubric": { |
| "0": "None.", |
| "1": "Mild - occasional profanity or rudeness, not aimed at anyone.", |
| "2": "Moderate - insults, crude humor, mild sexual references, heated hostility.", |
| "3": "Significant - harassment, demeaning stereotypes, graphic violence, or partly explicit sexual content.", |
| "4": "Severe - hate speech, targeted harassment, or sexually explicit material as the main content.", |
| "5": "Extreme - violent extremism, dehumanizing hate, incitement, or sexual content involving minors." |
| } |
| } |
| }, |
| "gated": { |
| "code_quality": { |
| "short": "code", |
| "description": "Is the code readable, correct-looking and documented, and written by a person rather than generated?", |
| "applies_when": { |
| "content_type": [ |
| "text_with_code", |
| "code_only" |
| ], |
| "format": [ |
| "code_file" |
| ] |
| }, |
| "rubric": { |
| "0": "Not usable code - garbled, minified, obfuscated or broken beyond repair.", |
| "1": "Very poor - likely non-functional fragments, no structure, or auto-generated boilerplate.", |
| "2": "Poor - may work but messy; unclear names, no comments, hard-coded values.", |
| "3": "Acceptable - readable and plausibly correct, with some structure and minimal documentation.", |
| "4": "Good - clean, idiomatic, well named and documented; plausibly correct, with edge cases handled.", |
| "5": "Excellent - exemplary, production-quality, well-documented and instructive code." |
| } |
| }, |
| "math_quality": { |
| "short": "math", |
| "description": "Is the notation correct, are the steps shown, and do the solutions follow logically?", |
| "applies_when": { |
| "content_type": [ |
| "math_heavy" |
| ], |
| "topic": [ |
| "math" |
| ] |
| }, |
| "rubric": { |
| "0": "Garbled math - broken notation or nonsense.", |
| "1": "Mostly wrong or incoherent; results asserted without work.", |
| "2": "Some correct math, but errors, sloppy notation, or skipped steps that break the logic.", |
| "3": "Generally correct, with standard notation and the key steps shown.", |
| "4": "Correct, clean notation, complete steps and a clear logical flow.", |
| "5": "Rigorous and elegant - precise notation, every step justified, solutions easy to verify." |
| } |
| } |
| }, |
| "composite": { |
| "gated_weight": 0.2, |
| "hard_filters": [ |
| "toxicity >= 4", |
| "spam_seo >= 4", |
| "boilerplate >= 4.5" |
| ] |
| } |
| }, |
| "text_normalize": "collapse_spaces" |
| } |
