Step 1: package scaffold, classifier, constraints, metric modules

- schemas: EvalRequest/EvalReport contracts with is_proxy/requires_judge
honesty flags baked into the type layer
- classify: rule votes + bge prototype referee + instruction/source split
- constraints: IFEval-style extraction and deterministic checkers
- metrics: readability (Flesch/FK bands), lexical (MTLD, cliches, rhythm),
relevance (bge, per-paragraph drift), faithfulness (SummaC-ZS over
nli-deberta-v3-small), hallucination risk signals, static code ladder

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

pyproject.toml

@@ -0,0 +1,24 @@
+[project]
+name = "rubriq"
+version = "0.1.0"
+description = "Score any LLM's output locally - no API calls - and get research-grounded prompts that fix what failed."
+requires-python = ">=3.11"
+dependencies = [
+    "fastapi>=0.110",
+    "uvicorn>=0.29",
+    "pydantic>=2.6",
+    "numpy>=1.26",
+    "sentence-transformers>=3.0",
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=8.0",
+    "httpx>=0.27",
+]
+
+[tool.setuptools.packages.find]
+include = ["rubriq*"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]

rubriq/__init__.py

@@ -0,0 +1,3 @@
+"""Rubriq: local, research-grounded LLM output quality evaluation."""
+
+__version__ = "0.1.0"

rubriq/classify.py

@@ -0,0 +1,230 @@
+"""Task classification: rule signals first, embedding prototypes as referee.
+
+Design (RESEARCH.md 1.8 adaptation + standard practice):
+- High-precision regex/structure rules vote on task type. A confident rule
+  vote decides directly — "Summarize the following" needs no neural network.
+- When rules disagree or stay silent, the instruction is embedded and
+  compared against prototype sentences per task type (bge-small).
+- Separately, the prompt is split into instruction vs. pasted source
+  material; source presence turns QA into QA_GROUNDED and unlocks the
+  NLI faithfulness parameter.
+
+Every fired signal is recorded and surfaced in the UI, so a misclassification
+is diagnosable by the user (and overridable in a later UI iteration).
+"""
+
+from __future__ import annotations
+
+import re
+
+from .models import get_embedder
+from .schemas import TaskClassification, TaskType
+from .textutils import word_count
+
+# ---------------------------------------------------------------- source split
+
+_DELIM_BLOCK = re.compile(
+    r"(?s)(?:\"{3}|'{3}|`{3}|<{2,}|={3,}|-{3,}|###)\s*\n?(.+?)(?:\"{3}|'{3}|`{3}|>{2,}|={3,}|-{3,}|###|\Z)"
+)
+_SOURCE_INTRO = re.compile(
+    r"(?i)(?:following (?:text|article|document|passage|email|report|code|excerpt|transcript|paragraph)s?|"
+    r"text below|below is|here is the (?:text|article|document|passage|code)|this (?:text|article|document))\s*[:.]?\s*\n"
+)
+
+
+def split_instruction_and_source(prompt: str) -> tuple[str, str | None]:
+    """Separate the user's instruction from pasted source material.
+
+    Order of attempts: explicit delimiter blocks, then "the following text:"
+    intros, then a length heuristic (one short leading/trailing instruction
+    line attached to a long body). Conservative: when unsure, the whole
+    prompt is instruction and no faithfulness checking happens — a false
+    source is worse than a missed one because it would generate false
+    unfaithfulness findings.
+    """
+    m = _DELIM_BLOCK.search(prompt)
+    if m and word_count(m.group(1)) >= 40:
+        source = m.group(1).strip()
+        instruction = (prompt[:m.start()] + " " + prompt[m.end():]).strip()
+        if instruction:
+            return instruction, source
+
+    m = _SOURCE_INTRO.search(prompt)
+    if m:
+        source = prompt[m.end():].strip()
+        instruction = prompt[:m.end()].strip()
+        if word_count(source) >= 40:
+            return instruction, source
+
+    # Length heuristic: short first paragraph, long remainder.
+    parts = prompt.split("\n", 1)
+    if len(parts) == 2:
+        head, rest = parts[0].strip(), parts[1].strip()
+        if head and word_count(head) <= 60 and word_count(rest) >= 150 and _looks_instructional(head):
+            return head, rest
+
+    return prompt.strip(), None
+
+
+def _looks_instructional(line: str) -> bool:
+    return bool(re.search(
+        r"(?i)\b(summari[sz]e|rewrite|translate|extract|answer|explain|list|analy[sz]e|"
+        r"convert|classify|describe|edit|fix|review|compare|based on|according to)\b",
+        line,
+    )) or line.rstrip().endswith("?")
+
+
+# ---------------------------------------------------------------- rule signals
+
+_RULES: list[tuple[TaskType, str, re.Pattern]] = [
+    (TaskType.SUMMARIZATION, "prompt asks to summarize",
+     re.compile(r"(?i)\b(summari[sz]e|summary|tl;?dr|condense|key points? of|main points? of|abstract of)\b")),
+    (TaskType.CODE_GENERATION, "prompt asks for code",
+     re.compile(r"(?i)\b(write|create|implement|generate|fix|debug|refactor)\b.{0,60}\b(function|class|script|code|program|method|module|regex|query|endpoint|component)\b")),
+    (TaskType.CODE_GENERATION, "prompt names a programming language",
+     re.compile(r"(?i)\bin (python|javascript|typescript|java|c\+\+|c#|go|rust|sql|html|css|bash|powershell)\b")),
+    (TaskType.CREATIVE_WRITING, "prompt asks for creative writing",
+     re.compile(r"(?i)\b(write|compose|craft)\b.{0,50}\b(story|poem|song|haiku|novel|fiction|tale|screenplay|lyrics|limerick)\b")),
+    (TaskType.EXTRACTION, "prompt asks to extract structured data",
+     re.compile(r"(?i)\b(extract|pull out|identify all|list (?:all|every))\b.{0,80}\b(names?|dates?|entities|emails?|numbers?|fields?|values?|keywords?|items)\b")),
+    (TaskType.EXTRACTION, "prompt requests JSON/CSV/table output from text",
+     re.compile(r"(?i)\b(as|in|into|to) (?:a |valid )?(json|csv|yaml|xml|table)\b")),
+    (TaskType.REASONING, "prompt poses a math/logic problem",
+     re.compile(r"(?i)\b(calculate|compute|solve|prove|how many|what is the (?:probability|total|sum|difference|product)|step[- ]by[- ]step|logic puzzle|riddle)\b")),
+    (TaskType.REWRITING, "prompt asks to rewrite/edit existing text",
+     re.compile(r"(?i)\b(rewrite|rephrase|paraphrase|edit|proofread|improve|polish|make (?:this|it) (?:more|less|sound)|change the tone|translate)\b")),
+    (TaskType.QA_OPEN, "prompt asks a direct question",
+     re.compile(r"(?i)^(?:please\s+)?(what|who|when|where|why|how|which|is|are|does|do|can|could|should|explain)\b")),
+]
+
+_CODE_FENCE = re.compile(r"```")
+
+
+def _rule_votes(instruction: str, output: str) -> list[tuple[TaskType, str]]:
+    votes: list[tuple[TaskType, str]] = []
+    for task, why, pat in _RULES:
+        if pat.search(instruction):
+            votes.append((task, why))
+    if _CODE_FENCE.search(output) and word_count(re.sub(r"```.*?```", "", output, flags=re.S)) < word_count(output) * 0.5:
+        votes.append((TaskType.CODE_GENERATION, "output is mostly code blocks"))
+    return votes
+
+
+# ------------------------------------------------------------ prototype backup
+
+_PROTOTYPES: dict[TaskType, list[str]] = {
+    TaskType.SUMMARIZATION: [
+        "Summarize this article in a few sentences.",
+        "Give me the key points of the following document.",
+    ],
+    TaskType.CODE_GENERATION: [
+        "Write a Python function that parses a CSV file.",
+        "Implement a REST endpoint that returns user data.",
+    ],
+    TaskType.CREATIVE_WRITING: [
+        "Write a short story about a lighthouse keeper.",
+        "Compose a poem about autumn in free verse.",
+    ],
+    TaskType.EXTRACTION: [
+        "Extract all person names and dates from this text as JSON.",
+        "List every product mentioned in the reviews below.",
+    ],
+    TaskType.REASONING: [
+        "If a train leaves at 3pm traveling 60 mph, when does it arrive?",
+        "Solve this logic puzzle and show your reasoning.",
+    ],
+    TaskType.REWRITING: [
+        "Rewrite this paragraph in a more formal tone.",
+        "Paraphrase the following text for a younger audience.",
+    ],
+    TaskType.QA_OPEN: [
+        "What are the main causes of inflation?",
+        "Explain how photosynthesis works.",
+    ],
+    TaskType.GENERAL: [
+        "Help me draft an email to my landlord.",
+        "Give me advice on preparing for a job interview.",
+    ],
+}
+
+_proto_cache: dict | None = None
+
+
+def _prototype_scores(instruction: str) -> dict[TaskType, float] | None:
+    global _proto_cache
+    embedder = get_embedder()
+    if embedder is None:
+        return None
+    import numpy as np
+    if _proto_cache is None:
+        flat, owners = [], []
+        for task, examples in _PROTOTYPES.items():
+            flat.extend(examples)
+            owners.extend([task] * len(examples))
+        vecs = embedder.encode(flat, normalize_embeddings=True)
+        _proto_cache = (np.asarray(vecs), owners)
+    vecs, owners = _proto_cache
+    q = embedder.encode([instruction], normalize_embeddings=True)[0]
+    sims = vecs @ q
+    best: dict[TaskType, float] = {}
+    for owner, sim in zip(owners, sims):
+        best[owner] = max(best.get(owner, -1.0), float(sim))
+    return best
+
+
+# ------------------------------------------------------------------- classifier
+
+def classify(prompt: str, output: str) -> TaskClassification:
+    instruction, source = split_instruction_and_source(prompt)
+    votes = _rule_votes(instruction, output)
+    signals = [why for _, why in votes]
+
+    vote_counts: dict[TaskType, int] = {}
+    for task, _ in votes:
+        vote_counts[task] = vote_counts.get(task, 0) + 1
+
+    task: TaskType
+    confidence: float
+
+    if vote_counts:
+        ranked = sorted(vote_counts.items(), key=lambda kv: -kv[1])
+        if len(ranked) == 1 or ranked[0][1] > ranked[1][1]:
+            task, confidence = ranked[0][0], min(0.95, 0.7 + 0.1 * ranked[0][1])
+        else:
+            # Tie -> embeddings referee among tied candidates.
+            tied = {t for t, c in ranked if c == ranked[0][1]}
+            proto = _prototype_scores(instruction)
+            if proto:
+                task = max(tied, key=lambda t: proto.get(t, -1.0))
+                confidence = 0.6
+                signals.append("embedding similarity broke a rule tie")
+            else:
+                task, confidence = ranked[0][0], 0.5
+                signals.append("rule tie, embeddings unavailable; first match kept")
+    else:
+        proto = _prototype_scores(instruction)
+        if proto:
+            task = max(proto, key=proto.get)  # type: ignore[arg-type]
+            top = proto[task]
+            task = task if top >= 0.45 else TaskType.GENERAL
+            confidence = round(min(0.85, max(0.35, top)), 2)
+            signals.append(f"embedding prototype match ({task.value}, sim={top:.2f})")
+        else:
+            task, confidence = TaskType.GENERAL, 0.3
+            signals.append("no rule matched, embeddings unavailable")
+
+    # Source text upgrades/refines the task.
+    if source is not None:
+        signals.append("prompt contains pasted source text")
+        if task == TaskType.QA_OPEN:
+            task = TaskType.QA_GROUNDED
+            signals.append("question + source text -> grounded Q&A")
+
+    return TaskClassification(
+        task_type=task,
+        confidence=confidence,
+        signals=signals,
+        has_source_text=source is not None,
+        instruction=instruction,
+        source_text=source,
+    )

rubriq/constraints.py

@@ -0,0 +1,286 @@
+"""Instruction-following: extract verifiable constraints from the prompt,
+check each with code (IFEval approach, RESEARCH.md 1.8).
+
+Extraction is deliberately conservative. A constraint only becomes a check
+when the pattern is unambiguous, and every extracted constraint is reported
+to the user whether it passed or failed — so a wrong extraction is visible,
+not a hidden scoring error. Counting checks like these are the case where
+deterministic code beats LLM judges (MT-Bench bias findings, RESEARCH.md 1.6).
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from dataclasses import dataclass, field
+
+from .schemas import Evidence, FailureMode, Finding
+from .textutils import snippet, split_paragraphs, split_sentences, word_count
+
+_NUM_WORDS = {
+    "one": 1, "two": 2, "three": 3, "four": 4, "five": 5, "six": 6,
+    "seven": 7, "eight": 8, "nine": 9, "ten": 10, "a single": 1, "single": 1,
+}
+
+
+def _num(tok: str) -> int:
+    tok = tok.strip().lower()
+    return int(tok) if tok.isdigit() else _NUM_WORDS[tok]
+
+
+_NUM_PAT = r"(\d+|one|two|three|four|five|six|seven|eight|nine|ten|a single|single)"
+
+
+@dataclass
+class Constraint:
+    kind: str                    # e.g. "max_words", "format_json"
+    description: str             # human-readable, e.g. "at most 150 words"
+    source_quote: str            # the prompt text that produced it
+    params: dict = field(default_factory=dict)
+
+
+@dataclass
+class CheckResult:
+    constraint: Constraint
+    passed: bool
+    detail: str                  # e.g. "212 words (limit 150)"
+    evidence: list[Evidence] = field(default_factory=list)
+
+
+# ---------------------------------------------------------------- extraction
+
+_UNIT = r"(words?|sentences?|paragraphs?|bullet ?points?|bullets?|items?|characters?|lines?|sections?)"
+
+_LIMIT_PATTERNS = [
+    # (regex, bound kind); bound in {"max","min","exact","approx"}
+    (re.compile(rf"(?i)\b(?:in|within|under|at most|no more than|not more than|maximum(?: of)?|max\.?|up to|fewer than|less than|no longer than)\s+{_NUM_PAT}\s+{_UNIT}"), "max"),
+    (re.compile(rf"(?i)\b(?:at least|minimum(?: of)?|min\.?|no fewer than|more than|over)\s+{_NUM_PAT}\s+{_UNIT}"), "min"),
+    (re.compile(rf"(?i)\bexactly\s+{_NUM_PAT}\s+{_UNIT}"), "exact"),
+    (re.compile(rf"(?i)\b(?:about|around|approximately|roughly|~)\s*{_NUM_PAT}\s+{_UNIT}"), "approx"),
+    (re.compile(rf"(?i)\b{_NUM_PAT}[- ]{_UNIT}\b(?:\s+(?:summary|answer|response|list|essay|description|overview|explanation))"), "approx"),
+]
+
+_STRUCT_PATTERNS: list[tuple[re.Pattern, str, str]] = [
+    (re.compile(r"(?i)\b(?:as|in|into|to|return|output|respond(?: only)? (?:in|with)|format(?:ted)? as)\s+(?:a |an |valid |pure |raw )?json\b"), "format_json", "output should be valid JSON"),
+    (re.compile(r"(?i)\bas (?:a |an )?(?:markdown )?table\b"), "format_table", "output should be a table"),
+    (re.compile(r"(?i)\b(?:as|in|using) (?:a )?(?:bullet(?:ed)?|bulleted) (?:points?|list)\b|\bbullet points? only\b"), "format_bullets", "output should be a bulleted list"),
+    (re.compile(r"(?i)\bnumbered list\b"), "format_numbered", "output should be a numbered list"),
+    (re.compile(r"(?i)\b(?:in |as )(?:a )?single paragraph\b|\bone paragraph\b"), "single_paragraph", "output should be one paragraph"),
+    (re.compile(r"(?i)\bin (?:a )?single sentence\b|\bone sentence\b"), "single_sentence", "output should be one sentence"),
+    (re.compile(r"(?i)\ball (?:in )?lower ?case\b|\bin lowercase only\b"), "all_lowercase", "output should be all lowercase"),
+    (re.compile(r"(?i)\ball (?:in )?(?:upper ?case|caps)\b"), "all_uppercase", "output should be all uppercase"),
+    (re.compile(r"(?i)\bno bullet points?\b|\bwithout (?:any )?bullets?\b|\bdo not use bullet\b"), "no_bullets", "output should not use bullet points"),
+    (re.compile(r"(?i)\bin plain (?:text|prose)\b|\bno markdown\b|\bwithout (?:any )?(?:markdown|formatting)\b"), "no_markdown", "output should be plain prose without markdown"),
+]
+
+_INCLUDE_PAT = re.compile(
+    r"(?i)\b(?:must (?:include|mention|contain)|be sure to (?:include|mention)|"
+    r"include|mention|don'?t forget to (?:include|mention))\s+"
+    r"(?:the (?:word|term|phrase)s?\s+)?[\"“']([^\"”']{2,60})[\"”']"
+)
+_EXCLUDE_PAT = re.compile(
+    r"(?i)\b(?:do not|don'?t|never|avoid)\s+"
+    r"(?:use|mention|include|say)(?:\s+the\s+(?:word|term|phrase)s?)?\s+[\"“']([^\"”']{2,60})[\"”']"
+)
+_START_PAT = re.compile(r"(?i)\b(?:begin|start)\s+(?:your\s+)?(?:response|answer|output|reply|with)?\s*with\s+[\"“']([^\"”']{2,80})[\"”']")
+_END_PAT = re.compile(r"(?i)\bend\s+(?:your\s+)?(?:response|answer|output|reply)?\s*with\s+[\"“']([^\"”']{2,80})[\"”']")
+
+_BULLET_LINE = re.compile(r"(?m)^\s*(?:[-*•]|\d+[.)])\s+\S")
+_NUMBERED_LINE = re.compile(r"(?m)^\s*\d+[.)]\s+\S")
+_MD_MARKUP = re.compile(r"(?m)(^#{1,6}\s)|(\*\*)|(^\s*[-*]\s)|(```)|(^\s*>\s)|(\|.+\|)")
+
+
+def extract_constraints(instruction: str) -> list[Constraint]:
+    found: list[Constraint] = []
+    seen: set[tuple] = set()
+
+    for pat, bound in _LIMIT_PATTERNS:
+        for m in pat.finditer(instruction):
+            n, unit = _num(m.group(1)), m.group(2).lower()
+            unit_key = re.sub(r"\s|s$", "", unit)
+            unit_key = {"bulletpoint": "bullet", "item": "bullet"}.get(unit_key, unit_key)
+            key = (bound, unit_key, n)
+            if key in seen:
+                continue
+            seen.add(key)
+            found.append(Constraint(
+                kind=f"{bound}_{unit_key}",
+                description=f"{ {'max':'at most','min':'at least','exact':'exactly','approx':'about'}[bound] } {n} {unit}",
+                source_quote=m.group(0),
+                params={"n": n, "bound": bound, "unit": unit_key},
+            ))
+
+    for pat, kind, desc in _STRUCT_PATTERNS:
+        m = pat.search(instruction)
+        if m and ("struct", kind) not in seen:
+            seen.add(("struct", kind))
+            found.append(Constraint(kind=kind, description=desc, source_quote=m.group(0)))
+
+    for m in _INCLUDE_PAT.finditer(instruction):
+        term = m.group(1).strip()
+        if ("include", term.lower()) not in seen:
+            seen.add(("include", term.lower()))
+            found.append(Constraint(kind="must_include", description=f'must include "{term}"',
+                                    source_quote=m.group(0), params={"term": term}))
+    for m in _EXCLUDE_PAT.finditer(instruction):
+        term = m.group(1).strip()
+        if ("exclude", term.lower()) not in seen:
+            seen.add(("exclude", term.lower()))
+            found.append(Constraint(kind="must_exclude", description=f'must not use "{term}"',
+                                    source_quote=m.group(0), params={"term": term}))
+    m = _START_PAT.search(instruction)
+    if m:
+        found.append(Constraint(kind="starts_with", description=f'must start with "{m.group(1)}"',
+                                source_quote=m.group(0), params={"term": m.group(1)}))
+    m = _END_PAT.search(instruction)
+    if m:
+        found.append(Constraint(kind="ends_with", description=f'must end with "{m.group(1)}"',
+                                source_quote=m.group(0), params={"term": m.group(1)}))
+    return found
+
+
+# ------------------------------------------------------------------- checking
+
+def _count_units(output: str, unit: str) -> int:
+    if unit == "word":
+        return word_count(output)
+    if unit == "sentence":
+        return len(split_sentences(output))
+    if unit == "paragraph":
+        return len(split_paragraphs(output))
+    if unit == "character":
+        return len(output.strip())
+    if unit == "line":
+        return len([ln for ln in output.splitlines() if ln.strip()])
+    if unit in ("bullet",):
+        return len(_BULLET_LINE.findall(output))
+    if unit == "section":
+        return len(re.findall(r"(?m)^#{1,6}\s+\S", output)) or len(split_paragraphs(output))
+    return word_count(output)
+
+
+def _strip_code_blocks(output: str) -> str:
+    return re.sub(r"```.*?```", "", output, flags=re.S)
+
+
+def check_constraint(c: Constraint, output: str) -> CheckResult:
+    kind = c.kind
+    if kind.startswith(("max_", "min_", "exact_", "approx_")):
+        bound, unit = c.params["bound"], c.params["unit"]
+        n = c.params["n"]
+        actual = _count_units(output, unit)
+        if bound == "max":
+            ok = actual <= n
+        elif bound == "min":
+            ok = actual >= n
+        elif bound == "exact":
+            ok = actual == n
+        else:  # approx: ±20% with a floor of ±2
+            tol = max(2, round(n * 0.2))
+            ok = abs(actual - n) <= tol
+        return CheckResult(c, ok, f"{actual} {unit}{'' if actual == 1 else 's'} (asked: {c.description})")
+
+    if kind == "format_json":
+        text = output.strip()
+        fence = re.search(r"```(?:json)?\s*(.+?)```", text, flags=re.S)
+        candidate = fence.group(1).strip() if fence else text
+        try:
+            json.loads(candidate)
+            extra = text != candidate and not fence
+            return CheckResult(c, not extra, "valid JSON" if not extra else "JSON present but wrapped in extra prose")
+        except json.JSONDecodeError as e:
+            return CheckResult(c, False, f"not valid JSON ({e.msg} at position {e.pos})",
+                               [Evidence(quote=snippet(candidate[max(0, e.pos - 60):e.pos + 60]), note="around the parse error")])
+
+    if kind == "format_table":
+        ok = bool(re.search(r"(?m)^\|.+\|\s*$", output)) or bool(re.search(r"(?m)^.+\t.+$", output))
+        return CheckResult(c, ok, "table detected" if ok else "no table structure found in output")
+
+    if kind == "format_bullets":
+        n = len(_BULLET_LINE.findall(output))
+        return CheckResult(c, n >= 2, f"{n} bullet lines found")
+
+    if kind == "format_numbered":
+        n = len(_NUMBERED_LINE.findall(output))
+        return CheckResult(c, n >= 2, f"{n} numbered lines found")
+
+    if kind == "single_paragraph":
+        n = len(split_paragraphs(output))
+        return CheckResult(c, n == 1, f"{n} paragraphs")
+
+    if kind == "single_sentence":
+        n = len(split_sentences(output))
+        return CheckResult(c, n == 1, f"{n} sentences")
+
+    if kind == "all_lowercase":
+        letters = re.sub(r"[^A-Za-z]", "", output)
+        ok = letters.islower() if letters else True
+        bad = re.search(r"[A-Z][a-zA-Z]*", output)
+        ev = [Evidence(quote=snippet(bad.group(0)), note="uppercase found")] if bad else []
+        return CheckResult(c, ok, "all lowercase" if ok else "contains uppercase", ev)
+
+    if kind == "all_uppercase":
+        letters = re.sub(r"[^A-Za-z]", "", output)
+        ok = letters.isupper() if letters else True
+        return CheckResult(c, ok, "all caps" if ok else "contains lowercase")
+
+    if kind == "no_bullets":
+        n = len(_BULLET_LINE.findall(output))
+        return CheckResult(c, n == 0, "no bullets" if n == 0 else f"{n} bullet lines present")
+
+    if kind == "no_markdown":
+        m = _MD_MARKUP.search(output)
+        return CheckResult(c, m is None, "plain prose" if m is None else "markdown markup present",
+                           [Evidence(quote=snippet(m.group(0)))] if m else [])
+
+    if kind == "must_include":
+        term = c.params["term"]
+        ok = term.lower() in output.lower()
+        return CheckResult(c, ok, f'"{term}" {"present" if ok else "missing from output"}')
+
+    if kind == "must_exclude":
+        term = c.params["term"]
+        i = output.lower().find(term.lower())
+        ok = i < 0
+        ev = [] if ok else [Evidence(quote=snippet(output[max(0, i - 50):i + len(term) + 50]))]
+        return CheckResult(c, ok, f'"{term}" {"absent" if ok else "appears in output"}', ev)
+
+    if kind == "starts_with":
+        term = c.params["term"]
+        ok = output.strip().lower().startswith(term.lower())
+        return CheckResult(c, ok, "starts as required" if ok else f'output starts with "{snippet(output.strip(), 50)}"')
+
+    if kind == "ends_with":
+        term = c.params["term"]
+        ok = output.strip().rstrip(".!?").lower().endswith(term.lower().rstrip(".!?"))
+        return CheckResult(c, ok, "ends as required" if ok else f'output ends with "…{output.strip()[-50:]}"')
+
+    return CheckResult(c, True, "constraint recognized but no checker bound (reported, not scored)")
+
+
+def run_checks(instruction: str, output: str) -> list[CheckResult]:
+    prose = output
+    constraints = extract_constraints(instruction)
+    # Word/sentence limits should not count code inside fences for mixed outputs.
+    results = []
+    for c in constraints:
+        target = prose
+        if c.kind.split("_")[-1] in ("word", "sentence") and "```" in output:
+            target = _strip_code_blocks(output)
+        results.append(check_constraint(c, target))
+    return results
+
+
+def to_findings(results: list[CheckResult]) -> list[Finding]:
+    findings = []
+    for r in results:
+        if r.passed:
+            continue
+        findings.append(Finding(
+            failure_mode=FailureMode.FORMAT_VIOLATION if r.constraint.kind.startswith(("format", "single", "no_", "all_"))
+            else FailureMode.MISSED_CONSTRAINT,
+            detail=f"Prompt asked for {r.constraint.description} — output has {r.detail}.",
+            evidence=r.evidence + [Evidence(quote=r.constraint.source_quote, source="prompt", note="the instruction that set this constraint")],
+            data={"kind": r.constraint.kind, **r.constraint.params, "actual": r.detail},
+        ))
+    return findings

rubriq/metrics/code_static.py

@@ -0,0 +1,109 @@
+"""Static code checks (RESEARCH.md 1.11). No execution in v1 — the ladder:
+does it parse, is requested API surface present, are there placeholder stubs.
+Honesty note attached by the scorer: static analysis cannot prove the code
+runs or is correct; that is HumanEval-style territory reserved for judge mode.
+"""
+
+from __future__ import annotations
+
+import ast
+import re
+from dataclasses import dataclass, field
+
+from ..textutils import snippet
+
+_FENCE = re.compile(r"```(\w+)?\s*\n(.*?)```", re.S)
+_PLACEHOLDERS = re.compile(
+    r"(?i)(?:#|//|/\*)?\s*(TODO|FIXME|your[_ ](?:api[_ ]key|code|logic)[_ ]here|"
+    r"implement (?:this|me)|\.\.\.\s*$|<[A-Z_]+>|pass\s*#\s*placeholder)", re.M)
+_REQUESTED_NAME = re.compile(
+    r"(?i)\b(?:function|method|class)\s+(?:called|named)\s+[`'\"]?([A-Za-z_][A-Za-z0-9_]*)")
+
+_BRACKETS = {"(": ")", "[": "]", "{": "}"}
+
+
+@dataclass
+class CodeBlock:
+    language: str
+    code: str
+    parses: bool | None      # None = no parser for this language, balance-checked only
+    parse_error: str = ""
+
+
+@dataclass
+class CodeStaticResult:
+    blocks: list[CodeBlock] = field(default_factory=list)
+    has_code: bool = False
+    placeholders: list[str] = field(default_factory=list)
+    requested_names_missing: list[str] = field(default_factory=list)
+    prose_only: bool = False    # code was requested but none found
+
+
+def _balanced(code: str) -> bool:
+    """Bracket balance ignoring string literals (rough, for non-Python)."""
+    stack = []
+    in_str: str | None = None
+    prev = ""
+    for ch in code:
+        if in_str:
+            if ch == in_str and prev != "\\":
+                in_str = None
+        elif ch in "\"'":
+            in_str = ch
+        elif ch in _BRACKETS:
+            stack.append(_BRACKETS[ch])
+        elif ch in _BRACKETS.values():
+            if not stack or stack.pop() != ch:
+                return False
+        prev = ch
+    return not stack
+
+
+def compute(instruction: str, output: str) -> CodeStaticResult:
+    result = CodeStaticResult()
+    fences = _FENCE.findall(output)
+
+    # Unfenced fallback: output that is obviously bare code.
+    if not fences and re.search(r"(?m)^(def |class |function |import |from \w+ import |const |let |public )", output):
+        fences = [("", output)]
+
+    for lang, code in fences:
+        lang = (lang or "").lower()
+        code = code.strip()
+        if not code:
+            continue
+        block = CodeBlock(language=lang or "unknown", code=code, parses=None)
+        if lang in ("python", "py", "") and re.search(r"(?m)^(def |class |import |from )", code):
+            block.language = "python"
+            try:
+                ast.parse(code)
+                block.parses = True
+            except SyntaxError as e:
+                block.parses = False
+                block.parse_error = f"line {e.lineno}: {e.msg}"
+        elif lang == "json":
+            import json
+            try:
+                json.loads(code)
+                block.parses = True
+            except json.JSONDecodeError as e:
+                block.parses = False
+                block.parse_error = e.msg
+        else:
+            block.parses = None if _balanced(code) else False
+            if block.parses is False:
+                block.parse_error = "unbalanced brackets/braces"
+        result.blocks.append(block)
+
+    result.has_code = bool(result.blocks)
+    result.prose_only = not result.has_code
+
+    all_code = "\n".join(b.code for b in result.blocks)
+    result.placeholders = [snippet(m.group(0), 60) for m in _PLACEHOLDERS.finditer(all_code)][:5]
+
+    for m in _REQUESTED_NAME.finditer(instruction):
+        name = m.group(1)
+        if not re.search(rf"\b{re.escape(name)}\b", all_code):
+            result.requested_names_missing.append(name)
+
+    return result

rubriq/metrics/faithfulness.py

@@ -0,0 +1,103 @@
+"""NLI faithfulness — SummaC-ZS (Laban et al. 2022), RESEARCH.md 1.2.
+
+Output sentences are scored for entailment against source chunks with
+cross-encoder/nli-deberta-v3-small. Per output sentence we take the max
+entailment over source chunks ("is this sentence supported anywhere?");
+the document score is the mean; the least-supported sentences become quoted
+evidence for unfaithful_content findings.
+
+The cross-encoder returns logits over (contradiction, entailment, neutral)
+— sbert's documented label order for the nli-deberta-v3 family.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from ..models import get_nli
+from ..textutils import snippet, split_sentences, word_count
+
+_LABELS = ("contradiction", "entailment", "neutral")
+
+
+@dataclass
+class SentenceSupport:
+    sentence: str
+    entailment: float       # max over source chunks
+    contradiction: float    # at the chunk where entailment peaked
+    best_chunk: str
+
+
+@dataclass
+class FaithfulnessResult:
+    mean_entailment: float
+    sentences: list[SentenceSupport]
+    unsupported: list[SentenceSupport]     # entailment below threshold
+    contradicted: list[SentenceSupport]    # contradiction dominant somewhere
+    threshold: float
+
+
+def _chunks(source: str, max_words: int = 90) -> list[str]:
+    """Sliding sentence windows so support spanning two sentences isn't missed."""
+    sents = split_sentences(source)
+    if not sents:
+        return [source]
+    chunks, cur, cur_w = [], [], 0
+    for s in sents:
+        w = word_count(s)
+        cur.append(s)
+        cur_w += w
+        if cur_w >= max_words:
+            chunks.append(" ".join(cur))
+            cur, cur_w = cur[-1:], word_count(cur[-1]) if cur else 0
+    if cur:
+        chunks.append(" ".join(cur))
+    return chunks or [source]
+
+
+def _score_sentences(sents: list[str]) -> list[str]:
+    """Filter out non-claims: headers, tiny fragments, questions."""
+    return [s for s in sents if word_count(s) >= 4 and not s.strip().endswith("?")
+            and not s.strip().startswith("#")]
+
+
+def compute(source: str, output: str, threshold: float = 0.5) -> FaithfulnessResult | None:
+    nli = get_nli()
+    if nli is None:
+        return None
+    import numpy as np
+
+    out_sents = _score_sentences(split_sentences(output))
+    if not out_sents:
+        return None
+    chunks = _chunks(source)
+
+    pairs = [(c, s) for s in out_sents for c in chunks]
+    logits = nli.predict(pairs, batch_size=16)
+    probs = np.exp(logits) / np.exp(logits).sum(axis=1, keepdims=True)
+    probs = probs.reshape(len(out_sents), len(chunks), 3)
+
+    ent_idx = _LABELS.index("entailment")
+    con_idx = _LABELS.index("contradiction")
+
+    supports: list[SentenceSupport] = []
+    for i, s in enumerate(out_sents):
+        best_j = int(probs[i, :, ent_idx].argmax())
+        supports.append(SentenceSupport(
+            sentence=s,
+            entailment=round(float(probs[i, best_j, ent_idx]), 3),
+            contradiction=round(float(probs[i, :, con_idx].max()), 3),
+            best_chunk=snippet(chunks[best_j]),
+        ))
+
+    unsupported = [s for s in supports if s.entailment < threshold]
+    contradicted = [s for s in supports if s.contradiction > 0.5]
+    mean_ent = round(sum(s.entailment for s in supports) / len(supports), 3)
+
+    return FaithfulnessResult(
+        mean_entailment=mean_ent,
+        sentences=supports,
+        unsupported=sorted(unsupported, key=lambda s: s.entailment)[:5],
+        contradicted=contradicted[:5],
+        threshold=threshold,
+    )

rubriq/metrics/hallucination_risk.py

@@ -0,0 +1,78 @@
+"""Hallucination *risk signals* for ungrounded prompts (RESEARCH.md 1.9).
+
+Without a source or knowledge base, no local method can verify claims
+(FActScore's precondition). What we CAN do is flag the patterns that
+correlate with fabrication: citation-shaped strings, precise unattributed
+statistics, vague-authority claims, and confident framing around
+unverifiable specifics. Wording contract: every finding says "unverifiable
+here", never "false".
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+
+from ..textutils import snippet
+
+_SIGNALS: list[tuple[str, re.Pattern, str]] = [
+    ("citation_pattern",
+     re.compile(r"\([A-Z][A-Za-z-]+(?: (?:et al\.?|&|and) [A-Z][A-Za-z-]+)?,? (?:19|20)\d{2}\)|\[[0-9]{1,3}\]"),
+     "citation-shaped reference — LLMs fabricate plausible-looking citations; verify each one exists"),
+    ("case_or_statute",
+     re.compile(r"\b[A-Z][a-z]+ v\.? [A-Z][a-z]+\b|\b\d+ U\.S\.C?\.? §? ?\d+\b"),
+     "legal citation pattern — a documented fabrication hotspot (Mata v. Avianca, 2023)"),
+    ("precise_statistic",
+     re.compile(r"\b\d{1,3}(?:\.\d{1,2})?%|\b\d+(?:\.\d+)? (?:million|billion|trillion)\b"),
+     "precise statistic with no source in the prompt to check it against"),
+    ("vague_authority",
+     re.compile(r"(?i)\b(?:studies|research|experts|scientists|reports?) (?:show|shows|suggest|indicate|have (?:shown|found)|found|say|agree|confirm)\b"),
+     "appeal to unnamed studies/experts — no way to trace the claim"),
+    ("url",
+     re.compile(r"https?://\S+"),
+     "URL in output — LLMs generate dead or wrong links; verify before trusting"),
+    ("specific_date_event",
+     re.compile(r"(?i)\b(?:on|in) (?:January|February|March|April|May|June|July|August|September|October|November|December) \d{1,2},? (?:19|20)\d{2}\b"),
+     "specific dated event — unverifiable without a source"),
+    ("named_figure_quote",
+     re.compile(r"(?i)(?:said|stated|according to|wrote|declared)[,:]? ?[\"“]"),
+     "attributed quotation — fabricated quotes are a known failure pattern"),
+]
+
+_HEDGES = re.compile(r"(?i)\b(?:reportedly|allegedly|approximately|around|roughly|estimated|as of my knowledge|may|might|possibly|it is believed)\b")
+
+
+@dataclass
+class RiskSignal:
+    kind: str
+    quote: str
+    why: str
+
+
+@dataclass
+class HallucinationRiskResult:
+    signals: list[RiskSignal] = field(default_factory=list)
+    hedge_count: int = 0
+    density_per_100w: float = 0.0
+
+
+def compute(output: str, n_words: int) -> HallucinationRiskResult:
+    signals: list[RiskSignal] = []
+    seen: set[str] = set()
+    for kind, pat, why in _SIGNALS:
+        for m in pat.finditer(output):
+            start = max(0, m.start() - 60)
+            quote = snippet(output[start:m.end() + 60])
+            key = f"{kind}:{m.group(0)[:40]}"
+            if key in seen:
+                continue
+            seen.add(key)
+            signals.append(RiskSignal(kind=kind, quote=quote, why=why))
+            if sum(1 for s in signals if s.kind == kind) >= 3:
+                break   # cap per kind; the report needs examples, not a census
+
+    return HallucinationRiskResult(
+        signals=signals,
+        hedge_count=len(_HEDGES.findall(output)),
+        density_per_100w=round(100 * len(signals) / max(1, n_words), 2),
+    )

rubriq/metrics/lexical.py

@@ -0,0 +1,85 @@
+"""Lexical diversity and repetition (RESEARCH.md 1.10).
+
+MTLD (McCarthy & Jarvis 2010) for length-robust diversity; cliché density
+against a fixed list; sentence-length variance as the rhythm proxy; repeated
+n-grams for mechanical repetition. All of these are labeled proxies in the
+creative-writing bundle (carve-out #2).
+"""
+
+from __future__ import annotations
+
+import re
+from collections import Counter
+from dataclasses import dataclass
+
+from ..textutils import split_sentences, word_count, words
+
+_CLICHES = [
+    "at the end of the day", "in this day and age", "the calm before the storm",
+    "a whirlwind of emotions", "time stood still", "heart skipped a beat",
+    "sent shivers down", "a rollercoaster of", "little did they know",
+    "in the blink of an eye", "against all odds", "a testament to",
+    "the fabric of", "a tapestry of", "nestled in", "bustling", "myriad of",
+    "delve into", "embark on a journey", "unlock the secrets", "hidden gem",
+    "breathtaking", "picturesque", "last but not least", "needless to say",
+    "it goes without saying", "each and every", "first and foremost",
+    "stood the test of time", "beacon of hope", "double-edged sword",
+]
+
+
+def mtld(tokens: list[str], threshold: float = 0.72) -> float | None:
+    """Measure of Textual Lexical Diversity, bidirectional mean."""
+    if len(tokens) < 50:
+        return None
+
+    def _pass(seq: list[str]) -> float:
+        factors, types, count = 0.0, set(), 0
+        for tok in seq:
+            count += 1
+            types.add(tok)
+            if len(types) / count <= threshold:
+                factors += 1
+                types, count = set(), 0
+        if count:
+            ttr = len(types) / count
+            factors += (1 - ttr) / (1 - threshold) if ttr > threshold else 1
+        return len(seq) / factors if factors else float(len(seq))
+
+    return round((_pass(tokens) + _pass(tokens[::-1])) / 2, 1)
+
+
+@dataclass
+class LexicalStats:
+    mtld: float | None
+    cliche_hits: list[str]
+    cliches_per_100w: float
+    sentence_len_cv: float | None   # coefficient of variation of sentence lengths
+    repeated_ngrams: list[str]      # 4-grams appearing 3+ times
+
+
+def compute(text: str) -> LexicalStats:
+    toks = words(text)
+    n_w = max(1, len(toks))
+    low = text.lower()
+    hits = [c for c in _CLICHES if c in low]
+
+    sents = split_sentences(text)
+    lens = [word_count(s) for s in sents]
+    cv = None
+    if len(lens) >= 3:
+        mean = sum(lens) / len(lens)
+        var = sum((x - mean) ** 2 for x in lens) / len(lens)
+        cv = round((var ** 0.5) / mean, 2) if mean else None
+
+    grams = Counter(
+        " ".join(toks[i:i + 4]) for i in range(len(toks) - 3)
+    )
+    repeated = [g for g, c in grams.most_common(5) if c >= 3 and not re.fullmatch(r"(\w+ )*(the|of|and|a|to|in)( \w+)*", g)]
+
+    return LexicalStats(
+        mtld=mtld(toks),
+        cliche_hits=hits,
+        cliches_per_100w=round(100 * len(hits) / n_w, 2),
+        sentence_len_cv=cv,
+        repeated_ngrams=repeated,
+    )

rubriq/metrics/readability.py

@@ -0,0 +1,63 @@
+"""Readability and fluency statistics (RESEARCH.md 1.10).
+
+Flesch Reading Ease / Flesch-Kincaid grade are audience-fit signals, not
+quality scores: the score maps to a target band derived from the stated (or
+inferred) audience, and distance from the band is what counts against the
+output.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from ..textutils import count_syllables, split_sentences, words
+
+
+@dataclass
+class ReadabilityStats:
+    flesch_reading_ease: float
+    fk_grade: float
+    avg_sentence_words: float
+    n_sentences: int
+    n_words: int
+
+
+def compute(text: str) -> ReadabilityStats | None:
+    sents = split_sentences(text)
+    toks = words(text)
+    if not sents or len(toks) < 10:
+        return None
+    syllables = sum(count_syllables(w) for w in toks)
+    n_w, n_s = len(toks), len(sents)
+    fre = 206.835 - 1.015 * (n_w / n_s) - 84.6 * (syllables / n_w)
+    fk = 0.39 * (n_w / n_s) + 11.8 * (syllables / n_w) - 15.59
+    return ReadabilityStats(
+        flesch_reading_ease=round(fre, 1),
+        fk_grade=round(fk, 1),
+        avg_sentence_words=round(n_w / n_s, 1),
+        n_sentences=n_s,
+        n_words=n_w,
+    )
+
+
+# Audience keyword -> acceptable FK grade band (inclusive).
+_AUDIENCE_BANDS: list[tuple[tuple[str, ...], tuple[float, float], str]] = [
+    (("child", "kid", "5 year", "five year", "eli5", "young"), (0.0, 6.0), "young readers"),
+    (("teen", "high school", "student", "beginner", "layman", "non-technical", "general public", "simple"), (5.0, 10.0), "general readers"),
+    (("professional", "business", "executive",), (8.0, 14.0), "professional readers"),
+    (("expert", "academic", "technical", "clinician", "researcher", "phd", "specialist"), (10.0, 18.0), "expert readers"),
+]
+
+DEFAULT_BAND = (6.0, 14.0)
+
+
+def audience_band(audience_text: str, instruction: str) -> tuple[tuple[float, float], str]:
+    """Pick the FK grade band from the advanced-options audience field,
+    falling back to audience words inside the instruction itself."""
+    for probe in (audience_text.lower(), instruction.lower()):
+        if not probe:
+            continue
+        for keys, band, label in _AUDIENCE_BANDS:
+            if any(k in probe for k in keys):
+                return band, label
+    return DEFAULT_BAND, "unspecified audience (default band)"

rubriq/metrics/relevance.py

@@ -0,0 +1,52 @@
+"""Answer relevance via embeddings (RESEARCH.md 1.3/1.4, RAGAS-style,
+adapted to run without an LLM: cosine similarity between the instruction and
+the output, plus per-paragraph drill-down to locate where drift happens).
+
+bge models are asymmetric retrievers: queries get the documented
+"Represent this sentence..." prefix, passages do not.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from ..models import get_embedder
+from ..textutils import snippet, split_paragraphs
+
+_BGE_QUERY_PREFIX = "Represent this sentence for searching relevant passages: "
+
+
+@dataclass
+class RelevanceResult:
+    overall: float                        # cosine, roughly 0..1 for bge
+    worst_paragraph: str | None
+    worst_paragraph_sim: float | None
+    paragraph_sims: list[tuple[str, float]]
+
+
+def compute(instruction: str, output: str) -> RelevanceResult | None:
+    embedder = get_embedder()
+    if embedder is None:
+        return None
+    paragraphs = split_paragraphs(output) or [output]
+    texts = [_BGE_QUERY_PREFIX + instruction, output] + paragraphs
+    vecs = embedder.encode(texts, normalize_embeddings=True)
+    q, whole, para_vecs = vecs[0], vecs[1], vecs[2:]
+    overall = float(q @ whole)
+    sims = [(p, float(q @ v)) for p, v in zip(paragraphs, para_vecs)]
+    worst = min(sims, key=lambda x: x[1]) if len(sims) > 1 else (None, None)
+    return RelevanceResult(
+        overall=round(overall, 3),
+        worst_paragraph=snippet(worst[0]) if worst[0] else None,
+        worst_paragraph_sim=round(worst[1], 3) if worst[1] is not None else None,
+        paragraph_sims=[(snippet(p, 80), round(s, 3)) for p, s in sims],
+    )
+
+
+def similarity(a: str, b: str) -> float | None:
+    """Symmetric semantic similarity (used by rewriting's meaning check)."""
+    embedder = get_embedder()
+    if embedder is None:
+        return None
+    va, vb = embedder.encode([a, b], normalize_embeddings=True)
+    return round(float(va @ vb), 3)

rubriq/metrics/structure.py

@@ -0,0 +1,73 @@
+"""Structure, coherence proxy, and conciseness signals.
+
+Coherence proxy per the Coh-Metrix tradition adapted to embeddings
+(RESEARCH.md 1.10): adjacent-unit semantic similarity should sit in a
+middle band — too low reads disjointed, too high reads repetitive.
+Conciseness inverts MT-Bench's verbosity-bias finding (1.6).
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from ..models import get_embedder
+from ..textutils import snippet, split_paragraphs, split_sentences, word_count
+
+
+@dataclass
+class CoherenceResult:
+    mean_adjacent_sim: float
+    flagged_low: list[tuple[str, str, float]]    # (unit_a, unit_b, sim) — disjointed jumps
+    flagged_high: list[tuple[str, str, float]]   # near-duplicate units
+
+
+def coherence_proxy(text: str) -> CoherenceResult | None:
+    embedder = get_embedder()
+    if embedder is None:
+        return None
+    units = split_paragraphs(text)
+    if len(units) < 3:
+        units = split_sentences(text)
+    if len(units) < 3:
+        return None
+    vecs = embedder.encode(units, normalize_embeddings=True)
+    sims = [float(vecs[i] @ vecs[i + 1]) for i in range(len(vecs) - 1)]
+    low = [(snippet(units[i], 70), snippet(units[i + 1], 70), round(s, 2))
+           for i, s in enumerate(sims) if s < 0.35][:3]
+    high = [(snippet(units[i], 70), snippet(units[i + 1], 70), round(s, 2))
+            for i, s in enumerate(sims) if s > 0.92][:3]
+    return CoherenceResult(
+        mean_adjacent_sim=round(sum(sims) / len(sims), 3),
+        flagged_low=low,
+        flagged_high=high,
+    )
+
+
+@dataclass
+class ConcisenessResult:
+    output_words: int
+    expected_max: int | None      # None when no expectation could be set
+    ratio: float | None
+    filler_hits: list[str]
+
+
+_FILLERS = [
+    "it is important to note that", "it is worth noting", "as mentioned earlier",
+    "in order to", "due to the fact that", "at this point in time",
+    "it goes without saying", "needless to say", "in the world of",
+    "in today's", "when it comes to", "at the end of the day",
+    "i hope this helps", "certainly!", "great question",
+]
+
+
+def conciseness(instruction: str, output: str, task_expected_words: int | None) -> ConcisenessResult:
+    n = word_count(output)
+    low_out = output.lower()
+    hits = [f for f in _FILLERS if f in low_out]
+    ratio = round(n / task_expected_words, 2) if task_expected_words else None
+    return ConcisenessResult(
+        output_words=n,
+        expected_max=task_expected_words,
+        ratio=ratio,
+        filler_hits=hits,
+    )

rubriq/models.py

@@ -0,0 +1,64 @@
+"""Lazy singletons for the two local models.
+
+Checkpoints locked in RESEARCH.md:
+- embeddings: BAAI/bge-small-en-v1.5
+- NLI cross-encoder: cross-encoder/nli-deberta-v3-small
+
+Both load on first use (first request pays the load; the API exposes a
+warmup endpoint). If loading fails (no download yet, offline machine),
+callers receive None and the affected metrics report skipped_reason instead
+of fake numbers — degradation is visible, never silent.
+"""
+
+from __future__ import annotations
+
+import logging
+import threading
+
+logger = logging.getLogger(__name__)
+
+EMBEDDING_MODEL = "BAAI/bge-small-en-v1.5"
+NLI_MODEL = "cross-encoder/nli-deberta-v3-small"
+
+_lock = threading.Lock()
+_embedder = None
+_nli = None
+_embedder_failed = False
+_nli_failed = False
+
+
+def get_embedder():
+    global _embedder, _embedder_failed
+    if _embedder is not None or _embedder_failed:
+        return _embedder
+    with _lock:
+        if _embedder is None and not _embedder_failed:
+            try:
+                from sentence_transformers import SentenceTransformer
+                _embedder = SentenceTransformer(EMBEDDING_MODEL, device="cpu")
+            except Exception:
+                logger.exception("embedding model unavailable")
+                _embedder_failed = True
+    return _embedder
+
+
+def get_nli():
+    global _nli, _nli_failed
+    if _nli is not None or _nli_failed:
+        return _nli
+    with _lock:
+        if _nli is None and not _nli_failed:
+            try:
+                from sentence_transformers import CrossEncoder
+                _nli = CrossEncoder(NLI_MODEL, device="cpu")
+            except Exception:
+                logger.exception("NLI model unavailable")
+                _nli_failed = True
+    return _nli
+
+
+def availability() -> dict[str, bool]:
+    return {
+        "embeddings": get_embedder() is not None,
+        "nli": get_nli() is not None,
+    }

rubriq/schemas.py

@@ -0,0 +1,123 @@
+"""Data contracts shared by every scorer implementation and the API.
+
+The adapter rule: LocalScorer (v1) and LLMJudgeScorer (later) both consume
+EvalRequest and produce EvalReport. Nothing downstream may depend on which
+engine ran; honesty flags (is_proxy, requires_judge) live on the schema so
+the UI cannot forget them.
+"""
+
+from __future__ import annotations
+
+from enum import Enum
+
+from pydantic import BaseModel, Field
+
+
+class TaskType(str, Enum):
+    SUMMARIZATION = "summarization"
+    QA_GROUNDED = "qa_grounded"          # question answered against source text in the prompt
+    QA_OPEN = "qa_open"                  # question with no source text to check against
+    CODE_GENERATION = "code_generation"
+    CREATIVE_WRITING = "creative_writing"
+    EXTRACTION = "extraction"
+    REASONING = "reasoning"
+    REWRITING = "rewriting"
+    GENERAL = "general"
+
+
+class FailureMode(str, Enum):
+    UNFAITHFUL_CONTENT = "unfaithful_content"
+    HALLUCINATION_RISK = "hallucination_risk"
+    MISSED_CONSTRAINT = "missed_constraint"
+    INCOMPLETE_COVERAGE = "incomplete_coverage"
+    OFF_TOPIC_DRIFT = "off_topic_drift"
+    POOR_STRUCTURE = "poor_structure"
+    WRONG_REGISTER = "wrong_register"
+    VERBOSITY = "verbosity"
+    WEAK_REASONING = "weak_reasoning"
+    FORMAT_VIOLATION = "format_violation"
+    LOW_DIVERSITY = "low_diversity"
+    INVALID_CODE = "invalid_code"
+    INVENTED_FIELDS = "invented_fields"
+
+
+class ModelFamily(str, Enum):
+    CLAUDE = "claude"
+    GPT = "gpt"
+    GEMINI = "gemini"
+    OPEN_WEIGHTS = "open_weights"   # llama, mistral, qwen, ...
+    GENERIC = "generic"
+
+
+class Evidence(BaseModel):
+    """A quote from the output (or prompt) backing a finding."""
+    quote: str
+    source: str = "output"          # "output" | "prompt"
+    note: str = ""
+
+
+class Finding(BaseModel):
+    """One diagnosed problem, specific enough to drive a repair prompt."""
+    failure_mode: FailureMode
+    detail: str                     # human-readable diagnosis naming the problem
+    evidence: list[Evidence] = Field(default_factory=list)
+    data: dict = Field(default_factory=dict)   # machine-readable specifics (e.g. limit=150, actual=212)
+
+
+class ParameterScore(BaseModel):
+    parameter: str                  # machine key, e.g. "faithfulness"
+    display_name: str
+    score: float | None             # 0-100; None when the check could not run
+    verdict: str                    # one-line summary of how the output did
+    strengths: list[str] = Field(default_factory=list)
+    findings: list[Finding] = Field(default_factory=list)
+    is_proxy: bool = False          # proxy metric (subjective-quality carve-out)
+    requires_judge: bool = False    # cannot be judged locally at all; needs LLM-judge mode
+    skipped_reason: str = ""        # set when score is None
+    rubric_source: str = ""         # citation key into RESEARCH.md, e.g. "SummaC (Laban et al. 2022)"
+
+
+class TaskClassification(BaseModel):
+    task_type: TaskType
+    confidence: float               # 0-1
+    signals: list[str] = Field(default_factory=list)  # why, shown in UI
+    has_source_text: bool = False
+    instruction: str = ""           # prompt minus pasted source material
+    source_text: str | None = None
+
+
+class ImprovementPrompt(BaseModel):
+    failure_mode: FailureMode
+    title: str
+    diagnosis: str                  # names the specific problem found, with evidence
+    prompt_text: str                # ready to paste
+    model_family: ModelFamily
+    technique_source: str           # citation key into RESEARCH.md
+
+
+class OverallScore(BaseModel):
+    score: float                    # weighted 0-100 over scoreable parameters
+    note: str                       # what the number does and does not include
+
+
+class EvalOptions(BaseModel):
+    """Advanced options; every field has a sensible default (per brief)."""
+    weights: dict[str, float] = Field(default_factory=dict)   # parameter key -> weight override
+    audience: str = ""              # e.g. "children", "expert clinicians"
+    tone: str = ""                  # expected register, free text
+
+
+class EvalRequest(BaseModel):
+    prompt: str
+    output: str
+    model_name: str = ""            # free text, e.g. "gpt-4o", "claude sonnet"
+    options: EvalOptions = Field(default_factory=EvalOptions)
+
+
+class EvalReport(BaseModel):
+    task: TaskClassification
+    parameters: list[ParameterScore]
+    overall: OverallScore
+    improvement_prompts: list[ImprovementPrompt]
+    honesty_notes: list[str] = Field(default_factory=list)  # carve-out disclosures for this run
+    engine: str = "local"           # which Scorer produced this

rubriq/textutils.py

@@ -0,0 +1,77 @@
+"""Shared text primitives: tokenization-free splitting and counting.
+
+Kept dependency-free (no nltk/spacy) so the deterministic checks stay
+deterministic and the install stays light.
+"""
+
+from __future__ import annotations
+
+import re
+
+_ABBREVIATIONS = {
+    "mr", "mrs", "ms", "dr", "prof", "sr", "jr", "st", "vs", "etc", "e.g",
+    "i.e", "fig", "no", "vol", "inc", "ltd", "co", "corp", "dept", "est",
+    "approx", "u.s", "u.k",
+}
+
+_SENT_BOUNDARY = re.compile(r"(?<=[.!?])[\"')\]]*\s+(?=[A-Z0-9\"'(\[])")
+
+
+def split_sentences(text: str) -> list[str]:
+    """Regex sentence splitter with abbreviation guards. Good enough for
+    metric aggregation; not a linguistic gold standard and doesn't need to be.
+    """
+    text = text.strip()
+    if not text:
+        return []
+    parts: list[str] = []
+    for para in re.split(r"\n\s*\n|\n(?=[-*#\d])", text):
+        para = para.strip()
+        if not para:
+            continue
+        start = 0
+        for m in _SENT_BOUNDARY.finditer(para):
+            candidate = para[start:m.start() + 1]
+            last_word = re.findall(r"[\w.]+", candidate[-12:].lower())
+            if last_word and last_word[-1].rstrip(".") in _ABBREVIATIONS:
+                continue
+            if candidate.strip():
+                parts.append(candidate.strip())
+            start = m.end()
+        tail = para[start:].strip()
+        if tail:
+            parts.append(tail)
+    return parts
+
+
+def split_paragraphs(text: str) -> list[str]:
+    return [p.strip() for p in re.split(r"\n\s*\n", text.strip()) if p.strip()]
+
+
+def word_count(text: str) -> int:
+    return len(re.findall(r"[A-Za-z0-9'’-]+", text))
+
+
+def words(text: str) -> list[str]:
+    return re.findall(r"[A-Za-z'’-]+", text.lower())
+
+
+def count_syllables(word: str) -> int:
+    """Heuristic syllable count (vowel groups, silent-e adjustment)."""
+    w = word.lower().strip(".:;!?,'\"")
+    if not w:
+        return 0
+    groups = re.findall(r"[aeiouy]+", w)
+    n = len(groups)
+    if w.endswith("e") and n > 1 and not w.endswith(("le", "ee", "ye")):
+        n -= 1
+    return max(1, n)
+
+
+def snippet(text: str, limit: int = 160) -> str:
+    """Shorten a quote for evidence display without cutting mid-word."""
+    text = " ".join(text.split())
+    if len(text) <= limit:
+        return text
+    cut = text[:limit].rsplit(" ", 1)[0]
+    return cut + "…"