One unified repair prompt + tone selector replaces Advanced section
Browse files- generator rewrite: failure modes contribute rules/problems/role/pre to
a single composed prompt built around the original instruction; per-
family mechanisms preserved (XML rules for Claude, ### Requirements +
tail emphasis for GPT, few-shot skeleton for Gemini); test contract
rewritten for the single-prompt guarantee (46 green)
- tone chips (Neutral/Simple/Professional/Technical) in the main flow;
each maps to an audience band so the choice drives the reading-level
check; Advanced section removed
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
- rubriq/prompts/generator.py +228 -400
- tests/test_prompts.py +47 -54
- web/app.js +6 -3
- web/index.html +20 -10
- web/styles.css +34 -9
rubriq/prompts/generator.py
CHANGED
|
@@ -1,20 +1,23 @@
|
|
| 1 |
"""Improvement-prompt generation — the flagship.
|
| 2 |
|
| 3 |
-
Contract (RESEARCH.md Part 4, enforced by tests):
|
| 4 |
-
|
|
|
|
| 5 |
(b) prompt_text is a complete, ready-to-paste replacement prompt,
|
| 6 |
-
(c) the
|
| 7 |
"Be more specific" is a bug, not a template.
|
| 8 |
|
| 9 |
-
|
| 10 |
-
|
|
|
|
| 11 |
"""
|
| 12 |
|
| 13 |
from __future__ import annotations
|
| 14 |
|
| 15 |
-
from
|
| 16 |
-
|
| 17 |
-
from ..
|
|
|
|
| 18 |
from .model_profiles import Profile, get_profile
|
| 19 |
|
| 20 |
_SEVERITY = [
|
|
@@ -33,14 +36,42 @@ _SEVERITY = [
|
|
| 33 |
FailureMode.LOW_DIVERSITY,
|
| 34 |
]
|
| 35 |
|
| 36 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 37 |
|
| 38 |
|
| 39 |
-
def _quotes(findings, limit=3) -> list[str]:
|
| 40 |
out = []
|
| 41 |
for f in findings:
|
| 42 |
for e in f.evidence:
|
| 43 |
-
if e.source ==
|
| 44 |
out.append(e.quote)
|
| 45 |
break
|
| 46 |
if len(out) >= limit:
|
|
@@ -48,388 +79,147 @@ def _quotes(findings, limit=3) -> list[str]:
|
|
| 48 |
return out
|
| 49 |
|
| 50 |
|
| 51 |
-
|
| 52 |
-
|
| 53 |
-
|
| 54 |
-
|
| 55 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 56 |
|
| 57 |
def _rules_block(profile: Profile, rules: list[str]) -> str:
|
| 58 |
-
"""Render hard rules in the family's preferred structure."""
|
| 59 |
if profile.family == ModelFamily.CLAUDE:
|
| 60 |
-
|
| 61 |
-
return f"<rules>\n{inner}\n</rules>"
|
| 62 |
if profile.family == ModelFamily.GPT:
|
| 63 |
-
|
| 64 |
-
|
| 65 |
-
inner = "\n".join(f"- {r}" for r in rules)
|
| 66 |
-
return f"Rules:\n{inner}"
|
| 67 |
-
|
| 68 |
-
|
| 69 |
-
def _tail_emphasis(profile: Profile, critical: str) -> str:
|
| 70 |
-
"""GPT/open-weights/generic get the critical constraint repeated last
|
| 71 |
-
(lost-in-the-middle, RESEARCH.md 2.7). Claude/Gemini get their own levers
|
| 72 |
-
elsewhere, so no noisy repetition."""
|
| 73 |
-
if profile.family in (ModelFamily.OPEN_WEIGHTS, ModelFamily.GPT, ModelFamily.GENERIC):
|
| 74 |
-
return f"\n\nIMPORTANT — before finishing, verify: {critical}"
|
| 75 |
-
return ""
|
| 76 |
-
|
| 77 |
-
|
| 78 |
-
# --------------------------------------------------------------- builders
|
| 79 |
-
|
| 80 |
-
def _build_unfaithful(findings, task, profile: Profile) -> ImprovementPrompt:
|
| 81 |
-
bad = _quotes(findings)
|
| 82 |
-
bad_list = "\n".join(f'- "{q}"' for q in bad) if bad else "- (see report)"
|
| 83 |
-
grounding_rules = [
|
| 84 |
-
"Use ONLY information stated in the source text. If the source does not say it, do not write it.",
|
| 85 |
-
"If something the task needs is missing from the source, write \"the source does not specify\" rather than filling the gap.",
|
| 86 |
-
]
|
| 87 |
-
if profile.family == ModelFamily.CLAUDE:
|
| 88 |
-
pre = ("First, copy the exact sentences from the source that support your answer into <quotes> tags. "
|
| 89 |
-
"Then write the final answer in <answer> tags using only what you quoted.\n\n")
|
| 90 |
-
elif profile.family == ModelFamily.GEMINI:
|
| 91 |
-
pre = "Base every sentence of your answer on the text above. Quote or closely paraphrase the source.\n\n"
|
| 92 |
-
else:
|
| 93 |
-
pre = "Answer using only the provided text. Quote the source where possible.\n\n"
|
| 94 |
-
body = (
|
| 95 |
-
f"{_source_block(profile, task)}"
|
| 96 |
-
f"{task.instruction.strip()}\n\n"
|
| 97 |
-
f"{pre}"
|
| 98 |
-
f"{_rules_block(profile, grounding_rules)}\n\n"
|
| 99 |
-
f"A previous attempt invented content the source does not support, including:\n{bad_list}\n"
|
| 100 |
-
f"Do not repeat these claims unless the source states them."
|
| 101 |
-
f"{_tail_emphasis(profile, 'every claim traces to a sentence in the source text')}"
|
| 102 |
-
)
|
| 103 |
-
return ImprovementPrompt(
|
| 104 |
-
failure_mode=FailureMode.UNFAITHFUL_CONTENT,
|
| 105 |
-
title="Pin the output to the source",
|
| 106 |
-
diagnosis=(f"The output makes claims the source text does not support"
|
| 107 |
-
+ (f', e.g. "{bad[0]}"' if bad else "") + ". "
|
| 108 |
-
f"Repair: {profile.grounding_mechanism}"),
|
| 109 |
-
prompt_text=body,
|
| 110 |
-
model_family=profile.family,
|
| 111 |
-
technique_source=profile.source,
|
| 112 |
-
)
|
| 113 |
-
|
| 114 |
-
|
| 115 |
-
def _build_constraints(findings, task, profile: Profile) -> ImprovementPrompt:
|
| 116 |
-
missed = []
|
| 117 |
-
for f in findings:
|
| 118 |
-
kind = f.data.get("kind", "")
|
| 119 |
-
actual = f.data.get("actual", "")
|
| 120 |
-
missed.append((f.detail, kind, actual))
|
| 121 |
-
rules = []
|
| 122 |
-
for f in findings:
|
| 123 |
-
prompt_ev = next((e.quote for e in f.evidence if e.source == "prompt"), None)
|
| 124 |
-
rules.append(f.detail.split(" — ")[0].removeprefix("Prompt asked for ").strip().capitalize()
|
| 125 |
-
+ (f" (your prompt said: \"{prompt_ev}\")" if prompt_ev else ""))
|
| 126 |
-
if profile.family == ModelFamily.GEMINI:
|
| 127 |
-
example = ("\n\nHere is an example of a response with the right shape (replace the content, keep the form):\n"
|
| 128 |
-
"[write one short example that satisfies every rule above]")
|
| 129 |
-
else:
|
| 130 |
-
example = ""
|
| 131 |
-
critical = rules[0] if rules else "all stated constraints are met"
|
| 132 |
-
body = (
|
| 133 |
-
f"{_source_block(profile, task)}"
|
| 134 |
-
f"{task.instruction.strip()}\n\n"
|
| 135 |
-
f"{_rules_block(profile, rules)}"
|
| 136 |
-
f"{example}"
|
| 137 |
-
f"\n\nAfter drafting, count and check each requirement above before giving your final answer."
|
| 138 |
-
f"{_tail_emphasis(profile, critical)}"
|
| 139 |
-
)
|
| 140 |
-
first = findings[0]
|
| 141 |
-
return ImprovementPrompt(
|
| 142 |
-
failure_mode=first.failure_mode,
|
| 143 |
-
title="Make the constraints unmissable",
|
| 144 |
-
diagnosis=(f"The output missed {len(findings)} verifiable constraint(s): "
|
| 145 |
-
+ "; ".join(f.detail for f in findings[:3]) + " "
|
| 146 |
-
f"Repair: {profile.format_mechanism}"),
|
| 147 |
-
prompt_text=body,
|
| 148 |
-
model_family=profile.family,
|
| 149 |
-
technique_source=profile.source,
|
| 150 |
-
)
|
| 151 |
-
|
| 152 |
-
|
| 153 |
-
def _build_coverage(findings, task, profile: Profile) -> ImprovementPrompt | None:
|
| 154 |
-
parts = []
|
| 155 |
-
for f in findings:
|
| 156 |
-
q = next((e.quote for e in f.evidence if e.source == "prompt"), None)
|
| 157 |
-
if q:
|
| 158 |
-
parts.append(q)
|
| 159 |
-
if not parts:
|
| 160 |
-
return None # nothing quotable to enumerate; no honest repair possible
|
| 161 |
-
part_list = "\n".join(f"{i+1}. {p}" for i, p in enumerate(parts))
|
| 162 |
-
body = (
|
| 163 |
-
f"{_source_block(profile, task)}"
|
| 164 |
-
f"{task.instruction.strip()}\n\n"
|
| 165 |
-
f"Answer every numbered part separately, under its own heading:\n{part_list}\n\n"
|
| 166 |
-
f"If you cannot answer a part, say so under that heading rather than skipping it."
|
| 167 |
-
f"{_tail_emphasis(profile, f'all {len(parts)} parts have their own answer')}"
|
| 168 |
-
)
|
| 169 |
-
return ImprovementPrompt(
|
| 170 |
-
failure_mode=FailureMode.INCOMPLETE_COVERAGE,
|
| 171 |
-
title="Force full coverage of the question",
|
| 172 |
-
diagnosis=(f"{len(parts)} part(s) of the request went unanswered, e.g. \"{parts[0] if parts else ''}\". "
|
| 173 |
-
"Repair: enumerate the parts and demand one answer per part — models skip sub-questions "
|
| 174 |
-
"buried in prose but not numbered lists."),
|
| 175 |
-
prompt_text=body,
|
| 176 |
-
model_family=profile.family,
|
| 177 |
-
technique_source=profile.source,
|
| 178 |
-
)
|
| 179 |
-
|
| 180 |
-
|
| 181 |
-
def _build_hallucination(findings, task, profile: Profile) -> ImprovementPrompt:
|
| 182 |
-
flagged = _quotes(findings)
|
| 183 |
-
fl = "\n".join(f'- "{q}"' for q in flagged) if flagged else ""
|
| 184 |
-
rules = [
|
| 185 |
-
"For every statistic, name its source inline, or present it as an estimate in words (\"roughly\", \"on the order of\").",
|
| 186 |
-
"Do not cite papers, cases, or URLs unless you are certain they exist; prefer \"research on X suggests\" with no fake citation.",
|
| 187 |
-
"Mark claims you are unsure about explicitly (\"I may be wrong about the exact figure\").",
|
| 188 |
-
]
|
| 189 |
-
body = (
|
| 190 |
-
f"{task.instruction.strip()}\n\n"
|
| 191 |
-
f"{_rules_block(profile, rules)}\n\n"
|
| 192 |
-
+ (f"A previous attempt stated the following as fact; each is unverifiable as written:\n{fl}\n"
|
| 193 |
-
f"Restate these with sources or with honest uncertainty, or drop them.\n" if fl else "")
|
| 194 |
-
+ "\nEnd your answer with a one-line note: which claims above should I double-check independently?"
|
| 195 |
-
f"{_tail_emphasis(profile, 'no statistic or citation appears without a named source or an uncertainty marker')}"
|
| 196 |
-
)
|
| 197 |
-
return ImprovementPrompt(
|
| 198 |
-
failure_mode=FailureMode.HALLUCINATION_RISK,
|
| 199 |
-
title="Demand attribution or honest uncertainty",
|
| 200 |
-
diagnosis=(f"The output states unverifiable specifics as fact"
|
| 201 |
-
+ (f', e.g. "{flagged[0]}"' if flagged else "") + ". "
|
| 202 |
-
"No local check can verify these; the repair makes the model attribute or hedge them, "
|
| 203 |
-
"and hands you a verification checklist."),
|
| 204 |
-
prompt_text=body,
|
| 205 |
-
model_family=profile.family,
|
| 206 |
-
technique_source=profile.source,
|
| 207 |
-
)
|
| 208 |
-
|
| 209 |
-
|
| 210 |
-
def _build_reasoning(findings, task, profile: Profile) -> ImprovementPrompt:
|
| 211 |
-
if profile.family == ModelFamily.CLAUDE:
|
| 212 |
-
think = ("Work through the problem step by step inside <thinking> tags first. "
|
| 213 |
-
"Then give the final answer in <answer> tags.")
|
| 214 |
-
else:
|
| 215 |
-
think = ("Solve this step by step. Show each step of your working on its own line, "
|
| 216 |
-
"then state the final answer on a last line beginning \"Answer:\".")
|
| 217 |
-
body = (
|
| 218 |
-
f"{task.instruction.strip()}\n\n"
|
| 219 |
-
f"{think}\n\n"
|
| 220 |
-
f"After stating the answer, re-derive it a second way if possible and confirm both routes agree; "
|
| 221 |
-
f"if they disagree, say which one you trust and why."
|
| 222 |
-
f"{_tail_emphasis(profile, 'working shown, final answer explicitly labeled')}"
|
| 223 |
-
)
|
| 224 |
-
return ImprovementPrompt(
|
| 225 |
-
failure_mode=FailureMode.WEAK_REASONING,
|
| 226 |
-
title="Make the model show its working",
|
| 227 |
-
diagnosis=("The output gives a conclusion without checkable working. "
|
| 228 |
-
"Repair: request the reasoning *before* the answer (chain-of-thought), plus a second "
|
| 229 |
-
"independent derivation — the manual form of self-consistency."),
|
| 230 |
-
prompt_text=body,
|
| 231 |
-
model_family=profile.family,
|
| 232 |
-
technique_source="Wei et al. 2022 (CoT); Wang et al. 2023 (self-consistency); " + profile.source,
|
| 233 |
-
)
|
| 234 |
-
|
| 235 |
-
|
| 236 |
-
def _build_verbosity(findings, task, profile: Profile) -> ImprovementPrompt:
|
| 237 |
-
fillers = _quotes(findings)
|
| 238 |
-
limit_hint = next((f.data.get("ratio") for f in findings if "ratio" in f.data), None)
|
| 239 |
-
rules = [
|
| 240 |
-
"Open with the answer itself — no preamble, no restating the question.",
|
| 241 |
-
"Cut framing phrases (e.g. " + ", ".join(f'"{f}"' for f in fillers[:3]) + ")." if fillers
|
| 242 |
-
else "Cut framing phrases and meta-commentary.",
|
| 243 |
-
"One idea per sentence; delete any sentence that repeats an earlier one.",
|
| 244 |
-
]
|
| 245 |
-
if profile.family == ModelFamily.CLAUDE:
|
| 246 |
-
opener = "\nBegin your response directly with the substance (no introduction)."
|
| 247 |
-
else:
|
| 248 |
-
opener = ""
|
| 249 |
-
body = (
|
| 250 |
-
f"{_source_block(profile, task)}"
|
| 251 |
-
f"{task.instruction.strip()}\n\n"
|
| 252 |
-
f"{_rules_block(profile, rules)}{opener}"
|
| 253 |
-
f"{_tail_emphasis(profile, 'the response starts with substance and contains no filler phrases')}"
|
| 254 |
-
)
|
| 255 |
-
return ImprovementPrompt(
|
| 256 |
-
failure_mode=FailureMode.VERBOSITY,
|
| 257 |
-
title="Strip the padding",
|
| 258 |
-
diagnosis=("The output pads its length" + (f" ({limit_hint}x the asked-for size)" if limit_hint else "")
|
| 259 |
-
+ (f' with filler like "{fillers[0]}"' if fillers else "") + ". "
|
| 260 |
-
"Repair: answer-first structure plus an explicit ban on the exact filler found."),
|
| 261 |
-
prompt_text=body,
|
| 262 |
-
model_family=profile.family,
|
| 263 |
-
technique_source=profile.source,
|
| 264 |
-
)
|
| 265 |
-
|
| 266 |
-
|
| 267 |
-
def _build_register(findings, task, profile: Profile) -> ImprovementPrompt:
|
| 268 |
-
data = findings[0].data
|
| 269 |
-
fk, band = data.get("fk_grade"), data.get("band", [6, 14])
|
| 270 |
-
audience = data.get("audience", "the intended audience")
|
| 271 |
-
too_hard = fk is not None and fk > band[1]
|
| 272 |
-
style = ("short sentences, everyday words, one idea at a time"
|
| 273 |
-
if too_hard else "precise terminology and compact, information-dense prose")
|
| 274 |
-
role = f"You are writing for {audience}."
|
| 275 |
-
body = (
|
| 276 |
-
f"{role}\n\n"
|
| 277 |
-
f"{_source_block(profile, task)}"
|
| 278 |
-
f"{task.instruction.strip()}\n\n"
|
| 279 |
-
f"Write in a register that fits that audience: {style}. "
|
| 280 |
-
f"Read your draft as that reader before finalizing; rewrite any sentence they would stumble on."
|
| 281 |
-
f"{_tail_emphasis(profile, f'the text reads naturally for {audience}')}"
|
| 282 |
-
)
|
| 283 |
-
return ImprovementPrompt(
|
| 284 |
-
failure_mode=FailureMode.WRONG_REGISTER,
|
| 285 |
-
title="Match the audience's reading level",
|
| 286 |
-
diagnosis=(f"The output reads at US grade level {fk}, outside the {band[0]:g}-{band[1]:g} band for {audience} — "
|
| 287 |
-
f"{'harder' if too_hard else 'simpler'} than it should. "
|
| 288 |
-
"Repair: an explicit audience role. Role prompting reliably shifts register (it does not fix facts)."),
|
| 289 |
-
prompt_text=body,
|
| 290 |
-
model_family=profile.family,
|
| 291 |
-
technique_source="role prompting for register (RESEARCH.md 2.8); " + profile.source,
|
| 292 |
-
)
|
| 293 |
-
|
| 294 |
-
|
| 295 |
-
def _build_structure(findings, task, profile: Profile) -> ImprovementPrompt:
|
| 296 |
-
repeats = any("repeat" in f.detail.lower() or "duplicate" in f.detail.lower() for f in findings)
|
| 297 |
-
rules = [
|
| 298 |
-
"Before writing, produce a 3-6 point outline; then write one section per point.",
|
| 299 |
-
"Each section must add new information — no restating earlier sections." if repeats
|
| 300 |
-
else "Order sections so each follows from the previous one.",
|
| 301 |
-
]
|
| 302 |
-
body = (
|
| 303 |
-
f"{_source_block(profile, task)}"
|
| 304 |
-
f"{task.instruction.strip()}\n\n"
|
| 305 |
-
f"{_rules_block(profile, rules)}"
|
| 306 |
-
f"{_tail_emphasis(profile, 'sections follow the outline and none repeats another')}"
|
| 307 |
-
)
|
| 308 |
-
return ImprovementPrompt(
|
| 309 |
-
failure_mode=FailureMode.POOR_STRUCTURE,
|
| 310 |
-
title="Impose an outline",
|
| 311 |
-
diagnosis=("Adjacent sections " + ("repeat each other" if repeats else "jump between topics without connection")
|
| 312 |
-
+ " (flagged by flow analysis). Repair: outline-first generation pins the structure before prose exists."),
|
| 313 |
-
prompt_text=body,
|
| 314 |
-
model_family=profile.family,
|
| 315 |
-
technique_source=profile.source,
|
| 316 |
-
)
|
| 317 |
-
|
| 318 |
-
|
| 319 |
-
def _build_drift(findings, task, profile: Profile) -> ImprovementPrompt:
|
| 320 |
-
drifting = _quotes(findings, 1)
|
| 321 |
-
body = (
|
| 322 |
-
f"{_source_block(profile, task)}"
|
| 323 |
-
f"{task.instruction.strip()}\n\n"
|
| 324 |
-
f"Stay strictly on this request. Do not add background, related topics, or general advice unless asked."
|
| 325 |
-
+ (f"\n\nA previous attempt drifted into: \"{drifting[0]}\" — leave this out unless the request explicitly needs it."
|
| 326 |
-
if drifting else "")
|
| 327 |
-
+ f"{_tail_emphasis(profile, 'every paragraph directly serves the request')}"
|
| 328 |
-
)
|
| 329 |
-
return ImprovementPrompt(
|
| 330 |
-
failure_mode=FailureMode.OFF_TOPIC_DRIFT,
|
| 331 |
-
title="Fence the scope",
|
| 332 |
-
diagnosis=(f"Part of the output drifts off the request"
|
| 333 |
-
+ (f' ("{drifting[0]}")' if drifting else "") +
|
| 334 |
-
". Repair: an explicit scope fence naming the drift to exclude."),
|
| 335 |
-
prompt_text=body,
|
| 336 |
-
model_family=profile.family,
|
| 337 |
-
technique_source=profile.source,
|
| 338 |
-
)
|
| 339 |
-
|
| 340 |
-
|
| 341 |
-
def _build_code(findings, task, profile: Profile) -> ImprovementPrompt:
|
| 342 |
-
problems = [f.detail for f in findings]
|
| 343 |
-
plist = "\n".join(f"- {p}" for p in problems[:4])
|
| 344 |
-
rules = [
|
| 345 |
-
"Return complete, runnable code — no TODOs, placeholders, or elided sections.",
|
| 346 |
-
"If the code needs assumptions (versions, inputs), state them in a comment at the top.",
|
| 347 |
-
"After the code, list 2-3 quick test cases I can run to verify it.",
|
| 348 |
-
]
|
| 349 |
-
body = (
|
| 350 |
-
f"{task.instruction.strip()}\n\n"
|
| 351 |
-
f"{_rules_block(profile, rules)}\n\n"
|
| 352 |
-
f"A previous attempt had these problems — fix each:\n{plist}"
|
| 353 |
-
f"{_tail_emphasis(profile, 'the code parses, is complete, and includes the requested names')}"
|
| 354 |
-
)
|
| 355 |
-
return ImprovementPrompt(
|
| 356 |
-
failure_mode=FailureMode.INVALID_CODE,
|
| 357 |
-
title="Demand complete, verifiable code",
|
| 358 |
-
diagnosis=("Static analysis found concrete defects: " + "; ".join(problems[:2]) + " "
|
| 359 |
-
"Repair: completeness rules plus self-supplied test cases give you an immediate way to check the redo."),
|
| 360 |
-
prompt_text=body,
|
| 361 |
-
model_family=profile.family,
|
| 362 |
-
technique_source=profile.source,
|
| 363 |
-
)
|
| 364 |
-
|
| 365 |
-
|
| 366 |
-
def _build_invented(findings, task, profile: Profile) -> ImprovementPrompt:
|
| 367 |
-
invented = _quotes(findings)
|
| 368 |
-
il = "\n".join(f'- "{q}"' for q in invented) if invented else ""
|
| 369 |
-
rules = [
|
| 370 |
-
"Copy every extracted value verbatim from the source text — no paraphrasing, no normalizing.",
|
| 371 |
-
"If a requested field is not in the text, output null for it. Never fill gaps from general knowledge.",
|
| 372 |
-
]
|
| 373 |
-
body = (
|
| 374 |
-
f"{_source_block(profile, task)}"
|
| 375 |
-
f"{task.instruction.strip()}\n\n"
|
| 376 |
-
f"{_rules_block(profile, rules)}\n\n"
|
| 377 |
-
+ (f"A previous attempt output values that are not in the source:\n{il}\n" if il else "")
|
| 378 |
-
+ f"{_tail_emphasis(profile, 'every value string-matches the source text or is null')}"
|
| 379 |
-
)
|
| 380 |
-
return ImprovementPrompt(
|
| 381 |
-
failure_mode=FailureMode.INVENTED_FIELDS,
|
| 382 |
-
title="Verbatim-only extraction",
|
| 383 |
-
diagnosis=("Extracted values don't appear in the source text"
|
| 384 |
-
+ (f', e.g. "{invented[0]}"' if invented else "") +
|
| 385 |
-
". Repair: verbatim-copy rule plus explicit null for missing fields — the two rules that stop "
|
| 386 |
-
"extraction models from 'helpfully' completing data."),
|
| 387 |
-
prompt_text=body,
|
| 388 |
-
model_family=profile.family,
|
| 389 |
-
technique_source=profile.source,
|
| 390 |
-
)
|
| 391 |
-
|
| 392 |
-
|
| 393 |
-
def _build_diversity(findings, task, profile: Profile) -> ImprovementPrompt:
|
| 394 |
-
cliches = _quotes(findings)
|
| 395 |
-
cl = ", ".join(f'"{c}"' for c in cliches[:5])
|
| 396 |
-
rules = [
|
| 397 |
-
(f"Banned phrases: {cl}. Find a fresh image instead." if cliches else
|
| 398 |
-
"Avoid stock phrases; when a familiar expression appears in your draft, replace it with a concrete image."),
|
| 399 |
-
"Vary sentence length deliberately: mix sentences under 8 words with longer ones.",
|
| 400 |
-
"Prefer specific, sensory detail over abstract description.",
|
| 401 |
-
]
|
| 402 |
-
body = (
|
| 403 |
-
f"{task.instruction.strip()}\n\n"
|
| 404 |
-
f"{_rules_block(profile, rules)}"
|
| 405 |
-
f"{_tail_emphasis(profile, 'no banned phrase appears and sentence lengths vary')}"
|
| 406 |
-
)
|
| 407 |
-
return ImprovementPrompt(
|
| 408 |
-
failure_mode=FailureMode.LOW_DIVERSITY,
|
| 409 |
-
title="Ban the clichés it reached for",
|
| 410 |
-
diagnosis=("Style statistics flagged " + (f"stock phrases ({cl})" if cliches else "flat, repetitive prose") +
|
| 411 |
-
". Repair: ban the exact phrases found and constrain rhythm — negative examples work when they are this specific."),
|
| 412 |
-
prompt_text=body,
|
| 413 |
-
model_family=profile.family,
|
| 414 |
-
technique_source=profile.source,
|
| 415 |
-
)
|
| 416 |
-
|
| 417 |
-
|
| 418 |
-
_BUILDERS = {
|
| 419 |
-
FailureMode.UNFAITHFUL_CONTENT: _build_unfaithful,
|
| 420 |
-
FailureMode.MISSED_CONSTRAINT: _build_constraints,
|
| 421 |
-
FailureMode.FORMAT_VIOLATION: _build_constraints,
|
| 422 |
-
FailureMode.INCOMPLETE_COVERAGE: _build_coverage,
|
| 423 |
-
FailureMode.HALLUCINATION_RISK: _build_hallucination,
|
| 424 |
-
FailureMode.WEAK_REASONING: _build_reasoning,
|
| 425 |
-
FailureMode.VERBOSITY: _build_verbosity,
|
| 426 |
-
FailureMode.WRONG_REGISTER: _build_register,
|
| 427 |
-
FailureMode.POOR_STRUCTURE: _build_structure,
|
| 428 |
-
FailureMode.OFF_TOPIC_DRIFT: _build_drift,
|
| 429 |
-
FailureMode.INVALID_CODE: _build_code,
|
| 430 |
-
FailureMode.INVENTED_FIELDS: _build_invented,
|
| 431 |
-
FailureMode.LOW_DIVERSITY: _build_diversity,
|
| 432 |
-
}
|
| 433 |
|
| 434 |
|
| 435 |
def generate_improvement_prompts(
|
|
@@ -443,16 +233,54 @@ def generate_improvement_prompts(
|
|
| 443 |
for ps in parameters:
|
| 444 |
for f in ps.findings:
|
| 445 |
by_mode.setdefault(f.failure_mode, []).append(f)
|
| 446 |
-
|
| 447 |
-
|
| 448 |
-
by_mode[FailureMode.MISSED_CONSTRAINT].extend(by_mode.pop(FailureMode.FORMAT_VIOLATION))
|
| 449 |
|
| 450 |
-
|
|
|
|
|
|
|
| 451 |
for mode in _SEVERITY:
|
| 452 |
-
if mode
|
| 453 |
-
|
| 454 |
-
|
| 455 |
-
|
| 456 |
-
|
| 457 |
-
|
| 458 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
"""Improvement-prompt generation — the flagship.
|
| 2 |
|
| 3 |
+
Contract (RESEARCH.md Part 4, enforced by tests): the generator produces ONE
|
| 4 |
+
unified repair prompt that addresses every diagnosed failure at once —
|
| 5 |
+
(a) the diagnosis names each failure with the evidence that triggered it,
|
| 6 |
(b) prompt_text is a complete, ready-to-paste replacement prompt,
|
| 7 |
+
(c) the repairs use the named model family's documented mechanisms.
|
| 8 |
"Be more specific" is a bug, not a template.
|
| 9 |
|
| 10 |
+
Each failure mode contributes pieces (a role line, hard rules, previous-
|
| 11 |
+
attempt problems, a pre-instruction) and the composer assembles them around
|
| 12 |
+
the user's original instruction.
|
| 13 |
"""
|
| 14 |
|
| 15 |
from __future__ import annotations
|
| 16 |
|
| 17 |
+
from dataclasses import dataclass, field
|
| 18 |
+
|
| 19 |
+
from ..schemas import (FailureMode, ImprovementPrompt, ModelFamily,
|
| 20 |
+
ParameterScore, TaskClassification)
|
| 21 |
from .model_profiles import Profile, get_profile
|
| 22 |
|
| 23 |
_SEVERITY = [
|
|
|
|
| 36 |
FailureMode.LOW_DIVERSITY,
|
| 37 |
]
|
| 38 |
|
| 39 |
+
_MODE_NAMES = {
|
| 40 |
+
FailureMode.UNFAITHFUL_CONTENT: "content the source does not support",
|
| 41 |
+
FailureMode.INVALID_CODE: "defective code",
|
| 42 |
+
FailureMode.INVENTED_FIELDS: "extracted values not present in the source",
|
| 43 |
+
FailureMode.MISSED_CONSTRAINT: "missed constraints",
|
| 44 |
+
FailureMode.FORMAT_VIOLATION: "format violations",
|
| 45 |
+
FailureMode.INCOMPLETE_COVERAGE: "unanswered parts of the request",
|
| 46 |
+
FailureMode.HALLUCINATION_RISK: "unverifiable claims stated as fact",
|
| 47 |
+
FailureMode.OFF_TOPIC_DRIFT: "off-topic drift",
|
| 48 |
+
FailureMode.WEAK_REASONING: "reasoning without checkable working",
|
| 49 |
+
FailureMode.VERBOSITY: "padding",
|
| 50 |
+
FailureMode.WRONG_REGISTER: "the wrong reading level for the audience",
|
| 51 |
+
FailureMode.POOR_STRUCTURE: "structural problems",
|
| 52 |
+
FailureMode.LOW_DIVERSITY: "flat, cliché-heavy style",
|
| 53 |
+
}
|
| 54 |
+
|
| 55 |
+
MAX_RULES = 9
|
| 56 |
+
MAX_PROBLEMS = 7
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
@dataclass
|
| 60 |
+
class _Parts:
|
| 61 |
+
role: str = ""
|
| 62 |
+
pre: str = "" # grounding / think-first instruction
|
| 63 |
+
rules: list[str] = field(default_factory=list)
|
| 64 |
+
problems: list[str] = field(default_factory=list) # "previous attempt" lines
|
| 65 |
+
wants_example: bool = False # gemini few-shot skeleton
|
| 66 |
+
criticals: list[str] = field(default_factory=list) # tail-emphasis candidates
|
| 67 |
+
sources: list[str] = field(default_factory=list)
|
| 68 |
|
| 69 |
|
| 70 |
+
def _quotes(findings, limit=3, source="output") -> list[str]:
|
| 71 |
out = []
|
| 72 |
for f in findings:
|
| 73 |
for e in f.evidence:
|
| 74 |
+
if e.source == source and e.quote:
|
| 75 |
out.append(e.quote)
|
| 76 |
break
|
| 77 |
if len(out) >= limit:
|
|
|
|
| 79 |
return out
|
| 80 |
|
| 81 |
|
| 82 |
+
# ------------------------------------------------------ per-mode contributions
|
| 83 |
+
|
| 84 |
+
def _contribute(mode: FailureMode, findings, task, profile: Profile, parts: _Parts) -> str | None:
|
| 85 |
+
"""Fold this mode's repair into parts. Returns a diagnosis fragment, or
|
| 86 |
+
None when the mode has nothing usable to say."""
|
| 87 |
+
fam = profile.family
|
| 88 |
+
|
| 89 |
+
if mode == FailureMode.UNFAITHFUL_CONTENT:
|
| 90 |
+
bad = _quotes(findings)
|
| 91 |
+
if fam == ModelFamily.CLAUDE:
|
| 92 |
+
parts.pre = ("First, copy the exact sentences from the source that support your answer into "
|
| 93 |
+
"<quotes> tags. Then write the final answer in <answer> tags using only what you quoted.")
|
| 94 |
+
elif fam == ModelFamily.GEMINI:
|
| 95 |
+
parts.pre = "Base every sentence of your answer on the source text above; quote or closely paraphrase it."
|
| 96 |
+
else:
|
| 97 |
+
parts.pre = "Answer using only the provided source text. Quote the source where possible."
|
| 98 |
+
parts.rules.append("Use ONLY information stated in the source text. If the source does not say it, do not write it.")
|
| 99 |
+
parts.rules.append('If something the task needs is missing from the source, write "the source does not specify" rather than filling the gap.')
|
| 100 |
+
parts.problems += [f'invented content: "{q}"' for q in bad]
|
| 101 |
+
parts.criticals.append("every claim traces to a sentence in the source text")
|
| 102 |
+
parts.sources.append(profile.source)
|
| 103 |
+
return "content the source does not support" + (f' (e.g. "{bad[0]}")' if bad else "")
|
| 104 |
+
|
| 105 |
+
if mode == FailureMode.INVALID_CODE:
|
| 106 |
+
parts.rules.append("Return complete, runnable code — no TODOs, placeholders, or elided sections.")
|
| 107 |
+
parts.rules.append("After the code, list 2-3 quick test cases I can run to verify it.")
|
| 108 |
+
parts.problems += [f.detail for f in findings[:3]]
|
| 109 |
+
parts.criticals.append("the code parses, is complete, and includes every requested name")
|
| 110 |
+
parts.sources.append(profile.source)
|
| 111 |
+
return "concrete code defects (" + "; ".join(f.detail.rstrip(".") for f in findings[:2]) + ")"
|
| 112 |
+
|
| 113 |
+
if mode == FailureMode.INVENTED_FIELDS:
|
| 114 |
+
invented = _quotes(findings)
|
| 115 |
+
parts.rules.append("Copy every extracted value verbatim from the source text — no paraphrasing, no normalizing.")
|
| 116 |
+
parts.rules.append("If a requested field is not in the text, output null for it. Never fill gaps from general knowledge.")
|
| 117 |
+
parts.problems += [f'value not in the source: "{q}"' for q in invented]
|
| 118 |
+
parts.criticals.append("every value string-matches the source text or is null")
|
| 119 |
+
parts.sources.append(profile.source)
|
| 120 |
+
return "extracted values that are not in the source" + (f' (e.g. "{invented[0]}")' if invented else "")
|
| 121 |
+
|
| 122 |
+
if mode in (FailureMode.MISSED_CONSTRAINT, FailureMode.FORMAT_VIOLATION):
|
| 123 |
+
for f in findings:
|
| 124 |
+
req = f.detail.split(" — ")[0].removeprefix("Prompt asked for ").strip().capitalize()
|
| 125 |
+
parts.rules.append(req + ". Count and check this before giving your final answer.")
|
| 126 |
+
parts.problems.append(f.detail)
|
| 127 |
+
parts.criticals.insert(0, req.lower())
|
| 128 |
+
parts.wants_example = fam == ModelFamily.GEMINI
|
| 129 |
+
parts.sources.append(profile.source)
|
| 130 |
+
return f"{len(findings)} verifiable constraint(s) missed"
|
| 131 |
+
|
| 132 |
+
if mode == FailureMode.INCOMPLETE_COVERAGE:
|
| 133 |
+
qparts = _quotes(findings, limit=6, source="prompt")
|
| 134 |
+
if not qparts:
|
| 135 |
+
return None
|
| 136 |
+
numbered = " ".join(f"({i + 1}) {p}" for i, p in enumerate(qparts))
|
| 137 |
+
parts.rules.append(f"Answer every part of the request separately, under its own heading: {numbered}. "
|
| 138 |
+
"If you cannot answer a part, say so under that heading rather than skipping it.")
|
| 139 |
+
parts.problems += [f'left unanswered: "{p}"' for p in qparts[:3]]
|
| 140 |
+
parts.criticals.append("every numbered part has its own answer")
|
| 141 |
+
parts.sources.append(profile.source)
|
| 142 |
+
return f"{len(qparts)} part(s) of the request left unanswered"
|
| 143 |
+
|
| 144 |
+
if mode == FailureMode.HALLUCINATION_RISK:
|
| 145 |
+
flagged = _quotes(findings)
|
| 146 |
+
parts.rules.append("For every statistic or citation, name its real source inline, or state it as an "
|
| 147 |
+
'estimate in words ("roughly", "on the order of"). Never invent citations or URLs.')
|
| 148 |
+
parts.rules.append('Mark claims you are unsure about explicitly, and end with a one-line note: '
|
| 149 |
+
"which claims should I double-check independently?")
|
| 150 |
+
parts.problems += [f'stated as fact but unverifiable: "{q}"' for q in flagged]
|
| 151 |
+
parts.criticals.append("no statistic or citation appears without a named source or an uncertainty marker")
|
| 152 |
+
parts.sources.append(profile.source)
|
| 153 |
+
return "unverifiable specifics stated as fact" + (f' (e.g. "{flagged[0]}")' if flagged else "")
|
| 154 |
+
|
| 155 |
+
if mode == FailureMode.OFF_TOPIC_DRIFT:
|
| 156 |
+
drifting = _quotes(findings, 1)
|
| 157 |
+
parts.rules.append("Stay strictly on the request. No background, related topics, or general advice unless asked.")
|
| 158 |
+
if drifting:
|
| 159 |
+
parts.problems.append(f'drifted into: "{drifting[0]}"')
|
| 160 |
+
parts.criticals.append("every paragraph directly serves the request")
|
| 161 |
+
parts.sources.append(profile.source)
|
| 162 |
+
return "off-topic drift"
|
| 163 |
+
|
| 164 |
+
if mode == FailureMode.WEAK_REASONING:
|
| 165 |
+
if not parts.pre:
|
| 166 |
+
parts.pre = ("Work through the problem step by step inside <thinking> tags first, then give the final "
|
| 167 |
+
"answer in <answer> tags." if fam == ModelFamily.CLAUDE else
|
| 168 |
+
'Solve this step by step, each step on its own line, then state the final answer on a '
|
| 169 |
+
'last line beginning "Answer:".')
|
| 170 |
+
parts.rules.append("Show the working before the answer; if possible, re-derive the answer a second way and confirm both agree.")
|
| 171 |
+
parts.criticals.append("working shown, final answer explicitly labeled")
|
| 172 |
+
parts.sources.append("Wei et al. 2022 (CoT); Wang et al. 2023 (self-consistency)")
|
| 173 |
+
return "a conclusion without checkable working"
|
| 174 |
+
|
| 175 |
+
if mode == FailureMode.VERBOSITY:
|
| 176 |
+
fillers = _quotes(findings)
|
| 177 |
+
parts.rules.append("Open with the answer itself — no preamble, no restating the question."
|
| 178 |
+
+ (" Banned filler: " + ", ".join(f'"{q}"' for q in fillers[:3]) + "." if fillers else ""))
|
| 179 |
+
parts.problems += [f'filler: "{q}"' for q in fillers[:2]]
|
| 180 |
+
parts.criticals.append("the response starts with substance and contains no filler")
|
| 181 |
+
parts.sources.append(profile.source)
|
| 182 |
+
return "padding" + (f' (e.g. "{fillers[0]}")' if fillers else "")
|
| 183 |
+
|
| 184 |
+
if mode == FailureMode.WRONG_REGISTER:
|
| 185 |
+
data = findings[0].data
|
| 186 |
+
fk, band = data.get("fk_grade"), data.get("band", [6, 14])
|
| 187 |
+
audience = data.get("audience", "the intended audience")
|
| 188 |
+
too_hard = fk is not None and fk > band[1]
|
| 189 |
+
style = ("short sentences, everyday words, one idea at a time" if too_hard
|
| 190 |
+
else "precise terminology and compact, information-dense prose")
|
| 191 |
+
parts.role = f"You are writing for {audience}. Use {style}."
|
| 192 |
+
parts.criticals.append(f"the text reads naturally for {audience}")
|
| 193 |
+
parts.sources.append("role prompting for register (RESEARCH.md 2.8)")
|
| 194 |
+
return f"reading level {fk} outside the {band[0]:g}-{band[1]:g} band for {audience}"
|
| 195 |
+
|
| 196 |
+
if mode == FailureMode.POOR_STRUCTURE:
|
| 197 |
+
repeats = any("repeat" in f.detail.lower() or "duplicate" in f.detail.lower() for f in findings)
|
| 198 |
+
parts.rules.append("Before writing, produce a short outline; then write one section per point"
|
| 199 |
+
+ (" — no section may restate an earlier one." if repeats else ", each following from the previous."))
|
| 200 |
+
parts.sources.append(profile.source)
|
| 201 |
+
return "sections that repeat each other" if repeats else "abrupt jumps between sections"
|
| 202 |
+
|
| 203 |
+
if mode == FailureMode.LOW_DIVERSITY:
|
| 204 |
+
cliches = _quotes(findings)
|
| 205 |
+
parts.rules.append(("Banned phrases: " + ", ".join(f'"{c}"' for c in cliches[:5]) + ". Find a fresh image instead."
|
| 206 |
+
if cliches else "Replace stock phrases with concrete, sensory detail.")
|
| 207 |
+
+ " Vary sentence length deliberately.")
|
| 208 |
+
parts.problems += [f'stock phrase: "{c}"' for c in cliches[:2]]
|
| 209 |
+
parts.sources.append(profile.source)
|
| 210 |
+
return "flat style" + (f" (stock phrases like {cliches[0]!r})" if cliches else "")
|
| 211 |
+
|
| 212 |
+
return None
|
| 213 |
+
|
| 214 |
+
|
| 215 |
+
# --------------------------------------------------------------- composition
|
| 216 |
|
| 217 |
def _rules_block(profile: Profile, rules: list[str]) -> str:
|
|
|
|
| 218 |
if profile.family == ModelFamily.CLAUDE:
|
| 219 |
+
return "<rules>\n" + "\n".join(f"- {r}" for r in rules) + "\n</rules>"
|
|
|
|
| 220 |
if profile.family == ModelFamily.GPT:
|
| 221 |
+
return "### Requirements\n" + "\n".join(f"{i + 1}. {r}" for i, r in enumerate(rules))
|
| 222 |
+
return "Rules:\n" + "\n".join(f"- {r}" for r in rules)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 223 |
|
| 224 |
|
| 225 |
def generate_improvement_prompts(
|
|
|
|
| 233 |
for ps in parameters:
|
| 234 |
for f in ps.findings:
|
| 235 |
by_mode.setdefault(f.failure_mode, []).append(f)
|
| 236 |
+
if not by_mode:
|
| 237 |
+
return []
|
|
|
|
| 238 |
|
| 239 |
+
parts = _Parts()
|
| 240 |
+
fragments: list[str] = []
|
| 241 |
+
lead_mode: FailureMode | None = None
|
| 242 |
for mode in _SEVERITY:
|
| 243 |
+
if mode not in by_mode:
|
| 244 |
+
continue
|
| 245 |
+
fragment = _contribute(mode, by_mode[mode], task, profile, parts)
|
| 246 |
+
if fragment:
|
| 247 |
+
fragments.append(fragment)
|
| 248 |
+
lead_mode = lead_mode or mode
|
| 249 |
+
if lead_mode is None:
|
| 250 |
+
return []
|
| 251 |
+
|
| 252 |
+
parts.rules = list(dict.fromkeys(parts.rules))[:MAX_RULES]
|
| 253 |
+
parts.problems = list(dict.fromkeys(parts.problems))[:MAX_PROBLEMS]
|
| 254 |
+
|
| 255 |
+
segments: list[str] = []
|
| 256 |
+
if parts.role:
|
| 257 |
+
segments.append(parts.role)
|
| 258 |
+
if task.has_source_text:
|
| 259 |
+
segments.append(profile.wrap_source())
|
| 260 |
+
segments.append(task.instruction.strip())
|
| 261 |
+
if parts.pre:
|
| 262 |
+
segments.append(parts.pre)
|
| 263 |
+
if parts.rules:
|
| 264 |
+
segments.append(_rules_block(profile, parts.rules))
|
| 265 |
+
if parts.wants_example:
|
| 266 |
+
segments.append("Here is an example of a response with the right shape (replace the content, keep the form):\n"
|
| 267 |
+
"[write one short example that satisfies every rule above]")
|
| 268 |
+
if parts.problems:
|
| 269 |
+
segments.append("A previous attempt had these problems — do not repeat any of them:\n"
|
| 270 |
+
+ "\n".join(f"- {p}" for p in parts.problems))
|
| 271 |
+
if parts.criticals and profile.family in (ModelFamily.GPT, ModelFamily.OPEN_WEIGHTS, ModelFamily.GENERIC):
|
| 272 |
+
# lost-in-the-middle: repeat the most critical checks at the very end
|
| 273 |
+
segments.append("IMPORTANT — before finishing, verify: " + "; ".join(parts.criticals[:3]) + ".")
|
| 274 |
+
|
| 275 |
+
diagnosis = (f"{len(fragments)} problem(s) diagnosed: " + "; ".join(fragments) + ". "
|
| 276 |
+
f"This single prompt rebuilds your original instruction to fix all of them, using "
|
| 277 |
+
f"{profile.display} mechanisms — {profile.structure_hint}")
|
| 278 |
+
|
| 279 |
+
return [ImprovementPrompt(
|
| 280 |
+
failure_mode=lead_mode,
|
| 281 |
+
title="The repaired prompt",
|
| 282 |
+
diagnosis=diagnosis,
|
| 283 |
+
prompt_text="\n\n".join(segments),
|
| 284 |
+
model_family=profile.family,
|
| 285 |
+
technique_source="; ".join(dict.fromkeys(parts.sources)),
|
| 286 |
+
)]
|
tests/test_prompts.py
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
-
"""The flagship contract:
|
| 2 |
-
problem
|
| 3 |
-
|
| 4 |
|
| 5 |
import samples
|
| 6 |
from rubriq.schemas import EvalRequest, FailureMode, ModelFamily
|
|
@@ -18,78 +18,72 @@ def run(prompt, output, model=""):
|
|
| 18 |
return scorer.evaluate(EvalRequest(prompt=prompt, output=output, model_name=model))
|
| 19 |
|
| 20 |
|
| 21 |
-
def
|
| 22 |
-
report =
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
|
| 26 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 27 |
|
| 28 |
|
| 29 |
def test_no_generic_advice():
|
| 30 |
for pair in [(samples.SUMM_PROMPT, samples.SUMM_BAD), (samples.CODE_PROMPT, samples.CODE_BAD)]:
|
| 31 |
-
|
| 32 |
-
|
| 33 |
-
|
| 34 |
-
|
| 35 |
-
|
| 36 |
|
| 37 |
|
| 38 |
-
def
|
| 39 |
-
|
| 40 |
-
|
| 41 |
-
|
| 42 |
-
# The repair prompt must carry the actual invented claims back to the model.
|
| 43 |
-
assert '"' in unfaithful.prompt_text
|
| 44 |
-
assert "previous attempt" in unfaithful.prompt_text.lower()
|
| 45 |
|
| 46 |
|
| 47 |
def test_prompt_contains_original_instruction():
|
| 48 |
-
|
| 49 |
-
|
| 50 |
-
assert "60 words" in ip.prompt_text or "Summarize" in ip.prompt_text, \
|
| 51 |
-
"repair prompts must be complete replacements, not advice"
|
| 52 |
|
| 53 |
|
| 54 |
def test_claude_gets_xml_mechanism():
|
| 55 |
-
|
| 56 |
-
|
| 57 |
-
|
| 58 |
-
assert unfaithful.model_family == ModelFamily.CLAUDE
|
| 59 |
-
assert "<quotes>" in unfaithful.prompt_text or "<source>" in unfaithful.prompt_text
|
| 60 |
|
| 61 |
|
| 62 |
def test_gpt_gets_delimiters_and_tail_emphasis():
|
| 63 |
-
|
| 64 |
-
|
| 65 |
-
|
| 66 |
-
assert
|
| 67 |
-
assert "IMPORTANT" in unfaithful.prompt_text or "###" in unfaithful.prompt_text
|
| 68 |
|
| 69 |
|
| 70 |
def test_gemini_gets_fewshot_for_format():
|
| 71 |
-
|
| 72 |
-
|
| 73 |
-
|
| 74 |
-
if constraint:
|
| 75 |
-
assert "example" in constraint.prompt_text.lower()
|
| 76 |
|
| 77 |
|
| 78 |
def test_code_repair_names_defects():
|
| 79 |
-
|
| 80 |
-
|
| 81 |
-
assert "
|
| 82 |
-
assert "fix each" in code.prompt_text.lower()
|
| 83 |
|
| 84 |
|
| 85 |
def test_no_empty_coverage_repair_on_code():
|
| 86 |
-
"""Regression: missing requested function names must
|
| 87 |
-
|
| 88 |
-
|
| 89 |
-
|
| 90 |
-
|
| 91 |
-
if ip.failure_mode == FailureMode.INCOMPLETE_COVERAGE:
|
| 92 |
-
assert 'e.g. ""' not in ip.diagnosis
|
| 93 |
|
| 94 |
|
| 95 |
def test_family_detection_current_models():
|
|
@@ -113,6 +107,5 @@ def test_family_detection_current_models():
|
|
| 113 |
|
| 114 |
|
| 115 |
def test_technique_sources_cited():
|
| 116 |
-
|
| 117 |
-
|
| 118 |
-
assert ip.technique_source.strip(), "every template must cite its source"
|
|
|
|
| 1 |
+
"""The flagship contract: ONE unified repair prompt that names every
|
| 2 |
+
diagnosed problem with evidence, is paste-ready, and uses the named model
|
| 3 |
+
family's mechanisms. Generic advice is a test failure by design."""
|
| 4 |
|
| 5 |
import samples
|
| 6 |
from rubriq.schemas import EvalRequest, FailureMode, ModelFamily
|
|
|
|
| 18 |
return scorer.evaluate(EvalRequest(prompt=prompt, output=output, model_name=model))
|
| 19 |
|
| 20 |
|
| 21 |
+
def one(report):
|
| 22 |
+
assert len(report.improvement_prompts) == 1, \
|
| 23 |
+
f"expected exactly one unified repair prompt, got {len(report.improvement_prompts)}"
|
| 24 |
+
return report.improvement_prompts[0]
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def test_single_unified_prompt():
|
| 28 |
+
rep = run(samples.SUMM_PROMPT, samples.SUMM_BAD, model="OpenAI GPT-5.5")
|
| 29 |
+
ip = one(rep)
|
| 30 |
+
# the diagnosis names ALL the problem classes, not just one
|
| 31 |
+
assert "source does not support" in ip.diagnosis
|
| 32 |
+
assert "constraint" in ip.diagnosis
|
| 33 |
+
assert ip.failure_mode == FailureMode.UNFAITHFUL_CONTENT # most severe leads
|
| 34 |
|
| 35 |
|
| 36 |
def test_no_generic_advice():
|
| 37 |
for pair in [(samples.SUMM_PROMPT, samples.SUMM_BAD), (samples.CODE_PROMPT, samples.CODE_BAD)]:
|
| 38 |
+
rep = run(*pair, model="OpenAI GPT-5.5")
|
| 39 |
+
ip = one(rep)
|
| 40 |
+
low = (ip.diagnosis + " " + ip.prompt_text).lower()
|
| 41 |
+
for phrase in GENERIC_PHRASES:
|
| 42 |
+
assert phrase not in low, f"generic advice '{phrase}'"
|
| 43 |
|
| 44 |
|
| 45 |
+
def test_prompt_carries_evidence_back():
|
| 46 |
+
ip = one(run(samples.SUMM_PROMPT, samples.SUMM_BAD, model="OpenAI GPT-5.5"))
|
| 47 |
+
assert "previous attempt" in ip.prompt_text.lower()
|
| 48 |
+
assert "fifteen brave scientists" in ip.prompt_text # the invented claim travels into the repair
|
|
|
|
|
|
|
|
|
|
| 49 |
|
| 50 |
|
| 51 |
def test_prompt_contains_original_instruction():
|
| 52 |
+
ip = one(run(samples.SUMM_PROMPT, samples.SUMM_BAD, model="OpenAI GPT-5.5"))
|
| 53 |
+
assert "Summarize" in ip.prompt_text and "60 words" in ip.prompt_text
|
|
|
|
|
|
|
| 54 |
|
| 55 |
|
| 56 |
def test_claude_gets_xml_mechanism():
|
| 57 |
+
ip = one(run(samples.SUMM_PROMPT, samples.SUMM_BAD, model="Anthropic Claude Sonnet 5"))
|
| 58 |
+
assert ip.model_family == ModelFamily.CLAUDE
|
| 59 |
+
assert "<quotes>" in ip.prompt_text and "<rules>" in ip.prompt_text
|
|
|
|
|
|
|
| 60 |
|
| 61 |
|
| 62 |
def test_gpt_gets_delimiters_and_tail_emphasis():
|
| 63 |
+
ip = one(run(samples.SUMM_PROMPT, samples.SUMM_BAD, model="OpenAI GPT-5.5"))
|
| 64 |
+
assert ip.model_family == ModelFamily.GPT
|
| 65 |
+
assert "### Requirements" in ip.prompt_text
|
| 66 |
+
assert "IMPORTANT" in ip.prompt_text
|
|
|
|
| 67 |
|
| 68 |
|
| 69 |
def test_gemini_gets_fewshot_for_format():
|
| 70 |
+
ip = one(run(samples.SUMM_PROMPT, samples.SUMM_BAD, model="Google Gemini 3.5 Flash"))
|
| 71 |
+
assert ip.model_family == ModelFamily.GEMINI
|
| 72 |
+
assert "example" in ip.prompt_text.lower()
|
|
|
|
|
|
|
| 73 |
|
| 74 |
|
| 75 |
def test_code_repair_names_defects():
|
| 76 |
+
ip = one(run(samples.CODE_PROMPT, samples.CODE_BAD))
|
| 77 |
+
assert "parse_log_line" in ip.prompt_text
|
| 78 |
+
assert "does not parse" in ip.prompt_text or "does not parse" in ip.diagnosis
|
|
|
|
| 79 |
|
| 80 |
|
| 81 |
def test_no_empty_coverage_repair_on_code():
|
| 82 |
+
"""Regression: missing requested function names must never surface as a
|
| 83 |
+
'0 parts unanswered' coverage diagnosis."""
|
| 84 |
+
ip = one(run(samples.CODE_PROMPT, samples.CODE_BAD))
|
| 85 |
+
assert "0 part(s)" not in ip.diagnosis
|
| 86 |
+
assert 'e.g. ""' not in ip.diagnosis
|
|
|
|
|
|
|
| 87 |
|
| 88 |
|
| 89 |
def test_family_detection_current_models():
|
|
|
|
| 107 |
|
| 108 |
|
| 109 |
def test_technique_sources_cited():
|
| 110 |
+
ip = one(run(samples.SUMM_PROMPT, samples.SUMM_BAD, model="OpenAI GPT-5.5"))
|
| 111 |
+
assert ip.technique_source.strip()
|
|
|
web/app.js
CHANGED
|
@@ -231,10 +231,10 @@
|
|
| 231 |
<h2 class="sect">Parameter verdicts <span class="count">· ${scored.length} scored, ${nFindings} findings</span></h2>
|
| 232 |
${rep.parameters.map(verdictHtml).join("")}
|
| 233 |
|
| 234 |
-
<h2 class="sect">
|
| 235 |
${rep.improvement_prompts.length
|
| 236 |
? rep.improvement_prompts.map(repairHtml).join("")
|
| 237 |
-
: '<p class="no-repairs">No
|
| 238 |
</div>`;
|
| 239 |
|
| 240 |
// meter fill animation (double rAF so the 0-scale paints first)
|
|
@@ -268,7 +268,10 @@
|
|
| 268 |
prompt: $("promptIn").value,
|
| 269 |
output: $("outputIn").value,
|
| 270 |
model_name: composedModelName(),
|
| 271 |
-
options:
|
|
|
|
|
|
|
|
|
|
| 272 |
};
|
| 273 |
evalBtn.disabled = true;
|
| 274 |
evalBtn.textContent = "Evaluating…";
|
|
|
|
| 231 |
<h2 class="sect">Parameter verdicts <span class="count">· ${scored.length} scored, ${nFindings} findings</span></h2>
|
| 232 |
${rep.parameters.map(verdictHtml).join("")}
|
| 233 |
|
| 234 |
+
<h2 class="sect">The repaired prompt <span class="count">· one paste-ready fix for everything found · ${esc(FAMILY_LABELS[(rep.improvement_prompts[0] || {}).model_family] || "your model")}</span></h2>
|
| 235 |
${rep.improvement_prompts.length
|
| 236 |
? rep.improvement_prompts.map(repairHtml).join("")
|
| 237 |
+
: '<p class="no-repairs">No repair needed — every locally checkable parameter came back clean. If the output still feels wrong, the issue likely lives in the judge-mode territory listed above.</p>'}
|
| 238 |
</div>`;
|
| 239 |
|
| 240 |
// meter fill animation (double rAF so the 0-scale paints first)
|
|
|
|
| 268 |
prompt: $("promptIn").value,
|
| 269 |
output: $("outputIn").value,
|
| 270 |
model_name: composedModelName(),
|
| 271 |
+
options: (() => {
|
| 272 |
+
const tone = form.querySelector('input[name="tone"]:checked');
|
| 273 |
+
return { audience: tone?.dataset.audience || "", tone: tone?.value || "", weights: {} };
|
| 274 |
+
})(),
|
| 275 |
};
|
| 276 |
evalBtn.disabled = true;
|
| 277 |
evalBtn.textContent = "Evaluating…";
|
web/index.html
CHANGED
|
@@ -58,17 +58,27 @@
|
|
| 58 |
</div>
|
| 59 |
</fieldset>
|
| 60 |
|
| 61 |
-
<
|
| 62 |
-
<
|
| 63 |
-
<div class="
|
| 64 |
-
<label
|
| 65 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 66 |
</div>
|
| 67 |
-
|
| 68 |
-
<label for="toneIn">Expected tone</label>
|
| 69 |
-
<input id="toneIn" placeholder="e.g. formal, playful, neutral">
|
| 70 |
-
</div>
|
| 71 |
-
</details>
|
| 72 |
|
| 73 |
<div class="actions">
|
| 74 |
<button type="submit" class="btn btn-primary" id="evalBtn">Evaluate output</button>
|
|
|
|
| 58 |
</div>
|
| 59 |
</fieldset>
|
| 60 |
|
| 61 |
+
<fieldset class="field field-tone">
|
| 62 |
+
<legend>Expected tone <span class="opt">sets the reading-level check</span></legend>
|
| 63 |
+
<div class="tone-row" role="radiogroup" aria-label="Expected tone">
|
| 64 |
+
<label class="tone-chip">
|
| 65 |
+
<input type="radio" name="tone" value="" data-audience="" checked>
|
| 66 |
+
<span>Neutral</span>
|
| 67 |
+
</label>
|
| 68 |
+
<label class="tone-chip">
|
| 69 |
+
<input type="radio" name="tone" value="simple, plain language" data-audience="general readers">
|
| 70 |
+
<span>Simple</span>
|
| 71 |
+
</label>
|
| 72 |
+
<label class="tone-chip">
|
| 73 |
+
<input type="radio" name="tone" value="professional" data-audience="professional">
|
| 74 |
+
<span>Professional</span>
|
| 75 |
+
</label>
|
| 76 |
+
<label class="tone-chip">
|
| 77 |
+
<input type="radio" name="tone" value="technical" data-audience="expert">
|
| 78 |
+
<span>Technical</span>
|
| 79 |
+
</label>
|
| 80 |
</div>
|
| 81 |
+
</fieldset>
|
|
|
|
|
|
|
|
|
|
|
|
|
| 82 |
|
| 83 |
<div class="actions">
|
| 84 |
<button type="submit" class="btn btn-primary" id="evalBtn">Evaluate output</button>
|
web/styles.css
CHANGED
|
@@ -248,18 +248,43 @@ fieldset.field-model legend {
|
|
| 248 |
.combo-list li.co-empty { color: var(--muted); cursor: default; }
|
| 249 |
.combo-list li.co-empty:hover { background: transparent; }
|
| 250 |
|
| 251 |
-
|
| 252 |
-
.
|
| 253 |
-
|
| 254 |
font-size: 13px;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 255 |
font-weight: 500;
|
| 256 |
-
|
| 257 |
-
|
| 258 |
-
|
| 259 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 260 |
}
|
| 261 |
-
.advanced summary:hover { color: var(--ink); }
|
| 262 |
-
.advanced[open] summary { margin-bottom: 10px; }
|
| 263 |
|
| 264 |
.actions { display: flex; gap: 10px; margin: 20px 0 0; }
|
| 265 |
.btn {
|
|
|
|
| 248 |
.combo-list li.co-empty { color: var(--muted); cursor: default; }
|
| 249 |
.combo-list li.co-empty:hover { background: transparent; }
|
| 250 |
|
| 251 |
+
/* tone selector */
|
| 252 |
+
fieldset.field-tone { border: 0; padding: 0; margin: 0 0 16px; }
|
| 253 |
+
fieldset.field-tone legend {
|
| 254 |
font-size: 13px;
|
| 255 |
+
font-weight: 600;
|
| 256 |
+
margin-bottom: 6px;
|
| 257 |
+
padding: 0;
|
| 258 |
+
}
|
| 259 |
+
.tone-row { display: flex; flex-wrap: wrap; gap: 8px; }
|
| 260 |
+
.tone-chip input {
|
| 261 |
+
position: absolute;
|
| 262 |
+
opacity: 0;
|
| 263 |
+
pointer-events: none;
|
| 264 |
+
}
|
| 265 |
+
.tone-chip span {
|
| 266 |
+
display: inline-block;
|
| 267 |
+
border: 1px solid var(--line);
|
| 268 |
+
background: var(--bg);
|
| 269 |
+
color: var(--ink);
|
| 270 |
+
border-radius: 999px;
|
| 271 |
+
padding: 5px 14px;
|
| 272 |
+
font-size: 12.5px;
|
| 273 |
font-weight: 500;
|
| 274 |
+
cursor: pointer;
|
| 275 |
+
transition: border-color 150ms var(--ease), background 150ms var(--ease), color 150ms var(--ease);
|
| 276 |
+
}
|
| 277 |
+
.tone-chip span:hover { border-color: var(--primary); }
|
| 278 |
+
.tone-chip input:checked + span {
|
| 279 |
+
background: var(--primary-tint);
|
| 280 |
+
border-color: var(--primary);
|
| 281 |
+
color: var(--primary-deep);
|
| 282 |
+
font-weight: 600;
|
| 283 |
+
}
|
| 284 |
+
.tone-chip input:focus-visible + span {
|
| 285 |
+
outline: 2px solid var(--primary);
|
| 286 |
+
outline-offset: 2px;
|
| 287 |
}
|
|
|
|
|
|
|
| 288 |
|
| 289 |
.actions { display: flex; gap: 10px; margin: 20px 0 0; }
|
| 290 |
.btn {
|