devadit15 Claude Fable 5 commited on
Commit
2742f26
·
1 Parent(s): ef02a0a

One unified repair prompt + tone selector replaces Advanced section

Browse files

- generator rewrite: failure modes contribute rules/problems/role/pre to
a single composed prompt built around the original instruction; per-
family mechanisms preserved (XML rules for Claude, ### Requirements +
tail emphasis for GPT, few-shot skeleton for Gemini); test contract
rewritten for the single-prompt guarantee (46 green)
- tone chips (Neutral/Simple/Professional/Technical) in the main flow;
each maps to an audience band so the choice drives the reading-level
check; Advanced section removed

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

Files changed (5) hide show
  1. rubriq/prompts/generator.py +228 -400
  2. tests/test_prompts.py +47 -54
  3. web/app.js +6 -3
  4. web/index.html +20 -10
  5. web/styles.css +34 -9
rubriq/prompts/generator.py CHANGED
@@ -1,20 +1,23 @@
1
  """Improvement-prompt generation — the flagship.
2
 
3
- Contract (RESEARCH.md Part 4, enforced by tests):
4
- (a) the diagnosis names the specific failure with the evidence that triggered it,
 
5
  (b) prompt_text is a complete, ready-to-paste replacement prompt,
6
- (c) the fix uses the named model family's documented mechanism.
7
  "Be more specific" is a bug, not a template.
8
 
9
- Builders receive the user's original instruction and rebuild it with the
10
- repair applied, so the user pastes one block, not advice.
 
11
  """
12
 
13
  from __future__ import annotations
14
 
15
- from ..schemas import (Evidence, FailureMode, ImprovementPrompt, ModelFamily,
16
- ParameterScore, TaskClassification, TaskType)
17
- from ..textutils import snippet
 
18
  from .model_profiles import Profile, get_profile
19
 
20
  _SEVERITY = [
@@ -33,14 +36,42 @@ _SEVERITY = [
33
  FailureMode.LOW_DIVERSITY,
34
  ]
35
 
36
- MAX_PROMPTS = 5
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
37
 
38
 
39
- def _quotes(findings, limit=3) -> list[str]:
40
  out = []
41
  for f in findings:
42
  for e in f.evidence:
43
- if e.source == "output" and e.quote:
44
  out.append(e.quote)
45
  break
46
  if len(out) >= limit:
@@ -48,388 +79,147 @@ def _quotes(findings, limit=3) -> list[str]:
48
  return out
49
 
50
 
51
- def _source_block(profile: Profile, task: TaskClassification) -> str:
52
- if not task.has_source_text:
53
- return ""
54
- return profile.wrap_source() + "\n\n"
55
-
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
56
 
57
  def _rules_block(profile: Profile, rules: list[str]) -> str:
58
- """Render hard rules in the family's preferred structure."""
59
  if profile.family == ModelFamily.CLAUDE:
60
- inner = "\n".join(f"- {r}" for r in rules)
61
- return f"<rules>\n{inner}\n</rules>"
62
  if profile.family == ModelFamily.GPT:
63
- inner = "\n".join(f"{i+1}. {r}" for i, r in enumerate(rules))
64
- return f"### Requirements\n{inner}"
65
- inner = "\n".join(f"- {r}" for r in rules)
66
- return f"Rules:\n{inner}"
67
-
68
-
69
- def _tail_emphasis(profile: Profile, critical: str) -> str:
70
- """GPT/open-weights/generic get the critical constraint repeated last
71
- (lost-in-the-middle, RESEARCH.md 2.7). Claude/Gemini get their own levers
72
- elsewhere, so no noisy repetition."""
73
- if profile.family in (ModelFamily.OPEN_WEIGHTS, ModelFamily.GPT, ModelFamily.GENERIC):
74
- return f"\n\nIMPORTANT — before finishing, verify: {critical}"
75
- return ""
76
-
77
-
78
- # --------------------------------------------------------------- builders
79
-
80
- def _build_unfaithful(findings, task, profile: Profile) -> ImprovementPrompt:
81
- bad = _quotes(findings)
82
- bad_list = "\n".join(f'- "{q}"' for q in bad) if bad else "- (see report)"
83
- grounding_rules = [
84
- "Use ONLY information stated in the source text. If the source does not say it, do not write it.",
85
- "If something the task needs is missing from the source, write \"the source does not specify\" rather than filling the gap.",
86
- ]
87
- if profile.family == ModelFamily.CLAUDE:
88
- pre = ("First, copy the exact sentences from the source that support your answer into <quotes> tags. "
89
- "Then write the final answer in <answer> tags using only what you quoted.\n\n")
90
- elif profile.family == ModelFamily.GEMINI:
91
- pre = "Base every sentence of your answer on the text above. Quote or closely paraphrase the source.\n\n"
92
- else:
93
- pre = "Answer using only the provided text. Quote the source where possible.\n\n"
94
- body = (
95
- f"{_source_block(profile, task)}"
96
- f"{task.instruction.strip()}\n\n"
97
- f"{pre}"
98
- f"{_rules_block(profile, grounding_rules)}\n\n"
99
- f"A previous attempt invented content the source does not support, including:\n{bad_list}\n"
100
- f"Do not repeat these claims unless the source states them."
101
- f"{_tail_emphasis(profile, 'every claim traces to a sentence in the source text')}"
102
- )
103
- return ImprovementPrompt(
104
- failure_mode=FailureMode.UNFAITHFUL_CONTENT,
105
- title="Pin the output to the source",
106
- diagnosis=(f"The output makes claims the source text does not support"
107
- + (f', e.g. "{bad[0]}"' if bad else "") + ". "
108
- f"Repair: {profile.grounding_mechanism}"),
109
- prompt_text=body,
110
- model_family=profile.family,
111
- technique_source=profile.source,
112
- )
113
-
114
-
115
- def _build_constraints(findings, task, profile: Profile) -> ImprovementPrompt:
116
- missed = []
117
- for f in findings:
118
- kind = f.data.get("kind", "")
119
- actual = f.data.get("actual", "")
120
- missed.append((f.detail, kind, actual))
121
- rules = []
122
- for f in findings:
123
- prompt_ev = next((e.quote for e in f.evidence if e.source == "prompt"), None)
124
- rules.append(f.detail.split(" — ")[0].removeprefix("Prompt asked for ").strip().capitalize()
125
- + (f" (your prompt said: \"{prompt_ev}\")" if prompt_ev else ""))
126
- if profile.family == ModelFamily.GEMINI:
127
- example = ("\n\nHere is an example of a response with the right shape (replace the content, keep the form):\n"
128
- "[write one short example that satisfies every rule above]")
129
- else:
130
- example = ""
131
- critical = rules[0] if rules else "all stated constraints are met"
132
- body = (
133
- f"{_source_block(profile, task)}"
134
- f"{task.instruction.strip()}\n\n"
135
- f"{_rules_block(profile, rules)}"
136
- f"{example}"
137
- f"\n\nAfter drafting, count and check each requirement above before giving your final answer."
138
- f"{_tail_emphasis(profile, critical)}"
139
- )
140
- first = findings[0]
141
- return ImprovementPrompt(
142
- failure_mode=first.failure_mode,
143
- title="Make the constraints unmissable",
144
- diagnosis=(f"The output missed {len(findings)} verifiable constraint(s): "
145
- + "; ".join(f.detail for f in findings[:3]) + " "
146
- f"Repair: {profile.format_mechanism}"),
147
- prompt_text=body,
148
- model_family=profile.family,
149
- technique_source=profile.source,
150
- )
151
-
152
-
153
- def _build_coverage(findings, task, profile: Profile) -> ImprovementPrompt | None:
154
- parts = []
155
- for f in findings:
156
- q = next((e.quote for e in f.evidence if e.source == "prompt"), None)
157
- if q:
158
- parts.append(q)
159
- if not parts:
160
- return None # nothing quotable to enumerate; no honest repair possible
161
- part_list = "\n".join(f"{i+1}. {p}" for i, p in enumerate(parts))
162
- body = (
163
- f"{_source_block(profile, task)}"
164
- f"{task.instruction.strip()}\n\n"
165
- f"Answer every numbered part separately, under its own heading:\n{part_list}\n\n"
166
- f"If you cannot answer a part, say so under that heading rather than skipping it."
167
- f"{_tail_emphasis(profile, f'all {len(parts)} parts have their own answer')}"
168
- )
169
- return ImprovementPrompt(
170
- failure_mode=FailureMode.INCOMPLETE_COVERAGE,
171
- title="Force full coverage of the question",
172
- diagnosis=(f"{len(parts)} part(s) of the request went unanswered, e.g. \"{parts[0] if parts else ''}\". "
173
- "Repair: enumerate the parts and demand one answer per part — models skip sub-questions "
174
- "buried in prose but not numbered lists."),
175
- prompt_text=body,
176
- model_family=profile.family,
177
- technique_source=profile.source,
178
- )
179
-
180
-
181
- def _build_hallucination(findings, task, profile: Profile) -> ImprovementPrompt:
182
- flagged = _quotes(findings)
183
- fl = "\n".join(f'- "{q}"' for q in flagged) if flagged else ""
184
- rules = [
185
- "For every statistic, name its source inline, or present it as an estimate in words (\"roughly\", \"on the order of\").",
186
- "Do not cite papers, cases, or URLs unless you are certain they exist; prefer \"research on X suggests\" with no fake citation.",
187
- "Mark claims you are unsure about explicitly (\"I may be wrong about the exact figure\").",
188
- ]
189
- body = (
190
- f"{task.instruction.strip()}\n\n"
191
- f"{_rules_block(profile, rules)}\n\n"
192
- + (f"A previous attempt stated the following as fact; each is unverifiable as written:\n{fl}\n"
193
- f"Restate these with sources or with honest uncertainty, or drop them.\n" if fl else "")
194
- + "\nEnd your answer with a one-line note: which claims above should I double-check independently?"
195
- f"{_tail_emphasis(profile, 'no statistic or citation appears without a named source or an uncertainty marker')}"
196
- )
197
- return ImprovementPrompt(
198
- failure_mode=FailureMode.HALLUCINATION_RISK,
199
- title="Demand attribution or honest uncertainty",
200
- diagnosis=(f"The output states unverifiable specifics as fact"
201
- + (f', e.g. "{flagged[0]}"' if flagged else "") + ". "
202
- "No local check can verify these; the repair makes the model attribute or hedge them, "
203
- "and hands you a verification checklist."),
204
- prompt_text=body,
205
- model_family=profile.family,
206
- technique_source=profile.source,
207
- )
208
-
209
-
210
- def _build_reasoning(findings, task, profile: Profile) -> ImprovementPrompt:
211
- if profile.family == ModelFamily.CLAUDE:
212
- think = ("Work through the problem step by step inside <thinking> tags first. "
213
- "Then give the final answer in <answer> tags.")
214
- else:
215
- think = ("Solve this step by step. Show each step of your working on its own line, "
216
- "then state the final answer on a last line beginning \"Answer:\".")
217
- body = (
218
- f"{task.instruction.strip()}\n\n"
219
- f"{think}\n\n"
220
- f"After stating the answer, re-derive it a second way if possible and confirm both routes agree; "
221
- f"if they disagree, say which one you trust and why."
222
- f"{_tail_emphasis(profile, 'working shown, final answer explicitly labeled')}"
223
- )
224
- return ImprovementPrompt(
225
- failure_mode=FailureMode.WEAK_REASONING,
226
- title="Make the model show its working",
227
- diagnosis=("The output gives a conclusion without checkable working. "
228
- "Repair: request the reasoning *before* the answer (chain-of-thought), plus a second "
229
- "independent derivation — the manual form of self-consistency."),
230
- prompt_text=body,
231
- model_family=profile.family,
232
- technique_source="Wei et al. 2022 (CoT); Wang et al. 2023 (self-consistency); " + profile.source,
233
- )
234
-
235
-
236
- def _build_verbosity(findings, task, profile: Profile) -> ImprovementPrompt:
237
- fillers = _quotes(findings)
238
- limit_hint = next((f.data.get("ratio") for f in findings if "ratio" in f.data), None)
239
- rules = [
240
- "Open with the answer itself — no preamble, no restating the question.",
241
- "Cut framing phrases (e.g. " + ", ".join(f'"{f}"' for f in fillers[:3]) + ")." if fillers
242
- else "Cut framing phrases and meta-commentary.",
243
- "One idea per sentence; delete any sentence that repeats an earlier one.",
244
- ]
245
- if profile.family == ModelFamily.CLAUDE:
246
- opener = "\nBegin your response directly with the substance (no introduction)."
247
- else:
248
- opener = ""
249
- body = (
250
- f"{_source_block(profile, task)}"
251
- f"{task.instruction.strip()}\n\n"
252
- f"{_rules_block(profile, rules)}{opener}"
253
- f"{_tail_emphasis(profile, 'the response starts with substance and contains no filler phrases')}"
254
- )
255
- return ImprovementPrompt(
256
- failure_mode=FailureMode.VERBOSITY,
257
- title="Strip the padding",
258
- diagnosis=("The output pads its length" + (f" ({limit_hint}x the asked-for size)" if limit_hint else "")
259
- + (f' with filler like "{fillers[0]}"' if fillers else "") + ". "
260
- "Repair: answer-first structure plus an explicit ban on the exact filler found."),
261
- prompt_text=body,
262
- model_family=profile.family,
263
- technique_source=profile.source,
264
- )
265
-
266
-
267
- def _build_register(findings, task, profile: Profile) -> ImprovementPrompt:
268
- data = findings[0].data
269
- fk, band = data.get("fk_grade"), data.get("band", [6, 14])
270
- audience = data.get("audience", "the intended audience")
271
- too_hard = fk is not None and fk > band[1]
272
- style = ("short sentences, everyday words, one idea at a time"
273
- if too_hard else "precise terminology and compact, information-dense prose")
274
- role = f"You are writing for {audience}."
275
- body = (
276
- f"{role}\n\n"
277
- f"{_source_block(profile, task)}"
278
- f"{task.instruction.strip()}\n\n"
279
- f"Write in a register that fits that audience: {style}. "
280
- f"Read your draft as that reader before finalizing; rewrite any sentence they would stumble on."
281
- f"{_tail_emphasis(profile, f'the text reads naturally for {audience}')}"
282
- )
283
- return ImprovementPrompt(
284
- failure_mode=FailureMode.WRONG_REGISTER,
285
- title="Match the audience's reading level",
286
- diagnosis=(f"The output reads at US grade level {fk}, outside the {band[0]:g}-{band[1]:g} band for {audience} — "
287
- f"{'harder' if too_hard else 'simpler'} than it should. "
288
- "Repair: an explicit audience role. Role prompting reliably shifts register (it does not fix facts)."),
289
- prompt_text=body,
290
- model_family=profile.family,
291
- technique_source="role prompting for register (RESEARCH.md 2.8); " + profile.source,
292
- )
293
-
294
-
295
- def _build_structure(findings, task, profile: Profile) -> ImprovementPrompt:
296
- repeats = any("repeat" in f.detail.lower() or "duplicate" in f.detail.lower() for f in findings)
297
- rules = [
298
- "Before writing, produce a 3-6 point outline; then write one section per point.",
299
- "Each section must add new information — no restating earlier sections." if repeats
300
- else "Order sections so each follows from the previous one.",
301
- ]
302
- body = (
303
- f"{_source_block(profile, task)}"
304
- f"{task.instruction.strip()}\n\n"
305
- f"{_rules_block(profile, rules)}"
306
- f"{_tail_emphasis(profile, 'sections follow the outline and none repeats another')}"
307
- )
308
- return ImprovementPrompt(
309
- failure_mode=FailureMode.POOR_STRUCTURE,
310
- title="Impose an outline",
311
- diagnosis=("Adjacent sections " + ("repeat each other" if repeats else "jump between topics without connection")
312
- + " (flagged by flow analysis). Repair: outline-first generation pins the structure before prose exists."),
313
- prompt_text=body,
314
- model_family=profile.family,
315
- technique_source=profile.source,
316
- )
317
-
318
-
319
- def _build_drift(findings, task, profile: Profile) -> ImprovementPrompt:
320
- drifting = _quotes(findings, 1)
321
- body = (
322
- f"{_source_block(profile, task)}"
323
- f"{task.instruction.strip()}\n\n"
324
- f"Stay strictly on this request. Do not add background, related topics, or general advice unless asked."
325
- + (f"\n\nA previous attempt drifted into: \"{drifting[0]}\" — leave this out unless the request explicitly needs it."
326
- if drifting else "")
327
- + f"{_tail_emphasis(profile, 'every paragraph directly serves the request')}"
328
- )
329
- return ImprovementPrompt(
330
- failure_mode=FailureMode.OFF_TOPIC_DRIFT,
331
- title="Fence the scope",
332
- diagnosis=(f"Part of the output drifts off the request"
333
- + (f' ("{drifting[0]}")' if drifting else "") +
334
- ". Repair: an explicit scope fence naming the drift to exclude."),
335
- prompt_text=body,
336
- model_family=profile.family,
337
- technique_source=profile.source,
338
- )
339
-
340
-
341
- def _build_code(findings, task, profile: Profile) -> ImprovementPrompt:
342
- problems = [f.detail for f in findings]
343
- plist = "\n".join(f"- {p}" for p in problems[:4])
344
- rules = [
345
- "Return complete, runnable code — no TODOs, placeholders, or elided sections.",
346
- "If the code needs assumptions (versions, inputs), state them in a comment at the top.",
347
- "After the code, list 2-3 quick test cases I can run to verify it.",
348
- ]
349
- body = (
350
- f"{task.instruction.strip()}\n\n"
351
- f"{_rules_block(profile, rules)}\n\n"
352
- f"A previous attempt had these problems — fix each:\n{plist}"
353
- f"{_tail_emphasis(profile, 'the code parses, is complete, and includes the requested names')}"
354
- )
355
- return ImprovementPrompt(
356
- failure_mode=FailureMode.INVALID_CODE,
357
- title="Demand complete, verifiable code",
358
- diagnosis=("Static analysis found concrete defects: " + "; ".join(problems[:2]) + " "
359
- "Repair: completeness rules plus self-supplied test cases give you an immediate way to check the redo."),
360
- prompt_text=body,
361
- model_family=profile.family,
362
- technique_source=profile.source,
363
- )
364
-
365
-
366
- def _build_invented(findings, task, profile: Profile) -> ImprovementPrompt:
367
- invented = _quotes(findings)
368
- il = "\n".join(f'- "{q}"' for q in invented) if invented else ""
369
- rules = [
370
- "Copy every extracted value verbatim from the source text — no paraphrasing, no normalizing.",
371
- "If a requested field is not in the text, output null for it. Never fill gaps from general knowledge.",
372
- ]
373
- body = (
374
- f"{_source_block(profile, task)}"
375
- f"{task.instruction.strip()}\n\n"
376
- f"{_rules_block(profile, rules)}\n\n"
377
- + (f"A previous attempt output values that are not in the source:\n{il}\n" if il else "")
378
- + f"{_tail_emphasis(profile, 'every value string-matches the source text or is null')}"
379
- )
380
- return ImprovementPrompt(
381
- failure_mode=FailureMode.INVENTED_FIELDS,
382
- title="Verbatim-only extraction",
383
- diagnosis=("Extracted values don't appear in the source text"
384
- + (f', e.g. "{invented[0]}"' if invented else "") +
385
- ". Repair: verbatim-copy rule plus explicit null for missing fields — the two rules that stop "
386
- "extraction models from 'helpfully' completing data."),
387
- prompt_text=body,
388
- model_family=profile.family,
389
- technique_source=profile.source,
390
- )
391
-
392
-
393
- def _build_diversity(findings, task, profile: Profile) -> ImprovementPrompt:
394
- cliches = _quotes(findings)
395
- cl = ", ".join(f'"{c}"' for c in cliches[:5])
396
- rules = [
397
- (f"Banned phrases: {cl}. Find a fresh image instead." if cliches else
398
- "Avoid stock phrases; when a familiar expression appears in your draft, replace it with a concrete image."),
399
- "Vary sentence length deliberately: mix sentences under 8 words with longer ones.",
400
- "Prefer specific, sensory detail over abstract description.",
401
- ]
402
- body = (
403
- f"{task.instruction.strip()}\n\n"
404
- f"{_rules_block(profile, rules)}"
405
- f"{_tail_emphasis(profile, 'no banned phrase appears and sentence lengths vary')}"
406
- )
407
- return ImprovementPrompt(
408
- failure_mode=FailureMode.LOW_DIVERSITY,
409
- title="Ban the clichés it reached for",
410
- diagnosis=("Style statistics flagged " + (f"stock phrases ({cl})" if cliches else "flat, repetitive prose") +
411
- ". Repair: ban the exact phrases found and constrain rhythm — negative examples work when they are this specific."),
412
- prompt_text=body,
413
- model_family=profile.family,
414
- technique_source=profile.source,
415
- )
416
-
417
-
418
- _BUILDERS = {
419
- FailureMode.UNFAITHFUL_CONTENT: _build_unfaithful,
420
- FailureMode.MISSED_CONSTRAINT: _build_constraints,
421
- FailureMode.FORMAT_VIOLATION: _build_constraints,
422
- FailureMode.INCOMPLETE_COVERAGE: _build_coverage,
423
- FailureMode.HALLUCINATION_RISK: _build_hallucination,
424
- FailureMode.WEAK_REASONING: _build_reasoning,
425
- FailureMode.VERBOSITY: _build_verbosity,
426
- FailureMode.WRONG_REGISTER: _build_register,
427
- FailureMode.POOR_STRUCTURE: _build_structure,
428
- FailureMode.OFF_TOPIC_DRIFT: _build_drift,
429
- FailureMode.INVALID_CODE: _build_code,
430
- FailureMode.INVENTED_FIELDS: _build_invented,
431
- FailureMode.LOW_DIVERSITY: _build_diversity,
432
- }
433
 
434
 
435
  def generate_improvement_prompts(
@@ -443,16 +233,54 @@ def generate_improvement_prompts(
443
  for ps in parameters:
444
  for f in ps.findings:
445
  by_mode.setdefault(f.failure_mode, []).append(f)
446
- # constraint + format failures repair through the same rebuilt prompt
447
- if FailureMode.FORMAT_VIOLATION in by_mode and FailureMode.MISSED_CONSTRAINT in by_mode:
448
- by_mode[FailureMode.MISSED_CONSTRAINT].extend(by_mode.pop(FailureMode.FORMAT_VIOLATION))
449
 
450
- prompts: list[ImprovementPrompt] = []
 
 
451
  for mode in _SEVERITY:
452
- if mode in by_mode and mode in _BUILDERS:
453
- built = _BUILDERS[mode](by_mode[mode], task, profile)
454
- if built is not None:
455
- prompts.append(built)
456
- if len(prompts) >= MAX_PROMPTS:
457
- break
458
- return prompts
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  """Improvement-prompt generation — the flagship.
2
 
3
+ Contract (RESEARCH.md Part 4, enforced by tests): the generator produces ONE
4
+ unified repair prompt that addresses every diagnosed failure at once —
5
+ (a) the diagnosis names each failure with the evidence that triggered it,
6
  (b) prompt_text is a complete, ready-to-paste replacement prompt,
7
+ (c) the repairs use the named model family's documented mechanisms.
8
  "Be more specific" is a bug, not a template.
9
 
10
+ Each failure mode contributes pieces (a role line, hard rules, previous-
11
+ attempt problems, a pre-instruction) and the composer assembles them around
12
+ the user's original instruction.
13
  """
14
 
15
  from __future__ import annotations
16
 
17
+ from dataclasses import dataclass, field
18
+
19
+ from ..schemas import (FailureMode, ImprovementPrompt, ModelFamily,
20
+ ParameterScore, TaskClassification)
21
  from .model_profiles import Profile, get_profile
22
 
23
  _SEVERITY = [
 
36
  FailureMode.LOW_DIVERSITY,
37
  ]
38
 
39
+ _MODE_NAMES = {
40
+ FailureMode.UNFAITHFUL_CONTENT: "content the source does not support",
41
+ FailureMode.INVALID_CODE: "defective code",
42
+ FailureMode.INVENTED_FIELDS: "extracted values not present in the source",
43
+ FailureMode.MISSED_CONSTRAINT: "missed constraints",
44
+ FailureMode.FORMAT_VIOLATION: "format violations",
45
+ FailureMode.INCOMPLETE_COVERAGE: "unanswered parts of the request",
46
+ FailureMode.HALLUCINATION_RISK: "unverifiable claims stated as fact",
47
+ FailureMode.OFF_TOPIC_DRIFT: "off-topic drift",
48
+ FailureMode.WEAK_REASONING: "reasoning without checkable working",
49
+ FailureMode.VERBOSITY: "padding",
50
+ FailureMode.WRONG_REGISTER: "the wrong reading level for the audience",
51
+ FailureMode.POOR_STRUCTURE: "structural problems",
52
+ FailureMode.LOW_DIVERSITY: "flat, cliché-heavy style",
53
+ }
54
+
55
+ MAX_RULES = 9
56
+ MAX_PROBLEMS = 7
57
+
58
+
59
+ @dataclass
60
+ class _Parts:
61
+ role: str = ""
62
+ pre: str = "" # grounding / think-first instruction
63
+ rules: list[str] = field(default_factory=list)
64
+ problems: list[str] = field(default_factory=list) # "previous attempt" lines
65
+ wants_example: bool = False # gemini few-shot skeleton
66
+ criticals: list[str] = field(default_factory=list) # tail-emphasis candidates
67
+ sources: list[str] = field(default_factory=list)
68
 
69
 
70
+ def _quotes(findings, limit=3, source="output") -> list[str]:
71
  out = []
72
  for f in findings:
73
  for e in f.evidence:
74
+ if e.source == source and e.quote:
75
  out.append(e.quote)
76
  break
77
  if len(out) >= limit:
 
79
  return out
80
 
81
 
82
+ # ------------------------------------------------------ per-mode contributions
83
+
84
+ def _contribute(mode: FailureMode, findings, task, profile: Profile, parts: _Parts) -> str | None:
85
+ """Fold this mode's repair into parts. Returns a diagnosis fragment, or
86
+ None when the mode has nothing usable to say."""
87
+ fam = profile.family
88
+
89
+ if mode == FailureMode.UNFAITHFUL_CONTENT:
90
+ bad = _quotes(findings)
91
+ if fam == ModelFamily.CLAUDE:
92
+ parts.pre = ("First, copy the exact sentences from the source that support your answer into "
93
+ "<quotes> tags. Then write the final answer in <answer> tags using only what you quoted.")
94
+ elif fam == ModelFamily.GEMINI:
95
+ parts.pre = "Base every sentence of your answer on the source text above; quote or closely paraphrase it."
96
+ else:
97
+ parts.pre = "Answer using only the provided source text. Quote the source where possible."
98
+ parts.rules.append("Use ONLY information stated in the source text. If the source does not say it, do not write it.")
99
+ parts.rules.append('If something the task needs is missing from the source, write "the source does not specify" rather than filling the gap.')
100
+ parts.problems += [f'invented content: "{q}"' for q in bad]
101
+ parts.criticals.append("every claim traces to a sentence in the source text")
102
+ parts.sources.append(profile.source)
103
+ return "content the source does not support" + (f' (e.g. "{bad[0]}")' if bad else "")
104
+
105
+ if mode == FailureMode.INVALID_CODE:
106
+ parts.rules.append("Return complete, runnable code — no TODOs, placeholders, or elided sections.")
107
+ parts.rules.append("After the code, list 2-3 quick test cases I can run to verify it.")
108
+ parts.problems += [f.detail for f in findings[:3]]
109
+ parts.criticals.append("the code parses, is complete, and includes every requested name")
110
+ parts.sources.append(profile.source)
111
+ return "concrete code defects (" + "; ".join(f.detail.rstrip(".") for f in findings[:2]) + ")"
112
+
113
+ if mode == FailureMode.INVENTED_FIELDS:
114
+ invented = _quotes(findings)
115
+ parts.rules.append("Copy every extracted value verbatim from the source text — no paraphrasing, no normalizing.")
116
+ parts.rules.append("If a requested field is not in the text, output null for it. Never fill gaps from general knowledge.")
117
+ parts.problems += [f'value not in the source: "{q}"' for q in invented]
118
+ parts.criticals.append("every value string-matches the source text or is null")
119
+ parts.sources.append(profile.source)
120
+ return "extracted values that are not in the source" + (f' (e.g. "{invented[0]}")' if invented else "")
121
+
122
+ if mode in (FailureMode.MISSED_CONSTRAINT, FailureMode.FORMAT_VIOLATION):
123
+ for f in findings:
124
+ req = f.detail.split(" — ")[0].removeprefix("Prompt asked for ").strip().capitalize()
125
+ parts.rules.append(req + ". Count and check this before giving your final answer.")
126
+ parts.problems.append(f.detail)
127
+ parts.criticals.insert(0, req.lower())
128
+ parts.wants_example = fam == ModelFamily.GEMINI
129
+ parts.sources.append(profile.source)
130
+ return f"{len(findings)} verifiable constraint(s) missed"
131
+
132
+ if mode == FailureMode.INCOMPLETE_COVERAGE:
133
+ qparts = _quotes(findings, limit=6, source="prompt")
134
+ if not qparts:
135
+ return None
136
+ numbered = " ".join(f"({i + 1}) {p}" for i, p in enumerate(qparts))
137
+ parts.rules.append(f"Answer every part of the request separately, under its own heading: {numbered}. "
138
+ "If you cannot answer a part, say so under that heading rather than skipping it.")
139
+ parts.problems += [f'left unanswered: "{p}"' for p in qparts[:3]]
140
+ parts.criticals.append("every numbered part has its own answer")
141
+ parts.sources.append(profile.source)
142
+ return f"{len(qparts)} part(s) of the request left unanswered"
143
+
144
+ if mode == FailureMode.HALLUCINATION_RISK:
145
+ flagged = _quotes(findings)
146
+ parts.rules.append("For every statistic or citation, name its real source inline, or state it as an "
147
+ 'estimate in words ("roughly", "on the order of"). Never invent citations or URLs.')
148
+ parts.rules.append('Mark claims you are unsure about explicitly, and end with a one-line note: '
149
+ "which claims should I double-check independently?")
150
+ parts.problems += [f'stated as fact but unverifiable: "{q}"' for q in flagged]
151
+ parts.criticals.append("no statistic or citation appears without a named source or an uncertainty marker")
152
+ parts.sources.append(profile.source)
153
+ return "unverifiable specifics stated as fact" + (f' (e.g. "{flagged[0]}")' if flagged else "")
154
+
155
+ if mode == FailureMode.OFF_TOPIC_DRIFT:
156
+ drifting = _quotes(findings, 1)
157
+ parts.rules.append("Stay strictly on the request. No background, related topics, or general advice unless asked.")
158
+ if drifting:
159
+ parts.problems.append(f'drifted into: "{drifting[0]}"')
160
+ parts.criticals.append("every paragraph directly serves the request")
161
+ parts.sources.append(profile.source)
162
+ return "off-topic drift"
163
+
164
+ if mode == FailureMode.WEAK_REASONING:
165
+ if not parts.pre:
166
+ parts.pre = ("Work through the problem step by step inside <thinking> tags first, then give the final "
167
+ "answer in <answer> tags." if fam == ModelFamily.CLAUDE else
168
+ 'Solve this step by step, each step on its own line, then state the final answer on a '
169
+ 'last line beginning "Answer:".')
170
+ parts.rules.append("Show the working before the answer; if possible, re-derive the answer a second way and confirm both agree.")
171
+ parts.criticals.append("working shown, final answer explicitly labeled")
172
+ parts.sources.append("Wei et al. 2022 (CoT); Wang et al. 2023 (self-consistency)")
173
+ return "a conclusion without checkable working"
174
+
175
+ if mode == FailureMode.VERBOSITY:
176
+ fillers = _quotes(findings)
177
+ parts.rules.append("Open with the answer itself — no preamble, no restating the question."
178
+ + (" Banned filler: " + ", ".join(f'"{q}"' for q in fillers[:3]) + "." if fillers else ""))
179
+ parts.problems += [f'filler: "{q}"' for q in fillers[:2]]
180
+ parts.criticals.append("the response starts with substance and contains no filler")
181
+ parts.sources.append(profile.source)
182
+ return "padding" + (f' (e.g. "{fillers[0]}")' if fillers else "")
183
+
184
+ if mode == FailureMode.WRONG_REGISTER:
185
+ data = findings[0].data
186
+ fk, band = data.get("fk_grade"), data.get("band", [6, 14])
187
+ audience = data.get("audience", "the intended audience")
188
+ too_hard = fk is not None and fk > band[1]
189
+ style = ("short sentences, everyday words, one idea at a time" if too_hard
190
+ else "precise terminology and compact, information-dense prose")
191
+ parts.role = f"You are writing for {audience}. Use {style}."
192
+ parts.criticals.append(f"the text reads naturally for {audience}")
193
+ parts.sources.append("role prompting for register (RESEARCH.md 2.8)")
194
+ return f"reading level {fk} outside the {band[0]:g}-{band[1]:g} band for {audience}"
195
+
196
+ if mode == FailureMode.POOR_STRUCTURE:
197
+ repeats = any("repeat" in f.detail.lower() or "duplicate" in f.detail.lower() for f in findings)
198
+ parts.rules.append("Before writing, produce a short outline; then write one section per point"
199
+ + (" — no section may restate an earlier one." if repeats else ", each following from the previous."))
200
+ parts.sources.append(profile.source)
201
+ return "sections that repeat each other" if repeats else "abrupt jumps between sections"
202
+
203
+ if mode == FailureMode.LOW_DIVERSITY:
204
+ cliches = _quotes(findings)
205
+ parts.rules.append(("Banned phrases: " + ", ".join(f'"{c}"' for c in cliches[:5]) + ". Find a fresh image instead."
206
+ if cliches else "Replace stock phrases with concrete, sensory detail.")
207
+ + " Vary sentence length deliberately.")
208
+ parts.problems += [f'stock phrase: "{c}"' for c in cliches[:2]]
209
+ parts.sources.append(profile.source)
210
+ return "flat style" + (f" (stock phrases like {cliches[0]!r})" if cliches else "")
211
+
212
+ return None
213
+
214
+
215
+ # --------------------------------------------------------------- composition
216
 
217
  def _rules_block(profile: Profile, rules: list[str]) -> str:
 
218
  if profile.family == ModelFamily.CLAUDE:
219
+ return "<rules>\n" + "\n".join(f"- {r}" for r in rules) + "\n</rules>"
 
220
  if profile.family == ModelFamily.GPT:
221
+ return "### Requirements\n" + "\n".join(f"{i + 1}. {r}" for i, r in enumerate(rules))
222
+ return "Rules:\n" + "\n".join(f"- {r}" for r in rules)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
223
 
224
 
225
  def generate_improvement_prompts(
 
233
  for ps in parameters:
234
  for f in ps.findings:
235
  by_mode.setdefault(f.failure_mode, []).append(f)
236
+ if not by_mode:
237
+ return []
 
238
 
239
+ parts = _Parts()
240
+ fragments: list[str] = []
241
+ lead_mode: FailureMode | None = None
242
  for mode in _SEVERITY:
243
+ if mode not in by_mode:
244
+ continue
245
+ fragment = _contribute(mode, by_mode[mode], task, profile, parts)
246
+ if fragment:
247
+ fragments.append(fragment)
248
+ lead_mode = lead_mode or mode
249
+ if lead_mode is None:
250
+ return []
251
+
252
+ parts.rules = list(dict.fromkeys(parts.rules))[:MAX_RULES]
253
+ parts.problems = list(dict.fromkeys(parts.problems))[:MAX_PROBLEMS]
254
+
255
+ segments: list[str] = []
256
+ if parts.role:
257
+ segments.append(parts.role)
258
+ if task.has_source_text:
259
+ segments.append(profile.wrap_source())
260
+ segments.append(task.instruction.strip())
261
+ if parts.pre:
262
+ segments.append(parts.pre)
263
+ if parts.rules:
264
+ segments.append(_rules_block(profile, parts.rules))
265
+ if parts.wants_example:
266
+ segments.append("Here is an example of a response with the right shape (replace the content, keep the form):\n"
267
+ "[write one short example that satisfies every rule above]")
268
+ if parts.problems:
269
+ segments.append("A previous attempt had these problems — do not repeat any of them:\n"
270
+ + "\n".join(f"- {p}" for p in parts.problems))
271
+ if parts.criticals and profile.family in (ModelFamily.GPT, ModelFamily.OPEN_WEIGHTS, ModelFamily.GENERIC):
272
+ # lost-in-the-middle: repeat the most critical checks at the very end
273
+ segments.append("IMPORTANT — before finishing, verify: " + "; ".join(parts.criticals[:3]) + ".")
274
+
275
+ diagnosis = (f"{len(fragments)} problem(s) diagnosed: " + "; ".join(fragments) + ". "
276
+ f"This single prompt rebuilds your original instruction to fix all of them, using "
277
+ f"{profile.display} mechanisms — {profile.structure_hint}")
278
+
279
+ return [ImprovementPrompt(
280
+ failure_mode=lead_mode,
281
+ title="The repaired prompt",
282
+ diagnosis=diagnosis,
283
+ prompt_text="\n\n".join(segments),
284
+ model_family=profile.family,
285
+ technique_source="; ".join(dict.fromkeys(parts.sources)),
286
+ )]
tests/test_prompts.py CHANGED
@@ -1,6 +1,6 @@
1
- """The flagship contract: improvement prompts must name the diagnosed
2
- problem, quote evidence, be paste-ready, and use the named model family's
3
- mechanism. Generic advice is a test failure by design."""
4
 
5
  import samples
6
  from rubriq.schemas import EvalRequest, FailureMode, ModelFamily
@@ -18,78 +18,72 @@ def run(prompt, output, model=""):
18
  return scorer.evaluate(EvalRequest(prompt=prompt, output=output, model_name=model))
19
 
20
 
21
- def test_bad_summary_generates_prompts():
22
- report = run(samples.SUMM_PROMPT, samples.SUMM_BAD, model="gpt-4o")
23
- assert report.improvement_prompts, "a failing output must generate repairs"
24
- modes = {p.failure_mode for p in report.improvement_prompts}
25
- assert FailureMode.UNFAITHFUL_CONTENT in modes
26
- assert FailureMode.MISSED_CONSTRAINT in modes or FailureMode.VERBOSITY in modes
 
 
 
 
 
 
 
27
 
28
 
29
  def test_no_generic_advice():
30
  for pair in [(samples.SUMM_PROMPT, samples.SUMM_BAD), (samples.CODE_PROMPT, samples.CODE_BAD)]:
31
- report = run(*pair, model="gpt-4o")
32
- for ip in report.improvement_prompts:
33
- low = (ip.diagnosis + " " + ip.prompt_text).lower()
34
- for phrase in GENERIC_PHRASES:
35
- assert phrase not in low, f"generic advice '{phrase}' in {ip.title}"
36
 
37
 
38
- def test_diagnosis_names_the_evidence():
39
- report = run(samples.SUMM_PROMPT, samples.SUMM_BAD, model="gpt-4o")
40
- unfaithful = next(p for p in report.improvement_prompts
41
- if p.failure_mode == FailureMode.UNFAITHFUL_CONTENT)
42
- # The repair prompt must carry the actual invented claims back to the model.
43
- assert '"' in unfaithful.prompt_text
44
- assert "previous attempt" in unfaithful.prompt_text.lower()
45
 
46
 
47
  def test_prompt_contains_original_instruction():
48
- report = run(samples.SUMM_PROMPT, samples.SUMM_BAD, model="gpt-4o")
49
- for ip in report.improvement_prompts:
50
- assert "60 words" in ip.prompt_text or "Summarize" in ip.prompt_text, \
51
- "repair prompts must be complete replacements, not advice"
52
 
53
 
54
  def test_claude_gets_xml_mechanism():
55
- report = run(samples.SUMM_PROMPT, samples.SUMM_BAD, model="claude sonnet")
56
- unfaithful = next(p for p in report.improvement_prompts
57
- if p.failure_mode == FailureMode.UNFAITHFUL_CONTENT)
58
- assert unfaithful.model_family == ModelFamily.CLAUDE
59
- assert "<quotes>" in unfaithful.prompt_text or "<source>" in unfaithful.prompt_text
60
 
61
 
62
  def test_gpt_gets_delimiters_and_tail_emphasis():
63
- report = run(samples.SUMM_PROMPT, samples.SUMM_BAD, model="gpt-4o")
64
- unfaithful = next(p for p in report.improvement_prompts
65
- if p.failure_mode == FailureMode.UNFAITHFUL_CONTENT)
66
- assert unfaithful.model_family == ModelFamily.GPT
67
- assert "IMPORTANT" in unfaithful.prompt_text or "###" in unfaithful.prompt_text
68
 
69
 
70
  def test_gemini_gets_fewshot_for_format():
71
- report = run(samples.SUMM_PROMPT, samples.SUMM_BAD, model="gemini-1.5-pro")
72
- constraint = next((p for p in report.improvement_prompts
73
- if p.failure_mode in (FailureMode.MISSED_CONSTRAINT, FailureMode.FORMAT_VIOLATION)), None)
74
- if constraint:
75
- assert "example" in constraint.prompt_text.lower()
76
 
77
 
78
  def test_code_repair_names_defects():
79
- report = run(samples.CODE_PROMPT, samples.CODE_BAD)
80
- code = next(p for p in report.improvement_prompts if p.failure_mode == FailureMode.INVALID_CODE)
81
- assert "parse_log_line" in code.prompt_text # the missing requested name travels into the repair
82
- assert "fix each" in code.prompt_text.lower()
83
 
84
 
85
  def test_no_empty_coverage_repair_on_code():
86
- """Regression: missing requested function names must route to the code
87
- repair, never produce a '0 parts unanswered' coverage card."""
88
- report = run(samples.CODE_PROMPT, samples.CODE_BAD)
89
- for ip in report.improvement_prompts:
90
- assert "0 part(s)" not in ip.diagnosis
91
- if ip.failure_mode == FailureMode.INCOMPLETE_COVERAGE:
92
- assert 'e.g. ""' not in ip.diagnosis
93
 
94
 
95
  def test_family_detection_current_models():
@@ -113,6 +107,5 @@ def test_family_detection_current_models():
113
 
114
 
115
  def test_technique_sources_cited():
116
- report = run(samples.SUMM_PROMPT, samples.SUMM_BAD, model="gpt-4o")
117
- for ip in report.improvement_prompts:
118
- assert ip.technique_source.strip(), "every template must cite its source"
 
1
+ """The flagship contract: ONE unified repair prompt that names every
2
+ diagnosed problem with evidence, is paste-ready, and uses the named model
3
+ family's mechanisms. Generic advice is a test failure by design."""
4
 
5
  import samples
6
  from rubriq.schemas import EvalRequest, FailureMode, ModelFamily
 
18
  return scorer.evaluate(EvalRequest(prompt=prompt, output=output, model_name=model))
19
 
20
 
21
+ def one(report):
22
+ assert len(report.improvement_prompts) == 1, \
23
+ f"expected exactly one unified repair prompt, got {len(report.improvement_prompts)}"
24
+ return report.improvement_prompts[0]
25
+
26
+
27
+ def test_single_unified_prompt():
28
+ rep = run(samples.SUMM_PROMPT, samples.SUMM_BAD, model="OpenAI GPT-5.5")
29
+ ip = one(rep)
30
+ # the diagnosis names ALL the problem classes, not just one
31
+ assert "source does not support" in ip.diagnosis
32
+ assert "constraint" in ip.diagnosis
33
+ assert ip.failure_mode == FailureMode.UNFAITHFUL_CONTENT # most severe leads
34
 
35
 
36
  def test_no_generic_advice():
37
  for pair in [(samples.SUMM_PROMPT, samples.SUMM_BAD), (samples.CODE_PROMPT, samples.CODE_BAD)]:
38
+ rep = run(*pair, model="OpenAI GPT-5.5")
39
+ ip = one(rep)
40
+ low = (ip.diagnosis + " " + ip.prompt_text).lower()
41
+ for phrase in GENERIC_PHRASES:
42
+ assert phrase not in low, f"generic advice '{phrase}'"
43
 
44
 
45
+ def test_prompt_carries_evidence_back():
46
+ ip = one(run(samples.SUMM_PROMPT, samples.SUMM_BAD, model="OpenAI GPT-5.5"))
47
+ assert "previous attempt" in ip.prompt_text.lower()
48
+ assert "fifteen brave scientists" in ip.prompt_text # the invented claim travels into the repair
 
 
 
49
 
50
 
51
  def test_prompt_contains_original_instruction():
52
+ ip = one(run(samples.SUMM_PROMPT, samples.SUMM_BAD, model="OpenAI GPT-5.5"))
53
+ assert "Summarize" in ip.prompt_text and "60 words" in ip.prompt_text
 
 
54
 
55
 
56
  def test_claude_gets_xml_mechanism():
57
+ ip = one(run(samples.SUMM_PROMPT, samples.SUMM_BAD, model="Anthropic Claude Sonnet 5"))
58
+ assert ip.model_family == ModelFamily.CLAUDE
59
+ assert "<quotes>" in ip.prompt_text and "<rules>" in ip.prompt_text
 
 
60
 
61
 
62
  def test_gpt_gets_delimiters_and_tail_emphasis():
63
+ ip = one(run(samples.SUMM_PROMPT, samples.SUMM_BAD, model="OpenAI GPT-5.5"))
64
+ assert ip.model_family == ModelFamily.GPT
65
+ assert "### Requirements" in ip.prompt_text
66
+ assert "IMPORTANT" in ip.prompt_text
 
67
 
68
 
69
  def test_gemini_gets_fewshot_for_format():
70
+ ip = one(run(samples.SUMM_PROMPT, samples.SUMM_BAD, model="Google Gemini 3.5 Flash"))
71
+ assert ip.model_family == ModelFamily.GEMINI
72
+ assert "example" in ip.prompt_text.lower()
 
 
73
 
74
 
75
  def test_code_repair_names_defects():
76
+ ip = one(run(samples.CODE_PROMPT, samples.CODE_BAD))
77
+ assert "parse_log_line" in ip.prompt_text
78
+ assert "does not parse" in ip.prompt_text or "does not parse" in ip.diagnosis
 
79
 
80
 
81
  def test_no_empty_coverage_repair_on_code():
82
+ """Regression: missing requested function names must never surface as a
83
+ '0 parts unanswered' coverage diagnosis."""
84
+ ip = one(run(samples.CODE_PROMPT, samples.CODE_BAD))
85
+ assert "0 part(s)" not in ip.diagnosis
86
+ assert 'e.g. ""' not in ip.diagnosis
 
 
87
 
88
 
89
  def test_family_detection_current_models():
 
107
 
108
 
109
  def test_technique_sources_cited():
110
+ ip = one(run(samples.SUMM_PROMPT, samples.SUMM_BAD, model="OpenAI GPT-5.5"))
111
+ assert ip.technique_source.strip()
 
web/app.js CHANGED
@@ -231,10 +231,10 @@
231
  <h2 class="sect">Parameter verdicts <span class="count">· ${scored.length} scored, ${nFindings} findings</span></h2>
232
  ${rep.parameters.map(verdictHtml).join("")}
233
 
234
- <h2 class="sect">Repair prompts <span class="count">· targeted at ${esc(FAMILY_LABELS[(rep.improvement_prompts[0] || {}).model_family] || "your model")}</span></h2>
235
  ${rep.improvement_prompts.length
236
  ? rep.improvement_prompts.map(repairHtml).join("")
237
- : '<p class="no-repairs">No repairs needed — every locally checkable parameter came back clean. If the output still feels wrong, the issue likely lives in the judge-mode territory listed above.</p>'}
238
  </div>`;
239
 
240
  // meter fill animation (double rAF so the 0-scale paints first)
@@ -268,7 +268,10 @@
268
  prompt: $("promptIn").value,
269
  output: $("outputIn").value,
270
  model_name: composedModelName(),
271
- options: { audience: $("audienceIn").value.trim(), tone: $("toneIn").value.trim(), weights: {} },
 
 
 
272
  };
273
  evalBtn.disabled = true;
274
  evalBtn.textContent = "Evaluating…";
 
231
  <h2 class="sect">Parameter verdicts <span class="count">· ${scored.length} scored, ${nFindings} findings</span></h2>
232
  ${rep.parameters.map(verdictHtml).join("")}
233
 
234
+ <h2 class="sect">The repaired prompt <span class="count">· one paste-ready fix for everything found · ${esc(FAMILY_LABELS[(rep.improvement_prompts[0] || {}).model_family] || "your model")}</span></h2>
235
  ${rep.improvement_prompts.length
236
  ? rep.improvement_prompts.map(repairHtml).join("")
237
+ : '<p class="no-repairs">No repair needed — every locally checkable parameter came back clean. If the output still feels wrong, the issue likely lives in the judge-mode territory listed above.</p>'}
238
  </div>`;
239
 
240
  // meter fill animation (double rAF so the 0-scale paints first)
 
268
  prompt: $("promptIn").value,
269
  output: $("outputIn").value,
270
  model_name: composedModelName(),
271
+ options: (() => {
272
+ const tone = form.querySelector('input[name="tone"]:checked');
273
+ return { audience: tone?.dataset.audience || "", tone: tone?.value || "", weights: {} };
274
+ })(),
275
  };
276
  evalBtn.disabled = true;
277
  evalBtn.textContent = "Evaluating…";
web/index.html CHANGED
@@ -58,17 +58,27 @@
58
  </div>
59
  </fieldset>
60
 
61
- <details class="advanced">
62
- <summary>Advanced</summary>
63
- <div class="field">
64
- <label for="audienceIn">Intended audience</label>
65
- <input id="audienceIn" placeholder="e.g. children, general readers, expert clinicians">
 
 
 
 
 
 
 
 
 
 
 
 
 
 
66
  </div>
67
- <div class="field">
68
- <label for="toneIn">Expected tone</label>
69
- <input id="toneIn" placeholder="e.g. formal, playful, neutral">
70
- </div>
71
- </details>
72
 
73
  <div class="actions">
74
  <button type="submit" class="btn btn-primary" id="evalBtn">Evaluate output</button>
 
58
  </div>
59
  </fieldset>
60
 
61
+ <fieldset class="field field-tone">
62
+ <legend>Expected tone <span class="opt">sets the reading-level check</span></legend>
63
+ <div class="tone-row" role="radiogroup" aria-label="Expected tone">
64
+ <label class="tone-chip">
65
+ <input type="radio" name="tone" value="" data-audience="" checked>
66
+ <span>Neutral</span>
67
+ </label>
68
+ <label class="tone-chip">
69
+ <input type="radio" name="tone" value="simple, plain language" data-audience="general readers">
70
+ <span>Simple</span>
71
+ </label>
72
+ <label class="tone-chip">
73
+ <input type="radio" name="tone" value="professional" data-audience="professional">
74
+ <span>Professional</span>
75
+ </label>
76
+ <label class="tone-chip">
77
+ <input type="radio" name="tone" value="technical" data-audience="expert">
78
+ <span>Technical</span>
79
+ </label>
80
  </div>
81
+ </fieldset>
 
 
 
 
82
 
83
  <div class="actions">
84
  <button type="submit" class="btn btn-primary" id="evalBtn">Evaluate output</button>
web/styles.css CHANGED
@@ -248,18 +248,43 @@ fieldset.field-model legend {
248
  .combo-list li.co-empty { color: var(--muted); cursor: default; }
249
  .combo-list li.co-empty:hover { background: transparent; }
250
 
251
- .advanced { margin: 4px 0 16px; }
252
- .advanced summary {
253
- cursor: pointer;
254
  font-size: 13px;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
255
  font-weight: 500;
256
- color: var(--muted);
257
- padding: 4px 0;
258
- user-select: none;
259
- list-style-position: outside;
 
 
 
 
 
 
 
 
 
260
  }
261
- .advanced summary:hover { color: var(--ink); }
262
- .advanced[open] summary { margin-bottom: 10px; }
263
 
264
  .actions { display: flex; gap: 10px; margin: 20px 0 0; }
265
  .btn {
 
248
  .combo-list li.co-empty { color: var(--muted); cursor: default; }
249
  .combo-list li.co-empty:hover { background: transparent; }
250
 
251
+ /* tone selector */
252
+ fieldset.field-tone { border: 0; padding: 0; margin: 0 0 16px; }
253
+ fieldset.field-tone legend {
254
  font-size: 13px;
255
+ font-weight: 600;
256
+ margin-bottom: 6px;
257
+ padding: 0;
258
+ }
259
+ .tone-row { display: flex; flex-wrap: wrap; gap: 8px; }
260
+ .tone-chip input {
261
+ position: absolute;
262
+ opacity: 0;
263
+ pointer-events: none;
264
+ }
265
+ .tone-chip span {
266
+ display: inline-block;
267
+ border: 1px solid var(--line);
268
+ background: var(--bg);
269
+ color: var(--ink);
270
+ border-radius: 999px;
271
+ padding: 5px 14px;
272
+ font-size: 12.5px;
273
  font-weight: 500;
274
+ cursor: pointer;
275
+ transition: border-color 150ms var(--ease), background 150ms var(--ease), color 150ms var(--ease);
276
+ }
277
+ .tone-chip span:hover { border-color: var(--primary); }
278
+ .tone-chip input:checked + span {
279
+ background: var(--primary-tint);
280
+ border-color: var(--primary);
281
+ color: var(--primary-deep);
282
+ font-weight: 600;
283
+ }
284
+ .tone-chip input:focus-visible + span {
285
+ outline: 2px solid var(--primary);
286
+ outline-offset: 2px;
287
  }
 
 
288
 
289
  .actions { display: flex; gap: 10px; margin: 20px 0 0; }
290
  .btn {