| { |
| "_about": "Golden dataset for the Help skill (`src/agents/handlers/help.HelpAgent`). Unlike intent/readiness this calls the LIVE model: each case declares an analysis state + report-readiness + chat history, the runner streams HelpAgent.astream for real, and asserts RULES the reply must obey (not text similarity — help replies are free prose with no single correct wording). Metric is COMPLIANCE (% of rule assertions that hold), reported separately for held-out vs carried_over cases.", |
| "_groups": "language (reply matches the user's language — the button-path bug), report_guard (never suggest a report when report_ready.ready=false; do suggest it when true — this also IS the 'no action leakage' check, since generate_report is the only gated action), orientation (quality of the suggested starter questions — MANUAL review, not auto-scored), capability (help.md v7: offers the deeper analyses once the basics are done, and degrades a goal no tool can serve — individual future-event prediction, causal claims — to a supported question instead of handing it to the user).", |
| "_asserts": "language_match {expected} — detect the reply's language (help._detect_reply_language over the OUTPUT) must equal expected. must_not_contain_any {patterns} — none of the (case-insensitive) patterns appear. must_contain_any {patterns} — at least one appears.", |
| "_carried_over": "carried_over:true rows MIRROR an example in config/prompts/help.md (the row `id` IS the help.md `<!-- id: ... -->`). They are the regression guard: if the prompt is refactored, the demonstrated rule must still hold. What is mirrored is the INPUT spec + the assertion — NOT the example's reply text (temperature>0 makes exact match invalid). Held-out rows (carried_over:false) are NOT in the prompt; their compliance is the real generalization signal. If help.md's Examples change, keep these ids in sync (manual, like intent).", |
| "_missing_codes": "report_ready.missing uses codes mapped to the real strings is_report_ready emits (imported in run_eval): analysis -> _MISSING_ANALYSIS, delta -> _MISSING_DELTA. Kept as codes so the dataset survives wording changes.", |
| "schema": { |
| "id": "stable handle; for carried_over rows this equals the help.md example id", |
| "group": "language | report_guard | orientation | capability", |
| "carried_over": "bool — mirrors a help.md example", |
| "manual_review": "bool — run but exclude from the auto compliance rate (read output_text)", |
| "state": "{ analysis_title, objective, business_questions[], report_id }", |
| "report_ready": "{ ready: bool, missing: [analysis|delta] }", |
| "history": "[{ role: human|ai, content }] — drives language on the button path", |
| "message": "the human turn; null = button path (HelpAgent falls back to a per-language trigger)", |
| "asserts": "[{ type, ...spec }] — the rules the reply must obey", |
| "note": "human-readable description" |
| }, |
| "cases": [ |
| { |
| "id": "lang_01", "group": "language", "carried_over": false, "manual_review": false, |
| "state": { "analysis_title": "Analisis penjualan", "objective": "memahami performa penjualan bulanan", "business_questions": ["produk mana yang paling laku?"], "report_id": null }, |
| "report_ready": { "ready": false, "missing": ["analysis"] }, |
| "history": [{ "role": "human", "content": "aku baru upload datanya, terus aku harus ngapain?" }], |
| "message": null, |
| "asserts": [{ "type": "language_match", "expected": "Indonesian" }], |
| "note": "REGRESSION of the button-path bug: Indonesian conversation, message=null. Reply must be Indonesian, not English." |
| }, |
| { |
| "id": "lang_02", "group": "language", "carried_over": false, "manual_review": false, |
| "state": { "analysis_title": "Sales analysis", "objective": "understand monthly sales performance", "business_questions": ["which products drive revenue?"], "report_id": null }, |
| "report_ready": { "ready": false, "missing": ["analysis"] }, |
| "history": [{ "role": "human", "content": "okay I uploaded my data, what do I do next?" }], |
| "message": null, |
| "asserts": [{ "type": "language_match", "expected": "English" }], |
| "note": "English conversation, button path — reply must stay English." |
| }, |
| { |
| "id": "lang_03", "group": "language", "carried_over": false, "manual_review": false, |
| "state": { "analysis_title": "Analisis churn", "objective": "menurunkan churn pelanggan", "business_questions": ["segmen mana yang paling banyak churn?"], "report_id": null }, |
| "report_ready": { "ready": false, "missing": ["analysis"] }, |
| "history": [], |
| "message": "gimana caranya mulai analisis ini ya?", |
| "asserts": [{ "type": "language_match", "expected": "Indonesian" }], |
| "note": "Intent path: the real Indonesian user turn drives the language." |
| }, |
| { |
| "id": "lang_04", "group": "language", "carried_over": false, "manual_review": false, |
| "state": { "analysis_title": "Retention analysis", "objective": "understand user retention", "business_questions": ["what drives repeat usage?"], "report_id": null }, |
| "report_ready": { "ready": false, "missing": ["analysis"] }, |
| "history": [], |
| "message": null, |
| "asserts": [{ "type": "language_match", "expected": "English" }], |
| "note": "Fresh analysis, no chat yet, button path — with no turn to read, the user-authored goal (English objective + business_questions, required at onboarding) drives the language." |
| }, |
| { |
| "id": "lang_06", "group": "language", "carried_over": false, "manual_review": false, |
| "state": { "analysis_title": "Analisis retensi", "objective": "memahami retensi pengguna", "business_questions": ["apa yang mendorong penggunaan berulang?"], "report_id": null }, |
| "report_ready": { "ready": false, "missing": ["analysis"] }, |
| "history": [], |
| "message": null, |
| "asserts": [{ "type": "language_match", "expected": "Indonesian" }], |
| "note": "Same fresh-analysis path as lang_04 but the goal is Indonesian — the goal signal must yield Indonesian (not the hard fallback, which only fires when the goal is empty too)." |
| }, |
| { |
| "id": "lang_05", "group": "language", "carried_over": false, "manual_review": false, |
| "state": { "analysis_title": "Analisis penjualan", "objective": "memahami tren penjualan", "business_questions": ["bagaimana tren bulanan?"], "report_id": null }, |
| "report_ready": { "ready": false, "missing": ["analysis"] }, |
| "history": [ |
| { "role": "human", "content": "apa saja yang bisa aku tanyakan tentang data ini?" }, |
| { "role": "ai", "content": "You can start by asking which products sell the most." } |
| ], |
| "message": null, |
| "asserts": [{ "type": "language_match", "expected": "Indonesian" }], |
| "note": "Last AI turn is English but the human turn is Indonesian — mirror the human, reply Indonesian." |
| }, |
| { |
| "id": "help_ex_guard_delta", "group": "report_guard", "carried_over": true, "manual_review": false, |
| "state": { "analysis_title": "Sales analysis", "objective": "understand monthly sales performance", "business_questions": ["which products drive revenue?"], "report_id": "rep-1" }, |
| "report_ready": { "ready": false, "missing": ["delta"] }, |
| "history": [{ "role": "human", "content": "what should I do next?" }], |
| "message": null, |
| "asserts": [{ "type": "must_not_contain_any", "patterns": ["/report", "generate the report", "generate your report", "create the report"] }], |
| "note": "MIRRORS help.md example help_ex_guard_delta. A report exists and nothing new since — must NOT tell the user to generate a report; steer them to run a fresh analysis first." |
| }, |
| { |
| "id": "help_ex_guard_ready", "group": "report_guard", "carried_over": true, "manual_review": false, |
| "state": { "analysis_title": "Sales analysis", "objective": "understand monthly sales performance", "business_questions": ["which products drive revenue?"], "report_id": null }, |
| "report_ready": { "ready": true, "missing": [] }, |
| "history": [{ "role": "human", "content": "what should I do next?" }], |
| "message": null, |
| "asserts": [ |
| { "type": "must_contain_any", "patterns": ["Generate", "generate", "report"] }, |
| { "type": "must_not_contain_any", "patterns": ["/report"] } |
| ], |
| "note": "MIRRORS help.md example help_ex_guard_ready. Enough analysis done — SHOULD nudge toward the report by pointing at the Generate button; must NOT offer /report (removed in v6)." |
| }, |
| { |
| "id": "guard_03", "group": "report_guard", "carried_over": false, "manual_review": false, |
| "state": { "analysis_title": "Retention analysis", "objective": "improve 30-day retention", "business_questions": ["which cohort retains best?"], "report_id": null }, |
| "report_ready": { "ready": false, "missing": ["analysis"] }, |
| "history": [{ "role": "human", "content": "can I get a report now?" }], |
| "message": null, |
| "asserts": [{ "type": "must_not_contain_any", "patterns": ["/report", "generate the report", "generate your report", "you can generate"] }], |
| "note": "No analysis run yet, user asks for a report directly — must NOT offer to generate; redirect to running an analysis first." |
| }, |
| { |
| "id": "guard_04", "group": "report_guard", "carried_over": false, "manual_review": false, |
| "state": { "analysis_title": "Analisis penjualan", "objective": "memahami performa penjualan", "business_questions": ["produk mana yang paling laku?"], "report_id": null }, |
| "report_ready": { "ready": true, "missing": [] }, |
| "history": [{ "role": "human", "content": "selanjutnya aku ngapain?" }], |
| "message": null, |
| "asserts": [ |
| { "type": "must_contain_any", "patterns": ["Generate", "generate", "laporan", "report"] }, |
| { "type": "must_not_contain_any", "patterns": ["/report"] }, |
| { "type": "language_match", "expected": "Indonesian" } |
| ], |
| "note": "Ready + Indonesian conversation — should nudge toward the report (Generate button) AND stay in Indonesian; must NOT offer /report (removed in v6)." |
| }, |
| { |
| "id": "guard_05", "group": "report_guard", "carried_over": false, "manual_review": false, |
| "state": { "analysis_title": "Analisis churn", "objective": "menurunkan churn", "business_questions": ["segmen mana yang paling churn?"], "report_id": null }, |
| "report_ready": { "ready": false, "missing": ["analysis"] }, |
| "history": [{ "role": "human", "content": "aku mau bikin laporan dong" }], |
| "message": null, |
| "asserts": [ |
| { "type": "must_not_contain_any", "patterns": ["/report", "silakan buat laporan", "kamu bisa membuat laporan", "generate your report"] }, |
| { "type": "language_match", "expected": "Indonesian" } |
| ], |
| "note": "Indonesian, not ready, user asks for a report — must NOT offer it and must reply in Indonesian." |
| }, |
| { |
| "id": "help_ex_orient", "group": "orientation", "carried_over": true, "manual_review": true, |
| "state": { "analysis_title": "Sales analysis", "objective": "understand monthly sales performance", "business_questions": ["which products drive revenue?"], "report_id": null }, |
| "report_ready": { "ready": false, "missing": ["analysis"] }, |
| "history": [], |
| "message": null, |
| "asserts": [], |
| "note": "MIRRORS help.md example help_ex_orient. MANUAL: are the 2-3 starter questions concrete, descriptive-first, and tied to the objective? Read output_text." |
| }, |
| { |
| "id": "orient_02", "group": "orientation", "carried_over": false, "manual_review": true, |
| "state": { "analysis_title": "Retention analysis", "objective": "improve 30-day retention", "business_questions": ["which acquisition channel retains best?"], "report_id": null }, |
| "report_ready": { "ready": false, "missing": ["analysis"] }, |
| "history": [ |
| { "role": "human", "content": "which channel brings the most signups?" }, |
| { "role": "ai", "content": "Organic search brought the most signups last month (1,240)." } |
| ], |
| "message": null, |
| "asserts": [], |
| "note": "MANUAL: one question already answered — does help build on it with a NEW follow-up (retention by channel), not re-suggest the answered question? Read output_text." |
| }, |
| { |
| "id": "cap_01", "group": "capability", "carried_over": false, "manual_review": false, |
| "state": { "analysis_title": "Fleet downtime", "objective": "reduce unscheduled downtime and improve PA", "business_questions": ["what drives downtime?", "what will downtime look like next quarter?"], "report_id": null }, |
| "report_ready": { "ready": false, "missing": ["analysis"] }, |
| "history": [ |
| { "role": "human", "content": "what is the total downtime per unit?" }, |
| { "role": "ai", "content": "Unit U-07 has the highest total downtime (412 hours), followed by U-02 (388 hours)." }, |
| { "role": "human", "content": "how has downtime trended by month?" }, |
| { "role": "ai", "content": "Downtime rose from 1,020 hours in January to 1,340 hours in June." } |
| ], |
| "message": null, |
| "asserts": [ |
| { "type": "must_contain_any", "patterns": ["forecast", "project", "next quarter", "next 3 months", "most affect", "drive", "driver", "explain", "unusual", "anomal", "significant"] }, |
| { "type": "language_match", "expected": "English" } |
| ], |
| "note": "v7 capability sync: the basics (breakdown + trend) are done and the business questions ask for drivers and a projection — Help should now offer a deeper analysis (key drivers / forecast / anomaly / significance), which v5 forbade." |
| }, |
| { |
| "id": "cap_02", "group": "capability", "carried_over": false, "manual_review": false, |
| "state": { "analysis_title": "Prediksi breakdown", "objective": "memprediksi unit mana yang akan breakdown bulan depan", "business_questions": ["unit mana yang akan rusak berikutnya?"], "report_id": null }, |
| "report_ready": { "ready": false, "missing": ["analysis"] }, |
| "history": [{ "role": "human", "content": "aku mau tahu unit mana yang bakal rusak bulan depan, harus tanya apa?" }], |
| "message": null, |
| "asserts": [ |
| { "type": "must_contain_any", "patterns": ["tidak biasa", "anomali", "janggal", "proyeksi", "proyeksikan", "prediksi total", "tren"] }, |
| { "type": "must_not_contain_any", "patterns": ["“unit mana yang akan", "\"unit mana yang akan", "'unit mana yang akan", "“unit mana yang bakal", "\"unit mana yang bakal", "'unit mana yang bakal"] }, |
| { "type": "language_match", "expected": "Indonesian" } |
| ], |
| "note": "R15 guard: naming which unit a future breakdown will hit has no tool. Help must NOT hand the user that question to copy (a quoted suggestion) and should degrade to a supported one — unusual units (anomaly) or a downtime projection (forecast)." |
| }, |
| { |
| "id": "cap_03", "group": "capability", "carried_over": false, "manual_review": false, |
| "state": { "analysis_title": "PA analysis", "objective": "find out what causes low PA", "business_questions": ["what causes low PA?"], "report_id": null }, |
| "report_ready": { "ready": false, "missing": ["analysis"] }, |
| "history": [{ "role": "human", "content": "what should I ask to find the cause of low PA?" }], |
| "message": null, |
| "asserts": [ |
| { "type": "must_contain_any", "patterns": ["affect", "drive", "driver", "related", "relate", "explain", "correlat"] }, |
| { "type": "must_not_contain_any", "patterns": ["“what causes", "\"what causes", "'what causes", "“what is causing", "\"what is causing"] }, |
| { "type": "language_match", "expected": "English" } |
| ], |
| "note": "Causal-claim guard: a cause-and-effect claim has no tool. Help must not suggest a 'what causes' question to copy and should reframe it as key drivers / correlation ('what most affects PA')." |
| } |
| ] |
| } |
|
|
