Buckets:
| #!/usr/bin/env python3 | |
| """analysis_questionnaire_v1.py -- pre-registered-style analysis of the FIELDED short questionnaire (user_study_v1, 2026-09-22). | |
| Inputs (from parse_questionnaire_pdfs.py): questionnaire.csv (long, data_schema layout) and sessions.csv (session_index 1 = C, 2 = S per | |
| the owner's recording rule). Endpoints available with the fielded items: | |
| EP0 appropriateness = mean(APP1, APP2, APP3) [APP3 positively worded in the fielded form -> NOT reverse-scored; protocol deviation] | |
| ES1 timeliness = TIM1 | |
| ES2 annoyance = ANN1 (higher = more annoying) | |
| ES6 preference = END1 decoded to C / S / none (session 1 = C) | |
| Statistics (protocol section 11): participant-level paired bootstrap of the mean difference C - S (2000 resamples, seed 20260922, percentile | |
| 95 % CI) + two-sided Wilcoxon signed-rank (scipy, zero_method "wilcox"); verdict above / below / inconclusive; p beside the CI, never alone. | |
| Missing: cannot_judge / unanswered = missing; APP needs >= 2 of 3 items; primary = complete pairs; sensitivity NP-neutral = missing construct -> 4. | |
| Preference: shares with Wilson 95 % CI; (prefer C - prefer S) with a participant-level bootstrap CI. | |
| Reuses paired_endpoints.paired_table / boot_mean / analyse. | |
| """ | |
| import argparse, math, os, sys | |
| import numpy as np, pandas as pd | |
| sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) | |
| from paired_endpoints import paired_table, boot_mean, analyse # noqa: E402 | |
| CONSTRUCTS = {"EP0_appropriateness": (["APP1", "APP2", "APP3"], 2), "ES1_timeliness": (["TIM1"], 1), "ES2_annoyance": (["ANN1"], 1)} | |
| THEMES = { | |
| "C frequent / loud / distracting": ["constant", "frequent", "loud", "distracting", "annoying", "too many", "sound"], | |
| "C context-aware (pedestrians, traffic)": ["context", "pedestrian", "traffic", "stopping", "intelligent", "accurate", "quick"], | |
| "S only on hands-off / insensitive": ["hands", "steering", "insensitive", "less sensitive", "did not alert", "didn't warn", "not alert", "not accurate", "late"], | |
| "wants traffic-dependent prompting": ["heavy", "light", "straight", "danger"], | |
| } | |
| def wilson(k, n, z=1.96): | |
| if n == 0: | |
| return (float("nan"), float("nan")) | |
| p = k / n; d = 1 + z * z / n; c = p + z * z / (2 * n); h = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) | |
| return ((c - h) / d, (c + h) / d) | |
| def construct_scores(q, neutral_missing=False): | |
| s = q[q["form"] == "S"].copy() | |
| s["raw_value"] = pd.to_numeric(s["raw_value"], errors="coerce") | |
| s["session_index"] = s["session_index"].astype(int) | |
| out = {} | |
| for name, (items, need) in CONSTRUCTS.items(): | |
| g = s[s["item_id"].isin(items)].groupby(["participant_id", "session_index"])["raw_value"] | |
| cnt = g.count(); mean = g.mean() | |
| val = mean.where(cnt >= need) | |
| if neutral_missing: | |
| val = val.fillna(4.0) | |
| out[name] = val | |
| return pd.DataFrame(out) | |
| def preference(q, sessions, rng, n_res): | |
| e = q[(q["form"] == "E") & (q["item_id"] == "END1")].set_index("participant_id")["raw_value"].astype(str) | |
| cond = sessions.set_index(["participant_id", "session_index"])["condition"] | |
| dec = {} | |
| for pid, v in e.items(): | |
| if v in ("1", "2"): | |
| dec[pid] = cond.get((pid, int(v)), "?") | |
| elif v == "none": | |
| dec[pid] = "none" | |
| n = len(dec); kC = sum(1 for v in dec.values() if v == "C"); kS = sum(1 for v in dec.values() if v == "S"); kN = sum(1 for v in dec.values() if v == "none") | |
| rows = [] | |
| for lab, k in (("prefer C", kC), ("prefer S", kS), ("no difference", kN)): | |
| lo, hi = wilson(k, n); rows.append({"choice": lab, "k": k, "n": n, "share": k / n if n else float("nan"), "wilson_lo": lo, "wilson_hi": hi}) | |
| x = np.array([1 if v == "C" else (-1 if v == "S" else 0) for v in dec.values()], float) | |
| if n >= 2: | |
| b = boot_mean(x, rng, n_res); lo, hi = np.percentile(b, [2.5, 97.5]) | |
| else: | |
| lo = hi = float("nan") | |
| diff = {"metric": "share(prefer C) - share(prefer S)", "value": (kC - kS) / n if n else float("nan"), "ci95_lo": lo, "ci95_hi": hi, | |
| "verdict": ("above" if lo > 0 else ("below" if hi < 0 else "inconclusive")) if n >= 2 else "n/a", "n": n} | |
| return pd.DataFrame(rows), diff, dec | |
| def themes(q): | |
| e = q[(q["form"] == "E") & (q["item_id"] == "END5")].set_index("participant_id")["free_text"].fillna("") | |
| rows = [] | |
| for pid, txt in e.items(): | |
| t = txt.lower() | |
| hit = [k for k, kws in THEMES.items() if any(w in t for w in kws)] | |
| rows.append({"participant_id": pid, "themes": "; ".join(hit) if txt.strip() else "(no comment)", "comment": txt.strip()}) | |
| return pd.DataFrame(rows) | |
| def fmt(x, nd=2): | |
| return "n/a" if x is None or (isinstance(x, float) and math.isnan(x)) else f"{x:.{nd}f}" | |
| def main(argv=None): | |
| ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) | |
| ap.add_argument("--questionnaire", required=True); ap.add_argument("--sessions", required=True); ap.add_argument("--out", required=True) | |
| ap.add_argument("--resamples", type=int, default=2000); ap.add_argument("--seed", type=int, default=20260922) | |
| a = ap.parse_args(argv) | |
| q = pd.read_csv(a.questionnaire, dtype=str, keep_default_na=False) | |
| sessions = pd.read_csv(a.sessions); sessions["session_index"] = sessions["session_index"].astype(int) | |
| os.makedirs(a.out, exist_ok=True) | |
| n_participants = sessions["participant_id"].nunique() | |
| results = {} | |
| for label, neutral in (("primary_complete_pairs", False), ("sensitivity_NP_neutral", True)): | |
| cs = construct_scores(q, neutral_missing=neutral) | |
| tables = paired_table(cs, sessions[["participant_id", "session_index", "condition_code"]], "X", "Y") | |
| df = analyse(tables, None, a.resamples, a.seed, "C", "S") | |
| df.insert(0, "analysis", label); results[label] = (df, tables) | |
| endpoints = pd.concat([r[0] for r in results.values()], ignore_index=True) | |
| endpoints.to_csv(os.path.join(a.out, "endpoints.csv"), index=False) | |
| rng = np.random.default_rng(a.seed) | |
| pref, pref_diff, dec = preference(q, sessions, rng, a.resamples) | |
| pref.to_csv(os.path.join(a.out, "preference.csv"), index=False) | |
| # per-participant table (primary scoring) | |
| cs = construct_scores(q).reset_index() | |
| cs["condition"] = cs.apply(lambda r: sessions.set_index(["participant_id", "session_index"])["condition"].get((r["participant_id"], r["session_index"])), axis=1) | |
| wide = cs.pivot_table(index="participant_id", columns="condition", values=list(CONSTRUCTS), aggfunc="first") | |
| per = pd.DataFrame(index=sorted(cs["participant_id"].unique())) | |
| for ep in CONSTRUCTS: | |
| for c in ("C", "S"): | |
| per[f"{ep}_{c}"] = wide[(ep, c)] if (ep, c) in wide.columns else np.nan | |
| per[f"{ep}_diff"] = per[f"{ep}_C"] - per[f"{ep}_S"] | |
| per["preference"] = pd.Series(dec) | |
| th = themes(q).set_index("participant_id"); per["comment_themes"] = th["themes"]; per["comment"] = th["comment"] | |
| per.index.name = "participant_id"; per.to_csv(os.path.join(a.out, "per_participant.csv")) | |
| # markdown | |
| L = [f"# Questionnaire analysis v1 (fielded short form) — {n_participants} participants with data", "", | |
| f"Inputs: `{os.path.relpath(a.questionnaire)}`, `{os.path.relpath(a.sessions)}`; bootstrap {a.resamples} resamples, seed {a.seed}; Wilcoxon two-sided (scipy).", | |
| "Condition rule (owner, 2026-09-23): session 1 = CockpitWM (C), session 2 = stock openpilot (S) for every respondent. Differences are C − S.", "", | |
| "## Endpoints", "", "| analysis | endpoint | n pairs | mean C | mean S | mean Δ (C−S) | median Δ | 95 % CI | Wilcoxon p | verdict |", "|---|---|---|---|---|---|---|---|---|---|"] | |
| for _, r in endpoints.iterrows(): | |
| if "note" in r and isinstance(r.get("note"), str) and r["note"]: | |
| L.append(f"| {r['analysis']} | {r['endpoint']} | {r['n_pairs']} | — | — | — | — | — | — | {r['note']} |"); continue | |
| L.append(f"| {r['analysis']} | {r['endpoint']} | {int(r['n_pairs'])} | {fmt(r['mean_C'])} | {fmt(r['mean_S'])} | {fmt(r['mean_diff'], 2)} | {fmt(r['median_diff'], 2)} | [{fmt(r['ci95_lo'])}, {fmt(r['ci95_hi'])}] | {fmt(r['wilcoxon_p'], 4)} | {r['verdict']} |") | |
| L += ["", "* EP0 = mean(APP1, APP2, APP3), APP3 positively worded in the fielded form → not reverse-scored (deviation from protocol §9.1).", | |
| "* ES2 annoyance: higher = more annoying, so a positive Δ means C was rated MORE annoying.", | |
| "* primary_complete_pairs: participants with both constructs present (cannot_judge / unanswered = missing; APP needs ≥ 2 of 3 items). sensitivity_NP_neutral: missing construct scores set to 4.", "", | |
| "## Preference (END1, decoded: 'first drive' = C)", "", "| choice | k | n | share | Wilson 95 % CI |", "|---|---|---|---|---|"] | |
| for _, r in pref.iterrows(): | |
| L.append(f"| {r['choice']} | {int(r['k'])} | {int(r['n'])} | {fmt(r['share'])} | [{fmt(r['wilson_lo'])}, {fmt(r['wilson_hi'])}] |") | |
| L += ["", f"share(prefer C) − share(prefer S) = {fmt(pref_diff['value'])}, bootstrap 95 % CI [{fmt(pref_diff['ci95_lo'])}, {fmt(pref_diff['ci95_hi'])}] → {pref_diff['verdict']} (n {pref_diff['n']})", "", | |
| "## Per participant (primary scoring)", "", "| pid | APP C | APP S | Δ | TIM C | TIM S | Δ | ANN C | ANN S | Δ | prefers | comment themes |", "|---|---|---|---|---|---|---|---|---|---|---|---|"] | |
| for pid, r in per.iterrows(): | |
| L.append(f"| {pid} | {fmt(r['EP0_appropriateness_C'])} | {fmt(r['EP0_appropriateness_S'])} | {fmt(r['EP0_appropriateness_diff'])} | {fmt(r['ES1_timeliness_C'],0)} | {fmt(r['ES1_timeliness_S'],0)} | {fmt(r['ES1_timeliness_diff'],0)} | {fmt(r['ES2_annoyance_C'],0)} | {fmt(r['ES2_annoyance_S'],0)} | {fmt(r['ES2_annoyance_diff'],0)} | {r['preference'] if isinstance(r['preference'], str) else 'n/a'} | {r['comment_themes'] if isinstance(r['comment_themes'], str) else ''} |") | |
| # exposure confound (owner note 2026-09-23): S prompted rarely / late, so its low annoyance partly reflects under-alerting | |
| sS = q[(q["form"] == "S") & (q["session_index"].astype(str) == "2")].copy(); sS["raw_value"] = pd.to_numeric(sS["raw_value"], errors="coerce") | |
| tim_s = sS[sS["item_id"] == "TIM1"].dropna(subset=["raw_value"]); ann_s = sS[sS["item_id"] == "ANN1"].dropna(subset=["raw_value"]) | |
| ann_c = q[(q["form"] == "S") & (q["session_index"].astype(str) == "1") & (q["item_id"] == "ANN1")].copy(); ann_c["raw_value"] = pd.to_numeric(ann_c["raw_value"], errors="coerce"); ann_c = ann_c.dropna(subset=["raw_value"]) | |
| late_s = int((tim_s["raw_value"] <= 3).sum()); floor_s = int((ann_s["raw_value"] == 1).sum()); floor_c = int((ann_c["raw_value"] == 1).sum()) | |
| both = tim_s.set_index("participant_id")["raw_value"].to_frame("tim").join(ann_s.set_index("participant_id")["raw_value"].to_frame("ann"), how="inner") | |
| late_and_floor = int(((both["tim"] <= 3) & (both["ann"] == 1)).sum()) | |
| miss_theme = int(th["themes"].str.contains("S only on hands-off", na=False).sum()) if len(th) else 0 | |
| L += ["", "## Interpretation note: annoyance must be read together with exposure", "", | |
| "The stock openpilot monitor (S) prompts only while engaged and reacts to a narrow cue set (hands off the wheel, sustained gaze away); in these drives it prompted rarely and, per the respondents, late or not at all. Its low annoyance score is therefore largely a consequence of issuing few prompts, not evidence of better-calibrated prompting. Supporting numbers from this sample:", "", | |
| f"* S timeliness (TIM1) rated <= 3 by {late_s} of {len(tim_s)} respondents who could judge it; S annoyance at the floor value 1 for {floor_s} of {len(ann_s)} (C: {floor_c} of {len(ann_c)}).", | |
| f"* {late_and_floor} of {len(both)} respondents gave S BOTH timeliness <= 3 and annoyance = 1: the un-annoying S sessions are the ones where S did not prompt when the driver expected it.", | |
| f"* Free text: {miss_theme} respondents describe S as prompting only on hands-off or missing / late prompts.", | |
| "* Consequence for the endpoints: EP0 (appropriateness) and ES1 (timeliness) favour C with CIs excluding 0; ES2 (annoyance) favours S but is confounded with exposure. The fair comparison of intrusiveness is annoyance conditional on prompting (per-alert ratings, ES8) and prompts per hour (ES7) from the device logs; both are pending. Until then ES2 is to be reported as 'C prompted more and was rated more intrusive; S prompted rarely', not as 'S was better calibrated'.", | |
| "* Design implication for CockpitWM: the intrusiveness complaints concern prompt frequency and sound (60 s cooldown re-fires while the read-out stays above threshold; loud tone), not the situations chosen. An onset-persistence gate and a softer first-level tone address them without touching the model or thresholds."] | |
| L += ["", "## Free-text comments (END5, verbatim, P-ids only)", ""] | |
| for pid, r in per.iterrows(): | |
| if isinstance(r["comment"], str) and r["comment"].strip(): | |
| L.append(f"* **{pid}**: {r['comment'].strip()}") | |
| open(os.path.join(a.out, "endpoints.md"), "w").write("\n".join(L) + "\n") | |
| print("\n".join(L[:len(L)])) | |
| return 0 | |
| if __name__ == "__main__": | |
| sys.exit(main()) | |
Xet Storage Details
- Size:
- 13.3 kB
- Xet hash:
- f94dec09f61b67805beae676a883fe9a47ca7682433528b78ac9b480684e56ee
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.