DocDoeAI / src /lib /beta /betaReport.ts
asnannp's picture
deploy: sync backend to Space root (learn-lesson HF cache fix)
65d68f5
Raw History Blame Contribute Delete
13.4 kB
/**
* Day 7 — privacy-safe local beta report.
*
* Builds a small, derived summary from the existing local beta storage
* (access / analytics / feedback) so the builder can ask a beta student to
* copy/paste or download a report of what happened on their device, without
* ever exposing raw localStorage, free-text feedback messages, or any
* contact/identity field. Nothing here calls a backend or external service.
*/
import { getBetaAnalyticsSummary, loadBetaAnalytics, type BetaAnalyticsSummary } from "./betaAnalytics";
import { BETA_FEEDBACK_KIND_LABELS, loadBetaFeedback, type BetaFeedbackKind } from "./betaFeedback";
import { loadBetaAccess } from "./betaAccess";
export const BETA_REPORT_SCHEMA_VERSION = 1;
const MAX_REPORT_EVENTS = 20;
const MAX_REPORT_FEEDBACK = 10;
export type BetaReportAccess = {
hasAccess: boolean;
classLevel?: string;
board?: string;
languagePreference?: string;
grantedAt?: string;
};
export type BetaReportFunnel = {
grantFromViewedPercent: number;
classStartFromGrantedPercent: number;
feedbackFromClassStartsPercent: number;
};
export type BetaReportFeedbackByKind = {
kind: BetaFeedbackKind;
label: string;
count: number;
};
export type BetaReportRecentEvent = {
name: string;
route: string;
createdAt: string;
stepLabel?: string;
};
export type BetaReportRecentFeedback = {
kind: BetaFeedbackKind;
label: string;
rating?: number;
route: string;
createdAt: string;
status: string;
sessionId: string;
};
export type BetaReport = {
generatedAt: string;
schemaVersion: number;
access: BetaReportAccess;
analytics: BetaAnalyticsSummary;
funnel: BetaReportFunnel;
feedbackByKind: BetaReportFeedbackByKind[];
recentEvents: BetaReportRecentEvent[];
recentFeedback: BetaReportRecentFeedback[];
};
export type BetaLaunchReadinessStatus = "Not enough pilot signal" | "Pilot ready" | "Ready for 10-20" | "Needs review";
export type BetaPilotIssueStatus = "Good signal" | "Needs explanation fix" | "Too early";
export type BetaPilotIssueTracker = {
helpedUnderstand: number;
confusingReports: number;
wrongOrDoubtfulReports: number;
tooLong: number;
needMoreExamples: number;
totalTrackedFeedback: number;
status: BetaPilotIssueStatus;
};
export type BetaLessonQualityScore = {
score: number;
label: "No signal yet" | "Needs review" | "Pilot signal improving";
factors: Array<{ label: string; value: string; impact: number }>;
};
export type BetaSafeJsonExport = {
generatedAt: string;
schemaVersion: number;
privacy: string;
report: BetaReport;
pilotIssueTracker: BetaPilotIssueTracker;
lessonQualityScore: BetaLessonQualityScore;
};
export const BETA_LESSON_QUALITY_CHECKLIST = [
"Does the lesson explain concept before formula?",
"Are derivations clear?",
"Are examples included?",
"Are common mistakes shown?",
"Is board-answer style included?",
"Is revision saved?",
"Is feedback collected?",
] as const;
/**
* Defense-in-depth guard. Recursively strips any object key whose name
* matches a forbidden PII/contact/auth fragment, so a report can never carry
* those fields even if a future edit accidentally introduces one.
*/
const FORBIDDEN_KEY_FRAGMENTS = [
"email",
"phone",
"mobile",
"contact",
"fullname",
"full_name",
"school",
"location",
"address",
"token",
"auth",
"password",
"answertext",
"chat",
"messagetext",
"nickname",
"aadhaar",
"payment",
];
function isForbiddenKey(key: string): boolean {
const normalized = key.toLowerCase();
return FORBIDDEN_KEY_FRAGMENTS.some((fragment) => normalized.includes(fragment));
}
function stripUnsafeKeys(value: unknown): unknown {
if (Array.isArray(value)) return value.map(stripUnsafeKeys);
if (value && typeof value === "object") {
const out: Record<string, unknown> = {};
for (const [key, val] of Object.entries(value as Record<string, unknown>)) {
if (isForbiddenKey(key)) continue;
out[key] = stripUnsafeKeys(val);
}
return out;
}
return value;
}
export function assertSafeBetaReport<T>(report: T): T {
return stripUnsafeKeys(report) as T;
}
function percent(numerator: number, denominator: number): number {
if (denominator <= 0) return 0;
return Math.round((numerator / denominator) * 100);
}
/** Builds the current local beta report from access/analytics/feedback storage. */
export function buildBetaReport(): BetaReport {
const access = loadBetaAccess();
const events = loadBetaAnalytics();
const feedbackItems = loadBetaFeedback();
const analytics = getBetaAnalyticsSummary(events);
const byKind = new Map<BetaFeedbackKind, number>();
for (const item of feedbackItems) byKind.set(item.kind, (byKind.get(item.kind) ?? 0) + 1);
const report: BetaReport = {
generatedAt: new Date().toISOString(),
schemaVersion: BETA_REPORT_SCHEMA_VERSION,
access: {
hasAccess: access.hasAccess,
classLevel: access.classLevel,
board: access.board,
languagePreference: access.languagePreference,
grantedAt: access.grantedAt,
},
analytics,
funnel: {
grantFromViewedPercent: percent(analytics.betaAccessGranted, analytics.betaAccessViewed),
classStartFromGrantedPercent: percent(analytics.classStarts, analytics.betaAccessGranted),
feedbackFromClassStartsPercent: percent(analytics.feedbackCount, analytics.classStarts),
},
feedbackByKind: Array.from(byKind.entries()).map(([kind, count]) => ({
kind,
label: BETA_FEEDBACK_KIND_LABELS[kind],
count,
})),
recentEvents: events.slice(-MAX_REPORT_EVENTS).map((event) => ({
name: event.name,
route: event.route,
createdAt: event.createdAt,
stepLabel: event.stepLabel,
})),
recentFeedback: feedbackItems.slice(-MAX_REPORT_FEEDBACK).map((item) => ({
kind: item.kind,
label: BETA_FEEDBACK_KIND_LABELS[item.kind],
rating: item.rating,
route: item.route,
createdAt: item.createdAt,
status: item.status,
sessionId: item.sessionId,
})),
};
return assertSafeBetaReport(report);
}
/** Plain-text rendering suitable for clipboard copy or a downloaded .txt file. */
export function formatBetaReportForCopy(report: BetaReport = buildBetaReport()): string {
const lines: string[] = [];
lines.push("DocDoe SSLC Physics beta report (local device only)");
lines.push(`Generated: ${report.generatedAt}`);
lines.push(`Schema version: ${report.schemaVersion}`);
lines.push("");
lines.push("Access");
lines.push(` Has access: ${report.access.hasAccess ? "yes" : "no"}`);
if (report.access.classLevel) lines.push(` Class: ${report.access.classLevel}`);
if (report.access.board) lines.push(` Board: ${report.access.board}`);
if (report.access.languagePreference) lines.push(` Language: ${report.access.languagePreference}`);
lines.push("");
lines.push("Funnel");
lines.push(` Grant from viewed: ${report.funnel.grantFromViewedPercent}%`);
lines.push(` Class start from granted: ${report.funnel.classStartFromGrantedPercent}%`);
lines.push(` Feedback from class starts: ${report.funnel.feedbackFromClassStartsPercent}%`);
lines.push("");
lines.push("Analytics counts");
lines.push(` Total events: ${report.analytics.totalEvents}`);
lines.push(` Class starts: ${report.analytics.classStarts}`);
lines.push(` Class completions: ${report.analytics.classCompletions}`);
lines.push(` Revision saves: ${report.analytics.revisionSaves}`);
lines.push(` Feedback submitted: ${report.analytics.feedbackCount}`);
lines.push(` Bug reports: ${report.analytics.bugReportCount}`);
lines.push(` Confusing reports: ${report.analytics.confusingReportCount}`);
lines.push(` Wrong/doubtful reports: ${report.analytics.wrongReportCount}`);
lines.push("");
lines.push("Feedback by kind");
if (report.feedbackByKind.length === 0) {
lines.push(" (none yet)");
} else {
for (const entry of report.feedbackByKind) lines.push(` ${entry.label}: ${entry.count}`);
}
lines.push("");
lines.push(`Recent events (last ${report.recentEvents.length})`);
if (report.recentEvents.length === 0) lines.push(" (none yet)");
for (const event of report.recentEvents) lines.push(` ${event.createdAt} - ${event.name} (${event.route})`);
lines.push("");
lines.push(`Recent feedback (last ${report.recentFeedback.length})`);
if (report.recentFeedback.length === 0) lines.push(" (none yet)");
for (const item of report.recentFeedback) {
lines.push(` ${item.createdAt} - ${item.label}${item.rating ? ` (${item.rating}/5)` : ""} on ${item.route}`);
}
return lines.join("\n");
}
/** Short one-line summary for compact display (e.g. a "beta data health" card). */
export function getBetaReportSummary(report: BetaReport = buildBetaReport()): string {
return `${report.analytics.betaAccessGranted} granted, ${report.analytics.classStarts} class starts, ${report.analytics.feedbackCount} feedback, ${report.analytics.bugReportCount} bugs`;
}
function feedbackKindCount(report: BetaReport, kind: BetaFeedbackKind): number {
return report.feedbackByKind.find((entry) => entry.kind === kind)?.count ?? 0;
}
export function computeBetaPilotIssueTracker(report: BetaReport = buildBetaReport()): BetaPilotIssueTracker {
const helpedUnderstand = feedbackKindCount(report, "helped_understand");
const confusingReports = feedbackKindCount(report, "still_confusing");
const wrongOrDoubtfulReports = feedbackKindCount(report, "wrong_or_doubtful");
const tooLong = feedbackKindCount(report, "too_long");
const needMoreExamples = feedbackKindCount(report, "need_more_examples");
const totalTrackedFeedback = helpedUnderstand + confusingReports + wrongOrDoubtfulReports + tooLong + needMoreExamples;
const explanationIssues = confusingReports + wrongOrDoubtfulReports + tooLong + needMoreExamples;
let status: BetaPilotIssueStatus = "Too early";
if (totalTrackedFeedback > 0) {
status = explanationIssues > helpedUnderstand ? "Needs explanation fix" : "Good signal";
}
return {
helpedUnderstand,
confusingReports,
wrongOrDoubtfulReports,
tooLong,
needMoreExamples,
totalTrackedFeedback,
status,
};
}
function clampScore(score: number): number {
return Math.max(0, Math.min(100, score));
}
export function computeBetaLessonQualityScore(report: BetaReport = buildBetaReport()): BetaLessonQualityScore {
const issues = computeBetaPilotIssueTracker(report);
const factors: BetaLessonQualityScore["factors"] = [
{
label: "Class started",
value: report.analytics.classStarts > 0 ? "yes" : "not yet",
impact: report.analytics.classStarts > 0 ? 25 : 0,
},
{
label: "Revision saved",
value: report.analytics.revisionSaves > 0 ? "yes" : "not yet",
impact: report.analytics.revisionSaves > 0 ? 20 : 0,
},
{
label: "Feedback submitted",
value: report.analytics.feedbackCount > 0 ? "yes" : "not yet",
impact: report.analytics.feedbackCount > 0 ? 20 : 0,
},
{
label: "Helped understand",
value: String(issues.helpedUnderstand),
impact: issues.helpedUnderstand > 0 ? 20 : 0,
},
{
label: "Confusing or wrong reports",
value: String(issues.confusingReports + issues.wrongOrDoubtfulReports),
impact: -Math.min(30, (issues.confusingReports + issues.wrongOrDoubtfulReports) * 15),
},
{
label: "Needs more examples / too long",
value: String(issues.needMoreExamples + issues.tooLong),
impact: -Math.min(20, issues.needMoreExamples * 10 + issues.tooLong * 5),
},
];
const score = clampScore(factors.reduce((total, factor) => total + factor.impact, 0));
const label =
score === 0 ? "No signal yet" : issues.status === "Needs explanation fix" || score < 55 ? "Needs review" : "Pilot signal improving";
return { score, label, factors };
}
export function buildSafeBetaJsonExport(report: BetaReport = buildBetaReport()): BetaSafeJsonExport {
return assertSafeBetaReport({
generatedAt: new Date().toISOString(),
schemaVersion: report.schemaVersion,
privacy: "Sanitized local beta summary only. No raw localStorage dump, raw feedback text, contact fields, full answers, or private chat text.",
report,
pilotIssueTracker: computeBetaPilotIssueTracker(report),
lessonQualityScore: computeBetaLessonQualityScore(report),
});
}
/**
* Conservative pilot-readiness read from local analytics. Intentionally
* favors "not enough signal" / "needs review" over a false "ready" verdict.
*/
export function computeBetaLaunchReadiness(analytics: BetaAnalyticsSummary): BetaLaunchReadinessStatus {
const granted = analytics.betaAccessGranted;
const classStarts = analytics.classStarts;
const feedbackCount = analytics.feedbackCount;
const concerningCount = analytics.confusingReportCount + analytics.wrongReportCount;
const concerningRatio = feedbackCount > 0 ? concerningCount / feedbackCount : 0;
if (granted === 0 || classStarts === 0) return "Not enough pilot signal";
if (feedbackCount > 0 && concerningRatio >= 0.5) return "Needs review";
if (granted >= 10 && classStarts >= 10 && feedbackCount >= 5) return "Ready for 10-20";
if (granted >= 3 && classStarts >= 3 && feedbackCount >= 3) return "Pilot ready";
return "Not enough pilot signal";
}