Spaces:

asnannp
/

DocDoeAI

Runtime error

DocDoeAI / src /lib /teaching /physicsQaReport.ts

deploy: sync backend to Space root (learn-lesson HF cache fix)

65d68f5 about 1 month ago

8.51 kB

/**
 * Phase 6K — Physics full-subject QA + exam-readiness report.
 *
 * Proves DocDoe Physics is usable as a paid tuition product by checking every
 * curated seed across all 7 chapters for coverage, source refs, bilingual
 * support, visual boards, exam-answer quality and honesty (no fake claims, no
 * curated-PYQ). It reuses the seed validator and never weakens it.
 *
 * This module is read-only: it computes a report, it does not mutate seeds.
 */

import { getAllLessonSeeds, type TuitionLessonSeed } from "./physicsLessonSeeds";
import { validateLessonSeed } from "./lessonSeedValidation";

export type PhysicsQaCoverage = {
  hasConcept: boolean;
  hasFormulaOrRule: boolean;
  hasVisualBoard: boolean;
  hasNumerical: boolean;
  hasExamPractice: boolean;
  hasRevision: boolean;
  hasMalayalamSupport: boolean;
  hasManglishSupport: boolean;
};

export type PhysicsQaChapter = {
  chapterId: string;
  chapterTitle: string;
  seedCount: number;
  coverage: PhysicsQaCoverage;
  warnings: string[];
  needsHumanReviewSeeds: string[];
};

export type PhysicsQaReport = {
  totalChapters: number;
  totalSeeds: number;
  chapters: PhysicsQaChapter[];
  globalWarnings: string[];
  productionReadiness: {
    status: "ready" | "ready_with_review_flags" | "not_ready";
    blockers: string[];
    recommendedFixes: string[];
  };
};

// Placeholder phrases that would betray raw auto-extraction leaking into a seed.
const GENERIC_PHRASES = [
  "note 1",
  "core concept",
  "important keyword",
  "standard physics calculation",
  "refer to formulas.json",
  "textbook derivation",
  "lorem ipsum",
];

function coverageFor(seeds: TuitionLessonSeed[]): PhysicsQaCoverage {
  const types = new Set(seeds.map((s) => s.contentType));
  // Day 20 — a chapter can satisfy exam-answer/revision coverage two ways:
  // a dedicated mission of that contentType (the original structure), OR the
  // SSLC teaching-order structure where EVERY mission carries its own board
  // exam answers and its own recap/must-remember block (Sound Waves after the
  // reorder). What matters for the student is that the coverage exists, not
  // which mission label carries it.
  const everyMissionHasExamAnswers = seeds.length > 0 && seeds.every((s) => s.examAnswers.length > 0);
  const everyMissionHasRecap = seeds.length > 0 && seeds.every((s) => s.recap.summary.length > 0 && s.recap.mustRemember.length > 0);
  return {
    hasConcept: types.has("concept"),
    hasFormulaOrRule: types.has("formula") || seeds.some((s) => s.formulas.length > 0),
    hasVisualBoard: seeds.some((s) => (s.visualBoard?.frames?.length ?? 0) > 0),
    hasNumerical: types.has("numerical"),
    hasExamPractice: types.has("exam_answer") || everyMissionHasExamAnswers,
    hasRevision: types.has("revision") || everyMissionHasRecap,
    hasMalayalamSupport: seeds.every((s) => Boolean(s.teach.malayalamSupport?.trim())),
    hasManglishSupport: seeds.every((s) => Boolean(s.teach.manglishSupport?.trim())),
  };
}

function seedQaWarnings(seed: TuitionLessonSeed): string[] {
  const warnings: string[] = [];
  const where = seed.missionId || "<no id>";

  // Reuse the hard validator (source refs, 11 steps, exam keywords, quiz
  // answers/explanations, formula symbols/units, fake-guarantee, curated PYQ).
  for (const error of validateLessonSeed(seed).errors) {
    warnings.push(`[${where}] ${error}`);
  }

  if (!seed.missionTitle?.trim()) warnings.push(`[${where}] Empty mission title.`);
  if (!seed.goal?.trim()) warnings.push(`[${where}] Empty goal.`);

  // Every seed must carry at least one bilingual support snippet.
  if (!seed.teach.malayalamSupport?.trim() && !seed.teach.manglishSupport?.trim()) {
    warnings.push(`[${where}] Missing both Malayalam and Manglish support.`);
  }

  // Visual board must have frames.
  if ((seed.visualBoard?.frames?.length ?? 0) === 0) {
    warnings.push(`[${where}] Visual board has no frames.`);
  }

  // needsHumanReview must explain itself.
  if (seed.qualityFlags.needsHumanReview && !seed.qualityFlags.reason?.trim()) {
    warnings.push(`[${where}] needsHumanReview is set without a reason.`);
  }

  // Generic/placeholder wording must not survive into a curated seed.
  const text = JSON.stringify(seed).toLowerCase();
  for (const phrase of GENERIC_PHRASES) {
    if (text.includes(phrase)) warnings.push(`[${where}] Contains generic/placeholder wording: "${phrase}".`);
  }

  return warnings;
}

function chapterCoverageWarnings(coverage: PhysicsQaCoverage): string[] {
  const warnings: string[] = [];
  if (!coverage.hasConcept) warnings.push("No concept mission.");
  if (!coverage.hasExamPractice) warnings.push("No exam-answer mission.");
  if (!coverage.hasRevision) warnings.push("No revision mission.");
  if (!coverage.hasVisualBoard) warnings.push("No visual-board-supported mission.");
  if (!coverage.hasMalayalamSupport) warnings.push("Some seed is missing Malayalam support.");
  if (!coverage.hasManglishSupport) warnings.push("Some seed is missing Manglish support.");
  return warnings;
}

export function buildPhysicsQaReport(): PhysicsQaReport {
  // Day 20 — the seed pool now includes the first Chemistry draft; this
  // report is the PHYSICS exam-readiness QA, so it scopes to Physics seeds.
  const seeds = getAllLessonSeeds().filter((seed) => seed.subject === "Physics");
  const byChapter = new Map<string, TuitionLessonSeed[]>();
  for (const seed of seeds) {
    byChapter.set(seed.chapterId, [...(byChapter.get(seed.chapterId) ?? []), seed]);
  }

  const blockers: string[] = [];
  const chapters: PhysicsQaChapter[] = Array.from(byChapter.entries()).map(([chapterId, chapterSeeds]) => {
    const coverage = coverageFor(chapterSeeds);
    const seedWarnings = chapterSeeds.flatMap(seedQaWarnings);
    const coverageWarnings = chapterCoverageWarnings(coverage);
    const warnings = [...coverageWarnings, ...seedWarnings];
    const needsHumanReviewSeeds = chapterSeeds
      .filter((s) => s.qualityFlags.needsHumanReview)
      .map((s) => s.missionId);

    // Blockers: anything that makes the chapter unsafe to sell.
    if (chapterSeeds.length === 0) blockers.push(`${chapterId}: chapter has zero seeds.`);
    for (const w of coverageWarnings) {
      if (!w.startsWith("Some seed is missing")) blockers.push(`${chapterId}: ${w}`);
      else blockers.push(`${chapterId}: ${w}`);
    }
    // Any hard validator error is a blocker (source refs, 11 steps, fake claims, curated PYQ, etc.).
    for (const seed of chapterSeeds) {
      const errors = validateLessonSeed(seed).errors;
      if (errors.length > 0) blockers.push(`${chapterId}/${seed.missionId}: ${errors.length} validation error(s).`);
      if (seed.qualityFlags.needsHumanReview && !seed.qualityFlags.reason?.trim()) {
        blockers.push(`${chapterId}/${seed.missionId}: needsHumanReview without a reason.`);
      }
    }

    return {
      chapterId,
      chapterTitle: chapterSeeds[0]?.chapterTitle ?? chapterId,
      seedCount: chapterSeeds.length,
      coverage,
      warnings,
      needsHumanReviewSeeds,
    };
  });

  const globalWarnings: string[] = [];
  if (byChapter.size !== 7) globalWarnings.push(`Expected 7 curated Physics chapters, found ${byChapter.size}.`);
  const qualitativeNoNumerical = chapters.filter((c) => !c.coverage.hasNumerical).map((c) => c.chapterId);
  if (qualitativeNoNumerical.length) {
    globalWarnings.push(
      `Chapters without a numerical mission (qualitative — informational, not a blocker): ${qualitativeNoNumerical.join(", ")}.`,
    );
  }

  const reviewFlagged = chapters.flatMap((c) => c.needsHumanReviewSeeds.map((m) => `${c.chapterId}/${m}`));

  const uniqueBlockers = Array.from(new Set(blockers));
  let status: PhysicsQaReport["productionReadiness"]["status"];
  if (uniqueBlockers.length > 0) {
    status = "not_ready";
  } else if (reviewFlagged.length > 0) {
    status = "ready_with_review_flags";
  } else {
    status = "ready";
  }

  const recommendedFixes: string[] = [];
  for (const flagged of reviewFlagged) {
    recommendedFixes.push(`Verify ${flagged} against the SCERT textbook, then clear its needsHumanReview flag.`);
  }
  if (uniqueBlockers.length > 0) {
    recommendedFixes.push("Resolve all blockers above before selling Physics as a paid product.");
  }

  return {
    totalChapters: byChapter.size,
    totalSeeds: seeds.length,
    chapters,
    globalWarnings,
    productionReadiness: {
      status,
      blockers: uniqueBlockers,
      recommendedFixes,
    },
  };
}

	/**
	* Phase 6K — Physics full-subject QA + exam-readiness report.
	*
	* Proves DocDoe Physics is usable as a paid tuition product by checking every
	* curated seed across all 7 chapters for coverage, source refs, bilingual
	* support, visual boards, exam-answer quality and honesty (no fake claims, no
	* curated-PYQ). It reuses the seed validator and never weakens it.
	*
	* This module is read-only: it computes a report, it does not mutate seeds.
	*/

	import { getAllLessonSeeds, type TuitionLessonSeed } from "./physicsLessonSeeds";
	import { validateLessonSeed } from "./lessonSeedValidation";

	export type PhysicsQaCoverage = {
	hasConcept: boolean;
	hasFormulaOrRule: boolean;
	hasVisualBoard: boolean;
	hasNumerical: boolean;
	hasExamPractice: boolean;
	hasRevision: boolean;
	hasMalayalamSupport: boolean;
	hasManglishSupport: boolean;
	};

	export type PhysicsQaChapter = {
	chapterId: string;
	chapterTitle: string;
	seedCount: number;
	coverage: PhysicsQaCoverage;
	warnings: string[];
	needsHumanReviewSeeds: string[];
	};

	export type PhysicsQaReport = {
	totalChapters: number;
	totalSeeds: number;
	chapters: PhysicsQaChapter[];
	globalWarnings: string[];
	productionReadiness: {
	status: "ready" \| "ready_with_review_flags" \| "not_ready";
	blockers: string[];
	recommendedFixes: string[];
	};
	};

	// Placeholder phrases that would betray raw auto-extraction leaking into a seed.
	const GENERIC_PHRASES = [
	"note 1",
	"core concept",
	"important keyword",
	"standard physics calculation",
	"refer to formulas.json",
	"textbook derivation",
	"lorem ipsum",
	];

	function coverageFor(seeds: TuitionLessonSeed[]): PhysicsQaCoverage {
	const types = new Set(seeds.map((s) => s.contentType));
	// Day 20 — a chapter can satisfy exam-answer/revision coverage two ways:
	// a dedicated mission of that contentType (the original structure), OR the
	// SSLC teaching-order structure where EVERY mission carries its own board
	// exam answers and its own recap/must-remember block (Sound Waves after the
	// reorder). What matters for the student is that the coverage exists, not
	// which mission label carries it.
	const everyMissionHasExamAnswers = seeds.length > 0 && seeds.every((s) => s.examAnswers.length > 0);
	const everyMissionHasRecap = seeds.length > 0 && seeds.every((s) => s.recap.summary.length > 0 && s.recap.mustRemember.length > 0);
	return {
	hasConcept: types.has("concept"),
	hasFormulaOrRule: types.has("formula") \|\| seeds.some((s) => s.formulas.length > 0),
	hasVisualBoard: seeds.some((s) => (s.visualBoard?.frames?.length ?? 0) > 0),
	hasNumerical: types.has("numerical"),
	hasExamPractice: types.has("exam_answer") \|\| everyMissionHasExamAnswers,
	hasRevision: types.has("revision") \|\| everyMissionHasRecap,
	hasMalayalamSupport: seeds.every((s) => Boolean(s.teach.malayalamSupport?.trim())),
	hasManglishSupport: seeds.every((s) => Boolean(s.teach.manglishSupport?.trim())),
	};
	}

	function seedQaWarnings(seed: TuitionLessonSeed): string[] {
	const warnings: string[] = [];
	const where = seed.missionId \|\| "<no id>";

	// Reuse the hard validator (source refs, 11 steps, exam keywords, quiz
	// answers/explanations, formula symbols/units, fake-guarantee, curated PYQ).
	for (const error of validateLessonSeed(seed).errors) {
	warnings.push(`[${where}] ${error}`);
	}

	if (!seed.missionTitle?.trim()) warnings.push(`[${where}] Empty mission title.`);
	if (!seed.goal?.trim()) warnings.push(`[${where}] Empty goal.`);

	// Every seed must carry at least one bilingual support snippet.
	if (!seed.teach.malayalamSupport?.trim() && !seed.teach.manglishSupport?.trim()) {
	warnings.push(`[${where}] Missing both Malayalam and Manglish support.`);
	}

	// Visual board must have frames.
	if ((seed.visualBoard?.frames?.length ?? 0) === 0) {
	warnings.push(`[${where}] Visual board has no frames.`);
	}

	// needsHumanReview must explain itself.
	if (seed.qualityFlags.needsHumanReview && !seed.qualityFlags.reason?.trim()) {
	warnings.push(`[${where}] needsHumanReview is set without a reason.`);
	}

	// Generic/placeholder wording must not survive into a curated seed.
	const text = JSON.stringify(seed).toLowerCase();
	for (const phrase of GENERIC_PHRASES) {
	if (text.includes(phrase)) warnings.push(`[${where}] Contains generic/placeholder wording: "${phrase}".`);
	}

	return warnings;
	}

	function chapterCoverageWarnings(coverage: PhysicsQaCoverage): string[] {
	const warnings: string[] = [];
	if (!coverage.hasConcept) warnings.push("No concept mission.");
	if (!coverage.hasExamPractice) warnings.push("No exam-answer mission.");
	if (!coverage.hasRevision) warnings.push("No revision mission.");
	if (!coverage.hasVisualBoard) warnings.push("No visual-board-supported mission.");
	if (!coverage.hasMalayalamSupport) warnings.push("Some seed is missing Malayalam support.");
	if (!coverage.hasManglishSupport) warnings.push("Some seed is missing Manglish support.");
	return warnings;
	}

	export function buildPhysicsQaReport(): PhysicsQaReport {
	// Day 20 — the seed pool now includes the first Chemistry draft; this
	// report is the PHYSICS exam-readiness QA, so it scopes to Physics seeds.
	const seeds = getAllLessonSeeds().filter((seed) => seed.subject === "Physics");
	const byChapter = new Map<string, TuitionLessonSeed[]>();
	for (const seed of seeds) {
	byChapter.set(seed.chapterId, [...(byChapter.get(seed.chapterId) ?? []), seed]);
	}

	const blockers: string[] = [];
	const chapters: PhysicsQaChapter[] = Array.from(byChapter.entries()).map(([chapterId, chapterSeeds]) => {
	const coverage = coverageFor(chapterSeeds);
	const seedWarnings = chapterSeeds.flatMap(seedQaWarnings);
	const coverageWarnings = chapterCoverageWarnings(coverage);
	const warnings = [...coverageWarnings, ...seedWarnings];
	const needsHumanReviewSeeds = chapterSeeds
	.filter((s) => s.qualityFlags.needsHumanReview)
	.map((s) => s.missionId);

	// Blockers: anything that makes the chapter unsafe to sell.
	if (chapterSeeds.length === 0) blockers.push(`${chapterId}: chapter has zero seeds.`);
	for (const w of coverageWarnings) {
	if (!w.startsWith("Some seed is missing")) blockers.push(`${chapterId}: ${w}`);
	else blockers.push(`${chapterId}: ${w}`);
	}
	// Any hard validator error is a blocker (source refs, 11 steps, fake claims, curated PYQ, etc.).
	for (const seed of chapterSeeds) {
	const errors = validateLessonSeed(seed).errors;
	if (errors.length > 0) blockers.push(`${chapterId}/${seed.missionId}: ${errors.length} validation error(s).`);
	if (seed.qualityFlags.needsHumanReview && !seed.qualityFlags.reason?.trim()) {
	blockers.push(`${chapterId}/${seed.missionId}: needsHumanReview without a reason.`);
	}
	}

	return {
	chapterId,
	chapterTitle: chapterSeeds[0]?.chapterTitle ?? chapterId,
	seedCount: chapterSeeds.length,
	coverage,
	warnings,
	needsHumanReviewSeeds,
	};
	});

	const globalWarnings: string[] = [];
	if (byChapter.size !== 7) globalWarnings.push(`Expected 7 curated Physics chapters, found ${byChapter.size}.`);
	const qualitativeNoNumerical = chapters.filter((c) => !c.coverage.hasNumerical).map((c) => c.chapterId);
	if (qualitativeNoNumerical.length) {
	globalWarnings.push(
	`Chapters without a numerical mission (qualitative — informational, not a blocker): ${qualitativeNoNumerical.join(", ")}.`,
	);
	}

	const reviewFlagged = chapters.flatMap((c) => c.needsHumanReviewSeeds.map((m) => `${c.chapterId}/${m}`));

	const uniqueBlockers = Array.from(new Set(blockers));
	let status: PhysicsQaReport["productionReadiness"]["status"];
	if (uniqueBlockers.length > 0) {
	status = "not_ready";
	} else if (reviewFlagged.length > 0) {
	status = "ready_with_review_flags";
	} else {
	status = "ready";
	}

	const recommendedFixes: string[] = [];
	for (const flagged of reviewFlagged) {
	recommendedFixes.push(`Verify ${flagged} against the SCERT textbook, then clear its needsHumanReview flag.`);
	}
	if (uniqueBlockers.length > 0) {
	recommendedFixes.push("Resolve all blockers above before selling Physics as a paid product.");
	}

	return {
	totalChapters: byChapter.size,
	totalSeeds: seeds.length,
	chapters,
	globalWarnings,
	productionReadiness: {
	status,
	blockers: uniqueBlockers,
	recommendedFixes,
	},
	};
	}