Spaces:
Paused
Paused
| """Check that the configured provider actually produces valid output. | |
| An HTTP 200 proves nothing — llama2 7b/13b returned 200s full of wrong field | |
| names, inline comments, and fractional weights. Only a real generation that | |
| validates against the Pydantic schema counts as working. | |
| Run before any deploy, and again after changing OBE_LLM_MODEL: | |
| python scripts/verify_provider.py | |
| Exits non-zero if any generation fails. | |
| """ | |
| import logging | |
| import os | |
| import sys | |
| import time | |
| sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) | |
| from obe.course_plo_prompt import build_course_plo_messages | |
| from obe.llm_client import ( | |
| generate_course_plo_mapping, | |
| generate_obe_matrix, | |
| generate_question_set, | |
| get_provider, | |
| ) | |
| from obe.prompt import build_messages | |
| from obe.question_prompt import build_question_messages | |
| from obe.schema import BLOOM_LABELS | |
| logging.basicConfig(level=logging.INFO, format="%(levelname)s %(name)s: %(message)s") | |
| CHAPTER = """ | |
| Memory is the process by which information is encoded, stored, and retrieved. | |
| Encoding converts sensory input into a form the brain can process. Storage | |
| maintains that information over time, across sensory, short-term, and long-term | |
| stores. Retrieval brings stored information back into awareness. The multi-store | |
| model distinguishes these stages, while the levels-of-processing framework | |
| argues that depth of processing at encoding predicts later recall better than | |
| time spent in any one store. Interference and decay both explain forgetting. | |
| """ | |
| PLOS = [ | |
| "PLO-01: Apply core psychological theories to explain human behaviour.", | |
| "PLO-02: Evaluate research evidence critically.", | |
| ] | |
| def _check(label: str, fn) -> tuple[bool, float]: | |
| print(f"\n-- {label} " + "-" * (56 - len(label))) | |
| start = time.perf_counter() | |
| try: | |
| result = fn() | |
| except Exception as exc: | |
| elapsed = time.perf_counter() - start | |
| print(f" FAIL ({elapsed:.1f}s): {type(exc).__name__}: {exc}") | |
| return False, elapsed | |
| elapsed = time.perf_counter() - start | |
| print(f" OK ({elapsed:.1f}s)") | |
| return True, elapsed | |
| def _obe(): | |
| matrix = generate_obe_matrix(build_messages("Cognitive Psychology", PLOS, CHAPTER)) | |
| assert matrix.clos, "no CLOs generated" | |
| first = matrix.clos[0] | |
| weights = first.assessment_weights | |
| total = weights.quiz + weights.assignment + weights.midterm + weights.final | |
| print(f" {len(matrix.clos)} CLOs; first at Bloom {first.bloom_level} " | |
| f"({first.bloom_label}); weights sum to {total:g}") | |
| print(f" CLO: {first.clo_text[:80]}") | |
| return matrix | |
| def _questions(): | |
| qs = generate_question_set( | |
| build_question_messages( | |
| clo_text="Explain how encoding depth affects long-term recall.", | |
| content_text=CHAPTER, | |
| bloom_level=2, | |
| bloom_label=BLOOM_LABELS[2], | |
| difficulty="Medium", | |
| question_type="MCQ (single correct)", | |
| question_count=3, | |
| option_count=4, | |
| ) | |
| ) | |
| assert qs.questions, "no questions generated" | |
| first = qs.questions[0] | |
| assert len(first.options) == 4, f"expected 4 options, got {len(first.options)}" | |
| print(f" {len(qs.questions)} questions; {len(first.options)} options on the first") | |
| print(f" Q: {first.question_text[:80]}") | |
| return qs | |
| def _mapping(): | |
| result = generate_course_plo_mapping( | |
| build_course_plo_messages( | |
| program="BS Psychology", | |
| course_batch=[ | |
| { | |
| "Code": "PSY-101", | |
| "Course Title": "Introduction to Psychology", | |
| "Objectives": "Survey of major areas of psychology.", | |
| "Cr. Hrs.": "3", | |
| }, | |
| { | |
| "Code": "PSY-201", | |
| "Course Title": "Research Methods I", | |
| "Objectives": "Design and evaluation of psychological research.", | |
| "Cr. Hrs.": "3", | |
| }, | |
| ], | |
| plo_list=[ | |
| {"PLO Code": "PLO-01", "PLO Text": PLOS[0]}, | |
| {"PLO Code": "PLO-02", "PLO Text": PLOS[1]}, | |
| ], | |
| ) | |
| ) | |
| assert result.mappings, "no mappings generated" | |
| codes = [m.course_code for m in result.mappings] | |
| print(f" {len(result.mappings)} courses mapped: {', '.join(codes)}") | |
| return result | |
| def main() -> int: | |
| provider = get_provider() | |
| print(f"Provider : {provider.name}") | |
| print(f"Model : {provider.model}") | |
| print(f"Schema : {provider.schema_mode.value}") | |
| print(f"Timeout : {provider.timeout}s") | |
| if provider.schema_mode.value != "native": | |
| print("\n NOTE: no constrained decoding for this model — output is not") | |
| print(" schema-guaranteed. The runner will fall back and repair, but") | |
| print(" a strict-capable model is safer for production.") | |
| results = [ | |
| _check("OBEMatrix", _obe), | |
| _check("QuestionSet", _questions), | |
| _check("CoursePLOMappingSet", _mapping), | |
| ] | |
| passed = sum(1 for ok, _ in results if ok) | |
| total_time = sum(t for _, t in results) | |
| print("\n" + "=" * 60) | |
| print(f"{passed}/{len(results)} passed in {total_time:.1f}s") | |
| return 0 if passed == len(results) else 1 | |
| if __name__ == "__main__": | |
| sys.exit(main()) | |