Download src/utils/utils_quiz_quality.py from QuizGenCS/QuizGen: direct link, hf CLI and curl.
- Browser
- Download file 17 kB
-
https://huggingface.co/spaces/QuizGenCS/QuizGen/resolve/main/src/utils/utils_quiz_quality.py
- Command line
-
hf download hf://spaces/QuizGenCS/QuizGen/src/utils/utils_quiz_quality.py
-
curl -L -o utils_quiz_quality.py https://huggingface.co/spaces/QuizGenCS/QuizGen/resolve/main/src/utils/utils_quiz_quality.py
17 kB
| # utils_quiz_quality.py (EN-only, SMART TF-IDF version, returns exactly n questions) | |
| from __future__ import annotations | |
| import os, re, random, math | |
| from dataclasses import dataclass | |
| from typing import List, Dict, Tuple, Optional | |
| import numpy as np | |
| from sklearn.feature_extraction.text import TfidfVectorizer | |
| from sklearn.metrics.pairwise import cosine_similarity | |
| # --------------------------- | |
| # Reproducibility | |
| # --------------------------- | |
| def set_all_seeds(seed: int = 42): | |
| import numpy as _np, random as _random | |
| os.environ["PYTHONHASHSEED"] = str(seed) | |
| _random.seed(seed); _np.random.seed(seed) | |
| # --------------------------- | |
| # Small utils | |
| # --------------------------- | |
| _ws = re.compile(r"\s+") | |
| _tok = re.compile(r"[A-Za-z0-9\-]+") | |
| def norm(s: str) -> str: | |
| return _ws.sub(" ", s.strip().lower()) | |
| def too_long(s: str, max_chars: int) -> bool: | |
| return len(s) > max_chars | |
| def leaks_answer(q: str, a: str) -> bool: | |
| qn, an = norm(q), norm(a) | |
| if not a: | |
| return False | |
| if an and an in qn: | |
| return True | |
| toks = [t for t in _tok.findall(an) if len(t) >= 3] | |
| if not toks: | |
| return False | |
| overlap = sum(t in qn for t in toks) | |
| return overlap >= max(1, math.ceil(len(toks) * 0.6)) | |
| def dedup_keep_order(items: List[str]) -> List[str]: | |
| seen, out = set(), [] | |
| for x in items: | |
| k = norm(x) | |
| if k and k not in seen: | |
| seen.add(k); out.append(x) | |
| return out | |
| # --------------------------- | |
| # SMART Candidate Extraction | |
| # --------------------------- | |
| # 🔧 upgraded version: noun phrase extractor (fallbacks to regex) | |
| try: | |
| import spacy | |
| _nlp = spacy.load("en_core_web_sm", disable=["ner", "parser"]) | |
| except Exception: | |
| _nlp = None | |
| def extract_terms_en(txt: str, min_token_len=3, max_words=4) -> List[str]: | |
| """Smart English term extractor using spaCy noun chunks fallback to regex.""" | |
| if _nlp: | |
| doc = _nlp(txt) | |
| cands = set() | |
| for chunk in doc.noun_chunks: | |
| phrase = chunk.text.strip().lower() | |
| if 1 <= len(phrase.split()) <= max_words and len(phrase) >= min_token_len: | |
| cands.add(phrase) | |
| if cands: | |
| return list(cands)[:500] | |
| # fallback: regex bi-gram extractor | |
| words = [w for w in _tok.findall(txt) if len(w) >= min_token_len] | |
| if not words: | |
| return [] | |
| cands = set() | |
| for n in (1, 2): | |
| for i in range(len(words) - n + 1): | |
| phrase = " ".join(words[i:i+n]) | |
| if 1 <= len(phrase.split()) <= max_words: | |
| cands.add(phrase) | |
| stop = { | |
| "and","or","is","are","be","of","in","the","a","an","to","for","on","by", | |
| "with","as","at","from","that","this","it","its" | |
| } | |
| out = [p for p in cands if norm(p) not in stop and len(p) >= 3] | |
| return out[:500] | |
| # --------------------------- | |
| # TF-IDF index | |
| # --------------------------- | |
| class TFIDFIndex: | |
| vectorizer: TfidfVectorizer | |
| doc_mat: np.ndarray | |
| candidates: List[str] | |
| cand_mat: np.ndarray | |
| def _make_vectorizer() -> TfidfVectorizer: | |
| return TfidfVectorizer( | |
| analyzer="word", | |
| ngram_range=(1, 2), | |
| stop_words="english", | |
| min_df=1 | |
| ) | |
| def build_tfidf_index(slides: List[str]) -> TFIDFIndex: | |
| docs = [s for s in slides if s and s.strip()] | |
| if not docs: | |
| docs = ["(empty)"] | |
| if len(docs) < 2: | |
| docs = docs + docs | |
| corpus = [norm(s) for s in docs] | |
| vec = _make_vectorizer() | |
| doc_mat = vec.fit_transform(corpus) | |
| all_cands: List[str] = [] | |
| for s in docs: | |
| all_cands.extend(extract_terms_en(s)) | |
| candidates = dedup_keep_order(all_cands)[:3000] | |
| if not candidates: | |
| candidates = list(vec.vocabulary_.keys())[:200] | |
| cand_mat = vec.transform([norm(c) for c in candidates]) if candidates else doc_mat[:0] | |
| return TFIDFIndex(vec, doc_mat, candidates, cand_mat) | |
| # --------------------------- | |
| # Difficulty | |
| # --------------------------- | |
| DIFF = { | |
| "easy": {"n_options": 4, "sim_low": 0.02, "sim_high": 0.35}, | |
| "medium": {"n_options": 4, "sim_low": 0.06, "sim_high": 0.55}, | |
| "hard": {"n_options": 5, "sim_low": 0.18, "sim_high": 0.85}, | |
| } | |
| def clamp(level: str) -> str: | |
| return level if level in DIFF else "medium" | |
| # --------------------------- | |
| # SMART Similarity helpers | |
| # --------------------------- | |
| def word_overlap(a: str, b: str) -> float: | |
| sa, sb = set(a.split()), set(b.split()) | |
| if not sa or not sb: | |
| return 0.0 | |
| return len(sa & sb) / len(sa | sb) | |
| def weighted_similarity(a_vec, b_vec, a: str, b: str) -> float: | |
| sim_tfidf = cosine_similarity(a_vec, b_vec)[0, 0] | |
| sim_overlap = word_overlap(a, b) | |
| return 0.7 * sim_tfidf + 0.3 * sim_overlap | |
| # --------------------------- | |
| # Core selection (SMART) | |
| # --------------------------- | |
| def pick_answer(slide: str, tfidf: TFIDFIndex) -> Optional[str]: | |
| """Selects a key concept using weighted score (TF-IDF + length + position).""" | |
| if not tfidf.candidates: | |
| return None | |
| slide_vec = tfidf.vectorizer.transform([norm(slide)]) | |
| sims = cosine_similarity(slide_vec, tfidf.cand_mat)[0] | |
| if sims.size == 0: | |
| return None | |
| scores = [] | |
| for i, term in enumerate(tfidf.candidates): | |
| length_score = min(len(term) / 15, 1.0) | |
| pos_bonus = 0.2 if slide.lower().find(term.lower()) < len(slide) / 2 else 0.0 | |
| total = sims[i] * 0.7 + length_score * 0.2 + pos_bonus * 0.1 | |
| scores.append(total) | |
| best_idx = int(np.argmax(scores)) | |
| return tfidf.candidates[best_idx] | |
| def make_distractors(ans: str, tfidf: TFIDFIndex, k: int, lo: float, hi: float) -> List[str]: | |
| """Generate distractors using weighted similarity.""" | |
| if not ans or not tfidf.candidates: | |
| return [] | |
| ans_vec = tfidf.vectorizer.transform([norm(ans)]) | |
| out: List[str] = [] | |
| ansn = norm(ans) | |
| for cand in tfidf.candidates: | |
| if norm(cand) == ansn: | |
| continue | |
| c_vec = tfidf.vectorizer.transform([norm(cand)]) | |
| sim = weighted_similarity(ans_vec, c_vec, norm(ans), norm(cand)) | |
| if lo <= sim <= hi: | |
| out.append(cand) | |
| if len(out) >= k: | |
| break | |
| if len(out) < k: | |
| pool = [c for c in tfidf.candidates if norm(c) != ansn] | |
| random.shuffle(pool) | |
| out.extend(pool[:k - len(out)]) | |
| return dedup_keep_order(out)[:k] | |
| # --------------------------- | |
| # Question generator | |
| # --------------------------- | |
| def gen_question(slide: str, tfidf: TFIDFIndex, level="medium") -> Dict: | |
| level = clamp(level) | |
| ans = pick_answer(slide, tfidf) | |
| if not ans: | |
| return {} | |
| spec = DIFF[level] | |
| need = spec["n_options"] - 1 | |
| distracts = make_distractors(ans, tfidf, k=max(need + 2, 4), lo=spec["sim_low"], hi=spec["sim_high"]) | |
| if len(distracts) < need: | |
| pool = extract_terms_en(slide) | |
| random.shuffle(pool) | |
| for p in pool: | |
| if norm(p) != norm(ans) and p not in distracts: | |
| distracts.append(p) | |
| if len(distracts) >= need: | |
| break | |
| options = [ans] + distracts[:need] | |
| options = dedup_keep_order(options) | |
| random.shuffle(options) | |
| stem = { | |
| "easy": f"In this slide, which option best matches “{ans}”?", | |
| "medium": f"From the slide, which statement correctly relates to “{ans}”?", | |
| "hard": f"Which option most accurately explains “{ans}” in this slide's context?" | |
| }[level] | |
| return {"question": stem, "options": options, "answer": ans, "meta": {"level": level}} | |
| # --------------------------- | |
| # QC | |
| # --------------------------- | |
| def qc(q: Dict, relax: bool=False) -> Tuple[bool, str]: | |
| if not q or "question" not in q or "options" not in q or "answer" not in q: | |
| return False, "invalid" | |
| stem, opts, ans = q["question"], q["options"], q["answer"] | |
| if not relax: | |
| if too_long(stem, 180): | |
| return False, "q-too-long" | |
| if any(too_long(o, 120) for o in opts): | |
| return False, "opt-too-long" | |
| if leaks_answer(stem, ans): | |
| return False, "leak" | |
| if len(set(map(norm, opts))) != len(opts): | |
| return False, "dup-options" | |
| if all(norm(ans) != norm(o) for o in opts): | |
| return False, "ans-not-in-opts" | |
| return True, "ok" | |
| def qc_set(qq: List[Dict]) -> None: | |
| """Check overall diversity of generated questions.""" | |
| stems = [norm(q.get("question", "")) for q in qq] | |
| if len(set(stems)) < len(stems) * 0.8: | |
| print("⚠️ Warning: many similar question stems detected") | |
| def dedup_questions(qq: List[Dict]) -> List[Dict]: | |
| seen, out = set(), [] | |
| for q in qq: | |
| k = norm(q.get("question", "")) + "|" + norm(q.get("answer", "")) | |
| if k and k not in seen: | |
| seen.add(k) | |
| out.append(q) | |
| return out | |
| # --------------------------- | |
| # Fallback builders (force fill) | |
| # --------------------------- | |
| def _make_fallback_question(slide_text: str, tfidf: TFIDFIndex, level: str) -> Dict: | |
| pool_terms = extract_terms_en(slide_text) | |
| if not pool_terms: | |
| pool_terms = list(tfidf.candidates) if tfidf.candidates else ["topic", "concept", "method", "result", "analysis"] | |
| random.shuffle(pool_terms) | |
| answer = pool_terms[0] | |
| distracts = make_distractors(answer, tfidf, k=4, lo=0.02, hi=0.85) | |
| while len(distracts) < 3 and len(pool_terms) > 1: | |
| cand = pool_terms.pop() | |
| if norm(cand) != norm(answer) and cand not in distracts: | |
| distracts.append(cand) | |
| if len(distracts) < 3 and tfidf.candidates: | |
| extra = [c for c in tfidf.candidates if norm(c) != norm(answer)] | |
| random.shuffle(extra) | |
| distracts.extend(extra[:3 - len(distracts)]) | |
| options = [answer] + distracts[:3] | |
| options = dedup_keep_order(options) | |
| while len(options) < 4: | |
| options.append(f"option-{len(options)+1}") | |
| random.shuffle(options) | |
| stem = { | |
| "easy": f"In this slide, which option best matches “{answer}”?", | |
| "medium": f"From the slide, which statement correctly relates to “{answer}”?", | |
| "hard": f"Which option most accurately explains “{answer}” in this slide's context?" | |
| }[clamp(level)] | |
| return {"question": stem, "options": options, "answer": answer, "meta": {"level": clamp(level)}, "fallback": True} | |
| def _force_fill_to_n(slides: List[str], tfidf: TFIDFIndex, existing: List[Dict], n_questions: int, level: str) -> List[Dict]: | |
| out = list(existing) | |
| i = 0 | |
| seen = set(norm(q["question"]) + "|" + norm(q["answer"]) for q in out) | |
| while len(out) < n_questions and slides: | |
| s = slides[i % len(slides)] | |
| fb = _make_fallback_question(s, tfidf, level) | |
| key_new = norm(fb["question"]) + "|" + norm(fb["answer"]) | |
| if key_new not in seen: | |
| out.append(fb) | |
| seen.add(key_new) | |
| i += 1 | |
| if i > n_questions * 3: | |
| break | |
| return out[:n_questions] | |
| # --------------------------- | |
| # Distribution | |
| # --------------------------- | |
| def distribute(buckets: List[List[Dict]], total: int) -> List[Dict]: | |
| bs = [list(x) for x in buckets if x] | |
| out, i = [], 0 | |
| while len(out) < total and bs: | |
| b = bs[i % len(bs)] | |
| if b: | |
| out.append(b.pop(0)) | |
| i += 1 | |
| else: | |
| bs.pop(i % len(bs) if bs else 0) | |
| return out | |
| # --------------------------- | |
| # Apply question types (MCQ / CLOZE / T/F) | |
| # --------------------------- | |
| def _apply_qtypes( | |
| qq: List[Dict], | |
| q_types: Optional[List[str]], | |
| seed: int, | |
| ) -> List[Dict]: | |
| if not qq: | |
| return qq | |
| if not q_types: | |
| q_types = ["MCQ"] | |
| # normalize | |
| norm_types = [t.upper() for t in q_types if t] | |
| if not norm_types: | |
| norm_types = ["MCQ"] | |
| rng = random.Random(seed) | |
| out: List[Dict] = [] | |
| for i, q in enumerate(qq): | |
| stem = str(q.get("question", "")) | |
| ans = str(q.get("answer", "")) | |
| opts = [ | |
| str(o) | |
| for o in q.get("options", []) | |
| if isinstance(o, (str, int, float)) and str(o).strip() | |
| ] | |
| # ensure answer in options | |
| if ans and all(norm(ans) != norm(o) for o in opts): | |
| opts.insert(0, ans) | |
| if not opts and ans: | |
| opts = [ans] | |
| while len(opts) < 4: | |
| opts.append(f"option-{len(opts)+1}") | |
| opts = dedup_keep_order(opts)[:4] | |
| # เลือก type | |
| target_type = norm_types[i % len(norm_types)] | |
| # ---- CLOZE ---- | |
| if target_type == "CLOZE": | |
| if ans: | |
| pattern = re.compile(re.escape(ans), re.IGNORECASE) | |
| new_stem, nsub = pattern.subn("____", stem, count=1) | |
| if nsub > 0: | |
| stem_cloze = new_stem | |
| else: | |
| stem_cloze = f"____ {stem}" | |
| else: | |
| stem_cloze = stem | |
| out.append({ | |
| "type": "CLOZE", | |
| "question": stem_cloze, | |
| "options": opts, | |
| "answer": ans, | |
| "explanation": "", | |
| "meta": q.get("meta", {}), | |
| }) | |
| # ---- T/F ---- | |
| elif target_type in ("T/F", "TF", "TRUE/FALSE"): | |
| # ทำ statement จริง/เท็จจาก answer / distractor | |
| wrong = next( | |
| (o for o in opts if ans and norm(o) != norm(ans)), | |
| None, | |
| ) | |
| make_true = (i % 2 == 0 or not wrong) | |
| if make_true: | |
| statement = f"\"{ans}\" ปรากฏเป็นคำสำคัญในสไลด์นี้" | |
| tf_ans = "True" | |
| expl = "" | |
| else: | |
| statement = f"\"{wrong}\" ปรากฏเป็นคำสำคัญในสไลด์นี้" | |
| tf_ans = "False" | |
| expl = f"คำตอบที่ถูกคือ {ans}" | |
| out.append({ | |
| "type": "T/F", | |
| "question": statement, | |
| "options": ["True", "False"], | |
| "answer": tf_ans, | |
| "explanation": expl, | |
| "meta": q.get("meta", {}), | |
| }) | |
| # ---- MCQ ---- | |
| else: | |
| out.append({ | |
| "type": "MCQ", | |
| "question": stem, | |
| "options": opts, | |
| "answer": ans, | |
| "explanation": "", | |
| "meta": q.get("meta", {}), | |
| }) | |
| return out | |
| # --------------------------- | |
| # Public API (force-complete) | |
| # --------------------------- | |
| def generate_quiz( | |
| slides: List[str], | |
| n_questions=12, | |
| level="medium", | |
| seed=42, | |
| q_types: Optional[List[str]] = None, | |
| ) -> Dict: | |
| set_all_seeds(seed) | |
| slides = [s for s in slides if s and s.strip()] | |
| if not slides: | |
| slides = ["(empty)"] | |
| tfidf = build_tfidf_index(slides) | |
| # Dynamic attempts based on slide length | |
| draft_by_slide: List[List[Dict]] = [] | |
| for s in slides: | |
| ATTEMPTS_PER_SLIDE = min(10, max(3, len(s.split()) // 40)) | |
| local: List[Dict] = [] | |
| for _ in range(ATTEMPTS_PER_SLIDE): | |
| q = gen_question(s, tfidf, level=level) | |
| if not q: | |
| continue | |
| ok, _ = qc(q, relax=False) | |
| if ok: | |
| local.append(q) | |
| if not local: | |
| q = gen_question(s, tfidf, level=level) | |
| if q and qc(q, relax=True)[0]: | |
| local.append(q) | |
| draft_by_slide.append(dedup_questions(local)) | |
| picked = distribute(draft_by_slide, total=n_questions) | |
| if len(picked) < n_questions: | |
| pool = [q for b in draft_by_slide for q in b] | |
| exist_ids = set(id(q) for q in picked) | |
| extras = [q for q in pool if id(q) not in exist_ids] | |
| needed = n_questions - len(picked) | |
| picked.extend(extras[:needed]) | |
| final: List[Dict] = [] | |
| for q in picked: | |
| ok, _ = qc(q, relax=False) | |
| if ok: | |
| final.append(q) | |
| final = dedup_questions(final) | |
| if len(final) < n_questions: | |
| relax_pool = [q for q in picked if qc(q, relax=True)[0]] | |
| exist = set(norm(x["question"]) + "|" + norm(x["answer"]) for x in final) | |
| relax_pool = [q for q in relax_pool if (norm(q["question"]) + "|" + norm(q["answer"])) not in exist] | |
| need = n_questions - len(final) | |
| final.extend(relax_pool[:need]) | |
| if len(final) < n_questions: | |
| final = _force_fill_to_n(slides, tfidf, final, n_questions, level) | |
| qc_set(final) | |
| # apply question types (MCQ / CLOZE / T/F) | |
| typed_questions = _apply_qtypes(final[:n_questions], q_types, seed) | |
| total_raw = sum(len(x) for x in draft_by_slide) | |
| return { | |
| "questions": typed_questions, | |
| "qc_report": { | |
| "slides": len(slides), | |
| "draft_generated": total_raw, | |
| "selected": len(typed_questions), | |
| "level": clamp(level), | |
| "seed": seed, | |
| "forced_fill_used": len(final) < n_questions | |
| } | |
| } | |