QuizGen / src /utils /utils_quiz_quality.py
pluem9
fixminilmTFIDF
3d94eac
Raw History Blame Contribute Delete
17 kB
# utils_quiz_quality.py (EN-only, SMART TF-IDF version, returns exactly n questions)
from __future__ import annotations
import os, re, random, math
from dataclasses import dataclass
from typing import List, Dict, Tuple, Optional
import numpy as np
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity
# ---------------------------
# Reproducibility
# ---------------------------
def set_all_seeds(seed: int = 42):
import numpy as _np, random as _random
os.environ["PYTHONHASHSEED"] = str(seed)
_random.seed(seed); _np.random.seed(seed)
# ---------------------------
# Small utils
# ---------------------------
_ws = re.compile(r"\s+")
_tok = re.compile(r"[A-Za-z0-9\-]+")
def norm(s: str) -> str:
return _ws.sub(" ", s.strip().lower())
def too_long(s: str, max_chars: int) -> bool:
return len(s) > max_chars
def leaks_answer(q: str, a: str) -> bool:
qn, an = norm(q), norm(a)
if not a:
return False
if an and an in qn:
return True
toks = [t for t in _tok.findall(an) if len(t) >= 3]
if not toks:
return False
overlap = sum(t in qn for t in toks)
return overlap >= max(1, math.ceil(len(toks) * 0.6))
def dedup_keep_order(items: List[str]) -> List[str]:
seen, out = set(), []
for x in items:
k = norm(x)
if k and k not in seen:
seen.add(k); out.append(x)
return out
# ---------------------------
# SMART Candidate Extraction
# ---------------------------
# 🔧 upgraded version: noun phrase extractor (fallbacks to regex)
try:
import spacy
_nlp = spacy.load("en_core_web_sm", disable=["ner", "parser"])
except Exception:
_nlp = None
def extract_terms_en(txt: str, min_token_len=3, max_words=4) -> List[str]:
"""Smart English term extractor using spaCy noun chunks fallback to regex."""
if _nlp:
doc = _nlp(txt)
cands = set()
for chunk in doc.noun_chunks:
phrase = chunk.text.strip().lower()
if 1 <= len(phrase.split()) <= max_words and len(phrase) >= min_token_len:
cands.add(phrase)
if cands:
return list(cands)[:500]
# fallback: regex bi-gram extractor
words = [w for w in _tok.findall(txt) if len(w) >= min_token_len]
if not words:
return []
cands = set()
for n in (1, 2):
for i in range(len(words) - n + 1):
phrase = " ".join(words[i:i+n])
if 1 <= len(phrase.split()) <= max_words:
cands.add(phrase)
stop = {
"and","or","is","are","be","of","in","the","a","an","to","for","on","by",
"with","as","at","from","that","this","it","its"
}
out = [p for p in cands if norm(p) not in stop and len(p) >= 3]
return out[:500]
# ---------------------------
# TF-IDF index
# ---------------------------
@dataclass
class TFIDFIndex:
vectorizer: TfidfVectorizer
doc_mat: np.ndarray
candidates: List[str]
cand_mat: np.ndarray
def _make_vectorizer() -> TfidfVectorizer:
return TfidfVectorizer(
analyzer="word",
ngram_range=(1, 2),
stop_words="english",
min_df=1
)
def build_tfidf_index(slides: List[str]) -> TFIDFIndex:
docs = [s for s in slides if s and s.strip()]
if not docs:
docs = ["(empty)"]
if len(docs) < 2:
docs = docs + docs
corpus = [norm(s) for s in docs]
vec = _make_vectorizer()
doc_mat = vec.fit_transform(corpus)
all_cands: List[str] = []
for s in docs:
all_cands.extend(extract_terms_en(s))
candidates = dedup_keep_order(all_cands)[:3000]
if not candidates:
candidates = list(vec.vocabulary_.keys())[:200]
cand_mat = vec.transform([norm(c) for c in candidates]) if candidates else doc_mat[:0]
return TFIDFIndex(vec, doc_mat, candidates, cand_mat)
# ---------------------------
# Difficulty
# ---------------------------
DIFF = {
"easy": {"n_options": 4, "sim_low": 0.02, "sim_high": 0.35},
"medium": {"n_options": 4, "sim_low": 0.06, "sim_high": 0.55},
"hard": {"n_options": 5, "sim_low": 0.18, "sim_high": 0.85},
}
def clamp(level: str) -> str:
return level if level in DIFF else "medium"
# ---------------------------
# SMART Similarity helpers
# ---------------------------
def word_overlap(a: str, b: str) -> float:
sa, sb = set(a.split()), set(b.split())
if not sa or not sb:
return 0.0
return len(sa & sb) / len(sa | sb)
def weighted_similarity(a_vec, b_vec, a: str, b: str) -> float:
sim_tfidf = cosine_similarity(a_vec, b_vec)[0, 0]
sim_overlap = word_overlap(a, b)
return 0.7 * sim_tfidf + 0.3 * sim_overlap
# ---------------------------
# Core selection (SMART)
# ---------------------------
def pick_answer(slide: str, tfidf: TFIDFIndex) -> Optional[str]:
"""Selects a key concept using weighted score (TF-IDF + length + position)."""
if not tfidf.candidates:
return None
slide_vec = tfidf.vectorizer.transform([norm(slide)])
sims = cosine_similarity(slide_vec, tfidf.cand_mat)[0]
if sims.size == 0:
return None
scores = []
for i, term in enumerate(tfidf.candidates):
length_score = min(len(term) / 15, 1.0)
pos_bonus = 0.2 if slide.lower().find(term.lower()) < len(slide) / 2 else 0.0
total = sims[i] * 0.7 + length_score * 0.2 + pos_bonus * 0.1
scores.append(total)
best_idx = int(np.argmax(scores))
return tfidf.candidates[best_idx]
def make_distractors(ans: str, tfidf: TFIDFIndex, k: int, lo: float, hi: float) -> List[str]:
"""Generate distractors using weighted similarity."""
if not ans or not tfidf.candidates:
return []
ans_vec = tfidf.vectorizer.transform([norm(ans)])
out: List[str] = []
ansn = norm(ans)
for cand in tfidf.candidates:
if norm(cand) == ansn:
continue
c_vec = tfidf.vectorizer.transform([norm(cand)])
sim = weighted_similarity(ans_vec, c_vec, norm(ans), norm(cand))
if lo <= sim <= hi:
out.append(cand)
if len(out) >= k:
break
if len(out) < k:
pool = [c for c in tfidf.candidates if norm(c) != ansn]
random.shuffle(pool)
out.extend(pool[:k - len(out)])
return dedup_keep_order(out)[:k]
# ---------------------------
# Question generator
# ---------------------------
def gen_question(slide: str, tfidf: TFIDFIndex, level="medium") -> Dict:
level = clamp(level)
ans = pick_answer(slide, tfidf)
if not ans:
return {}
spec = DIFF[level]
need = spec["n_options"] - 1
distracts = make_distractors(ans, tfidf, k=max(need + 2, 4), lo=spec["sim_low"], hi=spec["sim_high"])
if len(distracts) < need:
pool = extract_terms_en(slide)
random.shuffle(pool)
for p in pool:
if norm(p) != norm(ans) and p not in distracts:
distracts.append(p)
if len(distracts) >= need:
break
options = [ans] + distracts[:need]
options = dedup_keep_order(options)
random.shuffle(options)
stem = {
"easy": f"In this slide, which option best matches “{ans}”?",
"medium": f"From the slide, which statement correctly relates to “{ans}”?",
"hard": f"Which option most accurately explains “{ans}” in this slide's context?"
}[level]
return {"question": stem, "options": options, "answer": ans, "meta": {"level": level}}
# ---------------------------
# QC
# ---------------------------
def qc(q: Dict, relax: bool=False) -> Tuple[bool, str]:
if not q or "question" not in q or "options" not in q or "answer" not in q:
return False, "invalid"
stem, opts, ans = q["question"], q["options"], q["answer"]
if not relax:
if too_long(stem, 180):
return False, "q-too-long"
if any(too_long(o, 120) for o in opts):
return False, "opt-too-long"
if leaks_answer(stem, ans):
return False, "leak"
if len(set(map(norm, opts))) != len(opts):
return False, "dup-options"
if all(norm(ans) != norm(o) for o in opts):
return False, "ans-not-in-opts"
return True, "ok"
def qc_set(qq: List[Dict]) -> None:
"""Check overall diversity of generated questions."""
stems = [norm(q.get("question", "")) for q in qq]
if len(set(stems)) < len(stems) * 0.8:
print("⚠️ Warning: many similar question stems detected")
def dedup_questions(qq: List[Dict]) -> List[Dict]:
seen, out = set(), []
for q in qq:
k = norm(q.get("question", "")) + "|" + norm(q.get("answer", ""))
if k and k not in seen:
seen.add(k)
out.append(q)
return out
# ---------------------------
# Fallback builders (force fill)
# ---------------------------
def _make_fallback_question(slide_text: str, tfidf: TFIDFIndex, level: str) -> Dict:
pool_terms = extract_terms_en(slide_text)
if not pool_terms:
pool_terms = list(tfidf.candidates) if tfidf.candidates else ["topic", "concept", "method", "result", "analysis"]
random.shuffle(pool_terms)
answer = pool_terms[0]
distracts = make_distractors(answer, tfidf, k=4, lo=0.02, hi=0.85)
while len(distracts) < 3 and len(pool_terms) > 1:
cand = pool_terms.pop()
if norm(cand) != norm(answer) and cand not in distracts:
distracts.append(cand)
if len(distracts) < 3 and tfidf.candidates:
extra = [c for c in tfidf.candidates if norm(c) != norm(answer)]
random.shuffle(extra)
distracts.extend(extra[:3 - len(distracts)])
options = [answer] + distracts[:3]
options = dedup_keep_order(options)
while len(options) < 4:
options.append(f"option-{len(options)+1}")
random.shuffle(options)
stem = {
"easy": f"In this slide, which option best matches “{answer}”?",
"medium": f"From the slide, which statement correctly relates to “{answer}”?",
"hard": f"Which option most accurately explains “{answer}” in this slide's context?"
}[clamp(level)]
return {"question": stem, "options": options, "answer": answer, "meta": {"level": clamp(level)}, "fallback": True}
def _force_fill_to_n(slides: List[str], tfidf: TFIDFIndex, existing: List[Dict], n_questions: int, level: str) -> List[Dict]:
out = list(existing)
i = 0
seen = set(norm(q["question"]) + "|" + norm(q["answer"]) for q in out)
while len(out) < n_questions and slides:
s = slides[i % len(slides)]
fb = _make_fallback_question(s, tfidf, level)
key_new = norm(fb["question"]) + "|" + norm(fb["answer"])
if key_new not in seen:
out.append(fb)
seen.add(key_new)
i += 1
if i > n_questions * 3:
break
return out[:n_questions]
# ---------------------------
# Distribution
# ---------------------------
def distribute(buckets: List[List[Dict]], total: int) -> List[Dict]:
bs = [list(x) for x in buckets if x]
out, i = [], 0
while len(out) < total and bs:
b = bs[i % len(bs)]
if b:
out.append(b.pop(0))
i += 1
else:
bs.pop(i % len(bs) if bs else 0)
return out
# ---------------------------
# Apply question types (MCQ / CLOZE / T/F)
# ---------------------------
def _apply_qtypes(
qq: List[Dict],
q_types: Optional[List[str]],
seed: int,
) -> List[Dict]:
if not qq:
return qq
if not q_types:
q_types = ["MCQ"]
# normalize
norm_types = [t.upper() for t in q_types if t]
if not norm_types:
norm_types = ["MCQ"]
rng = random.Random(seed)
out: List[Dict] = []
for i, q in enumerate(qq):
stem = str(q.get("question", ""))
ans = str(q.get("answer", ""))
opts = [
str(o)
for o in q.get("options", [])
if isinstance(o, (str, int, float)) and str(o).strip()
]
# ensure answer in options
if ans and all(norm(ans) != norm(o) for o in opts):
opts.insert(0, ans)
if not opts and ans:
opts = [ans]
while len(opts) < 4:
opts.append(f"option-{len(opts)+1}")
opts = dedup_keep_order(opts)[:4]
# เลือก type
target_type = norm_types[i % len(norm_types)]
# ---- CLOZE ----
if target_type == "CLOZE":
if ans:
pattern = re.compile(re.escape(ans), re.IGNORECASE)
new_stem, nsub = pattern.subn("____", stem, count=1)
if nsub > 0:
stem_cloze = new_stem
else:
stem_cloze = f"____ {stem}"
else:
stem_cloze = stem
out.append({
"type": "CLOZE",
"question": stem_cloze,
"options": opts,
"answer": ans,
"explanation": "",
"meta": q.get("meta", {}),
})
# ---- T/F ----
elif target_type in ("T/F", "TF", "TRUE/FALSE"):
# ทำ statement จริง/เท็จจาก answer / distractor
wrong = next(
(o for o in opts if ans and norm(o) != norm(ans)),
None,
)
make_true = (i % 2 == 0 or not wrong)
if make_true:
statement = f"\"{ans}\" ปรากฏเป็นคำสำคัญในสไลด์นี้"
tf_ans = "True"
expl = ""
else:
statement = f"\"{wrong}\" ปรากฏเป็นคำสำคัญในสไลด์นี้"
tf_ans = "False"
expl = f"คำตอบที่ถูกคือ {ans}"
out.append({
"type": "T/F",
"question": statement,
"options": ["True", "False"],
"answer": tf_ans,
"explanation": expl,
"meta": q.get("meta", {}),
})
# ---- MCQ ----
else:
out.append({
"type": "MCQ",
"question": stem,
"options": opts,
"answer": ans,
"explanation": "",
"meta": q.get("meta", {}),
})
return out
# ---------------------------
# Public API (force-complete)
# ---------------------------
def generate_quiz(
slides: List[str],
n_questions=12,
level="medium",
seed=42,
q_types: Optional[List[str]] = None,
) -> Dict:
set_all_seeds(seed)
slides = [s for s in slides if s and s.strip()]
if not slides:
slides = ["(empty)"]
tfidf = build_tfidf_index(slides)
# Dynamic attempts based on slide length
draft_by_slide: List[List[Dict]] = []
for s in slides:
ATTEMPTS_PER_SLIDE = min(10, max(3, len(s.split()) // 40))
local: List[Dict] = []
for _ in range(ATTEMPTS_PER_SLIDE):
q = gen_question(s, tfidf, level=level)
if not q:
continue
ok, _ = qc(q, relax=False)
if ok:
local.append(q)
if not local:
q = gen_question(s, tfidf, level=level)
if q and qc(q, relax=True)[0]:
local.append(q)
draft_by_slide.append(dedup_questions(local))
picked = distribute(draft_by_slide, total=n_questions)
if len(picked) < n_questions:
pool = [q for b in draft_by_slide for q in b]
exist_ids = set(id(q) for q in picked)
extras = [q for q in pool if id(q) not in exist_ids]
needed = n_questions - len(picked)
picked.extend(extras[:needed])
final: List[Dict] = []
for q in picked:
ok, _ = qc(q, relax=False)
if ok:
final.append(q)
final = dedup_questions(final)
if len(final) < n_questions:
relax_pool = [q for q in picked if qc(q, relax=True)[0]]
exist = set(norm(x["question"]) + "|" + norm(x["answer"]) for x in final)
relax_pool = [q for q in relax_pool if (norm(q["question"]) + "|" + norm(q["answer"])) not in exist]
need = n_questions - len(final)
final.extend(relax_pool[:need])
if len(final) < n_questions:
final = _force_fill_to_n(slides, tfidf, final, n_questions, level)
qc_set(final)
# apply question types (MCQ / CLOZE / T/F)
typed_questions = _apply_qtypes(final[:n_questions], q_types, seed)
total_raw = sum(len(x) for x in draft_by_slide)
return {
"questions": typed_questions,
"qc_report": {
"slides": len(slides),
"draft_generated": total_raw,
"selected": len(typed_questions),
"level": clamp(level),
"seed": seed,
"forced_fill_used": len(final) < n_questions
}
}