Spaces:
Sleeping
Sleeping
File size: 2,188 Bytes
d544dca e0ddbe9 d544dca e0ddbe9 d544dca | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 | """
Difficulty scorer: combines
- semantic distance between claim and context (hard if very distant)
- answer length entropy (hard if longer numerical / multi-step)
- distractor similarity (hard if distractors almost correct)
- curriculum mapping (maps grade level → easy / medium / hard)
"""
from __future__ import annotations
import re
import numpy as np
from .config import Settings
class DifficultyScorer:
def __init__(self, settings: Settings, dense_encoder):
self.s = settings
self.dense = dense_encoder
def score(
self,
question: str,
answer: str,
distractors: list[str],
context: str,
) -> str:
ctx_emb = self.dense.encode([context], normalize_embeddings=True)
ans_emb = self.dense.encode([answer], normalize_embeddings=True)
q_emb = self.dense.encode([question], normalize_embeddings=True)
# semantic distance between question and context (hard if very distant)
dist_q_ctx = float(np.dot(q_emb[0], ctx_emb[0]))
distractor_scores = []
for d in distractors:
d_emb = self.dense.encode([d], normalize_embeddings=True)
distractor_scores.append(float(np.dot(ans_emb[0], d_emb[0])))
avg_distractor_sim = float(np.mean(distractor_scores)) if distractor_scores else 0.0
answer_words = len(answer.split())
numeric_tokens = len(re.findall(r"\d[\d.,]*", answer))
# Heuristic score (0 → easy, 1 → hard)
score = 0.0
if dist_q_ctx < self.s.difficulty_easy_threshold:
score += 0.2
elif dist_q_ctx > self.s.difficulty_hard_threshold:
score += 0.6
else:
score += 0.4
# distractors: if >0.75 overlap → hard
if avg_distractor_sim > 0.75:
score += 0.3
elif avg_distractor_sim > 0.55:
score += 0.15
# complex answers bump difficulty
if numeric_tokens > 1 or answer_words > 12:
score += 0.2
score = min(1.0, score)
if score > 0.7:
return "hard"
elif score > 0.4:
return "medium"
return "easy"
|