multiq/difficulty.py

"""
Difficulty scorer: combines
- semantic distance between claim and context  (hard if very distant)
- answer length entropy  (hard if longer numerical / multi-step)
- distractor similarity  (hard if distractors almost correct)
- curriculum mapping     (maps grade level → easy / medium / hard)
"""
from __future__ import annotations

import re

import numpy as np

from .config import Settings


class DifficultyScorer:
    def __init__(self, settings: Settings, dense_encoder):
        self.s = settings
        self.dense = dense_encoder

    def score(
        self,
        question: str,
        answer: str,
        distractors: list[str],
        context: str,
    ) -> str:
        ctx_emb = self.dense.encode([context], normalize_embeddings=True)
        ans_emb = self.dense.encode([answer], normalize_embeddings=True)
        q_emb = self.dense.encode([question], normalize_embeddings=True)

        # semantic distance between question and context (hard if very distant)
        dist_q_ctx = float(np.dot(q_emb[0], ctx_emb[0]))

        distractor_scores = []
        for d in distractors:
            d_emb = self.dense.encode([d], normalize_embeddings=True)
            distractor_scores.append(float(np.dot(ans_emb[0], d_emb[0])))
        avg_distractor_sim = float(np.mean(distractor_scores)) if distractor_scores else 0.0

        answer_words = len(answer.split())
        numeric_tokens = len(re.findall(r"\d[\d.,]*", answer))

        # Heuristic score (0 → easy, 1 → hard)
        score = 0.0
        if dist_q_ctx < self.s.difficulty_easy_threshold:
            score += 0.2
        elif dist_q_ctx > self.s.difficulty_hard_threshold:
            score += 0.6
        else:
            score += 0.4

        # distractors: if >0.75 overlap → hard
        if avg_distractor_sim > 0.75:
            score += 0.3
        elif avg_distractor_sim > 0.55:
            score += 0.15

        # complex answers bump difficulty
        if numeric_tokens > 1 or answer_words > 12:
            score += 0.2

        score = min(1.0, score)

        if score > 0.7:
            return "hard"
        elif score > 0.4:
            return "medium"
        return "easy"