sdg/preprocessing/dedupe/test_minhash.py

"""Tests for MinHashDeduplicator. Skipped if datasketch is not installed."""

from __future__ import annotations

import pytest

pytest.importorskip("datasketch")

from sdg.preprocessing.dedupe.minhash import MinHashDeduplicator


# ── Boundary cases ───────────────────────────────────────────────────────────

def test_empty_input_returns_empty():
    assert MinHashDeduplicator().dedup([], show_progress=False) == []


def test_single_item_returns_single():
    assert MinHashDeduplicator().dedup(["only one"], show_progress=False) == [0]


# ── Identity / disjoint ──────────────────────────────────────────────────────

def test_two_identical_items_collapse_to_one():
    text = "Solve the equation x squared minus four equals zero showing every step"
    keep = MinHashDeduplicator().dedup([text, text], show_progress=False)
    assert len(keep) == 1


def test_two_completely_different_items_both_kept():
    keep = MinHashDeduplicator().dedup(
        [
            "describe the lifecycle of a butterfly in detail",
            "compute the integral of sine x from zero to pi",
        ],
        show_progress=False,
    )
    assert len(keep) == 2


# ── Threshold behavior ───────────────────────────────────────────────────────

def test_near_duplicate_above_threshold_collapses():
    # Tail-word edit only (one shingle differs) -> exact Jaccard ~0.93, safely
    # above the 0.7 threshold even after exact-Jaccard candidate verification.
    base = (
        "Solve the equation x squared minus four equals zero. Show all of your work "
        "step by step and explain your reasoning so that a beginner can follow it easily today."
    )
    near = (
        "Solve the equation x squared minus four equals zero. Show all of your work "
        "step by step and explain your reasoning so that a beginner can follow it easily now."
    )
    keep = MinHashDeduplicator(threshold=0.7).dedup([base, near], show_progress=False)
    assert len(keep) == 1


def test_distinct_items_below_threshold_both_kept():
    keep = MinHashDeduplicator(threshold=0.95).dedup(
        [
            "explain how photosynthesis works in plants",
            "describe the process of cellular respiration in animals",
        ],
        show_progress=False,
    )
    assert len(keep) == 2


# ── Cluster of three ─────────────────────────────────────────────────────────

def test_cluster_of_three_collapses_to_one():
    base = "the quick brown fox jumps over the lazy dog under a bright moon at night"
    near1 = base + " quietly"
    near2 = base + " softly"
    keep = MinHashDeduplicator(threshold=0.7).dedup([base, near1, near2], show_progress=False)
    assert len(keep) == 1


# ── LSH candidate verification (sub-threshold false positives) ───────────────

def test_distinct_prompts_sharing_long_boilerplate_block_not_merged():
    """Distinct questions that share a long fixed boilerplate block sit just
    below the threshold (~0.65) pairwise. LSH may surface them as candidates;
    without verifying the candidate's actual Jaccard, union-find transitively
    collapses them. Each distinct question must survive as its own record.
    """
    suffix = (
        "\n\nA) If both assertion and reason are true and reason is the correct "
        "explanation of assertion\n"
        "B) If both assertion and reason are true but reason is not the correct "
        "explanation of assertion\n"
        "C) If assertion is true but reason is false\n"
        "D) If both assertion and reason are false"
    )
    bodies = [
        "F is more electronegative than Cl F has high electron affinity than Cl.",
        "Leucoplasts perform photosynthesis Chloroplasts store fats, starch and proteins",
        "CN ion is an ambient nucleophile. Nucleophiles are electron rich species.",
        "Fructose is a reducing sugar. It has a ketonic group.",
        "The lactic acid shows the geometrical isomerism. Lactic acid has a double bond.",
    ]
    prompts = [b + suffix for b in bodies]
    keep = MinHashDeduplicator(threshold=0.8).dedup(prompts, show_progress=False)
    assert len(keep) == len(prompts)


def test_true_duplicates_still_merge_after_verification():
    """Verification must not suppress genuine near-duplicates: same question,
    trivial rewording, sharing the same boilerplate block -> still one record.
    """
    suffix = (
        "\n\nAnswer Choices:\n(A) 0.05 uC\n(B) 0.5 uC\n(C) 5 uC\n(D) 50 uC\n"
        "(E) 500 uC\n(F) 5000 uC\n(G) 0.005 uC\n(H) 0.0005 uC\n(I) 50 uC\n(J) 5 uC"
    )
    # Same question, one-word rewording (the->this) -> exact Jaccard ~0.83,
    # above 0.8 even after verification; the shared boilerplate doesn't prevent it.
    base = "A capacitor with a capacitance of 5 microfarads is charged to a voltage of 10 volts. What is the charge stored on the capacitor?"
    near = "A capacitor with a capacitance of 5 microfarads is charged to a voltage of 10 volts. What is the charge stored on this capacitor?"
    keep = MinHashDeduplicator(threshold=0.8).dedup([base + suffix, near + suffix], show_progress=False)
    assert len(keep) == 1


# ── Representative selection via key_fn ──────────────────────────────────────

def test_key_fn_selects_longest_response_proxy():
    text = "Solve the equation x squared minus four equals zero showing every step"
    response_lengths = [10, 500, 50]
    key_fn = lambda i: -response_lengths[i]  # longest wins
    keep = MinHashDeduplicator().dedup([text, text, text], key_fn=key_fn, show_progress=False)
    assert keep == [1]


# ── Normalization ────────────────────────────────────────────────────────────

def test_normalization_collapses_case_and_whitespace_variants():
    a = "Solve the equation x squared minus four equals zero showing every step"
    b = "  solve THE  equation x  squared minus four equals zero showing every step  "
    keep = MinHashDeduplicator(normalize=True, threshold=0.7).dedup([a, b], show_progress=False)
    assert len(keep) == 1


def test_normalization_off_keeps_case_variants():
    """Without normalization, formatting variants may not cross threshold."""
    a = "ALPHA BETA GAMMA DELTA EPSILON"
    b = "alpha beta gamma delta epsilon"
    # With small inputs and high threshold, expect both kept when normalize=False
    # (different shingles -> Jaccard 0)
    dedup = MinHashDeduplicator(normalize=False, threshold=0.9, shingle_size=2)
    keep = dedup.dedup([a, b], show_progress=False)
    assert len(keep) == 2


# ── Shingle kinds ────────────────────────────────────────────────────────────

def test_char_shingles_work():
    a = "abcdefghijklmnopqrstuvwxyz" * 3
    b = "abcdefghijklmnopqrstuvwxyz" * 3 + "ZZZ"
    keep = MinHashDeduplicator(shingle_kind="char", shingle_size=8, threshold=0.7).dedup(
        [a, b], show_progress=False
    )
    assert len(keep) == 1


def test_invalid_shingle_kind_raises():
    with pytest.raises(ValueError):
        MinHashDeduplicator(shingle_kind="bogus")


def test_invalid_threshold_raises():
    with pytest.raises(ValueError):
        MinHashDeduplicator(threshold=1.5)


# ── Determinism ──────────────────────────────────────────────────────────────

def test_determinism_same_seed_same_result():
    texts = ["alpha beta gamma delta epsilon"] * 5 + ["unrelated content here please"]
    a = MinHashDeduplicator(seed=7).dedup(texts, show_progress=False)
    b = MinHashDeduplicator(seed=7).dedup(texts, show_progress=False)
    assert a == b