File size: 2,967 Bytes
58258b8 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 | """Auto-detect and strip common instruction-template prefixes for dedup similarity.
Many SFT datasets (especially MCQ-heavy ones) use a small number of fixed
instruction templates as prompt prefixes, e.g.
"Answer the following multiple choice question. The last line of your
response should be in the following format: 'Answer: A/B/C/D' (e.g.
'Answer: A')."
These shared prefixes inflate Jaccard similarity (MinHash) and embedding
cosine similarity (semantic), causing distinct questions that share a
template to be falsely merged as duplicates.
This module provides:
- find_common_prefixes(): scan a corpus and auto-detect frequent prefixes
- strip_template(): remove the longest matching template from a text
- template_hit_stats(): audit helper showing per-template hit counts
Intended usage: pass the *stripped* prompts to MinHash/Semantic for the
similarity computation, but keep the *original* (unstripped) prompts in
the records that get persisted/uploaded.
"""
from __future__ import annotations
from collections import Counter
def find_common_prefixes(
texts: list[str],
min_count: int = 10,
prefix_lengths: tuple[int, ...] = (50, 100, 150, 200, 300, 400, 500, 600),
) -> list[str]:
"""Find character-prefix substrings that begin >= `min_count` texts.
Tries each length in `prefix_lengths` independently — this lets the
detector find both short and long template variants. The result is sorted
longest-first so that during stripping the most-specific (longest) match
wins (see `strip_template`).
"""
counter: Counter = Counter()
for t in texts:
for length in prefix_lengths:
if len(t) >= length:
counter[t[:length]] += 1
templates = [pfx for pfx, c in counter.items() if c >= min_count]
templates.sort(key=len, reverse=True)
return templates
def strip_template(text: str, templates: list[str]) -> str:
"""Return `text` with the longest matching template prefix removed.
`templates` should be sorted longest-first (as returned by
`find_common_prefixes`). Leading whitespace is stripped from the result.
Returns `text` unchanged when no template matches or `templates` is empty.
"""
for tmpl in templates:
if text.startswith(tmpl):
return text[len(tmpl):].lstrip()
return text
def template_hit_stats(texts: list[str], templates: list[str]) -> dict[str, int]:
"""For each template, count how many texts have it as the longest matching prefix.
Texts that don't match any template are counted under '<no template>'.
Sum of counts equals len(texts).
"""
counts: dict[str, int] = {tmpl: 0 for tmpl in templates}
counts["<no template>"] = 0
for t in texts:
for tmpl in templates:
if t.startswith(tmpl):
counts[tmpl] += 1
break
else:
counts["<no template>"] += 1
return counts
|