model_tools / PMPF_duplicate_patcher.py
Naphula's picture
Upload PMPF_duplicate_patcher.py
b8d1d4a verified
Raw History Blame Contribute Delete
11.3 kB
#!/usr/bin/env python3
"""
PMPF_duplicate_patcher.py
Inspects dataset_cache/unified_dataset.parquet for duplicate Q&A pairs
(matching question and answer content at >= 99% similarity), displays a report,
and provides an interactive prompt to deduplicate the dataset with an automatic backup.
"""
import difflib
import json
import os
import re
import shutil
import sys
import time
from typing import Any, Dict, List, Optional, Tuple
import numpy as np
import pandas as pd
# ================= CONFIGURATION =================
DEFAULT_PARQUET_FILE = os.path.join(os.getcwd(), "dataset_cache", "unified_dataset.parquet")
SIMILARITY_THRESHOLD = 0.99 # 99% similarity threshold
# =================================================
class Colors:
HEADER = '\033[95m'
OKGREEN = '\033[92m'
OKCYAN = '\033[96m'
WARNING = '\033[93m'
FAIL = '\033[91m'
ENDC = '\033[0m'
BOLD = '\033[1m'
# Windows CMD color support
os.system('color')
def normalize_text(text: Any) -> str:
"""Normalizes whitespace and lowercase for reliable similarity and hashing."""
if text is None:
return ""
text_str = str(text)
# Collapse irregular whitespace
return re.sub(r'\s+', ' ', text_str).strip().lower()
def extract_qa_pair(row: pd.Series) -> Tuple[str, str]:
"""
Extracts the actual Question (user) and Answer (assistant/gpt) from the row.
Ignores metadata, category labels, or topic tags.
"""
# 1. ShareGPT format: conversations = [{"from": "human", "value": "..."}, {"from": "gpt", "value": "..."}]
if "conversations" in row and isinstance(row["conversations"], (list, np.ndarray)):
turns = row["conversations"]
user_parts, gpt_parts = [], []
for turn in turns:
if isinstance(turn, dict):
role = str(turn.get("from", "")).lower()
val = str(turn.get("value", turn.get("content", "")))
if role in ("human", "user"):
user_parts.append(val)
elif role in ("gpt", "assistant", "bot"):
gpt_parts.append(val)
if user_parts or gpt_parts:
return "\n".join(user_parts), "\n".join(gpt_parts)
# 2. OpenAI / ChatML format: messages = [{"role": "user", "content": "..."}, {"role": "assistant", "content": "..."}]
if "messages" in row and isinstance(row["messages"], (list, np.ndarray)):
turns = row["messages"]
user_parts, gpt_parts = [], []
for turn in turns:
if isinstance(turn, dict):
role = str(turn.get("role", "")).lower()
val = str(turn.get("content", ""))
if role in ("user", "human"):
user_parts.append(val)
elif role in ("assistant", "system", "gpt"):
gpt_parts.append(val)
if user_parts or gpt_parts:
return "\n".join(user_parts), "\n".join(gpt_parts)
# 3. Explicit prompt / response or instruction / output columns
q, a = "", ""
if "prompt" in row and pd.notna(row["prompt"]):
q = str(row["prompt"])
elif "instruction" in row and pd.notna(row["instruction"]):
q = str(row["instruction"])
elif "question" in row and pd.notna(row["question"]):
q = str(row["question"])
if "response" in row and pd.notna(row["response"]):
a = str(row["response"])
elif "output" in row and pd.notna(row["output"]):
a = str(row["output"])
elif "answer" in row and pd.notna(row["answer"]):
a = str(row["answer"])
if q or a:
return q, a
# 4. Fallback to raw text field if present
if "text" in row and pd.notna(row["text"]):
return str(row["text"]), ""
return "", ""
def calculate_similarity(text1: str, text2: str) -> float:
"""Calculates sequence similarity ratio between two texts."""
if text1 == text2:
return 1.0
if not text1 or not text2:
return 0.0
# Quick length check optimization: if difference exceeds 2%, ratio cannot be >= 0.99
len1, len2 = len(text1), len(text2)
max_len = max(len1, len2)
if max_len > 0 and (abs(len1 - len2) / max_len) > 0.02:
return 0.0
return difflib.SequenceMatcher(None, text1, text2).ratio()
def main():
print(f"{Colors.HEADER}{'='*80}")
print(" PARQUET Q&A DUPLICATE SCANNER & DE-DUPLICATION PATCHER")
print(f"{'='*80}{Colors.ENDC}")
parquet_file = DEFAULT_PARQUET_FILE
if not os.path.isfile(parquet_file):
print(f"{Colors.FAIL}Error: File not found at '{parquet_file}'.{Colors.ENDC}")
print("Please verify that the current working directory contains 'dataset_cache/unified_dataset.parquet'.")
return
print(f"Loading Parquet from: {Colors.OKCYAN}{parquet_file}{Colors.ENDC} ...")
try:
df = pd.read_parquet(parquet_file)
except Exception as e:
print(f"{Colors.FAIL}Failed to read parquet file: {e}{Colors.ENDC}")
return
total_records = len(df)
print(f"Loaded {Colors.BOLD}{total_records}{Colors.ENDC} total records.\n")
print("Extracting question & answer contents from dialogue/records...")
qa_pairs = [extract_qa_pair(row) for _, row in df.iterrows()]
df['_extracted_q'] = [q for q, _ in qa_pairs]
df['_extracted_a'] = [a for _, a in qa_pairs]
df['_norm_q'] = df['_extracted_q'].apply(normalize_text)
df['_norm_a'] = df['_extracted_a'].apply(normalize_text)
# Filter out records where both Q and A are completely empty
valid_mask = (df['_norm_q'] != "") | (df['_norm_a'] != "")
df_eval = df[valid_mask].copy()
# Step 1: Exact matches via canonical combined hash key
df_eval['_qa_key'] = df_eval['_norm_q'] + " <||SEP||> " + df_eval['_norm_a']
exact_dup_groups = df_eval.groupby('_qa_key').filter(lambda g: len(g) > 1)
# Step 2: Near-duplicate pass (>= 99% similarity) among remaining rows
# We group by length bucket to avoid O(N^2) explosion
unique_keys = df_eval.drop_duplicates(subset=['_qa_key']).copy()
unique_keys['key_len'] = unique_keys['_qa_key'].str.len()
near_duplicates_map = {} # index -> canonical cluster parent index
sorted_unique = unique_keys.sort_values(by='key_len')
rows_list = sorted_unique[['_norm_q', '_norm_a', 'key_len']].to_dict('index')
indices = list(rows_list.keys())
print("Running 99% similarity verification pass on conversation text...")
for i in range(len(indices)):
idx_a = indices[i]
if idx_a in near_duplicates_map:
continue
row_a = rows_list[idx_a]
len_a = row_a['key_len']
# Compare only with items whose length differs by <= 2%
for j in range(i + 1, len(indices)):
idx_b = indices[j]
row_b = rows_list[idx_b]
len_b = row_b['key_len']
if len_b - len_a > max(3, int(len_a * 0.02)):
break
q_sim = calculate_similarity(row_a['_norm_q'], row_b['_norm_q'])
if q_sim >= SIMILARITY_THRESHOLD:
a_sim = calculate_similarity(row_a['_norm_a'], row_b['_norm_a'])
if a_sim >= SIMILARITY_THRESHOLD:
near_duplicates_map[idx_b] = idx_a
# Consolidate duplicate clusters
clusters: Dict[Any, List[int]] = {}
# Exact group assignments
for key, group in exact_dup_groups.groupby('_qa_key'):
primary_idx = group.index[0]
clusters[primary_idx] = group.index.tolist()
# Merge near-duplicate assignments
for dup_idx, canonical_idx in near_duplicates_map.items():
found = False
for root, members in clusters.items():
if canonical_idx in members or root == canonical_idx:
if dup_idx not in members:
members.append(dup_idx)
found = True
break
if not found:
clusters[canonical_idx] = [canonical_idx, dup_idx]
# Collect indices marked as duplicates to remove (all except the first in each cluster)
all_drop_indices = set()
for _, members in clusters.items():
for redundant in members[1:]:
all_drop_indices.add(redundant)
if not clusters:
print(f"\n{Colors.OKGREEN}🎉 Scan complete! No duplicate Q&A pairs (>=99% match) were found.{Colors.ENDC}")
return
redundant_count = len(all_drop_indices)
print(f"\n{Colors.WARNING}⚠️ Found {len(clusters)} duplicate clusters containing {redundant_count} redundant rows.{Colors.ENDC}")
print("=" * 80)
print("DUPLICATE GROUP AUDIT PREVIEW:")
print("=" * 80)
# Show preview of first 10 groups
preview_limit = 10
for g_num, (primary_idx, member_indices) in enumerate(list(clusters.items())[:preview_limit], 1):
q_prev = df.loc[primary_idx, '_extracted_q'].replace("\n", " ")[:90]
a_prev = df.loc[primary_idx, '_extracted_a'].replace("\n", " ")[:90]
print(f"\n{Colors.BOLD}Cluster #{g_num} (Matches {len(member_indices)} entries):{Colors.ENDC}")
print(f" {Colors.OKCYAN}Q:{Colors.ENDC} \"{q_prev}...\"")
print(f" {Colors.OKGREEN}A:{Colors.ENDC} \"{a_prev}...\"")
print(f" Row Indices: {member_indices} (Keep: {member_indices[0]}, Drop: {member_indices[1:]})")
if len(clusters) > preview_limit:
print(f"\n... and {len(clusters) - preview_limit} more duplicate groups not shown.")
print(f"\n{Colors.HEADER}{'='*80}{Colors.ENDC}")
print(f"Dataset summary before action:")
print(f" - Total entries: {len(df)}")
print(f" - Redundant entries to remove: {len(all_drop_indices)}")
print(f" - Cleaned dataset total entries: {len(df) - len(all_drop_indices)}")
# Prompt user for de-duplication action
choice = input(f"\n{Colors.BOLD}Would you like to de-duplicate and save the cleaned dataset? (y/N): {Colors.ENDC}").strip().lower()
if choice in ['y', 'yes']:
# 1. Create a backup
timestamp = time.strftime("%Y%m%d_%H%M%S")
backup_path = f"{parquet_file}.backup_{timestamp}"
print(f"\nCreating safety backup at: {backup_path}")
shutil.copy2(parquet_file, backup_path)
# 2. Drop redundant rows and temporary analysis columns
cleaned_df = df.drop(index=list(all_drop_indices)).reset_index(drop=True)
temp_cols = ['_extracted_q', '_extracted_a', '_norm_q', '_norm_a']
cleaned_df.drop(columns=[c for c in temp_cols if c in cleaned_df.columns], inplace=True)
# 3. Write back to parquet
print(f"Overwriting {parquet_file} with de-duplicated dataset...")
cleaned_df.to_parquet(parquet_file, index=False)
print(f"{Colors.OKGREEN}✅ Successfully saved cleaned parquet ({len(cleaned_df)} rows).{Colors.ENDC}")
print(f"Backup preserved at: {Colors.WARNING}{backup_path}{Colors.ENDC}")
else:
print(f"\n{Colors.WARNING}De-duplication aborted. No files were modified.{Colors.ENDC}")
if __name__ == "__main__":
main()