#!/usr/bin/env python3 """ PMPF_duplicate_patcher.py Inspects dataset_cache/unified_dataset.parquet for duplicate Q&A pairs (matching question and answer content at >= 99% similarity), displays a report, and provides an interactive prompt to deduplicate the dataset with an automatic backup. """ import difflib import json import os import re import shutil import sys import time from typing import Any, Dict, List, Optional, Tuple import numpy as np import pandas as pd # ================= CONFIGURATION ================= DEFAULT_PARQUET_FILE = os.path.join(os.getcwd(), "dataset_cache", "unified_dataset.parquet") SIMILARITY_THRESHOLD = 0.99 # 99% similarity threshold # ================================================= class Colors: HEADER = '\033[95m' OKGREEN = '\033[92m' OKCYAN = '\033[96m' WARNING = '\033[93m' FAIL = '\033[91m' ENDC = '\033[0m' BOLD = '\033[1m' # Windows CMD color support os.system('color') def normalize_text(text: Any) -> str: """Normalizes whitespace and lowercase for reliable similarity and hashing.""" if text is None: return "" text_str = str(text) # Collapse irregular whitespace return re.sub(r'\s+', ' ', text_str).strip().lower() def extract_qa_pair(row: pd.Series) -> Tuple[str, str]: """ Extracts the actual Question (user) and Answer (assistant/gpt) from the row. Ignores metadata, category labels, or topic tags. """ # 1. ShareGPT format: conversations = [{"from": "human", "value": "..."}, {"from": "gpt", "value": "..."}] if "conversations" in row and isinstance(row["conversations"], (list, np.ndarray)): turns = row["conversations"] user_parts, gpt_parts = [], [] for turn in turns: if isinstance(turn, dict): role = str(turn.get("from", "")).lower() val = str(turn.get("value", turn.get("content", ""))) if role in ("human", "user"): user_parts.append(val) elif role in ("gpt", "assistant", "bot"): gpt_parts.append(val) if user_parts or gpt_parts: return "\n".join(user_parts), "\n".join(gpt_parts) # 2. OpenAI / ChatML format: messages = [{"role": "user", "content": "..."}, {"role": "assistant", "content": "..."}] if "messages" in row and isinstance(row["messages"], (list, np.ndarray)): turns = row["messages"] user_parts, gpt_parts = [], [] for turn in turns: if isinstance(turn, dict): role = str(turn.get("role", "")).lower() val = str(turn.get("content", "")) if role in ("user", "human"): user_parts.append(val) elif role in ("assistant", "system", "gpt"): gpt_parts.append(val) if user_parts or gpt_parts: return "\n".join(user_parts), "\n".join(gpt_parts) # 3. Explicit prompt / response or instruction / output columns q, a = "", "" if "prompt" in row and pd.notna(row["prompt"]): q = str(row["prompt"]) elif "instruction" in row and pd.notna(row["instruction"]): q = str(row["instruction"]) elif "question" in row and pd.notna(row["question"]): q = str(row["question"]) if "response" in row and pd.notna(row["response"]): a = str(row["response"]) elif "output" in row and pd.notna(row["output"]): a = str(row["output"]) elif "answer" in row and pd.notna(row["answer"]): a = str(row["answer"]) if q or a: return q, a # 4. Fallback to raw text field if present if "text" in row and pd.notna(row["text"]): return str(row["text"]), "" return "", "" def calculate_similarity(text1: str, text2: str) -> float: """Calculates sequence similarity ratio between two texts.""" if text1 == text2: return 1.0 if not text1 or not text2: return 0.0 # Quick length check optimization: if difference exceeds 2%, ratio cannot be >= 0.99 len1, len2 = len(text1), len(text2) max_len = max(len1, len2) if max_len > 0 and (abs(len1 - len2) / max_len) > 0.02: return 0.0 return difflib.SequenceMatcher(None, text1, text2).ratio() def main(): print(f"{Colors.HEADER}{'='*80}") print(" PARQUET Q&A DUPLICATE SCANNER & DE-DUPLICATION PATCHER") print(f"{'='*80}{Colors.ENDC}") parquet_file = DEFAULT_PARQUET_FILE if not os.path.isfile(parquet_file): print(f"{Colors.FAIL}Error: File not found at '{parquet_file}'.{Colors.ENDC}") print("Please verify that the current working directory contains 'dataset_cache/unified_dataset.parquet'.") return print(f"Loading Parquet from: {Colors.OKCYAN}{parquet_file}{Colors.ENDC} ...") try: df = pd.read_parquet(parquet_file) except Exception as e: print(f"{Colors.FAIL}Failed to read parquet file: {e}{Colors.ENDC}") return total_records = len(df) print(f"Loaded {Colors.BOLD}{total_records}{Colors.ENDC} total records.\n") print("Extracting question & answer contents from dialogue/records...") qa_pairs = [extract_qa_pair(row) for _, row in df.iterrows()] df['_extracted_q'] = [q for q, _ in qa_pairs] df['_extracted_a'] = [a for _, a in qa_pairs] df['_norm_q'] = df['_extracted_q'].apply(normalize_text) df['_norm_a'] = df['_extracted_a'].apply(normalize_text) # Filter out records where both Q and A are completely empty valid_mask = (df['_norm_q'] != "") | (df['_norm_a'] != "") df_eval = df[valid_mask].copy() # Step 1: Exact matches via canonical combined hash key df_eval['_qa_key'] = df_eval['_norm_q'] + " <||SEP||> " + df_eval['_norm_a'] exact_dup_groups = df_eval.groupby('_qa_key').filter(lambda g: len(g) > 1) # Step 2: Near-duplicate pass (>= 99% similarity) among remaining rows # We group by length bucket to avoid O(N^2) explosion unique_keys = df_eval.drop_duplicates(subset=['_qa_key']).copy() unique_keys['key_len'] = unique_keys['_qa_key'].str.len() near_duplicates_map = {} # index -> canonical cluster parent index sorted_unique = unique_keys.sort_values(by='key_len') rows_list = sorted_unique[['_norm_q', '_norm_a', 'key_len']].to_dict('index') indices = list(rows_list.keys()) print("Running 99% similarity verification pass on conversation text...") for i in range(len(indices)): idx_a = indices[i] if idx_a in near_duplicates_map: continue row_a = rows_list[idx_a] len_a = row_a['key_len'] # Compare only with items whose length differs by <= 2% for j in range(i + 1, len(indices)): idx_b = indices[j] row_b = rows_list[idx_b] len_b = row_b['key_len'] if len_b - len_a > max(3, int(len_a * 0.02)): break q_sim = calculate_similarity(row_a['_norm_q'], row_b['_norm_q']) if q_sim >= SIMILARITY_THRESHOLD: a_sim = calculate_similarity(row_a['_norm_a'], row_b['_norm_a']) if a_sim >= SIMILARITY_THRESHOLD: near_duplicates_map[idx_b] = idx_a # Consolidate duplicate clusters clusters: Dict[Any, List[int]] = {} # Exact group assignments for key, group in exact_dup_groups.groupby('_qa_key'): primary_idx = group.index[0] clusters[primary_idx] = group.index.tolist() # Merge near-duplicate assignments for dup_idx, canonical_idx in near_duplicates_map.items(): found = False for root, members in clusters.items(): if canonical_idx in members or root == canonical_idx: if dup_idx not in members: members.append(dup_idx) found = True break if not found: clusters[canonical_idx] = [canonical_idx, dup_idx] # Collect indices marked as duplicates to remove (all except the first in each cluster) all_drop_indices = set() for _, members in clusters.items(): for redundant in members[1:]: all_drop_indices.add(redundant) if not clusters: print(f"\n{Colors.OKGREEN}🎉 Scan complete! No duplicate Q&A pairs (>=99% match) were found.{Colors.ENDC}") return redundant_count = len(all_drop_indices) print(f"\n{Colors.WARNING}⚠️ Found {len(clusters)} duplicate clusters containing {redundant_count} redundant rows.{Colors.ENDC}") print("=" * 80) print("DUPLICATE GROUP AUDIT PREVIEW:") print("=" * 80) # Show preview of first 10 groups preview_limit = 10 for g_num, (primary_idx, member_indices) in enumerate(list(clusters.items())[:preview_limit], 1): q_prev = df.loc[primary_idx, '_extracted_q'].replace("\n", " ")[:90] a_prev = df.loc[primary_idx, '_extracted_a'].replace("\n", " ")[:90] print(f"\n{Colors.BOLD}Cluster #{g_num} (Matches {len(member_indices)} entries):{Colors.ENDC}") print(f" {Colors.OKCYAN}Q:{Colors.ENDC} \"{q_prev}...\"") print(f" {Colors.OKGREEN}A:{Colors.ENDC} \"{a_prev}...\"") print(f" Row Indices: {member_indices} (Keep: {member_indices[0]}, Drop: {member_indices[1:]})") if len(clusters) > preview_limit: print(f"\n... and {len(clusters) - preview_limit} more duplicate groups not shown.") print(f"\n{Colors.HEADER}{'='*80}{Colors.ENDC}") print(f"Dataset summary before action:") print(f" - Total entries: {len(df)}") print(f" - Redundant entries to remove: {len(all_drop_indices)}") print(f" - Cleaned dataset total entries: {len(df) - len(all_drop_indices)}") # Prompt user for de-duplication action choice = input(f"\n{Colors.BOLD}Would you like to de-duplicate and save the cleaned dataset? (y/N): {Colors.ENDC}").strip().lower() if choice in ['y', 'yes']: # 1. Create a backup timestamp = time.strftime("%Y%m%d_%H%M%S") backup_path = f"{parquet_file}.backup_{timestamp}" print(f"\nCreating safety backup at: {backup_path}") shutil.copy2(parquet_file, backup_path) # 2. Drop redundant rows and temporary analysis columns cleaned_df = df.drop(index=list(all_drop_indices)).reset_index(drop=True) temp_cols = ['_extracted_q', '_extracted_a', '_norm_q', '_norm_a'] cleaned_df.drop(columns=[c for c in temp_cols if c in cleaned_df.columns], inplace=True) # 3. Write back to parquet print(f"Overwriting {parquet_file} with de-duplicated dataset...") cleaned_df.to_parquet(parquet_file, index=False) print(f"{Colors.OKGREEN}✅ Successfully saved cleaned parquet ({len(cleaned_df)} rows).{Colors.ENDC}") print(f"Backup preserved at: {Colors.WARNING}{backup_path}{Colors.ENDC}") else: print(f"\n{Colors.WARNING}De-duplication aborted. No files were modified.{Colors.ENDC}") if __name__ == "__main__": main()