Spaces:
Running
Running
Download PMPF_duplicate_patcher.py from Naphula/model_tools: direct link, hf CLI and curl.
- Browser
- Download file 11.3 kB
-
https://huggingface.co/spaces/Naphula/model_tools/resolve/main/PMPF_duplicate_patcher.py
- Command line
-
hf download hf://spaces/Naphula/model_tools/PMPF_duplicate_patcher.py
-
curl -L -o PMPF_duplicate_patcher.py https://huggingface.co/spaces/Naphula/model_tools/resolve/main/PMPF_duplicate_patcher.py
11.3 kB
| #!/usr/bin/env python3 | |
| """ | |
| PMPF_duplicate_patcher.py | |
| Inspects dataset_cache/unified_dataset.parquet for duplicate Q&A pairs | |
| (matching question and answer content at >= 99% similarity), displays a report, | |
| and provides an interactive prompt to deduplicate the dataset with an automatic backup. | |
| """ | |
| import difflib | |
| import json | |
| import os | |
| import re | |
| import shutil | |
| import sys | |
| import time | |
| from typing import Any, Dict, List, Optional, Tuple | |
| import numpy as np | |
| import pandas as pd | |
| # ================= CONFIGURATION ================= | |
| DEFAULT_PARQUET_FILE = os.path.join(os.getcwd(), "dataset_cache", "unified_dataset.parquet") | |
| SIMILARITY_THRESHOLD = 0.99 # 99% similarity threshold | |
| # ================================================= | |
| class Colors: | |
| HEADER = '\033[95m' | |
| OKGREEN = '\033[92m' | |
| OKCYAN = '\033[96m' | |
| WARNING = '\033[93m' | |
| FAIL = '\033[91m' | |
| ENDC = '\033[0m' | |
| BOLD = '\033[1m' | |
| # Windows CMD color support | |
| os.system('color') | |
| def normalize_text(text: Any) -> str: | |
| """Normalizes whitespace and lowercase for reliable similarity and hashing.""" | |
| if text is None: | |
| return "" | |
| text_str = str(text) | |
| # Collapse irregular whitespace | |
| return re.sub(r'\s+', ' ', text_str).strip().lower() | |
| def extract_qa_pair(row: pd.Series) -> Tuple[str, str]: | |
| """ | |
| Extracts the actual Question (user) and Answer (assistant/gpt) from the row. | |
| Ignores metadata, category labels, or topic tags. | |
| """ | |
| # 1. ShareGPT format: conversations = [{"from": "human", "value": "..."}, {"from": "gpt", "value": "..."}] | |
| if "conversations" in row and isinstance(row["conversations"], (list, np.ndarray)): | |
| turns = row["conversations"] | |
| user_parts, gpt_parts = [], [] | |
| for turn in turns: | |
| if isinstance(turn, dict): | |
| role = str(turn.get("from", "")).lower() | |
| val = str(turn.get("value", turn.get("content", ""))) | |
| if role in ("human", "user"): | |
| user_parts.append(val) | |
| elif role in ("gpt", "assistant", "bot"): | |
| gpt_parts.append(val) | |
| if user_parts or gpt_parts: | |
| return "\n".join(user_parts), "\n".join(gpt_parts) | |
| # 2. OpenAI / ChatML format: messages = [{"role": "user", "content": "..."}, {"role": "assistant", "content": "..."}] | |
| if "messages" in row and isinstance(row["messages"], (list, np.ndarray)): | |
| turns = row["messages"] | |
| user_parts, gpt_parts = [], [] | |
| for turn in turns: | |
| if isinstance(turn, dict): | |
| role = str(turn.get("role", "")).lower() | |
| val = str(turn.get("content", "")) | |
| if role in ("user", "human"): | |
| user_parts.append(val) | |
| elif role in ("assistant", "system", "gpt"): | |
| gpt_parts.append(val) | |
| if user_parts or gpt_parts: | |
| return "\n".join(user_parts), "\n".join(gpt_parts) | |
| # 3. Explicit prompt / response or instruction / output columns | |
| q, a = "", "" | |
| if "prompt" in row and pd.notna(row["prompt"]): | |
| q = str(row["prompt"]) | |
| elif "instruction" in row and pd.notna(row["instruction"]): | |
| q = str(row["instruction"]) | |
| elif "question" in row and pd.notna(row["question"]): | |
| q = str(row["question"]) | |
| if "response" in row and pd.notna(row["response"]): | |
| a = str(row["response"]) | |
| elif "output" in row and pd.notna(row["output"]): | |
| a = str(row["output"]) | |
| elif "answer" in row and pd.notna(row["answer"]): | |
| a = str(row["answer"]) | |
| if q or a: | |
| return q, a | |
| # 4. Fallback to raw text field if present | |
| if "text" in row and pd.notna(row["text"]): | |
| return str(row["text"]), "" | |
| return "", "" | |
| def calculate_similarity(text1: str, text2: str) -> float: | |
| """Calculates sequence similarity ratio between two texts.""" | |
| if text1 == text2: | |
| return 1.0 | |
| if not text1 or not text2: | |
| return 0.0 | |
| # Quick length check optimization: if difference exceeds 2%, ratio cannot be >= 0.99 | |
| len1, len2 = len(text1), len(text2) | |
| max_len = max(len1, len2) | |
| if max_len > 0 and (abs(len1 - len2) / max_len) > 0.02: | |
| return 0.0 | |
| return difflib.SequenceMatcher(None, text1, text2).ratio() | |
| def main(): | |
| print(f"{Colors.HEADER}{'='*80}") | |
| print(" PARQUET Q&A DUPLICATE SCANNER & DE-DUPLICATION PATCHER") | |
| print(f"{'='*80}{Colors.ENDC}") | |
| parquet_file = DEFAULT_PARQUET_FILE | |
| if not os.path.isfile(parquet_file): | |
| print(f"{Colors.FAIL}Error: File not found at '{parquet_file}'.{Colors.ENDC}") | |
| print("Please verify that the current working directory contains 'dataset_cache/unified_dataset.parquet'.") | |
| return | |
| print(f"Loading Parquet from: {Colors.OKCYAN}{parquet_file}{Colors.ENDC} ...") | |
| try: | |
| df = pd.read_parquet(parquet_file) | |
| except Exception as e: | |
| print(f"{Colors.FAIL}Failed to read parquet file: {e}{Colors.ENDC}") | |
| return | |
| total_records = len(df) | |
| print(f"Loaded {Colors.BOLD}{total_records}{Colors.ENDC} total records.\n") | |
| print("Extracting question & answer contents from dialogue/records...") | |
| qa_pairs = [extract_qa_pair(row) for _, row in df.iterrows()] | |
| df['_extracted_q'] = [q for q, _ in qa_pairs] | |
| df['_extracted_a'] = [a for _, a in qa_pairs] | |
| df['_norm_q'] = df['_extracted_q'].apply(normalize_text) | |
| df['_norm_a'] = df['_extracted_a'].apply(normalize_text) | |
| # Filter out records where both Q and A are completely empty | |
| valid_mask = (df['_norm_q'] != "") | (df['_norm_a'] != "") | |
| df_eval = df[valid_mask].copy() | |
| # Step 1: Exact matches via canonical combined hash key | |
| df_eval['_qa_key'] = df_eval['_norm_q'] + " <||SEP||> " + df_eval['_norm_a'] | |
| exact_dup_groups = df_eval.groupby('_qa_key').filter(lambda g: len(g) > 1) | |
| # Step 2: Near-duplicate pass (>= 99% similarity) among remaining rows | |
| # We group by length bucket to avoid O(N^2) explosion | |
| unique_keys = df_eval.drop_duplicates(subset=['_qa_key']).copy() | |
| unique_keys['key_len'] = unique_keys['_qa_key'].str.len() | |
| near_duplicates_map = {} # index -> canonical cluster parent index | |
| sorted_unique = unique_keys.sort_values(by='key_len') | |
| rows_list = sorted_unique[['_norm_q', '_norm_a', 'key_len']].to_dict('index') | |
| indices = list(rows_list.keys()) | |
| print("Running 99% similarity verification pass on conversation text...") | |
| for i in range(len(indices)): | |
| idx_a = indices[i] | |
| if idx_a in near_duplicates_map: | |
| continue | |
| row_a = rows_list[idx_a] | |
| len_a = row_a['key_len'] | |
| # Compare only with items whose length differs by <= 2% | |
| for j in range(i + 1, len(indices)): | |
| idx_b = indices[j] | |
| row_b = rows_list[idx_b] | |
| len_b = row_b['key_len'] | |
| if len_b - len_a > max(3, int(len_a * 0.02)): | |
| break | |
| q_sim = calculate_similarity(row_a['_norm_q'], row_b['_norm_q']) | |
| if q_sim >= SIMILARITY_THRESHOLD: | |
| a_sim = calculate_similarity(row_a['_norm_a'], row_b['_norm_a']) | |
| if a_sim >= SIMILARITY_THRESHOLD: | |
| near_duplicates_map[idx_b] = idx_a | |
| # Consolidate duplicate clusters | |
| clusters: Dict[Any, List[int]] = {} | |
| # Exact group assignments | |
| for key, group in exact_dup_groups.groupby('_qa_key'): | |
| primary_idx = group.index[0] | |
| clusters[primary_idx] = group.index.tolist() | |
| # Merge near-duplicate assignments | |
| for dup_idx, canonical_idx in near_duplicates_map.items(): | |
| found = False | |
| for root, members in clusters.items(): | |
| if canonical_idx in members or root == canonical_idx: | |
| if dup_idx not in members: | |
| members.append(dup_idx) | |
| found = True | |
| break | |
| if not found: | |
| clusters[canonical_idx] = [canonical_idx, dup_idx] | |
| # Collect indices marked as duplicates to remove (all except the first in each cluster) | |
| all_drop_indices = set() | |
| for _, members in clusters.items(): | |
| for redundant in members[1:]: | |
| all_drop_indices.add(redundant) | |
| if not clusters: | |
| print(f"\n{Colors.OKGREEN}🎉 Scan complete! No duplicate Q&A pairs (>=99% match) were found.{Colors.ENDC}") | |
| return | |
| redundant_count = len(all_drop_indices) | |
| print(f"\n{Colors.WARNING}⚠️ Found {len(clusters)} duplicate clusters containing {redundant_count} redundant rows.{Colors.ENDC}") | |
| print("=" * 80) | |
| print("DUPLICATE GROUP AUDIT PREVIEW:") | |
| print("=" * 80) | |
| # Show preview of first 10 groups | |
| preview_limit = 10 | |
| for g_num, (primary_idx, member_indices) in enumerate(list(clusters.items())[:preview_limit], 1): | |
| q_prev = df.loc[primary_idx, '_extracted_q'].replace("\n", " ")[:90] | |
| a_prev = df.loc[primary_idx, '_extracted_a'].replace("\n", " ")[:90] | |
| print(f"\n{Colors.BOLD}Cluster #{g_num} (Matches {len(member_indices)} entries):{Colors.ENDC}") | |
| print(f" {Colors.OKCYAN}Q:{Colors.ENDC} \"{q_prev}...\"") | |
| print(f" {Colors.OKGREEN}A:{Colors.ENDC} \"{a_prev}...\"") | |
| print(f" Row Indices: {member_indices} (Keep: {member_indices[0]}, Drop: {member_indices[1:]})") | |
| if len(clusters) > preview_limit: | |
| print(f"\n... and {len(clusters) - preview_limit} more duplicate groups not shown.") | |
| print(f"\n{Colors.HEADER}{'='*80}{Colors.ENDC}") | |
| print(f"Dataset summary before action:") | |
| print(f" - Total entries: {len(df)}") | |
| print(f" - Redundant entries to remove: {len(all_drop_indices)}") | |
| print(f" - Cleaned dataset total entries: {len(df) - len(all_drop_indices)}") | |
| # Prompt user for de-duplication action | |
| choice = input(f"\n{Colors.BOLD}Would you like to de-duplicate and save the cleaned dataset? (y/N): {Colors.ENDC}").strip().lower() | |
| if choice in ['y', 'yes']: | |
| # 1. Create a backup | |
| timestamp = time.strftime("%Y%m%d_%H%M%S") | |
| backup_path = f"{parquet_file}.backup_{timestamp}" | |
| print(f"\nCreating safety backup at: {backup_path}") | |
| shutil.copy2(parquet_file, backup_path) | |
| # 2. Drop redundant rows and temporary analysis columns | |
| cleaned_df = df.drop(index=list(all_drop_indices)).reset_index(drop=True) | |
| temp_cols = ['_extracted_q', '_extracted_a', '_norm_q', '_norm_a'] | |
| cleaned_df.drop(columns=[c for c in temp_cols if c in cleaned_df.columns], inplace=True) | |
| # 3. Write back to parquet | |
| print(f"Overwriting {parquet_file} with de-duplicated dataset...") | |
| cleaned_df.to_parquet(parquet_file, index=False) | |
| print(f"{Colors.OKGREEN}✅ Successfully saved cleaned parquet ({len(cleaned_df)} rows).{Colors.ENDC}") | |
| print(f"Backup preserved at: {Colors.WARNING}{backup_path}{Colors.ENDC}") | |
| else: | |
| print(f"\n{Colors.WARNING}De-duplication aborted. No files were modified.{Colors.ENDC}") | |
| if __name__ == "__main__": | |
| main() |