Naphula commited on
Commit
b8d1d4a
·
verified ·
1 Parent(s): 4528f21

Upload PMPF_duplicate_patcher.py

Browse files
Files changed (1) hide show
  1. PMPF_duplicate_patcher.py +275 -0
PMPF_duplicate_patcher.py ADDED
@@ -0,0 +1,275 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """
3
+ PMPF_duplicate_patcher.py
4
+
5
+ Inspects dataset_cache/unified_dataset.parquet for duplicate Q&A pairs
6
+ (matching question and answer content at >= 99% similarity), displays a report,
7
+ and provides an interactive prompt to deduplicate the dataset with an automatic backup.
8
+ """
9
+
10
+ import difflib
11
+ import json
12
+ import os
13
+ import re
14
+ import shutil
15
+ import sys
16
+ import time
17
+ from typing import Any, Dict, List, Optional, Tuple
18
+
19
+ import numpy as np
20
+ import pandas as pd
21
+
22
+ # ================= CONFIGURATION =================
23
+ DEFAULT_PARQUET_FILE = os.path.join(os.getcwd(), "dataset_cache", "unified_dataset.parquet")
24
+ SIMILARITY_THRESHOLD = 0.99 # 99% similarity threshold
25
+ # =================================================
26
+
27
+ class Colors:
28
+ HEADER = '\033[95m'
29
+ OKGREEN = '\033[92m'
30
+ OKCYAN = '\033[96m'
31
+ WARNING = '\033[93m'
32
+ FAIL = '\033[91m'
33
+ ENDC = '\033[0m'
34
+ BOLD = '\033[1m'
35
+
36
+ # Windows CMD color support
37
+ os.system('color')
38
+
39
+
40
+ def normalize_text(text: Any) -> str:
41
+ """Normalizes whitespace and lowercase for reliable similarity and hashing."""
42
+ if text is None:
43
+ return ""
44
+ text_str = str(text)
45
+ # Collapse irregular whitespace
46
+ return re.sub(r'\s+', ' ', text_str).strip().lower()
47
+
48
+
49
+ def extract_qa_pair(row: pd.Series) -> Tuple[str, str]:
50
+ """
51
+ Extracts the actual Question (user) and Answer (assistant/gpt) from the row.
52
+ Ignores metadata, category labels, or topic tags.
53
+ """
54
+ # 1. ShareGPT format: conversations = [{"from": "human", "value": "..."}, {"from": "gpt", "value": "..."}]
55
+ if "conversations" in row and isinstance(row["conversations"], (list, np.ndarray)):
56
+ turns = row["conversations"]
57
+ user_parts, gpt_parts = [], []
58
+ for turn in turns:
59
+ if isinstance(turn, dict):
60
+ role = str(turn.get("from", "")).lower()
61
+ val = str(turn.get("value", turn.get("content", "")))
62
+ if role in ("human", "user"):
63
+ user_parts.append(val)
64
+ elif role in ("gpt", "assistant", "bot"):
65
+ gpt_parts.append(val)
66
+ if user_parts or gpt_parts:
67
+ return "\n".join(user_parts), "\n".join(gpt_parts)
68
+
69
+ # 2. OpenAI / ChatML format: messages = [{"role": "user", "content": "..."}, {"role": "assistant", "content": "..."}]
70
+ if "messages" in row and isinstance(row["messages"], (list, np.ndarray)):
71
+ turns = row["messages"]
72
+ user_parts, gpt_parts = [], []
73
+ for turn in turns:
74
+ if isinstance(turn, dict):
75
+ role = str(turn.get("role", "")).lower()
76
+ val = str(turn.get("content", ""))
77
+ if role in ("user", "human"):
78
+ user_parts.append(val)
79
+ elif role in ("assistant", "system", "gpt"):
80
+ gpt_parts.append(val)
81
+ if user_parts or gpt_parts:
82
+ return "\n".join(user_parts), "\n".join(gpt_parts)
83
+
84
+ # 3. Explicit prompt / response or instruction / output columns
85
+ q, a = "", ""
86
+ if "prompt" in row and pd.notna(row["prompt"]):
87
+ q = str(row["prompt"])
88
+ elif "instruction" in row and pd.notna(row["instruction"]):
89
+ q = str(row["instruction"])
90
+ elif "question" in row and pd.notna(row["question"]):
91
+ q = str(row["question"])
92
+
93
+ if "response" in row and pd.notna(row["response"]):
94
+ a = str(row["response"])
95
+ elif "output" in row and pd.notna(row["output"]):
96
+ a = str(row["output"])
97
+ elif "answer" in row and pd.notna(row["answer"]):
98
+ a = str(row["answer"])
99
+
100
+ if q or a:
101
+ return q, a
102
+
103
+ # 4. Fallback to raw text field if present
104
+ if "text" in row and pd.notna(row["text"]):
105
+ return str(row["text"]), ""
106
+
107
+ return "", ""
108
+
109
+
110
+ def calculate_similarity(text1: str, text2: str) -> float:
111
+ """Calculates sequence similarity ratio between two texts."""
112
+ if text1 == text2:
113
+ return 1.0
114
+ if not text1 or not text2:
115
+ return 0.0
116
+ # Quick length check optimization: if difference exceeds 2%, ratio cannot be >= 0.99
117
+ len1, len2 = len(text1), len(text2)
118
+ max_len = max(len1, len2)
119
+ if max_len > 0 and (abs(len1 - len2) / max_len) > 0.02:
120
+ return 0.0
121
+ return difflib.SequenceMatcher(None, text1, text2).ratio()
122
+
123
+
124
+ def main():
125
+ print(f"{Colors.HEADER}{'='*80}")
126
+ print(" PARQUET Q&A DUPLICATE SCANNER & DE-DUPLICATION PATCHER")
127
+ print(f"{'='*80}{Colors.ENDC}")
128
+
129
+ parquet_file = DEFAULT_PARQUET_FILE
130
+ if not os.path.isfile(parquet_file):
131
+ print(f"{Colors.FAIL}Error: File not found at '{parquet_file}'.{Colors.ENDC}")
132
+ print("Please verify that the current working directory contains 'dataset_cache/unified_dataset.parquet'.")
133
+ return
134
+
135
+ print(f"Loading Parquet from: {Colors.OKCYAN}{parquet_file}{Colors.ENDC} ...")
136
+ try:
137
+ df = pd.read_parquet(parquet_file)
138
+ except Exception as e:
139
+ print(f"{Colors.FAIL}Failed to read parquet file: {e}{Colors.ENDC}")
140
+ return
141
+
142
+ total_records = len(df)
143
+ print(f"Loaded {Colors.BOLD}{total_records}{Colors.ENDC} total records.\n")
144
+
145
+ print("Extracting question & answer contents from dialogue/records...")
146
+ qa_pairs = [extract_qa_pair(row) for _, row in df.iterrows()]
147
+ df['_extracted_q'] = [q for q, _ in qa_pairs]
148
+ df['_extracted_a'] = [a for _, a in qa_pairs]
149
+ df['_norm_q'] = df['_extracted_q'].apply(normalize_text)
150
+ df['_norm_a'] = df['_extracted_a'].apply(normalize_text)
151
+
152
+ # Filter out records where both Q and A are completely empty
153
+ valid_mask = (df['_norm_q'] != "") | (df['_norm_a'] != "")
154
+ df_eval = df[valid_mask].copy()
155
+
156
+ # Step 1: Exact matches via canonical combined hash key
157
+ df_eval['_qa_key'] = df_eval['_norm_q'] + " <||SEP||> " + df_eval['_norm_a']
158
+ exact_dup_groups = df_eval.groupby('_qa_key').filter(lambda g: len(g) > 1)
159
+
160
+ # Step 2: Near-duplicate pass (>= 99% similarity) among remaining rows
161
+ # We group by length bucket to avoid O(N^2) explosion
162
+ unique_keys = df_eval.drop_duplicates(subset=['_qa_key']).copy()
163
+ unique_keys['key_len'] = unique_keys['_qa_key'].str.len()
164
+
165
+ near_duplicates_map = {} # index -> canonical cluster parent index
166
+ sorted_unique = unique_keys.sort_values(by='key_len')
167
+ rows_list = sorted_unique[['_norm_q', '_norm_a', 'key_len']].to_dict('index')
168
+ indices = list(rows_list.keys())
169
+
170
+ print("Running 99% similarity verification pass on conversation text...")
171
+ for i in range(len(indices)):
172
+ idx_a = indices[i]
173
+ if idx_a in near_duplicates_map:
174
+ continue
175
+ row_a = rows_list[idx_a]
176
+ len_a = row_a['key_len']
177
+
178
+ # Compare only with items whose length differs by <= 2%
179
+ for j in range(i + 1, len(indices)):
180
+ idx_b = indices[j]
181
+ row_b = rows_list[idx_b]
182
+ len_b = row_b['key_len']
183
+
184
+ if len_b - len_a > max(3, int(len_a * 0.02)):
185
+ break
186
+
187
+ q_sim = calculate_similarity(row_a['_norm_q'], row_b['_norm_q'])
188
+ if q_sim >= SIMILARITY_THRESHOLD:
189
+ a_sim = calculate_similarity(row_a['_norm_a'], row_b['_norm_a'])
190
+ if a_sim >= SIMILARITY_THRESHOLD:
191
+ near_duplicates_map[idx_b] = idx_a
192
+
193
+ # Consolidate duplicate clusters
194
+ clusters: Dict[Any, List[int]] = {}
195
+
196
+ # Exact group assignments
197
+ for key, group in exact_dup_groups.groupby('_qa_key'):
198
+ primary_idx = group.index[0]
199
+ clusters[primary_idx] = group.index.tolist()
200
+
201
+ # Merge near-duplicate assignments
202
+ for dup_idx, canonical_idx in near_duplicates_map.items():
203
+ found = False
204
+ for root, members in clusters.items():
205
+ if canonical_idx in members or root == canonical_idx:
206
+ if dup_idx not in members:
207
+ members.append(dup_idx)
208
+ found = True
209
+ break
210
+ if not found:
211
+ clusters[canonical_idx] = [canonical_idx, dup_idx]
212
+
213
+ # Collect indices marked as duplicates to remove (all except the first in each cluster)
214
+ all_drop_indices = set()
215
+ for _, members in clusters.items():
216
+ for redundant in members[1:]:
217
+ all_drop_indices.add(redundant)
218
+
219
+ if not clusters:
220
+ print(f"\n{Colors.OKGREEN}🎉 Scan complete! No duplicate Q&A pairs (>=99% match) were found.{Colors.ENDC}")
221
+ return
222
+
223
+ redundant_count = len(all_drop_indices)
224
+ print(f"\n{Colors.WARNING}⚠️ Found {len(clusters)} duplicate clusters containing {redundant_count} redundant rows.{Colors.ENDC}")
225
+ print("=" * 80)
226
+ print("DUPLICATE GROUP AUDIT PREVIEW:")
227
+ print("=" * 80)
228
+
229
+ # Show preview of first 10 groups
230
+ preview_limit = 10
231
+ for g_num, (primary_idx, member_indices) in enumerate(list(clusters.items())[:preview_limit], 1):
232
+ q_prev = df.loc[primary_idx, '_extracted_q'].replace("\n", " ")[:90]
233
+ a_prev = df.loc[primary_idx, '_extracted_a'].replace("\n", " ")[:90]
234
+
235
+ print(f"\n{Colors.BOLD}Cluster #{g_num} (Matches {len(member_indices)} entries):{Colors.ENDC}")
236
+ print(f" {Colors.OKCYAN}Q:{Colors.ENDC} \"{q_prev}...\"")
237
+ print(f" {Colors.OKGREEN}A:{Colors.ENDC} \"{a_prev}...\"")
238
+ print(f" Row Indices: {member_indices} (Keep: {member_indices[0]}, Drop: {member_indices[1:]})")
239
+
240
+ if len(clusters) > preview_limit:
241
+ print(f"\n... and {len(clusters) - preview_limit} more duplicate groups not shown.")
242
+
243
+ print(f"\n{Colors.HEADER}{'='*80}{Colors.ENDC}")
244
+ print(f"Dataset summary before action:")
245
+ print(f" - Total entries: {len(df)}")
246
+ print(f" - Redundant entries to remove: {len(all_drop_indices)}")
247
+ print(f" - Cleaned dataset total entries: {len(df) - len(all_drop_indices)}")
248
+
249
+ # Prompt user for de-duplication action
250
+ choice = input(f"\n{Colors.BOLD}Would you like to de-duplicate and save the cleaned dataset? (y/N): {Colors.ENDC}").strip().lower()
251
+
252
+ if choice in ['y', 'yes']:
253
+ # 1. Create a backup
254
+ timestamp = time.strftime("%Y%m%d_%H%M%S")
255
+ backup_path = f"{parquet_file}.backup_{timestamp}"
256
+ print(f"\nCreating safety backup at: {backup_path}")
257
+ shutil.copy2(parquet_file, backup_path)
258
+
259
+ # 2. Drop redundant rows and temporary analysis columns
260
+ cleaned_df = df.drop(index=list(all_drop_indices)).reset_index(drop=True)
261
+ temp_cols = ['_extracted_q', '_extracted_a', '_norm_q', '_norm_a']
262
+ cleaned_df.drop(columns=[c for c in temp_cols if c in cleaned_df.columns], inplace=True)
263
+
264
+ # 3. Write back to parquet
265
+ print(f"Overwriting {parquet_file} with de-duplicated dataset...")
266
+ cleaned_df.to_parquet(parquet_file, index=False)
267
+
268
+ print(f"{Colors.OKGREEN}✅ Successfully saved cleaned parquet ({len(cleaned_df)} rows).{Colors.ENDC}")
269
+ print(f"Backup preserved at: {Colors.WARNING}{backup_path}{Colors.ENDC}")
270
+ else:
271
+ print(f"\n{Colors.WARNING}De-duplication aborted. No files were modified.{Colors.ENDC}")
272
+
273
+
274
+ if __name__ == "__main__":
275
+ main()