Naphula commited on
Commit
6f2444f
·
verified ·
1 Parent(s): a4c4002

Upload PMPF_duplicate_scanner.py

Browse files
Files changed (1) hide show
  1. PMPF_duplicate_scanner.py +161 -0
PMPF_duplicate_scanner.py ADDED
@@ -0,0 +1,161 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import glob
3
+ import json
4
+ import pandas as pd
5
+ import numpy as np
6
+
7
+ # ================= CONFIGURATION =================
8
+ DATASET_FOLDER = os.path.join(os.getcwd(), "dataset")
9
+ # =================================================
10
+
11
+ class Colors:
12
+ HEADER = '\033[95m'
13
+ OKGREEN = '\033[92m'
14
+ WARNING = '\033[93m'
15
+ FAIL = '\033[91m'
16
+ ENDC = '\033[0m'
17
+ BOLD = '\033[1m'
18
+
19
+ # Windows CMD color support fix
20
+ os.system('color')
21
+
22
+ def make_hashable(val):
23
+ """Converts dialogue structures or lists into a stable string for hashing."""
24
+ if val is None:
25
+ return ""
26
+ if isinstance(val, (list, np.ndarray, dict)):
27
+ try:
28
+ return json.dumps(val, sort_keys=True)
29
+ except Exception:
30
+ return str(val)
31
+ return str(val).strip()
32
+
33
+ def load_file_to_df(filepath):
34
+ """Loads various file formats and returns a unified DataFrame with row tracking."""
35
+ ext = os.path.splitext(filepath)[1].lower()
36
+ filename = os.path.basename(filepath)
37
+ df = pd.DataFrame()
38
+
39
+ try:
40
+ if ext in ['.txt', '.md']:
41
+ with open(filepath, 'r', encoding='utf-8', errors='ignore') as f:
42
+ lines = f.read().split("\n\n") # Split by double newline as rough sample block
43
+ df = pd.DataFrame({'text': lines})
44
+ df['row_index'] = df.index + 1
45
+
46
+ elif ext == '.parquet':
47
+ df = pd.read_parquet(filepath)
48
+ df['row_index'] = df.index
49
+
50
+ elif ext == '.csv':
51
+ df = pd.read_csv(filepath)
52
+ df['row_index'] = df.index + 2 # +2 accounts for header offset in CSV viewers
53
+
54
+ elif ext in ['.json', '.jsonl']:
55
+ try:
56
+ df = pd.read_json(filepath, lines=True)
57
+ except ValueError:
58
+ df = pd.read_json(filepath)
59
+ df['row_index'] = df.index
60
+
61
+ if not df.empty:
62
+ df['source_file'] = filename
63
+
64
+ except Exception as e:
65
+ print(f"{Colors.FAIL}Error reading {filename}: {e}{Colors.ENDC}")
66
+
67
+ return df
68
+
69
+ def main():
70
+ print(f"{Colors.HEADER}{'='*80}")
71
+ print(" CROSS-FILE DATASET DUPLICATE TRACE SCANNER")
72
+ print(f"{'='*80}{Colors.ENDC}")
73
+
74
+ if not os.path.exists(DATASET_FOLDER):
75
+ print(f"{Colors.FAIL}Error: Dataset folder '{DATASET_FOLDER}' not found.{Colors.ENDC}")
76
+ return
77
+
78
+ files = glob.glob(os.path.join(DATASET_FOLDER, "*.*"))
79
+ if not files:
80
+ print(f"{Colors.WARNING}No files found in '{DATASET_FOLDER}'.{Colors.ENDC}")
81
+ return
82
+
83
+ # 1. Load and aggregate all files
84
+ all_data = []
85
+ print("Loading and indexing raw dataset files...")
86
+ for f in files:
87
+ df = load_file_to_df(f)
88
+ if not df.empty:
89
+ print(f" -> Indexed: {os.path.basename(f)} ({len(df)} rows)")
90
+ all_data.append(df)
91
+
92
+ if not all_data:
93
+ print(f"{Colors.FAIL}No valid data could be parsed.{Colors.ENDC}")
94
+ return
95
+
96
+ # Combine everything into a global tracking DataFrame
97
+ master_df = pd.concat(all_data, ignore_index=True)
98
+
99
+ # Determine which column to audit
100
+ target_column = None
101
+ for col in ['conversations', 'messages', 'text']:
102
+ if col in master_df.columns:
103
+ target_column = col
104
+ break
105
+
106
+ if not target_column:
107
+ # Fallback to the first non-meta column
108
+ meta_cols = {'source_file', 'row_index'}
109
+ valid_cols = [c for c in master_df.columns if c not in meta_cols]
110
+ target_column = valid_cols[0] if valid_cols else master_df.columns[0]
111
+
112
+ print(f"\nAuditing duplicates using key column: '{Colors.BOLD}{target_column}{Colors.ENDC}'")
113
+
114
+ # Hash values for exact matching
115
+ master_df['_hash'] = master_df[target_column].apply(make_hashable)
116
+
117
+ # 2. Find duplicates (excluding empty rows)
118
+ master_df = master_df[master_df['_hash'] != ""]
119
+
120
+ # Group by the hash and find groups with size > 1
121
+ dup_groups = master_df.groupby('_hash').filter(lambda x: len(x) > 1)
122
+
123
+ if dup_groups.empty:
124
+ print(f"\n{Colors.OKGREEN}🎉 Scan complete! No duplicate records found across any of the files.{Colors.ENDC}")
125
+ return
126
+
127
+ # 3. Report detailed trace
128
+ unique_dupe_hashes = dup_groups['_hash'].unique()
129
+ total_dupes_count = len(dup_groups) - len(unique_dupe_hashes)
130
+
131
+ print(f"\n{Colors.WARNING}⚠️ Found {len(unique_dupe_hashes)} unique texts that have duplicated copies ({total_dupes_count} redundant entries total).{Colors.ENDC}")
132
+ print("="*80)
133
+ print("DETAILED DUPLICATE TRACE REPORT:")
134
+ print("="*80)
135
+
136
+ # Show up to 15 duplicate groups to keep console readable
137
+ max_groups_to_show = 15
138
+ for idx, h in enumerate(unique_dupe_hashes[:max_groups_to_show]):
139
+ group = dup_groups[dup_groups['_hash'] == h]
140
+
141
+ # Extract a clean text preview
142
+ raw_val = group.iloc[0][target_column]
143
+ if isinstance(raw_val, list): # ShareGPT format preview
144
+ preview = f"[ShareGPT Conversation with {len(raw_val)} turns]"
145
+ else:
146
+ preview = str(raw_val).replace('\n', ' ')[:80] + "..."
147
+
148
+ print(f"\n{Colors.BOLD}Group #{idx+1} | Duplicate Content Preview:{Colors.ENDC}")
149
+ print(f" \"{Colors.OKGREEN}{preview}{Colors.ENDC}\"")
150
+ print(f" {Colors.BOLD}Found in following locations:{Colors.ENDC}")
151
+
152
+ for _, row in group.iterrows():
153
+ print(f" - File: {Colors.WARNING}{row['source_file']}{Colors.ENDC} | Row/Index: {row['row_index']}")
154
+
155
+ if len(unique_dupe_hashes) > max_groups_to_show:
156
+ print(f"\n... and {len(unique_dupe_hashes) - max_groups_to_show} more duplicate groups.")
157
+
158
+ print(f"\n{Colors.HEADER}{'='*80}{Colors.ENDC}")
159
+
160
+ if __name__ == "__main__":
161
+ main()