File size: 5,943 Bytes
6f2444f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
import os
import glob
import json
import pandas as pd
import numpy as np

# ================= CONFIGURATION =================
DATASET_FOLDER = os.path.join(os.getcwd(), "dataset")
# =================================================

class Colors:
    HEADER = '\033[95m'
    OKGREEN = '\033[92m'
    WARNING = '\033[93m'
    FAIL = '\033[91m'
    ENDC = '\033[0m'
    BOLD = '\033[1m'

# Windows CMD color support fix
os.system('color')

def make_hashable(val):
    """Converts dialogue structures or lists into a stable string for hashing."""
    if val is None:
        return ""
    if isinstance(val, (list, np.ndarray, dict)):
        try:
            return json.dumps(val, sort_keys=True)
        except Exception:
            return str(val)
    return str(val).strip()

def load_file_to_df(filepath):
    """Loads various file formats and returns a unified DataFrame with row tracking."""
    ext = os.path.splitext(filepath)[1].lower()
    filename = os.path.basename(filepath)
    df = pd.DataFrame()
    
    try:
        if ext in ['.txt', '.md']:
            with open(filepath, 'r', encoding='utf-8', errors='ignore') as f:
                lines = f.read().split("\n\n")  # Split by double newline as rough sample block
            df = pd.DataFrame({'text': lines})
            df['row_index'] = df.index + 1
            
        elif ext == '.parquet':
            df = pd.read_parquet(filepath)
            df['row_index'] = df.index
            
        elif ext == '.csv':
            df = pd.read_csv(filepath)
            df['row_index'] = df.index + 2  # +2 accounts for header offset in CSV viewers
            
        elif ext in ['.json', '.jsonl']:
            try:
                df = pd.read_json(filepath, lines=True)
            except ValueError:
                df = pd.read_json(filepath)
            df['row_index'] = df.index
            
        if not df.empty:
            df['source_file'] = filename
            
    except Exception as e:
        print(f"{Colors.FAIL}Error reading {filename}: {e}{Colors.ENDC}")
        
    return df

def main():
    print(f"{Colors.HEADER}{'='*80}")
    print("             CROSS-FILE DATASET DUPLICATE TRACE SCANNER")
    print(f"{'='*80}{Colors.ENDC}")

    if not os.path.exists(DATASET_FOLDER):
        print(f"{Colors.FAIL}Error: Dataset folder '{DATASET_FOLDER}' not found.{Colors.ENDC}")
        return

    files = glob.glob(os.path.join(DATASET_FOLDER, "*.*"))
    if not files:
        print(f"{Colors.WARNING}No files found in '{DATASET_FOLDER}'.{Colors.ENDC}")
        return

    # 1. Load and aggregate all files
    all_data = []
    print("Loading and indexing raw dataset files...")
    for f in files:
        df = load_file_to_df(f)
        if not df.empty:
            print(f"  -> Indexed: {os.path.basename(f)} ({len(df)} rows)")
            all_data.append(df)

    if not all_data:
        print(f"{Colors.FAIL}No valid data could be parsed.{Colors.ENDC}")
        return

    # Combine everything into a global tracking DataFrame
    master_df = pd.concat(all_data, ignore_index=True)
    
    # Determine which column to audit
    target_column = None
    for col in ['conversations', 'messages', 'text']:
        if col in master_df.columns:
            target_column = col
            break
            
    if not target_column:
        # Fallback to the first non-meta column
        meta_cols = {'source_file', 'row_index'}
        valid_cols = [c for c in master_df.columns if c not in meta_cols]
        target_column = valid_cols[0] if valid_cols else master_df.columns[0]

    print(f"\nAuditing duplicates using key column: '{Colors.BOLD}{target_column}{Colors.ENDC}'")
    
    # Hash values for exact matching
    master_df['_hash'] = master_df[target_column].apply(make_hashable)
    
    # 2. Find duplicates (excluding empty rows)
    master_df = master_df[master_df['_hash'] != ""]
    
    # Group by the hash and find groups with size > 1
    dup_groups = master_df.groupby('_hash').filter(lambda x: len(x) > 1)
    
    if dup_groups.empty:
        print(f"\n{Colors.OKGREEN}🎉 Scan complete! No duplicate records found across any of the files.{Colors.ENDC}")
        return

    # 3. Report detailed trace
    unique_dupe_hashes = dup_groups['_hash'].unique()
    total_dupes_count = len(dup_groups) - len(unique_dupe_hashes)
    
    print(f"\n{Colors.WARNING}⚠️ Found {len(unique_dupe_hashes)} unique texts that have duplicated copies ({total_dupes_count} redundant entries total).{Colors.ENDC}")
    print("="*80)
    print("DETAILED DUPLICATE TRACE REPORT:")
    print("="*80)

    # Show up to 15 duplicate groups to keep console readable
    max_groups_to_show = 15
    for idx, h in enumerate(unique_dupe_hashes[:max_groups_to_show]):
        group = dup_groups[dup_groups['_hash'] == h]
        
        # Extract a clean text preview
        raw_val = group.iloc[0][target_column]
        if isinstance(raw_val, list):  # ShareGPT format preview
            preview = f"[ShareGPT Conversation with {len(raw_val)} turns]"
        else:
            preview = str(raw_val).replace('\n', ' ')[:80] + "..."

        print(f"\n{Colors.BOLD}Group #{idx+1} | Duplicate Content Preview:{Colors.ENDC}")
        print(f"  \"{Colors.OKGREEN}{preview}{Colors.ENDC}\"")
        print(f"  {Colors.BOLD}Found in following locations:{Colors.ENDC}")
        
        for _, row in group.iterrows():
            print(f"    - File: {Colors.WARNING}{row['source_file']}{Colors.ENDC} | Row/Index: {row['row_index']}")

    if len(unique_dupe_hashes) > max_groups_to_show:
        print(f"\n... and {len(unique_dupe_hashes) - max_groups_to_show} more duplicate groups.")

    print(f"\n{Colors.HEADER}{'='*80}{Colors.ENDC}")

if __name__ == "__main__":
    main()