File size: 2,406 Bytes
042eb00
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
# data_preprocessor.py
import hashlib
import re
from collections import Counter

class DataPreprocessor:
    def __init__(self):
        self.seen_hashes = set()

    def deduplicate(self, texts):
        """Remove exact duplicate lines."""
        unique_texts = []
        for text in texts:
            text_hash = hashlib.md5(text.encode()).hexdigest()
            if text_hash not in self.seen_hashes:
                self.seen_hashes.add(text_hash)
                unique_texts.append(text)
        print(f"Removed {len(texts) - len(unique_texts)} duplicates.")
        return unique_texts

    def filter_quality(self, texts, min_words=5, max_word_ratio=0.9):
        """Filter short texts and those with too many repeated words."""
        filtered = []
        for text in texts:
            words = text.split()
            if len(words) < min_words:
                continue
            # Check if > 90% of words are the same (spam)
            if len(words) > 0 and max(Counter(words).values()) / len(words) > max_word_ratio:
                continue
            filtered.append(text)
        print(f"Removed {len(texts) - len(filtered)} low-quality texts.")
        return filtered

    def remove_toxic(self, texts, toxic_keywords=None):
        """Basic toxicity filter."""
        if toxic_keywords is None:
            toxic_keywords = ["hate", "violence", "kill", "racist", "sexist", "nazi"]
        filtered = []
        for text in texts:
            lower_text = text.lower()
            if any(kw in lower_text for kw in toxic_keywords):
                continue
            filtered.append(text)
        print(f"Removed {len(texts) - len(filtered)} potentially toxic texts.")
        return filtered

    def preprocess_file(self, input_path, output_path):
        with open(input_path, 'r', encoding='utf-8') as f:
            texts = [line.strip() for line in f if line.strip()]
        
        texts = self.deduplicate(texts)
        texts = self.filter_quality(texts)
        texts = self.remove_toxic(texts)
        
        with open(output_path, 'w', encoding='utf-8') as f:
            for text in texts:
                f.write(text + '\n')
        print(f"Saved cleaned data to {output_path}")

# Usage:
# preprocessor = DataPreprocessor()
# preprocessor.preprocess_file("data/raw_train.txt", "data/train.txt")