Download data_preprocessor.py from razz-123/disease_detection: direct link, hf CLI and curl.
- Browser
- Download file 2.41 kB
-
https://huggingface.co/razz-123/disease_detection/resolve/main/data_preprocessor.py
- Command line
-
hf download hf://razz-123/disease_detection/data_preprocessor.py
-
curl -L -o data_preprocessor.py https://huggingface.co/razz-123/disease_detection/resolve/main/data_preprocessor.py
2.41 kB
| # data_preprocessor.py | |
| import hashlib | |
| import re | |
| from collections import Counter | |
| class DataPreprocessor: | |
| def __init__(self): | |
| self.seen_hashes = set() | |
| def deduplicate(self, texts): | |
| """Remove exact duplicate lines.""" | |
| unique_texts = [] | |
| for text in texts: | |
| text_hash = hashlib.md5(text.encode()).hexdigest() | |
| if text_hash not in self.seen_hashes: | |
| self.seen_hashes.add(text_hash) | |
| unique_texts.append(text) | |
| print(f"Removed {len(texts) - len(unique_texts)} duplicates.") | |
| return unique_texts | |
| def filter_quality(self, texts, min_words=5, max_word_ratio=0.9): | |
| """Filter short texts and those with too many repeated words.""" | |
| filtered = [] | |
| for text in texts: | |
| words = text.split() | |
| if len(words) < min_words: | |
| continue | |
| # Check if > 90% of words are the same (spam) | |
| if len(words) > 0 and max(Counter(words).values()) / len(words) > max_word_ratio: | |
| continue | |
| filtered.append(text) | |
| print(f"Removed {len(texts) - len(filtered)} low-quality texts.") | |
| return filtered | |
| def remove_toxic(self, texts, toxic_keywords=None): | |
| """Basic toxicity filter.""" | |
| if toxic_keywords is None: | |
| toxic_keywords = ["hate", "violence", "kill", "racist", "sexist", "nazi"] | |
| filtered = [] | |
| for text in texts: | |
| lower_text = text.lower() | |
| if any(kw in lower_text for kw in toxic_keywords): | |
| continue | |
| filtered.append(text) | |
| print(f"Removed {len(texts) - len(filtered)} potentially toxic texts.") | |
| return filtered | |
| def preprocess_file(self, input_path, output_path): | |
| with open(input_path, 'r', encoding='utf-8') as f: | |
| texts = [line.strip() for line in f if line.strip()] | |
| texts = self.deduplicate(texts) | |
| texts = self.filter_quality(texts) | |
| texts = self.remove_toxic(texts) | |
| with open(output_path, 'w', encoding='utf-8') as f: | |
| for text in texts: | |
| f.write(text + '\n') | |
| print(f"Saved cleaned data to {output_path}") | |
| # Usage: | |
| # preprocessor = DataPreprocessor() | |
| # preprocessor.preprocess_file("data/raw_train.txt", "data/train.txt") |