Spaces:
Runtime error
Runtime error
Download scripts/utils/remove_duplicates.py from rb1337/Phishing-Detection-System: direct link, hf CLI and curl.
- Browser
- Download file 691 Bytes
-
https://huggingface.co/spaces/rb1337/Phishing-Detection-System/resolve/main/scripts/utils/remove_duplicates.py
- Command line
-
hf download hf://spaces/rb1337/Phishing-Detection-System/scripts/utils/remove_duplicates.py
-
curl -L -o remove_duplicates.py https://huggingface.co/spaces/rb1337/Phishing-Detection-System/resolve/main/scripts/utils/remove_duplicates.py
691 Bytes
| """ | |
| Remove duplicates from clean_dataset.csv | |
| """ | |
| import pandas as pd | |
| # Load dataset | |
| df = pd.read_csv('data/processed/clean_dataset.csv') | |
| print(f"Original: {len(df):,} URLs") | |
| # Check duplicates | |
| print(f"Duplicates: {df.duplicated(subset='url').sum():,}") | |
| # Keep first occurrence of each URL | |
| df_clean = df.drop_duplicates(subset='url', keep='first') | |
| print(f"After removing duplicates: {len(df_clean):,} URLs") | |
| # Check label distribution | |
| print(f"\nLabel distribution:") | |
| print(df_clean['label'].value_counts()) | |
| # Save | |
| df_clean.to_csv('data/processed/clean_dataset_no_duplicates.csv', index=False) | |
| print(f"\n✓ Saved to: data/processed/clean_dataset_no_duplicates.csv") |