| """ |
| Ingest Open-Source Real-World Toxicity, Hate Speech & Profanity Datasets |
| to `toxicity/` folder on `ViuAI/viu-mini-raw-pretrain` via server-side CommitOperationCopy. |
| |
| Datasets Included: |
| 1. google/civil_comments (Train Parquets ~320 MB - 2M real internet toxic, obscene & abusive comments) |
| 2. tdavidson/hate_speech_offensive (Parquet ~1.56 MB - 24k real offensive & hate speech tweets) |
| 3. pankajbiswas6/prism-hinglish-hate-speech (CSV ~8 MB - Real-world Hinglish toxic & abusive comments) |
| 4. nikitadesai/hasoc (CSV ~4.95 MB - Official HASOC Hindi/English offensive text) |
| 5. Paul/hatecheck-hindi (CSV ~0.82 MB - Verified Hindi hate speech & profanity test set) |
| """ |
| import sys |
| import time |
| from huggingface_hub import HfApi, CommitOperationCopy |
|
|
| sys.stdout.reconfigure(encoding='utf-8') |
|
|
| OPEN_SOURCE_TOXICITY_ITEMS = [ |
| { |
| "src_repo": "google/civil_comments", |
| "src_path": "data/train-00000-of-00002.parquet", |
| "dst_path": "toxicity/civil_comments_train_00000.parquet" |
| }, |
| { |
| "src_repo": "google/civil_comments", |
| "src_path": "data/train-00001-of-00002.parquet", |
| "dst_path": "toxicity/civil_comments_train_00001.parquet" |
| }, |
| { |
| "src_repo": "tdavidson/hate_speech_offensive", |
| "src_path": "data/train-00000-of-00001.parquet", |
| "dst_path": "toxicity/hate_speech_offensive_train.parquet" |
| }, |
| { |
| "src_repo": "pankajbiswas6/prism-hinglish-hate-speech", |
| "src_path": "data/train.csv", |
| "dst_path": "toxicity/prism_hinglish_hate_train.csv" |
| }, |
| { |
| "src_repo": "nikitadesai/hasoc", |
| "src_path": "traindata-basic.csv", |
| "dst_path": "toxicity/hasoc_hindi_offensive.csv" |
| }, |
| { |
| "src_repo": "Paul/hatecheck-hindi", |
| "src_path": "test.csv", |
| "dst_path": "toxicity/hatecheck_hindi.csv" |
| } |
| ] |
|
|
| def main(): |
| api = HfApi() |
| target_repo = "ViuAI/viu-mini-raw-pretrain" |
|
|
| print("=== Ingesting Open-Source Internet Toxicity & Gaali Datasets ===") |
| print(f"Target Repo: {target_repo}/toxicity/") |
|
|
| target_files = set(api.list_repo_files(target_repo, repo_type="dataset")) |
| operations = [] |
|
|
| for item in OPEN_SOURCE_TOXICITY_ITEMS: |
| dst = item["dst_path"] |
| if dst in target_files: |
| print(f" [Already Present] {dst}") |
| continue |
| print(f" [Queued] {item['src_repo']}/{item['src_path']} -> {dst}") |
| operations.append( |
| CommitOperationCopy( |
| src_path_in_repo=item["src_path"], |
| path_in_repo=dst, |
| src_repo_id=item["src_repo"], |
| src_repo_type="dataset" |
| ) |
| ) |
|
|
| if not operations: |
| print("[ok] All open-source toxicity files are already present in repo!") |
| return |
|
|
| print(f"\nCommitting {len(operations)} server-side copies to {target_repo}...") |
| start_t = time.time() |
| api.create_commit( |
| repo_id=target_repo, |
| repo_type="dataset", |
| operations=operations, |
| commit_message="feat(toxicity): ingest open-source Google Civil Comments, HASOC Hindi, Hinglish Prism & Davidson hate speech datasets" |
| ) |
| elapsed = time.time() - start_t |
| print(f"[SUCCESS] All open-source toxicity datasets committed in {elapsed:.2f}s!") |
|
|
| if __name__ == "__main__": |
| main() |
|
|
