ViuAI commited on
Commit
30b674c
·
verified ·
1 Parent(s): bc2430b

docs/feat: sync translation & hinglish expansion for data/scripts/add_curated_hinglish.py

Browse files
data/scripts/add_curated_hinglish.py ADDED
@@ -0,0 +1,97 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Add Curated High-Quality Hinglish Dialogues & Instruction Datasets
3
+ to ViuAI/viu-mini-raw-pretrain Hub via server-side CommitOperationCopy.
4
+ Datasets:
5
+ 1. NebulaByte/alpaca-gpt4-hindi-hinglish (56.89 MB)
6
+ 2. rvv-karma/English-Hinglish-TOP (25.27 MB)
7
+ 3. justinj92/hinglish_sharegpt_v0.1 (12.63 MB)
8
+ 4. rvv-karma/English-Hinglish (11.16 MB)
9
+ 5. harshraj/dolly15k_hinglish_dataset_cleaned (8.62 MB)
10
+ 6. fhai50032/HINGLISH-LIMA (3.04 MB)
11
+ """
12
+ import sys
13
+ import time
14
+ from huggingface_hub import HfApi, CommitOperationCopy
15
+
16
+ sys.stdout.reconfigure(encoding='utf-8')
17
+
18
+ CURATED_HINGLISH_SOURCES = [
19
+ {
20
+ "repo": "NebulaByte/alpaca-gpt4-hindi-hinglish",
21
+ "src_path": "data/train-00000-of-00001-d18143b18ee5a0f5.parquet",
22
+ "dst_filename": "alpaca-gpt4-hinglish.parquet"
23
+ },
24
+ {
25
+ "repo": "rvv-karma/English-Hinglish-TOP",
26
+ "src_path": "data/train-00000-of-00001.parquet",
27
+ "dst_filename": "english-hinglish-top-train.parquet"
28
+ },
29
+ {
30
+ "repo": "justinj92/hinglish_sharegpt_v0.1",
31
+ "src_path": "data/train-00000-of-00001.parquet",
32
+ "dst_filename": "sharegpt-hinglish-v0.1.parquet"
33
+ },
34
+ {
35
+ "repo": "rvv-karma/English-Hinglish",
36
+ "src_path": "data/train-00000-of-00001.parquet",
37
+ "dst_filename": "english-hinglish-train.parquet"
38
+ },
39
+ {
40
+ "repo": "harshraj/dolly15k_hinglish_dataset_cleaned",
41
+ "src_path": "data/train-00000-of-00001.parquet",
42
+ "dst_filename": "dolly15k-hinglish.parquet"
43
+ },
44
+ {
45
+ "repo": "fhai50032/HINGLISH-LIMA",
46
+ "src_path": "data/train-00000-of-00001.parquet",
47
+ "dst_filename": "lima-hinglish.parquet"
48
+ }
49
+ ]
50
+
51
+ def main():
52
+ api = HfApi()
53
+ target_repo = "ViuAI/viu-mini-raw-pretrain"
54
+ target_folder = "hinglish"
55
+
56
+ print("=== Adding Curated Hinglish Datasets to Pretraining Hub ===")
57
+ print(f"Target: {target_repo}/{target_folder}/")
58
+
59
+ target_files = set(api.list_repo_files(target_repo, repo_type="dataset"))
60
+ operations = []
61
+
62
+ for item in CURATED_HINGLISH_SOURCES:
63
+ repo = item["repo"]
64
+ src_path = item["src_path"]
65
+ dst_path = f"{target_folder}/{item['dst_filename']}"
66
+
67
+ if dst_path in target_files:
68
+ print(f" [Already Present] {dst_path}")
69
+ continue
70
+
71
+ print(f" [Queued] {repo}/{src_path} -> {dst_path}")
72
+ operations.append(
73
+ CommitOperationCopy(
74
+ src_path_in_repo=src_path,
75
+ path_in_repo=dst_path,
76
+ src_repo_id=repo,
77
+ src_repo_type="dataset"
78
+ )
79
+ )
80
+
81
+ if not operations:
82
+ print("[ok] All curated Hinglish datasets already present!")
83
+ return
84
+
85
+ print(f"\nCommitting {len(operations)} server-side copies to {target_repo}...")
86
+ start_t = time.time()
87
+ api.create_commit(
88
+ repo_id=target_repo,
89
+ repo_type="dataset",
90
+ operations=operations,
91
+ commit_message="feat(hinglish): ingest curated multi-turn dialogue, Alpaca, ShareGPT & Dolly Hinglish datasets"
92
+ )
93
+ elapsed = time.time() - start_t
94
+ print(f"[SUCCESS] Curated Hinglish shards committed in {elapsed:.2f}s!")
95
+
96
+ if __name__ == "__main__":
97
+ main()