ViuAI commited on
Commit
ca926e4
·
verified ·
1 Parent(s): aabd6f9

fix: exclude non-5col schema paths from HF stream (wikipedia/grammar/gsm8k CastError on shuffle)

Browse files
.gitattributes CHANGED
@@ -4,5 +4,7 @@
4
  *.onnx filter=lfs diff=lfs merge=lfs -text
5
  tokenizer/outputs/*.json filter=lfs diff=lfs merge=lfs -text
6
  data/processed/* filter=lfs diff=lfs merge=lfs -text
7
- data/processed_spiritual/train-p1p2_boost_sanatan_dharma_epics_upanishads-00000.parquet filter=lfs diff=lfs merge=lfs -text
8
- data/processed_spiritual/train-p1p2_boost_sanatan_dharma_gita_ramayana-00000.parquet filter=lfs diff=lfs merge=lfs -text
 
 
 
4
  *.onnx filter=lfs diff=lfs merge=lfs -text
5
  tokenizer/outputs/*.json filter=lfs diff=lfs merge=lfs -text
6
  data/processed/* filter=lfs diff=lfs merge=lfs -text
7
+ data/processed_spiritual/train-p1p2_boost_hindi_math_gsm8k-00000.parquet filter=lfs diff=lfs merge=lfs -text
8
+ data/processed_spiritual/train-p1p2_boost_hinduism_philosophy_qa-00000.parquet filter=lfs diff=lfs merge=lfs -text
9
+ data/processed_spiritual/train-p1p2_boost_hinglish_arena_chat-00000.parquet filter=lfs diff=lfs merge=lfs -text
10
+ data/processed_spiritual/train-p1p2_boost_indian_history_qa-00000.parquet filter=lfs diff=lfs merge=lfs -text
data/processed_spiritual/train-p1p2_boost_hindi_math_gsm8k-00000.parquet ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:35984a6198c38870c91b507e4436b95a76f7b271eba12afdab6156dc15974140
3
+ size 3216469
data/processed_spiritual/train-p1p2_boost_hinduism_philosophy_qa-00000.parquet CHANGED
Binary files a/data/processed_spiritual/train-p1p2_boost_hinduism_philosophy_qa-00000.parquet and b/data/processed_spiritual/train-p1p2_boost_hinduism_philosophy_qa-00000.parquet differ
 
data/processed_spiritual/train-p1p2_boost_hinglish_arena_chat-00000.parquet ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:51b7aa53e13f6784cca09cd46b9c6c9c410770da5ffa8633f8a2a71a01f35f02
3
+ size 6913323
data/processed_spiritual/train-p1p2_boost_indian_history_qa-00000.parquet ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:896cfefd36fb018ebf1412153c325d114871b0c521ecd56ada0e8b76ac59d1f9
3
+ size 650283
data/scripts/collect_spiritual_and_boost_corpus.py CHANGED
@@ -19,14 +19,14 @@ import os
19
  import sys
20
  import json
21
  import time
 
22
  import argparse
23
  from pathlib import Path
24
  import pandas as pd
25
  import pyarrow as pa
26
  import pyarrow.parquet as pq
27
- from huggingface_hub import HfApi, hf_hub_download
28
 
29
- # Enforce pure English logging
30
  print("======================================================================")
31
  print("Starting Sanatan Dharma & Knowledge Corpus Collector (ViuMini-MoE-242M)")
32
  print("======================================================================")
@@ -46,6 +46,11 @@ SCHEMA = pa.schema([
46
  def get_hf_token():
47
  """Retrieve Hugging Face token securely from environment or secret stores."""
48
  token = os.environ.get("HF_TOKEN") or os.environ.get("HUGGING_FACE_HUB_TOKEN")
 
 
 
 
 
49
  if not token:
50
  try:
51
  from kaggle_secrets import UserSecretsClient
@@ -318,26 +323,16 @@ def collect_hinduism_philosophy_qa():
318
  for item in data:
319
  title = str(item.get("Title", "")).strip()
320
  q_body = str(item.get("q_body", "")).strip()
321
- # Clean HTML tags simply
322
- import re
323
  clean_q = re.sub(r"<[^>]+>", " ", q_body).strip()
324
  clean_q = " ".join(clean_q.split())
325
 
326
- answers = item.get("Answers", [])
327
- ans_text = ""
328
- if isinstance(answers, list) and len(answers) > 0:
329
- # Find accepted answer or highest score answer
330
- best_ans = answers[0]
331
- for ans in answers:
332
- if ans.get("Id") == item.get("AcceptedAnswerId"):
333
- best_ans = ans
334
- break
335
- ans_body = str(best_ans.get("a_body", "")).strip()
336
- ans_text = re.sub(r"<[^>]+>", " ", ans_body).strip()
337
- ans_text = " ".join(ans_text.split())
338
-
339
- if title and ans_text and len(ans_text) > 40:
340
- text = f"<|user|> Question on Hindu Philosophy & Scriptures: {title}\n{clean_q}\n<|assistant|> <soch> Analyzing Hindu scriptures, context, and philosophical principles... </soch> {ans_text}"
341
  records.append({
342
  "text": text,
343
  "lang": "en",
@@ -408,8 +403,8 @@ def collect_hinglish_and_reasoning_boost():
408
  url_hist = "https://huggingface.co/datasets/kaifahmad/indian-history-hindi-QA-3.4k/resolve/main/data/train-00000-of-00001.parquet"
409
  df_hist = pd.read_parquet(url_hist)
410
  for _, row in df_hist.iterrows():
411
- q = str(row.get("question", "")).strip()
412
- a = str(row.get("answer", "")).strip()
413
  if q and a:
414
  text = f"<|user|> भारतीय इतिहास प्रश्न: {q}\n<|assistant|> {a}"
415
  records_hist.append({
@@ -451,20 +446,26 @@ def collect_hinglish_and_reasoning_boost():
451
  def upload_shards_to_hub(shards):
452
  """Upload all newly created boost shards to Hugging Face dataset repository."""
453
  print("\n[Step 5/5] Uploading Shards to Hugging Face Dataset Hub...")
454
- if not TOKEN:
455
- print(" - [Error] HF_TOKEN not found! Cannot push to Hub.")
 
456
  return False
457
 
 
 
458
  for file_path, row_count in shards:
 
 
 
459
  hub_path = f"domains/{file_path.name}"
460
  print(f" - Uploading {file_path.name} ({row_count:,} rows) -> {DATA_REPO}/{hub_path}...")
461
  try:
462
- api.upload_file(
463
  path_or_fileobj=str(file_path),
464
  path_in_repo=hub_path,
465
  repo_id=DATA_REPO,
466
  repo_type="dataset",
467
- commit_message=f"feat: add spiritual & reasoning boost {file_path.name} ({row_count:,} rows)"
468
  )
469
  print(f" ✓ Uploaded successfully!")
470
  except Exception as exc:
@@ -495,7 +496,7 @@ def main():
495
  all_shards.extend(more_shards)
496
 
497
  total_rows = sum(cnt for _, cnt in all_shards)
498
- total_bytes = sum(f.stat().st_size for f, _ in all_shards)
499
 
500
  print("\n" + "=" * 70)
501
  print("Collection Summary:")
 
19
  import sys
20
  import json
21
  import time
22
+ import re
23
  import argparse
24
  from pathlib import Path
25
  import pandas as pd
26
  import pyarrow as pa
27
  import pyarrow.parquet as pq
28
+ from huggingface_hub import HfApi, hf_hub_download, get_token
29
 
 
30
  print("======================================================================")
31
  print("Starting Sanatan Dharma & Knowledge Corpus Collector (ViuMini-MoE-242M)")
32
  print("======================================================================")
 
46
  def get_hf_token():
47
  """Retrieve Hugging Face token securely from environment or secret stores."""
48
  token = os.environ.get("HF_TOKEN") or os.environ.get("HUGGING_FACE_HUB_TOKEN")
49
+ if not token:
50
+ try:
51
+ token = get_token()
52
+ except Exception:
53
+ pass
54
  if not token:
55
  try:
56
  from kaggle_secrets import UserSecretsClient
 
323
  for item in data:
324
  title = str(item.get("Title", "")).strip()
325
  q_body = str(item.get("q_body", "")).strip()
326
+ aa_body = str(item.get("aa_body", "")).strip()
327
+
328
  clean_q = re.sub(r"<[^>]+>", " ", q_body).strip()
329
  clean_q = " ".join(clean_q.split())
330
 
331
+ clean_a = re.sub(r"<[^>]+>", " ", aa_body).strip()
332
+ clean_a = " ".join(clean_a.split())
333
+
334
+ if title and clean_a and len(clean_a) > 30:
335
+ text = f"<|user|> Question on Hindu Philosophy & Scriptures: {title}\n{clean_q}\n<|assistant|> <soch> Analyzing Hindu scriptures, context, and philosophical principles... </soch> {clean_a}"
 
 
 
 
 
 
 
 
 
 
336
  records.append({
337
  "text": text,
338
  "lang": "en",
 
403
  url_hist = "https://huggingface.co/datasets/kaifahmad/indian-history-hindi-QA-3.4k/resolve/main/data/train-00000-of-00001.parquet"
404
  df_hist = pd.read_parquet(url_hist)
405
  for _, row in df_hist.iterrows():
406
+ q = str(row.get("Question", "") or row.get("question", "")).strip()
407
+ a = str(row.get("Answer", "") or row.get("answer", "")).strip()
408
  if q and a:
409
  text = f"<|user|> भारतीय इतिहास प्रश्न: {q}\n<|assistant|> {a}"
410
  records_hist.append({
 
446
  def upload_shards_to_hub(shards):
447
  """Upload all newly created boost shards to Hugging Face dataset repository."""
448
  print("\n[Step 5/5] Uploading Shards to Hugging Face Dataset Hub...")
449
+ token = get_hf_token()
450
+ if not token:
451
+ print(" - [Error] Token not found! Cannot push to Hub.")
452
  return False
453
 
454
+ api_client = HfApi(token=token)
455
+
456
  for file_path, row_count in shards:
457
+ if row_count == 0:
458
+ print(f" - Skipping empty shard {file_path.name}")
459
+ continue
460
  hub_path = f"domains/{file_path.name}"
461
  print(f" - Uploading {file_path.name} ({row_count:,} rows) -> {DATA_REPO}/{hub_path}...")
462
  try:
463
+ api_client.upload_file(
464
  path_or_fileobj=str(file_path),
465
  path_in_repo=hub_path,
466
  repo_id=DATA_REPO,
467
  repo_type="dataset",
468
+ commit_message=f"feat: add boost {file_path.name} ({row_count:,} rows)"
469
  )
470
  print(f" ✓ Uploaded successfully!")
471
  except Exception as exc:
 
496
  all_shards.extend(more_shards)
497
 
498
  total_rows = sum(cnt for _, cnt in all_shards)
499
+ total_bytes = sum(f.stat().st_size for f, cnt in all_shards if cnt > 0)
500
 
501
  print("\n" + "=" * 70)
502
  print("Collection Summary:")
docs/PROGRESS.md CHANGED
@@ -348,8 +348,9 @@ Kaggle par manual file upload se version mismatch hota hai (v1 tokenizer wala ca
348
  - PASS: bootstrap, 2xT4, tok v2 mask_id=4, `rank 0/2 + 1/2`, `[dist broadcast r0->all]` (micro=4 accum=4), DDP wrapped synced, model 241.7M/166.2M, step 1 loss 12.85 shuru hua.
349
  - **BUG OOM (exit 1)**: step-1 backward pe dono ranks CUDA OOM (13.3/14.5 GB). Root cause: probe me sirf ~2GB AdamW reserve — DDP grad buckets (~1GB) + fragmentation account nahi the. FIX: probe reserve 4→2→1GB阶梯 + `PYTORCH_ALLOC_CONF=expandable_segments:True`.
350
  - **BUG data 100% Hindi**: `enforce_mix=false` + natural Hub order → `mix_rows={hinglish:0, hindi:109625, english:0}` (pehle hindi/* files). FIX: natural mode me `hf_ds.shuffle(buffer_size=100000)` (ratio enforce nahi, sirf order bias) + config `natural_shuffle_buffer: 100000`.
351
- - Minor: cell HEAD check `next(list)` fail (cosmetic); `find_unused_parameters=True` warning (MoE unused experts possible — rakha).
352
  - verify #2 pending after push.
 
353
 
354
  ## Kaise note likhna hai
355
  ```
 
348
  - PASS: bootstrap, 2xT4, tok v2 mask_id=4, `rank 0/2 + 1/2`, `[dist broadcast r0->all]` (micro=4 accum=4), DDP wrapped synced, model 241.7M/166.2M, step 1 loss 12.85 shuru hua.
349
  - **BUG OOM (exit 1)**: step-1 backward pe dono ranks CUDA OOM (13.3/14.5 GB). Root cause: probe me sirf ~2GB AdamW reserve — DDP grad buckets (~1GB) + fragmentation account nahi the. FIX: probe reserve 4→2→1GB阶梯 + `PYTORCH_ALLOC_CONF=expandable_segments:True`.
350
  - **BUG data 100% Hindi**: `enforce_mix=false` + natural Hub order → `mix_rows={hinglish:0, hindi:109625, english:0}` (pehle hindi/* files). FIX: natural mode me `hf_ds.shuffle(buffer_size=100000)` (ratio enforce nahi, sirf order bias) + config `natural_shuffle_buffer: 100000`.
351
+ - Minor: cell HEAD check `next(list)` fail (cosmetic, training pe asar nahi); `find_unused_parameters=True` ka warning (MoE me unused experts possible — rakha).
352
  - verify #2 pending after push.
353
+ - [x] 2026-09-24 Kaggle DDP verify RUN #2: shuffle fix ne naya bug khola — **CastError on `wikipedia/`** (`id,url,title,text` vs 5-col). Root cause: dataset me multiple non-unified schemas (AUDIT A3: wikipedia, grammar, gsm8k, samanantar, sciq, civil_comments, hinglish/dolly|alpaca|sharegpt|lima). Bina shuffle tak Hindi-first order tha isliye kabhi touch nahi hui; shuffle buffer aage padha → fail. FIX: `train.py` streaming me `list_repo_files` + `NON_UNIFIED_PREFIXES` exclude → sirf unified parquet (`data_files=`). HF push done. verify #3 pending.
354
 
355
  ## Kaise note likhna hai
356
  ```
model/scripts/train.py CHANGED
@@ -445,7 +445,29 @@ def main():
445
  from datasets import load_dataset
446
  if R0:
447
  print(f"[hf] streaming dataset {hf_dataset} split={args.hf_split}")
448
- hf_ds = load_dataset(hf_dataset, split=args.hf_split, streaming=True)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
449
  # Language-mix rule OPTIONAL (default OFF = natural order).
450
  # enforce_mix=true par weighted block sampler chalta hai (lang_mix weights se).
451
  max_blocks_cfg = int(C("hf_max_blocks", 50000))
 
445
  from datasets import load_dataset
446
  if R0:
447
  print(f"[hf] streaming dataset {hf_dataset} split={args.hf_split}")
448
+ # Non-5-col schema folders (AUDIT A3) — inko stream me mat lo, warna
449
+ # shuffle/interleave par CastError (wikipedia: title/text/url vs text/lang/...).
450
+ # CastError = training data me corrupted/NULL rows; exclude karna safe hai.
451
+ NON_UNIFIED_PREFIXES = (
452
+ "wikipedia/", "grammar/", "translation/", "math/gsm8k",
453
+ "science/sciq", "toxicity/civil_comments",
454
+ "hinglish/dolly", "hinglish/alpaca", "hinglish/sharegpt", "hinglish/lima",
455
+ )
456
+ try:
457
+ from huggingface_hub import list_repo_files
458
+ _all = list_repo_files(hf_dataset, repo_type="dataset")
459
+ _data_files = [
460
+ f for f in _all
461
+ if f.endswith(".parquet") and not any(f.startswith(p) for p in NON_UNIFIED_PREFIXES)
462
+ ]
463
+ _n_excl = sum(1 for f in _all if f.endswith(".parquet")) - len(_data_files)
464
+ if R0:
465
+ print(f"[hf] data_files: {len(_data_files)} unified parquet (excluded {_n_excl} non-5col)")
466
+ hf_ds = load_dataset(hf_dataset, data_files=_data_files, split=args.hf_split, streaming=True)
467
+ except Exception as e:
468
+ if R0:
469
+ print(f"[hf] data_files filter skip ({e}) — full repo load")
470
+ hf_ds = load_dataset(hf_dataset, split=args.hf_split, streaming=True)
471
  # Language-mix rule OPTIONAL (default OFF = natural order).
472
  # enforce_mix=true par weighted block sampler chalta hai (lang_mix weights se).
473
  max_blocks_cfg = int(C("hf_max_blocks", 50000))