fix: exclude non-5col schema paths from HF stream (wikipedia/grammar/gsm8k CastError on shuffle)
Browse files- .gitattributes +4 -2
- data/processed_spiritual/train-p1p2_boost_hindi_math_gsm8k-00000.parquet +3 -0
- data/processed_spiritual/train-p1p2_boost_hinduism_philosophy_qa-00000.parquet +0 -0
- data/processed_spiritual/train-p1p2_boost_hinglish_arena_chat-00000.parquet +3 -0
- data/processed_spiritual/train-p1p2_boost_indian_history_qa-00000.parquet +3 -0
- data/scripts/collect_spiritual_and_boost_corpus.py +27 -26
- docs/PROGRESS.md +2 -1
- model/scripts/train.py +23 -1
.gitattributes
CHANGED
|
@@ -4,5 +4,7 @@
|
|
| 4 |
*.onnx filter=lfs diff=lfs merge=lfs -text
|
| 5 |
tokenizer/outputs/*.json filter=lfs diff=lfs merge=lfs -text
|
| 6 |
data/processed/* filter=lfs diff=lfs merge=lfs -text
|
| 7 |
-
data/processed_spiritual/train-
|
| 8 |
-
data/processed_spiritual/train-
|
|
|
|
|
|
|
|
|
| 4 |
*.onnx filter=lfs diff=lfs merge=lfs -text
|
| 5 |
tokenizer/outputs/*.json filter=lfs diff=lfs merge=lfs -text
|
| 6 |
data/processed/* filter=lfs diff=lfs merge=lfs -text
|
| 7 |
+
data/processed_spiritual/train-p1p2_boost_hindi_math_gsm8k-00000.parquet filter=lfs diff=lfs merge=lfs -text
|
| 8 |
+
data/processed_spiritual/train-p1p2_boost_hinduism_philosophy_qa-00000.parquet filter=lfs diff=lfs merge=lfs -text
|
| 9 |
+
data/processed_spiritual/train-p1p2_boost_hinglish_arena_chat-00000.parquet filter=lfs diff=lfs merge=lfs -text
|
| 10 |
+
data/processed_spiritual/train-p1p2_boost_indian_history_qa-00000.parquet filter=lfs diff=lfs merge=lfs -text
|
data/processed_spiritual/train-p1p2_boost_hindi_math_gsm8k-00000.parquet
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:35984a6198c38870c91b507e4436b95a76f7b271eba12afdab6156dc15974140
|
| 3 |
+
size 3216469
|
data/processed_spiritual/train-p1p2_boost_hinduism_philosophy_qa-00000.parquet
CHANGED
|
Binary files a/data/processed_spiritual/train-p1p2_boost_hinduism_philosophy_qa-00000.parquet and b/data/processed_spiritual/train-p1p2_boost_hinduism_philosophy_qa-00000.parquet differ
|
|
|
data/processed_spiritual/train-p1p2_boost_hinglish_arena_chat-00000.parquet
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:51b7aa53e13f6784cca09cd46b9c6c9c410770da5ffa8633f8a2a71a01f35f02
|
| 3 |
+
size 6913323
|
data/processed_spiritual/train-p1p2_boost_indian_history_qa-00000.parquet
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:896cfefd36fb018ebf1412153c325d114871b0c521ecd56ada0e8b76ac59d1f9
|
| 3 |
+
size 650283
|
data/scripts/collect_spiritual_and_boost_corpus.py
CHANGED
|
@@ -19,14 +19,14 @@ import os
|
|
| 19 |
import sys
|
| 20 |
import json
|
| 21 |
import time
|
|
|
|
| 22 |
import argparse
|
| 23 |
from pathlib import Path
|
| 24 |
import pandas as pd
|
| 25 |
import pyarrow as pa
|
| 26 |
import pyarrow.parquet as pq
|
| 27 |
-
from huggingface_hub import HfApi, hf_hub_download
|
| 28 |
|
| 29 |
-
# Enforce pure English logging
|
| 30 |
print("======================================================================")
|
| 31 |
print("Starting Sanatan Dharma & Knowledge Corpus Collector (ViuMini-MoE-242M)")
|
| 32 |
print("======================================================================")
|
|
@@ -46,6 +46,11 @@ SCHEMA = pa.schema([
|
|
| 46 |
def get_hf_token():
|
| 47 |
"""Retrieve Hugging Face token securely from environment or secret stores."""
|
| 48 |
token = os.environ.get("HF_TOKEN") or os.environ.get("HUGGING_FACE_HUB_TOKEN")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 49 |
if not token:
|
| 50 |
try:
|
| 51 |
from kaggle_secrets import UserSecretsClient
|
|
@@ -318,26 +323,16 @@ def collect_hinduism_philosophy_qa():
|
|
| 318 |
for item in data:
|
| 319 |
title = str(item.get("Title", "")).strip()
|
| 320 |
q_body = str(item.get("q_body", "")).strip()
|
| 321 |
-
|
| 322 |
-
|
| 323 |
clean_q = re.sub(r"<[^>]+>", " ", q_body).strip()
|
| 324 |
clean_q = " ".join(clean_q.split())
|
| 325 |
|
| 326 |
-
|
| 327 |
-
|
| 328 |
-
|
| 329 |
-
|
| 330 |
-
|
| 331 |
-
for ans in answers:
|
| 332 |
-
if ans.get("Id") == item.get("AcceptedAnswerId"):
|
| 333 |
-
best_ans = ans
|
| 334 |
-
break
|
| 335 |
-
ans_body = str(best_ans.get("a_body", "")).strip()
|
| 336 |
-
ans_text = re.sub(r"<[^>]+>", " ", ans_body).strip()
|
| 337 |
-
ans_text = " ".join(ans_text.split())
|
| 338 |
-
|
| 339 |
-
if title and ans_text and len(ans_text) > 40:
|
| 340 |
-
text = f"<|user|> Question on Hindu Philosophy & Scriptures: {title}\n{clean_q}\n<|assistant|> <soch> Analyzing Hindu scriptures, context, and philosophical principles... </soch> {ans_text}"
|
| 341 |
records.append({
|
| 342 |
"text": text,
|
| 343 |
"lang": "en",
|
|
@@ -408,8 +403,8 @@ def collect_hinglish_and_reasoning_boost():
|
|
| 408 |
url_hist = "https://huggingface.co/datasets/kaifahmad/indian-history-hindi-QA-3.4k/resolve/main/data/train-00000-of-00001.parquet"
|
| 409 |
df_hist = pd.read_parquet(url_hist)
|
| 410 |
for _, row in df_hist.iterrows():
|
| 411 |
-
q = str(row.get("question", "")).strip()
|
| 412 |
-
a = str(row.get("answer", "")).strip()
|
| 413 |
if q and a:
|
| 414 |
text = f"<|user|> भारतीय इतिहास प्रश्न: {q}\n<|assistant|> {a}"
|
| 415 |
records_hist.append({
|
|
@@ -451,20 +446,26 @@ def collect_hinglish_and_reasoning_boost():
|
|
| 451 |
def upload_shards_to_hub(shards):
|
| 452 |
"""Upload all newly created boost shards to Hugging Face dataset repository."""
|
| 453 |
print("\n[Step 5/5] Uploading Shards to Hugging Face Dataset Hub...")
|
| 454 |
-
|
| 455 |
-
|
|
|
|
| 456 |
return False
|
| 457 |
|
|
|
|
|
|
|
| 458 |
for file_path, row_count in shards:
|
|
|
|
|
|
|
|
|
|
| 459 |
hub_path = f"domains/{file_path.name}"
|
| 460 |
print(f" - Uploading {file_path.name} ({row_count:,} rows) -> {DATA_REPO}/{hub_path}...")
|
| 461 |
try:
|
| 462 |
-
|
| 463 |
path_or_fileobj=str(file_path),
|
| 464 |
path_in_repo=hub_path,
|
| 465 |
repo_id=DATA_REPO,
|
| 466 |
repo_type="dataset",
|
| 467 |
-
commit_message=f"feat: add
|
| 468 |
)
|
| 469 |
print(f" ✓ Uploaded successfully!")
|
| 470 |
except Exception as exc:
|
|
@@ -495,7 +496,7 @@ def main():
|
|
| 495 |
all_shards.extend(more_shards)
|
| 496 |
|
| 497 |
total_rows = sum(cnt for _, cnt in all_shards)
|
| 498 |
-
total_bytes = sum(f.stat().st_size for f,
|
| 499 |
|
| 500 |
print("\n" + "=" * 70)
|
| 501 |
print("Collection Summary:")
|
|
|
|
| 19 |
import sys
|
| 20 |
import json
|
| 21 |
import time
|
| 22 |
+
import re
|
| 23 |
import argparse
|
| 24 |
from pathlib import Path
|
| 25 |
import pandas as pd
|
| 26 |
import pyarrow as pa
|
| 27 |
import pyarrow.parquet as pq
|
| 28 |
+
from huggingface_hub import HfApi, hf_hub_download, get_token
|
| 29 |
|
|
|
|
| 30 |
print("======================================================================")
|
| 31 |
print("Starting Sanatan Dharma & Knowledge Corpus Collector (ViuMini-MoE-242M)")
|
| 32 |
print("======================================================================")
|
|
|
|
| 46 |
def get_hf_token():
|
| 47 |
"""Retrieve Hugging Face token securely from environment or secret stores."""
|
| 48 |
token = os.environ.get("HF_TOKEN") or os.environ.get("HUGGING_FACE_HUB_TOKEN")
|
| 49 |
+
if not token:
|
| 50 |
+
try:
|
| 51 |
+
token = get_token()
|
| 52 |
+
except Exception:
|
| 53 |
+
pass
|
| 54 |
if not token:
|
| 55 |
try:
|
| 56 |
from kaggle_secrets import UserSecretsClient
|
|
|
|
| 323 |
for item in data:
|
| 324 |
title = str(item.get("Title", "")).strip()
|
| 325 |
q_body = str(item.get("q_body", "")).strip()
|
| 326 |
+
aa_body = str(item.get("aa_body", "")).strip()
|
| 327 |
+
|
| 328 |
clean_q = re.sub(r"<[^>]+>", " ", q_body).strip()
|
| 329 |
clean_q = " ".join(clean_q.split())
|
| 330 |
|
| 331 |
+
clean_a = re.sub(r"<[^>]+>", " ", aa_body).strip()
|
| 332 |
+
clean_a = " ".join(clean_a.split())
|
| 333 |
+
|
| 334 |
+
if title and clean_a and len(clean_a) > 30:
|
| 335 |
+
text = f"<|user|> Question on Hindu Philosophy & Scriptures: {title}\n{clean_q}\n<|assistant|> <soch> Analyzing Hindu scriptures, context, and philosophical principles... </soch> {clean_a}"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 336 |
records.append({
|
| 337 |
"text": text,
|
| 338 |
"lang": "en",
|
|
|
|
| 403 |
url_hist = "https://huggingface.co/datasets/kaifahmad/indian-history-hindi-QA-3.4k/resolve/main/data/train-00000-of-00001.parquet"
|
| 404 |
df_hist = pd.read_parquet(url_hist)
|
| 405 |
for _, row in df_hist.iterrows():
|
| 406 |
+
q = str(row.get("Question", "") or row.get("question", "")).strip()
|
| 407 |
+
a = str(row.get("Answer", "") or row.get("answer", "")).strip()
|
| 408 |
if q and a:
|
| 409 |
text = f"<|user|> भारतीय इतिहास प्रश्न: {q}\n<|assistant|> {a}"
|
| 410 |
records_hist.append({
|
|
|
|
| 446 |
def upload_shards_to_hub(shards):
|
| 447 |
"""Upload all newly created boost shards to Hugging Face dataset repository."""
|
| 448 |
print("\n[Step 5/5] Uploading Shards to Hugging Face Dataset Hub...")
|
| 449 |
+
token = get_hf_token()
|
| 450 |
+
if not token:
|
| 451 |
+
print(" - [Error] Token not found! Cannot push to Hub.")
|
| 452 |
return False
|
| 453 |
|
| 454 |
+
api_client = HfApi(token=token)
|
| 455 |
+
|
| 456 |
for file_path, row_count in shards:
|
| 457 |
+
if row_count == 0:
|
| 458 |
+
print(f" - Skipping empty shard {file_path.name}")
|
| 459 |
+
continue
|
| 460 |
hub_path = f"domains/{file_path.name}"
|
| 461 |
print(f" - Uploading {file_path.name} ({row_count:,} rows) -> {DATA_REPO}/{hub_path}...")
|
| 462 |
try:
|
| 463 |
+
api_client.upload_file(
|
| 464 |
path_or_fileobj=str(file_path),
|
| 465 |
path_in_repo=hub_path,
|
| 466 |
repo_id=DATA_REPO,
|
| 467 |
repo_type="dataset",
|
| 468 |
+
commit_message=f"feat: add boost {file_path.name} ({row_count:,} rows)"
|
| 469 |
)
|
| 470 |
print(f" ✓ Uploaded successfully!")
|
| 471 |
except Exception as exc:
|
|
|
|
| 496 |
all_shards.extend(more_shards)
|
| 497 |
|
| 498 |
total_rows = sum(cnt for _, cnt in all_shards)
|
| 499 |
+
total_bytes = sum(f.stat().st_size for f, cnt in all_shards if cnt > 0)
|
| 500 |
|
| 501 |
print("\n" + "=" * 70)
|
| 502 |
print("Collection Summary:")
|
docs/PROGRESS.md
CHANGED
|
@@ -348,8 +348,9 @@ Kaggle par manual file upload se version mismatch hota hai (v1 tokenizer wala ca
|
|
| 348 |
- PASS: bootstrap, 2xT4, tok v2 mask_id=4, `rank 0/2 + 1/2`, `[dist broadcast r0->all]` (micro=4 accum=4), DDP wrapped synced, model 241.7M/166.2M, step 1 loss 12.85 shuru hua.
|
| 349 |
- **BUG OOM (exit 1)**: step-1 backward pe dono ranks CUDA OOM (13.3/14.5 GB). Root cause: probe me sirf ~2GB AdamW reserve — DDP grad buckets (~1GB) + fragmentation account nahi the. FIX: probe reserve 4→2→1GB阶梯 + `PYTORCH_ALLOC_CONF=expandable_segments:True`.
|
| 350 |
- **BUG data 100% Hindi**: `enforce_mix=false` + natural Hub order → `mix_rows={hinglish:0, hindi:109625, english:0}` (pehle hindi/* files). FIX: natural mode me `hf_ds.shuffle(buffer_size=100000)` (ratio enforce nahi, sirf order bias) + config `natural_shuffle_buffer: 100000`.
|
| 351 |
-
- Minor: cell HEAD check `next(list)` fail (cosmetic); `find_unused_parameters=True` warning (MoE unused experts possible — rakha).
|
| 352 |
- verify #2 pending after push.
|
|
|
|
| 353 |
|
| 354 |
## Kaise note likhna hai
|
| 355 |
```
|
|
|
|
| 348 |
- PASS: bootstrap, 2xT4, tok v2 mask_id=4, `rank 0/2 + 1/2`, `[dist broadcast r0->all]` (micro=4 accum=4), DDP wrapped synced, model 241.7M/166.2M, step 1 loss 12.85 shuru hua.
|
| 349 |
- **BUG OOM (exit 1)**: step-1 backward pe dono ranks CUDA OOM (13.3/14.5 GB). Root cause: probe me sirf ~2GB AdamW reserve — DDP grad buckets (~1GB) + fragmentation account nahi the. FIX: probe reserve 4→2→1GB阶梯 + `PYTORCH_ALLOC_CONF=expandable_segments:True`.
|
| 350 |
- **BUG data 100% Hindi**: `enforce_mix=false` + natural Hub order → `mix_rows={hinglish:0, hindi:109625, english:0}` (pehle hindi/* files). FIX: natural mode me `hf_ds.shuffle(buffer_size=100000)` (ratio enforce nahi, sirf order bias) + config `natural_shuffle_buffer: 100000`.
|
| 351 |
+
- Minor: cell HEAD check `next(list)` fail (cosmetic, training pe asar nahi); `find_unused_parameters=True` ka warning (MoE me unused experts possible — rakha).
|
| 352 |
- verify #2 pending after push.
|
| 353 |
+
- [x] 2026-09-24 Kaggle DDP verify RUN #2: shuffle fix ne naya bug khola — **CastError on `wikipedia/`** (`id,url,title,text` vs 5-col). Root cause: dataset me multiple non-unified schemas (AUDIT A3: wikipedia, grammar, gsm8k, samanantar, sciq, civil_comments, hinglish/dolly|alpaca|sharegpt|lima). Bina shuffle tak Hindi-first order tha isliye kabhi touch nahi hui; shuffle buffer aage padha → fail. FIX: `train.py` streaming me `list_repo_files` + `NON_UNIFIED_PREFIXES` exclude → sirf unified parquet (`data_files=`). HF push done. verify #3 pending.
|
| 354 |
|
| 355 |
## Kaise note likhna hai
|
| 356 |
```
|
model/scripts/train.py
CHANGED
|
@@ -445,7 +445,29 @@ def main():
|
|
| 445 |
from datasets import load_dataset
|
| 446 |
if R0:
|
| 447 |
print(f"[hf] streaming dataset {hf_dataset} split={args.hf_split}")
|
| 448 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 449 |
# Language-mix rule OPTIONAL (default OFF = natural order).
|
| 450 |
# enforce_mix=true par weighted block sampler chalta hai (lang_mix weights se).
|
| 451 |
max_blocks_cfg = int(C("hf_max_blocks", 50000))
|
|
|
|
| 445 |
from datasets import load_dataset
|
| 446 |
if R0:
|
| 447 |
print(f"[hf] streaming dataset {hf_dataset} split={args.hf_split}")
|
| 448 |
+
# Non-5-col schema folders (AUDIT A3) — inko stream me mat lo, warna
|
| 449 |
+
# shuffle/interleave par CastError (wikipedia: title/text/url vs text/lang/...).
|
| 450 |
+
# CastError = training data me corrupted/NULL rows; exclude karna safe hai.
|
| 451 |
+
NON_UNIFIED_PREFIXES = (
|
| 452 |
+
"wikipedia/", "grammar/", "translation/", "math/gsm8k",
|
| 453 |
+
"science/sciq", "toxicity/civil_comments",
|
| 454 |
+
"hinglish/dolly", "hinglish/alpaca", "hinglish/sharegpt", "hinglish/lima",
|
| 455 |
+
)
|
| 456 |
+
try:
|
| 457 |
+
from huggingface_hub import list_repo_files
|
| 458 |
+
_all = list_repo_files(hf_dataset, repo_type="dataset")
|
| 459 |
+
_data_files = [
|
| 460 |
+
f for f in _all
|
| 461 |
+
if f.endswith(".parquet") and not any(f.startswith(p) for p in NON_UNIFIED_PREFIXES)
|
| 462 |
+
]
|
| 463 |
+
_n_excl = sum(1 for f in _all if f.endswith(".parquet")) - len(_data_files)
|
| 464 |
+
if R0:
|
| 465 |
+
print(f"[hf] data_files: {len(_data_files)} unified parquet (excluded {_n_excl} non-5col)")
|
| 466 |
+
hf_ds = load_dataset(hf_dataset, data_files=_data_files, split=args.hf_split, streaming=True)
|
| 467 |
+
except Exception as e:
|
| 468 |
+
if R0:
|
| 469 |
+
print(f"[hf] data_files filter skip ({e}) — full repo load")
|
| 470 |
+
hf_ds = load_dataset(hf_dataset, split=args.hf_split, streaming=True)
|
| 471 |
# Language-mix rule OPTIONAL (default OFF = natural order).
|
| 472 |
# enforce_mix=true par weighted block sampler chalta hai (lang_mix weights se).
|
| 473 |
max_blocks_cfg = int(C("hf_max_blocks", 50000))
|