Delete code/build_sft_v2_chat_dataset.py with huggingface_hub

code/build_sft_v2_chat_dataset.py

@@ -1,861 +0,0 @@
-# ==============================================================================
-# 💬 ViuAI Sarus-500M — Master SFT v2 Pure Chat Dataset Builder & Tokenizer
-# ==============================================================================
-# 100% Real Conversational & Knowledge Data | 0% Math | 0% Coding
-# Specially Designed for Sarus-500M Parameter Capacity:
-#   • D0: Identity & Persona (Sarus / ViuAI)           ~3,000
-#   • D1: Bilingual Greetings & Openers                ~4,000
-#   • D2: Conversational Dialogue & Multi-turn         ~35,000
-#   • D3: General Knowledge & Science (English)        ~40,000
-#   • D4: Hindi Factual QA & Knowledge (Breaks Greeting Loop) ~40,000
-#   • D5: Practical Writing & Everyday Tasks           ~30,000
-#   • D6: Culture, Recipes & Lifestyle                 ~25,000
-#   • D7: Bilingual Translation (EN <-> HI)            ~25,000
-# Total: ~202,000 High-Purity Conversational Samples
-# ==============================================================================
-
-import os
-import sys
-import json
-import re
-import random
-import hashlib
-import shutil
-import argparse
-import numpy as np
-from datasets import load_dataset
-from huggingface_hub import HfApi, hf_hub_download
-from tokenizers import Tokenizer
-
-# Fix console encoding
-if hasattr(sys.stdout, "reconfigure"):
-    sys.stdout.reconfigure(encoding="utf-8", errors="replace", line_buffering=True)
-if hasattr(sys.stderr, "reconfigure"):
-    sys.stderr.reconfigure(encoding="utf-8", errors="replace", line_buffering=True)
-
-BASE_DIR = os.getcwd()
-RAW_EXPORT_DIR = os.path.join(BASE_DIR, "sarus_sft_raw", "sft_v2")
-TOKENIZED_DIR = os.path.join(BASE_DIR, "sarus_sft_tokenized", "sft_v2")
-os.makedirs(RAW_EXPORT_DIR, exist_ok=True)
-os.makedirs(TOKENIZED_DIR, exist_ok=True)
-
-# SECURITY: HF token from env only. Never hardcode. Export HF_TOKEN before running.
-HF_TOKEN = os.environ.get("HF_TOKEN")
-RAW_REPO_ID = "ViuAI/sarus-sft-data-raw"
-TOKENIZED_REPO_ID = "ViuAI/viuai-500m-sft-tokenized"
-MODEL_REPO_ID = "ViuAI/ViuAI-500M"
-
-MAX_SEQ_LEN = 1024
-VAL_SPLIT_RATIO = 0.02
-MIN_TOKENS = 6
-
-USER_TOKEN = "<|user|>"
-ASSISTANT_TOKEN = "<|assistant|>"
-EOT_TOKEN = "<|endofturn|>"
-EOT_ID = 64002
-# EOT-as-pad. Never use <|user|> (64000) as pad.
-PAD_TOKEN_ID = EOT_ID
-
-DOMAIN_NAMES = {
-    0: "identity_sarus",
-    1: "greetings_cultural",
-    2: "conversational_dialogue",
-    3: "general_knowledge_science",
-    4: "hindi_knowledge_and_qa",
-    5: "practical_writing_tasks",
-    6: "culture_recipes_lifestyle",
-    7: "translation_bilingual"
-}
-
-DOMAIN_TARGETS = {
-    0: 3000,    # Identity & Attribution
-    1: 4000,    # Greetings & Icebreakers
-    2: 35000,   # Everyday Multi-Turn Chat
-    3: 40000,   # English Science & World Facts
-    4: 40000,   # Hindi Factual QA (NCERT Hindi + Indian GK)
-    5: 30000,   # Emails, Summaries & Everyday Writing
-    6: 25000,   # Indian Recipes, Culture & Wellness
-    7: 25000    # English <-> Hindi Translation
-}
-
-# ------------------------------------------------------------------------------
-# 1. Tokenizer Setup
-# ------------------------------------------------------------------------------
-def load_fast_tokenizer():
-    tok_candidates = [
-        os.path.join(BASE_DIR, "tokenizer", "tokenizer.json"),
-        os.path.join(BASE_DIR, "tokenizer.json")
-    ]
-    for p in tok_candidates:
-        if os.path.exists(p):
-            tok = Tokenizer.from_file(p)
-            print(f"✅ Tokenizer loaded from {p} ({tok.get_vocab_size():,} vocab)")
-            return tok
-
-    print("🌐 Downloading tokenizer.json from Hugging Face Model Hub...")
-    dl = hf_hub_download(repo_id=MODEL_REPO_ID, filename="tokenizer/tokenizer.json", token=HF_TOKEN)
-    tok = Tokenizer.from_file(dl)
-    os.makedirs(os.path.join(BASE_DIR, "tokenizer"), exist_ok=True)
-    shutil.copy(dl, os.path.join(BASE_DIR, "tokenizer", "tokenizer.json"))
-    print(f"✅ Tokenizer loaded ({tok.get_vocab_size():,} vocab)")
-    return tok
-
-TOKENIZER = None
-
-# ------------------------------------------------------------------------------
-# 2. Text Cleaner & Math/Code Filter (Guarantees Pure Chat)
-# ------------------------------------------------------------------------------
-GLOBAL_SEEN_HASHES = set()
-
-def is_duplicate(text: str) -> bool:
-    clean = re.sub(r"\s+", " ", text.lower().strip())
-    h = hashlib.sha256(clean.encode("utf-8")).hexdigest()
-    if h in GLOBAL_SEEN_HASHES:
-        return True
-    GLOBAL_SEEN_HASHES.add(h)
-    return False
-
-def clean_text(text: str) -> str:
-    if not text:
-        return ""
-    text = re.sub(r"(?:संदर्भ|सन्दर्भ|बाहरी कड़ियाँ|श्रेणियाँ|सम्पादन|References|External links|See also)[\s:]*$", "", text, flags=re.IGNORECASE)
-    text = re.sub(r"<[^>]+>", " ", text)
-    text = re.sub(r"[ \t]+", " ", text)
-    text = re.sub(r"\n{3,}", "\n\n", text)
-    return text.strip()
-
-def contains_math_or_code(text: str) -> bool:
-    """Strict filter: rejects any programming code or complex math formula."""
-    lower = text.lower()
-    # Code patterns
-    if "```python" in lower or "```java" in lower or "```cpp" in lower or "```c" in lower:
-        return True
-    if "def " in text and "):" in text:
-        return True
-    if "class " in text and ":" in text and "def __init__" in text:
-        return True
-    if "#include <" in text or "public static void main" in text:
-        return True
-    # Math / equation patterns
-    if "\\frac" in text or "\\sqrt" in text or "\\begin{equation}" in text:
-        return True
-    if re.search(r"\b(?:calculate|solve|evaluate)\s*:\s*\d+\s*[\+\-\*\/]\s*\d+", lower):
-        return True
-    return False
-
-# ------------------------------------------------------------------------------
-# 3. Domain Streamers & Builders
-# ------------------------------------------------------------------------------
-
-def load_json_or_jsonl(path):
-    if not path or not os.path.exists(path):
-        return []
-    try:
-        with open(path, "r", encoding="utf-8") as f:
-            c = f.read().strip()
-        if c.startswith("["):
-            return json.loads(c)
-        return [json.loads(line) for line in c.splitlines() if line.strip()]
-    except Exception as e:
-        print(f"  ⚠️ Could not parse {path}: {e}")
-        return []
-
-# --- D0: Identity (Sarus / ViuAI) ---
-def build_domain_identity(target=3000):
-    print(f"\n[D0: Identity] Sourcing authentic Sarus identity dataset (Target: {target:,})...")
-    samples = []
-    candidates = [
-        os.path.join(BASE_DIR, "data", "sarus_sft_raw", "d0_identity_sarus.jsonl"),
-        os.path.join(BASE_DIR, "data", "sft_identity.json"),
-        os.path.join(BASE_DIR, "sft_identity.json")
-    ]
-    p_to_use = None
-    for p in candidates:
-        if os.path.exists(p):
-            p_to_use = p
-            break
-
-    if p_to_use is None:
-        try:
-            print(f"  📥 Downloading d0_identity_sarus.jsonl from HF ({RAW_REPO_ID})...")
-            p_dl = os.path.join(BASE_DIR, "data", "sarus_sft_raw", "d0_identity_sarus.jsonl")
-            os.makedirs(os.path.dirname(p_dl), exist_ok=True)
-            dl = hf_hub_download(repo_id=RAW_REPO_ID, filename="d0_identity_sarus.jsonl", repo_type="dataset", token=HF_TOKEN)
-            shutil.copy(dl, p_dl)
-            p_to_use = p_dl
-        except Exception as e:
-            print(f"  ℹ️ HF fetch identity notice: {e}")
-            try:
-                dl2 = hf_hub_download(repo_id=MODEL_REPO_ID, filename="data/sft_identity.json", token=HF_TOKEN)
-                p_to_use = dl2
-            except Exception as e2:
-                print(f"  ℹ️ Fallback notice: {e2}")
-
-    raw_items = load_json_or_jsonl(p_to_use)
-    for item in raw_items:
-        q = clean_text((item.get("instruction") or "").replace("ViuAi", "ViuAI"))
-        r = clean_text((item.get("response") or "").replace("ViuAi", "ViuAI"))
-        if q and r and not is_duplicate(q):
-            samples.append({"instruction": q, "response": r, "domain_id": 0})
-        if len(samples) >= target:
-            break
-    print(f"  ✅ [D0: Identity] Loaded {len(samples):,} authentic Sarus identity samples.")
-    return samples[:target]
-
-# --- D1: Greetings & Cultural Openers ---
-def build_domain_greetings(target=4000):
-    print(f"\n[D1: Greetings] Sourcing authentic greetings dataset (Target: {target:,})...")
-    samples = []
-    candidates = [
-        os.path.join(BASE_DIR, "data", "sarus_sft_raw", "d1_greetings_cultural.jsonl"),
-        os.path.join(BASE_DIR, "data", "sft_greetings.json"),
-        os.path.join(BASE_DIR, "sft_greetings.json")
-    ]
-    p_to_use = None
-    for p in candidates:
-        if os.path.exists(p):
-            p_to_use = p
-            break
-
-    if p_to_use is None:
-        try:
-            print(f"  📥 Downloading d1_greetings_cultural.jsonl from HF ({RAW_REPO_ID})...")
-            p_dl = os.path.join(BASE_DIR, "data", "sarus_sft_raw", "d1_greetings_cultural.jsonl")
-            os.makedirs(os.path.dirname(p_dl), exist_ok=True)
-            dl = hf_hub_download(repo_id=RAW_REPO_ID, filename="d1_greetings_cultural.jsonl", repo_type="dataset", token=HF_TOKEN)
-            shutil.copy(dl, p_dl)
-            p_to_use = p_dl
-        except Exception as e:
-            print(f"  ℹ️ HF fetch greetings notice: {e}")
-            try:
-                dl2 = hf_hub_download(repo_id=MODEL_REPO_ID, filename="data/sft_greetings.json", token=HF_TOKEN)
-                p_to_use = dl2
-            except Exception as e2:
-                print(f"  ℹ️ Fallback notice: {e2}")
-
-    raw_items = load_json_or_jsonl(p_to_use)
-    for item in raw_items:
-        q = clean_text(item.get("instruction") or "")
-        r = clean_text(item.get("response") or "")
-        if q and r and not is_duplicate(q):
-            samples.append({"instruction": q, "response": r, "domain_id": 1})
-        if len(samples) >= target:
-            break
-    print(f"  ✅ [D1: Greetings] Loaded {len(samples):,} authentic greetings.")
-    return samples[:target]
-
-# --- D2: Conversational Dialogue & Multi-turn ---
-def build_domain_dialogue(target=35000):
-    print(f"\n[D2: Dialogue] Sourcing everyday conversational dialogue (Target: {target:,})...")
-    samples = []
-    
-    # 1. Check local pre-extracted multi-turn
-    local_p = os.path.join(BASE_DIR, "data", "sarus_sft_raw", "d8_multiturn_dialogue.jsonl")
-    if os.path.exists(local_p):
-        with open(local_p, "r", encoding="utf-8") as f:
-            for line in f:
-                try:
-                    item = json.loads(line)
-                except Exception:
-                    continue
-                q = clean_text(item.get("instruction") or "")
-                r = clean_text(item.get("response") or "")
-                if q and r and not is_duplicate(q) and not contains_math_or_code(r):
-                    samples.append({"instruction": q, "response": r, "domain_id": 2})
-                if len(samples) >= target:
-                    break
-
-    # 2. UltraChat 200k fallback/stream
-    if len(samples) < target:
-        try:
-            print("  🌐 Streaming everyday conversations from HuggingFaceH4/ultrachat_200k...")
-            ds = load_dataset("HuggingFaceH4/ultrachat_200k", split="train_sft", streaming=True)
-            for row in ds:
-                msgs = row.get("messages", [])
-                if len(msgs) >= 2 and msgs[0].get("role") == "user" and msgs[1].get("role") == "assistant":
-                    q = clean_text(msgs[0].get("content", ""))
-                    r = clean_text(msgs[1].get("content", ""))
-                    if q and r and len(q) < 600 and len(r) < 1200 and not is_duplicate(q) and not contains_math_or_code(r):
-                        samples.append({"instruction": q, "response": r, "domain_id": 2})
-                if len(samples) >= target:
-                    break
-        except Exception as e:
-            print(f"  ℹ️ UltraChat stream warning: {e}")
-
-    print(f"  ✅ [D2: Dialogue] Loaded {len(samples):,} conversational dialogue samples.")
-    return samples[:target]
-
-# --- D3: General Knowledge & Science (English) ---
-def build_domain_gk_science(target=40000):
-    print(f"\n[D3: GK & Science] Sourcing English Science & World Facts (Target: {target:,})...")
-    samples = []
-    
-    # 1. NCERT Science Class 9-12 (Biology, Chemistry, Physics)
-    try:
-        print("  🌐 Streaming NCERT Science from ParthKadam2003/NCERT_Dataset...")
-        ds_ncert = load_dataset("ParthKadam2003/NCERT_Dataset", split="train", streaming=True)
-        for row in ds_ncert:
-            q = (row.get("Question") or "").strip()
-            ans = (row.get("Answer") or "").strip()
-            exp = (row.get("Explanation") or "").strip()
-            if q and ans and not is_duplicate(q) and not contains_math_or_code(ans):
-                resp = ans
-                if exp and len(exp) > 20 and exp != ans and len(exp) < 600:
-                    resp = f"{ans}\n\nExplanation: {exp}"
-                samples.append({"instruction": clean_text(q), "response": clean_text(resp), "domain_id": 3})
-            if len(samples) >= 20000:
-                break
-    except Exception as e:
-        print(f"  ℹ️ NCERT dataset stream warning: {e}")
-
-    # 2. Local D5 General STEM (Filtered for clean science concepts, no math/code)
-    local_stem = os.path.join(BASE_DIR, "data", "sarus_sft_raw", "d5_general_stem.jsonl")
-    if os.path.exists(local_stem) and len(samples) < target:
-        with open(local_stem, "r", encoding="utf-8") as f:
-            for line in f:
-                try:
-                    item = json.loads(line)
-                except Exception:
-                    continue
-                q = clean_text(item.get("instruction") or "")
-                r = clean_text(item.get("response") or "")
-                if q and r and not is_duplicate(q) and not contains_math_or_code(r) and not contains_math_or_code(q):
-                    samples.append({"instruction": q, "response": r, "domain_id": 3})
-                if len(samples) >= target:
-                    break
-
-    # 3. Stanford Alpaca (General Science/World knowledge)
-    if len(samples) < target:
-        try:
-            print("  🌐 Streaming Stanford Alpaca for World Knowledge...")
-            ds_alpaca = load_dataset("tatsu-lab/alpaca", split="train", streaming=True)
-            for row in ds_alpaca:
-                q = (row.get("instruction") or "").strip()
-                inp = (row.get("input") or "").strip()
-                r = (row.get("output") or "").strip()
-                if inp:
-                    q = f"{q}\n\n{inp}"
-                if q and r and not is_duplicate(q) and not contains_math_or_code(q) and not contains_math_or_code(r):
-                    samples.append({"instruction": clean_text(q), "response": clean_text(r), "domain_id": 3})
-                if len(samples) >= target:
-                    break
-        except Exception as e:
-            print(f"  ℹ️ Alpaca stream warning: {e}")
-
-    print(f"  ✅ [D3: GK & Science] Loaded {len(samples):,} Science and World Knowledge samples.")
-    return samples[:target]
-
-# --- D4: Hindi Factual QA & Knowledge (BREAKS THE GREETING LOOP!) ---
-def build_domain_hindi_qa(target=40000):
-    print(f"\n[D4: Hindi Factual QA] Sourcing pure Hindi QA & Knowledge (Target: {target:,})...")
-    samples = []
-
-    # 1. Curated Ground-Truth Hindi Factual QA (Directly fixes observed failures)
-    ground_truth_hindi = [
-        ("भारत के प्रथम प्रधानमंत्री कौन थे?", "भारत के प्रथम प्रधानमंत्री पंडित जवाहरलाल नेहरू थे।"),
-        ("भारत के प्रथम राष्ट्रपति कौन थे?", "भारत के प्रथम राष्ट्रपति डॉ. राजेंद्र प्रसाद थे।"),
-        ("भारतीय संविधान के निर्माता कौन थे?", "भारतीय संविधान के मुख्य निर्माता और प्रारूप समिति (Drafting Committee) के अध्यक्ष डॉ. भीमराव अंबेडकर थे।"),
-        ("भारत की राजधानी क्या है?", "भारत की राजधानी नई दिल्ली है।"),
-        ("सूर्य किस दिशा से निकलता है?", "सूर्य पूर्व दिशा से निकलता है और पश्चिम दिशा में अस्त होता है।"),
-        ("पृथ्वी के सबसे नजदीक कौन सा तारा है?", "पृथ्वी के सबसे निकटतम तारा सूर्य है।"),
-        ("सौरमंडल का सबसे बड़ा ग्रह कौन सा है?", "सौरमंडल का सबसे बड़ा ग्रह बृहस्पति (Jupiter) है।"),
-        ("लाल ग्रह (Red Planet) किसे कहा जाता है?", "मंगल ग्रह (Mars) को लाल ग्रह कहा जाता है क्योंकि इसकी सतह पर आयरन ऑक्साइड प्रचुर मात्रा में है।"),
-        ("मानव शरीर में कुल कितनी हड्डियां होती हैं?", "एक वयस्क मानव शरीर में कुल 206 हड्डियां होती हैं।"),
-        ("पौधे प्रकाश संश्लेषण में कौन सी गैस लेते हैं?", "पौधे प्रकाश संश्लेषण (Photosynthesis) की प्रक्रिया में कार्बन डाइऑक्साइड (CO2) ग्रहण करते हैं और ऑक्सीजन (O2) छोड़ते हैं।"),
-        ("मनुष्य सांस लेने में कौन सी गैस उपयोग करते हैं?", "मनुष्य जीवित रहने के लिए सांस के रूप में ऑक्सीजन (O2) गैस ग्रहण करते हैं।"),
-        ("भारत का राष्ट्रीय पशु कौन सा है?", "भारत का राष्ट्रीय पशु बाघ (Bengal Tiger) है।"),
-        ("भारत का राष्ट्रीय पक्षी कौन सा है?", "भारत का राष्ट्रीय पक्षी मोर (Peacock) है।"),
-        ("जल का रासायनिक सूत्र क्या है?", "जल (पानी) का रासायनिक सूत्र H2O है।"),
-        ("दुनिया की सबसे ऊंची चोटी कौन सी है?", "दुनिया की सबसे ऊंची पर्वत चोटी माउंट एवरेस्ट है, जो नेपाल में स्थित है।"),
-        ("स्वतंत्र भारत का संविधान कब लागू हुआ था?", "भारतीय संविधान 26 जनवरी 1950 को पूर्ण रूप से लागू हुआ था।"),
-        ("राष्ट्रगान 'जन गण मन' किसने लिखा था?", "भारत का राष्ट्रगान 'जन गण मन' गुरुदेव रवींद्रनाथ टैगोर ने लिखा था।"),
-        ("राष्ट्रगीत 'वंदे मातरम' के रचयिता कौन हैं?", "राष्ट्रगीत 'वंदे मातरम' के रचयिता बंकिम चंद्र ��ट्टोपाध्याय हैं।"),
-        ("ताजमहल कहाँ स्थित है और किसने बनवाया था?", "ताजमहल उत्तर प्रदेश के आगरा शहर में यमुना नदी के किनारे स्थित है, जिसे मुगल सम्राट शाहजहाँ ने बनवाया था।"),
-        ("गाय के दूध का हल्का पीला रंग किस कारण होता है?", "गाय के दूध का हल्का पीला रंग कैरोटीन (Carotene) की उपस्थिति के कारण होता है।")
-    ]
-    for q, a in ground_truth_hindi:
-        if not is_duplicate(q):
-            samples.append({"instruction": q, "response": a, "domain_id": 4})
-
-    # 2. Real Indian History in Hindi (kaifahmad/indian-history-hindi-QA-3.4k)
-    try:
-        print("  🌐 Streaming Indian History QA in Hindi (kaifahmad/indian-history-hindi-QA-3.4k)...")
-        ds_hist = load_dataset("kaifahmad/indian-history-hindi-QA-3.4k", split="train", streaming=True)
-        for row in ds_hist:
-            q = (row.get("Question") or "").strip()
-            ans = (row.get("Answer") or "").strip()
-            if q and ans and len(q) > 8 and len(ans) > 10 and not is_duplicate(q) and not contains_math_or_code(ans):
-                samples.append({"instruction": clean_text(q), "response": clean_text(ans), "domain_id": 4})
-            if len(samples) >= 5000:
-                break
-    except Exception as e:
-        print(f"  ℹ️ Indian history QA stream warning: {e}")
-
-    # 3. Clean Hindi Alpaca (Filtered for Knowledge, strictly avoiding math/code)
-    try:
-        print("  🌐 Streaming clean Hindi knowledge from iamshnoo/alpaca-cleaned-hindi...")
-        ds_hi = load_dataset("iamshnoo/alpaca-cleaned-hindi", split="train", streaming=True)
-        for row in ds_hi:
-            q = (row.get("instruction") or "").strip()
-            inp = (row.get("input") or "").strip()
-            ans = (row.get("output") or "").strip()
-            if inp:
-                q = f"{q}\n{inp}"
-            if q and ans and not is_duplicate(q) and not contains_math_or_code(q) and not contains_math_or_code(ans):
-                samples.append({"instruction": clean_text(q), "response": clean_text(ans), "domain_id": 4})
-            if len(samples) >= target:
-                break
-    except Exception as e:
-        print(f"  ℹ️ Hindi Alpaca stream warning: {e}")
-
-    # 4. Fallback from local d6_hindi_culture if needed
-    local_hi = os.path.join(BASE_DIR, "data", "sarus_sft_raw", "d6_hindi_culture.jsonl")
-    if os.path.exists(local_hi) and len(samples) < target:
-        with open(local_hi, "r", encoding="utf-8") as f:
-            for line in f:
-                try:
-                    item = json.loads(line)
-                except Exception:
-                    continue
-                q = clean_text(item.get("instruction") or "")
-                r = clean_text(item.get("response") or "")
-                if q and r and not is_duplicate(q) and not contains_math_or_code(q) and not contains_math_or_code(r):
-                    samples.append({"instruction": q, "response": r, "domain_id": 4})
-                if len(samples) >= target:
-                    break
-
-    print(f"  ✅ [D4: Hindi Factual QA] Loaded {len(samples):,} real Hindi QA & Knowledge samples.")
-    return samples[:target]
-
-# --- D5: Practical Writing & Everyday Tasks ---
-def build_domain_practical_writing(target=30000):
-    print(f"\n[D5: Writing] Sourcing practical writing, emails & summaries (Target: {target:,})...")
-    samples = []
-    
-    # 1. Databricks Dolly 15k (human-written emails, summaries, brainstorming)
-    try:
-        print("  🌐 Streaming Databricks Dolly-15k writing tasks...")
-        ds_dolly = load_dataset("databricks/databricks-dolly-15k", split="train", streaming=True)
-        for row in ds_dolly:
-            cat = (row.get("category") or "").lower()
-            if cat in ["creative_writing", "summarization", "open_qa", "brainstorming", "general_qa"]:
-                q = (row.get("instruction") or "").strip()
-                ctx = (row.get("context") or "").strip()
-                ans = (row.get("response") or "").strip()
-                if ctx:
-                    q = f"{q}\n\nContext:\n{ctx}"
-                if q and ans and not is_duplicate(q) and not contains_math_or_code(ans):
-                    samples.append({"instruction": clean_text(q), "response": clean_text(ans), "domain_id": 5})
-            if len(samples) >= 12000:
-                break
-    except Exception as e:
-        print(f"  ℹ️ Dolly-15k stream warning: {e}")
-
-    # 2. Local D7 Summarization
-    local_d7 = os.path.join(BASE_DIR, "data", "sarus_sft_raw", "d7_summarization_translation.jsonl")
-    if os.path.exists(local_d7) and len(samples) < target:
-        with open(local_d7, "r", encoding="utf-8") as f:
-            for line in f:
-                try:
-                    item = json.loads(line)
-                except Exception:
-                    continue
-                q = clean_text(item.get("instruction") or "")
-                r = clean_text(item.get("response") or "")
-                if "summar" in q.lower() or "dialogue" in q.lower():
-                    if q and r and not is_duplicate(q) and not contains_math_or_code(r):
-                        samples.append({"instruction": q, "response": r, "domain_id": 5})
-                if len(samples) >= target:
-                    break
-
-    # 3. UltraChat everyday assistance
-    if len(samples) < target:
-        try:
-            ds = load_dataset("HuggingFaceH4/ultrachat_200k", split="train_sft", streaming=True)
-            for row in ds:
-                msgs = row.get("messages", [])
-                if len(msgs) >= 2:
-                    q = clean_text(msgs[0].get("content", ""))
-                    r = clean_text(msgs[1].get("content", ""))
-                    if ("email" in q.lower() or "letter" in q.lower() or "suggest" in q.lower() or "tips" in q.lower()):
-                        if q and r and not is_duplicate(q) and not contains_math_or_code(r):
-                            samples.append({"instruction": q, "response": r, "domain_id": 5})
-                if len(samples) >= target:
-                    break
-        except Exception as e:
-            print(f"  ℹ️ UltraChat writing stream warning: {e}")
-
-    print(f"  ✅ [D5: Writing] Loaded {len(samples):,} practical writing and summary samples.")
-    return samples[:target]
-
-# --- D6: Culture, Recipes & Lifestyle ---
-def build_domain_recipes_lifestyle(target=25000):
-    print(f"\n[D6: Recipes & Lifestyle] Sourcing Indian cooking, tea recipes & lifestyle tips (Target: {target:,})...")
-    samples = []
-    
-    # Hand-curated authentic Chai & Indian food guides
-    indian_culinary = [
-        ("Masala Chai banane ki sabse best recipe step by step batao.", 
-         "**स्वादिष्ट कड़क मसाला चाय की प्रामाणिक रेसिपी:**\n\n"
-         "**सामग्री (2 कप के लिए):**\n"
-         "• पानी: 1.5 कप\n"
-         "• ताजा दूध: 1 कप\n"
-         "• चाय पत्ती: 2 छोटे चम्मच\n"
-         "• चीनी: 2 छोटे चम्मच (स्वादानुसार)\n"
-         "• ताजा अदरक: 1 इंच टुकड़ा (कुटा हुआ)\n"
-         "• हरी इलायची: 2 (कुटी हुई)\n"
-         "• लौंग: 1-2 (वैकल्पिक)\n"
-         "• दालचीनी: छोटा सा टुकड़ा\n\n"
-         "**बनाने की विधि:**\n"
-         "1. एक पैन में 1.5 कप पानी गर्म करें। इसमें कुटा हुआ अदरक, इलायची और दालचीनी डालकर 2 मिनट तक उबालें ताकि मसालों का अर्क पानी में आ जाए।\n"
-         "2. अब इसमें चाय पत्ती डालें और मध्यम आंच पर 1-2 मिनट खौलाएं।\n"
-         "3. इसके बाद दूध और चीनी डालें। आंच तेज करें और चाय में पहला उबाल आने दें।\n"
-         "4. उबाल आने के बाद आंच धीमी करें और चाय को चम्मच से चलाते हुए 2-3 मिनट तक अच्छे से पकने दें।\n"
-         "5. जब सुंदर सुनहरा रंग आ जाए, तो गैस बंद करें और गरमा-गरम मसाला चाय छानकर परोसें।"),
-        ("How to make traditional Indian Masala Chai?",
-         "**Authentic Indian Masala Chai Recipe (Serves 2):**\n\n"
-         "**Ingredients:**\n"
-         "• Water: 1.5 cups\n"
-         "• Whole milk: 1 cup\n"
-         "• Strong black tea leaves: 2 tsp\n"
-         "• Sugar: 2 tsp (to taste)\n"
-         "• Fresh ginger: 1 inch (crushed)\n"
-         "• Green cardamom pods: 2 (crushed)\n"
-         "• Cinnamon stick: 1 small piece\n\n"
-         "**Instructions:**\n"
-         "1. Bring water to a simmer in a saucepan. Add crushed ginger, cardamom, and cinnamon. Boil for 2 minutes to extract flavors.\n"
-         "2. Add black tea leaves and simmer for another minute.\n"
-         "3. Pour in the milk and add sugar. Bring the mixture to a rolling boil.\n"
-         "4. Reduce heat and let it simmer gently for 2-3 minutes until a rich caramel-brown color forms.\n"
-         "5. Strain into cups and serve piping hot with biscuits or snacks."),
-        ("What are healthy morning habits for high energy?",
-         "**5 Simple Habits for High Morning Energy:**\n\n"
-         "1. **Hydrate Immediately:** Drink 1-2 glasses of warm water right after waking up to activate metabolism.\n"
-         "2. **Natural Sunlight Exposure:** Spend 10-15 minutes in morning sunlight to regulate your circadian rhythm and boost alertness.\n"
-         "3. **Light Stretching or Yoga:** 10 minutes of Surya Namaskar or gentle stretching improves blood circulation.\n"
-         "4. **Protein-Rich Breakfast:** Avoid high-sugar cereals; choose eggs, oats, sprouts, or paneer to avoid energy crashes.\n"
-         "5. **Delay Screen Time:** Avoid checking emails or social media for the first 30 minutes to reduce morning stress.")
-    ]
-    for q, a in indian_culinary:
-        if not is_duplicate(q):
-            samples.append({"instruction": q, "response": a, "domain_id": 6})
-
-    # Stream from Dolly & Alpaca (cooking, food, lifestyle categories)
-    try:
-        ds_dolly = load_dataset("databricks/databricks-dolly-15k", split="train", streaming=True)
-        for row in ds_dolly:
-            q = (row.get("instruction") or "").strip()
-            ans = (row.get("response") or "").strip()
-            lower = q.lower()
-            if any(k in lower for k in ["recipe", "cook", "food", "health", "exercise", "habit", "tea", "coffee", "fruit"]):
-                if q and ans and not is_duplicate(q) and not contains_math_or_code(ans):
-                    samples.append({"instruction": clean_text(q), "response": clean_text(ans), "domain_id": 6})
-            if len(samples) >= 10000:
-                break
-    except Exception as e:
-        print(f"  ℹ️ Dolly lifestyle stream warning: {e}")
-
-    # Fallback from d6_hindi_culture
-    local_d6 = os.path.join(BASE_DIR, "data", "sarus_sft_raw", "d6_hindi_culture.jsonl")
-    if not os.path.exists(local_d6) and len(samples) < target:
-        try:
-            print(f"  📥 Downloading d6_hindi_culture.jsonl from HF ({RAW_REPO_ID})...")
-            os.makedirs(os.path.dirname(local_d6), exist_ok=True)
-            dl = hf_hub_download(repo_id=RAW_REPO_ID, filename="d6_hindi_culture.jsonl", repo_type="dataset", token=HF_TOKEN)
-            shutil.copy(dl, local_d6)
-        except Exception as e:
-            print(f"  ℹ️ HF fetch d6 notice: {e}")
-
-    raw_items = load_json_or_jsonl(local_d6)
-    for item in raw_items:
-        q = clean_text(item.get("instruction") or "")
-        r = clean_text(item.get("response") or "")
-        if q and r and not is_duplicate(q) and not contains_math_or_code(q) and not contains_math_or_code(r):
-            samples.append({"instruction": q, "response": r, "domain_id": 6})
-        if len(samples) >= target:
-            break
-
-    print(f"  ✅ [D6: Recipes & Lifestyle] Loaded {len(samples):,} lifestyle and recipe samples.")
-    return samples[:target]
-
-# --- D7: Bilingual Translation (EN <-> HI) ---
-def build_domain_translation(target=25000):
-    print(f"\n[D7: Translation] Sourcing clean English <-> Hindi sentence translation (Target: {target:,})...")
-    samples = []
-    
-    # 1. Local pre-filtered translation
-    local_d7 = os.path.join(BASE_DIR, "data", "sarus_sft_raw", "d7_summarization_translation.jsonl")
-    if os.path.exists(local_d7):
-        with open(local_d7, "r", encoding="utf-8") as f:
-            for line in f:
-                try:
-                    item = json.loads(line)
-                except Exception:
-                    continue
-                q = clean_text(item.get("instruction") or "")
-                r = clean_text(item.get("response") or "")
-                if "translate" in q.lower() or "अनुवाद" in q:
-                    if q and r and not is_duplicate(q) and not contains_math_or_code(r):
-                        samples.append({"instruction": q, "response": r, "domain_id": 7})
-                if len(samples) >= target:
-                    break
-
-    # 2. IITB English-Hindi stream
-    if len(samples) < target:
-        try:
-            print("  🌐 Streaming authentic translation pairs from cfilt/iitb-english-hindi...")
-            ds = load_dataset("cfilt/iitb-english-hindi", split="train", streaming=True)
-            for row in ds:
-                trans = row.get("translation", {})
-                en = (trans.get("en") or "").strip()
-                hi = (trans.get("hi") or "").strip()
-                if en and hi and 15 < len(en) < 200 and 15 < len(hi) < 200:
-                    if not contains_math_or_code(en) and not contains_math_or_code(hi):
-                        # Add EN -> HI
-                        q_en = f"Translate to Hindi: '{en}'"
-                        if not is_duplicate(q_en):
-                            samples.append({"instruction": q_en, "response": hi, "domain_id": 7})
-                        # Add HI -> EN
-                        q_hi = f"Translate to English: '{hi}'"
-                        if not is_duplicate(q_hi):
-                            samples.append({"instruction": q_hi, "response": en, "domain_id": 7})
-                if len(samples) >= target:
-                    break
-        except Exception as e:
-            print(f"  ℹ️ IITB stream warning: {e}")
-
-    print(f"  ✅ [D7: Translation] Loaded {len(samples):,} translation pairs.")
-    return samples[:target]
-
-
-# ------------------------------------------------------------------------------
-# 4. Phase 1: Build & Export Raw JSONL
-# ------------------------------------------------------------------------------
-def run_phase_raw(upload_hf=False):
-    print("\n" + "=" * 85)
-    print("🚀 Commencing SFT v2 Pure Chat Dataset Creation across all 8 Domains...")
-    print("=" * 85)
-    
-    domain_builders = [
-        (0, build_domain_identity),
-        (1, build_domain_greetings),
-        (2, build_domain_dialogue),
-        (3, build_domain_gk_science),
-        (4, build_domain_hindi_qa),
-        (5, build_domain_practical_writing),
-        (6, build_domain_recipes_lifestyle),
-        (7, build_domain_translation),
-    ]
-
-    all_samples = []
-    domain_counts = {}
-
-    for did, builder in domain_builders:
-        tgt = DOMAIN_TARGETS[did]
-        d_name = DOMAIN_NAMES[did]
-        d_samples = builder(tgt)
-        domain_counts[d_name] = len(d_samples)
-        all_samples.extend(d_samples)
-        
-        raw_file = os.path.join(RAW_EXPORT_DIR, f"d{did}_{d_name}.jsonl")
-        with open(raw_file, "w", encoding="utf-8") as f:
-            for item in d_samples:
-                f.write(json.dumps(item, ensure_ascii=False) + "\n")
-        print(f"   📁 Saved raw JSONL -> {raw_file} ({len(d_samples):,} rows)")
-
-    total_samples = len(all_samples)
-    print("\n" + "=" * 85)
-    print(f"🏆 SFT v2 Raw Dataset Assembly Completed: {total_samples:,} High-Purity Chat Samples!")
-    print("=" * 85)
-
-    # Fail-fast validation: identity + greetings are non-negotiable (persona grounding),
-    # and streaming failures must never silently produce a tiny/empty dataset.
-    for _did, _need in [(0, 500), (1, 500)]:
-        _got = domain_counts.get(DOMAIN_NAMES[_did], 0)
-        if _got < _need:
-            raise RuntimeError(
-                f"[D{_did}: {DOMAIN_NAMES[_did]}] only {_got:,} samples (need >={_need:,}). "
-                f"Check local file data/sarus_sft_raw/d{_did}_*.jsonl and HF_TOKEN/network access."
-            )
-    if total_samples < 50000:
-        raise RuntimeError(
-            f"Total {total_samples:,} samples is far below the ~202k target. "
-            f"Counts: {domain_counts}. A streaming source probably failed — inspect warnings above."
-        )
-
-    # Optional HF Upload
-    if upload_hf:
-        if not HF_TOKEN:
-            raise ValueError("HF_TOKEN env var is not set! Export HF_TOKEN before --upload_hf.")
-        print(f"\n🌐 Uploading raw data to Hugging Face ({RAW_REPO_ID}/sft_v2)...")
-        api = HfApi(token=HF_TOKEN)
-        try:
-            api.upload_folder(
-                folder_path=RAW_EXPORT_DIR,
-                repo_id=RAW_REPO_ID,
-                path_in_repo="sft_v2_chat",
-                repo_type="dataset",
-                commit_message=f"Upload Sarus SFT v2 Pure Chat Raw Corpus ({total_samples:,} samples)"
-            )
-            print("✅ Successfully uploaded raw JSONL to Hugging Face!")
-        except Exception as e:
-            print(f"⚠️ Upload warning: {e}")
-
-    return all_samples
-
-# ------------------------------------------------------------------------------
-# 5. Phase 2: Tokenize & Pack NumPy Shards
-# ------------------------------------------------------------------------------
-def run_phase_tokenize(all_samples=None, upload_hf=False):
-    global TOKENIZER
-    if TOKENIZER is None:
-        TOKENIZER = load_fast_tokenizer()
-
-    if all_samples is None:
-        print("\n📥 Loading raw JSONL files from disk for tokenization...")
-        all_samples = []
-        for did in sorted(DOMAIN_NAMES.keys()):
-            d_name = DOMAIN_NAMES[did]
-            fpath = os.path.join(RAW_EXPORT_DIR, f"d{did}_{d_name}.jsonl")
-            if os.path.exists(fpath):
-                with open(fpath, "r", encoding="utf-8") as f:
-                    for line in f:
-                        if line.strip():
-                            all_samples.append(json.loads(line))
-        print(f"   ✅ Loaded {len(all_samples):,} raw samples.")
-
-    total_samples = len(all_samples)
-    if total_samples == 0:
-        print("❌ Error: No samples to tokenize!")
-        return
-
-    print("\n⚡ [PHASE 2] Tokenizing into Memory-Mapped NumPy Shards for GPU Training...")
-    random.seed(42)
-    random.shuffle(all_samples)
-
-    train_tokens = []
-    train_labels = []
-    train_offsets = [0]
-    train_domains = []
-
-    val_tokens = []
-    val_labels = []
-    val_offsets = [0]
-    val_domains = []
-
-    val_split_count = int(total_samples * VAL_SPLIT_RATIO)
-    print(f"   • Train Split: {total_samples - val_split_count:,} samples | Val Split: {val_split_count:,} samples")
-
-    domain_counts = {}
-    for did in DOMAIN_NAMES.keys():
-        domain_counts[DOMAIN_NAMES[did]] = sum(1 for x in all_samples if x.get("domain_id") == did)
-
-    for i, item in enumerate(all_samples):
-        inst = item["instruction"]
-        resp = item["response"]
-        did = item["domain_id"]
-
-        prompt_str = f"{USER_TOKEN}\n{inst}{EOT_TOKEN}\n{ASSISTANT_TOKEN}\n"
-        resp_str = f"{resp}{EOT_TOKEN}"
-
-        p_ids = TOKENIZER.encode(prompt_str).ids
-        r_ids = TOKENIZER.encode(resp_str).ids
-
-        if len(r_ids) < MIN_TOKENS:
-            continue
-
-        sample_tokens = p_ids + r_ids
-        sample_labels = [-100] * len(p_ids) + r_ids
-
-        if len(sample_tokens) > MAX_SEQ_LEN:
-            sample_tokens = sample_tokens[:MAX_SEQ_LEN]
-            sample_labels = sample_labels[:MAX_SEQ_LEN]
-            sample_tokens[-1] = EOT_ID
-            sample_labels[-1] = EOT_ID
-
-        is_val = (i < val_split_count)
-        if is_val:
-            val_tokens.extend(sample_tokens)
-            val_labels.extend(sample_labels)
-            val_offsets.append(len(val_tokens))
-            val_domains.append(did)
-        else:
-            train_tokens.extend(sample_tokens)
-            train_labels.extend(sample_labels)
-            train_offsets.append(len(train_tokens))
-            train_domains.append(did)
-
-        if (i + 1) % 50000 == 0 or (i + 1) == total_samples:
-            print(f"   • Tokenized {i + 1:,} / {total_samples:,} samples...")
-
-    # Export NumPy Arrays
-    print("\n📦 Writing memory-mapped NumPy shards...")
-    np.save(os.path.join(TOKENIZED_DIR, "train_tokens.npy"), np.array(train_tokens, dtype=np.int32))
-    np.save(os.path.join(TOKENIZED_DIR, "train_labels.npy"), np.array(train_labels, dtype=np.int32))
-    np.save(os.path.join(TOKENIZED_DIR, "train_offsets.npy"), np.array(train_offsets, dtype=np.int64))
-    np.save(os.path.join(TOKENIZED_DIR, "train_domains.npy"), np.array(train_domains, dtype=np.uint8))
-
-    np.save(os.path.join(TOKENIZED_DIR, "val_tokens.npy"), np.array(val_tokens, dtype=np.int32))
-    np.save(os.path.join(TOKENIZED_DIR, "val_labels.npy"), np.array(val_labels, dtype=np.int32))
-    np.save(os.path.join(TOKENIZED_DIR, "val_offsets.npy"), np.array(val_offsets, dtype=np.int64))
-    np.save(os.path.join(TOKENIZED_DIR, "val_domains.npy"), np.array(val_domains, dtype=np.uint8))
-
-    meta_payload = {
-        "version": "v2",
-        "model_name": "Sarus",
-        "developer": "ViuAI",
-        "dataset_type": "pure_chat",
-        "total_samples": total_samples,
-        "train_samples": len(train_offsets) - 1,
-        "val_samples": len(val_offsets) - 1,
-        "train_tokens_total": len(train_tokens),
-        "val_tokens_total": len(val_tokens),
-        "max_seq_len": MAX_SEQ_LEN,
-        "domain_names": DOMAIN_NAMES,
-        "domain_sample_counts": domain_counts
-    }
-
-    with open(os.path.join(TOKENIZED_DIR, "metadata.json"), "w", encoding="utf-8") as f:
-        json.dump(meta_payload, f, indent=2, ensure_ascii=False)
-
-    print(f"✅ Saved metadata.json with {len(DOMAIN_NAMES)} domains and stats!")
-
-    if upload_hf:
-        if not HF_TOKEN:
-            raise ValueError("HF_TOKEN env var is not set! Export HF_TOKEN before --upload_hf.")
-        print(f"\n🌐 Uploading tokenized shards to Hugging Face ({TOKENIZED_REPO_ID}/sft_v2)...")
-        api = HfApi(token=HF_TOKEN)
-        try:
-            api.upload_folder(
-                folder_path=TOKENIZED_DIR,
-                repo_id=TOKENIZED_REPO_ID,
-                path_in_repo="sft_v2",
-                repo_type="dataset",
-                commit_message=f"Upload Sarus SFT v2 Pure Chat Tokenized Shards ({total_samples:,} samples)"
-            )
-            print("✅ Successfully uploaded tokenized dataset to Hugging Face!")
-        except Exception as e:
-            print(f"⚠️ Upload warning: {e}")
-
-    print("\n" + "=" * 85)
-    print("🎉 [PHASE 2 COMPLETE] TOKENIZED SHARDS READY FOR HIGH-SPEED TRAINING!")
-    print(f"   • Train Tokens:    {len(train_tokens):,}")
-    print(f"   • Val Tokens:      {len(val_tokens):,}")
-    print(f"   • Shard Directory: {TOKENIZED_DIR}")
-    print("=" * 85)
-
-# ------------------------------------------------------------------------------
-# Main CLI
-# ------------------------------------------------------------------------------
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description="ViuAI Sarus-500M SFT v2 Pure Chat Dataset Builder")
-    parser.add_argument("--phase", type=str, default="all", choices=["raw", "tokenize", "all"], help="Execution phase")
-    parser.add_argument("--upload_hf", action="store_true", default=False, help="Upload outputs to Hugging Face")
-    args = parser.parse_args()
-
-    if args.phase in ["raw", "all"]:
-        samples = run_phase_raw(upload_hf=args.upload_hf)
-        if args.phase == "all":
-            run_phase_tokenize(all_samples=samples, upload_hf=args.upload_hf)
-    elif args.phase == "tokenize":
-        run_phase_tokenize(all_samples=None, upload_hf=args.upload_hf)