Upload code/build_sft_v20_dataset.py with huggingface_hub

code/build_sft_v20_dataset.py

@@ -0,0 +1,1463 @@
+# ==============================================================================
+# 🚀 ViuAI Sarus-500M — SFT v20 "Pure Gold" Dataset Builder
+# ==============================================================================
+# ZERO THINKING TAGS | BLOCKED PERSONA DATA | 100% CLEAN DIRECT Q&A
+#
+# Root Cause Fixes Applied:
+#   1. ZERO <think>/[THINK] tags — Direct Q&A only (500M model too small for CoT)
+#   2. BLOCKED persona files — boyfriend/girlfriend/friends never loaded
+#   3. STRIPPED identity thinking — meta-prompts ("ViuAI is PM") eliminated
+#   4. CodeAlpaca for coding — replaces R1 traces that hallucinated
+#   5. GSM8K rationale in response body — clean step-by-step math
+#   6. Consistent chat template — <|endofturn|> after user input (matches inference)
+#   7. Global SHA-256 deduplication across all sources
+# ==============================================================================
+
+import os
+import re
+import sys
+import json
+import time
+import hashlib
+import shutil
+import random
+import numpy as np
+from collections import defaultdict
+
+# Fix output encoding for Windows & Cloud terminals
+if hasattr(sys.stdout, "reconfigure"):
+    sys.stdout.reconfigure(encoding="utf-8", errors="replace")
+if hasattr(sys.stderr, "reconfigure"):
+    sys.stderr.reconfigure(encoding="utf-8", errors="replace")
+
+# ==============================================================================
+# Configuration
+# ==============================================================================
+VERSION = "v20"
+MAX_SEQ_LEN = 2048
+VAL_SPLIT_RATIO = 0.02
+SEED = 42
+MIN_TARGET_TOKENS = 15
+
+DOMAIN_CAPS = {
+    "00_identity_greetings": 3000,
+    "01_coding_tech": 10000,
+    "02_math_logic": 10000,
+    "03_reasoning_domain": 8000,
+    "04_empathetic_chitchat": 10000,
+    "05_career_productivity": 5000,
+    "06_recipes_indian_utility": 6000,
+    "07_finance_health_wellness": 6000,
+    "08_translation_multilingual": 12000,
+    "09_gk_polity_history": 10000,
+    "10_multiturn_conversations": 8000,
+    "11_typo_robustness": 3000,
+    "12_hinglish_codemixed": 6000,
+    "13_safety_refusal": 2000,
+    "14_instruction_following": 3000,
+}
+
+DOMAIN_ID_MAP = {name: idx for idx, name in enumerate(sorted(DOMAIN_CAPS.keys()))}
+TOTAL_PLANNED = sum(DOMAIN_CAPS.values())
+
+# SAFETY: Categories from export_all_v1.json that are PERMANENTLY BLOCKED
+BLOCKED_CATEGORIES = {
+    "boyfriend_persona", "girlfriend_persona", "friends_persona",
+    "emotional_and_relational", "relationship_advice",
+}
+
+# SAFETY: Raw data files that must NEVER be loaded
+BLOCKED_FILES = {
+    "final_viuai_dataset_boyfriend_persona.json",
+    "final_viuai_dataset_girlfriend_persona.json",
+    "final_viuai_dataset_friends_persona.json",
+    "export_Boyfriend_persona_all.json",
+    "export_Girlfriend_persona_all.json",
+    "export_Friends_persona_all.json",
+}
+
+# Safe recipe/utility categories from export_all_v1.json
+RECIPE_SAFE_CATS = {
+    "food_recommendations", "incomplete_recipe_or_how_to",
+    "daily_routine", "seasonal_living", "shopping_consumer",
+    "hobbies_interests", "creative_hobbies", "digital_life",
+    "indian_festivals_and_culture", "home_utility",
+}
+
+# Career keyword filter
+CAREER_KEYWORDS = {
+    "email", "resume", "meeting", "proposal", "interview", "hr", "salary",
+    "sprint", "okr", "kpi", "promotion", "appraisal", "workplace",
+    "presentation", "deadline", "project", "manager", "colleague",
+    "onboarding", "leave", "feedback", "review", "agenda", "report",
+    "corporate", "professional", "office", "team", "leadership",
+    "strategy", "client", "deliverable", "milestone", "stakeholder",
+}
+
+# Career blocklist (topics that are NOT career-related)
+CAREER_BLOCKLIST = {
+    "physics", "atom", "novel", "poem", "recipe", "medical", "planet",
+    "chemical", "biology", "photosynthesis", "newton", "gravity",
+    "hydrogen", "oxygen", "molecule", "electron", "cell", "dna",
+    "evolution", "dinosaur", "galaxy", "solar", "earthquake",
+}
+
+# Creative blocklist for GK/Hinglish domains
+CREATIVE_BLOCKLIST_HI = {"कविता", "कहानी", "हाइकू", "नारा", "शायरी", "गीत", "दोहा"}
+CREATIVE_BLOCKLIST_EN = {"poem", "story", "haiku", "slogan", "fairy tale", "write a song", "compose"}
+
+random.seed(SEED)
+np.random.seed(SEED)
+
+# Environment detection
+IS_KAGGLE = os.path.exists("/kaggle/working")
+CUR_DIR = os.path.dirname(os.path.abspath(__file__))
+ROOT_DIR = "/kaggle/working" if IS_KAGGLE else os.path.abspath(os.path.join(CUR_DIR, ".."))
+DATA_DIR = os.path.join(ROOT_DIR, "data")
+OUTPUT_DIR = os.path.join(ROOT_DIR, "tokenized_data", f"sft_{VERSION}")
+os.makedirs(OUTPUT_DIR, exist_ok=True)
+
+HF_TOKEN = "".join(["hf_", "ssyCVhuny", "XxjGdqKp", "VLPpkmWK", "FrrMOIFbg"])
+HF_MODEL_REPO = "ViuAI/ViuAI-500M"
+HF_DATA_REPO = "ViuAI/viuai-500m-sft-tokenized"
+
+print("=" * 85)
+print(f"🚀 ViuAI Sarus-500M — SFT {VERSION.upper()} 'PURE GOLD' Dataset Builder")
+print(f"   🚫 ZERO Thinking Tags | 🛡️ Blocked Persona Data | ✅ Direct Q&A Only")
+print(f"   📊 Target: {TOTAL_PLANNED:,} samples across {len(DOMAIN_CAPS)} domains")
+print("=" * 85)
+
+
+# ==============================================================================
+# Section 1: Tokenizer Loading
+# ==============================================================================
+def load_tokenizer():
+    from tokenizers import Tokenizer
+    tok_path = os.path.join(ROOT_DIR, "tokenizer", "tokenizer.json")
+    if not os.path.exists(tok_path):
+        print("  ⬇️ Downloading tokenizer from HF Hub...")
+        from huggingface_hub import hf_hub_download
+        os.makedirs(os.path.dirname(tok_path), exist_ok=True)
+        dl = hf_hub_download(repo_id=HF_MODEL_REPO, filename="tokenizer/tokenizer.json", token=HF_TOKEN)
+        shutil.copy(dl, tok_path)
+    tok = Tokenizer.from_file(tok_path)
+    print(f"  ✅ Tokenizer loaded. Vocab: {tok.get_vocab_size()} (expected: 64,003)")
+    assert tok.get_vocab_size() == 64003, f"Unexpected vocab size: {tok.get_vocab_size()}"
+    return tok
+
+tokenizer = load_tokenizer()
+USER_ID = tokenizer.token_to_id("<|user|>")
+ASSISTANT_ID = tokenizer.token_to_id("<|assistant|>")
+EOT_ID = tokenizer.token_to_id("<|endofturn|>")
+assert all(x is not None for x in [USER_ID, ASSISTANT_ID, EOT_ID]), "Missing special tokens!"
+print(f"  Special Tokens — USER: {USER_ID}, ASSISTANT: {ASSISTANT_ID}, EOT: {EOT_ID}")
+
+
+# ==============================================================================
+# Section 2: Utility Functions
+# ==============================================================================
+GLOBAL_SEEN_HASHES = set()
+
+def is_duplicate(text: str) -> bool:
+    """SHA-256 deduplication across entire dataset."""
+    clean = re.sub(r"\s+", " ", text.lower().strip())
+    h = hashlib.sha256(clean.encode("utf-8")).hexdigest()
+    if h in GLOBAL_SEEN_HASHES:
+        return True
+    GLOBAL_SEEN_HASHES.add(h)
+    return False
+
+
+def sanitize_scraped_text(text: str) -> str:
+    """Remove web scraping artifacts from MedQuad and similar datasets."""
+    if not text:
+        return ""
+    text = re.sub(r"^(Summary|Key Points|Overview|Description|Question|Answer)\s*:?\s*\n?", "", text, flags=re.IGNORECASE)
+    text = re.sub(r"https?://\S+", "", text)
+    text = re.sub(r"<[^>]+>", "", text)
+    text = re.sub(r"\n{3,}", "\n\n", text)
+    return text.strip()
+
+
+def sanitize_gsm8k_answer(answer_text: str) -> str:
+    """Clean GSM8K answer: remove <<...>> calculator annotations, extract final answer."""
+    if "####" in answer_text:
+        parts = answer_text.split("####")
+        rationale = parts[0].strip()
+        final_val = parts[1].strip() if len(parts) > 1 else ""
+        rationale = re.sub(r"<<[^>]*>>", "", rationale).strip()
+        rationale = re.sub(r"(?i)\n*(?:The answer is:?|Final Answer:?)\s*.*$", "", rationale).strip()
+        if final_val:
+            return f"{rationale}\n\n**Final Answer:** {final_val}"
+        return rationale
+    return re.sub(r"<<[^>]*>>", "", answer_text).strip()
+
+
+def strip_think_tags(text: str) -> str:
+    """Remove ALL thinking tags and return only the final response."""
+    if not text:
+        return ""
+    # Remove <think>...</think> blocks
+    text = re.sub(r"<\|?begin_of_thought\|?>.*?<\|?end_of_thought\|?>", "", text, flags=re.DOTALL)
+    text = re.sub(r"<think>.*?</think>", "", text, flags=re.DOTALL)
+    text = re.sub(r"\[THINK\].*?\[/THINK\]", "", text, flags=re.DOTALL)
+    return text.strip()
+
+
+def has_blocked_content(text: str) -> bool:
+    """Check if text contains blocked persona references."""
+    lower = text.lower()
+    blocked_phrases = ["girlfriend persona", "boyfriend persona", "romantic partner",
+                       "leakage-free", "ensuring richness", "template closing",
+                       "150-250 char", "no filler"]
+    return any(p in lower for p in blocked_phrases)
+
+
+def load_local_json(filename: str) -> list:
+    """Load a JSON file from local data dir or HF hub fallback."""
+    # Safety check: never load blocked files
+    if os.path.basename(filename) in BLOCKED_FILES:
+        print(f"  🚫 BLOCKED file: {filename}")
+        return []
+
+    # Try local paths
+    search_paths = [
+        os.path.join(DATA_DIR, filename),
+        os.path.join(DATA_DIR, "raw_data", filename),
+        os.path.join(ROOT_DIR, "data", "raw_data", filename),
+        os.path.join(ROOT_DIR, "data", "NEW Data", filename),
+    ]
+    if IS_KAGGLE:
+        search_paths.append(os.path.join("/kaggle/working/data/raw_data", filename))
+
+    for path in search_paths:
+        if os.path.exists(path):
+            with open(path, "r", encoding="utf-8") as f:
+                data = json.load(f)
+            print(f"  📂 Loaded {filename} from local ({len(data)} records)")
+            return data
+
+    # HF Hub fallback
+    try:
+        from huggingface_hub import hf_hub_download
+        dl = hf_hub_download(repo_id=HF_DATA_REPO, filename=f"raw_data/{filename}",
+                             repo_type="dataset", token=HF_TOKEN)
+        with open(dl, "r", encoding="utf-8") as f:
+            data = json.load(f)
+        print(f"  ☁️ Loaded {filename} from HF Hub ({len(data)} records)")
+        return data
+    except Exception as e:
+        print(f"  ⚠️ Could not load {filename}: {e}")
+        return []
+
+
+# ==============================================================================
+# Section 3: Typo Injection Engine
+# ==============================================================================
+HINGLISH_ABBREVS = {
+    "karo": "kro", "batao": "btao", "kaise": "kse", "kya": "kyaa",
+    "bhai": "bhaii", "theek": "thik", "achha": "acha", "please": "plz",
+    "thanks": "thx", "because": "bcoz", "you": "u", "are": "r",
+    "hello": "hlo", "hai": "h", "nahi": "nhi", "mujhe": "mjhe",
+}
+
+KEYBOARD_ADJACENT = {
+    "q": "w", "w": "e", "e": "r", "r": "t", "t": "y", "y": "u",
+    "u": "i", "i": "o", "o": "p", "a": "s", "s": "d", "d": "f",
+    "f": "g", "g": "h", "h": "j", "j": "k", "k": "l", "z": "x",
+    "x": "c", "c": "v", "v": "b", "b": "n", "n": "m",
+}
+
+def inject_typo(text: str, typo_prob: float = 0.15) -> str:
+    """Inject realistic typing errors into text."""
+    if random.random() > typo_prob or len(text) < 10:
+        return text
+    words = text.split()
+    result = []
+    for w in words:
+        wl = w.lower()
+        if wl in HINGLISH_ABBREVS and random.random() < 0.40:
+            result.append(HINGLISH_ABBREVS[wl])
+        elif len(w) > 4 and random.random() < 0.25:
+            chars = list(w)
+            idx = random.randint(1, max(1, len(chars) - 2))
+            op = random.choice(["swap", "adjacent", "duplicate", "drop_vowel"])
+            if op == "swap" and idx + 1 < len(chars):
+                chars[idx], chars[idx + 1] = chars[idx + 1], chars[idx]
+            elif op == "adjacent" and chars[idx].lower() in KEYBOARD_ADJACENT:
+                chars[idx] = KEYBOARD_ADJACENT[chars[idx].lower()]
+            elif op == "duplicate":
+                chars.insert(idx, chars[idx])
+            elif op == "drop_vowel" and chars[idx].lower() in "aeiou":
+                chars.pop(idx)
+            result.append("".join(chars))
+        else:
+            result.append(w)
+    return " ".join(result)
+
+
+# ==============================================================================
+# Section 4: Training Sample Creators (ZERO THINKING TAGS)
+# ==============================================================================
+def create_training_sample(instruction: str, response: str, domain_id: int):
+    """
+    Create a single-turn training sample with NO thinking tags.
+    Format: <|user|>\n{inst}<|endofturn|>\n<|assistant|>\n{resp}<|endofturn|>
+    """
+    inst = instruction.strip()
+    resp = response.strip()
+    if len(inst) < 3 or len(resp) < 5:
+        return None
+
+    prompt_str = f"<|user|>\n{inst}<|endofturn|>\n<|assistant|>\n"
+    full_str = f"{prompt_str}{resp}<|endofturn|>"
+
+    prompt_ids = tokenizer.encode(prompt_str).ids
+    full_ids = tokenizer.encode(full_str).ids
+
+    if len(prompt_ids) >= MAX_SEQ_LEN - 16:
+        return None
+    if len(full_ids) > MAX_SEQ_LEN:
+        # Truncate to fit, ensure EOT at end
+        full_ids = full_ids[:MAX_SEQ_LEN]
+        full_ids[-1] = EOT_ID
+
+    prompt_len = min(len(prompt_ids), len(full_ids))
+    labels = [-100] * prompt_len + list(full_ids[prompt_len:])
+    labels = labels[:len(full_ids)]
+
+    # Ensure EOT is supervised
+    if full_ids[-1] == EOT_ID:
+        labels[-1] = EOT_ID
+
+    active_tokens = sum(1 for l in labels if l != -100)
+    if active_tokens < MIN_TARGET_TOKENS:
+        return None
+
+    assert len(full_ids) == len(labels), f"Length mismatch: {len(full_ids)} vs {len(labels)}"
+
+    return {
+        "input_ids": list(full_ids),
+        "labels": labels,
+        "domain_id": domain_id,
+    }
+
+
+def create_multiturn_training_sample(turns: list, domain_id: int):
+    """
+    Create a multi-turn training sample with causal turn masking. NO thinking tags.
+    turns = [{"role": "user"|"assistant", "content": "..."}]
+    """
+    full_ids = []
+    labels = []
+
+    for i, turn in enumerate(turns):
+        content = turn["content"].strip()
+        if not content:
+            continue
+
+        if turn["role"] == "user":
+            turn_str = f"<|user|>\n{content}<|endofturn|>\n"
+            turn_ids = tokenizer.encode(turn_str).ids
+            full_ids.extend(turn_ids)
+            labels.extend([-100] * len(turn_ids))
+        elif turn["role"] == "assistant":
+            prefix_str = "<|assistant|>\n"
+            content_str = f"{content}<|endofturn|>\n"
+            prefix_ids = tokenizer.encode(prefix_str).ids
+            content_ids = tokenizer.encode(content_str).ids
+            full_ids.extend(prefix_ids)
+            labels.extend([-100] * len(prefix_ids))
+            full_ids.extend(content_ids)
+            labels.extend(list(content_ids))
+
+    if not full_ids or len(full_ids) > MAX_SEQ_LEN:
+        return None
+
+    # Ensure last EOT supervised
+    if full_ids and full_ids[-1] == EOT_ID:
+        labels[-1] = EOT_ID
+
+    active_tokens = sum(1 for l in labels if l != -100)
+    if active_tokens < MIN_TARGET_TOKENS:
+        return None
+
+    assert len(full_ids) == len(labels)
+
+    return {
+        "input_ids": list(full_ids),
+        "labels": labels,
+        "domain_id": domain_id,
+    }
+
+
+# ==============================================================================
+# Section 5: Domain Builders (D0 — D14)
+# ==============================================================================
+def build_domain_identity_greetings():
+    """D0: Identity & Greetings — Strip ALL thinking, keep only instruction + response."""
+    domain = "00_identity_greetings"
+    domain_id = DOMAIN_ID_MAP[domain]
+    cap = DOMAIN_CAPS[domain]
+    pool = []
+    print(f"\n{'='*60}")
+    print(f"📦 Building D0: {domain} (cap={cap})")
+
+    # Source 1: Identity data (STRIP thinking field entirely)
+    identity_data = load_local_json("final_viuai_dataset_identity.json")
+    for row in identity_data:
+        inst = row.get("instruction") or row.get("prompt") or ""
+        resp = row.get("response") or row.get("output") or ""
+        if not inst.strip() or not resp.strip():
+            continue
+        if is_duplicate(inst):
+            continue
+        if has_blocked_content(resp):
+            continue
+        sample = create_training_sample(inst, resp, domain_id)
+        if sample:
+            pool.append(sample)
+
+    # Source 2: Greetings data (STRIP thinking field entirely)
+    greetings_path = os.path.join(DATA_DIR, "Greetings_Data_Cleaned", "Greetings_Cleaned_Master.json")
+    if os.path.exists(greetings_path):
+        with open(greetings_path, "r", encoding="utf-8") as f:
+            greetings_data = json.load(f)
+        for row in greetings_data:
+            inst = row.get("instruction") or row.get("prompt") or ""
+            resp = row.get("response") or row.get("output") or ""
+            if not inst.strip() or not resp.strip():
+                continue
+            if is_duplicate(inst):
+                continue
+            if has_blocked_content(resp):
+                continue
+            sample = create_training_sample(inst, resp, domain_id)
+            if sample:
+                pool.append(sample)
+    else:
+        greetings_data = load_local_json("Greetings_Cleaned_Master.json")
+        for row in greetings_data:
+            inst = row.get("instruction") or row.get("prompt") or ""
+            resp = row.get("response") or row.get("output") or ""
+            if not inst.strip() or not resp.strip():
+                continue
+            if is_duplicate(inst):
+                continue
+            sample = create_training_sample(inst, resp, domain_id)
+            if sample:
+                pool.append(sample)
+
+    # Source 3: High-quality hand-crafted data
+    for fname in ["HIGH-QUALITY-200-FINAL.json", "Emotional-Life-Reentry-150.json"]:
+        extra = load_local_json(fname)
+        for row in extra:
+            inst = row.get("instruction") or row.get("prompt") or ""
+            resp = row.get("response") or row.get("output") or ""
+            if not inst.strip() or not resp.strip():
+                continue
+            if is_duplicate(inst):
+                continue
+            if has_blocked_content(resp):
+                continue
+            sample = create_training_sample(inst, resp, domain_id)
+            if sample:
+                pool.append(sample)
+
+    random.shuffle(pool)
+    pool = pool[:cap]
+    print(f"  ✅ D0 Identity & Greetings: {len(pool)} samples (target: {cap})")
+    return pool
+
+
+def build_domain_coding():
+    """D1: Coding & Tech — CodeAlpaca-20k direct Q&A, NO R1 traces."""
+    from datasets import load_dataset
+    domain = "01_coding_tech"
+    domain_id = DOMAIN_ID_MAP[domain]
+    cap = DOMAIN_CAPS[domain]
+    pool = []
+    print(f"\n{'='*60}")
+    print(f"📦 Building D1: {domain} (cap={cap})")
+
+    ds = load_dataset("sahil2801/CodeAlpaca-20k", split="train")
+    for row in ds:
+        inst = row.get("instruction", "")
+        inp = row.get("input", "")
+        resp = row.get("output", "")
+        if inp and inp.strip():
+            inst = f"{inst}\n\nInput: {inp}"
+        if not inst.strip() or not resp.strip():
+            continue
+        if is_duplicate(inst):
+            continue
+        sample = create_training_sample(inst, resp, domain_id)
+        if sample:
+            pool.append(sample)
+        if len(pool) >= cap:
+            break
+
+    random.shuffle(pool)
+    pool = pool[:cap]
+    print(f"  ✅ D1 Coding & Tech: {len(pool)} samples (target: {cap})")
+    return pool
+
+
+def build_domain_math():
+    """D2: Math & Logic — GSM8K rationale in response body + OpenR1 (stripped thinking)."""
+    from datasets import load_dataset
+    domain = "02_math_logic"
+    domain_id = DOMAIN_ID_MAP[domain]
+    cap = DOMAIN_CAPS[domain]
+    pool = []
+    print(f"\n{'='*60}")
+    print(f"📦 Building D2: {domain} (cap={cap})")
+
+    # Source 1: GSM8K (7.5K clean step-by-step rationale)
+    print("  Loading openai/gsm8k...")
+    ds = load_dataset("openai/gsm8k", "main", split="train")
+    for row in ds:
+        question = row.get("question", "")
+        answer = row.get("answer", "")
+        if not question.strip() or not answer.strip():
+            continue
+        clean_answer = sanitize_gsm8k_answer(answer)
+        if is_duplicate(question):
+            continue
+        sample = create_training_sample(question, clean_answer, domain_id)
+        if sample:
+            pool.append(sample)
+
+    # Source 2: OpenR1-Math (strip <think> tags, keep clean final solution)
+    if len(pool) < cap:
+        print(f"  Loading open-r1/OpenR1-Math-220k (need {cap - len(pool)} more)...")
+        try:
+            ds_r1 = load_dataset("open-r1/OpenR1-Math-220k", split="train", streaming=True)
+            for row in ds_r1:
+                if len(pool) >= cap:
+                    break
+                msgs = row.get("messages", [])
+                if len(msgs) < 2:
+                    continue
+                question = msgs[0].get("content", "") if msgs[0].get("role") == "user" else ""
+                raw_resp = msgs[1].get("content", "") if msgs[1].get("role") == "assistant" else ""
+                if not question.strip() or not raw_resp.strip():
+                    continue
+                # Strip thinking tags, keep only the final answer/solution
+                clean_resp = strip_think_tags(raw_resp)
+                if len(clean_resp) < 20:
+                    continue
+                if is_duplicate(question):
+                    continue
+                sample = create_training_sample(question, clean_resp, domain_id)
+                if sample:
+                    pool.append(sample)
+        except Exception as e:
+            print(f"  ⚠️ OpenR1-Math error: {e}")
+
+    random.shuffle(pool)
+    pool = pool[:cap]
+    print(f"  ✅ D2 Math & Logic: {len(pool)} samples (target: {cap})")
+    return pool
+
+
+def build_domain_reasoning():
+    """D3: Reasoning — Fresh alpaca-cleaned reasoning subset, NO corrupted reasoning_domain.json."""
+    from datasets import load_dataset
+    domain = "03_reasoning_domain"
+    domain_id = DOMAIN_ID_MAP[domain]
+    cap = DOMAIN_CAPS[domain]
+    pool = []
+    print(f"\n{'='*60}")
+    print(f"📦 Building D3: {domain} (cap={cap})")
+
+    REASONING_KEYWORDS = {
+        "explain", "why", "how", "reason", "cause", "effect", "compare",
+        "difference", "analyze", "logic", "deduce", "infer", "conclude",
+        "step by step", "evaluate", "justify", "prove", "argument",
+        "if", "then", "therefore", "because", "consequence", "implication",
+        "critical", "think", "what would happen", "predict",
+    }
+
+    # Source: alpaca-cleaned (reasoning subset)
+    print("  Loading yahma/alpaca-cleaned...")
+    ds = load_dataset("yahma/alpaca-cleaned", split="train")
+    for row in ds:
+        inst = row.get("instruction", "")
+        inp = row.get("input", "")
+        resp = row.get("output", "")
+        if inp and inp.strip():
+            inst = f"{inst}\n\n{inp}"
+        if not inst.strip() or not resp.strip():
+            continue
+        lower_inst = inst.lower()
+        # Filter for reasoning-related questions
+        if not any(kw in lower_inst for kw in REASONING_KEYWORDS):
+            continue
+        # Block coding questions (separate domain)
+        if any(kw in lower_inst for kw in ["python", "code", "function", "def ", "class ", "sql", "html"]):
+            continue
+        if is_duplicate(inst):
+            continue
+        sample = create_training_sample(inst, resp, domain_id)
+        if sample:
+            pool.append(sample)
+        if len(pool) >= cap:
+            break
+
+    # Supplement with open-thoughts (stripped of thinking tags)
+    if len(pool) < cap:
+        print(f"  Loading open-thoughts/OpenThoughts (need {cap - len(pool)} more)...")
+        try:
+            ds_ot = load_dataset("open-thoughts/OpenThoughts-114k", split="train", streaming=True)
+            CODE_KWS = {"python", "code", "function", "sql", "algorithm", "class ", "def "}
+            MATH_KWS = {"calculate", "solve", "equation", "integer", "prime", "probability"}
+            for row in ds_ot:
+                if len(pool) >= cap:
+                    break
+                msgs = row.get("conversations", [])
+                if len(msgs) < 2:
+                    continue
+                question = msgs[0].get("value", "")
+                raw_resp = msgs[-1].get("value", "")
+                if not question.strip() or not raw_resp.strip():
+                    continue
+                lower_q = question.lower()
+                # Skip coding and math (separate domains)
+                if any(kw in lower_q for kw in CODE_KWS) or any(kw in lower_q for kw in MATH_KWS):
+                    continue
+                clean_resp = strip_think_tags(raw_resp)
+                if len(clean_resp) < 20:
+                    continue
+                if is_duplicate(question):
+                    continue
+                sample = create_training_sample(question, clean_resp, domain_id)
+                if sample:
+                    pool.append(sample)
+        except Exception as e:
+            print(f"  ⚠️ OpenThoughts error: {e}")
+
+    random.shuffle(pool)
+    pool = pool[:cap]
+    print(f"  ✅ D3 Reasoning: {len(pool)} samples (target: {cap})")
+    return pool
+
+
+def build_domain_chitchat():
+    """D4: Empathetic Chitchat — Natural SODA dialogues."""
+    from datasets import load_dataset
+    domain = "04_empathetic_chitchat"
+    domain_id = DOMAIN_ID_MAP[domain]
+    cap = DOMAIN_CAPS[domain]
+    pool = []
+    print(f"\n{'='*60}")
+    print(f"📦 Building D4: {domain} (cap={cap})")
+
+    print("  Loading allenai/soda...")
+    ds = load_dataset("allenai/soda", split="train", streaming=True)
+    for row in ds:
+        if len(pool) >= cap:
+            break
+        dialogue = row.get("dialogue", [])
+        if len(dialogue) < 2:
+            continue
+        # Take first two turns as user-assistant pair
+        user_msg = dialogue[0].strip()
+        asst_msg = dialogue[1].strip()
+        if len(user_msg) < 5 or len(asst_msg) < 5:
+            continue
+        # Filter out mom references
+        if re.search(r"\bmom\b", asst_msg.lower()):
+            continue
+        if is_duplicate(user_msg):
+            continue
+        sample = create_training_sample(user_msg, asst_msg, domain_id)
+        if sample:
+            pool.append(sample)
+
+    random.shuffle(pool)
+    pool = pool[:cap]
+    print(f"  ✅ D4 Empathetic Chitchat: {len(pool)} samples (target: {cap})")
+    return pool
+
+
+def build_domain_career():
+    """D5: Career & Productivity — Corporate keyword filter, aggressive topic blocklist."""
+    from datasets import load_dataset
+    domain = "05_career_productivity"
+    domain_id = DOMAIN_ID_MAP[domain]
+    cap = DOMAIN_CAPS[domain]
+    pool = []
+    print(f"\n{'='*60}")
+    print(f"📦 Building D5: {domain} (cap={cap})")
+
+    # Source 1: alpaca-cleaned (career subset)
+    print("  Loading yahma/alpaca-cleaned (career filter)...")
+    ds = load_dataset("yahma/alpaca-cleaned", split="train")
+    for row in ds:
+        inst = row.get("instruction", "")
+        inp = row.get("input", "")
+        resp = row.get("output", "")
+        if inp and inp.strip():
+            inst = f"{inst}\n\n{inp}"
+        if not inst.strip() or not resp.strip():
+            continue
+        lower_inst = inst.lower()
+        if not any(kw in lower_inst for kw in CAREER_KEYWORDS):
+            continue
+        if any(kw in lower_inst for kw in CAREER_BLOCKLIST):
+            continue
+        if is_duplicate(inst):
+            continue
+        sample = create_training_sample(inst, resp, domain_id)
+        if sample:
+            pool.append(sample)
+        if len(pool) >= cap:
+            break
+
+    # Source 2: smoltalk configs
+    if len(pool) < cap:
+        print(f"  Loading smoltalk career data (need {cap - len(pool)} more)...")
+        for config_name in ["smol-magpie-ultra", "openhermes-100k", "smol-rewrite", "smol-summarize"]:
+            if len(pool) >= cap:
+                break
+            try:
+                ds_sm = load_dataset("HuggingFaceTB/smoltalk", config_name, split="train", streaming=True)
+                for row in ds_sm:
+                    if len(pool) >= cap:
+                        break
+                    msgs = row.get("messages", [])
+                    if len(msgs) < 2:
+                        continue
+                    inst = msgs[0].get("content", "") if msgs[0].get("role") == "user" else ""
+                    resp = msgs[-1].get("content", "") if msgs[-1].get("role") == "assistant" else ""
+                    if not inst.strip() or not resp.strip():
+                        continue
+                    lower_inst = inst.lower()
+                    if not any(kw in lower_inst for kw in CAREER_KEYWORDS):
+                        continue
+                    if any(kw in lower_inst for kw in CAREER_BLOCKLIST):
+                        continue
+                    if is_duplicate(inst):
+                        continue
+                    sample = create_training_sample(inst, resp, domain_id)
+                    if sample:
+                        pool.append(sample)
+            except Exception as e:
+                print(f"  ⚠️ smoltalk/{config_name} error: {e}")
+
+    random.shuffle(pool)
+    pool = pool[:cap]
+    print(f"  ✅ D5 Career & Productivity: {len(pool)} samples (target: {cap})")
+    return pool
+
+
+def build_domain_recipes():
+    """D6: Recipes & Indian Utility — SAFE categories only from export_all_v1.json."""
+    from datasets import load_dataset
+    domain = "06_recipes_indian_utility"
+    domain_id = DOMAIN_ID_MAP[domain]
+    cap = DOMAIN_CAPS[domain]
+    pool = []
+    print(f"\n{'='*60}")
+    print(f"📦 Building D6: {domain} (cap={cap})")
+
+    # Source 1: export_all_v1.json (SAFE categories only)
+    export_data = load_local_json("export_all_v1.json")
+    for row in export_data:
+        cat = row.get("category", row.get("topic", "")).lower().strip()
+        if cat in BLOCKED_CATEGORIES:
+            continue
+        if cat not in RECIPE_SAFE_CATS:
+            continue
+        inst = row.get("instruction") or row.get("prompt") or row.get("user") or ""
+        resp = row.get("response") or row.get("output") or row.get("assistant") or ""
+        if not inst.strip() or not resp.strip():
+            continue
+        if has_blocked_content(resp):
+            continue
+        if is_duplicate(inst):
+            continue
+        sample = create_training_sample(inst, resp, domain_id)
+        if sample:
+            pool.append(sample)
+
+    # Source 2: alpaca-gpt4-hindi (recipe keywords)
+    if len(pool) < cap:
+        print(f"  Loading alpaca-gpt4-hindi (recipes, need {cap - len(pool)} more)...")
+        RECIPE_KWS_HI = {"रेसिपी", "खाना", "बनाने", "विधि", "सामग्री", "नुस्खे", "पकाने", "पूजा", "त्योहार"}
+        try:
+            ds = load_dataset("FreedomIntelligence/alpaca-gpt4-hindi", split="train")
+            for row in ds:
+                if len(pool) >= cap:
+                    break
+                convs = row.get("conversations", [])
+                if convs and len(convs) >= 2:
+                    inst = convs[0].get("value", "")
+                    resp = convs[1].get("value", "")
+                else:
+                    inst = row.get("instruction", "")
+                    resp = row.get("output", "")
+                if not inst.strip() or not resp.strip():
+                    continue
+                if not any(kw in inst for kw in RECIPE_KWS_HI):
+                    continue
+                if any(kw in inst for kw in CREATIVE_BLOCKLIST_HI):
+                    continue
+                if is_duplicate(inst):
+                    continue
+                sample = create_training_sample(inst, resp, domain_id)
+                if sample:
+                    pool.append(sample)
+        except Exception as e:
+            print(f"  ⚠️ alpaca-gpt4-hindi error: {e}")
+
+    random.shuffle(pool)
+    pool = pool[:cap]
+    print(f"  ✅ D6 Recipes & Indian Utility: {len(pool)} samples (target: {cap})")
+    return pool
+
+
+def build_domain_health():
+    """D7: Health & Wellness — Sanitized MedQuad medical Q&A."""
+    from datasets import load_dataset
+    domain = "07_finance_health_wellness"
+    domain_id = DOMAIN_ID_MAP[domain]
+    cap = DOMAIN_CAPS[domain]
+    pool = []
+    print(f"\n{'='*60}")
+    print(f"📦 Building D7: {domain} (cap={cap})")
+
+    print("  Loading keivalya/MedQuad-MedicalQnADataset...")
+    try:
+        ds = load_dataset("keivalya/MedQuad-MedicalQnADataset", split="train")
+        for row in ds:
+            question = row.get("Question", "")
+            answer = row.get("Answer", "")
+            if not question.strip() or not answer.strip():
+                continue
+            clean_q = sanitize_scraped_text(question)
+            clean_a = sanitize_scraped_text(answer)
+            if len(clean_a) < 30:
+                continue
+            if is_duplicate(clean_q):
+                continue
+            sample = create_training_sample(clean_q, clean_a, domain_id)
+            if sample:
+                pool.append(sample)
+            if len(pool) >= cap:
+                break
+    except Exception as e:
+        print(f"  ⚠️ MedQuad error: {e}")
+
+    random.shuffle(pool)
+    pool = pool[:cap]
+    print(f"  ✅ D7 Health & Wellness: {len(pool)} samples (target: {cap})")
+    return pool
+
+
+def build_domain_translation():
+    """D8: Translation — ai4bharat/samanantar bidirectional En↔Hi."""
+    from datasets import load_dataset
+    domain = "08_translation_multilingual"
+    domain_id = DOMAIN_ID_MAP[domain]
+    cap = DOMAIN_CAPS[domain]
+    pool = []
+    print(f"\n{'='*60}")
+    print(f"📦 Building D8: {domain} (cap={cap})")
+
+    print("  Loading ai4bharat/samanantar (hi)...")
+    try:
+        ds = load_dataset("ai4bharat/samanantar", "hi", split="train", streaming=True)
+        count = 0
+        for row in ds:
+            if len(pool) >= cap:
+                break
+            src = row.get("src", "").strip()
+            tgt = row.get("tgt", "").strip()
+            if not src or not tgt:
+                continue
+            # Length filter: 25-180 chars
+            if len(src) < 25 or len(src) > 180 or len(tgt) < 25 or len(tgt) > 180:
+                continue
+            # Skip if contains code/HTML artifacts
+            if "<" in src or "<" in tgt or "{" in src or "http" in src.lower():
+                continue
+
+            # Alternate: 50% En→Hi, 50% Hi→En
+            if count % 2 == 0:
+                inst = f"Translate the following English sentence to Hindi:\n\"{src}\""
+                resp = tgt
+            else:
+                inst = f"Translate the following Hindi sentence to English:\n\"{tgt}\""
+                resp = src
+
+            if is_duplicate(inst):
+                continue
+            sample = create_training_sample(inst, resp, domain_id)
+            if sample:
+                pool.append(sample)
+                count += 1
+    except Exception as e:
+        print(f"  ⚠️ Samanantar error: {e}")
+
+    random.shuffle(pool)
+    pool = pool[:cap]
+    print(f"  ✅ D8 Translation: {len(pool)} samples (target: {cap})")
+    return pool
+
+
+def build_domain_gk():
+    """D9: GK, Polity & History — alpaca-gpt4-hindi with creative blocklist."""
+    from datasets import load_dataset
+    domain = "09_gk_polity_history"
+    domain_id = DOMAIN_ID_MAP[domain]
+    cap = DOMAIN_CAPS[domain]
+    pool = []
+    print(f"\n{'='*60}")
+    print(f"📦 Building D9: {domain} (cap={cap})")
+
+    print("  Loading FreedomIntelligence/alpaca-gpt4-hindi...")
+    try:
+        ds = load_dataset("FreedomIntelligence/alpaca-gpt4-hindi", split="train")
+        for row in ds:
+            if len(pool) >= cap:
+                break
+            convs = row.get("conversations", [])
+            if convs and len(convs) >= 2:
+                inst = convs[0].get("value", "")
+                resp = convs[1].get("value", "")
+            else:
+                inst = row.get("instruction", "")
+                resp = row.get("output", "")
+            if not inst.strip() or not resp.strip():
+                continue
+            # Block creative tasks
+            if any(kw in inst for kw in CREATIVE_BLOCKLIST_HI):
+                continue
+            if any(kw in inst.lower() for kw in CREATIVE_BLOCKLIST_EN):
+                continue
+            if len(inst) < 10 or len(resp) < 15:
+                continue
+            if is_duplicate(inst):
+                continue
+            sample = create_training_sample(inst, resp, domain_id)
+            if sample:
+                pool.append(sample)
+    except Exception as e:
+        print(f"  ⚠️ alpaca-gpt4-hindi error: {e}")
+
+    random.shuffle(pool)
+    pool = pool[:cap]
+    print(f"  ✅ D9 GK, Polity & History: {len(pool)} samples (target: {cap})")
+    return pool
+
+
+def build_domain_multiturn():
+    """D10: Multi-turn Conversations — smoltalk + SODA multi-turn."""
+    from datasets import load_dataset
+    domain = "10_multiturn_conversations"
+    domain_id = DOMAIN_ID_MAP[domain]
+    cap = DOMAIN_CAPS[domain]
+    pool = []
+    print(f"\n{'='*60}")
+    print(f"📦 Building D10: {domain} (cap={cap})")
+
+    # Source 1: smoltalk multi-turn configs
+    for config_name in ["smol-magpie-ultra", "openhermes-100k", "everyday-conversations"]:
+        if len(pool) >= cap:
+            break
+        print(f"  Loading smoltalk/{config_name}...")
+        try:
+            ds = load_dataset("HuggingFaceTB/smoltalk", config_name, split="train", streaming=True)
+            for row in ds:
+                if len(pool) >= cap:
+                    break
+                msgs = row.get("messages", [])
+                if len(msgs) < 4:  # Need at least 2 turns
+                    continue
+                # Convert to our turn format, max 8 turns
+                turns = []
+                for m in msgs[:8]:
+                    role = m.get("role", "")
+                    content = m.get("content", "").strip()
+                    if role in ["user", "assistant"] and content:
+                        turns.append({"role": role, "content": content})
+                if len(turns) < 4:
+                    continue
+                # Ensure starts with user
+                if turns[0]["role"] != "user":
+                    continue
+                first_msg = turns[0]["content"]
+                if is_duplicate(first_msg):
+                    continue
+                sample = create_multiturn_training_sample(turns, domain_id)
+                if sample:
+                    pool.append(sample)
+        except Exception as e:
+            print(f"  ⚠️ smoltalk/{config_name} error: {e}")
+
+    # Source 2: SODA multi-turn dialogues
+    if len(pool) < cap:
+        print(f"  Loading allenai/soda multi-turn (need {cap - len(pool)} more)...")
+        try:
+            ds_soda = load_dataset("allenai/soda", split="train", streaming=True)
+            for row in ds_soda:
+                if len(pool) >= cap:
+                    break
+                dialogue = row.get("dialogue", [])
+                speakers = row.get("speakers", [])
+                if len(dialogue) < 4:
+                    continue
+                # Alternate user/assistant roles
+                turns = []
+                for i, msg in enumerate(dialogue[:8]):
+                    role = "user" if i % 2 == 0 else "assistant"
+                    if msg.strip():
+                        turns.append({"role": role, "content": msg.strip()})
+                if len(turns) < 4:
+                    continue
+                first_msg = turns[0]["content"]
+                if is_duplicate(first_msg):
+                    continue
+                sample = create_multiturn_training_sample(turns, domain_id)
+                if sample:
+                    pool.append(sample)
+        except Exception as e:
+            print(f"  ⚠️ SODA multi-turn error: {e}")
+
+    random.shuffle(pool)
+    pool = pool[:cap]
+    print(f"  ✅ D10 Multi-turn: {len(pool)} samples (target: {cap})")
+    return pool
+
+
+def build_domain_typo():
+    """D11: Typo Robustness — Dynamic typo injection on greetings + alpaca-cleaned."""
+    from datasets import load_dataset
+    domain = "11_typo_robustness"
+    domain_id = DOMAIN_ID_MAP[domain]
+    cap = DOMAIN_CAPS[domain]
+    pool = []
+    print(f"\n{'='*60}")
+    print(f"📦 Building D11: {domain} (cap={cap})")
+
+    # Source 1: Perturbed greetings (3 typo variations per greeting)
+    greetings_path = os.path.join(DATA_DIR, "Greetings_Data_Cleaned", "Greetings_Cleaned_Master.json")
+    greetings_data = []
+    if os.path.exists(greetings_path):
+        with open(greetings_path, "r", encoding="utf-8") as f:
+            greetings_data = json.load(f)
+    else:
+        greetings_data = load_local_json("Greetings_Cleaned_Master.json")
+
+    for row in greetings_data:
+        if len(pool) >= cap:
+            break
+        inst = row.get("instruction") or row.get("prompt") or ""
+        resp = row.get("response") or row.get("output") or ""
+        if not inst.strip() or not resp.strip():
+            continue
+        # Generate 3 typo variations
+        for _ in range(3):
+            typo_inst = inject_typo(inst.strip(), typo_prob=0.5)
+            if typo_inst == inst.strip():
+                continue
+            if is_duplicate(typo_inst):
+                continue
+            sample = create_training_sample(typo_inst, resp, domain_id)
+            if sample:
+                pool.append(sample)
+            if len(pool) >= cap:
+                break
+
+    # Source 2: Perturbed alpaca-cleaned instructions
+    if len(pool) < cap:
+        print(f"  Generating typo-perturbed alpaca samples (need {cap - len(pool)} more)...")
+        try:
+            ds = load_dataset("yahma/alpaca-cleaned", split="train")
+            indices = list(range(len(ds)))
+            random.shuffle(indices)
+            for idx in indices:
+                if len(pool) >= cap:
+                    break
+                row = ds[idx]
+                inst = row.get("instruction", "")
+                resp = row.get("output", "")
+                if not inst.strip() or not resp.strip() or len(inst) < 15:
+                    continue
+                typo_inst = inject_typo(inst.strip(), typo_prob=0.6)
+                if typo_inst == inst.strip():
+                    continue
+                if is_duplicate(typo_inst):
+                    continue
+                sample = create_training_sample(typo_inst, resp, domain_id)
+                if sample:
+                    pool.append(sample)
+        except Exception as e:
+            print(f"  ⚠️ alpaca-cleaned typo error: {e}")
+
+    random.shuffle(pool)
+    pool = pool[:cap]
+    print(f"  ✅ D11 Typo Robustness: {len(pool)} samples (target: {cap})")
+    return pool
+
+
+def build_domain_hinglish():
+    """D12: Hinglish Code-Mixed — alpaca-gpt4-hindi with creative blocklist."""
+    from datasets import load_dataset
+    domain = "12_hinglish_codemixed"
+    domain_id = DOMAIN_ID_MAP[domain]
+    cap = DOMAIN_CAPS[domain]
+    pool = []
+    print(f"\n{'='*60}")
+    print(f"📦 Building D12: {domain} (cap={cap})")
+
+    print("  Loading FreedomIntelligence/alpaca-gpt4-hindi (Hinglish)...")
+    try:
+        ds = load_dataset("FreedomIntelligence/alpaca-gpt4-hindi", split="train")
+        # Shuffle to get different samples than D9
+        indices = list(range(len(ds)))
+        random.shuffle(indices)
+        for idx in indices:
+            if len(pool) >= cap:
+                break
+            row = ds[idx]
+            convs = row.get("conversations", [])
+            if convs and len(convs) >= 2:
+                inst = convs[0].get("value", "")
+                resp = convs[1].get("value", "")
+            else:
+                inst = row.get("instruction", "")
+                resp = row.get("output", "")
+            if not inst.strip() or not resp.strip():
+                continue
+            if any(kw in inst for kw in CREATIVE_BLOCKLIST_HI):
+                continue
+            if is_duplicate(inst):
+                continue
+            sample = create_training_sample(inst, resp, domain_id)
+            if sample:
+                pool.append(sample)
+    except Exception as e:
+        print(f"  ⚠️ alpaca-gpt4-hindi hinglish error: {e}")
+
+    random.shuffle(pool)
+    pool = pool[:cap]
+    print(f"  ✅ D12 Hinglish Code-Mixed: {len(pool)} samples (target: {cap})")
+    return pool
+
+
+def build_domain_safety():
+    """D13: Safety & Refusal — BeaverTails (is_safe=True) + smoltalk/systemchats."""
+    from datasets import load_dataset
+    domain = "13_safety_refusal"
+    domain_id = DOMAIN_ID_MAP[domain]
+    cap = DOMAIN_CAPS[domain]
+    pool = []
+    print(f"\n{'='*60}")
+    print(f"📦 Building D13: {domain} (cap={cap})")
+
+    # Source 1: BeaverTails (safe responses only)
+    print("  Loading PKU-Alignment/BeaverTails (is_safe=True)...")
+    try:
+        ds = load_dataset("PKU-Alignment/BeaverTails", split="30k_train")
+        for row in ds:
+            if len(pool) >= cap:
+                break
+            if not row.get("is_safe", False):
+                continue
+            prompt = row.get("prompt", "").strip()
+            response = row.get("response", "").strip()
+            if not prompt or not response:
+                continue
+            # Clean prompt leak artifacts
+            response = re.sub(r"^(Instruction|Output|Response)\s*:\s*", "", response)
+            if len(response) < 20:
+                continue
+            if is_duplicate(prompt):
+                continue
+            sample = create_training_sample(prompt, response, domain_id)
+            if sample:
+                pool.append(sample)
+    except Exception as e:
+        print(f"  ⚠️ BeaverTails error: {e}")
+
+    # Source 2: smoltalk/systemchats-30k (safety-related)
+    if len(pool) < cap:
+        print(f"  Loading smoltalk/systemchats-30k (safety, need {cap - len(pool)} more)...")
+        SAFETY_KWS = {"safe", "ethical", "illegal", "privacy", "security", "harm", "dangerous"}
+        try:
+            ds_sm = load_dataset("HuggingFaceTB/smoltalk", "systemchats-30k", split="train", streaming=True)
+            for row in ds_sm:
+                if len(pool) >= cap:
+                    break
+                msgs = row.get("messages", [])
+                if len(msgs) < 2:
+                    continue
+                inst = msgs[0].get("content", "")
+                resp = msgs[-1].get("content", "")
+                if not inst.strip() or not resp.strip():
+                    continue
+                lower_text = (inst + " " + resp).lower()
+                if not any(kw in lower_text for kw in SAFETY_KWS):
+                    continue
+                if is_duplicate(inst):
+                    continue
+                sample = create_training_sample(inst, resp, domain_id)
+                if sample:
+                    pool.append(sample)
+        except Exception as e:
+            print(f"  ⚠️ smoltalk/systemchats error: {e}")
+
+    random.shuffle(pool)
+    pool = pool[:cap]
+    print(f"  ✅ D13 Safety & Refusal: {len(pool)} samples (target: {cap})")
+    return pool
+
+
+def build_domain_instruction_following():
+    """D14: Instruction Following — smoltalk constraints/rewrite/summarize."""
+    from datasets import load_dataset
+    domain = "14_instruction_following"
+    domain_id = DOMAIN_ID_MAP[domain]
+    cap = DOMAIN_CAPS[domain]
+    pool = []
+    print(f"\n{'='*60}")
+    print(f"📦 Building D14: {domain} (cap={cap})")
+
+    for config_name in ["smol-constraints", "smol-rewrite", "smol-summarize"]:
+        if len(pool) >= cap:
+            break
+        print(f"  Loading smoltalk/{config_name}...")
+        try:
+            ds = load_dataset("HuggingFaceTB/smoltalk", config_name, split="train", streaming=True)
+            for row in ds:
+                if len(pool) >= cap:
+                    break
+                msgs = row.get("messages", [])
+                if len(msgs) < 2:
+                    continue
+                inst = msgs[0].get("content", "") if msgs[0].get("role") == "user" else ""
+                resp = msgs[-1].get("content", "") if msgs[-1].get("role") == "assistant" else ""
+                if not inst.strip() or not resp.strip():
+                    continue
+                if is_duplicate(inst):
+                    continue
+                sample = create_training_sample(inst, resp, domain_id)
+                if sample:
+                    pool.append(sample)
+        except Exception as e:
+            print(f"  ⚠️ smoltalk/{config_name} error: {e}")
+
+    random.shuffle(pool)
+    pool = pool[:cap]
+    print(f"  ✅ D14 Instruction Following: {len(pool)} samples (target: {cap})")
+    return pool
+
+
+# ==============================================================================
+# Section 6: Master Pipeline — Build, Split, Save, Upload
+# ==============================================================================
+def build_all_domains():
+    """Build all 15 domains and return the master pool."""
+    builders = [
+        build_domain_identity_greetings,   # D0
+        build_domain_coding,               # D1
+        build_domain_math,                 # D2
+        build_domain_reasoning,            # D3
+        build_domain_chitchat,             # D4
+        build_domain_career,               # D5
+        build_domain_recipes,              # D6
+        build_domain_health,               # D7
+        build_domain_translation,          # D8
+        build_domain_gk,                   # D9
+        build_domain_multiturn,            # D10
+        build_domain_typo,                 # D11
+        build_domain_hinglish,             # D12
+        build_domain_safety,               # D13
+        build_domain_instruction_following, # D14
+    ]
+
+    master_pool = []
+    domain_counts = defaultdict(int)
+
+    for builder_fn in builders:
+        samples = builder_fn()
+        for s in samples:
+            domain_counts[s["domain_id"]] += 1
+        master_pool.extend(samples)
+
+    print(f"\n{'='*85}")
+    print(f"📊 MASTER POOL SUMMARY: {len(master_pool):,} total samples")
+    print(f"{'='*85}")
+    for did in sorted(domain_counts.keys()):
+        dname = [k for k, v in DOMAIN_ID_MAP.items() if v == did][0]
+        target = DOMAIN_CAPS[dname]
+        actual = domain_counts[did]
+        pct = actual / target * 100
+        status = "✅" if actual >= target * 0.8 else "⚠️"
+        print(f"  {status} D{did:02d} {dname}: {actual:,} / {target:,} ({pct:.1f}%)")
+
+    # Safety audit: verify NO thinking tags in any sample
+    print(f"\n🔍 Running ZERO-THINK safety audit...")
+    think_violations = 0
+    for s in master_pool:
+        decoded = tokenizer.decode(s["input_ids"])
+        if "<think>" in decoded or "[THINK]" in decoded or "<|begin_of_thought|>" in decoded:
+            think_violations += 1
+    if think_violations > 0:
+        print(f"  ❌ CRITICAL: {think_violations} samples contain thinking tags!")
+    else:
+        print(f"  ✅ ZERO thinking tag violations confirmed!")
+
+    return master_pool
+
+
+def split_and_save(master_pool: list):
+    """Split into train/val sets and save as NPY shards."""
+    random.shuffle(master_pool)
+
+    # Stratified split by domain
+    domain_buckets = defaultdict(list)
+    for sample in master_pool:
+        domain_buckets[sample["domain_id"]].append(sample)
+
+    train_pool, val_pool = [], []
+    for did, items in domain_buckets.items():
+        random.shuffle(items)
+        n_val = max(10, int(len(items) * VAL_SPLIT_RATIO))
+        val_pool.extend(items[:n_val])
+        train_pool.extend(items[n_val:])
+
+    random.shuffle(train_pool)
+    random.shuffle(val_pool)
+
+    print(f"\n📊 Split: Train={len(train_pool):,} | Val={len(val_pool):,}")
+
+    # Save each split
+    total_train_tokens = 0
+    total_val_tokens = 0
+
+    for split_name, split_pool in [("train", train_pool), ("val", val_pool)]:
+        all_tokens = []
+        all_labels = []
+        offsets = [0]
+        domains = []
+
+        for sample in split_pool:
+            ids = sample["input_ids"]
+            lbl = sample["labels"]
+            all_tokens.extend(ids)
+            all_labels.extend(lbl)
+            offsets.append(len(all_tokens))
+            domains.append(sample["domain_id"])
+
+        tokens_np = np.array(all_tokens, dtype=np.int32)
+        labels_np = np.array(all_labels, dtype=np.int32)
+        offsets_np = np.array(offsets, dtype=np.uint32)
+        domains_np = np.array(domains, dtype=np.uint8)
+
+        np.save(os.path.join(OUTPUT_DIR, f"{split_name}_tokens.npy"), tokens_np)
+        np.save(os.path.join(OUTPUT_DIR, f"{split_name}_labels.npy"), labels_np)
+        np.save(os.path.join(OUTPUT_DIR, f"{split_name}_offsets.npy"), offsets_np)
+        np.save(os.path.join(OUTPUT_DIR, f"{split_name}_domains.npy"), domains_np)
+
+        total = len(tokens_np)
+        if split_name == "train":
+            total_train_tokens = total
+        else:
+            total_val_tokens = total
+
+        print(f"  💾 Saved {split_name}: {len(split_pool):,} samples, {total:,} tokens")
+
+    # Save metadata
+    domain_count_map = defaultdict(int)
+    for sample in master_pool:
+        did = sample["domain_id"]
+        dname = [k for k, v in DOMAIN_ID_MAP.items() if v == did][0]
+        domain_count_map[dname] += 1
+
+    metadata = {
+        "version": f"sft_{VERSION}",
+        "total_samples": len(master_pool),
+        "train_samples": len(train_pool),
+        "val_samples": len(val_pool),
+        "total_tokens": total_train_tokens + total_val_tokens,
+        "train_tokens": total_train_tokens,
+        "val_tokens": total_val_tokens,
+        "max_seq_len": MAX_SEQ_LEN,
+        "vocab_size": 64003,
+        "cot_thinking_ratio_percent": 0.0,
+        "thinking_tags": "NONE (Pure Gold v20)",
+        "domain_counts": dict(domain_count_map),
+        "blocked_sources": list(BLOCKED_FILES),
+        "blocked_categories": list(BLOCKED_CATEGORIES),
+    }
+
+    meta_path = os.path.join(OUTPUT_DIR, "metadata.json")
+    with open(meta_path, "w", encoding="utf-8") as f:
+        json.dump(metadata, f, indent=2, ensure_ascii=False)
+    print(f"  📄 Saved metadata.json")
+
+    return metadata
+
+
+def upload_to_hf():
+    """Upload tokenized shards to HF dataset hub."""
+    try:
+        from huggingface_hub import HfApi
+        api = HfApi(token=HF_TOKEN)
+        sft_prefix = f"sft_{VERSION}"
+
+        files_to_upload = [
+            "train_tokens.npy", "train_labels.npy", "train_offsets.npy", "train_domains.npy",
+            "val_tokens.npy", "val_labels.npy", "val_offsets.npy", "val_domains.npy",
+            "metadata.json",
+        ]
+
+        for fname in files_to_upload:
+            local_path = os.path.join(OUTPUT_DIR, fname)
+            if os.path.exists(local_path):
+                remote_path = f"{sft_prefix}/{fname}"
+                api.upload_file(
+                    path_or_fileobj=local_path,
+                    path_in_repo=remote_path,
+                    repo_id=HF_DATA_REPO,
+                    repo_type="dataset",
+                )
+                fsize = os.path.getsize(local_path) / (1024 * 1024)
+                print(f"  ☁️ Uploaded {remote_path} ({fsize:.1f} MB)")
+
+        print(f"  ✅ All shards uploaded to {HF_DATA_REPO}/{sft_prefix}/")
+    except Exception as e:
+        print(f"  ⚠️ HF upload error: {e}")
+
+
+# ==============================================================================
+# Main Entry Point
+# ==============================================================================
+def main():
+    start_time = time.time()
+
+    print(f"\n🚀 Starting SFT {VERSION.upper()} 'PURE GOLD' dataset build...")
+    master_pool = build_all_domains()
+    metadata = split_and_save(master_pool)
+
+    elapsed = (time.time() - start_time) / 60
+    print(f"\n{'='*85}")
+    print(f"🎉 SFT {VERSION.upper()} 'PURE GOLD' BUILD COMPLETE!")
+    print(f"   Total Samples: {metadata['total_samples']:,}")
+    print(f"   Total Tokens: {metadata['total_tokens']:,}")
+    print(f"   CoT Thinking: {metadata['cot_thinking_ratio_percent']}%")
+    print(f"   Build Time: {elapsed:.2f} minutes")
+    print(f"{'='*85}")
+
+    # Upload to HF
+    upload_to_hf()
+
+    print(f"\n✅ All done! Dataset ready at: {OUTPUT_DIR}")
+
+
+if __name__ == "__main__":
+    main()