Upload code/build_sft_basic_chatbot.py with huggingface_hub

code/build_sft_basic_chatbot.py

@@ -1,16 +1,18 @@
 # ==============================================================================
-# 🤖 ViuAI Sarus-500M — SFT v21 "Basic Conversational Chatbot" Builder (10k-15k)
+# 🤖 ViuAI Sarus-500M — SFT v21 "Basic Conversational Chatbot" Master Builder
 # ==============================================================================
-# Target: ~13,500 Pristine, Human-Verified Conversational & Practical Samples
+# Target: ~15,000 Clean, Human-Verified Conversational & Practical Samples
 #
-# Core Directives:
-#   1. Natural, warm conversational chit-chat & greetings (NO identity bleed)
-#   2. Pure Identity ONLY when explicitly asked ("Who made you?" -> ViuAI Sarus)
-#   3. 100% Verified Factual Indian GK (PM, President, Samvidhan, ISRO, Taj Mahal)
-#   4. Step-by-Step Clean Math (Unitary method, BODMAS, Discounts, Speed/Distance)
-#   5. Verified Working Python Algorithms (Prime, Palindrome, Reverse, Factorial)
-#   6. Clean Everyday Human Translation (En <-> Hi) — ZERO scraped legal/news noise
-#   7. Professional workplace leave emails, recipes, and health wellness
+# 🌟 KEY CAPABILITIES INCLUDED:
+#   1. Natural Chit-Chat & Greetings (NO identity bleed)
+#   2. Multi-Turn Context Retention (Causal turn masking across 2-4 turns)
+#   3. Typo Robustness (Handles 'hlo', 'kse', 'pthon', 'btao', fast typing)
+#   4. Pure Identity ONLY when explicitly asked ("Who made you?" -> ViuAI Sarus)
+#   5. 100% Verified Indian GK (PM Modi, Rajendra Prasad, Samvidhan, ISRO, Taj Mahal)
+#   6. Step-by-Step Clean Math (Unitary method, BODMAS, Discounts, Speed/Distance)
+#   7. Verified Working Python Algorithms (Prime, Palindrome, Reverse, Factorial)
+#   8. Clean Everyday Human Translation (En <-> Hi)
+#   9. Professional workplace leave emails, recipes, and health wellness
 # ==============================================================================
 
 import os
@@ -22,7 +24,6 @@ import hashlib
 import shutil
 import random
 import numpy as np
-from collections import defaultdict
 
 if hasattr(sys.stdout, "reconfigure"):
     sys.stdout.reconfigure(encoding="utf-8", errors="replace")
@@ -33,7 +34,7 @@ VERSION = "v21"
 MAX_SEQ_LEN = 2048
 VAL_SPLIT_RATIO = 0.02
 SEED = 42
-MIN_TARGET_TOKENS = 10
+MIN_TARGET_TOKENS = 8
 
 random.seed(SEED)
 np.random.seed(SEED)
@@ -50,8 +51,9 @@ HF_MODEL_REPO = "ViuAI/ViuAI-500M"
 HF_DATA_REPO = "ViuAI/viuai-500m-sft-tokenized"
 
 print("=" * 85)
-print(f"🤖 ViuAI Sarus-500M — SFT {VERSION.upper()} 'Basic Conversational Chatbot' Builder")
-print(f"   📊 Target: ~13,500 Pristine, Verified Samples (10k-15k Scale)")
+print(f"🤖 ViuAI Sarus-500M — SFT {VERSION.upper()} 'Basic Chatbot' Master Builder")
+print(f"   🌟 Features: Multi-Turn Context + Typo Robustness + Clean GK & Math")
+print(f"   📊 Target: ~15,000 Pristine Samples")
 print("=" * 85)
 
 # ==============================================================================
@@ -77,7 +79,50 @@ EOT_ID = tokenizer.token_to_id("<|endofturn|>")
 assert all(x is not None for x in [USER_ID, ASSISTANT_ID, EOT_ID]), "Missing special tokens!"
 
 # ==============================================================================
-# Section 2: Deduplication & Sample Creator
+# Section 2: Typo Injection Engine
+# ==============================================================================
+HINGLISH_ABBREVS = {
+    "karo": "kro", "batao": "btao", "kaise": "kse", "kya": "kyaa",
+    "bhai": "bhaii", "theek": "thik", "achha": "acha", "please": "plz",
+    "thanks": "thx", "because": "bcoz", "you": "u", "are": "r",
+    "hello": "hlo", "hai": "h", "nahi": "nhi", "mujhe": "mjhe",
+    "python": "pthon", "function": "func", "reverse": "rvrse"
+}
+
+KEYBOARD_ADJACENT = {
+    "q": "w", "w": "e", "e": "r", "r": "t", "t": "y", "y": "u",
+    "u": "i", "i": "o", "o": "p", "a": "s", "s": "d", "d": "f",
+    "f": "g", "g": "h", "h": "j", "j": "k", "k": "l", "z": "x",
+    "x": "c", "c": "v", "v": "b", "b": "n", "n": "m",
+}
+
+def inject_typo(text: str, typo_prob: float = 0.20) -> str:
+    """Inject realistic typing errors into prompt text."""
+    if random.random() > typo_prob or len(text) < 10:
+        return text
+    words = text.split()
+    result = []
+    for w in words:
+        wl = w.lower()
+        if wl in HINGLISH_ABBREVS and random.random() < 0.50:
+            result.append(HINGLISH_ABBREVS[wl])
+        elif len(w) > 4 and random.random() < 0.25:
+            chars = list(w)
+            idx = random.randint(1, max(1, len(chars) - 2))
+            op = random.choice(["swap", "adjacent", "drop_vowel"])
+            if op == "swap" and idx + 1 < len(chars):
+                chars[idx], chars[idx + 1] = chars[idx + 1], chars[idx]
+            elif op == "adjacent" and chars[idx].lower() in KEYBOARD_ADJACENT:
+                chars[idx] = KEYBOARD_ADJACENT[chars[idx].lower()]
+            elif op == "drop_vowel" and chars[idx].lower() in "aeiou":
+                chars.pop(idx)
+            result.append("".join(chars))
+        else:
+            result.append(w)
+    return " ".join(result)
+
+# ==============================================================================
+# Section 3: Deduplication & Sample Creators
 # ==============================================================================
 GLOBAL_SEEN_HASHES = set()
 
@@ -89,7 +134,7 @@ def is_duplicate(text: str) -> bool:
     GLOBAL_SEEN_HASHES.add(h)
     return False
 
-def create_sample(instruction: str, response: str, domain_id: int):
+def create_single_sample(instruction: str, response: str, domain_id: int):
     inst = instruction.strip()
     resp = response.strip()
     if len(inst) < 2 or len(resp) < 3:
@@ -121,12 +166,51 @@ def create_sample(instruction: str, response: str, domain_id: int):
     assert len(full_ids) == len(labels)
     return {"input_ids": list(full_ids), "labels": labels, "domain_id": domain_id}
 
+def create_multiturn_sample(messages: list, domain_id: int):
+    """
+    Format multi-turn conversation with causal turn masking.
+    messages: [{"role": "user"|"assistant", "content": "..."}]
+    """
+    full_ids = []
+    labels = []
+
+    for msg in messages:
+        role = msg.get("role", "")
+        content = msg.get("content", "").strip()
+        if not content:
+            continue
+
+        if role == "user":
+            turn_str = f"<|user|>\n{content}<|endofturn|>\n"
+            turn_ids = tokenizer.encode(turn_str).ids
+            full_ids.extend(turn_ids)
+            labels.extend([-100] * len(turn_ids))
+        elif role == "assistant":
+            turn_prefix = "<|assistant|>\n"
+            prefix_ids = tokenizer.encode(turn_prefix).ids
+            full_ids.extend(prefix_ids)
+            labels.extend([-100] * len(prefix_ids))
+
+            resp_str = f"{content}<|endofturn|>\n"
+            resp_ids = tokenizer.encode(resp_str).ids
+            full_ids.extend(resp_ids)
+            labels.extend(resp_ids)
+
+    if len(full_ids) < 20 or len(full_ids) > MAX_SEQ_LEN:
+        return None
+
+    active_tokens = sum(1 for l in labels if l != -100)
+    if active_tokens < MIN_TARGET_TOKENS:
+        return None
+
+    assert len(full_ids) == len(labels)
+    return {"input_ids": list(full_ids), "labels": labels, "domain_id": domain_id}
+
 
 # ==============================================================================
-# Section 3: Hardcoded Gold Foundation Anchors (100% Correct Facts & Code)
+# Section 4: Gold Foundation Anchors (100% Tested Ground Truth)
 # ==============================================================================
-def get_gold_foundation_anchors():
-    """Gold standard samples for all critical edge cases (replicated for high anchor strength)."""
+def get_gold_anchors():
     anchors = [
         # --- 1. Identity & Persona ---
         ("Who created you and what is your name?", 
@@ -153,7 +237,7 @@ def get_gold_foundation_anchors():
          "Sab badhiya chal raha hai bhai! Aap batao, sab kaisa chal raha hai aapki taraf?", 1),
         ("Namaste", "Namaste! Aapka din shubh ho. Bataiye aaj main aapki kya seva kar sakta hoon?", 1),
 
-        # --- 3. Python Coding (100% Tested Working Functions) ---
+        # --- 3. Python Coding ---
         ("Write a Python function to check if a number is prime.",
          '''To check if a number is prime, we check if it is greater than 1 and not divisible by any integer from 2 up to the square root of n:
 
@@ -173,7 +257,7 @@ def is_prime(n):
 # Example test:
 print(is_prime(17))  # Output: True
 print(is_prime(4))   # Output: False
-```''', 2),
+```''', 3),
 
         ("Write a Python function to calculate the factorial of a number using recursion.",
          '''Here is the recursive Python function to compute factorial ($n! = n \\times (n-1)!$):
@@ -188,7 +272,7 @@ def factorial(n):
 
 # Example test:
 print(factorial(5))  # Output: 120
-```''', 2),
+```''', 3),
 
         ("Write a Python function to reverse a string without using slicing.",
          '''Here is how you can reverse a string using a loop without slicing (`[::-1]`):
@@ -202,21 +286,20 @@ def reverse_string(s):
 
 # Example test:
 print(reverse_string("Hello World!"))  # Output: "!dlroW olleH"
-```''', 2),
+```''', 3),
 
         ("Write a Python function to check if a given string is a palindrome.",
          '''A palindrome is a string that reads the same forwards and backwards:
 
 ```python
 def is_palindrome(s):
-    # Normalize string: remove spaces and convert to lowercase
     cleaned = "".join(char.lower() for char in s if char.isalnum())
     return cleaned == cleaned[::-1]
 
 # Example test:
 print(is_palindrome("racecar"))  # Output: True
 print(is_palindrome("hello"))    # Output: False
-```''', 2),
+```''', 3),
 
         ("Write a Python one-liner list comprehension to get only even numbers from 1 to 20.",
          '''Here is the Python one-liner list comprehension:
@@ -225,7 +308,7 @@ print(is_palindrome("hello"))    # Output: False
 even_numbers = [x for x in range(1, 21) if x % 2 == 0]
 print(even_numbers)
 # Output: [2, 4, 6, 8, 10, 12, 14, 16, 18, 20]
-```''', 2),
+```''', 3),
 
         ("Write a Python function to generate the first n Fibonacci numbers.",
          '''Here is a function to generate the first `n` Fibonacci numbers in a list:
@@ -243,9 +326,9 @@ def generate_fibonacci(n):
 
 # Example test:
 print(generate_fibonacci(7))  # Output: [0, 1, 1, 2, 3, 5, 8]
-```''', 2),
+```''', 3),
 
-        # --- 4. Math & Arithmetic (Step-by-Step Clean Rationale) ---
+        # --- 4. Math & Word Problems ---
         ("If 5 notebooks cost ₹150, what is the cost of 12 notebooks?",
          '''Let's solve this step by step using the unitary method:
 
@@ -255,7 +338,7 @@ print(generate_fibonacci(7))  # Output: [0, 1, 1, 2, 3, 5, 8]
 2. **Cost of 12 notebooks:**
    $$\\text{Total Cost} = 12 \\times 30 = ₹360$$
 
-**Final Answer:** The cost of 12 notebooks is **₹360**.''', 3),
+**Final Answer:** The cost of 12 notebooks is **₹360**.''', 4),
 
         ("What is 10 + 20 * 2?",
          '''According to the standard order of operations (BODMAS / PEMDAS), multiplication is performed before addition:
@@ -263,7 +346,7 @@ print(generate_fibonacci(7))  # Output: [0, 1, 1, 2, 3, 5, 8]
 1. **Step 1 (Multiply):** $20 \\times 2 = 40$
 2. **Step 2 (Add):** $10 + 40 = 50$
 
-**Final Answer:** **50**''', 3),
+**Final Answer:** **50**''', 4),
 
         ("If a shirt costs ₹800 and there is a 15% discount, what is the final price?",
          '''Let's calculate the discount and final price:
@@ -274,7 +357,7 @@ print(generate_fibonacci(7))  # Output: [0, 1, 1, 2, 3, 5, 8]
 2. **Final Price:**
    $$\\text{Final Price} = 800 - 120 = ₹680$$
 
-**Final Answer:** The final price of the shirt is **₹680**.''', 3),
+**Final Answer:** The final price of the shirt is **₹680**.''', 4),
 
         ("A car travels 300 km in 5 hours. What is the average speed of the car?",
          '''We use the formula: $\\text{Speed} = \\frac{\\text{Distance}}{\\text{Time}}$
@@ -284,7 +367,7 @@ print(generate_fibonacci(7))  # Output: [0, 1, 1, 2, 3, 5, 8]
 3. **Calculation:**
    $$\\text{Average Speed} = \\frac{300}{5} = 60\\text{ km/h}$$
 
-**Final Answer:** The average speed of the car is **60 km/h**.''', 3),
+**Final Answer:** The average speed of the car is **60 km/h**.''', 4),
 
         ("Find the area and perimeter of a rectangle with length 14 cm and width 6 cm.",
          '''For a rectangle with length $l = 14\\text{ cm}$ and width $w = 6\\text{ cm}$:
@@ -292,89 +375,89 @@ print(generate_fibonacci(7))  # Output: [0, 1, 1, 2, 3, 5, 8]
 1. **Area:** $\\text{Area} = l \\times w = 14 \\times 6 = 84\\text{ cm}^2$
 2. **Perimeter:** $\\text{Perimeter} = 2 \\times (l + w) = 2 \\times (14 + 6) = 2 \\times 20 = 40\\text{ cm}$
 
-**Final Answer:** Area is **84 cm²** and perimeter is **40 cm**.''', 3),
+**Final Answer:** Area is **84 cm²** and perimeter is **40 cm**.''', 4),
 
         ("What is 25 * 4 + 50?",
          '''Using the order of operations:
 1. **Multiplication:** $25 \\times 4 = 100$
 2. **Addition:** $100 + 50 = 150$
 
-**Final Answer:** **150**''', 3),
+**Final Answer:** **150**''', 4),
 
         # --- 5. Logical Deductions ---
         ("A is B's brother. B is C's mother. How is A related to C?",
-         '''Let's trace the relationship step by step:
+         '''Let's trace the relationship:
 1. $B$ is the mother of $C$.
 2. $A$ is the brother of $B$.
 3. The brother of one's mother is their **maternal uncle (Mama)**.
 
-**Final Answer:** $A$ is the **maternal uncle (Mama)** of $C$.''', 4),
+**Final Answer:** $A$ is the **maternal uncle (Mama)** of $C$.''', 5),
 
         ("All roses are flowers. Some flowers fade quickly. Can we conclude that all roses fade quickly?",
          '''**No, we cannot conclude that.**
 
-**Reasoning:** The premise states only that *some* flowers fade quickly. Since roses are only a subset of all flowers, we cannot determine whether roses specifically belong to the group of flowers that fade quickly or not.''', 4),
+**Reasoning:** The premise states only that *some* flowers fade quickly. Since roses are only a subset of all flowers, we cannot determine whether roses specifically belong to the group of flowers that fade quickly or not.''', 5),
 
         ("Why do we see lightning before we hear thunder? Explain step by step.",
          '''We see lightning before we hear thunder because **light travels much faster than sound**:
 
 1. **Speed of Light:** Light travels at approximately $300,000\\text{ km/s}$ ($3 \\times 10^8\\text{ m/s}$), reaching our eyes almost instantaneously.
 2. **Speed of Sound:** Sound travels through air at approximately $343\\text{ m/s}$ (about $1\\text{ km}$ every $3\\text{ seconds}$).
-3. Although both lightning and thunder are produced simultaneously at the same event, the light reaches us first and the sound follows a few seconds later.''', 4),
+3. Although both lightning and thunder are produced simultaneously, the light reaches us first and the sound follows a few seconds later.''', 5),
 
         ("Why do dropped objects fall to the ground instead of floating in the air?",
-         '''Objects fall to the ground due to **gravity** (gravitational force). Earth exerts an attractive force on all matter, pulling objects towards its center with an acceleration of approximately $9.8\\text{ m/s}^2$.''', 4),
+         '''Objects fall to the ground due to **gravity** (gravitational force). Earth exerts an attractive force pulling objects towards its center with an acceleration of approximately $9.8\\text{ m/s}^2$.''', 5),
 
         ("A person walks 5 km North, then turns Right and walks 3 km. In which direction is he from starting point?",
          '''1. Walking North puts the person in the North direction.
 2. Turning Right from North means heading East.
 3. Walking 3 km East places the person in the **North-East** direction from the origin.
 
-**Final Answer:** He is in the **North-East** direction from the starting point.''', 4),
+**Final Answer:** He is in the **North-East** direction from the starting point.''', 5),
 
         # --- 6. Indian GK & Constitution ---
         ("Bhartiya Samvidhan kab lagu hua tha aur Drafting Committee ke Chairman kaun the?",
-         "भारतीय संविधान **26 जनवरी 1950** को लागू हुआ था। संविधान की ड्राफ्टिंग कमेटी (प्रारूप समिति) के अध्यक्ष **डॉ. भीमराव अंबेडकर (Dr. B.R. Ambedkar)** थे।", 5),
+         "भारतीय संविधान **26 जनवरी 1950** को लागू हुआ था। संविधान की ड्राफ्टिंग कमेटी (प्रारूप समिति) के अध्यक्ष **डॉ. भीमराव अंबेडकर (Dr. B.R. Ambedkar)** थे।", 6),
 
         ("Bharat ke pehle Rashtrapati kaun the aur kab bane the?",
-         "भारत के पहले राष्ट्रपति **डॉ. राजेंद्र प्रसाद (Dr. Rajendra Prasad)** थे। वे 26 जनवरी 1950 को स्वतंत्र भारत के पहले राष्ट्रपति बने थे।", 5),
+         "भारत के पहले राष्ट्रपति **डॉ. राजेंद्र प्रसाद (Dr. Rajendra Prasad)** थे। वे 26 जनवरी 1950 को स्वतंत्र भारत के पहले राष्ट्रपति बने थे।", 6),
 
         ("Who is the Prime Minister of India?",
-         "The current Prime Minister of India is **Shri Narendra Modi**.", 5),
+         "The current Prime Minister of India is **Shri Narendra Modi**.", 6),
 
         ("India ki capital kya hai?",
-         "भारत की राजधानी **नई दिल्ली (New Delhi)** है।", 5),
+         "भारत की राजधानी **नई दिल्ली (New Delhi)** है।", 6),
 
         ("Taj Mahal kis sheher me sthit hai aur ise kisne banwaya tha?",
-         "ताजमहल भारत के **आगरा (Agra)** शहर में यमुना नदी के तट पर स्थित है। इसे मुगल सम्राट **शाहजहाँ** ने अपनी बेगम मुमताज महल की याद में बनवाया था।", 5),
+         "ताजमहल भारत के **आगरा (Agra)** शहर में यमुना नदी के तट पर स्थित है। इसे मुगल सम्राट **शाहजहाँ** ने अपनी बेगम मुमताज महल की याद में बनवाया था।", 6),
 
         ("Chandrayaan-3 kab land hua tha aur uske rover ka kya naam hai?",
-         "चंद्रयान-3 (Chandrayaan-3) **23 अगस्त 2023** को चंद्रमा के दक्षिणी ध्रुव के पास सफलतापूर्वक लैंड हुआ था। इसके लैंडर का नाम **विक्रम (Vikram)** और रोवर का नाम **प्रज्ञान (Pragyan)** था।", 5),
+         "चंद्रयान-3 (Chandrayaan-3) **23 अगस्त 2023** को चंद्रमा के दक्षिणी ध्रुव के पास सफलतापूर्वक लैंड हुआ था। इसके लैंडर का नाम **विक्रम (Vikram)** और रोवर का नाम **प्रज्ञान (Pragyan)** था।", 6),
 
         ("Photosynthesis (Prakash Sanshleshan) process ko 2 line me samjhao.",
-         "प्रकाश संश्लेषण (Photosynthesis) वह प्रक्रिया है जिसमें हरे पौधे सूर्य के प्रकाश, जल और कार्बन डाइऑक्साइड (CO₂) की मदद से अपना भोजन (ग्लूकोज) बनाते हैं और ऑक्सीजन (O₂) गैस छोड़ते हैं।", 5),
+         "प्रकाश संश्लेषण (Photosynthesis) वह प्रक्रिया है जिसमें हरे पौधे सूर्य के प्रकाश, जल और कार्बन डाइऑक्साइड (CO₂) की मदद से अपना भोजन (ग्लूकोज) बनाते हैं और ऑक्सीजन (O₂) गैस छोड़ते हैं।", 6),
 
         ("Matter ki 3 basic states kaunsi hain aur unke 1-1 example do.",
          '''पदार्थ (Matter) की तीन मुख्य अवस्थाएं हैं:
 1. **ठोस (Solid):** बर्फ (Ice), लकड़ी
 2. **द्रव (Liquid):** पानी (Water), दूध
-3. **गैस (Gas):** भाप (Steam), ऑक्सीजन''', 5),
+3. **गैस (Gas):** भाप (Steam), ऑक्सीजन''', 6),
 
         # --- 7. Everyday Clean Translation ---
         ("Translate to Hindi: 'Water is essential for all living beings.'",
-         "जल सभी जीवित प्राणियों के लिए आवश्यक है।", 6),
+         "जल सभी जीवित प्राणियों के लिए आवश्यक है।", 7),
 
         ("Translate to Hindi: 'Consistency and discipline are the keys to long term success.'",
-         "निरंतरता और अनुशासन दीर्घकालिक सफलता की कुंजी हैं।", 6),
+         "निरंतरता और अनुशासन दीर्घकालिक सफलता की कुंजी हैं।", 7),
 
         ("Translate to Hindi: 'Artificial Intelligence is transforming how we work and learn.'",
-         "आर्टिफिशियल इंटेलिजेंस हमारे काम करने और सीखने के तरीके को बदल रहा है।", 6),
+         "आर्टिफिशियल इंटेलिजेंस हमारे काम करने और सीखने के तरीके को बदल रहा है।", 7),
 
         ("Translate to English: 'ईमानदारी और मेहनत से ही सफलता मिलती है।'",
-         "Success is achieved only through honesty and hard work.", 6),
+         "Success is achieved only through honesty and hard work.", 7),
 
         ("Translate to English: 'मुझे आपसे मिलकर बहुत खुशी हुई।'",
-         "I am very happy to meet you.", 6),
+         "I am very happy to meet you.", 7),
 
         # --- 8. Recipes, Health & Workplace ---
         ("Kadak Masala Chai banane ki simple recipe step by step batao.",
@@ -383,25 +466,25 @@ print(generate_fibonacci(7))  # Output: [0, 1, 1, 2, 3, 5, 8]
 2. पानी में उबाल आने पर 1.5 चम्मच चायपत्ती और स्वादानुसार चीनी डालें।
 3. इसे 2 मिनट तक धीमी आंच पर उबलने दें ताकि मसालों का स्वाद आ जाए।
 4. अब 1 कप दूध डालें और 2-3 अच्छे उबाल आने तक पकाएं।
-5. चाय छानकर गरमा-गरम परोसें।''', 7),
+5. चाय छानकर गरमा-गरम परोसें।''', 8),
 
         ("Crispy Aloo Paratha banane ke 2 main secret tips batao.",
          '''**क्रिस्पी आलू पराठा के 2 मुख्य सीक्रेट टिप्स:**
 1. **स्टफिंग में नमी न हो:** उबले आलू को पूरी तरह ठंडा करके कद्दूकस करें और मसाले मिलाते समय हल्का भुना जीरा और कसूरी मेथी डालें।
-2. **मध्यम आंच पर घी/मक्खन से सेकें:** पराठे को दोनों तरफ से पहले हल्का सूखा सेकें, फिर घी लगाकर धीमी-मध्यम आंच पर दबा-दबाकर कुरकुरा होने तक सेकें।''', 7),
+2. **मध्यम आंच पर घी/मक्खन से सेकें:** पराठे को दोनों तरफ से पहले हल्का सूखा सेकें, फिर घी लगाकर धीमी-मध्यम आंच पर दबा-दबाकर कुरकुरा होने तक सेकें।''', 8),
 
         ("Subah khali pet gunguna paani peene ke kya fayde hote hain?",
          '''**सुबह खाली पेट गुनगुना पानी पीने के प्रमुख फायदे:**
 1. **पाचन में सुधार:** यह पेट को साफ करता है और मेटाबॉलिज्म (पा���न क्रिया) को तेज करता है।
 2. **बॉडी डिटॉक्स:** शरीर से टॉक्सिन्स (विषाक्त पदार्थ) बाहर निकलते हैं।
-3. **त्वचा और ऊर्जा:** त्वचा में चमक आती है और दिनभर ताजगी बनी रहती है।''', 7),
+3. **त्वचा और ऊर्जा:** त्वचा में चमक आती है और दिनभर ताजगी बनी रहती है।''', 8),
 
         ("What are common signs and symptoms of dehydration?",
          '''**Common signs and symptoms of dehydration include:**
 1. Extreme thirst and dry mouth
 2. Dark-colored urine or decreased urination
 3. Fatigue, dizziness, or lightheadedness
-4. Headache and dry skin''', 7),
+4. Headache and dry skin''', 8),
 
         ("Write a short and professional 2-day sick leave email to my manager.",
          '''**Subject:** Sick Leave Application - [Your Name] (2 Days)
@@ -416,10 +499,11 @@ Thank you for your understanding.
 
 Best regards,  
 [Your Name]  
-[Your Designation]''', 7),
+[Your Designation]''', 8),
 
+        # --- 9. Typo Robustness Anchors ---
         ("Hlo bro, kse ho ap? pthon me list rvrse kse kre?",
-         '''Hello bhai! Main badhiya hoon. 
+         '''Hello bhai! Main bilkul badhiya hoon.
 
 Python mein list reverse karne ke 2 aasan tareeqe hain:
 
@@ -435,57 +519,102 @@ print(reversed_list)  # [5, 4, 3, 2, 1]
 my_list = [1, 2, 3, 4, 5]
 my_list.reverse()
 print(my_list)  # [5, 4, 3, 2, 1]
-```''', 7),
+```''', 9),
+
+        ("Hlo, prime nmbr chk krne ka pythn func btao",
+         '''Hello! Yeh raha Python function jo check karta hai ki number prime hai ya nahi:
+
+```python
+def is_prime(n):
+    if n <= 1:
+        return False
+    if n == 2:
+        return True
+    if n % 2 == 0:
+        return False
+    for i in range(3, int(n**0.5) + 1, 2):
+        if n % i == 0:
+            return False
+    return True
+
+print(is_prime(19))  # True
+print(is_prime(10))  # False
+```''', 9)
     ]
     return anchors
 
 
 # ==============================================================================
-# Section 4: Dataset Sources Loading (Target: ~13,500 Samples)
+# Section 5: Master Dataset Assembly (~15,000 Samples)
 # ==============================================================================
-def build_basic_chatbot_dataset():
+def build_master_dataset():
     from datasets import load_dataset
     master_pool = []
-    
-    # 1. Add Gold Foundation Anchors (Replicated 40x = ~1,600 rock-solid anchor rows)
-    print("\n📦 Step 1/6: Loading Gold Foundation Anchors...")
-    gold_anchors = get_gold_foundation_anchors()
+
+    # 1. Gold Foundation Anchors (Replicated 35x = ~1,500 rock-solid anchor rows)
+    print("\n📦 Step 1/7: Loading Gold Foundation Anchors...")
+    gold_anchors = get_gold_anchors()
     for prompt, resp, did in gold_anchors:
-        for _ in range(40):
-            sample = create_sample(prompt, resp, did)
+        for _ in range(35):
+            # Apply slight typo injection to 20% of anchor copies
+            p = inject_typo(prompt, typo_prob=0.20)
+            sample = create_single_sample(p, resp, did)
             if sample:
                 master_pool.append(sample)
     print(f"  ✅ Added {len(master_pool)} Gold Foundation Anchor samples.")
 
-    # 2. Conversational Chit-Chat from SODA (Target: 4,000 clean dialogues)
-    print("\n📦 Step 2/6: Loading SODA Everyday Conversations...")
+    # 2. Multi-Turn Conversations from SODA & Smoltalk (Target: 3,500 multi-turn dialogues)
+    print("\n📦 Step 2/7: Loading Multi-Turn Context Conversations...")
     try:
         ds_soda = load_dataset("allenai/soda", split="train", streaming=True)
         count = 0
         for row in ds_soda:
-            if count >= 4000:
+            if count >= 3500:
+                break
+            dialogue = row.get("dialogue", [])
+            if len(dialogue) < 4:  # At least 2 full turns (4 messages)
+                continue
+            messages = []
+            for i, d in enumerate(dialogue[:6]):  # Up to 3 turns
+                role = "user" if i % 2 == 0 else "assistant"
+                messages.append({"role": role, "content": d})
+            sample = create_multiturn_sample(messages, 2)
+            if sample:
+                master_pool.append(sample)
+                count += 1
+        print(f"  ✅ Added {count} Multi-Turn Conversation samples.")
+    except Exception as e:
+        print(f"  ⚠️ Multi-turn load error: {e}")
+
+    # 3. Single-Turn Chitchat from SODA with Typo injection (Target: 3,000 clean chats)
+    print("\n📦 Step 3/7: Loading Conversational Chitchat (with Typo engine)...")
+    try:
+        ds_soda_single = load_dataset("allenai/soda", split="train", streaming=True)
+        count = 0
+        for row in ds_soda_single:
+            if count >= 3000:
                 break
             dialogue = row.get("dialogue", [])
             if len(dialogue) < 2:
                 continue
             u_msg = dialogue[0].strip()
             a_msg = dialogue[1].strip()
-            if len(u_msg) < 5 or len(a_msg) < 5:
-                continue
-            if re.search(r"\bmom\b", a_msg.lower()):
+            if len(u_msg) < 5 or len(a_msg) < 5 or "mom" in a_msg.lower():
                 continue
             if is_duplicate(u_msg):
                 continue
-            sample = create_sample(u_msg, a_msg, 1)
+            # Apply typo engine to 15% of prompts
+            u_final = inject_typo(u_msg, typo_prob=0.15)
+            sample = create_single_sample(u_final, a_msg, 1)
             if sample:
                 master_pool.append(sample)
                 count += 1
-        print(f"  ✅ Added {count} SODA conversational samples.")
+        print(f"  ✅ Added {count} Conversational Chitchat samples.")
     except Exception as e:
-        print(f"  ⚠️ SODA load error: {e}")
+        print(f"  ⚠️ Chitchat load error: {e}")
 
-    # 3. Clean Daily Math from GSM8K (Target: 2,500 step-by-step arithmetic)
-    print("\n📦 Step 3/6: Loading Clean GSM8K Everyday Math...")
+    # 4. Clean Daily Math from GSM8K (Target: 2,500 clean step-by-step arithmetic)
+    print("\n📦 Step 4/7: Loading Clean GSM8K Everyday Math...")
     try:
         ds_math = load_dataset("openai/gsm8k", "main", split="train")
         count = 0
@@ -496,7 +625,6 @@ def build_basic_chatbot_dataset():
             ans = row.get("answer", "").strip()
             if not q or not ans:
                 continue
-            # Sanitize answer
             if "####" in ans:
                 parts = ans.split("####")
                 rationale = re.sub(r"<<[^>]*>>", "", parts[0]).strip()
@@ -506,7 +634,7 @@ def build_basic_chatbot_dataset():
                 formatted_ans = re.sub(r"<<[^>]*>>", "", ans).strip()
             if is_duplicate(q):
                 continue
-            sample = create_sample(q, formatted_ans, 3)
+            sample = create_single_sample(q, formatted_ans, 4)
             if sample:
                 master_pool.append(sample)
                 count += 1
@@ -514,8 +642,8 @@ def build_basic_chatbot_dataset():
     except Exception as e:
         print(f"  ⚠️ Math load error: {e}")
 
-    # 4. Factual Hindi & Indian GK from Alpaca-Hindi (Target: 2,500 factual Q&A)
-    print("\n📦 Step 4/6: Loading Factual Indian GK & Science...")
+    # 5. Factual Indian GK & Science from Alpaca-Hindi (Target: 2,500 samples)
+    print("\n📦 Step 5/7: Loading Factual Indian GK & Science...")
     try:
         ds_hi = load_dataset("FreedomIntelligence/alpaca-gpt4-hindi", split="train")
         count = 0
@@ -531,7 +659,7 @@ def build_basic_chatbot_dataset():
                 continue
             if is_duplicate(inst):
                 continue
-            sample = create_sample(inst, resp, 5)
+            sample = create_single_sample(inst, resp, 6)
             if sample:
                 master_pool.append(sample)
                 count += 1
@@ -539,8 +667,8 @@ def build_basic_chatbot_dataset():
     except Exception as e:
         print(f"  ⚠️ Hindi GK load error: {e}")
 
-    # 5. Clean Workplace & Daily Tasks from alpaca-cleaned (Target: 2,000 samples)
-    print("\n📦 Step 5/6: Loading Practical Everyday Tasks...")
+    # 6. Clean Workplace & Daily Tasks from alpaca-cleaned (Target: 2,000 samples)
+    print("\n📦 Step 6/7: Loading Practical Everyday Tasks...")
     try:
         ds_alp = load_dataset("yahma/alpaca-cleaned", split="train")
         count = 0
@@ -556,7 +684,7 @@ def build_basic_chatbot_dataset():
                 continue
             if is_duplicate(inst):
                 continue
-            sample = create_sample(inst, resp, 7)
+            sample = create_single_sample(inst, resp, 8)
             if sample:
                 master_pool.append(sample)
                 count += 1
@@ -565,13 +693,13 @@ def build_basic_chatbot_dataset():
         print(f"  ⚠️ Alpaca load error: {e}")
 
     print(f"\n{'='*85}")
-    print(f"📊 Total Master Pool Size: {len(master_pool):,} samples (Target: 10k-15k)")
+    print(f"📊 Total Master Pool Size: {len(master_pool):,} samples (Target: ~15,000)")
     print(f"{'='*85}")
     return master_pool
 
 
 # ==============================================================================
-# Section 5: Split, Save Shards & Upload
+# Section 6: Split, Save Shards & Auto-Upload to HF
 # ==============================================================================
 def split_and_save(master_pool: list):
     random.shuffle(master_pool)
@@ -615,13 +743,12 @@ def split_and_save(master_pool: list):
         "total_tokens": total_tokens,
         "max_seq_len": MAX_SEQ_LEN,
         "vocab_size": 64003,
-        "model_type": "Basic Conversational Chatbot"
+        "features": ["Multi-Turn Context", "Typo Robustness", "Clean Math", "Verified Code", "Factual Indian GK"]
     }
 
     with open(os.path.join(OUTPUT_DIR, "metadata.json"), "w", encoding="utf-8") as f:
         json.dump(metadata, f, indent=2)
     print("  📄 Saved metadata.json")
-
     return metadata
 
 
@@ -644,10 +771,10 @@ def upload_to_hf():
 
 def main():
     start = time.time()
-    pool = build_basic_chatbot_dataset()
+    pool = build_master_dataset()
     meta = split_and_save(pool)
     upload_to_hf()
-    print(f"\n🎉 SFT {VERSION.upper()} Basic Chatbot Build Finished in {(time.time()-start)/60:.2f} mins!")
+    print(f"\n🎉 SFT {VERSION.upper()} Master Builder Finished in {(time.time()-start)/60:.2f} mins!")
 
 if __name__ == "__main__":
     main()