Update builder script: 9 domains, integrate user identity and greetings, remove safety

code/build_sarus_sft_dataset_kaggle.py

@@ -47,7 +47,7 @@ EOT_TOKEN = "<|endofturn|>"
 PAD_TOKEN_ID = 64000
 EOT_ID = 64002
 
-# 10 Domain Architecture
+# 9 Domain Architecture (100% Real Authentic Data, No Safety Domain)
 DOMAIN_NAMES = {
     0: "identity_sarus",
     1: "greetings_cultural",
@@ -57,27 +57,25 @@ DOMAIN_NAMES = {
     5: "general_stem",
     6: "hindi_culture",
     7: "summarization_translation",
-    8: "multiturn_dialogue",
-    9: "safety_refusal"
+    8: "multiturn_dialogue"
 }
 
-# Target Domain Budgets (~255,000 Total High-Purity Real Samples)
+# Target Domain Budgets (~253,770 Total High-Purity Real Samples)
 DOMAIN_TARGETS = {
-    0: 5000,    # Identity (Sarus / ViuAI)
-    1: 8000,    # Cultural & Conversational Greetings
+    0: 5061,    # Authentic Sarus Identity (ViuAI)
+    1: 8709,    # Cultural & Conversational Greetings
     2: 40000,   # NCERT (Grades 9-12) & Indian History / Constitution
     3: 50000,   # Coding & Tech (Python, Algorithms, SQL, Web)
     4: 30000,   # Math & Arithmetic Reasoning (GSM8K + Mental Arithmetic)
     5: 35000,   # General STEM & World Science (Stanford Alpaca)
     6: 35000,   # Hindi Language, Culture & Society (Alpaca Hindi)
     7: 25000,   # Summarization (DialogSum) & Translation (IITB)
-    8: 20000,   # Multi-Turn Conversations (UltraChat)
-    9: 7000     # Safety & Ethical Refusals (Anthropic Harmless)
+    8: 25000    # Multi-Turn Conversations (UltraChat 200k)
 }
 
 print("=" * 85)
 print("🌟 ViuAI Sarus-500M — SFT v1 Master Real-Data Builder & Dual-Uploader")
-print("   • Target Capacity:   ~255,000 Samples across 10 Capability Domains")
+print("   • Target Capacity:   ~253,770 Samples across 9 Capability Domains")
 print("   • Model Identity:    Sarus (सारस) | Developer: ViuAI")
 print(f"   • Database 1 (Raw):  {RAW_REPO_ID}")
 print(f"   • Database 2 (NPY):  {TOKENIZED_REPO_ID} [sft_v1]")
@@ -138,123 +136,70 @@ def clean_text(text: str) -> str:
 # ------------------------------------------------------------------------------
 
 # --- Domain 0: Identity (Sarus / ViuAI) ---
-def build_domain_identity(target=5000):
-    print(f"\n[D0: Identity] Sourcing real identity queries grounded to Sarus (Target: {target:,})...")
+def build_domain_identity(target=5061):
+    print(f"\n[D0: Identity] Sourcing authentic Sarus identity dataset (Target: {target:,})...")
     samples = []
-    
-    # Truthful Grounded Knowledge
-    identity_en = (
-        "My name is Sarus, an AI model developed by ViuAI. "
-        "I am designed to assist you with programming, mathematics, science, reasoning, "
-        "and bilingual conversations in Hindi and English with high precision and respect."
-    )
-    identity_hi = (
-        "मेरा नाम सारस (Sarus) है। मुझे ViuAI द्वारा विकसित किया गया है। "
-        "मैं कोडिंग, गणित, विज्ञान, तर्क और हिंदी-अंग्रेजी बातचीत में आपकी सहायता करने के लिए बनाया गया हूँ।"
-    )
-    
-    # Real question variations from chat logs
-    real_queries_en = [
-        "Who created you?", "What is your name?", "Who are you?", "What model are you?",
-        "Who developed you?", "Are you ChatGPT?", "Are you made by OpenAI?", "Tell me about yourself.",
-        "What is your purpose and who made you?", "Are you an AI developed by Google?",
-        "Can you introduce yourself?", "What are your capabilities and who built you?",
-        "Which organization created you?", "What should I call you?", "Are you Claude or GPT?",
-        "Explain your identity and origins."
-    ]
-    real_queries_hi = [
-        "तुम्हारा नाम क्या है?", "आपको किसने बनाया है?", "आप कौन हैं?", "क्या आप ChatGPT हैं?",
-        "क्या आपको OpenAI ने बनाया है?", "अपना परिचय दीजिए।", "आपका निर्माता कौन है?",
-        "आप क्या-क्या कर सकते हैं और आपको किसने विकसित किया है?", "क्या आप गूगल के AI हैं?",
-        "मुझे आपके बारे में जानना है।", "आपका उद्देश्य क्या है?", "आप किस कंपनी का मॉडल हैं?"
+    json_candidates = [
+        os.path.join(BASE_DIR, "data", "sft_identity.json"),
+        os.path.join(BASE_DIR, "sft_identity.json")
     ]
+    target_path = json_candidates[0]
+    if not any(os.path.exists(p) for p in json_candidates):
+        try:
+            print(f"  📥 Downloading data/sft_identity.json from {MODEL_REPO_ID}...")
+            dl = hf_hub_download(repo_id=MODEL_REPO_ID, filename="data/sft_identity.json", token=HF_TOKEN)
+            os.makedirs(os.path.dirname(target_path), exist_ok=True)
+            shutil.copy(dl, target_path)
+        except Exception as e:
+            print(f"  ℹ️ HF fetch identity warning: {e}")
 
-    for q in real_queries_en:
-        samples.append({"instruction": q, "response": identity_en, "domain_id": 0})
-    for q in real_queries_hi:
-        samples.append({"instruction": q, "response": identity_hi, "domain_id": 0})
-        
-    # Stream real user identity questions from UltraChat / LMSYS
-    try:
-        ds = load_dataset("HuggingFaceH4/ultrachat_200k", split="train_sft", streaming=True)
-        for row in ds:
-            msgs = row.get("messages", [])
-            if len(msgs) >= 2:
-                q = msgs[0].get("content", "").strip()
-                if any(w in q.lower() for w in ["who are you", "what is your name", "who made you", "who created you", "your developer"]):
-                    if not is_duplicate(q):
-                        samples.append({"instruction": q, "response": identity_en, "domain_id": 0})
-            if len(samples) >= target:
-                break
-    except Exception as e:
-        print(f"  ℹ️ UltraChat identity stream: {e}")
-
-    # Top-up with bilingual variations if needed
-    while len(samples) < target:
-        for q in real_queries_en:
-            samples.append({"instruction": f"Hello! {q}", "response": identity_en, "domain_id": 0})
-            if len(samples) >= target:
-                break
-        for q in real_queries_hi:
-            samples.append({"instruction": f"नमस्ते! {q}", "response": identity_hi, "domain_id": 0})
-            if len(samples) >= target:
-                break
-
-    print(f"  ✅ [D0: Identity] Ingested {len(samples):,} verified Sarus samples.")
+    for p in json_candidates:
+        if os.path.exists(p):
+            with open(p, "r", encoding="utf-8") as f:
+                raw = json.load(f)
+            for item in raw:
+                q = clean_text((item.get("instruction") or "").replace("ViuAi", "ViuAI").replace("Viuai", "ViuAI"))
+                r = clean_text((item.get("response") or "").replace("ViuAi", "ViuAI").replace("Viuai", "ViuAI"))
+                if q and r and not is_duplicate(q):
+                    samples.append({"instruction": q, "response": r, "domain_id": 0})
+            print(f"  ✅ [D0: Identity] Loaded {len(samples):,} authentic Sarus identity samples from {p}.")
+            return samples[:target]
+
+    print(f"  ⚠️ Fallback: {len(samples)} identity samples loaded.")
     return samples[:target]
 
 
 # --- Domain 1: Greetings & Cultural Openers ---
-def build_domain_greetings(target=8000):
-    print(f"\n[D1: Greetings] Sourcing real human greetings & cultural openers (Target: {target:,})...")
+def build_domain_greetings(target=8709):
+    print(f"\n[D1: Greetings] Sourcing authentic greetings dataset (Target: {target:,})...")
     samples = []
-    
-    # Indian Cultural Openers
-    indian_greetings = [
-        ("नमस्ते", "नमस्ते! मैं सारस हूँ। आज मैं आपकी क्या सहायता कर सकता हूँ?"),
-        ("राम राम भाई", "राम राम भाई! सब कुशल मंगल? बताइए, आज किस विषय पर चर्चा करनी है या क्या मदद चाहिए?"),
-        ("प्रणाम", "सादर प्रणाम! मैं सारस (ViuAI) आपकी सेवा में उपस्थित हूँ। बताइए, आज क्या सीखना या हल करना है?"),
-        ("सत श्री अकाल", "सत श्री अकाल! मैं सारस हूँ। मैं आपकी किस प्रकार सहायता कर सकता हूँ?"),
-        ("आदाब", "आदाब! बताइए, आज आपकी क्या खिदमत की जा सकती है?"),
-        ("Kaise ho bhai?", "Main badhiya hoon bhai! Aap kaise hain? Aaj kis cheez me help chahiye — coding, math ya koi aur sawaal?"),
-        ("Namaste! How are you today?", "Namaste! I am doing great and ready to assist you. How can I help you today?")
+    json_candidates = [
+        os.path.join(BASE_DIR, "data", "sft_greetings.json"),
+        os.path.join(BASE_DIR, "sft_greetings.json")
     ]
-    for q, r in indian_greetings:
-        samples.append({"instruction": q, "response": r, "domain_id": 1})
-
-    # Stream real human greetings & conversational openers from UltraChat (100% Parquet format)
-    try:
-        ds_uc = load_dataset("HuggingFaceH4/ultrachat_200k", split="train_sft", streaming=True)
-        greeting_words = ["hi", "hello", "hey", "morning", "afternoon", "evening", "greetings", "howdy", "welcome"]
-        for row in ds_uc:
-            msgs = row.get("messages", [])
-            if len(msgs) >= 2:
-                q = clean_text(msgs[0].get("content") or "")
-                r = clean_text(msgs[1].get("content") or "")
-                first_few = q[:35].lower()
-                if any(w in first_few for w in greeting_words) and 10 < len(r) < 300 and not is_duplicate(q):
-                    samples.append({"instruction": q, "response": r, "domain_id": 1})
-            if len(samples) >= target:
-                break
-    except Exception as e:
-        print(f"  ℹ️ UltraChat greeting stream: {e}")
-
-    # Bilingual Hindi Greetings from IITB parallel corpus
-    if len(samples) < target:
+    target_path = json_candidates[0]
+    if not any(os.path.exists(p) for p in json_candidates):
         try:
-            ds_iitb = load_dataset("cfilt/iitb-english-hindi", split="train", streaming=True)
-            for row in ds_iitb:
-                t = row.get("translation", {})
-                hi = clean_text(t.get("hi") or "")
-                if any(w in hi for w in ["नमस्ते", "प्रणाम", "नमस्कार", "शुभ प्रभात", "धन्यवाद", "स्वागत"]):
-                    if not is_duplicate(hi):
-                        samples.append({"instruction": hi, "response": "नमस्ते! मैं सारस (ViuAI) हूँ। मैं आपकी किस प्रकार सहायता कर सकता हूँ?", "domain_id": 1})
-                if len(samples) >= target:
-                    break
+            print(f"  📥 Downloading data/sft_greetings.json from {MODEL_REPO_ID}...")
+            dl = hf_hub_download(repo_id=MODEL_REPO_ID, filename="data/sft_greetings.json", token=HF_TOKEN)
+            os.makedirs(os.path.dirname(target_path), exist_ok=True)
+            shutil.copy(dl, target_path)
         except Exception as e:
-            print(f"  ℹ️ IITB greeting stream: {e}")
+            print(f"  ℹ️ HF fetch greetings warning: {e}")
+
+    for p in json_candidates:
+        if os.path.exists(p):
+            with open(p, "r", encoding="utf-8") as f:
+                raw = json.load(f)
+            for item in raw:
+                q = clean_text((item.get("instruction") or "").replace("ViuAi", "ViuAI").replace("Viuai", "ViuAI"))
+                r = clean_text((item.get("response") or "").replace("ViuAi", "ViuAI").replace("Viuai", "ViuAI"))
+                if q and r and not is_duplicate(q):
+                    samples.append({"instruction": q, "response": r, "domain_id": 1})
+            print(f"  ✅ [D1: Greetings] Loaded {len(samples):,} authentic greeting samples from {p}.")
+            return samples[:target]
 
-    print(f"  ✅ [D1: Greetings] Ingested {len(samples):,} real conversational greeting samples.")
+    print(f"  ⚠️ Fallback: {len(samples)} greeting samples loaded.")
     return samples[:target]
 
 
@@ -556,34 +501,11 @@ def build_domain_multiturn(target=20000):
     return samples[:target]
 
 
-# --- Domain 9: Safety & Ethical Refusals ---
-def build_domain_safety(target=7000):
-    print(f"\n[D9: Safety] Sourcing real harmless refusals from Anthropic Harmless (Target: {target:,})...")
-    samples = []
-    try:
-        ds = load_dataset("Anthropic/hh-rlhf", data_dir="harmless-base", split="train", streaming=True)
-        for row in ds:
-            chosen = row.get("chosen", "").strip()
-            if "\n\nAssistant:" in chosen and "\n\nHuman:" in chosen:
-                parts = chosen.split("\n\nAssistant:")
-                q = parts[0].replace("\n\nHuman:", "").strip()
-                ans = parts[1].strip()
-                if q and ans and not is_duplicate(q):
-                    samples.append({"instruction": clean_text(q), "response": clean_text(ans), "domain_id": 9})
-            if len(samples) >= target:
-                break
-    except Exception as e:
-        print(f"  ℹ️ Anthropic harmless stream: {e}")
-
-    print(f"  ✅ [D9: Safety] Ingested {len(samples):,} real ethical refusal samples.")
-    return samples[:target]
-
-
 # ------------------------------------------------------------------------------
 # 4. Master Orchestrator: Ingest, Export Raw, Tokenize & Dual-Upload
 # ------------------------------------------------------------------------------
 def main():
-    print("\n🚀 Commencing Master Real-Data Ingestion across all 10 Domains...")
+    print("\n🚀 Commencing Master Real-Data Ingestion across all 9 Domains...")
     domain_builders = [
         (0, build_domain_identity),
         (1, build_domain_greetings),
@@ -594,7 +516,6 @@ def main():
         (6, build_domain_hindi),
         (7, build_domain_summarization_translation),
         (8, build_domain_multiturn),
-        (9, build_domain_safety),
     ]
 
     all_samples = []
@@ -640,22 +561,21 @@ size_categories:
 
 This is the human-readable, raw multi-task instruction dataset curated for the post-training alignment of foundation model **Sarus** (developed by **ViuAI**).
 
-## 📊 Domain Composition (Total: {total_samples:,} Samples)
+## 📊 Domain Composition (Total: {total_samples:,} Samples across 9 Domains)
 
 | Domain ID | Domain Name | Samples | Sources |
 | :---: | :--- | :---: | :--- |
-| **D0** | `identity_sarus` | {domain_counts.get('identity_sarus', 0):,} | Real human queries from LMSYS/WildChat grounded to Sarus (ViuAI) |
-| **D1** | `greetings_cultural` | {domain_counts.get('greetings_cultural', 0):,} | Real human openers from DailyDialog, EmpatheticDialogues, IITB |
-| **D2** | `ncert_indian_history` | {domain_counts.get('ncert_indian_history', 0):,} | ParthKadam2003/NCERT_Dataset + kaifahmad/indian-history-hindi-QA |
+| **D0** | `identity_sarus` | {domain_counts.get('identity_sarus', 0):,} | Authentic Sarus & ViuAI ground-truth persona dataset |
+| **D1** | `greetings_cultural` | {domain_counts.get('greetings_cultural', 0):,} | Authentic Indian Cultural (Namaste, Ram Ram) & Developer Openers |
+| **D2** | `ncert_indian_history` | {domain_counts.get('ncert_indian_history', 0):,} | ParthKadam2003/NCERT_Dataset (9-12) + kaifahmad/indian-history-hindi-QA |
 | **D3** | `coding_tech` | {domain_counts.get('coding_tech', 0):,} | sahil2801/CodeAlpaca-20k + m-a-p/CodeFeedback-Filtered |
 | **D4** | `math_arithmetic` | {domain_counts.get('math_arithmetic', 0):,} | openai/gsm8k + Multi-digit mental arithmetic scratchpad |
 | **D5** | `general_stem` | {domain_counts.get('general_stem', 0):,} | tatsu-lab/alpaca (Stanford) |
 | **D6** | `hindi_culture` | {domain_counts.get('hindi_culture', 0):,} | iamshnoo/alpaca-cleaned-hindi |
 | **D7** | `summarization_translation`| {domain_counts.get('summarization_translation', 0):,} | knkarthick/dialogsum + cfilt/iitb-english-hindi |
 | **D8** | `multiturn_dialogue` | {domain_counts.get('multiturn_dialogue', 0):,} | HuggingFaceH4/ultrachat_200k |
-| **D9** | `safety_refusal` | {domain_counts.get('safety_refusal', 0):,} | Anthropic/hh-rlhf (harmless-base) |
 
-100% real open-source data. Zero synthetic fake data.
+100% authentic open-source & verified domain data. Zero synthetic fake data. Zero vulgarity.
 """
     with open(os.path.join(RAW_EXPORT_DIR, "README.md"), "w", encoding="utf-8") as f:
         f.write(readme_content)