Upload code/build_sft_basic_chatbot.py with huggingface_hub

code/build_sft_basic_chatbot.py

@@ -570,6 +570,24 @@ The old man was deeply moved by the boy's compassion. Realizing that even small
     return anchors
 
 
+def extract_qa(row):
+    convs = row.get("conversations", [])
+    if isinstance(convs, list) and len(convs) >= 2:
+        u = convs[0].get("value") or convs[0].get("content") or ""
+        a = convs[1].get("value") or convs[1].get("content") or ""
+        return u.strip(), a.strip()
+    msgs = row.get("messages", [])
+    if isinstance(msgs, list) and len(msgs) >= 2:
+        u = msgs[0].get("content") or msgs[0].get("value") or ""
+        a = msgs[1].get("content") or msgs[1].get("value") or ""
+        return u.strip(), a.strip()
+    inst = row.get("instruction") or row.get("prompt") or row.get("question") or ""
+    inp = row.get("input") or ""
+    out = row.get("output") or row.get("response") or row.get("answer") or ""
+    if inp and inp.strip():
+        inst = f"{inst}\n\n{inp}"
+    return inst.strip(), out.strip()
+
 # ==============================================================================
 # Section 5: Master Dataset Assembly (~15,000 Samples)
 # ==============================================================================
@@ -577,8 +595,8 @@ def build_master_dataset():
     from datasets import load_dataset
     master_pool = []
 
-    # 1. Gold Foundation Anchors (Replicated 35x = ~1,600 rock-solid anchor rows)
-    print("\n📦 Step 1/8: Loading Gold Foundation Anchors...")
+    # 1. Gold Foundation Anchors (Replicated 35x = ~1,800 rock-solid anchor rows)
+    print("\n📦 Step 1/9: Loading Gold Foundation Anchors...")
     gold_anchors = get_gold_anchors()
     for prompt, resp, did in gold_anchors:
         for _ in range(35):
@@ -589,7 +607,7 @@ def build_master_dataset():
     print(f"  ✅ Added {len(master_pool)} Gold Foundation Anchor samples.")
 
     # 2. Multi-Turn Conversations from SODA (Target: 3,000 multi-turn dialogues)
-    print("\n📦 Step 2/8: Loading Multi-Turn Context Conversations...")
+    print("\n📦 Step 2/9: Loading Multi-Turn Context Conversations...")
     try:
         ds_soda = load_dataset("allenai/soda", split="train", streaming=True)
         count = 0
@@ -612,7 +630,7 @@ def build_master_dataset():
         print(f"  ⚠️ Multi-turn load error: {e}")
 
     # 3. Single-Turn Chitchat from SODA with Typo injection (Target: 2,500 clean chats)
-    print("\n📦 Step 3/8: Loading Conversational Chitchat (with Typo engine)...")
+    print("\n📦 Step 3/9: Loading Conversational Chitchat (with Typo engine)...")
     try:
         ds_soda_single = load_dataset("allenai/soda", split="train", streaming=True)
         count = 0
@@ -638,7 +656,7 @@ def build_master_dataset():
         print(f"  ⚠️ Chitchat load error: {e}")
 
     # 4. Clean Daily Math from GSM8K (Target: 2,200 clean step arithmetic)
-    print("\n📦 Step 4/8: Loading Clean GSM8K Everyday Math...")
+    print("\n📦 Step 4/9: Loading Clean GSM8K Everyday Math...")
     try:
         ds_math = load_dataset("openai/gsm8k", "main", split="train")
         count = 0
@@ -666,17 +684,16 @@ def build_master_dataset():
     except Exception as e:
         print(f"  ⚠️ Math load error: {e}")
 
-    # 5. Factual Indian GK & Science from Alpaca-Hindi (Target: 2,200 samples)
-    print("\n📦 Step 5/8: Loading Factual Indian GK & Science...")
+    # 5. Factual Indian GK & Science from Alpaca-Hindi (Target: 2,000 samples)
+    print("\n📦 Step 5/9: Loading Factual Indian GK & Science...")
     try:
         ds_hi = load_dataset("FreedomIntelligence/alpaca-gpt4-hindi", split="train")
         count = 0
-        GK_CREATIVE_BLOCK = {"कविता", "हाइकू", "नारा", "शायरी", "गीत", "दोहा"}
+        GK_CREATIVE_BLOCK = {"कविता", "हाइकू", "नारा", "शायरी", "गीत", "दोहा", "कहानी", "कथा"}
         for row in ds_hi:
-            if count >= 2200:
+            if count >= 2000:
                 break
-            inst = row.get("instruction", "").strip()
-            resp = row.get("output", "").strip()
+            inst, resp = extract_qa(row)
             if not inst or not resp or len(inst) < 10 or len(resp) < 15:
                 continue
             if any(kw in inst for kw in GK_CREATIVE_BLOCK):
@@ -691,18 +708,17 @@ def build_master_dataset():
     except Exception as e:
         print(f"  ⚠️ Hindi GK load error: {e}")
 
-    # 6. Hindi Stories & Moral Tales from Alpaca-Hindi (Target: 1,200 clean stories)
-    print("\n📦 Step 6/8: Loading Hindi Stories & Moral Tales...")
+    # 6. Hindi Stories & Moral Tales (Target: 1,000 clean stories)
+    print("\n📦 Step 6/9: Loading Hindi Stories & Moral Tales...")
     try:
         ds_hi_story = load_dataset("FreedomIntelligence/alpaca-gpt4-hindi", split="train")
         count = 0
-        STORY_KEYWORDS = {"कहानी", "कथा", "किस्सा", "शिक्षाप्रद", "पंचतंत्र", "नैतिक"}
+        STORY_KEYWORDS = ["कहानी", "कथा", "किस्��ा", "शिक्षाप्रद", "पंचतंत्र", "नैतिक"]
         for row in ds_hi_story:
-            if count >= 1200:
+            if count >= 1000:
                 break
-            inst = row.get("instruction", "").strip()
-            resp = row.get("output", "").strip()
-            if not inst or not resp or len(resp) < 100:
+            inst, resp = extract_qa(row)
+            if not inst or not resp or len(resp) < 80:
                 continue
             if any(kw in inst for kw in STORY_KEYWORDS):
                 if is_duplicate(inst):
@@ -715,21 +731,17 @@ def build_master_dataset():
     except Exception as e:
         print(f"  ⚠️ Hindi Story load error: {e}")
 
-    # 7. English Stories & Creative Narratives (Target: 1,200 clean stories)
-    print("\n📦 Step 7/8: Loading English Stories & Creative Narratives...")
+    # 7. English Stories & Creative Narratives (Target: 1,000 clean stories)
+    print("\n📦 Step 7/9: Loading English Stories & Creative Narratives...")
     try:
         ds_en_story = load_dataset("yahma/alpaca-cleaned", split="train")
         count = 0
-        EN_STORY_KEYWORDS = {"story", "tale", "fable", "narrative", "parable", "bedtime", "legend"}
+        EN_STORY_KEYWORDS = ["story", "tale", "fable", "narrative", "parable", "bedtime", "legend"]
         for row in ds_en_story:
-            if count >= 1200:
+            if count >= 1000:
                 break
-            inst = row.get("instruction", "").strip()
-            inp = row.get("input", "").strip()
-            resp = row.get("output", "").strip()
-            if inp:
-                inst = f"{inst}\n\n{inp}"
-            if not inst or not resp or len(resp) < 100:
+            inst, resp = extract_qa(row)
+            if not inst or not resp or len(resp) < 80:
                 continue
             lower_inst = inst.lower()
             if any(kw in lower_inst for kw in EN_STORY_KEYWORDS):
@@ -743,19 +755,15 @@ def build_master_dataset():
     except Exception as e:
         print(f"  ⚠️ English Story load error: {e}")
 
-    # 8. Clean Workplace & Daily Tasks (Target: 1,800 samples)
-    print("\n📦 Step 8/8: Loading Practical Everyday Tasks & Emails...")
+    # 8. Clean Workplace & Daily Tasks (Target: 1,500 samples)
+    print("\n📦 Step 8/9: Loading Practical Everyday Tasks & Emails...")
     try:
         ds_alp = load_dataset("yahma/alpaca-cleaned", split="train")
         count = 0
         for row in ds_alp:
-            if count >= 1800:
+            if count >= 1500:
                 break
-            inst = row.get("instruction", "").strip()
-            inp = row.get("input", "").strip()
-            resp = row.get("output", "").strip()
-            if inp:
-                inst = f"{inst}\n\n{inp}"
+            inst, resp = extract_qa(row)
             if not inst or not resp or len(resp) < 20:
                 continue
             if is_duplicate(inst):