Upload runners/KAGGLE_BUILD_SARUS_SFT_V2_DATASET.txt with huggingface_hub

runners/KAGGLE_BUILD_SARUS_SFT_V2_DATASET.txt

@@ -1,85 +1,26 @@
 # ==============================================================================
-# 🚀 ViuAI Sarus-500M — STEP 1: Kaggle SFT v2 Pure Chat Dataset Builder & Tokenizer
+# 🚀 ViuAI Sarus-500M — SFT v2 Pure Chat Dataset Builder (Kaggle 1-Click)
 # ==============================================================================
-# Run this entire block in a single Kaggle Notebook cell (CPU or GPU)
-#
-# What it does:
-#   1. Sets up dependencies (tokenizers, datasets, huggingface_hub).
-#   2. Downloads build_sft_v2_chat_dataset.py from ViuAI/ViuAI-500M.
-#   3. Builds ~200,000 Pure Chat & Assistant samples across 8 domains:
-#      • D0: Identity & Persona (Sarus / ViuAI)
-#      • D1: Bilingual Greetings & Openers
-#      • D2: Everyday Conversational Dialogue (Multi-Turn)
-#      • D3: General Knowledge & Science (English)
-#      • D4: Hindi Factual QA & Knowledge (Breaks Greeting Loop)
-#      • D5: Practical Writing & Everyday Tasks (Emails, Summaries)
-#      • D6: Culture, Recipes (Masala Chai) & Lifestyle Tips
-#      • D7: Bilingual Translation (EN <-> HI)
-#      (0% Math, 0% Coding — Pure Conversational Focus)
-#   4. Tokenizes directly into memory-mapped NumPy shards.
-#   5. Auto-uploads to Hugging Face:
-#      • Raw JSONL: ViuAI/sarus-sft-data-raw (folder: sft_v2_chat)
-#      • Tokenized .npy: ViuAI/viuai-500m-sft-tokenized (folder: sft_v2)
+# Run this entire block in a single Kaggle Notebook cell.
+# It streams and builds ~200,000 pure conversational samples (0% math, 0% code)
+# and uploads directly to Hugging Face!
 # ==============================================================================
 
-import os, sys, shutil, subprocess
+# 1. Install required packages
+!pip install -q tokenizers datasets huggingface_hub numpy
 
-# 1. Install & Configure Environment
-subprocess.run([sys.executable, "-m", "pip", "install", "-q", "tokenizers", "datasets", "huggingface_hub", "numpy"], check=True)
-
-from huggingface_hub import HfApi, hf_hub_download
+# 2. Fetch the builder script from Hugging Face
+import os, shutil
+from huggingface_hub import hf_hub_download
 
 HF_TOKEN = os.environ.get("HF_TOKEN") or ("".join(["hf_", "ssyCVhuny", "XxjGdqKp", "VLPpkmWK", "FrrMOIFbg"]))
+os.environ["HF_TOKEN"] = HF_TOKEN
 REPO_ID = "ViuAI/ViuAI-500M"
 
-print("=" * 85)
-print("🚀 ViuAI Sarus-500M — SFT v2 Pure Chat Dataset Builder")
-print("   • Target: ~200,000 High-Purity Conversational Samples across 8 Domains")
-print("   • Repositories:")
-print("     - Model Hub:       ViuAI/ViuAI-500M")
-print("     - Raw Corpus:      ViuAI/sarus-sft-data-raw [sft_v2_chat]")
-print("     - Tokenized Shards: ViuAI/viuai-500m-sft-tokenized [sft_v2]")
-print("=" * 85)
-
-# 2. Sync Required Scripts & Data Assets
 os.makedirs("code", exist_ok=True)
-os.makedirs("data", exist_ok=True)
-os.makedirs("tokenizer", exist_ok=True)
-
-files_to_sync = [
-    ("code/build_sft_v2_chat_dataset.py", "code/build_sft_v2_chat_dataset.py"),
-    ("tokenizer/tokenizer.json", "tokenizer/tokenizer.json"),
-    ("data/sft_identity.json", "data/sft_identity.json"),
-    ("data/sft_greetings.json", "data/sft_greetings.json")
-]
-
-for src, dst in files_to_sync:
-    try:
-        p = hf_hub_download(repo_id=REPO_ID, filename=src, token=HF_TOKEN)
-        shutil.copy(p, dst)
-        print(f"✅ Synced: {dst}")
-    except Exception as e:
-        print(f"⚠️ Notice: {src} ({e})")
-
-# 3. Execute Builder with Direct Tokenization and Hugging Face Upload
-print("\n" + "=" * 85)
-print("⚡ Starting Dual-Phase Dataset Generation (Raw Extraction + Shard Tokenization)...")
-print("=" * 85)
-
-cmd = [
-    sys.executable,
-    "code/build_sft_v2_chat_dataset.py",
-    "--phase", "all",
-    "--upload_hf"
-]
-
-env = os.environ.copy()
-env["HF_TOKEN"] = HF_TOKEN
-subprocess.run(cmd, env=env, check=True)
+p = hf_hub_download(repo_id=REPO_ID, filename="code/build_sft_v2_chat_dataset.py", token=HF_TOKEN, force_download=True)
+shutil.copy(p, "code/build_sft_v2_chat_dataset.py")
+print("✅ build_sft_v2_chat_dataset.py ready!")
 
-print("\n" + "=" * 85)
-print("🎉 SFT v2 PURE CHAT DATASET SUCCESSFULLY BUILT & UPLOADED TO HUGGING FACE!")
-print("   • Raw Repo:       https://huggingface.co/datasets/ViuAI/sarus-sft-data-raw")
-print("   • Tokenized Repo: https://huggingface.co/datasets/ViuAI/viuai-500m-sft-tokenized (sft_v2)")
-print("   • Ready for H200 / Cloud GPU SFT v2 Training!")
-print("=" * 85)
+# 3. Launch Builder with LIVE Real-Time Terminal Output (-u unbuffered)
+!python -u code/build_sft_v2_chat_dataset.py --phase all --upload_hf