Delete KAGGLE_VIUTRANSLATE_TEST.txt with huggingface_hub

KAGGLE_VIUTRANSLATE_TEST.txt

@@ -1,161 +0,0 @@
-# ==============================================================================
-# 🚀 ViuTranslate — 1-Click Kaggle Testing & Benchmark Suite
-# ==============================================================================
-# Evaluates the trained ViuTranslate model on real translation tasks.
-# Includes both Automated Benchmark and Interactive CLI Mode.
-# ==============================================================================
-
-import os
-import sys
-import torch
-import shutil
-
-print("=" * 80)
-print("🚀 ViuTranslate Evaluation & Interactive Test Suite")
-print("=" * 80)
-
-# 1. Ensure Dependencies
-try:
-    from tokenizers import Tokenizer
-    from huggingface_hub import hf_hub_download
-except ImportError:
-    import subprocess
-    subprocess.run([sys.executable, "-m", "pip", "install", "-q", "tokenizers", "huggingface_hub"], check=True)
-    from tokenizers import Tokenizer
-    from huggingface_hub import hf_hub_download
-
-HF_TOKEN = ''.join(['hf_','ssyCVhuny','XxjGdqKp','VLPpkmWK','FrrMOIFbg'])
-
-# 2. Download Model Architecture & Tokenizer if missing
-cur_dir = os.path.dirname(os.path.abspath(__file__)) if "__file__" in locals() else os.getcwd()
-if cur_dir not in sys.path:
-    sys.path.insert(0, cur_dir)
-
-for f in ["code/model.py", "code/config.py"]:
-    dest = os.path.basename(f)
-    if not os.path.exists(dest):
-        p = hf_hub_download(repo_id="ViuAI/ViuAI-500M", filename=f, token=HF_TOKEN)
-        shutil.copy(p, dest)
-
-if not os.path.exists("tokenizer.json"):
-    if os.path.exists("tokenizer/tokenizer.json"):
-        tok_path = "tokenizer/tokenizer.json"
-    else:
-        tok_p = hf_hub_download(repo_id="ViuAI/ViuAI-500M", filename="tokenizer/tokenizer.json", token=HF_TOKEN)
-        tok_path = tok_p
-else:
-    tok_path = "tokenizer.json"
-
-from model import ViuAI
-from config import ViuAIConfig
-
-# 3. Locate & Download Trained ViuTranslate Checkpoint
-ckpt_candidates = [
-    "sft_checkpoints/ViuTranslate/viutranslate_final.pt",
-    "viutranslate_final.pt",
-    "sft_checkpoints/ViuTranslate/ckpt_best.pt"
-]
-ckpt_path = None
-for c in ckpt_candidates:
-    if os.path.exists(c):
-        ckpt_path = c
-        break
-
-if ckpt_path is None:
-    print("🌐 Downloading viutranslate_final.pt from Hugging Face...")
-    os.makedirs("sft_checkpoints/ViuTranslate", exist_ok=True)
-    ckpt_path = "sft_checkpoints/ViuTranslate/viutranslate_final.pt"
-    try:
-        print("   • Checking dedicated repo ViuAI/ViuTranslate...")
-        dl_p = hf_hub_download(
-            repo_id="ViuAI/ViuTranslate",
-            filename="viutranslate_final.pt",
-            token=HF_TOKEN
-        )
-    except Exception:
-        print("   • Fallback: Checking ViuAI/ViuAI-500M...")
-        dl_p = hf_hub_download(
-            repo_id="ViuAI/ViuAI-500M",
-            filename="sft_checkpoints/ViuTranslate/viutranslate_final.pt",
-            token=HF_TOKEN
-        )
-    shutil.copy(dl_p, ckpt_path)
-
-print(f"📦 Loading weights from: {ckpt_path}")
-device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-
-cfg = ViuAIConfig(vocab_size=64003, context_length=2048)
-model = ViuAI(cfg).to(device)
-
-state = torch.load(ckpt_path, map_location=device, weights_only=False)
-weights = state.get("model_state_dict", state)
-model.load_state_dict(weights, strict=False)
-model.eval()
-
-tokenizer = Tokenizer.from_file(tok_path)
-EOT_ID = 64002
-
-print(f"✅ ViuTranslate 500M Model loaded successfully on {device.type.upper()}!")
-
-# 4. Pure Google-Translate Inference Engine
-@torch.no_grad()
-def translate_text(text: str, mode: str = "direct", src_lang: str = "auto") -> str:
-    if mode == "direct":
-        prompt = f"<|user|>\n{text.strip()}<|endofturn|>\n<|assistant|>\n"
-    elif mode == "to_hi":
-        prompt = f"<|user|>\nTranslate to Hindi: '{text.strip()}'<|endofturn|>\n<|assistant|>\n"
-    elif mode == "to_en":
-        prompt = f"<|user|>\nTranslate to English: '{text.strip()}'<|endofturn|>\n<|assistant|>\n"
-    else:
-        prompt = f"<|user|>\n{text.strip()}<|endofturn|>\n<|assistant|>\n"
-        
-    input_ids = torch.tensor([tokenizer.encode(prompt).ids], dtype=torch.long, device=device)
-    prompt_len = input_ids.shape[1]
-    
-    out = model.generate(
-        input_ids,
-        max_new_tokens=150,
-        temperature=0.2,
-        top_p=0.9,
-        repetition_penalty=1.15,
-        eos_token_id=EOT_ID
-    )
-    gen_tokens = out[0][prompt_len:].tolist()
-    if EOT_ID in gen_tokens:
-        gen_tokens = gen_tokens[:gen_tokens.index(EOT_ID)]
-    return tokenizer.decode(gen_tokens).strip()
-
-# 5. Benchmark Suite
-benchmark_cases = [
-    # English -> Hindi (Direct)
-    ("The sun rises in the east and sets in the west.", "direct", "Direct EN -> HI"),
-    ("Consistency and discipline are the keys to long term success.", "direct", "Proverb EN -> HI"),
-    ("Artificial intelligence is transforming industries across the globe.", "direct", "Tech EN -> HI"),
-    ("Please keep your passport and ticket ready at the check-in counter.", "direct", "Travel EN -> HI"),
-    ("Regular physical exercise improves cardiovascular health and boosts mental well-being.", "direct", "Health EN -> HI"),
-    
-    # Hindi -> English (Direct)
-    ("सूरज पूर्व में उगता है और पश्चिम में डूबता है।", "direct", "Direct HI -> EN"),
-    ("सफलता का कोई शॉर्टकट नहीं होता, केवल कठिन परिश्रम और लगन ही रंग लाती है।", "direct", "Proverb HI -> EN"),
-    ("स्वास्थ्य ही सबसे बड़ा धन है और संतुष्टि सबसे बड़ा सुख।", "direct", "Philosophy HI -> EN"),
-    ("भारतीय संविधान विश्व का सबसे बड़ा लिखित संविधान है।", "direct", "Civics HI -> EN"),
-    ("मुझे कल सुबह दिल्ली के लिए ट्रेन पकड़नी है।", "direct", "Daily Conversation HI -> EN"),
-
-    # Command Mode
-    ("Where is the nearest hospital and how can I get there?", "to_hi", "Command EN -> HI"),
-    ("भारत विविध संस्कृतियों और भाषाओं का एक अनूठा देश है।", "to_en", "Command HI -> EN"),
-]
-
-print("\n" + "=" * 80)
-print("📊 RUNNING OFFICIAL VIUTRANSLATE BENCHMARK SUITE")
-print("=" * 80)
-
-for text, mode, label in benchmark_cases:
-    print(f"\n🔹 [{label}]")
-    print(f"   INPUT:  {text}")
-    res = translate_text(text, mode=mode)
-    print(f"   OUTPUT: {res}")
-
-print("\n" + "=" * 80)
-print("✅ Benchmark Completed!")
-print("=" * 80)