Update translator_utils.py

translator_utils.py

@@ -2,7 +2,7 @@ import os
 import pandas as pd
 from transformers import pipeline
 import torch
-from huggingface_hub import HfApi
+from huggingface_hub import HfApi, hf_hub_download
 import spacy
 from spacy.lang.en import English
 
@@ -21,6 +21,7 @@ _nlp = None
 def download_dataset(force=False):
     """
     Downloads the dataset if it doesn't exist locally or if forced.
+    Uses hf_hub_download to ensure raw file integrity (avoids parsing errors).
     """
     if force and os.path.exists(LOCAL_FILENAME):
         try:
@@ -29,13 +30,19 @@ def download_dataset(force=False):
             pass
         
     if not os.path.exists(LOCAL_FILENAME):
-        print(f"Downloading dataset from {DATASET_URL}...")
+        print(f"Downloading dataset from {ENV_REPO_ID}...")
         try:
-            # Try reading with pandas directly from URL
-            # We use on_bad_lines='skip' to be robust
-            df = pd.read_csv(DATASET_URL, on_bad_lines='skip')
-            df.to_csv(LOCAL_FILENAME, index=False)
-            print("Download complete.")
+            # Use hf_hub_download to get the raw file
+            # local_dir="." downloads it to the current directory
+            downloaded_path = hf_hub_download(
+                repo_id=ENV_REPO_ID,
+                filename=DATA_FILE,
+                repo_type="dataset",
+                token=ENV_HF_TOKEN if ENV_HF_TOKEN else None,
+                local_dir=".",
+                local_dir_use_symlinks=False
+            )
+            print(f"Download complete: {downloaded_path}")
         except Exception as e:
             print(f"Error downloading dataset: {e}")
             pass
@@ -60,6 +67,7 @@ def read_csv_robust(filepath):
     
     for sep in separators:
         try:
+            # Try reading with specific separator
             df = pd.read_csv(filepath, sep=sep, on_bad_lines='skip')
             # Heuristic: If we have > 1 columns, it's likely correct
             if len(df.columns) > 1:
@@ -126,7 +134,7 @@ def load_resources():
 
 def smart_translate(text, translator, nlp):
     """
-    Translates text by chunking it into sentences to avoid truncation.
+    Translates text by chunking it into sentences AND splitting long sentences to avoid truncation.
     Strictly preserves full content.
     """
     if not text or not isinstance(text, str) or text.strip() == "":
@@ -135,44 +143,59 @@ def smart_translate(text, translator, nlp):
     doc = nlp(text)
     sentences = [sent.text.strip() for sent in doc.sents if sent.text.strip()]
     
-    chunks = []
-    current_chunk = []
-    current_len = 0
-    MAX_LEN = 250 
+    final_chunks = []
     
+    # 1. Processing Sentences
     for sent in sentences:
-        sent_len = len(sent.split())
+        words = sent.split()
         
-        if sent_len > MAX_LEN:
-            if current_chunk:
-                chunks.append(" ".join(current_chunk))
-                current_chunk = []
-                current_len = 0
-            chunks.append(sent) 
-            continue
+        # 2. Safety Check: If a single sentence is still huge (> 200 words)
+        # We split it by word count strictly
+        if len(words) > 200:
+            # Word-level chunking
+            chunk_size = 150 # Safe buffer
+            for i in range(0, len(words), chunk_size):
+                sub_chunk = " ".join(words[i:i+chunk_size])
+                final_chunks.append(sub_chunk)
+        else:
+            final_chunks.append(sent)
             
-        if current_len + sent_len > MAX_LEN:
-            chunks.append(" ".join(current_chunk))
-            current_chunk = [sent]
-            current_len = sent_len
+    # 3. Grouping small chunks back together to optimize calls (optional but good)
+    # We can skip this if we want maximum safety, but grouping saves time.
+    # Let's group them again carefully.
+    grouped_chunks = []
+    current_group = []
+    current_len = 0
+    MAX_GROUP_LEN = 200 # Words
+    
+    for chunk in final_chunks:
+        chunk_len = len(chunk.split())
+        
+        if current_len + chunk_len > MAX_GROUP_LEN:
+            grouped_chunks.append(" ".join(current_group))
+            current_group = [chunk]
+            current_len = chunk_len
         else:
-            current_chunk.append(sent)
-            current_len += sent_len
+            current_group.append(chunk)
+            current_len += chunk_len
             
-    if current_chunk:
-        chunks.append(" ".join(current_chunk))
+    if current_group:
+        grouped_chunks.append(" ".join(current_group))
         
-    translated_chunks = []
-    for chunk in chunks:
+    # 4. Translation
+    translated_parts = []
+    for chunk in grouped_chunks:
         if not chunk.strip(): continue
         try:
+            # Truncation=True handles the edge case where even our safe chunk is slightly over token limit
+            # But since we split words at 200, it's very unlikely to hit 512 tokens.
             res = translator(chunk, max_length=512, truncation=True)
-            translated_chunks.append(res[0]['translation_text'])
+            translated_parts.append(res[0]['translation_text'])
         except Exception as e:
             print(f"Chunk translation error: {e}")
-            translated_chunks.append(chunk) 
+            translated_parts.append(chunk) 
             
-    return " ".join(translated_chunks)
+    return " ".join(translated_parts)
 
 def process_rows(indices, token=None, progress=None):
     """