data.dir

app.py

@@ -14,30 +14,39 @@ def log(message):
     print(f"✅ {message}")
 
 
+import os
+import json
+from datasets import load_dataset
+
 DATA_DIR = "data"
+os.makedirs(DATA_DIR, exist_ok=True)  # Ensure directory exists
 
-def load_local_dataset(dataset_name):
-    """Load a dataset from a JSON file."""
-    file_path = os.path.join(DATA_DIR, f"{dataset_name}.json")
-    
-    if os.path.exists(file_path):
-        with open(file_path, "r") as f:
-            data = json.load(f)
-        print(f"✅ Loaded {dataset_name} from {file_path}")
-        return data
-    else:
-        print(f"❌ ERROR: {dataset_name} file not found!")
-        return None
-
-# ✅ Load all datasets from local storage
+# ✅ List of datasets
 datasets = {
-    "sales": load_local_dataset("sales"),
-    "blended": load_local_dataset("blended"),
-    "dialog": load_local_dataset("dialog"),
-    "multiwoz": load_local_dataset("multiwoz"),
+    "sales": "goendalf666/sales-conversations",
+    "blended": "blended_skill_talk",
+    "dialog": "daily_dialog",
+    "multiwoz": "multi_woz_v22",
 }
 
-print("✅ Datasets loaded from local storage!")
+# ✅ Save datasets to JSON
+for name, hf_name in datasets.items():
+    print(f"📥 Downloading {name} dataset...")
+    dataset = load_dataset(hf_name)
+
+    # Extract training data
+    train_data = dataset["train"]
+
+    # Convert dataset to list of dictionaries
+    data_list = [dict(row) for row in train_data]
+
+    # Save to JSON
+    file_path = os.path.join(DATA_DIR, f"{name}.json")
+    with open(file_path, "w") as f:
+        json.dump(data_list, f, indent=2)
+
+    print(f"✅ {name} dataset saved to {file_path}")
+
 
 
 # ✅ Step 1: Run Embedding Script (Import and Run)