Merge branch 'اسم-الفرع-الجديد' into main

app.py

@@ -218,10 +218,9 @@ def train():
                 epochs = int(request.form.get("epochs", 1))
                 batch_size = int(request.form.get("batch_size", 1))
             else:
-                # لو مفيش file ولا JSON → خطأ
                 return jsonify({"error": "No file uploaded or JSON provided"}), 400
 
-        # Merge & remove duplicates
+        # دمج وإزالة التكرارات بناءً على محتوى الرسائل فقط
         old_data = []
         if os.path.exists(DATASET_FILE):
             with open(DATASET_FILE, "r", encoding="utf-8") as f:
@@ -231,18 +230,26 @@ def train():
                     old_data = []
 
         combined = old_data + data
-        unique_data, seen = [], set()
+        unique_data, seen_texts = [], set()
+
         for item in combined:
-            key = json.dumps(item, ensure_ascii=False)
-            if key not in seen:
-                seen.add(key)
+            # نحول محتوى الرسائل فقط لسلسلة نصية لتحديد التكرار
+            if "messages" in item:
+                text_key = "\n".join([f"{m['role'].strip().lower()}:{m['content'].strip()}" for m in item["messages"]])
+            elif "text" in item:
+                text_key = item["text"].strip()
+            else:
+                continue  # تجاهل أي item فارغ أو غير صالح
+
+            if text_key not in seen_texts:
+                seen_texts.add(text_key)
                 unique_data.append(item)
 
-        # Save dataset
+        # الحفظ بعد إزالة التكرارات
         with open(DATASET_FILE, "w", encoding="utf-8") as f:
             json.dump(unique_data, f, ensure_ascii=False, indent=2)
 
-        # Prepare dataset
+        # تجهيز dataset للتدريب
         formatted = []
         for item in unique_data:
             if "messages" in item:
@@ -272,6 +279,7 @@ def train():
 
         trainer = Trainer(model=model, args=training_args, train_dataset=tokenized_dataset)
 
+        # تشغيل التدريب في Thread منفصل
         def run_training():
             print("🔄 Training started...")
             trainer.train()
@@ -280,7 +288,12 @@ def train():
 
         threading.Thread(target=run_training, daemon=True).start()
 
-        return jsonify({"status": "training started", "epochs": epochs, "batch_size": batch_size, "records": len(unique_data)})
+        return jsonify({
+            "status": "training started",
+            "epochs": epochs,
+            "batch_size": batch_size,
+            "records": len(unique_data)
+        })
 
     except Exception as e:
         return jsonify({"error": str(e)})