{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "455e3ffe-b0f3-4c51-9e43-c0c13eb3add9",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from arabert.preprocess import ArabertPreprocessor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "2c1af2d3-1c84-42c6-8d4a-6f8312ab2038",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:root:Model provided is not in the accepted model list. Preprocessor will default to a base Arabic preprocessor\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done! Saved to /home/skiredj.abderrahman/khalil/sbert_training/third_training/benchmark/benchmark15_preprocessed_arabertv02.csv\n",
      "                                       sentence1  \\\n",
      "0       شخص ما يحمل لوح التزلج ليلا على الرصيف .   \n",
      "1          تتسابق النساء في سباق الدايتونا 500 .   \n",
      "2                     تمشي النساء جنبا إلى جنب .   \n",
      "3  يقفز الرجل ذو القميص الأخضر عاليا على العشب .   \n",
      "4            رجلان يجلسان على العشب ومعهما موز .   \n",
      "\n",
      "                                           sentence2  score  \n",
      "0  رجل جالس بمفرده يقرأ على طاولة مستديرة ، خارج ...    0.8  \n",
      "1              يتسابق بعض الرجال ضمن مسابقة التزلج .    1.0  \n",
      "2                          هناك فتيات يمشين متجاورات    2.6  \n",
      "3  يمشي الرجل ذو القميص الأبيض على العشب الطويل م...    2.2  \n",
      "4          ثلاثة رجال يتسكعون عند فرشة بيع الفاكهة .    1.4  \n"
     ]
    }
   ],
   "source": [
    "#mteb/sts17-crosslingual-sts\n",
    "\n",
    "\n",
    "# Load CSV\n",
    "df = pd.read_csv('/home/skiredj.abderrahman/khalil/sbert_training/third_training/benchmark/benchmark15.csv')\n",
    "\n",
    "# Drop lang column\n",
    "df = df.drop(columns=['lang'], errors='ignore')\n",
    "\n",
    "# Initialize ArabertPreprocessor\n",
    "model_name = \"/home/skiredj.abderrahman/khalil/sbert_training/third_training/bert-base-arabertv02\"\n",
    "arabert_prep = ArabertPreprocessor(model_name=model_name)\n",
    "\n",
    "# Preprocess all text columns (adjust column names as needed)\n",
    "text_columns = df.select_dtypes(include='object').columns.tolist()\n",
    "\n",
    "for col in text_columns:\n",
    "    df[col] = df[col].apply(lambda x: arabert_prep.preprocess(str(x)) if pd.notna(x) else x)\n",
    "\n",
    "# Save preprocessed CSV\n",
    "output_path = '/home/skiredj.abderrahman/khalil/sbert_training/third_training/benchmark/benchmark15_preprocessed_arabertv02.csv'\n",
    "df.to_csv(output_path, index=False)\n",
    "\n",
    "print(f\"Done! Saved to {output_path}\")\n",
    "print(df.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "fdfe7c48-5358-4cde-a81c-b3300634c766",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/skiredj.abderrahman/.conda/envs/sbert_khalil/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading CSV...\n",
      "  250 pairs loaded.\n",
      "                                       sentence1  \\\n",
      "0       شخص ما يحمل لوح التزلج ليلا على الرصيف .   \n",
      "1          تتسابق النساء في سباق الدايتونا 500 .   \n",
      "2                     تمشي النساء جنبا إلى جنب .   \n",
      "3  يقفز الرجل ذو القميص الأخضر عاليا على العشب .   \n",
      "4            رجلان يجلسان على العشب ومعهما موز .   \n",
      "\n",
      "                                           sentence2  score  \n",
      "0  رجل جالس بمفرده يقرأ على طاولة مستديرة ، خارج ...    0.8  \n",
      "1              يتسابق بعض الرجال ضمن مسابقة التزلج .    1.0  \n",
      "2                          هناك فتيات يمشين متجاورات    2.6  \n",
      "3  يمشي الرجل ذو القميص الأبيض على العشب الطويل م...    2.2  \n",
      "4          ثلاثة رجال يتسكعون عند فرشة بيع الفاكهة .    1.4  \n",
      "\n",
      "Loading SBERT model and encoding sentences...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading weights: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████| 199/199 [00:00<00:00, 856.85it/s, Materializing param=pooler.dense.weight]\n",
      "Batches: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 8/8 [00:00<00:00, 22.09it/s]\n",
      "Batches: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 8/8 [00:00<00:00, 156.77it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Computing cosine similarities...\n",
      "\n",
      "─── STS Correlation Results ───────────────────────────────\n",
      "  Pearson  r : 0.8219  (p=1.5414e-62)\n",
      "  Spearman r : 0.8230  (p=7.4959e-63)\n",
      "\n",
      "─── Hit Rate & MRR ────────────────────────────────────────\n",
      "  (Relevance threshold on gold score: >= 0.5)\n",
      "  Hit Rate @ 1 : 0.5668  (123/217)\n",
      "  Hit Rate @ 3 : 0.7189  (156/217)\n",
      "  Hit Rate @ 5 : 0.7604  (165/217)\n",
      "  MRR        : 0.6610\n",
      "\n",
      "Results saved to: /home/skiredj.abderrahman/khalil/sbert_training/third_training/benchmark/benchmark15_preprocessed_arabertv02_benchmark_results.csv\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sentence_transformers import SentenceTransformer\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "from scipy.stats import pearsonr, spearmanr\n",
    "\n",
    "# ─── CONFIG ───────────────────────────────────────────────────────────────────\n",
    "MODEL_PATH  = \"/home/skiredj.abderrahman/khalil/sbert_training/third_training/output/arabert_20260326_1254/final\"\n",
    "CSV_PATH    = \"/home/skiredj.abderrahman/khalil/sbert_training/third_training/benchmark/benchmark15_preprocessed_arabertv02.csv\"\n",
    "TOP_K       = [1, 3, 5]\n",
    "SCORE_THRESHOLD = 0.5             # adjust if your score scale is not [0,1]\n",
    "# ──────────────────────────────────────────────────────────────────────────────\n",
    "\n",
    "# 1. Load CSV\n",
    "print(\"Loading CSV...\")\n",
    "df = pd.read_csv(CSV_PATH)\n",
    "if 'lang' in df.columns:\n",
    "    df = df.drop(columns=['lang'])\n",
    "\n",
    "print(f\"  {len(df)} pairs loaded.\")\n",
    "print(df.head())\n",
    "\n",
    "# 2. Encode\n",
    "print(\"\\nLoading SBERT model and encoding sentences...\")\n",
    "model = SentenceTransformer(MODEL_PATH)\n",
    "emb1 = model.encode(df['sentence1'].tolist(), batch_size=32, show_progress_bar=True, convert_to_numpy=True)\n",
    "emb2 = model.encode(df['sentence2'].tolist(), batch_size=32, show_progress_bar=True, convert_to_numpy=True)\n",
    "\n",
    "# 3. Cosine similarity per pair\n",
    "print(\"\\nComputing cosine similarities...\")\n",
    "cos_scores = np.array([\n",
    "    cosine_similarity(emb1[i].reshape(1, -1), emb2[i].reshape(1, -1))[0][0]\n",
    "    for i in range(len(df))\n",
    "])\n",
    "\n",
    "# 4. Pearson & Spearman\n",
    "gold_scores = df['score'].astype(float).values\n",
    "pearson_corr,  p_val_p = pearsonr(cos_scores, gold_scores)\n",
    "spearman_corr, p_val_s = spearmanr(cos_scores, gold_scores)\n",
    "\n",
    "print(\"\\n─── STS Correlation Results ───────────────────────────────\")\n",
    "print(f\"  Pearson  r : {pearson_corr:.4f}  (p={p_val_p:.4e})\")\n",
    "print(f\"  Spearman r : {spearman_corr:.4f}  (p={p_val_s:.4e})\")\n",
    "\n",
    "# 5. Hit Rate & MRR\n",
    "print(\"\\n─── Hit Rate & MRR ────────────────────────────────────────\")\n",
    "print(f\"  (Relevance threshold on gold score: >= {SCORE_THRESHOLD})\")\n",
    "\n",
    "hit_counts = {k: 0 for k in TOP_K}\n",
    "reciprocal_ranks = []\n",
    "n_queries = 0\n",
    "\n",
    "for i in range(len(df)):\n",
    "    if gold_scores[i] < SCORE_THRESHOLD:\n",
    "        continue\n",
    "\n",
    "    query_emb   = emb1[i].reshape(1, -1)\n",
    "    sims        = cosine_similarity(query_emb, emb2)[0]\n",
    "    ranked_idxs = np.argsort(sims)[::-1]\n",
    "\n",
    "    rank = np.where(ranked_idxs == i)[0][0] + 1  # 1-based\n",
    "\n",
    "    for k in TOP_K:\n",
    "        if rank <= k:\n",
    "            hit_counts[k] += 1\n",
    "\n",
    "    reciprocal_ranks.append(1.0 / rank)\n",
    "    n_queries += 1\n",
    "\n",
    "if n_queries == 0:\n",
    "    print(\"  No relevant pairs found — lower SCORE_THRESHOLD.\")\n",
    "else:\n",
    "    for k in TOP_K:\n",
    "        hr = hit_counts[k] / n_queries\n",
    "        print(f\"  Hit Rate @{k:2d} : {hr:.4f}  ({hit_counts[k]}/{n_queries})\")\n",
    "\n",
    "    mrr = np.mean(reciprocal_ranks)\n",
    "    print(f\"  MRR        : {mrr:.4f}\")\n",
    "\n",
    "# 6. Save results\n",
    "df['predicted_cos_sim'] = cos_scores\n",
    "output_path = CSV_PATH.replace('.csv', '_benchmark_results.csv')\n",
    "df.to_csv(output_path, index=False)\n",
    "print(f\"\\nResults saved to: {output_path}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "0596225c-0dbe-4cbc-a97e-6f4be5e19cfa",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading CSV...\n",
      "  250 pairs loaded.\n",
      "                                      sentence1  \\\n",
      "0       شخص ما يحمل لوح التزلج ليلا على الرصيف.   \n",
      "1          تتسابق النساء في سباق الدايتونا 500.   \n",
      "2                     تمشي النساء جنبا إلى جنب.   \n",
      "3  يقفز الرجل ذو القميص الأخضر عاليا على العشب.   \n",
      "4            رجلان يجلسان على العشب ومعهما موز.   \n",
      "\n",
      "                                           sentence2  score  \n",
      "0  رجل جالس بمفرده يقرأ على طاولة مستديرة ، خارج ...    0.8  \n",
      "1               يتسابق بعض الرجال ضمن مسابقة التزلج.    1.0  \n",
      "2                          هناك فتيات يمشين متجاورات    2.6  \n",
      "3  يمشي الرجل ذو القميص الأبيض على العشب الطويل م...    2.2  \n",
      "4           ثلاثة رجال يتسكعون عند فرشة بيع الفاكهة.    1.4  \n",
      "\n",
      "Loading multilingual-e5-base and encoding sentences...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads.\n",
      "WARNING:huggingface_hub.utils._http:Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads.\n",
      "Loading weights: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████| 199/199 [00:00<00:00, 743.94it/s, Materializing param=pooler.dense.weight]\n",
      "\u001b[1mXLMRobertaModel LOAD REPORT\u001b[0m from: intfloat/multilingual-e5-base\n",
      "Key                     | Status     |  | \n",
      "------------------------+------------+--+-\n",
      "embeddings.position_ids | UNEXPECTED |  | \n",
      "\n",
      "\u001b[3mNotes:\n",
      "- UNEXPECTED\u001b[3m\t:can be ignored when loading from different task/architecture; not ok if you expect identical arch.\u001b[0m\n",
      "Batches: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 8/8 [00:00<00:00, 78.36it/s]\n",
      "Batches: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 8/8 [00:00<00:00, 123.73it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Computing cosine similarities...\n",
      "\n",
      "─── STS Correlation Results ───────────────────────────────\n",
      "  Pearson  r : 0.7446  (p=1.9457e-45)\n",
      "  Spearman r : 0.7450  (p=1.6963e-45)\n",
      "\n",
      "─── Hit Rate & MRR ────────────────────────────────────────\n",
      "  (Relevance threshold on gold score: >= 0.5)\n",
      "  Hit Rate @ 1 : 0.5991  (130/217)\n",
      "  Hit Rate @ 3 : 0.7327  (159/217)\n",
      "  Hit Rate @ 5 : 0.7696  (167/217)\n",
      "  MRR        : 0.6794\n",
      "\n",
      "Results saved to: /home/skiredj.abderrahman/khalil/sbert_training/third_training/benchmark/benchmark15_e5_benchmark_results.csv\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sentence_transformers import SentenceTransformer\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "from scipy.stats import pearsonr, spearmanr\n",
    "\n",
    "# ─── CONFIG ───────────────────────────────────────────────────────────────────\n",
    "MODEL_NAME      = \"intfloat/multilingual-e5-base\"\n",
    "CSV_PATH        = \"/home/skiredj.abderrahman/khalil/sbert_training/third_training/benchmark/benchmark15.csv\"\n",
    "TOP_K           = [1, 3, 5]\n",
    "SCORE_THRESHOLD = 0.5             # adjust to your score scale\n",
    "PREFIX          = \"query: \"       # symmetric STS — same prefix for both sides\n",
    "# ──────────────────────────────────────────────────────────────────────────────\n",
    "\n",
    "# 1. Load CSV\n",
    "print(\"Loading CSV...\")\n",
    "df = pd.read_csv(CSV_PATH)\n",
    "if 'lang' in df.columns:\n",
    "    df = df.drop(columns=['lang'])\n",
    "\n",
    "print(f\"  {len(df)} pairs loaded.\")\n",
    "print(df.head())\n",
    "\n",
    "# 2. Apply E5 prefix\n",
    "sentences1 = [PREFIX + str(s) for s in df['sentence1'].tolist()]\n",
    "sentences2 = [PREFIX + str(s) for s in df['sentence2'].tolist()]\n",
    "\n",
    "# 3. Encode\n",
    "print(\"\\nLoading multilingual-e5-base and encoding sentences...\")\n",
    "model = SentenceTransformer(MODEL_NAME)\n",
    "emb1 = model.encode(sentences1, batch_size=32, show_progress_bar=True, normalize_embeddings=True, convert_to_numpy=True)\n",
    "emb2 = model.encode(sentences2, batch_size=32, show_progress_bar=True, normalize_embeddings=True, convert_to_numpy=True)\n",
    "\n",
    "# 4. Cosine similarity per pair\n",
    "# Note: since embeddings are L2-normalized, cosine sim == dot product\n",
    "print(\"\\nComputing cosine similarities...\")\n",
    "cos_scores = np.array([\n",
    "    cosine_similarity(emb1[i].reshape(1, -1), emb2[i].reshape(1, -1))[0][0]\n",
    "    for i in range(len(df))\n",
    "])\n",
    "\n",
    "# 5. Pearson & Spearman\n",
    "gold_scores = df['score'].astype(float).values\n",
    "pearson_corr,  p_val_p = pearsonr(cos_scores, gold_scores)\n",
    "spearman_corr, p_val_s = spearmanr(cos_scores, gold_scores)\n",
    "\n",
    "print(\"\\n─── STS Correlation Results ───────────────────────────────\")\n",
    "print(f\"  Pearson  r : {pearson_corr:.4f}  (p={p_val_p:.4e})\")\n",
    "print(f\"  Spearman r : {spearman_corr:.4f}  (p={p_val_s:.4e})\")\n",
    "\n",
    "# 6. Hit Rate & MRR\n",
    "print(\"\\n─── Hit Rate & MRR ────────────────────────────────────────\")\n",
    "print(f\"  (Relevance threshold on gold score: >= {SCORE_THRESHOLD})\")\n",
    "\n",
    "hit_counts = {k: 0 for k in TOP_K}\n",
    "reciprocal_ranks = []\n",
    "n_queries = 0\n",
    "\n",
    "for i in range(len(df)):\n",
    "    if gold_scores[i] < SCORE_THRESHOLD:\n",
    "        continue\n",
    "\n",
    "    query_emb   = emb1[i].reshape(1, -1)\n",
    "    sims        = cosine_similarity(query_emb, emb2)[0]\n",
    "    ranked_idxs = np.argsort(sims)[::-1]\n",
    "\n",
    "    rank = np.where(ranked_idxs == i)[0][0] + 1  # 1-based\n",
    "\n",
    "    for k in TOP_K:\n",
    "        if rank <= k:\n",
    "            hit_counts[k] += 1\n",
    "\n",
    "    reciprocal_ranks.append(1.0 / rank)\n",
    "    n_queries += 1\n",
    "\n",
    "if n_queries == 0:\n",
    "    print(\"  No relevant pairs found — lower SCORE_THRESHOLD.\")\n",
    "else:\n",
    "    for k in TOP_K:\n",
    "        hr = hit_counts[k] / n_queries\n",
    "        print(f\"  Hit Rate @{k:2d} : {hr:.4f}  ({hit_counts[k]}/{n_queries})\")\n",
    "\n",
    "    mrr = np.mean(reciprocal_ranks)\n",
    "    print(f\"  MRR        : {mrr:.4f}\")\n",
    "\n",
    "# 7. Save results\n",
    "df['predicted_cos_sim'] = cos_scores\n",
    "output_path = CSV_PATH.replace('.csv', '_e5_benchmark_results.csv')\n",
    "df.to_csv(output_path, index=False)\n",
    "print(f\"\\nResults saved to: {output_path}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "374c893e-c7ec-4ba8-9c3c-b23c79e56b8c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Successfully converted to /home/skiredj.abderrahman/khalil/sbert_training/third_training/benchmark/ArabicaQA_test.csv\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "import csv\n",
    "\n",
    "# Load your json file\n",
    "with open('/home/skiredj.abderrahman/khalil/sbert_training/third_training/benchmark/ArabicaQA_test.json', 'r', encoding='utf-8') as f:\n",
    "    json_data = json.load(f)\n",
    "\n",
    "output_file = '/home/skiredj.abderrahman/khalil/sbert_training/third_training/benchmark/ArabicaQA_test.csv'\n",
    "\n",
    "with open(output_file, 'w', newline='', encoding='utf-8-sig') as csvfile:\n",
    "    writer = csv.writer(csvfile)\n",
    "    # Writing the header\n",
    "    writer.writerow(['query', 'passage'])\n",
    "\n",
    "    # Navigating the SQuAD structure\n",
    "    for entry in json_data.get('data', []):\n",
    "        for paragraph in entry.get('paragraphs', []):\n",
    "            # If you want the full context as the passage, use this:\n",
    "            # context = paragraph.get('context', '') \n",
    "            \n",
    "            for qa in paragraph.get('qas', []):\n",
    "                query = qa.get('question')\n",
    "                \n",
    "                # Extracting the first answer text as the passage\n",
    "                answers = qa.get('answers', [])\n",
    "                if answers:\n",
    "                    passage = answers[0].get('text')\n",
    "                    writer.writerow([query, passage])\n",
    "\n",
    "print(f\"Successfully converted to {output_file}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "1a8404ab-ab43-4123-9fda-7ea7f9bbb987",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>query</th>\n",
       "      <th>passage</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>من الذي يترأس المرصد الحضري لمدينة الرياض؟</td>\n",
       "      <td>يترأس مجلس المرصد أمير المنطقة بالإضافة إلى أر...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ما هي الإنجازات البارزة التي حققها المرصد الحض...</td>\n",
       "      <td>في مارس 2017 حصل المرصد الحضري بمدينة الرياض ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ما هو المرصد الحضري لمدينة الرياض؟</td>\n",
       "      <td>هو مركز متخصص في وضع نظم مراقبة حضرية لرصد سير...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ما هو الهدف من المرصد الحضري لمدينة الرياض؟</td>\n",
       "      <td>يهدف إلى فهم الوضع الحالي للمدينة ووضع تصور لل...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ما هي مهام المرصد الحضري لمدينة الرياض؟</td>\n",
       "      <td>ومراقبة الإنجاز لتحقيق الأهداف واتخاذ القرارات.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13965</th>\n",
       "      <td>ما هو الموقف الفلسفي؟\\n</td>\n",
       "      <td>هي وجهة نظر تحاول شرح أو تفسير مشكلة معينة في ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13966</th>\n",
       "      <td>ما هو مصطلح \"نظرية\"؟\\n</td>\n",
       "      <td>هو بيان للغة الإنجليزية العامية وليس انعكاسًا...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13967</th>\n",
       "      <td>ما هو بنك دبي؟</td>\n",
       "      <td>مصرف إسلامي مقره في دبي، الإمارات العربية المتحدة</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13968</th>\n",
       "      <td>متي تأسس بنك دبي؟</td>\n",
       "      <td>سبتمبر 2002</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13969</th>\n",
       "      <td>متي أستحوز عليه استحوذ عليه بنك الإمارات دبي ا...</td>\n",
       "      <td>1 ديسمبر 2012</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>13970 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   query  \\\n",
       "0             من الذي يترأس المرصد الحضري لمدينة الرياض؟   \n",
       "1      ما هي الإنجازات البارزة التي حققها المرصد الحض...   \n",
       "2                     ما هو المرصد الحضري لمدينة الرياض؟   \n",
       "3            ما هو الهدف من المرصد الحضري لمدينة الرياض؟   \n",
       "4                ما هي مهام المرصد الحضري لمدينة الرياض؟   \n",
       "...                                                  ...   \n",
       "13965                            ما هو الموقف الفلسفي؟\\n   \n",
       "13966                             ما هو مصطلح \"نظرية\"؟\\n   \n",
       "13967                                     ما هو بنك دبي؟   \n",
       "13968                                  متي تأسس بنك دبي؟   \n",
       "13969  متي أستحوز عليه استحوذ عليه بنك الإمارات دبي ا...   \n",
       "\n",
       "                                                 passage  \n",
       "0      يترأس مجلس المرصد أمير المنطقة بالإضافة إلى أر...  \n",
       "1       في مارس 2017 حصل المرصد الحضري بمدينة الرياض ...  \n",
       "2      هو مركز متخصص في وضع نظم مراقبة حضرية لرصد سير...  \n",
       "3      يهدف إلى فهم الوضع الحالي للمدينة ووضع تصور لل...  \n",
       "4        ومراقبة الإنجاز لتحقيق الأهداف واتخاذ القرارات.  \n",
       "...                                                  ...  \n",
       "13965  هي وجهة نظر تحاول شرح أو تفسير مشكلة معينة في ...  \n",
       "13966   هو بيان للغة الإنجليزية العامية وليس انعكاسًا...  \n",
       "13967  مصرف إسلامي مقره في دبي، الإمارات العربية المتحدة  \n",
       "13968                                        سبتمبر 2002  \n",
       "13969                                     1 ديسمبر 2012   \n",
       "\n",
       "[13970 rows x 2 columns]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df=pd.read_csv('/home/skiredj.abderrahman/khalil/sbert_training/third_training/benchmark/ArabicaQA_test.csv')\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "9ae7575c-4edd-4c85-b3db-4598e92e5f95",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:root:Model provided is not in the accepted model list. Preprocessor will default to a base Arabic preprocessor\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done! Saved to /home/skiredj.abderrahman/khalil/sbert_training/third_training/benchmark/ArabicaQA_test_preprocessed_arabertv02.csv\n",
      "                                               query  \\\n",
      "0        من الذي يترأس المرصد الحضري لمدينة الرياض ؟   \n",
      "1  ما هي الإنجازات البارزة التي حققها المرصد الحض...   \n",
      "2                ما هو المرصد الحضري لمدينة الرياض ؟   \n",
      "3       ما هو الهدف من المرصد الحضري لمدينة الرياض ؟   \n",
      "4           ما هي مهام المرصد الحضري لمدينة الرياض ؟   \n",
      "\n",
      "                                             passage  \n",
      "0  يترأس مجلس المرصد أمير المنطقة بالإضافة إلى أر...  \n",
      "1  في مارس 2017 حصل المرصد الحضري بمدينة الرياض ع...  \n",
      "2  هو مركز متخصص في وضع نظم مراقبة حضرية لرصد سير...  \n",
      "3  يهدف إلى فهم الوضع الحالي للمدينة ووضع تصور لل...  \n",
      "4   ومراقبة الإنجاز لتحقيق الأهداف واتخاذ القرارات .  \n"
     ]
    }
   ],
   "source": [
    "#mteb/sts17-crosslingual-sts\n",
    "\n",
    "\n",
    "# Load CSV\n",
    "df = pd.read_csv('/home/skiredj.abderrahman/khalil/sbert_training/third_training/benchmark/ArabicaQA_test.csv')\n",
    "\n",
    "# Drop lang column\n",
    "df = df.drop(columns=['lang'], errors='ignore')\n",
    "\n",
    "# Initialize ArabertPreprocessor\n",
    "model_name = \"/home/skiredj.abderrahman/khalil/sbert_training/third_training/bert-base-arabertv02\"\n",
    "arabert_prep = ArabertPreprocessor(model_name=model_name)\n",
    "\n",
    "# Preprocess all text columns (adjust column names as needed)\n",
    "text_columns = df.select_dtypes(include='object').columns.tolist()\n",
    "\n",
    "for col in text_columns:\n",
    "    df[col] = df[col].apply(lambda x: arabert_prep.preprocess(str(x)) if pd.notna(x) else x)\n",
    "\n",
    "# Save preprocessed CSV\n",
    "output_path = '/home/skiredj.abderrahman/khalil/sbert_training/third_training/benchmark/ArabicaQA_test_preprocessed_arabertv02.csv'\n",
    "df.to_csv(output_path, index=False)\n",
    "\n",
    "print(f\"Done! Saved to {output_path}\")\n",
    "print(df.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "f649e0ac-70e6-45f3-b8c4-d2598152f942",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading dataset...\n",
      "  13970 query-passage pairs loaded.\n",
      "                                               query  \\\n",
      "0        من الذي يترأس المرصد الحضري لمدينة الرياض ؟   \n",
      "1  ما هي الإنجازات البارزة التي حققها المرصد الحض...   \n",
      "2                ما هو المرصد الحضري لمدينة الرياض ؟   \n",
      "3       ما هو الهدف من المرصد الحضري لمدينة الرياض ؟   \n",
      "4           ما هي مهام المرصد الحضري لمدينة الرياض ؟   \n",
      "\n",
      "                                             passage  \n",
      "0  يترأس مجلس المرصد أمير المنطقة بالإضافة إلى أر...  \n",
      "1  في مارس 2017 حصل المرصد الحضري بمدينة الرياض ع...  \n",
      "2  هو مركز متخصص في وضع نظم مراقبة حضرية لرصد سير...  \n",
      "3  يهدف إلى فهم الوضع الحالي للمدينة ووضع تصور لل...  \n",
      "4   ومراقبة الإنجاز لتحقيق الأهداف واتخاذ القرارات .  \n",
      "  Corpus size (unique passages): 12641\n",
      "  Number of queries            : 13970\n",
      "\n",
      "Loading AraBERT SBERT model...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading weights: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████| 199/199 [00:00<00:00, 916.83it/s, Materializing param=pooler.dense.weight]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Encoding queries...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batches: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 437/437 [00:03<00:00, 144.00it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Encoding corpus passages...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batches: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 396/396 [00:03<00:00, 129.01it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Running retrieval evaluation...\n",
      "  Processed 500/13970 queries...\n",
      "  Processed 1000/13970 queries...\n",
      "  Processed 1500/13970 queries...\n",
      "  Processed 2000/13970 queries...\n",
      "  Processed 2500/13970 queries...\n",
      "  Processed 3000/13970 queries...\n",
      "  Processed 3500/13970 queries...\n",
      "  Processed 4000/13970 queries...\n",
      "  Processed 4500/13970 queries...\n",
      "  Processed 5000/13970 queries...\n",
      "  Processed 5500/13970 queries...\n",
      "  Processed 6000/13970 queries...\n",
      "  Processed 6500/13970 queries...\n",
      "  Processed 7000/13970 queries...\n",
      "  Processed 7500/13970 queries...\n",
      "  Processed 8000/13970 queries...\n",
      "  Processed 8500/13970 queries...\n",
      "  Processed 9000/13970 queries...\n",
      "  Processed 9500/13970 queries...\n",
      "  Processed 10000/13970 queries...\n",
      "  Processed 10500/13970 queries...\n",
      "  Processed 11000/13970 queries...\n",
      "  Processed 11500/13970 queries...\n",
      "  Processed 12000/13970 queries...\n",
      "  Processed 12500/13970 queries...\n",
      "  Processed 13000/13970 queries...\n",
      "  Processed 13500/13970 queries...\n",
      "\n",
      "─── AraBERT SBERT — Asymmetric Retrieval Results ─────────\n",
      "  Corpus size : 12641 passages\n",
      "  Queries     : 13970\n",
      "\n",
      "  Hit Rate @ 1 : 0.1447  (2022/13970)\n",
      "  Hit Rate @ 3 : 0.2288  (3196/13970)\n",
      "  Hit Rate @ 5 : 0.2654  (3707/13970)\n",
      "  Hit Rate @10 : 0.3187  (4452/13970)\n",
      "  MRR         : 0.2041\n",
      "\n",
      "Detailed results saved to: /home/skiredj.abderrahman/khalil/sbert_training/third_training/benchmark/ArabicaQA_test_preprocessed_arabertv02_arabert_retrieval_results.csv\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sentence_transformers import SentenceTransformer\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "\n",
    "# ─── CONFIG ───────────────────────────────────────────────────────────────────\n",
    "MODEL_PATH  = \"/home/skiredj.abderrahman/khalil/sbert_training/third_training/output/arabert_20260326_1254/final\"\n",
    "CSV_PATH    = \"/home/skiredj.abderrahman/khalil/sbert_training/third_training/benchmark/ArabicaQA_test_preprocessed_arabertv02.csv\"          # update this\n",
    "TOP_K       = [1, 3, 5, 10]\n",
    "BATCH_SIZE  = 32\n",
    "# ──────────────────────────────────────────────────────────────────────────────\n",
    "\n",
    "# 1. Load CSV\n",
    "print(\"Loading dataset...\")\n",
    "df = pd.read_csv(CSV_PATH)\n",
    "print(f\"  {len(df)} query-passage pairs loaded.\")\n",
    "print(df.head())\n",
    "\n",
    "# 2. Deduplicate passages (build corpus)\n",
    "# Each passage is a candidate; the matching passage for query[i] is passage[i]\n",
    "passages     = df['passage'].tolist()\n",
    "queries      = df['query'].tolist()\n",
    "\n",
    "unique_passages = list(dict.fromkeys(passages))   # deduplicated, order preserved\n",
    "passage_to_idx  = {p: i for i, p in enumerate(unique_passages)}\n",
    "ground_truth    = [passage_to_idx[p] for p in passages]  # index in corpus for each query\n",
    "\n",
    "print(f\"  Corpus size (unique passages): {len(unique_passages)}\")\n",
    "print(f\"  Number of queries            : {len(queries)}\")\n",
    "\n",
    "# 3. Encode\n",
    "# Asymmetric: NO prefix for AraBERT SBERT (it was trained without them)\n",
    "print(\"\\nLoading AraBERT SBERT model...\")\n",
    "model = SentenceTransformer(MODEL_PATH)\n",
    "\n",
    "print(\"Encoding queries...\")\n",
    "query_embs = model.encode(\n",
    "    queries,\n",
    "    batch_size=BATCH_SIZE,\n",
    "    show_progress_bar=True,\n",
    "    convert_to_numpy=True\n",
    ")\n",
    "\n",
    "print(\"Encoding corpus passages...\")\n",
    "corpus_embs = model.encode(\n",
    "    unique_passages,\n",
    "    batch_size=BATCH_SIZE,\n",
    "    show_progress_bar=True,\n",
    "    convert_to_numpy=True\n",
    ")\n",
    "\n",
    "# 4. Retrieval — Hit Rate & MRR\n",
    "print(\"\\nRunning retrieval evaluation...\")\n",
    "\n",
    "hit_counts       = {k: 0 for k in TOP_K}\n",
    "reciprocal_ranks = []\n",
    "\n",
    "for i in range(len(queries)):\n",
    "    query_emb   = query_embs[i].reshape(1, -1)\n",
    "    sims        = cosine_similarity(query_emb, corpus_embs)[0]\n",
    "    ranked_idxs = np.argsort(sims)[::-1]           # descending\n",
    "\n",
    "    true_idx = ground_truth[i]\n",
    "    rank     = np.where(ranked_idxs == true_idx)[0][0] + 1  # 1-based\n",
    "\n",
    "    for k in TOP_K:\n",
    "        if rank <= k:\n",
    "            hit_counts[k] += 1\n",
    "\n",
    "    reciprocal_ranks.append(1.0 / rank)\n",
    "\n",
    "    if (i + 1) % 500 == 0:\n",
    "        print(f\"  Processed {i + 1}/{len(queries)} queries...\")\n",
    "\n",
    "# 5. Print results\n",
    "n = len(queries)\n",
    "print(\"\\n─── AraBERT SBERT — Asymmetric Retrieval Results ─────────\")\n",
    "print(f\"  Corpus size : {len(unique_passages)} passages\")\n",
    "print(f\"  Queries     : {n}\")\n",
    "print()\n",
    "for k in TOP_K:\n",
    "    hr = hit_counts[k] / n\n",
    "    print(f\"  Hit Rate @{k:2d} : {hr:.4f}  ({hit_counts[k]}/{n})\")\n",
    "\n",
    "mrr = np.mean(reciprocal_ranks)\n",
    "print(f\"  MRR         : {mrr:.4f}\")\n",
    "\n",
    "# 6. Save detailed results\n",
    "df['rank']            = [\n",
    "    int(np.where(np.argsort(\n",
    "        cosine_similarity(query_embs[i].reshape(1, -1), corpus_embs)[0]\n",
    "    )[::-1] == ground_truth[i])[0][0] + 1)\n",
    "    for i in range(len(queries))\n",
    "]\n",
    "df['reciprocal_rank'] = 1.0 / df['rank']\n",
    "\n",
    "output_path = CSV_PATH.replace('.csv', '_arabert_retrieval_results.csv')\n",
    "df.to_csv(output_path, index=False)\n",
    "print(f\"\\nDetailed results saved to: {output_path}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "56273b8e-a0cc-4668-8160-b774a9609cd0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading dataset...\n",
      "  13970 query-passage pairs loaded.\n",
      "                                               query  \\\n",
      "0         من الذي يترأس المرصد الحضري لمدينة الرياض؟   \n",
      "1  ما هي الإنجازات البارزة التي حققها المرصد الحض...   \n",
      "2                 ما هو المرصد الحضري لمدينة الرياض؟   \n",
      "3        ما هو الهدف من المرصد الحضري لمدينة الرياض؟   \n",
      "4            ما هي مهام المرصد الحضري لمدينة الرياض؟   \n",
      "\n",
      "                                             passage  \n",
      "0  يترأس مجلس المرصد أمير المنطقة بالإضافة إلى أر...  \n",
      "1   في مارس 2017 حصل المرصد الحضري بمدينة الرياض ...  \n",
      "2  هو مركز متخصص في وضع نظم مراقبة حضرية لرصد سير...  \n",
      "3  يهدف إلى فهم الوضع الحالي للمدينة ووضع تصور لل...  \n",
      "4    ومراقبة الإنجاز لتحقيق الأهداف واتخاذ القرارات.  \n",
      "  Corpus size (unique passages): 12845\n",
      "  Number of queries            : 13970\n",
      "\n",
      "Loading multilingual-e5-base model...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading weights: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████| 199/199 [00:00<00:00, 745.13it/s, Materializing param=pooler.dense.weight]\n",
      "\u001b[1mXLMRobertaModel LOAD REPORT\u001b[0m from: intfloat/multilingual-e5-base\n",
      "Key                     | Status     |  | \n",
      "------------------------+------------+--+-\n",
      "embeddings.position_ids | UNEXPECTED |  | \n",
      "\n",
      "\u001b[3mNotes:\n",
      "- UNEXPECTED\u001b[3m\t:can be ignored when loading from different task/architecture; not ok if you expect identical arch.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Encoding queries...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batches: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 437/437 [00:03<00:00, 122.76it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Encoding corpus passages...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batches: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 402/402 [00:03<00:00, 107.36it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Running retrieval evaluation...\n",
      "  Processed 500/13970 queries...\n",
      "  Processed 1000/13970 queries...\n",
      "  Processed 1500/13970 queries...\n",
      "  Processed 2000/13970 queries...\n",
      "  Processed 2500/13970 queries...\n",
      "  Processed 3000/13970 queries...\n",
      "  Processed 3500/13970 queries...\n",
      "  Processed 4000/13970 queries...\n",
      "  Processed 4500/13970 queries...\n",
      "  Processed 5000/13970 queries...\n",
      "  Processed 5500/13970 queries...\n",
      "  Processed 6000/13970 queries...\n",
      "  Processed 6500/13970 queries...\n",
      "  Processed 7000/13970 queries...\n",
      "  Processed 7500/13970 queries...\n",
      "  Processed 8000/13970 queries...\n",
      "  Processed 8500/13970 queries...\n",
      "  Processed 9000/13970 queries...\n",
      "  Processed 9500/13970 queries...\n",
      "  Processed 10000/13970 queries...\n",
      "  Processed 10500/13970 queries...\n",
      "  Processed 11000/13970 queries...\n",
      "  Processed 11500/13970 queries...\n",
      "  Processed 12000/13970 queries...\n",
      "  Processed 12500/13970 queries...\n",
      "  Processed 13000/13970 queries...\n",
      "  Processed 13500/13970 queries...\n",
      "\n",
      "─── multilingual-e5-base — Asymmetric Retrieval Results ──\n",
      "  Corpus size : 12845 passages\n",
      "  Queries     : 13970\n",
      "\n",
      "  Hit Rate @ 1 : 0.1844  (2576/13970)\n",
      "  Hit Rate @ 3 : 0.2814  (3931/13970)\n",
      "  Hit Rate @ 5 : 0.3226  (4507/13970)\n",
      "  Hit Rate @10 : 0.3802  (5312/13970)\n",
      "  MRR         : 0.2527\n",
      "\n",
      "Saving detailed results...\n",
      "Detailed results saved to: /home/skiredj.abderrahman/khalil/sbert_training/third_training/benchmark/ArabicaQA_test_e5_retrieval_results.csv\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sentence_transformers import SentenceTransformer\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "\n",
    "# ─── CONFIG ───────────────────────────────────────────────────────────────────\n",
    "MODEL_NAME  = 'intfloat/multilingual-e5-base'\n",
    "CSV_PATH    = '/home/skiredj.abderrahman/khalil/sbert_training/third_training/benchmark/ArabicaQA_test.csv'         # update this\n",
    "TOP_K       = [1, 3, 5, 10]\n",
    "BATCH_SIZE  = 32\n",
    "# ──────────────────────────────────────────────────────────────────────────────\n",
    "\n",
    "# 1. Load CSV\n",
    "print(\"Loading dataset...\")\n",
    "df = pd.read_csv(CSV_PATH)\n",
    "print(f\"  {len(df)} query-passage pairs loaded.\")\n",
    "print(df.head())\n",
    "\n",
    "# 2. Deduplicate passages (build corpus)\n",
    "passages        = df['passage'].tolist()\n",
    "queries         = df['query'].tolist()\n",
    "\n",
    "unique_passages = list(dict.fromkeys(passages))\n",
    "passage_to_idx  = {p: i for i, p in enumerate(unique_passages)}\n",
    "ground_truth    = [passage_to_idx[p] for p in passages]\n",
    "\n",
    "print(f\"  Corpus size (unique passages): {len(unique_passages)}\")\n",
    "print(f\"  Number of queries            : {len(queries)}\")\n",
    "\n",
    "# 3. Apply E5 asymmetric prefixes\n",
    "# \"query: \" for questions, \"passage: \" for documents — this is the key\n",
    "# difference vs AraBERT SBERT\n",
    "queries_prefixed  = [\"query: \"   + q for q in queries]\n",
    "passages_prefixed = [\"passage: \" + p for p in unique_passages]\n",
    "\n",
    "# 4. Encode\n",
    "print(\"\\nLoading multilingual-e5-base model...\")\n",
    "model = SentenceTransformer(MODEL_NAME)\n",
    "\n",
    "print(\"Encoding queries...\")\n",
    "query_embs = model.encode(\n",
    "    queries_prefixed,\n",
    "    batch_size=BATCH_SIZE,\n",
    "    show_progress_bar=True,\n",
    "    normalize_embeddings=True,    # required for E5\n",
    "    convert_to_numpy=True\n",
    ")\n",
    "\n",
    "print(\"Encoding corpus passages...\")\n",
    "corpus_embs = model.encode(\n",
    "    passages_prefixed,\n",
    "    batch_size=BATCH_SIZE,\n",
    "    show_progress_bar=True,\n",
    "    normalize_embeddings=True,    # required for E5\n",
    "    convert_to_numpy=True\n",
    ")\n",
    "\n",
    "# 5. Retrieval — Hit Rate & MRR\n",
    "print(\"\\nRunning retrieval evaluation...\")\n",
    "\n",
    "hit_counts       = {k: 0 for k in TOP_K}\n",
    "reciprocal_ranks = []\n",
    "\n",
    "for i in range(len(queries)):\n",
    "    query_emb   = query_embs[i].reshape(1, -1)\n",
    "    sims        = cosine_similarity(query_emb, corpus_embs)[0]\n",
    "    ranked_idxs = np.argsort(sims)[::-1]\n",
    "\n",
    "    true_idx = ground_truth[i]\n",
    "    rank     = np.where(ranked_idxs == true_idx)[0][0] + 1  # 1-based\n",
    "\n",
    "    for k in TOP_K:\n",
    "        if rank <= k:\n",
    "            hit_counts[k] += 1\n",
    "\n",
    "    reciprocal_ranks.append(1.0 / rank)\n",
    "\n",
    "    if (i + 1) % 500 == 0:\n",
    "        print(f\"  Processed {i + 1}/{len(queries)} queries...\")\n",
    "\n",
    "# 6. Print results\n",
    "n = len(queries)\n",
    "print(\"\\n─── multilingual-e5-base — Asymmetric Retrieval Results ──\")\n",
    "print(f\"  Corpus size : {len(unique_passages)} passages\")\n",
    "print(f\"  Queries     : {n}\")\n",
    "print()\n",
    "for k in TOP_K:\n",
    "    hr = hit_counts[k] / n\n",
    "    print(f\"  Hit Rate @{k:2d} : {hr:.4f}  ({hit_counts[k]}/{n})\")\n",
    "\n",
    "mrr = np.mean(reciprocal_ranks)\n",
    "print(f\"  MRR         : {mrr:.4f}\")\n",
    "\n",
    "# 7. Save detailed results\n",
    "print(\"\\nSaving detailed results...\")\n",
    "ranks = []\n",
    "for i in range(len(queries)):\n",
    "    sims        = cosine_similarity(query_embs[i].reshape(1, -1), corpus_embs)[0]\n",
    "    ranked_idxs = np.argsort(sims)[::-1]\n",
    "    rank        = int(np.where(ranked_idxs == ground_truth[i])[0][0] + 1)\n",
    "    ranks.append(rank)\n",
    "\n",
    "df['rank']            = ranks\n",
    "df['reciprocal_rank'] = 1.0 / df['rank']\n",
    "\n",
    "output_path = CSV_PATH.replace('.csv', '_e5_retrieval_results.csv')\n",
    "df.to_csv(output_path, index=False)\n",
    "print(f\"Detailed results saved to: {output_path}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "17cafbbc-d4b6-4e8b-84d5-d0c306bda1c4",
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.set_option('display.max_colwidth', None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "5a283221-d7bf-42ab-b9c3-4469b563b114",
   "metadata": {},
   "outputs": [],
   "source": [
    "df= pd.read_csv('/home/skiredj.abderrahman/khalil/sbert_training/third_training/benchmark/ArabicaQA_test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "da37d167-1068-4634-85c1-16249bf19252",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>query</th>\n",
       "      <th>passage</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>من الذي يترأس المرصد الحضري لمدينة الرياض؟</td>\n",
       "      <td>يترأس مجلس المرصد أمير المنطقة بالإضافة إلى أربعة عشر عضوا،</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ما هي الإنجازات البارزة التي حققها المرصد الحضري بمدينة الرياض؟</td>\n",
       "      <td>في مارس 2017 حصل المرصد الحضري بمدينة الرياض على الشهادة الذهبية WCCD ISO 37120 من المجلس العالمي لبيانات المدن في كندا (WCCD)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ما هو المرصد الحضري لمدينة الرياض؟</td>\n",
       "      <td>هو مركز متخصص في وضع نظم مراقبة حضرية لرصد سير عمليات التنمية الحضرية للمدينة،</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ما هو الهدف من المرصد الحضري لمدينة الرياض؟</td>\n",
       "      <td>يهدف إلى فهم الوضع الحالي للمدينة ووضع تصور للاتجاهات المستقبلية،</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ما هي مهام المرصد الحضري لمدينة الرياض؟</td>\n",
       "      <td>ومراقبة الإنجاز لتحقيق الأهداف واتخاذ القرارات.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13965</th>\n",
       "      <td>ما هو الموقف الفلسفي؟\\n</td>\n",
       "      <td>هي وجهة نظر تحاول شرح أو تفسير مشكلة معينة في الفلسفة.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13966</th>\n",
       "      <td>ما هو مصطلح \"نظرية\"؟\\n</td>\n",
       "      <td>هو بيان للغة الإنجليزية العامية وليس انعكاسًا لمصطلح نظرية</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13967</th>\n",
       "      <td>ما هو بنك دبي؟</td>\n",
       "      <td>مصرف إسلامي مقره في دبي، الإمارات العربية المتحدة</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13968</th>\n",
       "      <td>متي تأسس بنك دبي؟</td>\n",
       "      <td>سبتمبر 2002</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13969</th>\n",
       "      <td>متي أستحوز عليه استحوذ عليه بنك الإمارات دبي الوطني.\\n؟</td>\n",
       "      <td>1 ديسمبر 2012</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>13970 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                 query  \\\n",
       "0                           من الذي يترأس المرصد الحضري لمدينة الرياض؟   \n",
       "1      ما هي الإنجازات البارزة التي حققها المرصد الحضري بمدينة الرياض؟   \n",
       "2                                   ما هو المرصد الحضري لمدينة الرياض؟   \n",
       "3                          ما هو الهدف من المرصد الحضري لمدينة الرياض؟   \n",
       "4                              ما هي مهام المرصد الحضري لمدينة الرياض؟   \n",
       "...                                                                ...   \n",
       "13965                                          ما هو الموقف الفلسفي؟\\n   \n",
       "13966                                           ما هو مصطلح \"نظرية\"؟\\n   \n",
       "13967                                                   ما هو بنك دبي؟   \n",
       "13968                                                متي تأسس بنك دبي؟   \n",
       "13969          متي أستحوز عليه استحوذ عليه بنك الإمارات دبي الوطني.\\n؟   \n",
       "\n",
       "                                                                                                                               passage  \n",
       "0                                                                          يترأس مجلس المرصد أمير المنطقة بالإضافة إلى أربعة عشر عضوا،  \n",
       "1       في مارس 2017 حصل المرصد الحضري بمدينة الرياض على الشهادة الذهبية WCCD ISO 37120 من المجلس العالمي لبيانات المدن في كندا (WCCD)  \n",
       "2                                                       هو مركز متخصص في وضع نظم مراقبة حضرية لرصد سير عمليات التنمية الحضرية للمدينة،  \n",
       "3                                                                    يهدف إلى فهم الوضع الحالي للمدينة ووضع تصور للاتجاهات المستقبلية،  \n",
       "4                                                                                      ومراقبة الإنجاز لتحقيق الأهداف واتخاذ القرارات.  \n",
       "...                                                                                                                                ...  \n",
       "13965                                                                           هي وجهة نظر تحاول شرح أو تفسير مشكلة معينة في الفلسفة.  \n",
       "13966                                                                       هو بيان للغة الإنجليزية العامية وليس انعكاسًا لمصطلح نظرية  \n",
       "13967                                                                                مصرف إسلامي مقره في دبي، الإمارات العربية المتحدة  \n",
       "13968                                                                                                                      سبتمبر 2002  \n",
       "13969                                                                                                                   1 ديسمبر 2012   \n",
       "\n",
       "[13970 rows x 2 columns]"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9dd51f18-3d63-4f3c-9bcc-13457d50ee4b",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.19"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}