{ "cells": [ { "cell_type": "markdown", "id": "049c28f8-b525-41c2-9a0e-07b67774fc67", "metadata": { "jp-MarkdownHeadingCollapsed": true }, "source": [ "# chandra 2 avec split " ] }, { "cell_type": "code", "execution_count": null, "id": "e02915f2-5724-4aee-8813-c037c31fe1ba", "metadata": {}, "outputs": [], "source": [ "# ============================================================================\n", "# PIPELINE OCR - IMAGES LOCALES (sans prétraitement)\n", "# Entrée : dossier avec sous-dossiers contenant des PNG (déjà zoom x4)\n", "# Sortie : un fichier .md par sous-dossier\n", "# ============================================================================\n", "\n", "import os\n", "import re\n", "import time\n", "import base64\n", "import requests\n", "import glob\n", "import shutil\n", "import cv2\n", "import numpy as np\n", "from typing import Tuple, Union, Dict, Optional\n", "from concurrent.futures import ThreadPoolExecutor, as_completed\n", "from tqdm import tqdm\n", "\n", "# ========== CONFIGURATION ==========\n", "\n", "VLLM_API_URL = \"http://localhost:9996/v1/chat/completions\"\n", "MODEL_PATH = \"/home/skiredj.abderrahman/lina/chandra-ocr-2\"\n", "\n", "GENERATION_PARAMS = {\n", " \"temperature\": 0.1,\n", " \"max_tokens\": 30000,\n", " \"top_p\": 0.9,\n", "}\n", "\n", "REQUEST_TIMEOUT_SECONDS = 180\n", "NUM_WORKERS = 20\n", "\n", "OCR_PROMPT = \"\"\"Extract all text exactly as it appears in this image.\n", "Preserve the original layout, reading order, formatting, and structure.\n", "Keep all tables as proper Markdown tables.\n", "Maintain all mathematical formulas, equations, and special characters.\n", "Do not rewrite, summarize, or modify any content.\n", "If any text is unclear or unreadable, write [UNCLEAR].\n", "Output only the extracted text without any preamble or explanation.\"\"\"\n", "\n", "# ── Dossiers ──────────────────────────────────────────────────────────────────\n", "# Dossier racine contenant les sous-dossiers d'images\n", "IMAGES_DATASET_FOLDER = r\"/home/skiredj.abderrahman/khalil/Benchmarking_OCR/ground_truth_claude_100_images/100images\"\n", "\n", "# Dossier de sortie pour les .md et les images splittées\n", "OUTPUT_FOLDER = r\"/home/skiredj.abderrahman/khalil/Benchmarking_OCR/final_resulsts_benchmark/chandra2_with_split/output_markdown\"\n", "SPLIT_BASE_FOLDER =r\"/home/skiredj.abderrahman/khalil/Benchmarking_OCR/final_resulsts_benchmark/chandra2_with_split/split_images\"\n", "TXT_TEMP_FOLDER = r\"/home/skiredj.abderrahman/khalil/Benchmarking_OCR/final_resulsts_benchmark/chandra2_with_split/temp_txt\"\n", "\n", "\n", "# ========== LECTURE D'IMAGE SANS PRÉTRAITEMENT ==========\n", "\n", "def _read_image_raw(image_or_path: Union[str, np.ndarray]) -> np.ndarray:\n", " \"\"\"\n", " Lit une image telle quelle, SANS aucun prétraitement.\n", " Les images sont déjà en zoom x4 et prêtes pour l'OCR.\n", " \"\"\"\n", " if isinstance(image_or_path, str):\n", " img = cv2.imread(image_or_path, cv2.IMREAD_COLOR)\n", " if img is None:\n", " raise ValueError(f\"Impossible de lire l'image : {image_or_path}\")\n", " elif isinstance(image_or_path, np.ndarray):\n", " img = image_or_path.copy()\n", " else:\n", " raise TypeError(\"image_or_path doit être un chemin (str) ou un tableau numpy.\")\n", " return img\n", "\n", "\n", "# ========== DÉTECTION DE TABLEAU ==========\n", "\n", "def is_big_table(img, debug=False, base_name=\"\"):\n", " H, W = img.shape[:2]\n", " gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)\n", " _, th = cv2.threshold(gray, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)\n", " bin_img = th if np.mean(th) > 127 else cv2.bitwise_not(th)\n", " bin_img = cv2.medianBlur(bin_img, 3)\n", "\n", " # Lignes verticales\n", " vert_len = max(10, H // 20)\n", " vert_kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (1, vert_len))\n", " vertical = cv2.morphologyEx(255 - bin_img, cv2.MORPH_OPEN, vert_kernel, iterations=1)\n", " vertical = cv2.morphologyEx(vertical, cv2.MORPH_CLOSE, vert_kernel, iterations=1)\n", " _, vertical = cv2.threshold(vertical, 0, 255, cv2.THRESH_BINARY)\n", "\n", " tall_verticals = []\n", " total_v_width = 0\n", " contours_vert, _ = cv2.findContours(vertical, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)\n", " for c in contours_vert:\n", " x, y, w, h = cv2.boundingRect(c)\n", " if h >= int(0.85 * H) and (1 <= w <= int(0.03 * W)):\n", " tall_verticals.append((x, y, w, h))\n", " total_v_width += w\n", "\n", " spread_ok = False\n", " if len(tall_verticals) >= 5:\n", " xs = np.array([x + w / 2.0 for x, _, w, _ in tall_verticals], dtype=float)\n", " if np.ptp(xs) >= 0.5 * W:\n", " spread_ok = True\n", "\n", " width_coverage = total_v_width / float(W) if W > 0 else 0.0\n", " looks_like_big_table = (len(tall_verticals) >= 5) and spread_ok and (width_coverage >= 0.06)\n", "\n", " # Lignes horizontales\n", " horiz_len = max(10, W // 20)\n", " horiz_kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (horiz_len, 1))\n", " horizontal = cv2.morphologyEx(255 - bin_img, cv2.MORPH_OPEN, horiz_kernel, iterations=1)\n", " horizontal = cv2.morphologyEx(horizontal, cv2.MORPH_CLOSE, horiz_kernel, iterations=1)\n", " _, horizontal = cv2.threshold(horizontal, 0, 255, cv2.THRESH_BINARY)\n", " contours_horiz, _ = cv2.findContours(horizontal, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)\n", "\n", " long_h = [c for c in contours_horiz if cv2.boundingRect(c)[2] / W > 0.5]\n", " medium_h = [c for c in contours_horiz if 0.2 < cv2.boundingRect(c)[2] / W <= 0.5]\n", " many_horizontal = len(long_h)\n", "\n", " line_density = many_horizontal / (H / 100.0)\n", " if long_h:\n", " ys = [cv2.boundingRect(c)[1] + cv2.boundingRect(c)[3] / 2 for c in long_h]\n", " v_spread = (max(ys) - min(ys)) / H if len(ys) > 1 else 0\n", " else:\n", " v_spread = 0\n", "\n", " spacing_regularity = 0\n", " if many_horizontal >= 3:\n", " ys_sorted = sorted([cv2.boundingRect(c)[1] + cv2.boundingRect(c)[3] / 2 for c in long_h])\n", " spacings = [ys_sorted[i+1] - ys_sorted[i] for i in range(len(ys_sorted)-1)]\n", " if spacings:\n", " avg = np.mean(spacings)\n", " spacing_regularity = 1.0 / (1.0 + np.var(spacings) / (avg + 1))\n", "\n", " text_pixels = np.sum(255 - bin_img > 127)\n", " line_pixels = np.sum(horizontal > 127) + np.sum(vertical > 127)\n", " line_to_text_ratio = line_pixels / (text_pixels + 1)\n", "\n", " return (\n", " looks_like_big_table\n", " or many_horizontal >= 4\n", " or (many_horizontal >= 5 and line_density >= 2.0 and v_spread >= 0.6)\n", " or (many_horizontal >= 4 and spacing_regularity >= 0.7 and v_spread >= 0.5)\n", " or (many_horizontal >= 5 and line_to_text_ratio >= 0.25\n", " and (len(medium_h) + many_horizontal) >= 12 and v_spread >= 0.4)\n", " )\n", "\n", "\n", "# ========== SPLITTING DE COLONNES ==========\n", "\n", "def has_significant_text(img_region, min_text_ratio=0.01):\n", " gray = cv2.cvtColor(img_region, cv2.COLOR_BGR2GRAY) if len(img_region.shape) == 3 else img_region\n", " _, binary = cv2.threshold(gray, 0, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU)\n", " return (np.sum(binary > 0) / binary.size) > min_text_ratio\n", "\n", "\n", "def find_text_density_split(img, debug=False):\n", " gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) if len(img.shape) == 3 else img\n", " H, W = gray.shape\n", " _, binary = cv2.threshold(gray, 0, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU)\n", "\n", " # Vérification grille (tableaux complexes)\n", " h_kern = cv2.getStructuringElement(cv2.MORPH_RECT, (max(40, W // 15), 1))\n", " h_lines = cv2.morphologyEx(binary, cv2.MORPH_OPEN, h_kern)\n", " v_kern = cv2.getStructuringElement(cv2.MORPH_RECT, (1, max(30, H // 20)))\n", " v_lines = cv2.morphologyEx(binary, cv2.MORPH_OPEN, v_kern)\n", "\n", " nh = sum(1 for c in cv2.findContours(h_lines, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)[0]\n", " if cv2.boundingRect(c)[2] > 0.4 * W)\n", " nv = sum(1 for c in cv2.findContours(v_lines, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)[0]\n", " if cv2.boundingRect(c)[3] > 0.3 * H)\n", "\n", " if nh >= 2 and nv >= 2:\n", " grid = cv2.bitwise_and(h_lines, v_lines)\n", " intersection_contours, _ = cv2.findContours(grid, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)\n", " intersections = len(intersection_contours)\n", " if intersections > 0:\n", " ipa = intersections / ((W * H) / 10000)\n", "\n", " # Calcul du regularity_score (régularité de la grille) — identique pipeline de base\n", " intersection_points = []\n", " for c in intersection_contours:\n", " bx, by, bw, bh = cv2.boundingRect(c)\n", " intersection_points.append((bx + bw // 2, by + bh // 2))\n", "\n", " if len(intersection_points) >= 4:\n", " distances = [\n", " ((x2-x1)**2 + (y2-y1)**2) ** 0.5\n", " for i, (x1, y1) in enumerate(intersection_points)\n", " for (x2, y2) in intersection_points[i+1:]\n", " ]\n", " if distances:\n", " dist_mean = np.mean(distances)\n", " regularity_score = np.var(distances) / (dist_mean ** 2) if dist_mean > 0 else 0\n", " else:\n", " regularity_score = 0\n", " else:\n", " regularity_score = 0\n", "\n", " if intersections >= 20 and regularity_score < 0.1 and ipa > 0.3:\n", " return None\n", " if nh >= 6 and intersections >= 15 and ipa > 0.2:\n", " return None\n", " if nh >= 4 and nv >= 4 and intersections >= 25:\n", " return None\n", "\n", " vert_proj = np.sum(binary, axis=0) / 255\n", " win = max(1, int(W * 0.02))\n", " maxpv = vert_proj.max() if vert_proj.size else 0.0\n", " candidates = [\n", " (x, np.mean(vert_proj[max(0, x - win//2):min(W, x + win//2)]))\n", " for x in range(win, W - win)\n", " if np.mean(vert_proj[max(0, x - win//2):min(W, x + win//2)]) < maxpv * 0.1\n", " and int(0.25 * W) <= x <= int(0.75 * W)\n", " ]\n", " if not candidates:\n", " return None\n", "\n", " best = min(candidates, key=lambda p: abs(p[0] - W // 2))[0]\n", "\n", " def text_ratio(region):\n", " g = cv2.cvtColor(region, cv2.COLOR_BGR2GRAY) if len(region.shape) == 3 else region\n", " _, b = cv2.threshold(g, 0, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU)\n", " return (np.sum(b > 0) / float(b.size)) if b.size else 0.0\n", "\n", " lr, rr = text_ratio(img[:, :best]), text_ratio(img[:, best:])\n", " if lr < 0.01 or rr < 0.01 or min(lr, rr) < 0.5 * max(lr, rr):\n", " return None\n", " return best\n", "\n", "\n", "def has_unique_central_vertical_rule(image_or_path, center_band_ratio=0.20,\n", " min_height_ratio=0.70, max_line_width_ratio=0.03,\n", " min_line_width_px=1, max_gap_ratio=0.10,\n", " max_gap_runs=2, gap_run_ratio=0.01,\n", " max_candidates_allowed=3, debug=False, base_name=\"\"):\n", " bgr_orig = _read_image_raw(image_or_path)\n", " orig_H, orig_W = bgr_orig.shape[:2]\n", " scale_factor = 1.0\n", " bgr = bgr_orig\n", " max_side = max(orig_H, orig_W)\n", " if max_side > 2000:\n", " scale_factor = 2000.0 / max_side\n", " bgr = cv2.resize(bgr_orig, (int(orig_W * scale_factor), int(orig_H * scale_factor)),\n", " interpolation=cv2.INTER_AREA)\n", " H, W = bgr.shape[:2]\n", "\n", " gray = cv2.cvtColor(bgr, cv2.COLOR_BGR2GRAY)\n", " clahe = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8, 8))\n", " gray = clahe.apply(gray)\n", " blur = cv2.GaussianBlur(gray, (3, 3), 0)\n", " _, th_otsu = cv2.threshold(blur, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)\n", " bin_img = th_otsu if np.mean(th_otsu) > 127 else cv2.bitwise_not(th_otsu)\n", " bin_img = cv2.medianBlur(bin_img, 3)\n", "\n", " vert_len = max(10, H // 20)\n", " vert_kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (1, vert_len))\n", " vertical = cv2.morphologyEx(255 - bin_img, cv2.MORPH_OPEN, vert_kernel, iterations=1)\n", " vertical = cv2.morphologyEx(vertical, cv2.MORPH_CLOSE, vert_kernel, iterations=1)\n", " _, vertical = cv2.threshold(vertical, 0, 255, cv2.THRESH_BINARY)\n", "\n", " # Heuristique grand tableau\n", " tall_verticals, total_v_width = [], 0\n", " contours_all, _ = cv2.findContours(vertical, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)\n", " for c in contours_all:\n", " x, y, w, h = cv2.boundingRect(c)\n", " if h >= int(0.85 * H) and (min_line_width_px <= w <= int(max_line_width_ratio * W)):\n", " tall_verticals.append((x, y, w, h))\n", " total_v_width += w\n", " spread_ok = False\n", " if len(tall_verticals) >= 5:\n", " xs = np.array([x + w / 2.0 for x, _, w, _ in tall_verticals], dtype=float)\n", " spread_ok = np.ptp(xs) >= 0.50 * W\n", " width_coverage = total_v_width / float(W) if W > 0 else 0.0\n", " looks_like_big_table = len(tall_verticals) >= 5 and spread_ok and width_coverage >= 0.06\n", "\n", " mid, band_half = W // 2, int(center_band_ratio * W)\n", " x0, x1 = max(0, mid - band_half), min(W, mid + band_half)\n", " central_band = np.zeros_like(vertical)\n", " central_band[:, x0:x1] = vertical[:, x0:x1]\n", " contours, _ = cv2.findContours(central_band, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)\n", "\n", " candidates = []\n", " for cnt in contours:\n", " x, y, w, h = cv2.boundingRect(cnt)\n", " if h < int(min_height_ratio * H): continue\n", " if w < min_line_width_px or w > int(max_line_width_ratio * W): continue\n", " if not (mid - band_half <= x + w / 2.0 <= mid + band_half): continue\n", " roi = (255 - bin_img)[y:y+h, max(0, x):min(W, x+w)]\n", " row_has_ink = (np.max(roi, axis=1) > 0).astype(np.uint8)\n", " min_gap = max(3, int(gap_run_ratio * H))\n", " gaps, run = [], 0\n", " for val in row_has_ink:\n", " if val == 0: run += 1\n", " else:\n", " if run >= min_gap: gaps.append(run)\n", " run = 0\n", " if run >= min_gap: gaps.append(run)\n", " if sum(gaps) > max_gap_ratio * H or len(gaps) > max_gap_runs: continue\n", " candidates.append((x, y, w, h, {\"gaps\": gaps, \"total_gap\": sum(gaps)}))\n", "\n", " central_tall = sum(\n", " 1 for cnt in contours\n", " if cv2.boundingRect(cnt)[3] >= int(min_height_ratio * H)\n", " and min_line_width_px <= cv2.boundingRect(cnt)[2] <= int(max_line_width_ratio * W)\n", " )\n", " global_like = sum(\n", " 1 for c in contours_all\n", " if cv2.boundingRect(c)[3] >= int(min_height_ratio * H)\n", " and min_line_width_px <= cv2.boundingRect(c)[2] <= int(max_line_width_ratio * W)\n", " )\n", "\n", " details = {\n", " \"original_image_size\": (orig_H, orig_W), \"processed_image_size\": (H, W),\n", " \"scale_factor\": scale_factor, \"center_band\": (x0, x1),\n", " \"candidates_found\": len(candidates), \"global_vertical_candidates\": global_like,\n", " \"candidates\": [{\"bbox\": (int(x), int(y), int(w), int(h)), **stats}\n", " for (x, y, w, h, stats) in candidates],\n", " \"big_table_heuristic\": {\"looks_like_big_table\": bool(looks_like_big_table),\n", " \"very_tall_verticals\": len(tall_verticals),\n", " \"spread_ok\": bool(spread_ok), \"width_coverage\": float(width_coverage)}\n", " }\n", "\n", " if central_tall > 1 or global_like > max_candidates_allowed or looks_like_big_table:\n", " return (False, details)\n", " return (len(candidates) == 1, details)\n", "\n", "\n", "def get_split_x_from_details(details):\n", " if details.get(\"candidates_found\", 0) != 1:\n", " return None\n", " x, y, w, h = details[\"candidates\"][0][\"bbox\"]\n", " scale = float(details.get(\"scale_factor\", 1.0)) or 1.0\n", " split_x = int(round((x + w / 2) / scale))\n", " orig_W = details.get(\"original_image_size\", (0, 0))[1]\n", " if orig_W:\n", " split_x = max(1, min(orig_W - 1, split_x))\n", " return split_x\n", "\n", "\n", "def _derive_names(image_or_path, base_name=None):\n", " if isinstance(image_or_path, str):\n", " stem, ext = os.path.splitext(os.path.basename(image_or_path))\n", " return stem, ext or \".png\"\n", " stem, ext = os.path.splitext(base_name or \"\")\n", " return stem, ext or \".png\"\n", "\n", "\n", "def perform_split(img, split_x, dest_dir, stem, ext, left_suffix, right_suffix, details):\n", " os.makedirs(dest_dir, exist_ok=True)\n", " left_path = os.path.join(dest_dir, f\"{stem}{left_suffix}{ext}\")\n", " right_path = os.path.join(dest_dir, f\"{stem}{right_suffix}{ext}\")\n", " ok_l = cv2.imwrite(left_path, img[:, :split_x])\n", " ok_r = cv2.imwrite(right_path, img[:, split_x:])\n", " if ok_l and ok_r:\n", " return {\"split\": True, \"left_path\": left_path, \"right_path\": right_path,\n", " \"split_x\": split_x, \"details\": details}\n", " out = os.path.join(dest_dir, f\"{stem}{ext}\")\n", " cv2.imwrite(out, img)\n", " return {\"split\": False, \"reason\": \"write_error\", \"copied_path\": out}\n", "\n", "\n", "def split_if_two_column(image_or_path, dest_dir, base_name=None, debug=False):\n", " \"\"\"\n", " Détecte si l'image est en deux colonnes et la découpe le cas échéant.\n", " Aucun prétraitement n'est appliqué à l'image.\n", " \"\"\"\n", " img = _read_image_raw(image_or_path)\n", " stem, ext = _derive_names(image_or_path, base_name)\n", " os.makedirs(dest_dir, exist_ok=True)\n", "\n", " # Étape 1 : grand tableau → pas de split\n", " if is_big_table(img, debug=debug, base_name=stem):\n", " if debug: print(f\"[{stem}] Grand tableau détecté → pas de split.\")\n", " out = os.path.join(dest_dir, f\"{stem}{ext}\")\n", " if isinstance(image_or_path, str) and os.path.isfile(image_or_path):\n", " shutil.copy2(image_or_path, out)\n", " else:\n", " cv2.imwrite(out, img)\n", " return {\"split\": False, \"reason\": \"big_table_detected\", \"copied_path\": out}\n", "\n", " orig_H, orig_W = img.shape[:2]\n", "\n", " # Étape 2 : trait vertical central\n", " is_two_col, details = has_unique_central_vertical_rule(\n", " image_or_path, debug=debug, base_name=stem)\n", " if is_two_col:\n", " split_x = get_split_x_from_details(details)\n", " if split_x and has_significant_text(img[:, :split_x]) and has_significant_text(img[:, split_x:]):\n", " return perform_split(img, split_x, dest_dir, stem, ext, \"_left\", \"_right\", details)\n", "\n", " # Étape 3 : densité de texte\n", " alt_x = find_text_density_split(img, debug=debug)\n", " if alt_x and has_significant_text(img[:, :alt_x]) and has_significant_text(img[:, alt_x:]):\n", " return perform_split(img, alt_x, dest_dir, stem, ext, \"_left\", \"_right\",\n", " {\"method\": \"text_density\", \"split_x\": alt_x})\n", "\n", " # Pas de split : copie telle quelle\n", " out = os.path.join(dest_dir, f\"{stem}{ext}\")\n", " if isinstance(image_or_path, str) and os.path.isfile(image_or_path):\n", " shutil.copy2(image_or_path, out)\n", " else:\n", " cv2.imwrite(out, img)\n", " return {\"split\": False, \"reason\": \"no_valid_split\", \"copied_path\": out}\n", "\n", "\n", "# ========== OCR ==========\n", "\n", "def call_vllm_ocr(image_path, retries=3):\n", " for attempt in range(1, retries + 1):\n", " try:\n", " with open(image_path, \"rb\") as f:\n", " img_b64 = base64.b64encode(f.read()).decode(\"utf-8\")\n", " payload = {\n", " \"model\": MODEL_PATH,\n", " \"messages\": [{\"role\": \"user\", \"content\": [\n", " {\"type\": \"image_url\", \"image_url\": {\"url\": f\"data:image/png;base64,{img_b64}\"}},\n", " {\"type\": \"text\", \"text\": OCR_PROMPT}\n", " ]}],\n", " **GENERATION_PARAMS,\n", " }\n", " resp = requests.post(VLLM_API_URL, json=payload, timeout=REQUEST_TIMEOUT_SECONDS)\n", " resp.raise_for_status()\n", " data = resp.json()\n", " if \"choices\" in data and data[\"choices\"]:\n", " return data[\"choices\"][0][\"message\"][\"content\"]\n", " raise Exception(f\"Format de réponse inattendu : {data}\")\n", " except requests.exceptions.Timeout:\n", " print(f\" Timeout tentative {attempt}/{retries} ({image_path})\")\n", " if attempt < retries: time.sleep(5)\n", " except Exception as e:\n", " print(f\" Erreur tentative {attempt}/{retries} ({image_path}): {e}\")\n", " if attempt < retries: time.sleep(3)\n", " raise Exception(f\"OCR échoué après {retries} tentatives : {image_path}\")\n", "\n", "\n", "def process_image(input_path, output_txt_path):\n", " if os.path.exists(output_txt_path):\n", " return # déjà traité\n", " try:\n", " text = call_vllm_ocr(input_path)\n", " with open(output_txt_path, \"w\", encoding=\"utf-8\") as f:\n", " f.write(text)\n", " except Exception as e:\n", " with open(output_txt_path, \"w\", encoding=\"utf-8\") as f:\n", " f.write(f\"[OCR FAILED: {e}]\")\n", "\n", "\n", "def clean_unnecessary_linebreaks(text):\n", " lines = text.split(\"\\n\")\n", " out = []\n", " for i, line in enumerate(lines):\n", " if line.startswith(\"=== Page\"):\n", " out.append(line)\n", " if i + 1 < len(lines) and lines[i + 1].strip() == \"\":\n", " out.append(\"\")\n", " else:\n", " prev_is_page_header = i > 0 and lines[i - 1].startswith(\"=== Page\")\n", " prev_ends_sentence = i > 0 and lines[i - 1].rstrip().endswith(\".\")\n", " if not prev_is_page_header and not prev_ends_sentence and out and out[-1] != \"\":\n", " out[-1] = out[-1].rstrip() + \" \" + line.lstrip()\n", " else:\n", " out.append(line)\n", " return \"\\n\".join(out)\n", "\n", "\n", "# ========== TRI DES IMAGES (arabe : right avant left) ==========\n", "\n", "def arabic_sort_key(path):\n", " name = os.path.splitext(os.path.basename(path))[0]\n", " if name.endswith(\"_right\"):\n", " base, order = name[:-6], 0 # right = 0 → passe en premier\n", " elif name.endswith(\"_left\"):\n", " base, order = name[:-5], 1 # left = 1 → passe en second\n", " else:\n", " base, order = name, 0\n", " # Clé primaire = nom complet du document (tri alphabétique)\n", " # Clé secondaire = right avant left pour la même page\n", " return (base, order)\n", "\n", "\n", "# ========== PIPELINE PRINCIPAL ==========\n", "\n", "def process_subfolder(subfolder_path, subfolder_name):\n", " \"\"\"\n", " Traite un sous-dossier : layout detection → split → OCR → .md\n", " \"\"\"\n", " print(f\"\\n{'='*60}\")\n", " print(f\"Traitement : {subfolder_name}\")\n", " print(f\"{'='*60}\")\n", "\n", " # Dossiers de travail pour ce sous-dossier\n", " split_dir = os.path.join(SPLIT_BASE_FOLDER, subfolder_name)\n", " txt_dir = os.path.join(TXT_TEMP_FOLDER, subfolder_name)\n", " output_md = os.path.join(OUTPUT_FOLDER, subfolder_name + \".md\")\n", " os.makedirs(split_dir, exist_ok=True)\n", " os.makedirs(txt_dir, exist_ok=True)\n", " os.makedirs(OUTPUT_FOLDER, exist_ok=True)\n", "\n", " if os.path.exists(output_md):\n", " print(f\" → Déjà traité, on passe.\")\n", " return\n", "\n", " # ── Étape 1 : Détection de layout + split ─────────────────────────────────\n", " png_files = sorted(glob.glob(os.path.join(subfolder_path, \"*.png\")))\n", " if not png_files:\n", " print(f\" ⚠️ Aucune image PNG trouvée dans {subfolder_path}\")\n", " return\n", "\n", " print(f\" {len(png_files)} image(s) trouvée(s). Détection de layout...\")\n", " for img_path in tqdm(png_files, desc=\" Layout detection\", leave=False):\n", " split_if_two_column(img_path, split_dir, debug=False)\n", "\n", " # ── Étape 2 : Tri et OCR en parallèle ─────────────────────────────────────\n", " split_images = sorted(glob.glob(os.path.join(split_dir, \"*.png\")), key=arabic_sort_key)\n", " if not split_images:\n", " print(f\" ⚠️ Aucune image après split dans {split_dir}\")\n", " return\n", "\n", " tasks = [\n", " (img_p, os.path.join(txt_dir, os.path.splitext(os.path.basename(img_p))[0] + \".txt\"))\n", " for img_p in split_images\n", " ]\n", "\n", " print(f\" {len(split_images)} image(s) à envoyer à Chandra-2...\")\n", " with ThreadPoolExecutor(max_workers=NUM_WORKERS) as executor:\n", " futures = {executor.submit(process_image, ip, op): (ip, op) for ip, op in tasks}\n", " for future in tqdm(as_completed(futures), total=len(futures),\n", " desc=\" OCR\", leave=False):\n", " try:\n", " future.result()\n", " except Exception as e:\n", " print(f\" ❌ Erreur non gérée : {e}\")\n", "\n", " # ── Étape 3 : Assemblage en Markdown ──────────────────────────────────────\n", " parts = []\n", " for i, img_p in enumerate(split_images, 1):\n", " img_name = os.path.splitext(os.path.basename(img_p))[0]\n", " txt_path = os.path.join(txt_dir, img_name + \".txt\")\n", " if os.path.exists(txt_path):\n", " content = open(txt_path, encoding=\"utf-8\").read()\n", " else:\n", " content = \"[OCR failed for this page]\"\n", " parts.append(f\"=== Page {i} ({img_name}) ===\\n{content}\\n\")\n", "\n", " final = clean_unnecessary_linebreaks(\"\\n\".join(parts))\n", " with open(output_md, \"w\", encoding=\"utf-8\") as f:\n", " f.write(final)\n", "\n", " print(f\" ✅ Markdown sauvegardé → {output_md}\")\n", "\n", "\n", "def run_pipeline():\n", " \"\"\"\n", " Point d'entrée principal.\n", " Parcourt tous les sous-dossiers de IMAGES_DATASET_FOLDER,\n", " affiche un récapitulatif complet, puis lance le traitement.\n", " \"\"\"\n", " subfolders = sorted([\n", " d for d in os.listdir(IMAGES_DATASET_FOLDER)\n", " if os.path.isdir(os.path.join(IMAGES_DATASET_FOLDER, d))\n", " ])\n", "\n", " if not subfolders:\n", " print(f\"Aucun sous-dossier trouvé dans {IMAGES_DATASET_FOLDER}\")\n", " return\n", "\n", " # ── Scan préalable : compter toutes les images PNG ────────────────────────\n", " print(f\"\\n{'='*60}\")\n", " print(f\" SCAN DU DATASET\")\n", " print(f\"{'='*60}\")\n", " print(f\" Dossier racine : {IMAGES_DATASET_FOLDER}\\n\")\n", "\n", " folder_image_counts = {} # { nom_sous_dossier : [liste de chemins PNG] }\n", " max_name_len = max(len(d) for d in subfolders)\n", "\n", " for name in subfolders:\n", " path = os.path.join(IMAGES_DATASET_FOLDER, name)\n", " pngs = sorted(glob.glob(os.path.join(path, \"*.png\")))\n", " folder_image_counts[name] = pngs\n", " status = f\"{len(pngs):>4} image(s)\" if pngs else \" ── aucune image PNG\"\n", " print(f\" 📁 {name:<{max_name_len}} → {status}\")\n", "\n", " total_images = sum(len(v) for v in folder_image_counts.values())\n", " total_folders = len(subfolders)\n", " empty_folders = sum(1 for v in folder_image_counts.values() if not v)\n", " active_folders = total_folders - empty_folders\n", "\n", " print(f\"\\n{'─'*60}\")\n", " print(f\" Sous-dossiers trouvés : {total_folders}\")\n", " print(f\" Sous-dossiers avec PNG : {active_folders}\")\n", " print(f\" Sous-dossiers vides : {empty_folders}\")\n", " print(f\" Total images PNG : {total_images}\")\n", " print(f\"{'─'*60}\")\n", " print(f\" Sortie markdown : {OUTPUT_FOLDER}\")\n", " print(f\"{'='*60}\\n\")\n", "\n", " if total_images == 0:\n", " print(\"Aucune image à traiter. Arrêt.\")\n", " return\n", "\n", " # ── Traitement ────────────────────────────────────────────────────────────\n", " for name in tqdm(subfolders, desc=\"Sous-dossiers\"):\n", " if not folder_image_counts[name]:\n", " continue # sous-dossier vide, on saute\n", " path = os.path.join(IMAGES_DATASET_FOLDER, name)\n", " try:\n", " process_subfolder(path, name)\n", " except Exception as e:\n", " print(f\"❌ Erreur sur {name} : {e}\")\n", "\n", " print(f\"\\n{'='*60}\")\n", " print(\"PIPELINE TERMINÉ\")\n", " print(f\" {active_folders} sous-dossier(s) traité(s), {total_images} image(s) au total\")\n", " print(f\" Résultats dans : {OUTPUT_FOLDER}\")\n", " print(f\"{'='*60}\")\n", "\n", "\n", "# ========== LANCEMENT ==========\n", "\n", "if __name__ == \"__main__\":\n", " run_pipeline()\n", "\n" ] }, { "cell_type": "markdown", "id": "6ee1befe-ad31-46b1-8fb3-6fec1c6a7ca8", "metadata": {}, "source": [ "## from html to md " ] }, { "cell_type": "code", "execution_count": null, "id": "56186fc4-33dd-4d76-8c08-f30ef49c42d8", "metadata": {}, "outputs": [], "source": [ "# exectued tomd.pbs\n", "from pathlib import Path\n", "from markdownify import markdownify as md\n", "import re\n", "import time\n", "\n", "def convert_markdown_file(file_path, output_folder=None):\n", " \"\"\"Convert HTML in markdown file to proper markdown.\"\"\"\n", " print(f\"Processing: {file_path.name}\")\n", "\n", " # Read the file\n", " content = file_path.read_text(encoding='utf-8')\n", "\n", " # Split by page sections\n", " page_pattern = r'(=== Page \\d+ \\([^)]+\\) ===)'\n", " sections = re.split(page_pattern, content)\n", "\n", " # Convert HTML sections to markdown\n", " converted = []\n", " for section in sections:\n", " if section.startswith('=== Page'):\n", " converted.append(section)\n", " elif section.strip():\n", " # Convert HTML to markdown\n", " markdown = md(section, heading_style=\"ATX\")\n", " converted.append('\\n' + markdown.strip() + '\\n')\n", "\n", " # Save the result\n", " output_path = output_folder / file_path.name if output_folder else file_path.parent / f\"{file_path.stem}_converted.md\"\n", " output_path.write_text('\\n'.join(converted), encoding='utf-8')\n", "\n", " print(f\"✓ Saved: {output_path.name}\")\n", "\n", "def convert_all_files(input_folder, output_folder=None, batch_size=50, wait_seconds=5):\n", " \"\"\"Convert all markdown files in folder with batching.\"\"\"\n", " input_path = Path(input_folder)\n", " output_path = Path(output_folder) if output_folder else None\n", "\n", " if output_path:\n", " output_path.mkdir(parents=True, exist_ok=True)\n", "\n", " md_files = list(input_path.glob('*.md'))\n", " total_files = len(md_files)\n", " print(f\"Found {total_files} files\")\n", " print(f\"Processing in batches of {batch_size} with {wait_seconds}s wait\\n\")\n", "\n", " converted = 0\n", " failed = 0\n", "\n", " for i, md_file in enumerate(md_files, 1):\n", " try:\n", " convert_markdown_file(md_file, output_path)\n", " converted += 1\n", " except Exception as e:\n", " print(f\"✗ Error: {md_file.name} - {e}\")\n", " failed += 1\n", "\n", " # Batch wait\n", " if i % batch_size == 0 and i < total_files:\n", " print(f\"\\n--- Processed {i}/{total_files} files ---\")\n", " print(f\"Waiting {wait_seconds} seconds...\\n\")\n", " time.sleep(wait_seconds)\n", "\n", " print(f\"\\n{'='*60}\")\n", " print(f\"✓ Complete! Converted: {converted} | Failed: {failed} | Total: {total_files}\")\n", "\n", "if __name__ == \"__main__\":\n", " input_folder = r\"/home/skiredj.abderrahman/khalil/Benchmarking_OCR/final_resulsts_benchmark/chandra2_with_split/output_markdown\"\n", " output_folder = r\"/home/skiredj.abderrahman/khalil/Benchmarking_OCR/final_resulsts_benchmark/chandra2_with_split/output_markdown_nohtml\"\n", "\n", " convert_all_files(input_folder, output_folder, batch_size=25, wait_seconds=10)\n", "\n", " # Option 2: Save to new folder with custom batch settings (uncomment)\n", " # output_folder = r\"C:\\...\\converted_markdowns\"\n", " # convert_all_files(input_folder, output_folder, batch_size=100, wait_seconds=3)\n" ] }, { "cell_type": "markdown", "id": "6098b496-9585-4684-bcdf-3e8ee2a860e9", "metadata": {}, "source": [ "## remove page number + merge left and right pages in one single page in the markdowns " ] }, { "cell_type": "code", "execution_count": null, "id": "8aa6f7a0-0d45-473f-926e-4779c27ffab0", "metadata": {}, "outputs": [], "source": [ "\n", "import os\n", "import re\n", "from pathlib import Path\n", "\n", "# ── Change these two paths ──────────────────────────────────────────\n", "INPUT_FOLDER = r\"/home/skiredj.abderrahman/khalil/Benchmarking_OCR/final_resulsts_benchmark/chandra2_with_split/output_markdown_nohtml\"\n", "OUTPUT_FOLDER = r\"/home/skiredj.abderrahman/khalil/Benchmarking_OCR/final_resulsts_benchmark/chandra2_with_split/output_markdown_nohtml_no_left_right_or_pagen\"\n", "# ────────────────────────────────────────────────────────────────────\n", "\n", "def parse_pages(content):\n", " pattern = r'(=== Page (\\d+) \\(([^)]+)\\) ===)'\n", " parts = re.split(pattern, content)\n", " \n", " preamble = parts[0]\n", " pages = []\n", " i = 1\n", " while i < len(parts):\n", " pages.append({\n", " 'header': parts[i],\n", " 'num': int(parts[i+1]),\n", " 'id': parts[i+2],\n", " 'body': parts[i+3]\n", " })\n", " i += 4\n", " return preamble, pages\n", "\n", "def merge_pages(pages):\n", " merged = []\n", " skip_ids = set()\n", " \n", " for i, page in enumerate(pages):\n", " pid = page['id']\n", " if pid in skip_ids:\n", " continue\n", " \n", " if pid.endswith('_right'):\n", " base_id = pid[:-len('_right')]\n", " left_page = None\n", " for j in range(i+1, len(pages)):\n", " if pages[j]['id'] == base_id + '_left':\n", " left_page = pages[j]\n", " skip_ids.add(pages[j]['id'])\n", " break\n", " \n", " combined_body = (page['body'].rstrip() + '\\n' + left_page['body']) if left_page else page['body']\n", " merged.append({'id': base_id, 'body': combined_body})\n", "\n", " elif pid.endswith('_left'):\n", " # Orphan left — strip suffix, keep content\n", " merged.append({'id': pid[:-len('_left')], 'body': page['body']})\n", " else:\n", " merged.append({'id': pid, 'body': page['body']})\n", " \n", " # Renumber sequentially\n", " return [{'num': n, 'id': p['id'], 'body': p['body']} for n, p in enumerate(merged, start=1)]\n", "\n", "def rebuild_content(preamble, pages):\n", " result = preamble\n", " for page in pages:\n", " result += f\"=== {page['id']} ===\\n\"\n", " result += page['body']\n", " return result\n", "\n", "def process_folder(input_folder, output_folder):\n", " input_path = Path(input_folder)\n", " output_path = Path(output_folder)\n", " output_path.mkdir(parents=True, exist_ok=True)\n", " \n", " md_files = list(input_path.glob('*.md'))\n", " if not md_files:\n", " print(f\"No .md files found in {input_folder}\")\n", " return\n", " \n", " for md_file in md_files:\n", " content = md_file.read_text(encoding='utf-8')\n", " preamble, pages = parse_pages(content)\n", " \n", " if not pages:\n", " (output_path / md_file.name).write_text(content, encoding='utf-8')\n", " print(f\"{md_file.name} — no pages found, copied as-is.\")\n", " continue\n", " \n", " merged = merge_pages(pages)\n", " output_path.joinpath(md_file.name).write_text(rebuild_content(preamble, merged), encoding='utf-8')\n", " \n", " print(f\"{md_file.name} — {len(pages)} pages → {len(merged)} pages ({len(pages)-len(merged)} pair(s) merged)\")\n", " \n", " print(f\"\\n✅ Done! Files saved to: {output_path.resolve()}\")\n", "\n", "process_folder(INPUT_FOLDER, OUTPUT_FOLDER)" ] }, { "cell_type": "code", "execution_count": null, "id": "0dcb0107-ff20-497f-a74d-71c0ff87d7ed", "metadata": {}, "outputs": [], "source": [ "## benchmarking " ] }, { "cell_type": "code", "execution_count": null, "id": "b0065aaa-edf6-4c40-bf98-2e44ff941ada", "metadata": {}, "outputs": [], "source": [ "import os\n", "import re\n", "import unicodedata\n", "import html as _html\n", "import json\n", "import editdistance\n", "import jiwer\n", "import sacrebleu\n", "from rouge_score import rouge_scorer as rs\n", "from apted import APTED, Config\n", "from lxml import etree\n", "from typing import Optional\n", "from pathlib import Path\n", "\n", "# ── Paths ─────────────────────────────────────────────────────────────────────\n", "GT_FOLDER = Path(\"/home/skiredj.abderrahman/khalil/Benchmarking_OCR/ground_truth_claude_100_images/GT\")\n", "HYP_FOLDER = Path(\"/home/skiredj.abderrahman/khalil/Benchmarking_OCR/final_resulsts_benchmark/chandra2_with_split/output_markdown_nohtml_no_left_right_or_pagen\")\n", "MODEL_NAME = \"chandra2_with_split\"\n", "\n", "# ── Normalization ─────────────────────────────────────────────────────────────\n", "def normalize_arabic_ocr(text: str, remove_markdown: bool = False) -> str:\n", " if not isinstance(text, str) or not text.strip():\n", " return \"\"\n", " if remove_markdown:\n", " text = re.sub(r'#{1,6}\\s*', '', text)\n", " text = re.sub(r'^\\s*[-*+]\\s+', '', text, flags=re.MULTILINE)\n", " text = re.sub(r'\\*{1,3}(.*?)\\*{1,3}', r'\\1', text, flags=re.DOTALL)\n", " text = re.sub(r'`{1,3}.*?`{1,3}', ' ', text, flags=re.DOTALL)\n", " text = re.sub(r'\\[([^\\]]*)\\]\\([^\\)]*\\)', r'\\1', text)\n", " text = re.sub(r'!\\[[^\\]]*\\]\\([^\\)]*\\)', ' ', text)\n", " text = re.sub(r'\\|', ' ', text)\n", " text = unicodedata.normalize('NFKC', text)\n", " text = re.sub(r'[\\u200B-\\u200F\\u202A-\\u202E\\u2060-\\u2064\\uFEFF\\u00AD]', '', text)\n", " text = re.sub(\n", " r'[\\u064B-\\u065F\\u0610-\\u061A\\u06D6-\\u06DC\\u06DF-\\u06E4\\u06E7\\u06E8\\u06EA-\\u06ED]',\n", " '', text\n", " )\n", " text = text.replace('\\u0640', '')\n", " text = re.sub(r'[\\u0622\\u0623\\u0625\\u0671\\u0672\\u0673\\u0675]', '\\u0627', text)\n", " text = text.replace('\\u0649', '\\u064A')\n", " text = text.replace('\\u0629', '\\u0647')\n", " text = text.replace('\\u0624', '\\u0648')\n", " text = text.replace('\\u0626', '\\u064A')\n", " text = re.sub(r'[\\u060C\\u061B\\u061F\\u066A-\\u066D\\u0600-\\u0605]', ' ', text)\n", " text = re.sub(r'[.,:;()\\[\\]{}\\-«»\"\\'!?/\\\\~@#$%^&*_+=<>|]', ' ', text)\n", " text = text.lower()\n", " text = re.sub(r'[^\\u0600-\\u06FF\\u0750-\\u077Fa-z0-9\\s]', ' ', text)\n", " text = text.translate(str.maketrans('٠١٢٣٤٥٦٧٨٩', '0123456789'))\n", " text = text.lower()\n", " return re.sub(r'\\s+', ' ', text).strip()\n", "\n", "# ── TEDS helpers ──────────────────────────────────────────────────────────────\n", "def _is_separator_row(row: str) -> bool:\n", " cells = [c.strip() for c in row.strip().strip('|').split('|')]\n", " return bool(cells) and all(re.match(r'^[-:\\s]+$', c) for c in cells if c)\n", "\n", "\n", "def _expand_inline_table(line: str) -> Optional[str]:\n", " if not re.search(r'\\|\\s*-{2,}[\\s:]*\\|', line):\n", " return None\n", " raw_cells = [c.strip() for c in line.split('|')]\n", " while raw_cells and raw_cells[0] == '':\n", " raw_cells.pop(0)\n", " while raw_cells and raw_cells[-1] == '':\n", " raw_cells.pop()\n", " if not raw_cells:\n", " return None\n", " sep_positions = [i for i, c in enumerate(raw_cells) if re.match(r'^[-:\\s]+$', c) and len(c) >= 2]\n", " if not sep_positions:\n", " return None\n", " n_cols = len(sep_positions)\n", " sep_start = sep_positions[0]\n", " header_cells = raw_cells[:sep_start]\n", " if len(header_cells) > n_cols:\n", " header_cells = header_cells[len(header_cells) - n_cols:]\n", " elif len(header_cells) < n_cols:\n", " header_cells = [''] * (n_cols - len(header_cells)) + header_cells\n", " data_cells = raw_cells[sep_start + n_cols:]\n", " rows = []\n", " for i in range(0, len(data_cells), n_cols):\n", " chunk = data_cells[i:i + n_cols]\n", " while len(chunk) < n_cols:\n", " chunk.append('')\n", " rows.append(chunk)\n", " if not rows:\n", " return None\n", " sep_row = '| ' + ' | '.join(['---'] * n_cols) + ' |'\n", " header_row = '| ' + ' | '.join(header_cells) + ' |'\n", " data_rows = ['| ' + ' | '.join(r) + ' |' for r in rows]\n", " return '\\n'.join([header_row, sep_row] + data_rows)\n", "\n", "\n", "def extract_md_tables(text: str) -> list:\n", " tables = []\n", " lines = text.split('\\n')\n", " cur = []\n", " for line in lines:\n", " stripped = line.strip()\n", " if re.search(r'\\|\\s*-{2,}[\\s:]*\\|', stripped):\n", " if len(cur) >= 2:\n", " tables.append('\\n'.join(cur))\n", " cur = []\n", " expanded = _expand_inline_table(stripped)\n", " if expanded:\n", " tables.append(expanded)\n", " continue\n", " if stripped.startswith('|') or stripped.count('|') >= 2:\n", " cur.append(stripped)\n", " else:\n", " if len(cur) >= 2:\n", " tables.append('\\n'.join(cur))\n", " cur = []\n", " if len(cur) >= 2:\n", " tables.append('\\n'.join(cur))\n", " return tables\n", "\n", "\n", "def md_table_to_html(md: str) -> str:\n", " lines = [l.strip() for l in md.strip().split('\\n') if l.strip()]\n", " lines = [l for l in lines if not _is_separator_row(l)]\n", " if not lines:\n", " return '
'\n", " rows = []\n", " for i, line in enumerate(lines):\n", " cells = [re.sub(r'\\s+', ' ', c.strip()) for c in line.strip('|').split('|')]\n", " tag = 'th' if i == 0 else 'td'\n", " rows.append('' + ''.join(f'<{tag}>{_html.escape(c)}' for c in cells) + '')\n", " return '' + ''.join(rows) + '
'\n", "\n", "\n", "class AptedConfig(Config):\n", " def rename(self, node1, node2):\n", " t1 = re.sub(r'\\s+', ' ', (node1.text or '').strip())\n", " t2 = re.sub(r'\\s+', ' ', (node2.text or '').strip())\n", " return 0 if (node1.tag == node2.tag and t1 == t2) else 1\n", " def children(self, node):\n", " return list(node)\n", "\n", "\n", "def teds_score(ref_html: str, hyp_html: str) -> Optional[float]:\n", " try:\n", " rt = etree.fromstring(f'{ref_html}')\n", " ht = etree.fromstring(f'{hyp_html}')\n", " dist = APTED(rt, ht, AptedConfig()).compute_edit_distance()\n", " n = max(len(list(rt.iter())), len(list(ht.iter())))\n", " return round((1.0 - dist / n) * 100, 2) if n > 0 else 0.0\n", " except Exception as e:\n", " print(f'TEDS error: {e}')\n", " return None\n", "\n", "# ── Per-file evaluation ───────────────────────────────────────────────────────\n", "def evaluate_page(ref_raw: str, hyp_raw: str) -> dict:\n", " ref = normalize_arabic_ocr(ref_raw, remove_markdown=True)\n", " hyp = normalize_arabic_ocr(hyp_raw, remove_markdown=True)\n", "\n", " if not ref or not hyp:\n", " return {'status': 'empty_after_norm'}\n", "\n", " rouge_sc = rs.RougeScorer(['rouge1', 'rougeL'], use_stemmer=False)\n", " pg = {'status': 'ok', 'ref_chars': len(ref_raw), 'hyp_chars': len(hyp_raw)}\n", "\n", " # NED\n", " try:\n", " ed = editdistance.eval(ref, hyp)\n", " pg['NED'] = round(ed / max(len(ref), len(hyp), 1) * 100, 2)\n", " except:\n", " pg['NED'] = None\n", "\n", " # CER / WER — jiwer\n", " try:\n", " tf = jiwer.Compose([jiwer.Strip(), jiwer.ReduceToListOfListOfWords()])\n", " pg['CER_jiwer'] = round(jiwer.cer(ref, hyp) * 100, 2)\n", " pg['WER_jiwer'] = round(jiwer.wer(ref, hyp, reference_transform=tf, hypothesis_transform=tf) * 100, 2)\n", " except:\n", " pg['CER_jiwer'] = pg['WER_jiwer'] = None\n", "\n", " # CER / WER — dinglehopper\n", " try:\n", " from dinglehopper.character_error_rate import character_error_rate as dce\n", " from dinglehopper.word_error_rate import word_error_rate as dwe\n", " pg['CER_dh'] = round(float(dce(ref, hyp)) * 100, 2)\n", " pg['WER_dh'] = round(float(dwe(ref, hyp)) * 100, 2)\n", " except:\n", " pg['CER_dh'] = pg['WER_dh'] = None\n", "\n", " # BLEU\n", " try:\n", " pg['BLEU'] = round(sacrebleu.sentence_bleu(hyp, [ref]).score, 2)\n", " except:\n", " pg['BLEU'] = None\n", "\n", " # ROUGE\n", " try:\n", " r = rouge_sc.score(ref, hyp)\n", " pg['ROUGE_1'] = round(r['rouge1'].fmeasure * 100, 2)\n", " pg['ROUGE_L'] = round(r['rougeL'].fmeasure * 100, 2)\n", " except:\n", " pg['ROUGE_1'] = pg['ROUGE_L'] = None\n", "\n", " # TEDS\n", " ref_tables = extract_md_tables(ref_raw)\n", " hyp_tables = extract_md_tables(hyp_raw)\n", " pg['n_tables_ref'] = len(ref_tables)\n", " pg['n_tables_hyp'] = len(hyp_tables)\n", " if ref_tables:\n", " teds_list = []\n", " for i, rt in enumerate(ref_tables):\n", " score = teds_score(md_table_to_html(rt), md_table_to_html(hyp_tables[i])) \\\n", " if i < len(hyp_tables) else 0.0\n", " teds_list.append(score if score is not None else 0.0)\n", " pg['TEDS'] = round(sum(teds_list) / len(teds_list), 2)\n", " else:\n", " pg['TEDS'] = None\n", "\n", " return pg\n", "\n", "# ── Page splitter ─────────────────────────────────────────────────────────────\n", "def split_pages_raw(content: str) -> dict:\n", " pattern = re.compile(r'^===\\s*(.+?)\\s*===$', re.MULTILINE)\n", " matches = list(pattern.finditer(content))\n", " if not matches:\n", " print(\"⚠️ No page separators found\")\n", " return {\"page_01\": content.strip()}\n", " pages = {}\n", " for idx, m in enumerate(matches):\n", " page_id = m.group(1).strip()\n", " start = m.end()\n", " end = matches[idx + 1].start() if idx + 1 < len(matches) else len(content)\n", " text = content[start:end].strip()\n", " if text:\n", " pages[page_id] = text\n", " return pages\n", "\n", "# ── Discover and match files ──────────────────────────────────────────────────\n", "gt_files = {f.name: f for f in sorted(GT_FOLDER.glob('*.md'))}\n", "hyp_files = {f.name: f for f in sorted(HYP_FOLDER.glob('*.md'))}\n", "\n", "print(f'GT folder : {len(gt_files)} .md files found')\n", "print(f'HYP folder : {len(hyp_files)} .md files found')\n", "\n", "common = sorted(set(gt_files) & set(hyp_files))\n", "only_gt = set(gt_files) - set(hyp_files)\n", "only_hyp = set(hyp_files) - set(gt_files)\n", "\n", "if only_gt:\n", " print(f'⚠️ Only in GT : {sorted(only_gt)}')\n", "if only_hyp:\n", " print(f'⚠️ Only in HYP : {sorted(only_hyp)}')\n", "print(f'\\n✅ Matched files to evaluate : {len(common)}')\n", "\n", "# ── Evaluate each matched pair — page by page ─────────────────────────────────\n", "# per_page stores ALL pages across ALL files: key = \"filename::page_id\"\n", "per_page = {}\n", "\n", "for fname in common:\n", " gt_raw = gt_files[fname].read_text(encoding='utf-8').strip()\n", " hyp_raw = hyp_files[fname].read_text(encoding='utf-8').strip()\n", "\n", " gt_pages = split_pages_raw(gt_raw)\n", " hyp_pages = split_pages_raw(hyp_raw)\n", "\n", " common_pages = sorted(set(gt_pages) & set(hyp_pages))\n", " only_gt_p = set(gt_pages) - set(hyp_pages)\n", " only_hyp_p = set(hyp_pages) - set(gt_pages)\n", "\n", " print(f'\\n 📄 {fname} → GT:{len(gt_pages)} pages HYP:{len(hyp_pages)} pages matched:{len(common_pages)}')\n", " if only_gt_p:\n", " print(f' ⚠️ Only in GT : {sorted(only_gt_p)}')\n", " if only_hyp_p:\n", " print(f' ⚠️ Only in HYP : {sorted(only_hyp_p)}')\n", "\n", " for page_id in common_pages:\n", " key = f\"{fname}::{page_id}\"\n", " result = evaluate_page(gt_pages[page_id], hyp_pages[page_id])\n", " per_page[key] = result\n", " if result.get('status') == 'ok':\n", " print(f' ✓ {page_id} NED:{result[\"NED\"]}% CER:{result[\"CER_jiwer\"]}% WER:{result[\"WER_jiwer\"]}%')\n", " else:\n", " print(f' ⚠️ {page_id} → {result.get(\"status\")}')\n", "\n", "# ── Global averages across all pages of all files ─────────────────────────────\n", "def avg(k):\n", " vals = [v[k] for v in per_page.values()\n", " if isinstance(v, dict) and v.get('status') == 'ok' and v.get(k) is not None]\n", " return round(sum(vals) / len(vals), 2) if vals else None\n", "\n", "overall = {k: avg(k) for k in ['NED','CER_jiwer','WER_jiwer','CER_dh','WER_dh','BLEU','ROUGE_1','ROUGE_L','TEDS']}\n", "total_pages = sum(1 for v in per_page.values() if isinstance(v, dict) and v.get('status') == 'ok')\n", "\n", "# ── Print final results ───────────────────────────────────────────────────────\n", "SEP = '=' * 60\n", "print(f'\\n{SEP}')\n", "print(f' {MODEL_NAME} — {len(common)} files / {total_pages} pages evaluated')\n", "print(SEP)\n", "print(f\" NED : {overall['NED']}% [↓ principal]\")\n", "print(f\" CER_jiwer : {overall['CER_jiwer']}% WER_jiwer : {overall['WER_jiwer']}%\")\n", "print(f\" CER_dh : {overall['CER_dh']}% WER_dh : {overall['WER_dh']}%\")\n", "print(f\" BLEU : {overall['BLEU']} ROUGE-1 : {overall['ROUGE_1']}% ROUGE-L : {overall['ROUGE_L']}%\")\n", "print(f\" TEDS : {overall['TEDS']}% [None = no tables in GT page]\")\n", "print(SEP)\n", "\n", "print(f\"\\n{'─'*60}\")\n", "print(f' Per-page breakdown')\n", "print(f\"{'─'*60}\")\n", "for key, pg in per_page.items():\n", " fname, page_id = key.split('::', 1)\n", " if pg.get('status') == 'ok':\n", " print(f'\\n [{fname}] {page_id}')\n", " print(f\" NED:{pg['NED']}% CER:{pg['CER_jiwer']}% WER:{pg['WER_jiwer']}% BLEU:{pg['BLEU']} ROUGE-1:{pg['ROUGE_1']}% TEDS:{pg['TEDS']}%\")\n", " if pg['n_tables_ref'] or pg['n_tables_hyp']:\n", " print(f\" tables → GT:{pg['n_tables_ref']} HYP:{pg['n_tables_hyp']}\")\n", " else:\n", " print(f\"\\n [{fname}] {page_id} ⚠️ {pg.get('status')}\")\n", "\n", "# ── Save results to JSON ──────────────────────────────────────────────────────\n", "RESULTS_DIR = \"/home/skiredj.abderrahman/khalil/Benchmarking_OCR/final_resulsts_benchmark/\"\n", "os.makedirs(RESULTS_DIR, exist_ok=True)\n", "\n", "def save_results(report: dict):\n", " name = report[\"model\"].replace(\" \", \"_\")\n", " path = os.path.join(RESULTS_DIR, f\"{name}.json\")\n", " json.dump(report, open(path, \"w\", encoding=\"utf-8\"), ensure_ascii=False, indent=2)\n", " print(f\"\\n✅ Sauvegardé : {path}\")\n", "\n", "report = {\n", " \"model\" : MODEL_NAME,\n", " \"n_files\" : len(common),\n", " \"n_pages\" : total_pages,\n", " \"overall\" : overall,\n", " \"per_page\": per_page\n", "}\n", "save_results(report)" ] }, { "cell_type": "markdown", "id": "d5b0afdf-c51b-4880-8048-930a60fb7d22", "metadata": { "jp-MarkdownHeadingCollapsed": true }, "source": [ "# chandra 2 no split " ] }, { "cell_type": "markdown", "id": "03639969-d6a8-4a0d-8ac7-ec7d4aa0771e", "metadata": { "jp-MarkdownHeadingCollapsed": true }, "source": [ "## OCR" ] }, { "cell_type": "code", "execution_count": null, "id": "b275631d-62ec-4450-9901-9ffa8fadfe09", "metadata": {}, "outputs": [], "source": [ "# ============================================================================\n", "# PIPELINE OCR - IMAGES LOCALES (sans prétraitement)\n", "# Entrée : dossier avec sous-dossiers contenant des PNG (déjà zoom x4)\n", "# Sortie : un fichier .md par sous-dossier\n", "# ============================================================================\n", "\n", "import os\n", "import re\n", "import time\n", "import base64\n", "import requests\n", "import glob\n", "import shutil\n", "from typing import Union, Optional\n", "from concurrent.futures import ThreadPoolExecutor, as_completed\n", "from tqdm import tqdm\n", "\n", "# ========== CONFIGURATION ==========\n", "\n", "VLLM_API_URL = \"http://localhost:9996/v1/chat/completions\"\n", "MODEL_PATH = \"/home/skiredj.abderrahman/lina/chandra-ocr-2\"\n", "\n", "GENERATION_PARAMS = {\n", " \"temperature\": 0.1,\n", " \"max_tokens\": 30000,\n", " \"top_p\": 0.9,\n", "}\n", "\n", "REQUEST_TIMEOUT_SECONDS = 180\n", "NUM_WORKERS = 20\n", "\n", "OCR_PROMPT = \"\"\"Extract all text exactly as it appears in this image.\n", "Preserve the original layout, reading order, formatting, and structure.\n", "Keep all tables as proper Markdown tables.\n", "Maintain all mathematical formulas, equations, and special characters.\n", "Do not rewrite, summarize, or modify any content.\n", "If any text is unclear or unreadable, write [UNCLEAR].\n", "Output only the extracted text without any preamble or explanation.\"\"\"\n", "\n", "# ── Dossiers ──────────────────────────────────────────────────────────────────\n", "IMAGES_DATASET_FOLDER = r\"/home/skiredj.abderrahman/khalil/Benchmarking_OCR/ground_truth_claude_100_images/100images\"\n", "\n", "OUTPUT_FOLDER = r\"/home/skiredj.abderrahman/khalil/Benchmarking_OCR/final_resulsts_benchmark/chandra2_nosplit/output_markdown\"\n", "TXT_TEMP_FOLDER = r\"/home/skiredj.abderrahman/khalil/Benchmarking_OCR/final_resulsts_benchmark/chandra2_nosplit/temp_txt\"\n", "\n", "\n", "# ========== TRI DES IMAGES (arabe : right avant left) ==========\n", "\n", "def arabic_sort_key(path):\n", " name = os.path.splitext(os.path.basename(path))[0]\n", " if name.endswith(\"_right\"):\n", " base, order = name[:-6], 0 # right = 0 → passe en premier\n", " elif name.endswith(\"_left\"):\n", " base, order = name[:-5], 1 # left = 1 → passe en second\n", " else:\n", " base, order = name, 0\n", " return (base, order)\n", "\n", "\n", "# ========== OCR ==========\n", "\n", "def call_vllm_ocr(image_path, retries=3):\n", " for attempt in range(1, retries + 1):\n", " try:\n", " with open(image_path, \"rb\") as f:\n", " img_b64 = base64.b64encode(f.read()).decode(\"utf-8\")\n", " payload = {\n", " \"model\": MODEL_PATH,\n", " \"messages\": [{\"role\": \"user\", \"content\": [\n", " {\"type\": \"image_url\", \"image_url\": {\"url\": f\"data:image/png;base64,{img_b64}\"}},\n", " {\"type\": \"text\", \"text\": OCR_PROMPT}\n", " ]}],\n", " **GENERATION_PARAMS,\n", " }\n", " resp = requests.post(VLLM_API_URL, json=payload, timeout=REQUEST_TIMEOUT_SECONDS)\n", " resp.raise_for_status()\n", " data = resp.json()\n", " if \"choices\" in data and data[\"choices\"]:\n", " return data[\"choices\"][0][\"message\"][\"content\"]\n", " raise Exception(f\"Format de réponse inattendu : {data}\")\n", " except requests.exceptions.Timeout:\n", " print(f\" Timeout tentative {attempt}/{retries} ({image_path})\")\n", " if attempt < retries: time.sleep(5)\n", " except Exception as e:\n", " print(f\" Erreur tentative {attempt}/{retries} ({image_path}): {e}\")\n", " if attempt < retries: time.sleep(3)\n", " raise Exception(f\"OCR échoué après {retries} tentatives : {image_path}\")\n", "\n", "\n", "def process_image(input_path, output_txt_path):\n", " if os.path.exists(output_txt_path):\n", " return # déjà traité\n", " try:\n", " text = call_vllm_ocr(input_path)\n", " with open(output_txt_path, \"w\", encoding=\"utf-8\") as f:\n", " f.write(text)\n", " except Exception as e:\n", " with open(output_txt_path, \"w\", encoding=\"utf-8\") as f:\n", " f.write(f\"[OCR FAILED: {e}]\")\n", "\n", "\n", "def clean_unnecessary_linebreaks(text):\n", " lines = text.split(\"\\n\")\n", " out = []\n", " for i, line in enumerate(lines):\n", " if line.startswith(\"=== Page\"):\n", " out.append(line)\n", " if i + 1 < len(lines) and lines[i + 1].strip() == \"\":\n", " out.append(\"\")\n", " else:\n", " prev_is_page_header = i > 0 and lines[i - 1].startswith(\"=== Page\")\n", " prev_ends_sentence = i > 0 and lines[i - 1].rstrip().endswith(\".\")\n", " if not prev_is_page_header and not prev_ends_sentence and out and out[-1] != \"\":\n", " out[-1] = out[-1].rstrip() + \" \" + line.lstrip()\n", " else:\n", " out.append(line)\n", " return \"\\n\".join(out)\n", "\n", "\n", "# ========== PIPELINE PRINCIPAL ==========\n", "\n", "def process_subfolder(subfolder_path, subfolder_name):\n", " \"\"\"\n", " Traite un sous-dossier : OCR direct sur les PNG → .md\n", " \"\"\"\n", " print(f\"\\n{'='*60}\")\n", " print(f\"Traitement : {subfolder_name}\")\n", " print(f\"{'='*60}\")\n", "\n", " txt_dir = os.path.join(TXT_TEMP_FOLDER, subfolder_name)\n", " output_md = os.path.join(OUTPUT_FOLDER, subfolder_name + \".md\")\n", " os.makedirs(txt_dir, exist_ok=True)\n", " os.makedirs(OUTPUT_FOLDER, exist_ok=True)\n", "\n", " if os.path.exists(output_md):\n", " print(f\" → Déjà traité, on passe.\")\n", " return\n", "\n", " # ── Étape 1 : Récupération et tri des images ───────────────────────────────\n", " png_files = sorted(\n", " glob.glob(os.path.join(subfolder_path, \"*.png\")),\n", " key=arabic_sort_key\n", " )\n", " if not png_files:\n", " print(f\" ⚠️ Aucune image PNG trouvée dans {subfolder_path}\")\n", " return\n", "\n", " # ── Étape 2 : OCR en parallèle ────────────────────────────────────────────\n", " tasks = [\n", " (img_p, os.path.join(txt_dir, os.path.splitext(os.path.basename(img_p))[0] + \".txt\"))\n", " for img_p in png_files\n", " ]\n", "\n", " print(f\" {len(png_files)} image(s) à envoyer à Chandra-2...\")\n", " with ThreadPoolExecutor(max_workers=NUM_WORKERS) as executor:\n", " futures = {executor.submit(process_image, ip, op): (ip, op) for ip, op in tasks}\n", " for future in tqdm(as_completed(futures), total=len(futures),\n", " desc=\" OCR\", leave=False):\n", " try:\n", " future.result()\n", " except Exception as e:\n", " print(f\" ❌ Erreur non gérée : {e}\")\n", "\n", " # ── Étape 3 : Assemblage en Markdown ──────────────────────────────────────\n", " parts = []\n", " for i, img_p in enumerate(png_files, 1):\n", " img_name = os.path.splitext(os.path.basename(img_p))[0]\n", " txt_path = os.path.join(txt_dir, img_name + \".txt\")\n", " if os.path.exists(txt_path):\n", " content = open(txt_path, encoding=\"utf-8\").read()\n", " else:\n", " content = \"[OCR failed for this page]\"\n", " parts.append(f\"=== Page {i} ({img_name}) ===\\n{content}\\n\")\n", "\n", " final = clean_unnecessary_linebreaks(\"\\n\".join(parts))\n", " with open(output_md, \"w\", encoding=\"utf-8\") as f:\n", " f.write(final)\n", "\n", " print(f\" ✅ Markdown sauvegardé → {output_md}\")\n", "\n", "\n", "def run_pipeline():\n", " \"\"\"\n", " Point d'entrée principal.\n", " Parcourt tous les sous-dossiers de IMAGES_DATASET_FOLDER,\n", " affiche un récapitulatif complet, puis lance le traitement.\n", " \"\"\"\n", " subfolders = sorted([\n", " d for d in os.listdir(IMAGES_DATASET_FOLDER)\n", " if os.path.isdir(os.path.join(IMAGES_DATASET_FOLDER, d))\n", " ])\n", "\n", " if not subfolders:\n", " print(f\"Aucun sous-dossier trouvé dans {IMAGES_DATASET_FOLDER}\")\n", " return\n", "\n", " # ── Scan préalable ────────────────────────────────────────────────────────\n", " print(f\"\\n{'='*60}\")\n", " print(f\" SCAN DU DATASET\")\n", " print(f\"{'='*60}\")\n", " print(f\" Dossier racine : {IMAGES_DATASET_FOLDER}\\n\")\n", "\n", " folder_image_counts = {}\n", " max_name_len = max(len(d) for d in subfolders)\n", "\n", " for name in subfolders:\n", " path = os.path.join(IMAGES_DATASET_FOLDER, name)\n", " pngs = sorted(glob.glob(os.path.join(path, \"*.png\")))\n", " folder_image_counts[name] = pngs\n", " status = f\"{len(pngs):>4} image(s)\" if pngs else \" ── aucune image PNG\"\n", " print(f\" 📁 {name:<{max_name_len}} → {status}\")\n", "\n", " total_images = sum(len(v) for v in folder_image_counts.values())\n", " total_folders = len(subfolders)\n", " empty_folders = sum(1 for v in folder_image_counts.values() if not v)\n", " active_folders = total_folders - empty_folders\n", "\n", " print(f\"\\n{'─'*60}\")\n", " print(f\" Sous-dossiers trouvés : {total_folders}\")\n", " print(f\" Sous-dossiers avec PNG : {active_folders}\")\n", " print(f\" Sous-dossiers vides : {empty_folders}\")\n", " print(f\" Total images PNG : {total_images}\")\n", " print(f\"{'─'*60}\")\n", " print(f\" Sortie markdown : {OUTPUT_FOLDER}\")\n", " print(f\"{'='*60}\\n\")\n", "\n", " if total_images == 0:\n", " print(\"Aucune image à traiter. Arrêt.\")\n", " return\n", "\n", " # ── Traitement ────────────────────────────────────────────────────────────\n", " for name in tqdm(subfolders, desc=\"Sous-dossiers\"):\n", " if not folder_image_counts[name]:\n", " continue\n", " path = os.path.join(IMAGES_DATASET_FOLDER, name)\n", " try:\n", " process_subfolder(path, name)\n", " except Exception as e:\n", " print(f\"❌ Erreur sur {name} : {e}\")\n", "\n", " print(f\"\\n{'='*60}\")\n", " print(\"PIPELINE TERMINÉ\")\n", " print(f\" {active_folders} sous-dossier(s) traité(s), {total_images} image(s) au total\")\n", " print(f\" Résultats dans : {OUTPUT_FOLDER}\")\n", " print(f\"{'='*60}\")\n", "\n", "\n", "# ========== LANCEMENT ==========\n", "\n", "if __name__ == \"__main__\":\n", " run_pipeline()" ] }, { "cell_type": "markdown", "id": "f2b058be-1b01-4e25-bd77-e89aa6e17a80", "metadata": { "jp-MarkdownHeadingCollapsed": true }, "source": [ "## html to md " ] }, { "cell_type": "code", "execution_count": null, "id": "d68fce38-7245-420d-bd08-748ec54defe2", "metadata": {}, "outputs": [], "source": [ "# exectued tomd.pbs\n", "from pathlib import Path\n", "from markdownify import markdownify as md\n", "import re\n", "import time\n", "\n", "def convert_markdown_file(file_path, output_folder=None):\n", " \"\"\"Convert HTML in markdown file to proper markdown.\"\"\"\n", " print(f\"Processing: {file_path.name}\")\n", "\n", " # Read the file\n", " content = file_path.read_text(encoding='utf-8')\n", "\n", " # Split by page sections\n", " page_pattern = r'(=== Page \\d+ \\([^)]+\\) ===)'\n", " sections = re.split(page_pattern, content)\n", "\n", " # Convert HTML sections to markdown\n", " converted = []\n", " for section in sections:\n", " if section.startswith('=== Page'):\n", " converted.append(section)\n", " elif section.strip():\n", " # Convert HTML to markdown\n", " markdown = md(section, heading_style=\"ATX\")\n", " converted.append('\\n' + markdown.strip() + '\\n')\n", "\n", " # Save the result\n", " output_path = output_folder / file_path.name if output_folder else file_path.parent / f\"{file_path.stem}_converted.md\"\n", " output_path.write_text('\\n'.join(converted), encoding='utf-8')\n", "\n", " print(f\"✓ Saved: {output_path.name}\")\n", "\n", "def convert_all_files(input_folder, output_folder=None, batch_size=50, wait_seconds=5):\n", " \"\"\"Convert all markdown files in folder with batching.\"\"\"\n", " input_path = Path(input_folder)\n", " output_path = Path(output_folder) if output_folder else None\n", "\n", " if output_path:\n", " output_path.mkdir(parents=True, exist_ok=True)\n", "\n", " md_files = list(input_path.glob('*.md'))\n", " total_files = len(md_files)\n", " print(f\"Found {total_files} files\")\n", " print(f\"Processing in batches of {batch_size} with {wait_seconds}s wait\\n\")\n", "\n", " converted = 0\n", " failed = 0\n", "\n", " for i, md_file in enumerate(md_files, 1):\n", " try:\n", " convert_markdown_file(md_file, output_path)\n", " converted += 1\n", " except Exception as e:\n", " print(f\"✗ Error: {md_file.name} - {e}\")\n", " failed += 1\n", "\n", " # Batch wait\n", " if i % batch_size == 0 and i < total_files:\n", " print(f\"\\n--- Processed {i}/{total_files} files ---\")\n", " print(f\"Waiting {wait_seconds} seconds...\\n\")\n", " time.sleep(wait_seconds)\n", "\n", " print(f\"\\n{'='*60}\")\n", " print(f\"✓ Complete! Converted: {converted} | Failed: {failed} | Total: {total_files}\")\n", "\n", "if __name__ == \"__main__\":\n", " input_folder = r\"/home/skiredj.abderrahman/khalil/Benchmarking_OCR/final_resulsts_benchmark/chandra2_nosplit/output_markdown\"\n", " output_folder = r\"/home/skiredj.abderrahman/khalil/Benchmarking_OCR/final_resulsts_benchmark/chandra2_nosplit/output_markdown_no_html\"\n", "\n", " convert_all_files(input_folder, output_folder, batch_size=25, wait_seconds=10)\n", "\n", " # Option 2: Save to new folder with custom batch settings (uncomment)\n", " # output_folder = r\"C:\\...\\converted_markdowns\"\n", " # convert_all_files(input_folder, output_folder, batch_size=100, wait_seconds=3)\n" ] }, { "cell_type": "markdown", "id": "034f796a-9a00-4144-abbc-26d3d68a079a", "metadata": { "jp-MarkdownHeadingCollapsed": true }, "source": [ "## remove page n " ] }, { "cell_type": "code", "execution_count": null, "id": "3cf73c71-8665-4a47-a2a4-5eb17a67f7c0", "metadata": {}, "outputs": [], "source": [ "\n", "import os\n", "import re\n", "from pathlib import Path\n", "\n", "# ── Change these two paths ──────────────────────────────────────────\n", "INPUT_FOLDER = \"/home/skiredj.abderrahman/khalil/Benchmarking_OCR/final_resulsts_benchmark/chandra2_nosplit/output_markdown_no_html\"\n", "OUTPUT_FOLDER = \"/home/skiredj.abderrahman/khalil/Benchmarking_OCR/final_resulsts_benchmark/chandra2_nosplit/output_markdown_no_html_no_pageN\"\n", "# ────────────────────────────────────────────────────────────────────\n", "\n", "def parse_pages(content):\n", " pattern = r'(=== Page (\\d+) \\(([^)]+)\\) ===)'\n", " parts = re.split(pattern, content)\n", " \n", " preamble = parts[0]\n", " pages = []\n", " i = 1\n", " while i < len(parts):\n", " pages.append({\n", " 'header': parts[i],\n", " 'num': int(parts[i+1]),\n", " 'id': parts[i+2],\n", " 'body': parts[i+3]\n", " })\n", " i += 4\n", " return preamble, pages\n", "\n", "def merge_pages(pages):\n", " merged = []\n", " skip_ids = set()\n", " \n", " for i, page in enumerate(pages):\n", " pid = page['id']\n", " if pid in skip_ids:\n", " continue\n", " \n", " if pid.endswith('_right'):\n", " base_id = pid[:-len('_right')]\n", " left_page = None\n", " for j in range(i+1, len(pages)):\n", " if pages[j]['id'] == base_id + '_left':\n", " left_page = pages[j]\n", " skip_ids.add(pages[j]['id'])\n", " break\n", " \n", " combined_body = (page['body'].rstrip() + '\\n' + left_page['body']) if left_page else page['body']\n", " merged.append({'id': base_id, 'body': combined_body})\n", "\n", " elif pid.endswith('_left'):\n", " # Orphan left — strip suffix, keep content\n", " merged.append({'id': pid[:-len('_left')], 'body': page['body']})\n", " else:\n", " merged.append({'id': pid, 'body': page['body']})\n", " \n", " # Renumber sequentially\n", " return [{'num': n, 'id': p['id'], 'body': p['body']} for n, p in enumerate(merged, start=1)]\n", "\n", "def rebuild_content(preamble, pages):\n", " result = preamble\n", " for page in pages:\n", " result += f\"=== {page['id']} ===\\n\"\n", " result += page['body']\n", " return result\n", "\n", "def process_folder(input_folder, output_folder):\n", " input_path = Path(input_folder)\n", " output_path = Path(output_folder)\n", " output_path.mkdir(parents=True, exist_ok=True)\n", " \n", " md_files = list(input_path.glob('*.md'))\n", " if not md_files:\n", " print(f\"No .md files found in {input_folder}\")\n", " return\n", " \n", " for md_file in md_files:\n", " content = md_file.read_text(encoding='utf-8')\n", " preamble, pages = parse_pages(content)\n", " \n", " if not pages:\n", " (output_path / md_file.name).write_text(content, encoding='utf-8')\n", " print(f\"{md_file.name} — no pages found, copied as-is.\")\n", " continue\n", " \n", " merged = merge_pages(pages)\n", " output_path.joinpath(md_file.name).write_text(rebuild_content(preamble, merged), encoding='utf-8')\n", " \n", " print(f\"{md_file.name} — {len(pages)} pages → {len(merged)} pages ({len(pages)-len(merged)} pair(s) merged)\")\n", " \n", " print(f\"\\n✅ Done! Files saved to: {output_path.resolve()}\")\n", "\n", "process_folder(INPUT_FOLDER, OUTPUT_FOLDER)" ] }, { "cell_type": "markdown", "id": "27a2dd91-ca94-4d4d-858a-10f6beecc291", "metadata": {}, "source": [ "## Benchmarking" ] }, { "cell_type": "code", "execution_count": null, "id": "e04626f5-1bb6-4708-afdb-199c40a38b0d", "metadata": {}, "outputs": [], "source": [ "import os\n", "import re\n", "import unicodedata\n", "import html as _html\n", "import json\n", "import editdistance\n", "import jiwer\n", "import sacrebleu\n", "from rouge_score import rouge_scorer as rs\n", "from apted import APTED, Config\n", "from lxml import etree\n", "from typing import Optional\n", "from pathlib import Path\n", "\n", "# ── Paths ─────────────────────────────────────────────────────────────────────\n", "GT_FOLDER = Path(\"/home/skiredj.abderrahman/khalil/Benchmarking_OCR/ground_truth_claude_100_images/GT\")\n", "HYP_FOLDER = Path(\"/home/skiredj.abderrahman/khalil/Benchmarking_OCR/final_resulsts_benchmark/chandra2_nosplit/output_markdown_no_html_no_pageN\")\n", "MODEL_NAME = \"chandra2_without_split\"\n", "\n", "# ── Normalization ─────────────────────────────────────────────────────────────\n", "def normalize_arabic_ocr(text: str, remove_markdown: bool = False) -> str:\n", " if not isinstance(text, str) or not text.strip():\n", " return \"\"\n", " if remove_markdown:\n", " text = re.sub(r'#{1,6}\\s*', '', text)\n", " text = re.sub(r'^\\s*[-*+]\\s+', '', text, flags=re.MULTILINE)\n", " text = re.sub(r'\\*{1,3}(.*?)\\*{1,3}', r'\\1', text, flags=re.DOTALL)\n", " text = re.sub(r'`{1,3}.*?`{1,3}', ' ', text, flags=re.DOTALL)\n", " text = re.sub(r'\\[([^\\]]*)\\]\\([^\\)]*\\)', r'\\1', text)\n", " text = re.sub(r'!\\[[^\\]]*\\]\\([^\\)]*\\)', ' ', text)\n", " text = re.sub(r'\\|', ' ', text)\n", " text = unicodedata.normalize('NFKC', text)\n", " text = re.sub(r'[\\u200B-\\u200F\\u202A-\\u202E\\u2060-\\u2064\\uFEFF\\u00AD]', '', text)\n", " text = re.sub(\n", " r'[\\u064B-\\u065F\\u0610-\\u061A\\u06D6-\\u06DC\\u06DF-\\u06E4\\u06E7\\u06E8\\u06EA-\\u06ED]',\n", " '', text\n", " )\n", " text = text.replace('\\u0640', '')\n", " text = re.sub(r'[\\u0622\\u0623\\u0625\\u0671\\u0672\\u0673\\u0675]', '\\u0627', text)\n", " text = text.replace('\\u0649', '\\u064A')\n", " text = text.replace('\\u0629', '\\u0647')\n", " text = text.replace('\\u0624', '\\u0648')\n", " text = text.replace('\\u0626', '\\u064A')\n", " text = re.sub(r'[\\u060C\\u061B\\u061F\\u066A-\\u066D\\u0600-\\u0605]', ' ', text)\n", " text = re.sub(r'[.,:;()\\[\\]{}\\-«»\"\\'!?/\\\\~@#$%^&*_+=<>|]', ' ', text)\n", " text = text.lower()\n", " text = re.sub(r'[^\\u0600-\\u06FF\\u0750-\\u077Fa-z0-9\\s]', ' ', text)\n", " text = text.translate(str.maketrans('٠١٢٣٤٥٦٧٨٩', '0123456789'))\n", " text = text.lower()\n", " return re.sub(r'\\s+', ' ', text).strip()\n", "\n", "# ── TEDS helpers ──────────────────────────────────────────────────────────────\n", "def _is_separator_row(row: str) -> bool:\n", " cells = [c.strip() for c in row.strip().strip('|').split('|')]\n", " return bool(cells) and all(re.match(r'^[-:\\s]+$', c) for c in cells if c)\n", "\n", "\n", "def _expand_inline_table(line: str) -> Optional[str]:\n", " if not re.search(r'\\|\\s*-{2,}[\\s:]*\\|', line):\n", " return None\n", " raw_cells = [c.strip() for c in line.split('|')]\n", " while raw_cells and raw_cells[0] == '':\n", " raw_cells.pop(0)\n", " while raw_cells and raw_cells[-1] == '':\n", " raw_cells.pop()\n", " if not raw_cells:\n", " return None\n", " sep_positions = [i for i, c in enumerate(raw_cells) if re.match(r'^[-:\\s]+$', c) and len(c) >= 2]\n", " if not sep_positions:\n", " return None\n", " n_cols = len(sep_positions)\n", " sep_start = sep_positions[0]\n", " header_cells = raw_cells[:sep_start]\n", " if len(header_cells) > n_cols:\n", " header_cells = header_cells[len(header_cells) - n_cols:]\n", " elif len(header_cells) < n_cols:\n", " header_cells = [''] * (n_cols - len(header_cells)) + header_cells\n", " data_cells = raw_cells[sep_start + n_cols:]\n", " rows = []\n", " for i in range(0, len(data_cells), n_cols):\n", " chunk = data_cells[i:i + n_cols]\n", " while len(chunk) < n_cols:\n", " chunk.append('')\n", " rows.append(chunk)\n", " if not rows:\n", " return None\n", " sep_row = '| ' + ' | '.join(['---'] * n_cols) + ' |'\n", " header_row = '| ' + ' | '.join(header_cells) + ' |'\n", " data_rows = ['| ' + ' | '.join(r) + ' |' for r in rows]\n", " return '\\n'.join([header_row, sep_row] + data_rows)\n", "\n", "\n", "def extract_md_tables(text: str) -> list:\n", " tables = []\n", " lines = text.split('\\n')\n", " cur = []\n", " for line in lines:\n", " stripped = line.strip()\n", " if re.search(r'\\|\\s*-{2,}[\\s:]*\\|', stripped):\n", " if len(cur) >= 2:\n", " tables.append('\\n'.join(cur))\n", " cur = []\n", " expanded = _expand_inline_table(stripped)\n", " if expanded:\n", " tables.append(expanded)\n", " continue\n", " if stripped.startswith('|') or stripped.count('|') >= 2:\n", " cur.append(stripped)\n", " else:\n", " if len(cur) >= 2:\n", " tables.append('\\n'.join(cur))\n", " cur = []\n", " if len(cur) >= 2:\n", " tables.append('\\n'.join(cur))\n", " return tables\n", "\n", "\n", "def md_table_to_html(md: str) -> str:\n", " lines = [l.strip() for l in md.strip().split('\\n') if l.strip()]\n", " lines = [l for l in lines if not _is_separator_row(l)]\n", " if not lines:\n", " return '
'\n", " rows = []\n", " for i, line in enumerate(lines):\n", " cells = [re.sub(r'\\s+', ' ', c.strip()) for c in line.strip('|').split('|')]\n", " tag = 'th' if i == 0 else 'td'\n", " rows.append('' + ''.join(f'<{tag}>{_html.escape(c)}' for c in cells) + '')\n", " return '' + ''.join(rows) + '
'\n", "\n", "\n", "class AptedConfig(Config):\n", " def rename(self, node1, node2):\n", " t1 = re.sub(r'\\s+', ' ', (node1.text or '').strip())\n", " t2 = re.sub(r'\\s+', ' ', (node2.text or '').strip())\n", " return 0 if (node1.tag == node2.tag and t1 == t2) else 1\n", " def children(self, node):\n", " return list(node)\n", "\n", "\n", "def teds_score(ref_html: str, hyp_html: str) -> Optional[float]:\n", " try:\n", " rt = etree.fromstring(f'{ref_html}')\n", " ht = etree.fromstring(f'{hyp_html}')\n", " dist = APTED(rt, ht, AptedConfig()).compute_edit_distance()\n", " n = max(len(list(rt.iter())), len(list(ht.iter())))\n", " return round((1.0 - dist / n) * 100, 2) if n > 0 else 0.0\n", " except Exception as e:\n", " print(f'TEDS error: {e}')\n", " return None\n", "\n", "# ── Per-file evaluation ───────────────────────────────────────────────────────\n", "def evaluate_page(ref_raw: str, hyp_raw: str) -> dict:\n", " ref = normalize_arabic_ocr(ref_raw, remove_markdown=True)\n", " hyp = normalize_arabic_ocr(hyp_raw, remove_markdown=True)\n", "\n", " if not ref or not hyp:\n", " return {'status': 'empty_after_norm'}\n", "\n", " rouge_sc = rs.RougeScorer(['rouge1', 'rougeL'], use_stemmer=False)\n", " pg = {'status': 'ok', 'ref_chars': len(ref_raw), 'hyp_chars': len(hyp_raw)}\n", "\n", " # NED\n", " try:\n", " ed = editdistance.eval(ref, hyp)\n", " pg['NED'] = round(ed / max(len(ref), len(hyp), 1) * 100, 2)\n", " except:\n", " pg['NED'] = None\n", "\n", " # CER / WER — jiwer\n", " try:\n", " tf = jiwer.Compose([jiwer.Strip(), jiwer.ReduceToListOfListOfWords()])\n", " pg['CER_jiwer'] = round(jiwer.cer(ref, hyp) * 100, 2)\n", " pg['WER_jiwer'] = round(jiwer.wer(ref, hyp, reference_transform=tf, hypothesis_transform=tf) * 100, 2)\n", " except:\n", " pg['CER_jiwer'] = pg['WER_jiwer'] = None\n", "\n", " # CER / WER — dinglehopper\n", " try:\n", " from dinglehopper.character_error_rate import character_error_rate as dce\n", " from dinglehopper.word_error_rate import word_error_rate as dwe\n", " pg['CER_dh'] = round(float(dce(ref, hyp)) * 100, 2)\n", " pg['WER_dh'] = round(float(dwe(ref, hyp)) * 100, 2)\n", " except:\n", " pg['CER_dh'] = pg['WER_dh'] = None\n", "\n", " # BLEU\n", " try:\n", " pg['BLEU'] = round(sacrebleu.sentence_bleu(hyp, [ref]).score, 2)\n", " except:\n", " pg['BLEU'] = None\n", "\n", " # ROUGE\n", " try:\n", " r = rouge_sc.score(ref, hyp)\n", " pg['ROUGE_1'] = round(r['rouge1'].fmeasure * 100, 2)\n", " pg['ROUGE_L'] = round(r['rougeL'].fmeasure * 100, 2)\n", " except:\n", " pg['ROUGE_1'] = pg['ROUGE_L'] = None\n", "\n", " # TEDS\n", " ref_tables = extract_md_tables(ref_raw)\n", " hyp_tables = extract_md_tables(hyp_raw)\n", " pg['n_tables_ref'] = len(ref_tables)\n", " pg['n_tables_hyp'] = len(hyp_tables)\n", " if ref_tables:\n", " teds_list = []\n", " for i, rt in enumerate(ref_tables):\n", " score = teds_score(md_table_to_html(rt), md_table_to_html(hyp_tables[i])) \\\n", " if i < len(hyp_tables) else 0.0\n", " teds_list.append(score if score is not None else 0.0)\n", " pg['TEDS'] = round(sum(teds_list) / len(teds_list), 2)\n", " else:\n", " pg['TEDS'] = None\n", "\n", " return pg\n", "\n", "# ── Page splitter ─────────────────────────────────────────────────────────────\n", "def split_pages_raw(content: str) -> dict:\n", " pattern = re.compile(r'^===\\s*(.+?)\\s*===$', re.MULTILINE)\n", " matches = list(pattern.finditer(content))\n", " if not matches:\n", " print(\"⚠️ No page separators found\")\n", " return {\"page_01\": content.strip()}\n", " pages = {}\n", " for idx, m in enumerate(matches):\n", " page_id = m.group(1).strip()\n", " start = m.end()\n", " end = matches[idx + 1].start() if idx + 1 < len(matches) else len(content)\n", " text = content[start:end].strip()\n", " if text:\n", " pages[page_id] = text\n", " return pages\n", "\n", "# ── Discover and match files ──────────────────────────────────────────────────\n", "gt_files = {f.name: f for f in sorted(GT_FOLDER.glob('*.md'))}\n", "hyp_files = {f.name: f for f in sorted(HYP_FOLDER.glob('*.md'))}\n", "\n", "print(f'GT folder : {len(gt_files)} .md files found')\n", "print(f'HYP folder : {len(hyp_files)} .md files found')\n", "\n", "common = sorted(set(gt_files) & set(hyp_files))\n", "only_gt = set(gt_files) - set(hyp_files)\n", "only_hyp = set(hyp_files) - set(gt_files)\n", "\n", "if only_gt:\n", " print(f'⚠️ Only in GT : {sorted(only_gt)}')\n", "if only_hyp:\n", " print(f'⚠️ Only in HYP : {sorted(only_hyp)}')\n", "print(f'\\n✅ Matched files to evaluate : {len(common)}')\n", "\n", "# ── Evaluate each matched pair — page by page ─────────────────────────────────\n", "# per_page stores ALL pages across ALL files: key = \"filename::page_id\"\n", "per_page = {}\n", "\n", "for fname in common:\n", " gt_raw = gt_files[fname].read_text(encoding='utf-8').strip()\n", " hyp_raw = hyp_files[fname].read_text(encoding='utf-8').strip()\n", "\n", " gt_pages = split_pages_raw(gt_raw)\n", " hyp_pages = split_pages_raw(hyp_raw)\n", "\n", " common_pages = sorted(set(gt_pages) & set(hyp_pages))\n", " only_gt_p = set(gt_pages) - set(hyp_pages)\n", " only_hyp_p = set(hyp_pages) - set(gt_pages)\n", "\n", " print(f'\\n 📄 {fname} → GT:{len(gt_pages)} pages HYP:{len(hyp_pages)} pages matched:{len(common_pages)}')\n", " if only_gt_p:\n", " print(f' ⚠️ Only in GT : {sorted(only_gt_p)}')\n", " if only_hyp_p:\n", " print(f' ⚠️ Only in HYP : {sorted(only_hyp_p)}')\n", "\n", " for page_id in common_pages:\n", " key = f\"{fname}::{page_id}\"\n", " result = evaluate_page(gt_pages[page_id], hyp_pages[page_id])\n", " per_page[key] = result\n", " if result.get('status') == 'ok':\n", " print(f' ✓ {page_id} NED:{result[\"NED\"]}% CER:{result[\"CER_jiwer\"]}% WER:{result[\"WER_jiwer\"]}%')\n", " else:\n", " print(f' ⚠️ {page_id} → {result.get(\"status\")}')\n", "\n", "# ── Global averages across all pages of all files ─────────────────────────────\n", "def avg(k):\n", " vals = [v[k] for v in per_page.values()\n", " if isinstance(v, dict) and v.get('status') == 'ok' and v.get(k) is not None]\n", " return round(sum(vals) / len(vals), 2) if vals else None\n", "\n", "overall = {k: avg(k) for k in ['NED','CER_jiwer','WER_jiwer','CER_dh','WER_dh','BLEU','ROUGE_1','ROUGE_L','TEDS']}\n", "total_pages = sum(1 for v in per_page.values() if isinstance(v, dict) and v.get('status') == 'ok')\n", "\n", "# ── Print final results ───────────────────────────────────────────────────────\n", "SEP = '=' * 60\n", "print(f'\\n{SEP}')\n", "print(f' {MODEL_NAME} — {len(common)} files / {total_pages} pages evaluated')\n", "print(SEP)\n", "print(f\" NED : {overall['NED']}% [↓ principal]\")\n", "print(f\" CER_jiwer : {overall['CER_jiwer']}% WER_jiwer : {overall['WER_jiwer']}%\")\n", "print(f\" CER_dh : {overall['CER_dh']}% WER_dh : {overall['WER_dh']}%\")\n", "print(f\" BLEU : {overall['BLEU']} ROUGE-1 : {overall['ROUGE_1']}% ROUGE-L : {overall['ROUGE_L']}%\")\n", "print(f\" TEDS : {overall['TEDS']}% [None = no tables in GT page]\")\n", "print(SEP)\n", "\n", "print(f\"\\n{'─'*60}\")\n", "print(f' Per-page breakdown')\n", "print(f\"{'─'*60}\")\n", "for key, pg in per_page.items():\n", " fname, page_id = key.split('::', 1)\n", " if pg.get('status') == 'ok':\n", " print(f'\\n [{fname}] {page_id}')\n", " print(f\" NED:{pg['NED']}% CER:{pg['CER_jiwer']}% WER:{pg['WER_jiwer']}% BLEU:{pg['BLEU']} ROUGE-1:{pg['ROUGE_1']}% TEDS:{pg['TEDS']}%\")\n", " if pg['n_tables_ref'] or pg['n_tables_hyp']:\n", " print(f\" tables → GT:{pg['n_tables_ref']} HYP:{pg['n_tables_hyp']}\")\n", " else:\n", " print(f\"\\n [{fname}] {page_id} ⚠️ {pg.get('status')}\")\n", "\n", "# ── Save results to JSON ──────────────────────────────────────────────────────\n", "RESULTS_DIR = \"/home/skiredj.abderrahman/khalil/Benchmarking_OCR/final_resulsts_benchmark/\"\n", "os.makedirs(RESULTS_DIR, exist_ok=True)\n", "\n", "def save_results(report: dict):\n", " name = report[\"model\"].replace(\" \", \"_\")\n", " path = os.path.join(RESULTS_DIR, f\"{name}.json\")\n", " json.dump(report, open(path, \"w\", encoding=\"utf-8\"), ensure_ascii=False, indent=2)\n", " print(f\"\\n✅ Sauvegardé : {path}\")\n", "\n", "report = {\n", " \"model\" : MODEL_NAME,\n", " \"n_files\" : len(common),\n", " \"n_pages\" : total_pages,\n", " \"overall\" : overall,\n", " \"per_page\": per_page\n", "}\n", "save_results(report)" ] }, { "cell_type": "markdown", "id": "4a7564fa-5e70-4181-9a4e-ec10dc6b42ce", "metadata": { "jp-MarkdownHeadingCollapsed": true }, "source": [ "# chandra 1 " ] }, { "cell_type": "markdown", "id": "50419ea0-bc9d-43da-a9ba-b38a7339ac2d", "metadata": { "jp-MarkdownHeadingCollapsed": true }, "source": [ "## ocr" ] }, { "cell_type": "code", "execution_count": null, "id": "c65684ea-0496-4bad-bda6-617c51ef1321", "metadata": {}, "outputs": [], "source": [ "# ============================================================================\n", "# PIPELINE OCR - IMAGES LOCALES (sans prétraitement)\n", "# Entrée : dossier avec sous-dossiers contenant des PNG (déjà zoom x4)\n", "# Sortie : un fichier .md par sous-dossier\n", "# ============================================================================\n", "\n", "import os\n", "import re\n", "import time\n", "import base64\n", "import requests\n", "import glob\n", "import shutil\n", "from typing import Union, Optional\n", "from concurrent.futures import ThreadPoolExecutor, as_completed\n", "from tqdm import tqdm\n", "\n", "# ========== CONFIGURATION ==========\n", "\n", "VLLM_API_URL = \"http://localhost:9996/v1/chat/completions\"\n", "# Updated to the path used in your vLLM serve command\n", "MODEL_PATH = \"/home/skiredj.abderrahman/khalil/chandra\"\n", "\n", "GENERATION_PARAMS = {\n", " \"temperature\": 0.1,\n", " \"max_tokens\": 30000,\n", " \"top_p\": 0.9,\n", "}\n", "\n", "REQUEST_TIMEOUT_SECONDS = 180\n", "NUM_WORKERS = 20\n", "\n", "OCR_PROMPT = \"\"\"Extract all text exactly as it appears in this image.\n", "Preserve the original layout, reading order, formatting, and structure.\n", "Keep all tables as proper Markdown tables.\n", "Maintain all mathematical formulas, equations, and special characters.\n", "Do not rewrite, summarize, or modify any content.\n", "If any text is unclear or unreadable, write [UNCLEAR].\n", "Output only the extracted text without any preamble or explanation.\"\"\"\n", "\n", "# ── Dossiers ──────────────────────────────────────────────────────────────────\n", "IMAGES_DATASET_FOLDER = r\"/home/skiredj.abderrahman/khalil/Benchmarking_OCR/ground_truth_claude_100_images/100images\"\n", "\n", "OUTPUT_FOLDER = r\"/home/skiredj.abderrahman/khalil/Benchmarking_OCR/final_resulsts_benchmark/chandra1_nosplit/output_markdown\"\n", "TXT_TEMP_FOLDER = r\"/home/skiredj.abderrahman/khalil/Benchmarking_OCR/final_resulsts_benchmark/chandra1_nosplit/temp_txt\"\n", "\n", "\n", "# ========== TRI DES IMAGES (arabe : right avant left) ==========\n", "\n", "def arabic_sort_key(path):\n", " name = os.path.splitext(os.path.basename(path))[0]\n", " if name.endswith(\"_right\"):\n", " base, order = name[:-6], 0 # right = 0 → passe en premier\n", " elif name.endswith(\"_left\"):\n", " base, order = name[:-5], 1 # left = 1 → passe en second\n", " else:\n", " base, order = name, 0\n", " return (base, order)\n", "\n", "\n", "# ========== OCR ==========\n", "\n", "def call_vllm_ocr(image_path, retries=3):\n", " for attempt in range(1, retries + 1):\n", " try:\n", " with open(image_path, \"rb\") as f:\n", " img_b64 = base64.b64encode(f.read()).decode(\"utf-8\")\n", " payload = {\n", " \"model\": MODEL_PATH,\n", " \"messages\": [{\"role\": \"user\", \"content\": [\n", " {\"type\": \"image_url\", \"image_url\": {\"url\": f\"data:image/png;base64,{img_b64}\"}},\n", " {\"type\": \"text\", \"text\": OCR_PROMPT}\n", " ]}],\n", " **GENERATION_PARAMS,\n", " }\n", " resp = requests.post(VLLM_API_URL, json=payload, timeout=REQUEST_TIMEOUT_SECONDS)\n", " resp.raise_for_status()\n", " data = resp.json()\n", " if \"choices\" in data and data[\"choices\"]:\n", " return data[\"choices\"][0][\"message\"][\"content\"]\n", " raise Exception(f\"Format de réponse inattendu : {data}\")\n", " except requests.exceptions.Timeout:\n", " print(f\" Timeout tentative {attempt}/{retries} ({image_path})\")\n", " if attempt < retries: time.sleep(5)\n", " except Exception as e:\n", " print(f\" Erreur tentative {attempt}/{retries} ({image_path}): {e}\")\n", " if attempt < retries: time.sleep(3)\n", " raise Exception(f\"OCR échoué après {retries} tentatives : {image_path}\")\n", "\n", "\n", "def process_image(input_path, output_txt_path):\n", " if os.path.exists(output_txt_path):\n", " return # déjà traité\n", " try:\n", " text = call_vllm_ocr(input_path)\n", " with open(output_txt_path, \"w\", encoding=\"utf-8\") as f:\n", " f.write(text)\n", " except Exception as e:\n", " with open(output_txt_path, \"w\", encoding=\"utf-8\") as f:\n", " f.write(f\"[OCR FAILED: {e}]\")\n", "\n", "\n", "def clean_unnecessary_linebreaks(text):\n", " lines = text.split(\"\\n\")\n", " out = []\n", " for i, line in enumerate(lines):\n", " if line.startswith(\"=== Page\"):\n", " out.append(line)\n", " if i + 1 < len(lines) and lines[i + 1].strip() == \"\":\n", " out.append(\"\")\n", " else:\n", " prev_is_page_header = i > 0 and lines[i - 1].startswith(\"=== Page\")\n", " prev_ends_sentence = i > 0 and lines[i - 1].rstrip().endswith(\".\")\n", " if not prev_is_page_header and not prev_ends_sentence and out and out[-1] != \"\":\n", " out[-1] = out[-1].rstrip() + \" \" + line.lstrip()\n", " else:\n", " out.append(line)\n", " return \"\\n\".join(out)\n", "\n", "\n", "# ========== PIPELINE PRINCIPAL ==========\n", "\n", "def process_subfolder(subfolder_path, subfolder_name):\n", " \"\"\"\n", " Traite un sous-dossier : OCR direct sur les PNG → .md\n", " \"\"\"\n", " print(f\"\\n{'='*60}\")\n", " print(f\"Traitement : {subfolder_name}\")\n", " print(f\"{'='*60}\")\n", "\n", " txt_dir = os.path.join(TXT_TEMP_FOLDER, subfolder_name)\n", " output_md = os.path.join(OUTPUT_FOLDER, subfolder_name + \".md\")\n", " os.makedirs(txt_dir, exist_ok=True)\n", " os.makedirs(OUTPUT_FOLDER, exist_ok=True)\n", "\n", " if os.path.exists(output_md):\n", " print(f\" → Déjà traité, on passe.\")\n", " return\n", "\n", " # ── Étape 1 : Récupération et tri des images ───────────────────────────────\n", " png_files = sorted(\n", " glob.glob(os.path.join(subfolder_path, \"*.png\")),\n", " key=arabic_sort_key\n", " )\n", " if not png_files:\n", " print(f\" ⚠️ Aucune image PNG trouvée dans {subfolder_path}\")\n", " return\n", "\n", " # ── Étape 2 : OCR en parallèle ────────────────────────────────────────────\n", " tasks = [\n", " (img_p, os.path.join(txt_dir, os.path.splitext(os.path.basename(img_p))[0] + \".txt\"))\n", " for img_p in png_files\n", " ]\n", "\n", " print(f\" {len(png_files)} image(s) à envoyer à Chandra-2...\")\n", " with ThreadPoolExecutor(max_workers=NUM_WORKERS) as executor:\n", " futures = {executor.submit(process_image, ip, op): (ip, op) for ip, op in tasks}\n", " for future in tqdm(as_completed(futures), total=len(futures),\n", " desc=\" OCR\", leave=False):\n", " try:\n", " future.result()\n", " except Exception as e:\n", " print(f\" ❌ Erreur non gérée : {e}\")\n", "\n", " # ── Étape 3 : Assemblage en Markdown ──────────────────────────────────────\n", " parts = []\n", " for i, img_p in enumerate(png_files, 1):\n", " img_name = os.path.splitext(os.path.basename(img_p))[0]\n", " txt_path = os.path.join(txt_dir, img_name + \".txt\")\n", " if os.path.exists(txt_path):\n", " content = open(txt_path, encoding=\"utf-8\").read()\n", " else:\n", " content = \"[OCR failed for this page]\"\n", " parts.append(f\"=== Page {i} ({img_name}) ===\\n{content}\\n\")\n", "\n", " final = clean_unnecessary_linebreaks(\"\\n\".join(parts))\n", " with open(output_md, \"w\", encoding=\"utf-8\") as f:\n", " f.write(final)\n", "\n", " print(f\" ✅ Markdown sauvegardé → {output_md}\")\n", "\n", "\n", "def run_pipeline():\n", " \"\"\"\n", " Point d'entrée principal.\n", " Parcourt tous les sous-dossiers de IMAGES_DATASET_FOLDER,\n", " affiche un récapitulatif complet, puis lance le traitement.\n", " \"\"\"\n", " subfolders = sorted([\n", " d for d in os.listdir(IMAGES_DATASET_FOLDER)\n", " if os.path.isdir(os.path.join(IMAGES_DATASET_FOLDER, d))\n", " ])\n", "\n", " if not subfolders:\n", " print(f\"Aucun sous-dossier trouvé dans {IMAGES_DATASET_FOLDER}\")\n", " return\n", "\n", " # ── Scan préalable ────────────────────────────────────────────────────────\n", " print(f\"\\n{'='*60}\")\n", " print(f\" SCAN DU DATASET\")\n", " print(f\"{'='*60}\")\n", " print(f\" Dossier racine : {IMAGES_DATASET_FOLDER}\\n\")\n", "\n", " folder_image_counts = {}\n", " max_name_len = max(len(d) for d in subfolders)\n", "\n", " for name in subfolders:\n", " path = os.path.join(IMAGES_DATASET_FOLDER, name)\n", " pngs = sorted(glob.glob(os.path.join(path, \"*.png\")))\n", " folder_image_counts[name] = pngs\n", " status = f\"{len(pngs):>4} image(s)\" if pngs else \" ── aucune image PNG\"\n", " print(f\" 📁 {name:<{max_name_len}} → {status}\")\n", "\n", " total_images = sum(len(v) for v in folder_image_counts.values())\n", " total_folders = len(subfolders)\n", " empty_folders = sum(1 for v in folder_image_counts.values() if not v)\n", " active_folders = total_folders - empty_folders\n", "\n", " print(f\"\\n{'─'*60}\")\n", " print(f\" Sous-dossiers trouvés : {total_folders}\")\n", " print(f\" Sous-dossiers avec PNG : {active_folders}\")\n", " print(f\" Sous-dossiers vides : {empty_folders}\")\n", " print(f\" Total images PNG : {total_images}\")\n", " print(f\"{'─'*60}\")\n", " print(f\" Sortie markdown : {OUTPUT_FOLDER}\")\n", " print(f\"{'='*60}\\n\")\n", "\n", " if total_images == 0:\n", " print(\"Aucune image à traiter. Arrêt.\")\n", " return\n", "\n", " # ── Traitement ────────────────────────────────────────────────────────────\n", " for name in tqdm(subfolders, desc=\"Sous-dossiers\"):\n", " if not folder_image_counts[name]:\n", " continue\n", " path = os.path.join(IMAGES_DATASET_FOLDER, name)\n", " try:\n", " process_subfolder(path, name)\n", " except Exception as e:\n", " print(f\"❌ Erreur sur {name} : {e}\")\n", "\n", " print(f\"\\n{'='*60}\")\n", " print(\"PIPELINE TERMINÉ\")\n", " print(f\" {active_folders} sous-dossier(s) traité(s), {total_images} image(s) au total\")\n", " print(f\" Résultats dans : {OUTPUT_FOLDER}\")\n", " print(f\"{'='*60}\")\n", "\n", "\n", "# ========== LANCEMENT ==========\n", "\n", "if __name__ == \"__main__\":\n", " run_pipeline()" ] }, { "cell_type": "markdown", "id": "e1d3b382-fe68-4373-a565-a99c455f49a5", "metadata": { "jp-MarkdownHeadingCollapsed": true }, "source": [ "## html to md " ] }, { "cell_type": "code", "execution_count": null, "id": "c9069726-b479-4e0b-8f80-c0a64403943d", "metadata": {}, "outputs": [], "source": [ "# exectued tomd.pbs\n", "from pathlib import Path\n", "from markdownify import markdownify as md\n", "import re\n", "import time\n", "\n", "def convert_markdown_file(file_path, output_folder=None):\n", " \"\"\"Convert HTML in markdown file to proper markdown.\"\"\"\n", " print(f\"Processing: {file_path.name}\")\n", "\n", " # Read the file\n", " content = file_path.read_text(encoding='utf-8')\n", "\n", " # Split by page sections\n", " page_pattern = r'(=== Page \\d+ \\([^)]+\\) ===)'\n", " sections = re.split(page_pattern, content)\n", "\n", " # Convert HTML sections to markdown\n", " converted = []\n", " for section in sections:\n", " if section.startswith('=== Page'):\n", " converted.append(section)\n", " elif section.strip():\n", " # Convert HTML to markdown\n", " markdown = md(section, heading_style=\"ATX\")\n", " converted.append('\\n' + markdown.strip() + '\\n')\n", "\n", " # Save the result\n", " output_path = output_folder / file_path.name if output_folder else file_path.parent / f\"{file_path.stem}_converted.md\"\n", " output_path.write_text('\\n'.join(converted), encoding='utf-8')\n", "\n", " print(f\"✓ Saved: {output_path.name}\")\n", "\n", "def convert_all_files(input_folder, output_folder=None, batch_size=50, wait_seconds=5):\n", " \"\"\"Convert all markdown files in folder with batching.\"\"\"\n", " input_path = Path(input_folder)\n", " output_path = Path(output_folder) if output_folder else None\n", "\n", " if output_path:\n", " output_path.mkdir(parents=True, exist_ok=True)\n", "\n", " md_files = list(input_path.glob('*.md'))\n", " total_files = len(md_files)\n", " print(f\"Found {total_files} files\")\n", " print(f\"Processing in batches of {batch_size} with {wait_seconds}s wait\\n\")\n", "\n", " converted = 0\n", " failed = 0\n", "\n", " for i, md_file in enumerate(md_files, 1):\n", " try:\n", " convert_markdown_file(md_file, output_path)\n", " converted += 1\n", " except Exception as e:\n", " print(f\"✗ Error: {md_file.name} - {e}\")\n", " failed += 1\n", "\n", " # Batch wait\n", " if i % batch_size == 0 and i < total_files:\n", " print(f\"\\n--- Processed {i}/{total_files} files ---\")\n", " print(f\"Waiting {wait_seconds} seconds...\\n\")\n", " time.sleep(wait_seconds)\n", "\n", " print(f\"\\n{'='*60}\")\n", " print(f\"✓ Complete! Converted: {converted} | Failed: {failed} | Total: {total_files}\")\n", "\n", "if __name__ == \"__main__\":\n", " input_folder = r\"/home/skiredj.abderrahman/khalil/Benchmarking_OCR/final_resulsts_benchmark/chandra1_nosplit/output_markdown\"\n", " output_folder = r\"/home/skiredj.abderrahman/khalil/Benchmarking_OCR/final_resulsts_benchmark/chandra1_nosplit/output_markdown_no_html\"\n", "\n", " convert_all_files(input_folder, output_folder, batch_size=25, wait_seconds=10)\n", "\n", " # Option 2: Save to new folder with custom batch settings (uncomment)\n", " # output_folder = r\"C:\\...\\converted_markdowns\"\n", " # convert_all_files(input_folder, output_folder, batch_size=100, wait_seconds=3)\n" ] }, { "cell_type": "markdown", "id": "365ecf8f-d91a-49ba-ba8e-4b9dc49d2993", "metadata": { "jp-MarkdownHeadingCollapsed": true }, "source": [ "## no page n " ] }, { "cell_type": "code", "execution_count": null, "id": "9c5c9376-4e25-4a5f-92d9-272b000aa60e", "metadata": {}, "outputs": [], "source": [ "\n", "import os\n", "import re\n", "from pathlib import Path\n", "\n", "# ── Change these two paths ──────────────────────────────────────────\n", "INPUT_FOLDER = \"/home/skiredj.abderrahman/khalil/Benchmarking_OCR/final_resulsts_benchmark/chandra1_nosplit/output_markdown_no_html\"\n", "OUTPUT_FOLDER = \"/home/skiredj.abderrahman/khalil/Benchmarking_OCR/final_resulsts_benchmark/chandra1_nosplit/output_markdown_no_html_no_pageN\"\n", "# ────────────────────────────────────────────────────────────────────\n", "\n", "def parse_pages(content):\n", " pattern = r'(=== Page (\\d+) \\(([^)]+)\\) ===)'\n", " parts = re.split(pattern, content)\n", " \n", " preamble = parts[0]\n", " pages = []\n", " i = 1\n", " while i < len(parts):\n", " pages.append({\n", " 'header': parts[i],\n", " 'num': int(parts[i+1]),\n", " 'id': parts[i+2],\n", " 'body': parts[i+3]\n", " })\n", " i += 4\n", " return preamble, pages\n", "\n", "def merge_pages(pages):\n", " merged = []\n", " skip_ids = set()\n", " \n", " for i, page in enumerate(pages):\n", " pid = page['id']\n", " if pid in skip_ids:\n", " continue\n", " \n", " if pid.endswith('_right'):\n", " base_id = pid[:-len('_right')]\n", " left_page = None\n", " for j in range(i+1, len(pages)):\n", " if pages[j]['id'] == base_id + '_left':\n", " left_page = pages[j]\n", " skip_ids.add(pages[j]['id'])\n", " break\n", " \n", " combined_body = (page['body'].rstrip() + '\\n' + left_page['body']) if left_page else page['body']\n", " merged.append({'id': base_id, 'body': combined_body})\n", "\n", " elif pid.endswith('_left'):\n", " # Orphan left — strip suffix, keep content\n", " merged.append({'id': pid[:-len('_left')], 'body': page['body']})\n", " else:\n", " merged.append({'id': pid, 'body': page['body']})\n", " \n", " # Renumber sequentially\n", " return [{'num': n, 'id': p['id'], 'body': p['body']} for n, p in enumerate(merged, start=1)]\n", "\n", "def rebuild_content(preamble, pages):\n", " result = preamble\n", " for page in pages:\n", " result += f\"=== {page['id']} ===\\n\"\n", " result += page['body']\n", " return result\n", "\n", "def process_folder(input_folder, output_folder):\n", " input_path = Path(input_folder)\n", " output_path = Path(output_folder)\n", " output_path.mkdir(parents=True, exist_ok=True)\n", " \n", " md_files = list(input_path.glob('*.md'))\n", " if not md_files:\n", " print(f\"No .md files found in {input_folder}\")\n", " return\n", " \n", " for md_file in md_files:\n", " content = md_file.read_text(encoding='utf-8')\n", " preamble, pages = parse_pages(content)\n", " \n", " if not pages:\n", " (output_path / md_file.name).write_text(content, encoding='utf-8')\n", " print(f\"{md_file.name} — no pages found, copied as-is.\")\n", " continue\n", " \n", " merged = merge_pages(pages)\n", " output_path.joinpath(md_file.name).write_text(rebuild_content(preamble, merged), encoding='utf-8')\n", " \n", " print(f\"{md_file.name} — {len(pages)} pages → {len(merged)} pages ({len(pages)-len(merged)} pair(s) merged)\")\n", " \n", " print(f\"\\n✅ Done! Files saved to: {output_path.resolve()}\")\n", "\n", "process_folder(INPUT_FOLDER, OUTPUT_FOLDER)" ] }, { "cell_type": "markdown", "id": "269ad709-b4ca-4eca-a1e5-ab82efbb063c", "metadata": { "jp-MarkdownHeadingCollapsed": true }, "source": [ "## benchmark " ] }, { "cell_type": "code", "execution_count": null, "id": "c0308544-4e61-4bc5-9928-2bfc919ea388", "metadata": {}, "outputs": [], "source": [ "import os\n", "import re\n", "import unicodedata\n", "import html as _html\n", "import json\n", "import editdistance\n", "import jiwer\n", "import sacrebleu\n", "from rouge_score import rouge_scorer as rs\n", "from apted import APTED, Config\n", "from lxml import etree\n", "from typing import Optional\n", "from pathlib import Path\n", "\n", "# ── Paths ─────────────────────────────────────────────────────────────────────\n", "GT_FOLDER = Path(\"/home/skiredj.abderrahman/khalil/Benchmarking_OCR/ground_truth_claude_100_images/GT\")\n", "HYP_FOLDER = Path(\"/home/skiredj.abderrahman/khalil/Benchmarking_OCR/final_resulsts_benchmark/chandra1_nosplit/output_markdown_no_html_no_pageN\")\n", "MODEL_NAME = \"chandra1_without_split\"\n", "\n", "# ── Normalization ─────────────────────────────────────────────────────────────\n", "def normalize_arabic_ocr(text: str, remove_markdown: bool = False) -> str:\n", " if not isinstance(text, str) or not text.strip():\n", " return \"\"\n", " if remove_markdown:\n", " text = re.sub(r'#{1,6}\\s*', '', text)\n", " text = re.sub(r'^\\s*[-*+]\\s+', '', text, flags=re.MULTILINE)\n", " text = re.sub(r'\\*{1,3}(.*?)\\*{1,3}', r'\\1', text, flags=re.DOTALL)\n", " text = re.sub(r'`{1,3}.*?`{1,3}', ' ', text, flags=re.DOTALL)\n", " text = re.sub(r'\\[([^\\]]*)\\]\\([^\\)]*\\)', r'\\1', text)\n", " text = re.sub(r'!\\[[^\\]]*\\]\\([^\\)]*\\)', ' ', text)\n", " text = re.sub(r'\\|', ' ', text)\n", " text = unicodedata.normalize('NFKC', text)\n", " text = re.sub(r'[\\u200B-\\u200F\\u202A-\\u202E\\u2060-\\u2064\\uFEFF\\u00AD]', '', text)\n", " text = re.sub(\n", " r'[\\u064B-\\u065F\\u0610-\\u061A\\u06D6-\\u06DC\\u06DF-\\u06E4\\u06E7\\u06E8\\u06EA-\\u06ED]',\n", " '', text\n", " )\n", " text = text.replace('\\u0640', '')\n", " text = re.sub(r'[\\u0622\\u0623\\u0625\\u0671\\u0672\\u0673\\u0675]', '\\u0627', text)\n", " text = text.replace('\\u0649', '\\u064A')\n", " text = text.replace('\\u0629', '\\u0647')\n", " text = text.replace('\\u0624', '\\u0648')\n", " text = text.replace('\\u0626', '\\u064A')\n", " text = re.sub(r'[\\u060C\\u061B\\u061F\\u066A-\\u066D\\u0600-\\u0605]', ' ', text)\n", " text = re.sub(r'[.,:;()\\[\\]{}\\-«»\"\\'!?/\\\\~@#$%^&*_+=<>|]', ' ', text)\n", " text = text.lower()\n", " text = re.sub(r'[^\\u0600-\\u06FF\\u0750-\\u077Fa-z0-9\\s]', ' ', text)\n", " text = text.translate(str.maketrans('٠١٢٣٤٥٦٧٨٩', '0123456789'))\n", " text = text.lower()\n", " return re.sub(r'\\s+', ' ', text).strip()\n", "\n", "# ── TEDS helpers ──────────────────────────────────────────────────────────────\n", "def _is_separator_row(row: str) -> bool:\n", " cells = [c.strip() for c in row.strip().strip('|').split('|')]\n", " return bool(cells) and all(re.match(r'^[-:\\s]+$', c) for c in cells if c)\n", "\n", "\n", "def _expand_inline_table(line: str) -> Optional[str]:\n", " if not re.search(r'\\|\\s*-{2,}[\\s:]*\\|', line):\n", " return None\n", " raw_cells = [c.strip() for c in line.split('|')]\n", " while raw_cells and raw_cells[0] == '':\n", " raw_cells.pop(0)\n", " while raw_cells and raw_cells[-1] == '':\n", " raw_cells.pop()\n", " if not raw_cells:\n", " return None\n", " sep_positions = [i for i, c in enumerate(raw_cells) if re.match(r'^[-:\\s]+$', c) and len(c) >= 2]\n", " if not sep_positions:\n", " return None\n", " n_cols = len(sep_positions)\n", " sep_start = sep_positions[0]\n", " header_cells = raw_cells[:sep_start]\n", " if len(header_cells) > n_cols:\n", " header_cells = header_cells[len(header_cells) - n_cols:]\n", " elif len(header_cells) < n_cols:\n", " header_cells = [''] * (n_cols - len(header_cells)) + header_cells\n", " data_cells = raw_cells[sep_start + n_cols:]\n", " rows = []\n", " for i in range(0, len(data_cells), n_cols):\n", " chunk = data_cells[i:i + n_cols]\n", " while len(chunk) < n_cols:\n", " chunk.append('')\n", " rows.append(chunk)\n", " if not rows:\n", " return None\n", " sep_row = '| ' + ' | '.join(['---'] * n_cols) + ' |'\n", " header_row = '| ' + ' | '.join(header_cells) + ' |'\n", " data_rows = ['| ' + ' | '.join(r) + ' |' for r in rows]\n", " return '\\n'.join([header_row, sep_row] + data_rows)\n", "\n", "\n", "def extract_md_tables(text: str) -> list:\n", " tables = []\n", " lines = text.split('\\n')\n", " cur = []\n", " for line in lines:\n", " stripped = line.strip()\n", " if re.search(r'\\|\\s*-{2,}[\\s:]*\\|', stripped):\n", " if len(cur) >= 2:\n", " tables.append('\\n'.join(cur))\n", " cur = []\n", " expanded = _expand_inline_table(stripped)\n", " if expanded:\n", " tables.append(expanded)\n", " continue\n", " if stripped.startswith('|') or stripped.count('|') >= 2:\n", " cur.append(stripped)\n", " else:\n", " if len(cur) >= 2:\n", " tables.append('\\n'.join(cur))\n", " cur = []\n", " if len(cur) >= 2:\n", " tables.append('\\n'.join(cur))\n", " return tables\n", "\n", "\n", "def md_table_to_html(md: str) -> str:\n", " lines = [l.strip() for l in md.strip().split('\\n') if l.strip()]\n", " lines = [l for l in lines if not _is_separator_row(l)]\n", " if not lines:\n", " return '
'\n", " rows = []\n", " for i, line in enumerate(lines):\n", " cells = [re.sub(r'\\s+', ' ', c.strip()) for c in line.strip('|').split('|')]\n", " tag = 'th' if i == 0 else 'td'\n", " rows.append('' + ''.join(f'<{tag}>{_html.escape(c)}' for c in cells) + '')\n", " return '' + ''.join(rows) + '
'\n", "\n", "\n", "class AptedConfig(Config):\n", " def rename(self, node1, node2):\n", " t1 = re.sub(r'\\s+', ' ', (node1.text or '').strip())\n", " t2 = re.sub(r'\\s+', ' ', (node2.text or '').strip())\n", " return 0 if (node1.tag == node2.tag and t1 == t2) else 1\n", " def children(self, node):\n", " return list(node)\n", "\n", "\n", "def teds_score(ref_html: str, hyp_html: str) -> Optional[float]:\n", " try:\n", " rt = etree.fromstring(f'{ref_html}')\n", " ht = etree.fromstring(f'{hyp_html}')\n", " dist = APTED(rt, ht, AptedConfig()).compute_edit_distance()\n", " n = max(len(list(rt.iter())), len(list(ht.iter())))\n", " return round((1.0 - dist / n) * 100, 2) if n > 0 else 0.0\n", " except Exception as e:\n", " print(f'TEDS error: {e}')\n", " return None\n", "\n", "# ── Per-file evaluation ───────────────────────────────────────────────────────\n", "def evaluate_page(ref_raw: str, hyp_raw: str) -> dict:\n", " ref = normalize_arabic_ocr(ref_raw, remove_markdown=True)\n", " hyp = normalize_arabic_ocr(hyp_raw, remove_markdown=True)\n", "\n", " if not ref or not hyp:\n", " return {'status': 'empty_after_norm'}\n", "\n", " rouge_sc = rs.RougeScorer(['rouge1', 'rougeL'], use_stemmer=False)\n", " pg = {'status': 'ok', 'ref_chars': len(ref_raw), 'hyp_chars': len(hyp_raw)}\n", "\n", " # NED\n", " try:\n", " ed = editdistance.eval(ref, hyp)\n", " pg['NED'] = round(ed / max(len(ref), len(hyp), 1) * 100, 2)\n", " except:\n", " pg['NED'] = None\n", "\n", " # CER / WER — jiwer\n", " try:\n", " tf = jiwer.Compose([jiwer.Strip(), jiwer.ReduceToListOfListOfWords()])\n", " pg['CER_jiwer'] = round(jiwer.cer(ref, hyp) * 100, 2)\n", " pg['WER_jiwer'] = round(jiwer.wer(ref, hyp, reference_transform=tf, hypothesis_transform=tf) * 100, 2)\n", " except:\n", " pg['CER_jiwer'] = pg['WER_jiwer'] = None\n", "\n", " # CER / WER — dinglehopper\n", " try:\n", " from dinglehopper.character_error_rate import character_error_rate as dce\n", " from dinglehopper.word_error_rate import word_error_rate as dwe\n", " pg['CER_dh'] = round(float(dce(ref, hyp)) * 100, 2)\n", " pg['WER_dh'] = round(float(dwe(ref, hyp)) * 100, 2)\n", " except:\n", " pg['CER_dh'] = pg['WER_dh'] = None\n", "\n", " # BLEU\n", " try:\n", " pg['BLEU'] = round(sacrebleu.sentence_bleu(hyp, [ref]).score, 2)\n", " except:\n", " pg['BLEU'] = None\n", "\n", " # ROUGE\n", " try:\n", " r = rouge_sc.score(ref, hyp)\n", " pg['ROUGE_1'] = round(r['rouge1'].fmeasure * 100, 2)\n", " pg['ROUGE_L'] = round(r['rougeL'].fmeasure * 100, 2)\n", " except:\n", " pg['ROUGE_1'] = pg['ROUGE_L'] = None\n", "\n", " # TEDS\n", " ref_tables = extract_md_tables(ref_raw)\n", " hyp_tables = extract_md_tables(hyp_raw)\n", " pg['n_tables_ref'] = len(ref_tables)\n", " pg['n_tables_hyp'] = len(hyp_tables)\n", " if ref_tables:\n", " teds_list = []\n", " for i, rt in enumerate(ref_tables):\n", " score = teds_score(md_table_to_html(rt), md_table_to_html(hyp_tables[i])) \\\n", " if i < len(hyp_tables) else 0.0\n", " teds_list.append(score if score is not None else 0.0)\n", " pg['TEDS'] = round(sum(teds_list) / len(teds_list), 2)\n", " else:\n", " pg['TEDS'] = None\n", "\n", " return pg\n", "\n", "# ── Page splitter ─────────────────────────────────────────────────────────────\n", "def split_pages_raw(content: str) -> dict:\n", " pattern = re.compile(r'^===\\s*(.+?)\\s*===$', re.MULTILINE)\n", " matches = list(pattern.finditer(content))\n", " if not matches:\n", " print(\"⚠️ No page separators found\")\n", " return {\"page_01\": content.strip()}\n", " pages = {}\n", " for idx, m in enumerate(matches):\n", " page_id = m.group(1).strip()\n", " start = m.end()\n", " end = matches[idx + 1].start() if idx + 1 < len(matches) else len(content)\n", " text = content[start:end].strip()\n", " if text:\n", " pages[page_id] = text\n", " return pages\n", "\n", "# ── Discover and match files ──────────────────────────────────────────────────\n", "gt_files = {f.name: f for f in sorted(GT_FOLDER.glob('*.md'))}\n", "hyp_files = {f.name: f for f in sorted(HYP_FOLDER.glob('*.md'))}\n", "\n", "print(f'GT folder : {len(gt_files)} .md files found')\n", "print(f'HYP folder : {len(hyp_files)} .md files found')\n", "\n", "common = sorted(set(gt_files) & set(hyp_files))\n", "only_gt = set(gt_files) - set(hyp_files)\n", "only_hyp = set(hyp_files) - set(gt_files)\n", "\n", "if only_gt:\n", " print(f'⚠️ Only in GT : {sorted(only_gt)}')\n", "if only_hyp:\n", " print(f'⚠️ Only in HYP : {sorted(only_hyp)}')\n", "print(f'\\n✅ Matched files to evaluate : {len(common)}')\n", "\n", "# ── Evaluate each matched pair — page by page ─────────────────────────────────\n", "# per_page stores ALL pages across ALL files: key = \"filename::page_id\"\n", "per_page = {}\n", "\n", "for fname in common:\n", " gt_raw = gt_files[fname].read_text(encoding='utf-8').strip()\n", " hyp_raw = hyp_files[fname].read_text(encoding='utf-8').strip()\n", "\n", " gt_pages = split_pages_raw(gt_raw)\n", " hyp_pages = split_pages_raw(hyp_raw)\n", "\n", " common_pages = sorted(set(gt_pages) & set(hyp_pages))\n", " only_gt_p = set(gt_pages) - set(hyp_pages)\n", " only_hyp_p = set(hyp_pages) - set(gt_pages)\n", "\n", " print(f'\\n 📄 {fname} → GT:{len(gt_pages)} pages HYP:{len(hyp_pages)} pages matched:{len(common_pages)}')\n", " if only_gt_p:\n", " print(f' ⚠️ Only in GT : {sorted(only_gt_p)}')\n", " if only_hyp_p:\n", " print(f' ⚠️ Only in HYP : {sorted(only_hyp_p)}')\n", "\n", " for page_id in common_pages:\n", " key = f\"{fname}::{page_id}\"\n", " result = evaluate_page(gt_pages[page_id], hyp_pages[page_id])\n", " per_page[key] = result\n", " if result.get('status') == 'ok':\n", " print(f' ✓ {page_id} NED:{result[\"NED\"]}% CER:{result[\"CER_jiwer\"]}% WER:{result[\"WER_jiwer\"]}%')\n", " else:\n", " print(f' ⚠️ {page_id} → {result.get(\"status\")}')\n", "\n", "# ── Global averages across all pages of all files ─────────────────────────────\n", "def avg(k):\n", " vals = [v[k] for v in per_page.values()\n", " if isinstance(v, dict) and v.get('status') == 'ok' and v.get(k) is not None]\n", " return round(sum(vals) / len(vals), 2) if vals else None\n", "\n", "overall = {k: avg(k) for k in ['NED','CER_jiwer','WER_jiwer','CER_dh','WER_dh','BLEU','ROUGE_1','ROUGE_L','TEDS']}\n", "total_pages = sum(1 for v in per_page.values() if isinstance(v, dict) and v.get('status') == 'ok')\n", "\n", "# ── Print final results ───────────────────────────────────────────────────────\n", "SEP = '=' * 60\n", "print(f'\\n{SEP}')\n", "print(f' {MODEL_NAME} — {len(common)} files / {total_pages} pages evaluated')\n", "print(SEP)\n", "print(f\" NED : {overall['NED']}% [↓ principal]\")\n", "print(f\" CER_jiwer : {overall['CER_jiwer']}% WER_jiwer : {overall['WER_jiwer']}%\")\n", "print(f\" CER_dh : {overall['CER_dh']}% WER_dh : {overall['WER_dh']}%\")\n", "print(f\" BLEU : {overall['BLEU']} ROUGE-1 : {overall['ROUGE_1']}% ROUGE-L : {overall['ROUGE_L']}%\")\n", "print(f\" TEDS : {overall['TEDS']}% [None = no tables in GT page]\")\n", "print(SEP)\n", "\n", "print(f\"\\n{'─'*60}\")\n", "print(f' Per-page breakdown')\n", "print(f\"{'─'*60}\")\n", "for key, pg in per_page.items():\n", " fname, page_id = key.split('::', 1)\n", " if pg.get('status') == 'ok':\n", " print(f'\\n [{fname}] {page_id}')\n", " print(f\" NED:{pg['NED']}% CER:{pg['CER_jiwer']}% WER:{pg['WER_jiwer']}% BLEU:{pg['BLEU']} ROUGE-1:{pg['ROUGE_1']}% TEDS:{pg['TEDS']}%\")\n", " if pg['n_tables_ref'] or pg['n_tables_hyp']:\n", " print(f\" tables → GT:{pg['n_tables_ref']} HYP:{pg['n_tables_hyp']}\")\n", " else:\n", " print(f\"\\n [{fname}] {page_id} ⚠️ {pg.get('status')}\")\n", "\n", "# ── Save results to JSON ──────────────────────────────────────────────────────\n", "RESULTS_DIR = \"/home/skiredj.abderrahman/khalil/Benchmarking_OCR/final_resulsts_benchmark/\"\n", "os.makedirs(RESULTS_DIR, exist_ok=True)\n", "\n", "def save_results(report: dict):\n", " name = report[\"model\"].replace(\" \", \"_\")\n", " path = os.path.join(RESULTS_DIR, f\"{name}.json\")\n", " json.dump(report, open(path, \"w\", encoding=\"utf-8\"), ensure_ascii=False, indent=2)\n", " print(f\"\\n✅ Sauvegardé : {path}\")\n", "\n", "report = {\n", " \"model\" : MODEL_NAME,\n", " \"n_files\" : len(common),\n", " \"n_pages\" : total_pages,\n", " \"overall\" : overall,\n", " \"per_page\": per_page\n", "}\n", "save_results(report)" ] }, { "cell_type": "markdown", "id": "2ececfe2-8ea1-4471-89a4-56c94e6052ed", "metadata": { "jp-MarkdownHeadingCollapsed": true }, "source": [ "# deepseek 1 " ] }, { "cell_type": "markdown", "id": "13cc920e-8989-4d08-896d-17841b38284a", "metadata": { "jp-MarkdownHeadingCollapsed": true }, "source": [ "## OCR" ] }, { "cell_type": "code", "execution_count": null, "id": "1b315f36-763b-41e2-bb5c-5e09c0ef7034", "metadata": {}, "outputs": [], "source": [ "# ============================================================================\n", "# PIPELINE OCR - IMAGES LOCALES (sans prétraitement)\n", "# Entrée : dossier avec sous-dossiers contenant des PNG (déjà zoom x4)\n", "# Sortie : un fichier .md par sous-dossier\n", "# ============================================================================\n", "\n", "import os\n", "import re\n", "import time\n", "import base64\n", "import glob\n", "import shutil\n", "from pathlib import Path\n", "from typing import Union, Optional\n", "from concurrent.futures import ThreadPoolExecutor, as_completed\n", "from tqdm import tqdm\n", "from openai import OpenAI\n", "\n", "# ========== CONFIGURATION ==========\n", "\n", "MODEL_PATH = \"/home/skiredj.abderrahman/khalil/DeepSeek-OCR\"\n", "\n", "client = OpenAI(\n", " api_key=\"EMPTY\",\n", " base_url=\"http://localhost:8000/v1\",\n", " timeout=3600\n", ")\n", "\n", "GENERATION_PARAMS = {\n", " \"max_tokens\": 2048,\n", " \"temperature\": 0.0,\n", " \"extra_body\": {\n", " \"skip_special_tokens\": False,\n", " \"vllm_xargs\": {\n", " \"ngram_size\": 30,\n", " \"window_size\": 90,\n", " \"whitelist_token_ids\": [128821, 128822],\n", " },\n", " },\n", "}\n", "\n", "REQUEST_TIMEOUT_SECONDS = 180\n", "NUM_WORKERS = 1\n", "\n", "OCR_PROMPT = \"Convert the document to markdown.\"\n", "\n", "# ── Dossiers ──────────────────────────────────────────────────────────────────\n", "IMAGES_DATASET_FOLDER = r\"/home/skiredj.abderrahman/khalil/Benchmarking_OCR/ground_truth_claude_100_images/100images\"\n", "OUTPUT_FOLDER = r\"/home/skiredj.abderrahman/khalil/Benchmarking_OCR/final_resulsts_benchmark/deepseek1_nosplit/output_markdown\"\n", "TXT_TEMP_FOLDER = r\"/home/skiredj.abderrahman/khalil/Benchmarking_OCR/final_resulsts_benchmark/deepseek1_nosplit/temp_txt\"\n", "\n", "\n", "# ========== TRI DES IMAGES (arabe : right avant left) ==========\n", "\n", "def arabic_sort_key(path):\n", " name = os.path.splitext(os.path.basename(path))[0]\n", " if name.endswith(\"_right\"):\n", " base, order = name[:-6], 0 # right = 0 → passe en premier\n", " elif name.endswith(\"_left\"):\n", " base, order = name[:-5], 1 # left = 1 → passe en second\n", " else:\n", " base, order = name, 0\n", " return (base, order)\n", "\n", "\n", "# ========== OCR ==========\n", "\n", "def encode_image(path):\n", " with open(path, \"rb\") as f:\n", " return base64.b64encode(f.read()).decode(\"utf-8\")\n", "\n", "\n", "def call_deepseek_ocr(image_path, retries=3):\n", " for attempt in range(1, retries + 1):\n", " try:\n", " base64_image = encode_image(image_path)\n", " messages = [\n", " {\n", " \"role\": \"user\",\n", " \"content\": [\n", " {\n", " \"type\": \"image_url\",\n", " \"image_url\": {\n", " \"url\": f\"data:image/png;base64,{base64_image}\"\n", " }\n", " },\n", " {\n", " \"type\": \"text\",\n", " \"text\": OCR_PROMPT\n", " }\n", " ]\n", " }\n", " ]\n", " response = client.chat.completions.create(\n", " model=MODEL_PATH,\n", " messages=messages,\n", " **GENERATION_PARAMS,\n", " )\n", " return response.choices[0].message.content\n", " except Exception as e:\n", " print(f\" Erreur tentative {attempt}/{retries} ({image_path}): {e}\")\n", " if attempt < retries:\n", " time.sleep(3)\n", " raise Exception(f\"OCR échoué après {retries} tentatives : {image_path}\")\n", "\n", "\n", "def process_image(input_path, output_txt_path):\n", " if os.path.exists(output_txt_path):\n", " return # déjà traité\n", " try:\n", " text = call_deepseek_ocr(input_path)\n", " with open(output_txt_path, \"w\", encoding=\"utf-8\") as f:\n", " f.write(text)\n", " except Exception as e:\n", " with open(output_txt_path, \"w\", encoding=\"utf-8\") as f:\n", " f.write(f\"[OCR FAILED: {e}]\")\n", "\n", "\n", "def _is_table_line(line: str) -> bool:\n", " \"\"\"True if the line is part of a markdown table (starts with | or has 2+ pipes).\"\"\"\n", " s = line.strip()\n", " return s.startswith(\"|\") or s.count(\"|\") >= 2\n", "\n", "\n", "def clean_unnecessary_linebreaks(text):\n", " lines = text.split(\"\\n\")\n", " out = []\n", " for i, line in enumerate(lines):\n", " if line.startswith(\"=== Page\"):\n", " out.append(line)\n", " if i + 1 < len(lines) and lines[i + 1].strip() == \"\":\n", " out.append(\"\")\n", " else:\n", " prev_is_page_header = i > 0 and lines[i - 1].startswith(\"=== Page\")\n", " prev_ends_sentence = i > 0 and lines[i - 1].rstrip().endswith(\".\")\n", "\n", " # Never merge table rows — doing so collapses the entire table onto\n", " # one line and breaks all downstream table detection / TEDS scoring.\n", " current_is_table = _is_table_line(line)\n", " prev_is_table = bool(out) and _is_table_line(out[-1])\n", "\n", " if (not prev_is_page_header\n", " and not prev_ends_sentence\n", " and not current_is_table\n", " and not prev_is_table\n", " and out\n", " and out[-1] != \"\"):\n", " out[-1] = out[-1].rstrip() + \" \" + line.lstrip()\n", " else:\n", " out.append(line)\n", " return \"\\n\".join(out)\n", "\n", "\n", "# ========== PIPELINE PRINCIPAL ==========\n", "\n", "def process_subfolder(subfolder_path, subfolder_name):\n", " print(f\"\\n{'='*60}\")\n", " print(f\"Traitement : {subfolder_name}\")\n", " print(f\"{'='*60}\")\n", "\n", " txt_dir = os.path.join(TXT_TEMP_FOLDER, subfolder_name)\n", " output_md = os.path.join(OUTPUT_FOLDER, subfolder_name + \".md\")\n", " os.makedirs(txt_dir, exist_ok=True)\n", " os.makedirs(OUTPUT_FOLDER, exist_ok=True)\n", "\n", " if os.path.exists(output_md):\n", " print(f\" → Déjà traité, on passe.\")\n", " return\n", "\n", " # ── Étape 1 : Récupération et tri des images ───────────────────────────────\n", " png_files = sorted(\n", " glob.glob(os.path.join(subfolder_path, \"*.png\")),\n", " key=arabic_sort_key\n", " )\n", " if not png_files:\n", " print(f\" ⚠️ Aucune image PNG trouvée dans {subfolder_path}\")\n", " return\n", "\n", " # ── Étape 2 : OCR en parallèle ────────────────────────────────────────────\n", " tasks = [\n", " (img_p, os.path.join(txt_dir, os.path.splitext(os.path.basename(img_p))[0] + \".txt\"))\n", " for img_p in png_files\n", " ]\n", "\n", " print(f\" {len(png_files)} image(s) à envoyer à DeepSeek-OCR...\")\n", " with ThreadPoolExecutor(max_workers=NUM_WORKERS) as executor:\n", " futures = {executor.submit(process_image, ip, op): (ip, op) for ip, op in tasks}\n", " for future in tqdm(as_completed(futures), total=len(futures),\n", " desc=\" OCR\", leave=False):\n", " try:\n", " future.result()\n", " except Exception as e:\n", " print(f\" ❌ Erreur non gérée : {e}\")\n", "\n", " # ── Étape 3 : Assemblage en Markdown ──────────────────────────────────────\n", " parts = []\n", " for i, img_p in enumerate(png_files, 1):\n", " img_name = os.path.splitext(os.path.basename(img_p))[0]\n", " txt_path = os.path.join(txt_dir, img_name + \".txt\")\n", " if os.path.exists(txt_path):\n", " content = open(txt_path, encoding=\"utf-8\").read()\n", " else:\n", " content = \"[OCR failed for this page]\"\n", " parts.append(f\"=== Page {i} ({img_name}) ===\\n{content}\\n\")\n", "\n", " final = clean_unnecessary_linebreaks(\"\\n\".join(parts))\n", " with open(output_md, \"w\", encoding=\"utf-8\") as f:\n", " f.write(final)\n", "\n", " print(f\" ✅ Markdown sauvegardé → {output_md}\")\n", "\n", "\n", "def run_pipeline():\n", " subfolders = sorted([\n", " d for d in os.listdir(IMAGES_DATASET_FOLDER)\n", " if os.path.isdir(os.path.join(IMAGES_DATASET_FOLDER, d))\n", " ])\n", "\n", " if not subfolders:\n", " print(f\"Aucun sous-dossier trouvé dans {IMAGES_DATASET_FOLDER}\")\n", " return\n", "\n", " # ── Scan préalable ────────────────────────────────────────────────────────\n", " print(f\"\\n{'='*60}\")\n", " print(f\" SCAN DU DATASET\")\n", " print(f\"{'='*60}\")\n", " print(f\" Dossier racine : {IMAGES_DATASET_FOLDER}\\n\")\n", "\n", " folder_image_counts = {}\n", " max_name_len = max(len(d) for d in subfolders)\n", "\n", " for name in subfolders:\n", " path = os.path.join(IMAGES_DATASET_FOLDER, name)\n", " pngs = sorted(glob.glob(os.path.join(path, \"*.png\")))\n", " folder_image_counts[name] = pngs\n", " status = f\"{len(pngs):>4} image(s)\" if pngs else \" ── aucune image PNG\"\n", " print(f\" 📁 {name:<{max_name_len}} → {status}\")\n", "\n", " total_images = sum(len(v) for v in folder_image_counts.values())\n", " total_folders = len(subfolders)\n", " empty_folders = sum(1 for v in folder_image_counts.values() if not v)\n", " active_folders = total_folders - empty_folders\n", "\n", " print(f\"\\n{'─'*60}\")\n", " print(f\" Sous-dossiers trouvés : {total_folders}\")\n", " print(f\" Sous-dossiers avec PNG : {active_folders}\")\n", " print(f\" Sous-dossiers vides : {empty_folders}\")\n", " print(f\" Total images PNG : {total_images}\")\n", " print(f\"{'─'*60}\")\n", " print(f\" Sortie markdown : {OUTPUT_FOLDER}\")\n", " print(f\"{'='*60}\\n\")\n", "\n", " if total_images == 0:\n", " print(\"Aucune image à traiter. Arrêt.\")\n", " return\n", "\n", " # ── Traitement ────────────────────────────────────────────────────────────\n", " for name in tqdm(subfolders, desc=\"Sous-dossiers\"):\n", " if not folder_image_counts[name]:\n", " continue\n", " path = os.path.join(IMAGES_DATASET_FOLDER, name)\n", " try:\n", " process_subfolder(path, name)\n", " except Exception as e:\n", " print(f\"❌ Erreur sur {name} : {e}\")\n", "\n", " print(f\"\\n{'='*60}\")\n", " print(\"PIPELINE TERMINÉ\")\n", " print(f\" {active_folders} sous-dossier(s) traité(s), {total_images} image(s) au total\")\n", " print(f\" Résultats dans : {OUTPUT_FOLDER}\")\n", " print(f\"{'='*60}\")\n", "\n", "\n", "# ========== LANCEMENT ==========\n", "\n", "if __name__ == \"__main__\":\n", " run_pipeline()" ] }, { "cell_type": "markdown", "id": "7f041447-43d0-4cbd-9984-0c435a4cf472", "metadata": { "jp-MarkdownHeadingCollapsed": true }, "source": [ "## no page n " ] }, { "cell_type": "code", "execution_count": null, "id": "e71d046c-fc0b-45d9-8209-45cb76c3f396", "metadata": {}, "outputs": [], "source": [ "\n", "import os\n", "import re\n", "from pathlib import Path\n", "\n", "# ── Change these two paths ──────────────────────────────────────────\n", "INPUT_FOLDER = \"/home/skiredj.abderrahman/khalil/Benchmarking_OCR/final_resulsts_benchmark/deepseek1_nosplit/output_markdown\"\n", "OUTPUT_FOLDER = \"/home/skiredj.abderrahman/khalil/Benchmarking_OCR/final_resulsts_benchmark/deepseek1_nosplit/output_markdown_no_pageN\"\n", "# ────────────────────────────────────────────────────────────────────\n", "\n", "def parse_pages(content):\n", " pattern = r'(=== Page (\\d+) \\(([^)]+)\\) ===)'\n", " parts = re.split(pattern, content)\n", " \n", " preamble = parts[0]\n", " pages = []\n", " i = 1\n", " while i < len(parts):\n", " pages.append({\n", " 'header': parts[i],\n", " 'num': int(parts[i+1]),\n", " 'id': parts[i+2],\n", " 'body': parts[i+3]\n", " })\n", " i += 4\n", " return preamble, pages\n", "\n", "def merge_pages(pages):\n", " merged = []\n", " skip_ids = set()\n", " \n", " for i, page in enumerate(pages):\n", " pid = page['id']\n", " if pid in skip_ids:\n", " continue\n", " \n", " if pid.endswith('_right'):\n", " base_id = pid[:-len('_right')]\n", " left_page = None\n", " for j in range(i+1, len(pages)):\n", " if pages[j]['id'] == base_id + '_left':\n", " left_page = pages[j]\n", " skip_ids.add(pages[j]['id'])\n", " break\n", " \n", " combined_body = (page['body'].rstrip() + '\\n' + left_page['body']) if left_page else page['body']\n", " merged.append({'id': base_id, 'body': combined_body})\n", "\n", " elif pid.endswith('_left'):\n", " # Orphan left — strip suffix, keep content\n", " merged.append({'id': pid[:-len('_left')], 'body': page['body']})\n", " else:\n", " merged.append({'id': pid, 'body': page['body']})\n", " \n", " # Renumber sequentially\n", " return [{'num': n, 'id': p['id'], 'body': p['body']} for n, p in enumerate(merged, start=1)]\n", "\n", "def rebuild_content(preamble, pages):\n", " result = preamble\n", " for page in pages:\n", " result += f\"=== {page['id']} ===\\n\"\n", " result += page['body']\n", " return result\n", "\n", "def process_folder(input_folder, output_folder):\n", " input_path = Path(input_folder)\n", " output_path = Path(output_folder)\n", " output_path.mkdir(parents=True, exist_ok=True)\n", " \n", " md_files = list(input_path.glob('*.md'))\n", " if not md_files:\n", " print(f\"No .md files found in {input_folder}\")\n", " return\n", " \n", " for md_file in md_files:\n", " content = md_file.read_text(encoding='utf-8')\n", " preamble, pages = parse_pages(content)\n", " \n", " if not pages:\n", " (output_path / md_file.name).write_text(content, encoding='utf-8')\n", " print(f\"{md_file.name} — no pages found, copied as-is.\")\n", " continue\n", " \n", " merged = merge_pages(pages)\n", " output_path.joinpath(md_file.name).write_text(rebuild_content(preamble, merged), encoding='utf-8')\n", " \n", " print(f\"{md_file.name} — {len(pages)} pages → {len(merged)} pages ({len(pages)-len(merged)} pair(s) merged)\")\n", " \n", " print(f\"\\n✅ Done! Files saved to: {output_path.resolve()}\")\n", "\n", "process_folder(INPUT_FOLDER, OUTPUT_FOLDER)" ] }, { "cell_type": "markdown", "id": "9f12f1a8-5649-49de-b7d0-97434c34ad80", "metadata": { "jp-MarkdownHeadingCollapsed": true }, "source": [ "## benchmark " ] }, { "cell_type": "code", "execution_count": null, "id": "71cc585d-ffa2-4c8e-9b44-36566b863dae", "metadata": {}, "outputs": [], "source": [ "import os\n", "import re\n", "import unicodedata\n", "import html as _html\n", "import json\n", "import editdistance\n", "import jiwer\n", "import sacrebleu\n", "from rouge_score import rouge_scorer as rs\n", "from apted import APTED, Config\n", "from lxml import etree\n", "from typing import Optional\n", "from pathlib import Path\n", "\n", "# ── Paths ─────────────────────────────────────────────────────────────────────\n", "GT_FOLDER = Path(\"/home/skiredj.abderrahman/khalil/Benchmarking_OCR/ground_truth_claude_100_images/GT\")\n", "HYP_FOLDER = Path(\"/home/skiredj.abderrahman/khalil/Benchmarking_OCR/final_resulsts_benchmark/deepseek1_nosplit/output_markdown_no_pageN\")\n", "MODEL_NAME = \"deepseek1_no_split\"\n", "\n", "# ── Normalization ─────────────────────────────────────────────────────────────\n", "def normalize_arabic_ocr(text: str, remove_markdown: bool = False) -> str:\n", " if not isinstance(text, str) or not text.strip():\n", " return \"\"\n", " if remove_markdown:\n", " text = re.sub(r'#{1,6}\\s*', '', text)\n", " text = re.sub(r'^\\s*[-*+]\\s+', '', text, flags=re.MULTILINE)\n", " text = re.sub(r'\\*{1,3}(.*?)\\*{1,3}', r'\\1', text, flags=re.DOTALL)\n", " text = re.sub(r'`{1,3}.*?`{1,3}', ' ', text, flags=re.DOTALL)\n", " text = re.sub(r'\\[([^\\]]*)\\]\\([^\\)]*\\)', r'\\1', text)\n", " text = re.sub(r'!\\[[^\\]]*\\]\\([^\\)]*\\)', ' ', text)\n", " text = re.sub(r'\\|', ' ', text)\n", " text = unicodedata.normalize('NFKC', text)\n", " text = re.sub(r'[\\u200B-\\u200F\\u202A-\\u202E\\u2060-\\u2064\\uFEFF\\u00AD]', '', text)\n", " text = re.sub(\n", " r'[\\u064B-\\u065F\\u0610-\\u061A\\u06D6-\\u06DC\\u06DF-\\u06E4\\u06E7\\u06E8\\u06EA-\\u06ED]',\n", " '', text\n", " )\n", " text = text.replace('\\u0640', '')\n", " text = re.sub(r'[\\u0622\\u0623\\u0625\\u0671\\u0672\\u0673\\u0675]', '\\u0627', text)\n", " text = text.replace('\\u0649', '\\u064A')\n", " text = text.replace('\\u0629', '\\u0647')\n", " text = text.replace('\\u0624', '\\u0648')\n", " text = text.replace('\\u0626', '\\u064A')\n", " text = re.sub(r'[\\u060C\\u061B\\u061F\\u066A-\\u066D\\u0600-\\u0605]', ' ', text)\n", " text = re.sub(r'[.,:;()\\[\\]{}\\-«»\"\\'!?/\\\\~@#$%^&*_+=<>|]', ' ', text)\n", " text = text.lower()\n", " text = re.sub(r'[^\\u0600-\\u06FF\\u0750-\\u077Fa-z0-9\\s]', ' ', text)\n", " text = text.translate(str.maketrans('٠١٢٣٤٥٦٧٨٩', '0123456789'))\n", " text = text.lower()\n", " return re.sub(r'\\s+', ' ', text).strip()\n", "\n", "# ── TEDS helpers ──────────────────────────────────────────────────────────────\n", "def _is_separator_row(row: str) -> bool:\n", " cells = [c.strip() for c in row.strip().strip('|').split('|')]\n", " return bool(cells) and all(re.match(r'^[-:\\s]+$', c) for c in cells if c)\n", "\n", "\n", "def _expand_inline_table(line: str) -> Optional[str]:\n", " if not re.search(r'\\|\\s*-{2,}[\\s:]*\\|', line):\n", " return None\n", " raw_cells = [c.strip() for c in line.split('|')]\n", " while raw_cells and raw_cells[0] == '':\n", " raw_cells.pop(0)\n", " while raw_cells and raw_cells[-1] == '':\n", " raw_cells.pop()\n", " if not raw_cells:\n", " return None\n", " sep_positions = [i for i, c in enumerate(raw_cells) if re.match(r'^[-:\\s]+$', c) and len(c) >= 2]\n", " if not sep_positions:\n", " return None\n", " n_cols = len(sep_positions)\n", " sep_start = sep_positions[0]\n", " header_cells = raw_cells[:sep_start]\n", " if len(header_cells) > n_cols:\n", " header_cells = header_cells[len(header_cells) - n_cols:]\n", " elif len(header_cells) < n_cols:\n", " header_cells = [''] * (n_cols - len(header_cells)) + header_cells\n", " data_cells = raw_cells[sep_start + n_cols:]\n", " rows = []\n", " for i in range(0, len(data_cells), n_cols):\n", " chunk = data_cells[i:i + n_cols]\n", " while len(chunk) < n_cols:\n", " chunk.append('')\n", " rows.append(chunk)\n", " if not rows:\n", " return None\n", " sep_row = '| ' + ' | '.join(['---'] * n_cols) + ' |'\n", " header_row = '| ' + ' | '.join(header_cells) + ' |'\n", " data_rows = ['| ' + ' | '.join(r) + ' |' for r in rows]\n", " return '\\n'.join([header_row, sep_row] + data_rows)\n", "\n", "\n", "def extract_md_tables(text: str) -> list:\n", " tables = []\n", " lines = text.split('\\n')\n", " cur = []\n", " for line in lines:\n", " stripped = line.strip()\n", " if re.search(r'\\|\\s*-{2,}[\\s:]*\\|', stripped):\n", " if len(cur) >= 2:\n", " tables.append('\\n'.join(cur))\n", " cur = []\n", " expanded = _expand_inline_table(stripped)\n", " if expanded:\n", " tables.append(expanded)\n", " continue\n", " if stripped.startswith('|') or stripped.count('|') >= 2:\n", " cur.append(stripped)\n", " else:\n", " if len(cur) >= 2:\n", " tables.append('\\n'.join(cur))\n", " cur = []\n", " if len(cur) >= 2:\n", " tables.append('\\n'.join(cur))\n", " return tables\n", "\n", "\n", "def md_table_to_html(md: str) -> str:\n", " lines = [l.strip() for l in md.strip().split('\\n') if l.strip()]\n", " lines = [l for l in lines if not _is_separator_row(l)]\n", " if not lines:\n", " return '
'\n", " rows = []\n", " for i, line in enumerate(lines):\n", " cells = [re.sub(r'\\s+', ' ', c.strip()) for c in line.strip('|').split('|')]\n", " tag = 'th' if i == 0 else 'td'\n", " rows.append('' + ''.join(f'<{tag}>{_html.escape(c)}' for c in cells) + '')\n", " return '' + ''.join(rows) + '
'\n", "\n", "\n", "class AptedConfig(Config):\n", " def rename(self, node1, node2):\n", " t1 = re.sub(r'\\s+', ' ', (node1.text or '').strip())\n", " t2 = re.sub(r'\\s+', ' ', (node2.text or '').strip())\n", " return 0 if (node1.tag == node2.tag and t1 == t2) else 1\n", " def children(self, node):\n", " return list(node)\n", "\n", "\n", "def teds_score(ref_html: str, hyp_html: str) -> Optional[float]:\n", " try:\n", " rt = etree.fromstring(f'{ref_html}')\n", " ht = etree.fromstring(f'{hyp_html}')\n", " dist = APTED(rt, ht, AptedConfig()).compute_edit_distance()\n", " n = max(len(list(rt.iter())), len(list(ht.iter())))\n", " return round((1.0 - dist / n) * 100, 2) if n > 0 else 0.0\n", " except Exception as e:\n", " print(f'TEDS error: {e}')\n", " return None\n", "\n", "# ── Per-file evaluation ───────────────────────────────────────────────────────\n", "def evaluate_page(ref_raw: str, hyp_raw: str) -> dict:\n", " ref = normalize_arabic_ocr(ref_raw, remove_markdown=True)\n", " hyp = normalize_arabic_ocr(hyp_raw, remove_markdown=True)\n", "\n", " if not ref or not hyp:\n", " return {'status': 'empty_after_norm'}\n", "\n", " rouge_sc = rs.RougeScorer(['rouge1', 'rougeL'], use_stemmer=False)\n", " pg = {'status': 'ok', 'ref_chars': len(ref_raw), 'hyp_chars': len(hyp_raw)}\n", "\n", " # NED\n", " try:\n", " ed = editdistance.eval(ref, hyp)\n", " pg['NED'] = round(ed / max(len(ref), len(hyp), 1) * 100, 2)\n", " except:\n", " pg['NED'] = None\n", "\n", " # CER / WER — jiwer\n", " try:\n", " tf = jiwer.Compose([jiwer.Strip(), jiwer.ReduceToListOfListOfWords()])\n", " pg['CER_jiwer'] = round(jiwer.cer(ref, hyp) * 100, 2)\n", " pg['WER_jiwer'] = round(jiwer.wer(ref, hyp, reference_transform=tf, hypothesis_transform=tf) * 100, 2)\n", " except:\n", " pg['CER_jiwer'] = pg['WER_jiwer'] = None\n", "\n", " # CER / WER — dinglehopper\n", " try:\n", " from dinglehopper.character_error_rate import character_error_rate as dce\n", " from dinglehopper.word_error_rate import word_error_rate as dwe\n", " pg['CER_dh'] = round(float(dce(ref, hyp)) * 100, 2)\n", " pg['WER_dh'] = round(float(dwe(ref, hyp)) * 100, 2)\n", " except:\n", " pg['CER_dh'] = pg['WER_dh'] = None\n", "\n", " # BLEU\n", " try:\n", " pg['BLEU'] = round(sacrebleu.sentence_bleu(hyp, [ref]).score, 2)\n", " except:\n", " pg['BLEU'] = None\n", "\n", " # ROUGE\n", " try:\n", " r = rouge_sc.score(ref, hyp)\n", " pg['ROUGE_1'] = round(r['rouge1'].fmeasure * 100, 2)\n", " pg['ROUGE_L'] = round(r['rougeL'].fmeasure * 100, 2)\n", " except:\n", " pg['ROUGE_1'] = pg['ROUGE_L'] = None\n", "\n", " # TEDS\n", " ref_tables = extract_md_tables(ref_raw)\n", " hyp_tables = extract_md_tables(hyp_raw)\n", " pg['n_tables_ref'] = len(ref_tables)\n", " pg['n_tables_hyp'] = len(hyp_tables)\n", " if ref_tables:\n", " teds_list = []\n", " for i, rt in enumerate(ref_tables):\n", " score = teds_score(md_table_to_html(rt), md_table_to_html(hyp_tables[i])) \\\n", " if i < len(hyp_tables) else 0.0\n", " teds_list.append(score if score is not None else 0.0)\n", " pg['TEDS'] = round(sum(teds_list) / len(teds_list), 2)\n", " else:\n", " pg['TEDS'] = None\n", "\n", " return pg\n", "\n", "# ── Page splitter ─────────────────────────────────────────────────────────────\n", "def split_pages_raw(content: str) -> dict:\n", " pattern = re.compile(r'^===\\s*(.+?)\\s*===$', re.MULTILINE)\n", " matches = list(pattern.finditer(content))\n", " if not matches:\n", " print(\"⚠️ No page separators found\")\n", " return {\"page_01\": content.strip()}\n", " pages = {}\n", " for idx, m in enumerate(matches):\n", " page_id = m.group(1).strip()\n", " start = m.end()\n", " end = matches[idx + 1].start() if idx + 1 < len(matches) else len(content)\n", " text = content[start:end].strip()\n", " if text:\n", " pages[page_id] = text\n", " return pages\n", "\n", "# ── Discover and match files ──────────────────────────────────────────────────\n", "gt_files = {f.name: f for f in sorted(GT_FOLDER.glob('*.md'))}\n", "hyp_files = {f.name: f for f in sorted(HYP_FOLDER.glob('*.md'))}\n", "\n", "print(f'GT folder : {len(gt_files)} .md files found')\n", "print(f'HYP folder : {len(hyp_files)} .md files found')\n", "\n", "common = sorted(set(gt_files) & set(hyp_files))\n", "only_gt = set(gt_files) - set(hyp_files)\n", "only_hyp = set(hyp_files) - set(gt_files)\n", "\n", "if only_gt:\n", " print(f'⚠️ Only in GT : {sorted(only_gt)}')\n", "if only_hyp:\n", " print(f'⚠️ Only in HYP : {sorted(only_hyp)}')\n", "print(f'\\n✅ Matched files to evaluate : {len(common)}')\n", "\n", "# ── Evaluate each matched pair — page by page ─────────────────────────────────\n", "# per_page stores ALL pages across ALL files: key = \"filename::page_id\"\n", "per_page = {}\n", "\n", "for fname in common:\n", " gt_raw = gt_files[fname].read_text(encoding='utf-8').strip()\n", " hyp_raw = hyp_files[fname].read_text(encoding='utf-8').strip()\n", "\n", " gt_pages = split_pages_raw(gt_raw)\n", " hyp_pages = split_pages_raw(hyp_raw)\n", "\n", " common_pages = sorted(set(gt_pages) & set(hyp_pages))\n", " only_gt_p = set(gt_pages) - set(hyp_pages)\n", " only_hyp_p = set(hyp_pages) - set(gt_pages)\n", "\n", " print(f'\\n 📄 {fname} → GT:{len(gt_pages)} pages HYP:{len(hyp_pages)} pages matched:{len(common_pages)}')\n", " if only_gt_p:\n", " print(f' ⚠️ Only in GT : {sorted(only_gt_p)}')\n", " if only_hyp_p:\n", " print(f' ⚠️ Only in HYP : {sorted(only_hyp_p)}')\n", "\n", " for page_id in common_pages:\n", " key = f\"{fname}::{page_id}\"\n", " result = evaluate_page(gt_pages[page_id], hyp_pages[page_id])\n", " per_page[key] = result\n", " if result.get('status') == 'ok':\n", " print(f' ✓ {page_id} NED:{result[\"NED\"]}% CER:{result[\"CER_jiwer\"]}% WER:{result[\"WER_jiwer\"]}%')\n", " else:\n", " print(f' ⚠️ {page_id} → {result.get(\"status\")}')\n", "\n", "# ── Global averages across all pages of all files ─────────────────────────────\n", "def avg(k):\n", " vals = [v[k] for v in per_page.values()\n", " if isinstance(v, dict) and v.get('status') == 'ok' and v.get(k) is not None]\n", " return round(sum(vals) / len(vals), 2) if vals else None\n", "\n", "overall = {k: avg(k) for k in ['NED','CER_jiwer','WER_jiwer','CER_dh','WER_dh','BLEU','ROUGE_1','ROUGE_L','TEDS']}\n", "total_pages = sum(1 for v in per_page.values() if isinstance(v, dict) and v.get('status') == 'ok')\n", "\n", "# ── Print final results ───────────────────────────────────────────────────────\n", "SEP = '=' * 60\n", "print(f'\\n{SEP}')\n", "print(f' {MODEL_NAME} — {len(common)} files / {total_pages} pages evaluated')\n", "print(SEP)\n", "print(f\" NED : {overall['NED']}% [↓ principal]\")\n", "print(f\" CER_jiwer : {overall['CER_jiwer']}% WER_jiwer : {overall['WER_jiwer']}%\")\n", "print(f\" CER_dh : {overall['CER_dh']}% WER_dh : {overall['WER_dh']}%\")\n", "print(f\" BLEU : {overall['BLEU']} ROUGE-1 : {overall['ROUGE_1']}% ROUGE-L : {overall['ROUGE_L']}%\")\n", "print(f\" TEDS : {overall['TEDS']}% [None = no tables in GT page]\")\n", "print(SEP)\n", "\n", "print(f\"\\n{'─'*60}\")\n", "print(f' Per-page breakdown')\n", "print(f\"{'─'*60}\")\n", "for key, pg in per_page.items():\n", " fname, page_id = key.split('::', 1)\n", " if pg.get('status') == 'ok':\n", " print(f'\\n [{fname}] {page_id}')\n", " print(f\" NED:{pg['NED']}% CER:{pg['CER_jiwer']}% WER:{pg['WER_jiwer']}% BLEU:{pg['BLEU']} ROUGE-1:{pg['ROUGE_1']}% TEDS:{pg['TEDS']}%\")\n", " if pg['n_tables_ref'] or pg['n_tables_hyp']:\n", " print(f\" tables → GT:{pg['n_tables_ref']} HYP:{pg['n_tables_hyp']}\")\n", " else:\n", " print(f\"\\n [{fname}] {page_id} ⚠️ {pg.get('status')}\")\n", "\n", "# ── Save results to JSON ──────────────────────────────────────────────────────\n", "RESULTS_DIR = \"/home/skiredj.abderrahman/khalil/Benchmarking_OCR/final_resulsts_benchmark/\"\n", "os.makedirs(RESULTS_DIR, exist_ok=True)\n", "\n", "def save_results(report: dict):\n", " name = report[\"model\"].replace(\" \", \"_\")\n", " path = os.path.join(RESULTS_DIR, f\"{name}.json\")\n", " json.dump(report, open(path, \"w\", encoding=\"utf-8\"), ensure_ascii=False, indent=2)\n", " print(f\"\\n✅ Sauvegardé : {path}\")\n", "\n", "report = {\n", " \"model\" : MODEL_NAME,\n", " \"n_files\" : len(common),\n", " \"n_pages\" : total_pages,\n", " \"overall\" : overall,\n", " \"per_page\": per_page\n", "}\n", "save_results(report)" ] }, { "cell_type": "markdown", "id": "8843e94d-fb41-4ad7-bd2e-3a931ecce006", "metadata": { "jp-MarkdownHeadingCollapsed": true }, "source": [ "# deepseek 2 " ] }, { "cell_type": "markdown", "id": "ce72c305-1654-4c95-92b4-45403a9362a4", "metadata": { "jp-MarkdownHeadingCollapsed": true }, "source": [ "## OCR " ] }, { "cell_type": "code", "execution_count": null, "id": "63c473b6-a011-44d1-9e03-b96dd5d57f02", "metadata": {}, "outputs": [], "source": [ "\n", "\n", "\n", "############\n", "# ============================================================================\n", "# PIPELINE OCR - IMAGES LOCALES (sans prétraitement)\n", "# Entrée : dossier avec sous-dossiers contenant des PNG (déjà zoom x4)\n", "# Sortie : un fichier .md par sous-dossier\n", "# ============================================================================\n", "\n", "import os\n", "import re\n", "import time\n", "import base64\n", "import glob\n", "import shutil\n", "from pathlib import Path\n", "from typing import Union, Optional\n", "from concurrent.futures import ThreadPoolExecutor, as_completed\n", "from tqdm import tqdm\n", "from openai import OpenAI\n", "\n", "# ========== CONFIGURATION ==========\n", "\n", "MODEL_PATH = \"/home/skiredj.abderrahman/khalil/DeepSeek-OCR-2\"\n", "\n", "client = OpenAI(\n", " api_key=\"EMPTY\",\n", " base_url=\"http://localhost:9996/v1\",\n", " timeout=3600\n", ")\n", "\n", "GENERATION_PARAMS = {\n", " \"max_tokens\": 2048,\n", " \"temperature\": 0.0,\n", " \"extra_body\": {\n", " \"skip_special_tokens\": False,\n", " \"vllm_xargs\": {\n", " \"ngram_size\": 30,\n", " \"window_size\": 90,\n", " \"whitelist_token_ids\": [128821, 128822],\n", " },\n", " },\n", "}\n", "\n", "REQUEST_TIMEOUT_SECONDS = 180\n", "NUM_WORKERS = 1\n", "\n", "OCR_PROMPT = \"Convert the document to markdown.\"\n", "\n", "# ── Dossiers ──────────────────────────────────────────────────────────────────\n", "IMAGES_DATASET_FOLDER = r\"/home/skiredj.abderrahman/khalil/Benchmarking_OCR/ground_truth_claude_100_images/100images\"\n", "OUTPUT_FOLDER = r\"/home/skiredj.abderrahman/khalil/Benchmarking_OCR/final_resulsts_benchmark/deepseek2_nosplit/output_markdown\"\n", "TXT_TEMP_FOLDER = r\"/home/skiredj.abderrahman/khalil/Benchmarking_OCR/final_resulsts_benchmark/deepseek2_nosplit/temp_txt\"\n", "\n", "\n", "# ========== TRI DES IMAGES (arabe : right avant left) ==========\n", "\n", "def arabic_sort_key(path):\n", " name = os.path.splitext(os.path.basename(path))[0]\n", " if name.endswith(\"_right\"):\n", " base, order = name[:-6], 0 # right = 0 → passe en premier\n", " elif name.endswith(\"_left\"):\n", " base, order = name[:-5], 1 # left = 1 → passe en second\n", " else:\n", " base, order = name, 0\n", " return (base, order)\n", "\n", "\n", "# ========== OCR ==========\n", "\n", "def encode_image(path):\n", " with open(path, \"rb\") as f:\n", " return base64.b64encode(f.read()).decode(\"utf-8\")\n", "\n", "\n", "def call_deepseek_ocr(image_path, retries=3):\n", " for attempt in range(1, retries + 1):\n", " try:\n", " base64_image = encode_image(image_path)\n", " messages = [\n", " {\n", " \"role\": \"user\",\n", " \"content\": [\n", " {\n", " \"type\": \"image_url\",\n", " \"image_url\": {\n", " \"url\": f\"data:image/png;base64,{base64_image}\"\n", " }\n", " },\n", " {\n", " \"type\": \"text\",\n", " \"text\": OCR_PROMPT\n", " }\n", " ]\n", " }\n", " ]\n", " response = client.chat.completions.create(\n", " model=MODEL_PATH,\n", " messages=messages,\n", " **GENERATION_PARAMS,\n", " )\n", " return response.choices[0].message.content\n", " except Exception as e:\n", " print(f\" Erreur tentative {attempt}/{retries} ({image_path}): {e}\")\n", " if attempt < retries:\n", " time.sleep(3)\n", " raise Exception(f\"OCR échoué après {retries} tentatives : {image_path}\")\n", "\n", "\n", "def process_image(input_path, output_txt_path):\n", " if os.path.exists(output_txt_path):\n", " return # déjà traité\n", " try:\n", " text = call_deepseek_ocr(input_path)\n", " with open(output_txt_path, \"w\", encoding=\"utf-8\") as f:\n", " f.write(text)\n", " except Exception as e:\n", " with open(output_txt_path, \"w\", encoding=\"utf-8\") as f:\n", " f.write(f\"[OCR FAILED: {e}]\")\n", "\n", "\n", "def _is_table_line(line: str) -> bool:\n", " \"\"\"True if the line is part of a markdown table (starts with | or has 2+ pipes).\"\"\"\n", " s = line.strip()\n", " return s.startswith(\"|\") or s.count(\"|\") >= 2\n", "\n", "\n", "def clean_unnecessary_linebreaks(text):\n", " lines = text.split(\"\\n\")\n", " out = []\n", " for i, line in enumerate(lines):\n", " if line.startswith(\"=== Page\"):\n", " out.append(line)\n", " if i + 1 < len(lines) and lines[i + 1].strip() == \"\":\n", " out.append(\"\")\n", " else:\n", " prev_is_page_header = i > 0 and lines[i - 1].startswith(\"=== Page\")\n", " prev_ends_sentence = i > 0 and lines[i - 1].rstrip().endswith(\".\")\n", "\n", " # Never merge table rows — doing so collapses the entire table onto\n", " # one line and breaks all downstream table detection / TEDS scoring.\n", " current_is_table = _is_table_line(line)\n", " prev_is_table = bool(out) and _is_table_line(out[-1])\n", "\n", " if (not prev_is_page_header\n", " and not prev_ends_sentence\n", " and not current_is_table\n", " and not prev_is_table\n", " and out\n", " and out[-1] != \"\"):\n", " out[-1] = out[-1].rstrip() + \" \" + line.lstrip()\n", " else:\n", " out.append(line)\n", " return \"\\n\".join(out)\n", "\n", "\n", "# ========== PIPELINE PRINCIPAL ==========\n", "\n", "def process_subfolder(subfolder_path, subfolder_name):\n", " print(f\"\\n{'='*60}\")\n", " print(f\"Traitement : {subfolder_name}\")\n", " print(f\"{'='*60}\")\n", "\n", " txt_dir = os.path.join(TXT_TEMP_FOLDER, subfolder_name)\n", " output_md = os.path.join(OUTPUT_FOLDER, subfolder_name + \".md\")\n", " os.makedirs(txt_dir, exist_ok=True)\n", " os.makedirs(OUTPUT_FOLDER, exist_ok=True)\n", "\n", " if os.path.exists(output_md):\n", " print(f\" → Déjà traité, on passe.\")\n", " return\n", "\n", " # ── Étape 1 : Récupération et tri des images ───────────────────────────────\n", " png_files = sorted(\n", " glob.glob(os.path.join(subfolder_path, \"*.png\")),\n", " key=arabic_sort_key\n", " )\n", " if not png_files:\n", " print(f\" ⚠️ Aucune image PNG trouvée dans {subfolder_path}\")\n", " return\n", "\n", " # ── Étape 2 : OCR en parallèle ────────────────────────────────────────────\n", " tasks = [\n", " (img_p, os.path.join(txt_dir, os.path.splitext(os.path.basename(img_p))[0] + \".txt\"))\n", " for img_p in png_files\n", " ]\n", "\n", " print(f\" {len(png_files)} image(s) à envoyer à DeepSeek-OCR...\")\n", " with ThreadPoolExecutor(max_workers=NUM_WORKERS) as executor:\n", " futures = {executor.submit(process_image, ip, op): (ip, op) for ip, op in tasks}\n", " for future in tqdm(as_completed(futures), total=len(futures),\n", " desc=\" OCR\", leave=False):\n", " try:\n", " future.result()\n", " except Exception as e:\n", " print(f\" ❌ Erreur non gérée : {e}\")\n", "\n", " # ── Étape 3 : Assemblage en Markdown ──────────────────────────────────────\n", " parts = []\n", " for i, img_p in enumerate(png_files, 1):\n", " img_name = os.path.splitext(os.path.basename(img_p))[0]\n", " txt_path = os.path.join(txt_dir, img_name + \".txt\")\n", " if os.path.exists(txt_path):\n", " content = open(txt_path, encoding=\"utf-8\").read()\n", " else:\n", " content = \"[OCR failed for this page]\"\n", " parts.append(f\"=== Page {i} ({img_name}) ===\\n{content}\\n\")\n", "\n", " final = clean_unnecessary_linebreaks(\"\\n\".join(parts))\n", " with open(output_md, \"w\", encoding=\"utf-8\") as f:\n", " f.write(final)\n", "\n", " print(f\" ✅ Markdown sauvegardé → {output_md}\")\n", "\n", "\n", "def run_pipeline():\n", " subfolders = sorted([\n", " d for d in os.listdir(IMAGES_DATASET_FOLDER)\n", " if os.path.isdir(os.path.join(IMAGES_DATASET_FOLDER, d))\n", " ])\n", "\n", " if not subfolders:\n", " print(f\"Aucun sous-dossier trouvé dans {IMAGES_DATASET_FOLDER}\")\n", " return\n", "\n", " # ── Scan préalable ────────────────────────────────────────────────────────\n", " print(f\"\\n{'='*60}\")\n", " print(f\" SCAN DU DATASET\")\n", " print(f\"{'='*60}\")\n", " print(f\" Dossier racine : {IMAGES_DATASET_FOLDER}\\n\")\n", "\n", " folder_image_counts = {}\n", " max_name_len = max(len(d) for d in subfolders)\n", "\n", " for name in subfolders:\n", " path = os.path.join(IMAGES_DATASET_FOLDER, name)\n", " pngs = sorted(glob.glob(os.path.join(path, \"*.png\")))\n", " folder_image_counts[name] = pngs\n", " status = f\"{len(pngs):>4} image(s)\" if pngs else \" ── aucune image PNG\"\n", " print(f\" 📁 {name:<{max_name_len}} → {status}\")\n", "\n", " total_images = sum(len(v) for v in folder_image_counts.values())\n", " total_folders = len(subfolders)\n", " empty_folders = sum(1 for v in folder_image_counts.values() if not v)\n", " active_folders = total_folders - empty_folders\n", "\n", " print(f\"\\n{'─'*60}\")\n", " print(f\" Sous-dossiers trouvés : {total_folders}\")\n", " print(f\" Sous-dossiers avec PNG : {active_folders}\")\n", " print(f\" Sous-dossiers vides : {empty_folders}\")\n", " print(f\" Total images PNG : {total_images}\")\n", " print(f\"{'─'*60}\")\n", " print(f\" Sortie markdown : {OUTPUT_FOLDER}\")\n", " print(f\"{'='*60}\\n\")\n", "\n", " if total_images == 0:\n", " print(\"Aucune image à traiter. Arrêt.\")\n", " return\n", "\n", " # ── Traitement ────────────────────────────────────────────────────────────\n", " for name in tqdm(subfolders, desc=\"Sous-dossiers\"):\n", " if not folder_image_counts[name]:\n", " continue\n", " path = os.path.join(IMAGES_DATASET_FOLDER, name)\n", " try:\n", " process_subfolder(path, name)\n", " except Exception as e:\n", " print(f\"❌ Erreur sur {name} : {e}\")\n", "\n", " print(f\"\\n{'='*60}\")\n", " print(\"PIPELINE TERMINÉ\")\n", " print(f\" {active_folders} sous-dossier(s) traité(s), {total_images} image(s) au total\")\n", " print(f\" Résultats dans : {OUTPUT_FOLDER}\")\n", " print(f\"{'='*60}\")\n", "\n", "\n", "# ========== LANCEMENT ==========\n", "\n", "if __name__ == \"__main__\":\n", " run_pipeline()" ] }, { "cell_type": "markdown", "id": "af3e9661-ac6a-4683-b7df-38c8f91b0c1f", "metadata": { "jp-MarkdownHeadingCollapsed": true }, "source": [ "## NO page n " ] }, { "cell_type": "code", "execution_count": null, "id": "7b4c752a-294a-4567-8b12-09ca792781d4", "metadata": {}, "outputs": [], "source": [ "\n", "import os\n", "import re\n", "from pathlib import Path\n", "\n", "# ── Change these two paths ──────────────────────────────────────────\n", "INPUT_FOLDER = \"/home/skiredj.abderrahman/khalil/Benchmarking_OCR/final_resulsts_benchmark/deepseek2_nosplit/output_markdown\"\n", "OUTPUT_FOLDER = \"/home/skiredj.abderrahman/khalil/Benchmarking_OCR/final_resulsts_benchmark/deepseek2_nosplit/output_markdown_no_pageN\"\n", "# ────────────────────────────────────────────────────────────────────\n", "\n", "def parse_pages(content):\n", " pattern = r'(=== Page (\\d+) \\(([^)]+)\\) ===)'\n", " parts = re.split(pattern, content)\n", " \n", " preamble = parts[0]\n", " pages = []\n", " i = 1\n", " while i < len(parts):\n", " pages.append({\n", " 'header': parts[i],\n", " 'num': int(parts[i+1]),\n", " 'id': parts[i+2],\n", " 'body': parts[i+3]\n", " })\n", " i += 4\n", " return preamble, pages\n", "\n", "def merge_pages(pages):\n", " merged = []\n", " skip_ids = set()\n", " \n", " for i, page in enumerate(pages):\n", " pid = page['id']\n", " if pid in skip_ids:\n", " continue\n", " \n", " if pid.endswith('_right'):\n", " base_id = pid[:-len('_right')]\n", " left_page = None\n", " for j in range(i+1, len(pages)):\n", " if pages[j]['id'] == base_id + '_left':\n", " left_page = pages[j]\n", " skip_ids.add(pages[j]['id'])\n", " break\n", " \n", " combined_body = (page['body'].rstrip() + '\\n' + left_page['body']) if left_page else page['body']\n", " merged.append({'id': base_id, 'body': combined_body})\n", "\n", " elif pid.endswith('_left'):\n", " # Orphan left — strip suffix, keep content\n", " merged.append({'id': pid[:-len('_left')], 'body': page['body']})\n", " else:\n", " merged.append({'id': pid, 'body': page['body']})\n", " \n", " # Renumber sequentially\n", " return [{'num': n, 'id': p['id'], 'body': p['body']} for n, p in enumerate(merged, start=1)]\n", "\n", "def rebuild_content(preamble, pages):\n", " result = preamble\n", " for page in pages:\n", " result += f\"=== {page['id']} ===\\n\"\n", " result += page['body']\n", " return result\n", "\n", "def process_folder(input_folder, output_folder):\n", " input_path = Path(input_folder)\n", " output_path = Path(output_folder)\n", " output_path.mkdir(parents=True, exist_ok=True)\n", " \n", " md_files = list(input_path.glob('*.md'))\n", " if not md_files:\n", " print(f\"No .md files found in {input_folder}\")\n", " return\n", " \n", " for md_file in md_files:\n", " content = md_file.read_text(encoding='utf-8')\n", " preamble, pages = parse_pages(content)\n", " \n", " if not pages:\n", " (output_path / md_file.name).write_text(content, encoding='utf-8')\n", " print(f\"{md_file.name} — no pages found, copied as-is.\")\n", " continue\n", " \n", " merged = merge_pages(pages)\n", " output_path.joinpath(md_file.name).write_text(rebuild_content(preamble, merged), encoding='utf-8')\n", " \n", " print(f\"{md_file.name} — {len(pages)} pages → {len(merged)} pages ({len(pages)-len(merged)} pair(s) merged)\")\n", " \n", " print(f\"\\n✅ Done! Files saved to: {output_path.resolve()}\")\n", "\n", "process_folder(INPUT_FOLDER, OUTPUT_FOLDER)" ] }, { "cell_type": "markdown", "id": "7361a36b-d6c3-4111-b100-84026908e66a", "metadata": { "jp-MarkdownHeadingCollapsed": true }, "source": [ "## benchmark" ] }, { "cell_type": "code", "execution_count": null, "id": "5c88fa88-ea27-403a-bc67-89cccefb1906", "metadata": {}, "outputs": [], "source": [ "import os\n", "import re\n", "import unicodedata\n", "import html as _html\n", "import json\n", "import editdistance\n", "import jiwer\n", "import sacrebleu\n", "from rouge_score import rouge_scorer as rs\n", "from apted import APTED, Config\n", "from lxml import etree\n", "from typing import Optional\n", "from pathlib import Path\n", "\n", "# ── Paths ─────────────────────────────────────────────────────────────────────\n", "GT_FOLDER = Path(\"/home/skiredj.abderrahman/khalil/Benchmarking_OCR/ground_truth_claude_100_images/GT\")\n", "HYP_FOLDER = Path(\"/home/skiredj.abderrahman/khalil/Benchmarking_OCR/final_resulsts_benchmark/deepseek2_nosplit/output_markdown_no_pageN\")\n", "MODEL_NAME = \"deepseek2_no_split\"\n", "\n", "# ── Normalization ─────────────────────────────────────────────────────────────\n", "def normalize_arabic_ocr(text: str, remove_markdown: bool = False) -> str:\n", " if not isinstance(text, str) or not text.strip():\n", " return \"\"\n", " if remove_markdown:\n", " text = re.sub(r'#{1,6}\\s*', '', text)\n", " text = re.sub(r'^\\s*[-*+]\\s+', '', text, flags=re.MULTILINE)\n", " text = re.sub(r'\\*{1,3}(.*?)\\*{1,3}', r'\\1', text, flags=re.DOTALL)\n", " text = re.sub(r'`{1,3}.*?`{1,3}', ' ', text, flags=re.DOTALL)\n", " text = re.sub(r'\\[([^\\]]*)\\]\\([^\\)]*\\)', r'\\1', text)\n", " text = re.sub(r'!\\[[^\\]]*\\]\\([^\\)]*\\)', ' ', text)\n", " text = re.sub(r'\\|', ' ', text)\n", " text = unicodedata.normalize('NFKC', text)\n", " text = re.sub(r'[\\u200B-\\u200F\\u202A-\\u202E\\u2060-\\u2064\\uFEFF\\u00AD]', '', text)\n", " text = re.sub(\n", " r'[\\u064B-\\u065F\\u0610-\\u061A\\u06D6-\\u06DC\\u06DF-\\u06E4\\u06E7\\u06E8\\u06EA-\\u06ED]',\n", " '', text\n", " )\n", " text = text.replace('\\u0640', '')\n", " text = re.sub(r'[\\u0622\\u0623\\u0625\\u0671\\u0672\\u0673\\u0675]', '\\u0627', text)\n", " text = text.replace('\\u0649', '\\u064A')\n", " text = text.replace('\\u0629', '\\u0647')\n", " text = text.replace('\\u0624', '\\u0648')\n", " text = text.replace('\\u0626', '\\u064A')\n", " text = re.sub(r'[\\u060C\\u061B\\u061F\\u066A-\\u066D\\u0600-\\u0605]', ' ', text)\n", " text = re.sub(r'[.,:;()\\[\\]{}\\-«»\"\\'!?/\\\\~@#$%^&*_+=<>|]', ' ', text)\n", " text = text.lower()\n", " text = re.sub(r'[^\\u0600-\\u06FF\\u0750-\\u077Fa-z0-9\\s]', ' ', text)\n", " text = text.translate(str.maketrans('٠١٢٣٤٥٦٧٨٩', '0123456789'))\n", " text = text.lower()\n", " return re.sub(r'\\s+', ' ', text).strip()\n", "\n", "# ── TEDS helpers ──────────────────────────────────────────────────────────────\n", "def _is_separator_row(row: str) -> bool:\n", " cells = [c.strip() for c in row.strip().strip('|').split('|')]\n", " return bool(cells) and all(re.match(r'^[-:\\s]+$', c) for c in cells if c)\n", "\n", "\n", "def _expand_inline_table(line: str) -> Optional[str]:\n", " if not re.search(r'\\|\\s*-{2,}[\\s:]*\\|', line):\n", " return None\n", " raw_cells = [c.strip() for c in line.split('|')]\n", " while raw_cells and raw_cells[0] == '':\n", " raw_cells.pop(0)\n", " while raw_cells and raw_cells[-1] == '':\n", " raw_cells.pop()\n", " if not raw_cells:\n", " return None\n", " sep_positions = [i for i, c in enumerate(raw_cells) if re.match(r'^[-:\\s]+$', c) and len(c) >= 2]\n", " if not sep_positions:\n", " return None\n", " n_cols = len(sep_positions)\n", " sep_start = sep_positions[0]\n", " header_cells = raw_cells[:sep_start]\n", " if len(header_cells) > n_cols:\n", " header_cells = header_cells[len(header_cells) - n_cols:]\n", " elif len(header_cells) < n_cols:\n", " header_cells = [''] * (n_cols - len(header_cells)) + header_cells\n", " data_cells = raw_cells[sep_start + n_cols:]\n", " rows = []\n", " for i in range(0, len(data_cells), n_cols):\n", " chunk = data_cells[i:i + n_cols]\n", " while len(chunk) < n_cols:\n", " chunk.append('')\n", " rows.append(chunk)\n", " if not rows:\n", " return None\n", " sep_row = '| ' + ' | '.join(['---'] * n_cols) + ' |'\n", " header_row = '| ' + ' | '.join(header_cells) + ' |'\n", " data_rows = ['| ' + ' | '.join(r) + ' |' for r in rows]\n", " return '\\n'.join([header_row, sep_row] + data_rows)\n", "\n", "\n", "def extract_md_tables(text: str) -> list:\n", " tables = []\n", " lines = text.split('\\n')\n", " cur = []\n", " for line in lines:\n", " stripped = line.strip()\n", " if re.search(r'\\|\\s*-{2,}[\\s:]*\\|', stripped):\n", " if len(cur) >= 2:\n", " tables.append('\\n'.join(cur))\n", " cur = []\n", " expanded = _expand_inline_table(stripped)\n", " if expanded:\n", " tables.append(expanded)\n", " continue\n", " if stripped.startswith('|') or stripped.count('|') >= 2:\n", " cur.append(stripped)\n", " else:\n", " if len(cur) >= 2:\n", " tables.append('\\n'.join(cur))\n", " cur = []\n", " if len(cur) >= 2:\n", " tables.append('\\n'.join(cur))\n", " return tables\n", "\n", "\n", "def md_table_to_html(md: str) -> str:\n", " lines = [l.strip() for l in md.strip().split('\\n') if l.strip()]\n", " lines = [l for l in lines if not _is_separator_row(l)]\n", " if not lines:\n", " return '
'\n", " rows = []\n", " for i, line in enumerate(lines):\n", " cells = [re.sub(r'\\s+', ' ', c.strip()) for c in line.strip('|').split('|')]\n", " tag = 'th' if i == 0 else 'td'\n", " rows.append('' + ''.join(f'<{tag}>{_html.escape(c)}' for c in cells) + '')\n", " return '' + ''.join(rows) + '
'\n", "\n", "\n", "class AptedConfig(Config):\n", " def rename(self, node1, node2):\n", " t1 = re.sub(r'\\s+', ' ', (node1.text or '').strip())\n", " t2 = re.sub(r'\\s+', ' ', (node2.text or '').strip())\n", " return 0 if (node1.tag == node2.tag and t1 == t2) else 1\n", " def children(self, node):\n", " return list(node)\n", "\n", "\n", "def teds_score(ref_html: str, hyp_html: str) -> Optional[float]:\n", " try:\n", " rt = etree.fromstring(f'{ref_html}')\n", " ht = etree.fromstring(f'{hyp_html}')\n", " dist = APTED(rt, ht, AptedConfig()).compute_edit_distance()\n", " n = max(len(list(rt.iter())), len(list(ht.iter())))\n", " return round((1.0 - dist / n) * 100, 2) if n > 0 else 0.0\n", " except Exception as e:\n", " print(f'TEDS error: {e}')\n", " return None\n", "\n", "# ── Per-file evaluation ───────────────────────────────────────────────────────\n", "def evaluate_page(ref_raw: str, hyp_raw: str) -> dict:\n", " ref = normalize_arabic_ocr(ref_raw, remove_markdown=True)\n", " hyp = normalize_arabic_ocr(hyp_raw, remove_markdown=True)\n", "\n", " if not ref or not hyp:\n", " return {'status': 'empty_after_norm'}\n", "\n", " rouge_sc = rs.RougeScorer(['rouge1', 'rougeL'], use_stemmer=False)\n", " pg = {'status': 'ok', 'ref_chars': len(ref_raw), 'hyp_chars': len(hyp_raw)}\n", "\n", " # NED\n", " try:\n", " ed = editdistance.eval(ref, hyp)\n", " pg['NED'] = round(ed / max(len(ref), len(hyp), 1) * 100, 2)\n", " except:\n", " pg['NED'] = None\n", "\n", " # CER / WER — jiwer\n", " try:\n", " tf = jiwer.Compose([jiwer.Strip(), jiwer.ReduceToListOfListOfWords()])\n", " pg['CER_jiwer'] = round(jiwer.cer(ref, hyp) * 100, 2)\n", " pg['WER_jiwer'] = round(jiwer.wer(ref, hyp, reference_transform=tf, hypothesis_transform=tf) * 100, 2)\n", " except:\n", " pg['CER_jiwer'] = pg['WER_jiwer'] = None\n", "\n", " # CER / WER — dinglehopper\n", " try:\n", " from dinglehopper.character_error_rate import character_error_rate as dce\n", " from dinglehopper.word_error_rate import word_error_rate as dwe\n", " pg['CER_dh'] = round(float(dce(ref, hyp)) * 100, 2)\n", " pg['WER_dh'] = round(float(dwe(ref, hyp)) * 100, 2)\n", " except:\n", " pg['CER_dh'] = pg['WER_dh'] = None\n", "\n", " # BLEU\n", " try:\n", " pg['BLEU'] = round(sacrebleu.sentence_bleu(hyp, [ref]).score, 2)\n", " except:\n", " pg['BLEU'] = None\n", "\n", " # ROUGE\n", " try:\n", " r = rouge_sc.score(ref, hyp)\n", " pg['ROUGE_1'] = round(r['rouge1'].fmeasure * 100, 2)\n", " pg['ROUGE_L'] = round(r['rougeL'].fmeasure * 100, 2)\n", " except:\n", " pg['ROUGE_1'] = pg['ROUGE_L'] = None\n", "\n", " # TEDS\n", " ref_tables = extract_md_tables(ref_raw)\n", " hyp_tables = extract_md_tables(hyp_raw)\n", " pg['n_tables_ref'] = len(ref_tables)\n", " pg['n_tables_hyp'] = len(hyp_tables)\n", " if ref_tables:\n", " teds_list = []\n", " for i, rt in enumerate(ref_tables):\n", " score = teds_score(md_table_to_html(rt), md_table_to_html(hyp_tables[i])) \\\n", " if i < len(hyp_tables) else 0.0\n", " teds_list.append(score if score is not None else 0.0)\n", " pg['TEDS'] = round(sum(teds_list) / len(teds_list), 2)\n", " else:\n", " pg['TEDS'] = None\n", "\n", " return pg\n", "\n", "# ── Page splitter ─────────────────────────────────────────────────────────────\n", "def split_pages_raw(content: str) -> dict:\n", " pattern = re.compile(r'^===\\s*(.+?)\\s*===$', re.MULTILINE)\n", " matches = list(pattern.finditer(content))\n", " if not matches:\n", " print(\"⚠️ No page separators found\")\n", " return {\"page_01\": content.strip()}\n", " pages = {}\n", " for idx, m in enumerate(matches):\n", " page_id = m.group(1).strip()\n", " start = m.end()\n", " end = matches[idx + 1].start() if idx + 1 < len(matches) else len(content)\n", " text = content[start:end].strip()\n", " if text:\n", " pages[page_id] = text\n", " return pages\n", "\n", "# ── Discover and match files ──────────────────────────────────────────────────\n", "gt_files = {f.name: f for f in sorted(GT_FOLDER.glob('*.md'))}\n", "hyp_files = {f.name: f for f in sorted(HYP_FOLDER.glob('*.md'))}\n", "\n", "print(f'GT folder : {len(gt_files)} .md files found')\n", "print(f'HYP folder : {len(hyp_files)} .md files found')\n", "\n", "common = sorted(set(gt_files) & set(hyp_files))\n", "only_gt = set(gt_files) - set(hyp_files)\n", "only_hyp = set(hyp_files) - set(gt_files)\n", "\n", "if only_gt:\n", " print(f'⚠️ Only in GT : {sorted(only_gt)}')\n", "if only_hyp:\n", " print(f'⚠️ Only in HYP : {sorted(only_hyp)}')\n", "print(f'\\n✅ Matched files to evaluate : {len(common)}')\n", "\n", "# ── Evaluate each matched pair — page by page ─────────────────────────────────\n", "# per_page stores ALL pages across ALL files: key = \"filename::page_id\"\n", "per_page = {}\n", "\n", "for fname in common:\n", " gt_raw = gt_files[fname].read_text(encoding='utf-8').strip()\n", " hyp_raw = hyp_files[fname].read_text(encoding='utf-8').strip()\n", "\n", " gt_pages = split_pages_raw(gt_raw)\n", " hyp_pages = split_pages_raw(hyp_raw)\n", "\n", " common_pages = sorted(set(gt_pages) & set(hyp_pages))\n", " only_gt_p = set(gt_pages) - set(hyp_pages)\n", " only_hyp_p = set(hyp_pages) - set(gt_pages)\n", "\n", " print(f'\\n 📄 {fname} → GT:{len(gt_pages)} pages HYP:{len(hyp_pages)} pages matched:{len(common_pages)}')\n", " if only_gt_p:\n", " print(f' ⚠️ Only in GT : {sorted(only_gt_p)}')\n", " if only_hyp_p:\n", " print(f' ⚠️ Only in HYP : {sorted(only_hyp_p)}')\n", "\n", " for page_id in common_pages:\n", " key = f\"{fname}::{page_id}\"\n", " result = evaluate_page(gt_pages[page_id], hyp_pages[page_id])\n", " per_page[key] = result\n", " if result.get('status') == 'ok':\n", " print(f' ✓ {page_id} NED:{result[\"NED\"]}% CER:{result[\"CER_jiwer\"]}% WER:{result[\"WER_jiwer\"]}%')\n", " else:\n", " print(f' ⚠️ {page_id} → {result.get(\"status\")}')\n", "\n", "# ── Global averages across all pages of all files ─────────────────────────────\n", "def avg(k):\n", " vals = [v[k] for v in per_page.values()\n", " if isinstance(v, dict) and v.get('status') == 'ok' and v.get(k) is not None]\n", " return round(sum(vals) / len(vals), 2) if vals else None\n", "\n", "overall = {k: avg(k) for k in ['NED','CER_jiwer','WER_jiwer','CER_dh','WER_dh','BLEU','ROUGE_1','ROUGE_L','TEDS']}\n", "total_pages = sum(1 for v in per_page.values() if isinstance(v, dict) and v.get('status') == 'ok')\n", "\n", "# ── Print final results ───────────────────────────────────────────────────────\n", "SEP = '=' * 60\n", "print(f'\\n{SEP}')\n", "print(f' {MODEL_NAME} — {len(common)} files / {total_pages} pages evaluated')\n", "print(SEP)\n", "print(f\" NED : {overall['NED']}% [↓ principal]\")\n", "print(f\" CER_jiwer : {overall['CER_jiwer']}% WER_jiwer : {overall['WER_jiwer']}%\")\n", "print(f\" CER_dh : {overall['CER_dh']}% WER_dh : {overall['WER_dh']}%\")\n", "print(f\" BLEU : {overall['BLEU']} ROUGE-1 : {overall['ROUGE_1']}% ROUGE-L : {overall['ROUGE_L']}%\")\n", "print(f\" TEDS : {overall['TEDS']}% [None = no tables in GT page]\")\n", "print(SEP)\n", "\n", "print(f\"\\n{'─'*60}\")\n", "print(f' Per-page breakdown')\n", "print(f\"{'─'*60}\")\n", "for key, pg in per_page.items():\n", " fname, page_id = key.split('::', 1)\n", " if pg.get('status') == 'ok':\n", " print(f'\\n [{fname}] {page_id}')\n", " print(f\" NED:{pg['NED']}% CER:{pg['CER_jiwer']}% WER:{pg['WER_jiwer']}% BLEU:{pg['BLEU']} ROUGE-1:{pg['ROUGE_1']}% TEDS:{pg['TEDS']}%\")\n", " if pg['n_tables_ref'] or pg['n_tables_hyp']:\n", " print(f\" tables → GT:{pg['n_tables_ref']} HYP:{pg['n_tables_hyp']}\")\n", " else:\n", " print(f\"\\n [{fname}] {page_id} ⚠️ {pg.get('status')}\")\n", "\n", "# ── Save results to JSON ──────────────────────────────────────────────────────\n", "RESULTS_DIR = \"/home/skiredj.abderrahman/khalil/Benchmarking_OCR/final_resulsts_benchmark/\"\n", "os.makedirs(RESULTS_DIR, exist_ok=True)\n", "\n", "def save_results(report: dict):\n", " name = report[\"model\"].replace(\" \", \"_\")\n", " path = os.path.join(RESULTS_DIR, f\"{name}.json\")\n", " json.dump(report, open(path, \"w\", encoding=\"utf-8\"), ensure_ascii=False, indent=2)\n", " print(f\"\\n✅ Sauvegardé : {path}\")\n", "\n", "report = {\n", " \"model\" : MODEL_NAME,\n", " \"n_files\" : len(common),\n", " \"n_pages\" : total_pages,\n", " \"overall\" : overall,\n", " \"per_page\": per_page\n", "}\n", "save_results(report)" ] }, { "cell_type": "code", "execution_count": null, "id": "7fc3f012-ebfc-4acc-9319-36cc84610ea2", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "c773eaf1-607b-4e94-b993-51585cb154a6", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "bbaccf78-81c0-4b64-9296-60fc96a72012", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "feeb2a42-ceee-4eb8-abe0-1a14a46b69f1", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "e7a8ecec-8823-452a-a52f-96dc5a0e098d", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "4d8eca7b-6704-4a76-968b-a37dc98a9d03", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "ea58d8b5-2086-4591-b279-fa27ac2a45ee", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "ae89e37f-2756-4d70-9d7f-b0e5ef051407", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "8cae6f31-9a17-4df4-a19c-1d61ae8cdc36", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "c8345bf4-2f46-4d23-8d97-f1c577fabf24", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "2668dd22-732e-4632-8555-7b49acfe7ac1", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "e71656a0-0f7f-4c85-b8c0-77371a8e7404", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "701137c6-ce9c-455b-99de-5ccbfae1c1b8", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "01c954f0-f69f-433e-9367-1b0b1f0c85d7", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "44ddc5fd-3e9d-4da5-b64b-20ed7d03b06f", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "3fcd571e-dda2-4287-b90d-36d58f65a3ae", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "9ac1d4e0-5c5b-41d9-b24c-144595a32798", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "2425c27e-e17f-4330-8651-5ffa8f5b7e1b", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "6b0f4e2a-9b19-4c37-8aa5-b0b98595f64e", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "81ea6f15-4d51-427e-a289-df29bd1650ae", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "ba05743f-31cc-43a7-bb16-fc91445392f4", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "22e1fea2-d9db-468e-9a59-dfa83fa82cc9", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "6a4600f3-c96c-4ce1-94c9-d933d16c51f6", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "ecda2c4e-0c12-4f42-9b47-a7277d94b28b", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "31d6d0aa-0588-4753-9f38-cbc33fae70e8", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "48f593c5-2012-482f-91c4-78aefe012969", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "8153a5e3-466b-466b-b3c9-1df169c55fa6", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "77268b37-7030-4f28-9c6c-fe02fba971d3", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "3fb8043c-b824-408c-bbdc-08e17be880ab", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "dc27d8c1-7c50-4c34-a7fe-36d51ce3fcd7", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "0b18c933-2747-4684-a62c-965f701ceb1e", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "1c60fe23-5960-4ebc-ab65-f1b0fb18cc05", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.20" } }, "nbformat": 4, "nbformat_minor": 5 }