#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ Synthetic Text Detection Dataset Generator for Cham OCR (Phase 4). Generates realistic multi-line Cham document and manuscript pages with exact polygon bounding box labels in PaddleOCR SimpleDataSet format. """ import os import sys import cv2 import json import random import argparse import numpy as np from PIL import Image, ImageDraw, ImageFont # Force UTF-8 stdout encoding on Windows if sys.stdout and sys.stdout.encoding != 'utf-8': try: sys.stdout.reconfigure(encoding='utf-8') except AttributeError: pass # Monkeypatch NumPy 2.x compatibility if not hasattr(np, 'sctypes'): np.sctypes = { 'int': [np.int8, np.int16, np.int32, np.int64], 'uint': [np.uint8, np.uint16, np.uint32, np.uint64], 'float': [np.float16, np.float32, np.float64], 'complex': [np.complex64, np.complex128], 'others': [bool, object, bytes, str] } if not hasattr(np, 'bool'): np.bool = bool if not hasattr(np, 'int'): np.int = int if not hasattr(np, 'float'): np.float = float if not hasattr(np, 'typeDict'): np.typeDict = {} PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.insert(0, os.path.join(PROJECT_ROOT, "scripts")) _FONT_CACHE = {} def get_cached_font(font_path, size): key = (font_path, size) if key not in _FONT_CACHE: if os.path.exists(font_path): _FONT_CACHE[key] = ImageFont.truetype(font_path, size) else: _FONT_CACHE[key] = ImageFont.load_default() return _FONT_CACHE[key] def apply_old_paper_background(width, height): """ Generates a realistic antique parchment/paper background with varied tones, vignetting, and stains. """ base_r = random.randint(220, 245) base_g = random.randint(210, 235) base_b = random.randint(180, 210) img = np.full((height, width, 3), (base_b, base_g, base_r), dtype=np.uint8) # 1. Low frequency gradient/shading grid_h, grid_w = max(4, height // 60), max(4, width // 60) low_noise = np.random.normal(0, 14, (grid_h, grid_w)).astype(np.float32) low_noise = cv2.resize(low_noise, (width, height), interpolation=cv2.INTER_CUBIC) # 2. Fine grain noise fine_noise = np.random.normal(0, 5, (height, width)).astype(np.float32) total_noise = low_noise + fine_noise for c in range(3): channel = img[:, :, c].astype(np.float32) + total_noise img[:, :, c] = np.clip(channel, 0, 255).astype(np.uint8) # 3. Vignette effect (darker edges) X = np.linspace(-1, 1, width) Y = np.linspace(-1, 1, height) mesh_x, mesh_y = np.meshgrid(X, Y) dist = np.sqrt(mesh_x**2 + mesh_y**2) vignette = 1.0 - np.clip((dist - 0.5) * 0.35, 0, 0.45) for c in range(3): img[:, :, c] = (img[:, :, c].astype(np.float32) * vignette).astype(np.uint8) # 4. Optional water/ink stain spots if random.random() < 0.5: num_spots = random.randint(1, 3) for _ in range(num_spots): cx = random.randint(60, width - 60) cy = random.randint(60, height - 60) radius = random.randint(20, 50) cv2.circle(img, (cx, cy), radius, (max(0, base_b - 25), max(0, base_g - 25), max(0, base_r - 30)), -1) img = cv2.GaussianBlur(img, (31, 31), 0) return Image.fromarray(cv2.cvtColor(img, cv2.COLOR_BGR2RGB)) def warp_points(pts, matrix, amp, freq, phase): """ Applies perspective transformation and non-linear wave deformation to ground-truth polygon vertices. """ pts_arr = np.array(pts, dtype=np.float32).reshape(-1, 1, 2) # 1. Perspective transformation warped_persp = cv2.perspectiveTransform(pts_arr, matrix) # 2. Wave deformation (y_w = y + dy) final_pts = [] for pt in warped_persp: x, y = pt[0] dy = amp * np.sin(x * freq + phase) final_pts.append([round(float(x), 1), round(float(y + dy), 1)]) return final_pts def draw_distorted_page_with_labels(lines, fonts, width=800, height=650): font_path = random.choice(fonts) font_size = random.randint(20, 26) font = get_cached_font(font_path, font_size) bg = apply_old_paper_background(width, height) draw = ImageDraw.Draw(bg) y_spacing = random.randint(font_size + 24, font_size + 42) y_start = random.randint(40, 70) line_boxes = [] for idx, text in enumerate(lines): y_pos = y_start + idx * y_spacing if y_pos + font_size + 15 > height: break x_pos = random.randint(40, 80) ink_r = random.randint(20, 45) ink_g = random.randint(20, 40) ink_b = random.randint(20, 38) draw.text((x_pos, y_pos), text, font=font, fill=(ink_r, ink_g, ink_b)) try: bbox = font.getbbox(text) text_w = bbox[2] - bbox[0] text_h = bbox[3] - bbox[1] except Exception: text_w, text_h = len(text) * 14, font_size pad_x = 4 pad_y = 3 box = [ [x_pos - pad_x, y_pos - pad_y], [x_pos + text_w + pad_x, y_pos - pad_y], [x_pos + text_w + pad_x, y_pos + text_h + pad_y], [x_pos - pad_x, y_pos + text_h + pad_y] ] line_boxes.append((text, box)) img_np = np.array(bg) h, w = img_np.shape[:2] amp = random.uniform(3.0, 8.0) freq = random.uniform(0.004, 0.010) phase = random.uniform(0, 2 * np.pi) pts1 = np.float32([[0, 0], [w, 0], [0, h], [w, h]]) tx1 = random.uniform(0, w * 0.03) ty1 = random.uniform(0, h * 0.03) tx2 = random.uniform(0, w * 0.03) ty2 = random.uniform(0, h * 0.03) tx3 = random.uniform(0, w * 0.03) ty3 = random.uniform(0, h * 0.03) tx4 = random.uniform(0, w * 0.03) ty4 = random.uniform(0, h * 0.03) pts2 = np.float32([ [0 + tx1, 0 + ty1], [w - tx2, 0 + ty2], [0 + tx3, h - ty3], [w - tx4, h - ty4] ]) matrix = cv2.getPerspectiveTransform(pts1, pts2) x_idx = np.arange(w) y_idx = np.arange(h) X, Y = np.meshgrid(x_idx, y_idx) dy = amp * np.sin(X * freq + phase) map_x = X.astype(np.float32) map_y = (Y + dy).astype(np.float32) img_wave = cv2.remap(img_np, map_x, map_y, cv2.INTER_LINEAR, borderMode=cv2.BORDER_REPLICATE) img_warped = cv2.warpPerspective(img_wave, matrix, (w, h), borderMode=cv2.BORDER_REPLICATE) warped_labels = [] for text, box in line_boxes: warped_box = warp_points(box, matrix, amp, freq, phase) warped_labels.append({ "transcription": text, "points": warped_box }) return Image.fromarray(img_warped), warped_labels def main(): parser = argparse.ArgumentParser(description="Generate synthetic Cham Detection dataset") parser.add_argument("--num_train", type=int, default=50, help="Number of training pages") parser.add_argument("--num_val", type=int, default=10, help="Number of validation pages") parser.add_argument("--output_dir", type=str, default=None, help="Output directory") args = parser.parse_args() print("📂 Starting Cham Text Detection Synthetic Generator (Phase 4)...") corpus_file = os.path.join(PROJECT_ROOT, "data", "corpus", "cham_text.txt") if not os.path.exists(corpus_file): corpus_file = os.path.join(PROJECT_ROOT, "output_v22", "paddleocr_cham_finetune", "data", "corpus", "cham_text.txt") if not os.path.exists(corpus_file): print(f"❌ Corpus file not found at {corpus_file}!") return with open(corpus_file, "r", encoding="utf-8") as f: sentences = [line.strip() for line in f if line.strip() and not line.strip().startswith('#')] print(f" Loaded {len(sentences)} corpus lines.") fonts_dir = os.path.join(PROJECT_ROOT, "data", "fonts") if not os.path.exists(fonts_dir): fonts_dir = os.path.join(PROJECT_ROOT, "output_v22", "paddleocr_cham_finetune", "data", "fonts") fonts = [ os.path.join(fonts_dir, f) for f in ["NotoSansCham-Regular.ttf", "NotoSansCham-Bold.ttf", "NotoSansCham-Black.ttf"] if os.path.exists(os.path.join(fonts_dir, f)) ] if not fonts: fonts = [os.path.join(fonts_dir, "NotoSansCham-Regular.ttf")] print(f" Using fonts: {[os.path.basename(f) for f in fonts]}") det_root = args.output_dir or os.path.join(PROJECT_ROOT, "data", "detector") train_img_dir = os.path.join(det_root, "train_images") val_img_dir = os.path.join(det_root, "val_images") os.makedirs(train_img_dir, exist_ok=True) os.makedirs(val_img_dir, exist_ok=True) print(f" Sinh {args.num_train} trang huấn luyện (train_images)...") train_label_entries = [] for i in range(args.num_train): num_lines = random.randint(4, 9) lines = random.sample(sentences, min(num_lines, len(sentences))) img, labels = draw_distorted_page_with_labels(lines, fonts) img_name = f"det_train_{i:05d}.png" img.save(os.path.join(train_img_dir, img_name)) train_label_entries.append(f"train_images/{img_name}\t{json.dumps(labels, ensure_ascii=False)}") with open(os.path.join(det_root, "det_train_label.txt"), "w", encoding="utf-8", newline="\n") as f_out: f_out.write("\n".join(train_label_entries) + "\n") print(f" Sinh {args.num_val} trang kiểm thử (val_images)...") val_label_entries = [] for i in range(args.num_val): num_lines = random.randint(4, 9) lines = random.sample(sentences, min(num_lines, len(sentences))) img, labels = draw_distorted_page_with_labels(lines, fonts) img_name = f"det_val_{i:05d}.png" img.save(os.path.join(val_img_dir, img_name)) val_label_entries.append(f"val_images/{img_name}\t{json.dumps(labels, ensure_ascii=False)}") with open(os.path.join(det_root, "det_val_label.txt"), "w", encoding="utf-8", newline="\n") as f_out: f_out.write("\n".join(val_label_entries) + "\n") print(f"🎉 Hoàn tất sinh tập dữ liệu Detection tại: {det_root}") print(f" - Train: {len(train_label_entries)} ảnh") print(f" - Val: {len(val_label_entries)} ảnh") if __name__ == '__main__': main()