Spaces:
Running
Running
File size: 15,817 Bytes
e16aadc | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 | """文本场景比对管线:OCR提取 + 语义嵌入 + SSIM结构比对 + 印章/手写检测"""
import time
import cv2
import numpy as np
import torch
from skimage.metrics import structural_similarity as ssim
from module.config import (
TEXT_SIM_WEIGHT, SSIM_WEIGHT, SEAL_WEIGHT, TEXT_SCENE_THRESHOLD,
TEXT_SCENE_BGE_THRESHOLD,
)
def _cv_imread(path, flags=cv2.IMREAD_COLOR):
"""支持中文路径的cv2.imread"""
return cv2.imdecode(np.fromfile(path, dtype=np.uint8), flags)
# ==================== OCR文字提取 ====================
def extract_text(image_path, ocr_engine):
"""使用PaddleOCR提取图片中的全部文字
Args:
image_path: 图片路径
ocr_engine: PaddleOCR引擎实例
Returns:
full_text: 拼接后的完整文本
text_lines: 每行文字的列表(含置信度)
"""
result = ocr_engine.ocr(image_path, cls=True)
text_lines = []
if result and result[0]:
for line in result[0]:
text = line[1][0]
conf = line[1][1]
text_lines.append({'text': text, 'confidence': float(conf)})
full_text = ' '.join([t['text'] for t in text_lines])
return full_text, text_lines
# ==================== 语义嵌入比对 ====================
def compute_text_similarity(text1, text2, bge_tokenizer, bge_model):
"""使用BGE-small-zh计算两段文本的语义相似度
Args:
text1, text2: 待比较的两段文本
bge_tokenizer: BGE分词器实例
bge_model: BGE模型实例
Returns:
similarity: 余弦相似度,范围[0,1]
"""
if not text1.strip() or not text2.strip():
return 0.0
def _encode(text):
"""编码单段文本为归一化向量"""
encoded = bge_tokenizer(text, padding=True, truncation=True, return_tensors='pt', max_length=512)
if torch.cuda.is_available():
encoded = {k: v.cuda() for k, v in encoded.items()}
with torch.no_grad():
outputs = bge_model(**encoded)
cls_embedding = outputs.last_hidden_state[:, 0]
cls_embedding = torch.nn.functional.normalize(cls_embedding, p=2, dim=1)
return cls_embedding.cpu().numpy()
emb1 = _encode(text1)
emb2 = _encode(text2)
sim = float(np.dot(emb1[0], emb2[0]))
return sim
# ==================== SSIM结构比对 ====================
def compute_ssim(image_path1, image_path2):
"""计算两张图片的SSIM结构相似度
将图片缩放到相同尺寸后转为灰度计算SSIM
Returns:
ssim_score: SSIM值,范围[0,1]
"""
img1 = _cv_imread(image_path1)
img2 = _cv_imread(image_path2)
if img1 is None or img2 is None:
return 0.0
h = min(img1.shape[0], img2.shape[0])
w = min(img1.shape[1], img2.shape[1])
img1 = cv2.resize(img1, (w, h))
img2 = cv2.resize(img2, (w, h))
gray1 = cv2.cvtColor(img1, cv2.COLOR_BGR2GRAY)
gray2 = cv2.cvtColor(img2, cv2.COLOR_BGR2GRAY)
score, _ = ssim(gray1, gray2, full=True)
return float(score)
# ==================== 印章/手写检测 ====================
def detect_seals_and_handwriting(image_path, min_area_ratio=0.001):
"""检测图片中的红色印章和手写笔迹区域
印章检测原理:HSV颜色空间中提取红色区域
Args:
image_path: 图片路径
min_area_ratio: 最小区域面积占比,过滤噪声
Returns:
seals: 印章区域列表,每项含 {'bbox', 'area', 'area_ratio'}
seal_total_ratio: 印章总面积占比
"""
img = _cv_imread(image_path)
if img is None:
return [], 0.0
h, w = img.shape[:2]
total_area = h * w
# --- 印章检测:HSV红色区域 ---
hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
# 红色在HSV中有两段:0-10 和 160-180
mask1 = cv2.inRange(hsv, np.array([0, 70, 50]), np.array([10, 255, 255]))
mask2 = cv2.inRange(hsv, np.array([160, 70, 50]), np.array([180, 255, 255]))
red_mask = cv2.bitwise_or(mask1, mask2)
# 形态学操作去噪
kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (5, 5))
red_mask = cv2.morphologyEx(red_mask, cv2.MORPH_CLOSE, kernel, iterations=2)
red_mask = cv2.morphologyEx(red_mask, cv2.MORPH_OPEN, kernel, iterations=1)
# 查找轮廓
contours, _ = cv2.findContours(red_mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
seals = []
min_area = total_area * min_area_ratio
for cnt in contours:
area = cv2.contourArea(cnt)
if area >= min_area:
x, y, bw, bh = cv2.boundingRect(cnt)
seals.append({
'bbox': (x, y, bw, bh),
'area': int(area),
'area_ratio': round(area / total_area, 4),
})
seal_total_ratio = sum(s['area_ratio'] for s in seals)
return seals, round(seal_total_ratio, 4)
def compare_seals(seals1, seals2):
"""比对两张图的印章区域
比较逻辑:
- 两张图都有印章 → 比较数量和面积占比差异
- 一张有一张无 → 印章不匹配
- 两张都无 → 印章项跳过
Returns:
match: bool或None,印章是否匹配(None表示两张都无印章)
detail: 详细信息字典
"""
has_seal1 = len(seals1) > 0
has_seal2 = len(seals2) > 0
ratio1 = sum(s['area_ratio'] for s in seals1)
ratio2 = sum(s['area_ratio'] for s in seals2)
detail = {
'img1_seal_count': len(seals1),
'img2_seal_count': len(seals2),
'img1_seal_ratio': round(ratio1, 4),
'img2_seal_ratio': round(ratio2, 4),
}
if not has_seal1 and not has_seal2:
return None, detail
if has_seal1 != has_seal2:
detail['reason'] = '一张有印章一张无'
return False, detail
# 两张都有印章:比较数量和面积
count_match = len(seals1) == len(seals2)
ratio_diff = abs(ratio1 - ratio2)
area_match = ratio_diff < 0.02
match = count_match and area_match
detail['count_match'] = count_match
detail['area_diff'] = round(ratio_diff, 4)
detail['area_match'] = area_match
return match, detail
def compare_text_scene_cached(image_path, cached_h, ocr_engine,
bge_tokenizer, bge_model,
precomputed_text=None, precomputed_bge=None):
"""文本场景缓存比对: 仅实时 OCR/BGE/Seal 查询图,历史图从缓存读取
Args:
image_path: 查询图片路径
cached_h: 预存储的历史图缓存 dict (ocr_text, bge_vector, seal_count, seal_ratio)
ocr_engine: PaddleOCR引擎
bge_tokenizer: BGE分词器
bge_model: BGE模型
Returns:
result dict (与 compare_text_scene() 兼容的格式)
"""
start_time = time.time()
# 1. OCR 仅查询图 (优先使用预计算结果)
if precomputed_text is not None:
text1 = precomputed_text
lines1 = []
ocr_time = 0.0
else:
text1, lines1 = extract_text(image_path, ocr_engine)
ocr_time = time.time() - start_time
text2 = cached_h.get('ocr_text', '')
# 2. 语义嵌入 (优先使用预计算向量)
if precomputed_bge is not None:
emb_q = precomputed_bge
emb_h = cached_h.get('bge_vector', np.zeros(512, dtype=np.float32))
text_sim = float(np.dot(emb_q, emb_h)) if text1.strip() else 0.0
embed_time = 0.0
elif not text1.strip():
text_sim = 0.0
embed_time = 0.0
else:
t_embed = time.time()
def _encode_q(text):
encoded = bge_tokenizer(text, padding=True, truncation=True,
return_tensors='pt', max_length=512)
if torch.cuda.is_available():
encoded = {k: v.cuda() for k, v in encoded.items()}
with torch.no_grad():
outputs = bge_model(**encoded)
cls_embedding = outputs.last_hidden_state[:, 0]
cls_embedding = torch.nn.functional.normalize(cls_embedding, p=2, dim=1)
return cls_embedding.cpu().numpy()[0]
emb_q = _encode_q(text1)
emb_h = cached_h.get('bge_vector', np.zeros(512, dtype=np.float32))
text_sim = float(np.dot(emb_q, emb_h))
embed_time = time.time() - t_embed
# 3. SSIM 仍需两张图
t_ssim = time.time()
ssim_score = compute_ssim(image_path, cached_h['path'])
ssim_time = time.time() - t_ssim
# 4. 印章:仅查询图(历史图用缓存)
t_seal = time.time()
seals_q, ratio_q = detect_seals_and_handwriting(image_path)
cnt_h = cached_h.get('seal_count', 0)
ratio_h = cached_h.get('seal_ratio', 0.0)
has_q, has_h = len(seals_q) > 0, cnt_h > 0
if not has_q and not has_h:
seal_bonus = 0.5
elif has_q != has_h:
seal_bonus = 0.0
else:
count_match = len(seals_q) == cnt_h
area_match = abs(ratio_q - ratio_h) < 0.02
seal_bonus = 1.0 if (count_match and area_match) else 0.0
seal_time = time.time() - t_seal
# 5. 综合判定
final_score = text_sim * TEXT_SIM_WEIGHT + ssim_score * SSIM_WEIGHT + seal_bonus * SEAL_WEIGHT
is_same = final_score >= TEXT_SCENE_THRESHOLD
total_time = time.time() - start_time
result = {
'is_same_scene': bool(is_same),
'scene_type': 'text',
'final_score': round(final_score, 4),
'text_similarity': round(text_sim, 4),
'ssim_score': round(ssim_score, 4),
'seal_bonus': seal_bonus,
'seal_match': None if (not has_q and not has_h) else (has_q and has_h),
'seal_detail': {
'img1_seal_count': len(seals_q), 'img2_seal_count': cnt_h,
'img1_seal_ratio': ratio_q, 'img2_seal_ratio': ratio_h,
},
'img1_text_length': len(text1), 'img2_text_length': len(text2),
'img1_seal_count': len(seals_q), 'img2_seal_count': cnt_h,
'img1_seal_ratio': ratio_q, 'img2_seal_ratio': ratio_h,
'similarity_score': round(final_score, 4),
'match_count': len(seals_q) + cnt_h,
'inlier_ratio': round(text_sim, 4),
'avg_confidence': round(ssim_score, 4),
'time_breakdown': {
'ocr': round(ocr_time, 2), 'embedding': round(embed_time, 2),
'ssim': round(ssim_time, 2), 'seal': round(seal_time, 2),
'total': round(total_time, 2),
},
}
return result
def compare_text_scene_pure_bge(image_path, cached_h, ocr_engine,
bge_tokenizer, bge_model,
precomputed_text=None, precomputed_bge=None):
"""Pure BGE: only OCR->BGE cosine similarity, no SSIM/seal"""
start_time = time.time()
print(f' [TIME-PURE-BGE] enter, precomputed_text={precomputed_text is not None}, precomputed_bge={precomputed_bge is not None}')
if precomputed_text is not None:
text1 = precomputed_text
ocr_time = 0.0
else:
text1, _ = extract_text(image_path, ocr_engine)
ocr_time = time.time() - start_time
if precomputed_bge is not None:
emb_q = precomputed_bge
emb_h = cached_h.get('_vector', cached_h.get('bge_vector', np.zeros(512, dtype=np.float32)))
text_sim = float(np.dot(emb_q, emb_h)) if text1.strip() else 0.0
embed_time = 0.0
elif not text1.strip():
text_sim = 0.0
embed_time = 0.0
else:
t_embed = time.time()
def _encode_q(text):
encoded = bge_tokenizer(text, padding=True, truncation=True,
return_tensors='pt', max_length=512)
if torch.cuda.is_available():
encoded = {k: v.cuda() for k, v in encoded.items()}
with torch.no_grad():
outputs = bge_model(**encoded)
cls_embedding = outputs.last_hidden_state[:, 0]
cls_embedding = torch.nn.functional.normalize(cls_embedding, p=2, dim=1)
return cls_embedding.cpu().numpy()[0]
emb_q = _encode_q(text1)
emb_h = cached_h.get('_vector', cached_h.get('bge_vector', np.zeros(512, dtype=np.float32)))
text_sim = float(np.dot(emb_q, emb_h))
embed_time = time.time() - t_embed
is_same = text_sim >= TEXT_SCENE_BGE_THRESHOLD
total_time = time.time() - start_time
print(f' [TIME-PURE-BGE] text_sim={text_sim:.4f} is_same={is_same} total={total_time:.3f}s')
result = {
'is_same_scene': bool(is_same),
'scene_type': 'text',
'final_score': round(text_sim, 4),
'text_similarity': round(text_sim, 4),
'ssim_score': 0.0,
'seal_bonus': 0.0,
'similarity_score': round(text_sim, 4),
'match_count': 0,
'inlier_ratio': round(text_sim, 4),
'avg_confidence': 0.0,
'time_breakdown': {
'ocr': round(ocr_time, 2), 'embedding': round(embed_time, 2),
'ssim': 0.0, 'seal': 0.0, 'total': round(total_time, 2),
},
}
return result
# ==================== 综合判定 ====================
def compare_text_scene(image_path1, image_path2, ocr_engine, bge_tokenizer, bge_model):
"""文本场景比对主函数
Args:
image_path1: 第一张图片路径
image_path2: 第二张图片路径
ocr_engine: PaddleOCR引擎实例
bge_tokenizer: BGE分词器实例
bge_model: BGE模型实例
Returns:
result: dict,含各项分数和最终判定
"""
start_time = time.time()
# 1. OCR文字提取
text1, lines1 = extract_text(image_path1, ocr_engine)
text2, lines2 = extract_text(image_path2, ocr_engine)
ocr_time = time.time() - start_time
# 2. 语义嵌入比对
text_sim = compute_text_similarity(text1, text2, bge_tokenizer, bge_model)
embed_time = time.time() - start_time - ocr_time
# 3. SSIM结构比对
ssim_score = compute_ssim(image_path1, image_path2)
ssim_time = time.time() - start_time - ocr_time - embed_time
# 4. 印章检测与比对
seals1, seal_ratio1 = detect_seals_and_handwriting(image_path1)
seals2, seal_ratio2 = detect_seals_and_handwriting(image_path2)
seal_match, seal_detail = compare_seals(seals1, seals2)
seal_time = time.time() - start_time - ocr_time - embed_time - ssim_time
# 5. 综合判定
if seal_match is True:
seal_bonus = 1.0
elif seal_match is False:
seal_bonus = 0.0
else:
seal_bonus = 0.5 # 两张图都没有印章,中性
final_score = text_sim * TEXT_SIM_WEIGHT + ssim_score * SSIM_WEIGHT + seal_bonus * SEAL_WEIGHT
is_same = final_score >= TEXT_SCENE_THRESHOLD
total_time = time.time() - start_time
result = {
'is_same_scene': bool(is_same),
'scene_type': 'text',
'final_score': round(final_score, 4),
'text_similarity': round(text_sim, 4),
'ssim_score': round(ssim_score, 4),
'seal_bonus': seal_bonus,
'seal_match': seal_match,
'seal_detail': seal_detail,
'img1_text_length': len(text1),
'img2_text_length': len(text2),
'img1_seal_count': len(seals1),
'img2_seal_count': len(seals2),
'img1_seal_ratio': seal_ratio1,
'img2_seal_ratio': seal_ratio2,
'similarity_score': round(final_score, 4),
'match_count': len(seals1) + len(seals2), # 文本场景用印章数代替
'inlier_ratio': round(text_sim, 4),
'avg_confidence': round(ssim_score, 4),
'time_breakdown': {
'ocr': round(ocr_time, 2),
'embedding': round(embed_time, 2),
'ssim': round(ssim_time, 2),
'seal': round(seal_time, 2),
'total': round(total_time, 2),
},
}
return result
|