youssefreda9 commited on
Commit
de7e7ce
·
1 Parent(s): 8fe9501

fix: block grammar model from removing/normalizing punctuation marks

Browse files
Files changed (1) hide show
  1. src/app.py +19 -0
src/app.py CHANGED
@@ -1411,6 +1411,25 @@ def analyze_text():
1411
  )
1412
  continue
1413
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1414
  # Reject grammar hallucinations (e.g. جالس→جاكسون)
1415
  orig_text = d.get('original', '')
1416
  corr_text = d.get('correction', '')
 
1411
  )
1412
  continue
1413
 
1414
+ # ── Punctuation-protection guard ──
1415
+ # Grammar model sometimes strips/normalizes punctuation
1416
+ # (removes full stops, commas, etc.). Reject grammar diffs
1417
+ # that only differ in punctuation marks — grammar should
1418
+ # only fix grammar, not touch punctuation.
1419
+ _orig_t = d.get('original', '')
1420
+ _corr_t = d.get('correction', '')
1421
+ _PUNC_CHARS = set('.,;:!?؟،؛。·…•–—\u200f\u200e')
1422
+ _orig_no_punc = ''.join(c for c in _orig_t if c not in _PUNC_CHARS)
1423
+ _corr_no_punc = ''.join(c for c in _corr_t if c not in _PUNC_CHARS)
1424
+ if _orig_no_punc == _corr_no_punc:
1425
+ # Only punctuation was changed — block it
1426
+ if _orig_t != _corr_t:
1427
+ logger.info(
1428
+ f"[GRAMMAR] Blocked punctuation change: "
1429
+ f"'{_orig_t}' → '{_corr_t}' (grammar must not alter punctuation)"
1430
+ )
1431
+ continue
1432
+
1433
  # Reject grammar hallucinations (e.g. جالس→جاكسون)
1434
  orig_text = d.get('original', '')
1435
  corr_text = d.get('correction', '')