Commit ·
de7e7ce
1
Parent(s): 8fe9501
fix: block grammar model from removing/normalizing punctuation marks
Browse files- src/app.py +19 -0
src/app.py
CHANGED
|
@@ -1411,6 +1411,25 @@ def analyze_text():
|
|
| 1411 |
)
|
| 1412 |
continue
|
| 1413 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1414 |
# Reject grammar hallucinations (e.g. جالس→جاكسون)
|
| 1415 |
orig_text = d.get('original', '')
|
| 1416 |
corr_text = d.get('correction', '')
|
|
|
|
| 1411 |
)
|
| 1412 |
continue
|
| 1413 |
|
| 1414 |
+
# ── Punctuation-protection guard ──
|
| 1415 |
+
# Grammar model sometimes strips/normalizes punctuation
|
| 1416 |
+
# (removes full stops, commas, etc.). Reject grammar diffs
|
| 1417 |
+
# that only differ in punctuation marks — grammar should
|
| 1418 |
+
# only fix grammar, not touch punctuation.
|
| 1419 |
+
_orig_t = d.get('original', '')
|
| 1420 |
+
_corr_t = d.get('correction', '')
|
| 1421 |
+
_PUNC_CHARS = set('.,;:!?؟،؛。·…•–—\u200f\u200e')
|
| 1422 |
+
_orig_no_punc = ''.join(c for c in _orig_t if c not in _PUNC_CHARS)
|
| 1423 |
+
_corr_no_punc = ''.join(c for c in _corr_t if c not in _PUNC_CHARS)
|
| 1424 |
+
if _orig_no_punc == _corr_no_punc:
|
| 1425 |
+
# Only punctuation was changed — block it
|
| 1426 |
+
if _orig_t != _corr_t:
|
| 1427 |
+
logger.info(
|
| 1428 |
+
f"[GRAMMAR] Blocked punctuation change: "
|
| 1429 |
+
f"'{_orig_t}' → '{_corr_t}' (grammar must not alter punctuation)"
|
| 1430 |
+
)
|
| 1431 |
+
continue
|
| 1432 |
+
|
| 1433 |
# Reject grammar hallucinations (e.g. جالس→جاكسون)
|
| 1434 |
orig_text = d.get('original', '')
|
| 1435 |
corr_text = d.get('correction', '')
|