Commit ·
b7e415b
1
Parent(s): 03801c6
fix: Apply hamza+ta-marbuta corrections AFTER validation steps to prevent reversion
Browse filesRoot cause: Steps 8-10 in correct() compared edit distance to INPUT text,
which IS the error. The postprocessor's hamza fixes (انا→أنا) were being
reverted because 'انا' is closer to input 'انا' than 'أنا' is.
src/nlp/spelling/araspell_rules.py
CHANGED
|
@@ -1530,5 +1530,12 @@ class ArabicSpellChecker:
|
|
| 1530 |
raw_valid, _ = self.validator.validate(original, raw_model_output, "mixed")
|
| 1531 |
if raw_valid:
|
| 1532 |
result = raw_model_output
|
| 1533 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1534 |
return result
|
|
|
|
| 1530 |
raw_valid, _ = self.validator.validate(original, raw_model_output, "mixed")
|
| 1531 |
if raw_valid:
|
| 1532 |
result = raw_model_output
|
| 1533 |
+
# ── FINAL PASS: Hamza whitelist + Ta Marbuta fixes (unrevertable) ──
|
| 1534 |
+
# These are applied AFTER all validation/safety steps so they can't
|
| 1535 |
+
# be undone by Steps 8-10 which compare against raw_model_output.
|
| 1536 |
+
# The root issue: Steps 8-10 use edit distance to INPUT (which has errors)
|
| 1537 |
+
# so they revert corrections back to the erroneous form.
|
| 1538 |
+
result = AraSpellPostProcessor.fix_common_hamza(result)
|
| 1539 |
+
result = AraSpellPostProcessor.fix_ha_ta_marbuta(result, vocab_manager=self.vocab_manager)
|
| 1540 |
+
|
| 1541 |
return result
|