| from unlimited_ocr_rdna4.postprocess import EOS_STOP, clean_model_output, repetition_warning | |
| def test_clean_model_output_preserves_unicode_and_content() -> None: | |
| raw = ( | |
| "<|det|>header [1, 2, 3, 4]<|/det|>Überblick\n" | |
| "<|det|>table [1, 2, 3, 4]<|/det|><table><tr><td>€48.50</td></tr></table>\n" | |
| "[Non-Text]\n" | |
| "\\coloneqq" | |
| f"{EOS_STOP}" | |
| ) | |
| assert clean_model_output(raw) == "Überblick\n<table><tr><td>€48.50</td></tr></table>\n\\coloneqq" | |
| def test_clean_model_output_removes_combined_reference_tag() -> None: | |
| raw = "<|ref|>title<|/ref|><|det|>[[1, 2, 3, 4]]<|/det|>Actual title" | |
| assert clean_model_output(raw) == "titleActual title" | |
| def test_clean_model_output_preserves_ordinary_words_and_tex() -> None: | |
| raw = "NonTextile material\nA \\eqqcolon B\nembedded NonText label" | |
| assert clean_model_output(raw) == raw | |
| def test_clean_model_output_preserves_malformed_unclosed_tags() -> None: | |
| raw = "before <|det|>coordinate and recognized content" | |
| assert clean_model_output(raw) == raw | |
| def test_repetition_warning_detects_three_lines() -> None: | |
| text = "start\nrepeating output\nrepeating output\nrepeating output" | |
| assert "repeated line" in (repetition_warning(text) or "") | |
| def test_repetition_warning_accepts_normal_text() -> None: | |
| assert repetition_warning("first\nsecond\nthird") is None | |