Update app.py

app.py

@@ -2739,256 +2739,256 @@ def process_text(input_text):
             para_simplified = []
 
             for original_sent in sents:
-    hidden, lists = hide_paren_lists(original_sent)
-    working_set = deque([(hidden, lists)])
-    seen = set()
-    sentence_final = []
+                hidden, lists = hide_paren_lists(original_sent)
+                working_set = deque([(hidden, lists)])
+                seen = set()
+                sentence_final = []
+
+                iters = 0
+                while working_set:
+                    iters += 1
+
+                    if iters > MAX_ITERS:
+                        print(
+                            f"ABORT: exceeded {MAX_ITERS} iterations; "
+                            f"draining {len(working_set)} item(s) to final"
+                        )
+                        while working_set:
+                            t, ls = working_set.popleft()
+                            for s in expand_paren_lists(t, ls):
+                                sentence_final.append(clean_output(s))
+                break
 
-    iters = 0
-    while working_set:
-        iters += 1
+            current, lists = working_set.popleft()
+            current = current.strip()
+            if not current:
+                continue
+
+            current = clean_output(current)
+            if not current:
+                continue
+
+            key = re.sub(r"\s+", " ", current.lower())
+            if key in seen:
+                print(f"SKIP duplicate: {current[:80]}...")
+                for s in expand_paren_lists(current, lists):
+                    sentence_final.append(clean_output(s))
+                continue
+            seen.add(key)
 
-        if iters > MAX_ITERS:
             print(
-                f"ABORT: exceeded {MAX_ITERS} iterations; "
-                f"draining {len(working_set)} item(s) to final"
+                f"\n--- iter {iters} | queue={len(working_set)} | "
+                f"{current[:100]}..."
             )
-            while working_set:
-                t, ls = working_set.popleft()
-                for s in expand_paren_lists(t, ls):
-                    sentence_final.append(clean_output(s))
-            break
 
-        current, lists = working_set.popleft()
-        current = current.strip()
-        if not current:
-            continue
+            tagged_list = tag_all_signs_in_sent(current)
 
-        current = clean_output(current)
-        if not current:
-            continue
+            has_ssccv = any(
+                any(x in t for x in ["M:Y_SSCCV", "M:Y_ESCCV"])
+                for t in tagged_list
+            )
+            has_ssmn = any(
+                any(x in t for x in ["M:Y_SSMN", "M:Y_ESMN"])
+                for t in tagged_list
+            )
+            has_ccv = any(
+                "<M:Y_CCV>" in t or "M:Y_CCV" in t for t in tagged_list
+            )
+            has_cmv1 = any(
+                "<M:Y_CMV1>" in t or "M:Y_CMV1" in t for t in tagged_list
+            )
+            has_cmn1 = any(
+                "<M:Y_CMN1>" in t or "M:Y_CMN1" in t for t in tagged_list
+            )
+            has_cln = any(
+                "<M:Y_CLN>" in t or "M:Y_CLN" in t for t in tagged_list
+            )
 
-        key = re.sub(r"\s+", " ", current.lower())
-        if key in seen:
-            print(f"SKIP duplicate: {current[:80]}...")
-            for s in expand_paren_lists(current, lists):
-                sentence_final.append(clean_output(s))
-            continue
-        seen.add(key)
+            skipped_ssccv = set()
+            mode = xml_tag = None
+            simplifier = span_fn = None
+            bracketed_version = None
+
+            # SSCCV first, but skip cognitive/communicative VPs
+            if has_ssccv:
+                while True:
+                    bracketed_version = build_single_sign_bracketed(
+                        tagged_list,
+                        mode="ssccv",
+                        skip_indices=skipped_ssccv,
+                    )
+                    if not bracketed_version:
+                        break
 
-        print(
-            f"\n--- iter {iters} | queue={len(working_set)} | "
-            f"{current[:100]}..."
-        )
+                    print(f"\nProcessing: {current}")
+                    print(
+                        f"Bracketed version (ssccv): {bracketed_version}"
+                    )
 
-        tagged_list = tag_all_signs_in_sent(current)
+                    cleaned = clean_for_spaCy(bracketed_version)
+                    tokens, pos, lemmas, preds = tag_SSCCV_span(cleaned)
+                    tokens, pos, lemmas, preds = drop_bracket_tokens(
+                        tokens, pos, lemmas, preds
+                    )
 
-        has_ssccv = any(
-            any(x in t for x in ["M:Y_SSCCV", "M:Y_ESCCV"])
-            for t in tagged_list
-        )
-        has_ssmn = any(
-            any(x in t for x in ["M:Y_SSMN", "M:Y_ESMN"])
-            for t in tagged_list
-        )
-        has_ccv = any(
-            "<M:Y_CCV>" in t or "M:Y_CCV" in t for t in tagged_list
-        )
-        has_cmv1 = any(
-            "<M:Y_CMV1>" in t or "M:Y_CMV1" in t for t in tagged_list
-        )
-        has_cmn1 = any(
-            "<M:Y_CMN1>" in t or "M:Y_CMN1" in t for t in tagged_list
-        )
-        has_cln = any(
-            "<M:Y_CLN>" in t or "M:Y_CLN" in t for t in tagged_list
-        )
+                    if is_cognitive_communicative_vp(preds):
+                        ssccv_idxs = [
+                            i
+                            for i, t in enumerate(tagged_list)
+                            if "M:Y_SSCCV" in str(t)
+                            or "M:Y_ESCCV" in str(t)
+                        ]
+                        remaining = [
+                            i
+                            for i in ssccv_idxs
+                            if i not in skipped_ssccv
+                        ]
+                        if remaining:
+                            skipped_ssccv.add(remaining[0])
+                        print(
+                            "  Skip SSCCV "
+                            "(cognitive/communicative VP)"
+                        )
+                        continue
+
+                    mode = "ssccv"
+                    xml_tag = "M:Y_SSCCV"
+                    simplifier = SSCCVsimplify
+                    span_fn = tag_SSCCV_span
+                    break
 
-        skipped_ssccv = set()
-        mode = xml_tag = None
-        simplifier = span_fn = None
-        bracketed_version = None
+            if mode is None and has_ssmn:
+                mode = "ssmn"
+                xml_tag = "M:Y_SSMN"
+                simplifier = SSMNsimplify
+                span_fn = tag_SSMN_span
+            elif mode is None and has_ccv:
+                mode = "ccv"
+                xml_tag = "M:Y_CCV"
+                simplifier = CompoundSimplify
+                span_fn = tag_CCV_span
+            elif mode is None and has_cmv1:
+                mode = "cmv1"
+                xml_tag = "M:Y_CMV1"
+                simplifier = CompoundSimplify
+                span_fn = tag_CMV1_span
+            elif mode is None and has_cmn1:
+                mode = "cmn1"
+                xml_tag = "M:Y_CMN1"
+                simplifier = CompoundSimplify
+                span_fn = tag_CMN1_span
+            elif mode is None and has_cln:
+                mode = "cln"
+                xml_tag = "M:Y_CLN"
+                simplifier = CompoundSimplify
+                span_fn = tag_CLN_span
+            elif mode is None:
+                for s in expand_paren_lists(current, lists):
+                    sentence_final.append(clean_output(s))
+                continue
 
-        # SSCCV first, but skip cognitive/communicative VPs
-        if has_ssccv:
-            while True:
+            if mode != "ssccv":
                 bracketed_version = build_single_sign_bracketed(
-                    tagged_list,
-                    mode="ssccv",
-                    skip_indices=skipped_ssccv,
+                    tagged_list, mode=mode
                 )
                 if not bracketed_version:
-                    break
-
+                    for s in expand_paren_lists(current, lists):
+                        sentence_final.append(clean_output(s))
+                    continue
                 print(f"\nProcessing: {current}")
                 print(
-                    f"Bracketed version (ssccv): {bracketed_version}"
+                    f"Bracketed version ({mode}): {bracketed_version}"
                 )
-
                 cleaned = clean_for_spaCy(bracketed_version)
-                tokens, pos, lemmas, preds = tag_SSCCV_span(cleaned)
+                tokens, pos, lemmas, preds = span_fn(cleaned)
                 tokens, pos, lemmas, preds = drop_bracket_tokens(
                     tokens, pos, lemmas, preds
                 )
 
-                if is_cognitive_communicative_vp(preds):
-                    ssccv_idxs = [
-                        i
-                        for i, t in enumerate(tagged_list)
-                        if "M:Y_SSCCV" in str(t)
-                        or "M:Y_ESCCV" in str(t)
-                    ]
-                    remaining = [
-                        i
-                        for i in ssccv_idxs
-                        if i not in skipped_ssccv
-                    ]
-                    if remaining:
-                        skipped_ssccv.add(remaining[0])
-                    print(
-                        "  Skip SSCCV "
-                        "(cognitive/communicative VP)"
-                    )
-                    continue
-
-                mode = "ssccv"
-                xml_tag = "M:Y_SSCCV"
-                simplifier = SSCCVsimplify
-                span_fn = tag_SSCCV_span
-                break
-
-        if mode is None and has_ssmn:
-            mode = "ssmn"
-            xml_tag = "M:Y_SSMN"
-            simplifier = SSMNsimplify
-            span_fn = tag_SSMN_span
-        elif mode is None and has_ccv:
-            mode = "ccv"
-            xml_tag = "M:Y_CCV"
-            simplifier = CompoundSimplify
-            span_fn = tag_CCV_span
-        elif mode is None and has_cmv1:
-            mode = "cmv1"
-            xml_tag = "M:Y_CMV1"
-            simplifier = CompoundSimplify
-            span_fn = tag_CMV1_span
-        elif mode is None and has_cmn1:
-            mode = "cmn1"
-            xml_tag = "M:Y_CMN1"
-            simplifier = CompoundSimplify
-            span_fn = tag_CMN1_span
-        elif mode is None and has_cln:
-            mode = "cln"
-            xml_tag = "M:Y_CLN"
-            simplifier = CompoundSimplify
-            span_fn = tag_CLN_span
-        elif mode is None:
-            for s in expand_paren_lists(current, lists):
-                sentence_final.append(clean_output(s))
-            continue
-
-        if mode != "ssccv":
-            bracketed_version = build_single_sign_bracketed(
-                tagged_list, mode=mode
-            )
-            if not bracketed_version:
-                for s in expand_paren_lists(current, lists):
-                    sentence_final.append(clean_output(s))
-                continue
-            print(f"\nProcessing: {current}")
-            print(
-                f"Bracketed version ({mode}): {bracketed_version}"
+            cleaned_tokens, clean_pos, clean_lemmas, cleaned_preds = (
+                sign_cleaning(tokens, pos, lemmas, preds)
             )
-            cleaned = clean_for_spaCy(bracketed_version)
-            tokens, pos, lemmas, preds = span_fn(cleaned)
-            tokens, pos, lemmas, preds = drop_bracket_tokens(
-                tokens, pos, lemmas, preds
+            improved_preds = improve_predicted_tag_sequence(
+                cleaned_preds
             )
 
-        cleaned_tokens, clean_pos, clean_lemmas, cleaned_preds = (
-            sign_cleaning(tokens, pos, lemmas, preds)
-        )
-        improved_preds = improve_predicted_tag_sequence(
-            cleaned_preds
-        )
-
-        xml_sent = create_xml_sent(
-            cleaned_tokens,
-            clean_pos,
-            clean_lemmas,
-            improved_preds,
-            xml_tag,
-        )
-        simps = simplifier(xml_sent)
-        if simps:
-            simps = [fix_subject_verb_agreement(s) for s in simps]
-
-        if (
-            simps
-            and simplifier is CompoundSimplify
-            and xml_tag
-            in ("M:Y_CCV", "M:Y_CMV1", "M:Y_CMN1", "M:Y_CLN")
-        ):
-            coord = extract_coordinator_from_xml(xml_sent)
-            if coord:
-                rewritten = []
-                for s in simps:
-                    s2 = rewrite_residual_list_comma(s, coord)
-                    if s2 != s:
-                        print(
-                            f"  Residual list rewrite ({coord}): "
-                            f"{s!r} → {s2!r}"
-                        )
-                    rewritten.append(s2)
-                simps = rewritten
-
-        if simps and xml_tag == "M:Y_CMN1":
-            simps = [fix_subject_verb_agreement(s) for s in simps]
-
-        if not simps:
-            print(
-                "Simplifier returned []; forcing final (no re-queue)"
+            xml_sent = create_xml_sent(
+                cleaned_tokens,
+                clean_pos,
+                clean_lemmas,
+                improved_preds,
+                xml_tag,
             )
-            current = fix_subject_verb_agreement(current)
-            for s in expand_paren_lists(current, lists):
-                sentence_final.append(clean_output(s))
-            continue
+            simps = simplifier(xml_sent)
+            if simps:
+                simps = [fix_subject_verb_agreement(s) for s in simps]
 
-        queued_any = False
-        for s in simps:
-            plain_s = clean_output(s)
-            plain_s = fix_subject_verb_agreement(plain_s)
-            if not plain_s:
-                continue
-            plain_key = re.sub(r"\s+", " ", plain_s.lower())
-            if plain_key == key:
-                print(
-                    f"  No progress (same as input): "
-                    f"{plain_s[:80]}..."
-                )
-                for exp in expand_paren_lists(plain_s, lists):
-                    sentence_final.append(clean_output(exp))
-                continue
-            if plain_key in seen:
+            if (
+                simps
+                and simplifier is CompoundSimplify
+                and xml_tag
+                in ("M:Y_CCV", "M:Y_CMV1", "M:Y_CMN1", "M:Y_CLN")
+            ):
+                coord = extract_coordinator_from_xml(xml_sent)
+                if coord:
+                    rewritten = []
+                    for s in simps:
+                        s2 = rewrite_residual_list_comma(s, coord)
+                        if s2 != s:
+                            print(
+                                f"  Residual list rewrite ({coord}): "
+                                f"{s!r} → {s2!r}"
+                            )
+                        rewritten.append(s2)
+                    simps = rewritten
+
+            if simps and xml_tag == "M:Y_CMN1":
+                simps = [fix_subject_verb_agreement(s) for s in simps]
+
+            if not simps:
                 print(
-                    f"  Skip already-seen result: "
-                    f"{plain_s[:80]}..."
+                    "Simplifier returned []; forcing final (no re-queue)"
                 )
-                for exp in expand_paren_lists(plain_s, lists):
-                    sentence_final.append(clean_output(exp))
+                current = fix_subject_verb_agreement(current)
+                for s in expand_paren_lists(current, lists):
+                    sentence_final.append(clean_output(s))
                 continue
-            working_set.append((plain_s, lists))
-            queued_any = True
-            print(f"  → Queued: {plain_s}")
-
-        if not queued_any:
-            for s in expand_paren_lists(current, lists):
-                sentence_final.append(clean_output(s))
-
-    for s in order_sentences_for_discourse(sentence_final):
-        s = clean_output(s)
-        if s:
-            para_simplified.append(s)
+
+            queued_any = False
+            for s in simps:
+                plain_s = clean_output(s)
+                plain_s = fix_subject_verb_agreement(plain_s)
+                if not plain_s:
+                    continue
+                plain_key = re.sub(r"\s+", " ", plain_s.lower())
+                if plain_key == key:
+                    print(
+                        f"  No progress (same as input): "
+                        f"{plain_s[:80]}..."
+                    )
+                    for exp in expand_paren_lists(plain_s, lists):
+                        sentence_final.append(clean_output(exp))
+                    continue
+                if plain_key in seen:
+                    print(
+                        f"  Skip already-seen result: "
+                        f"{plain_s[:80]}..."
+                    )
+                    for exp in expand_paren_lists(plain_s, lists):
+                        sentence_final.append(clean_output(exp))
+                    continue
+                working_set.append((plain_s, lists))
+                queued_any = True
+                print(f"  → Queued: {plain_s}")
+
+            if not queued_any:
+                for s in expand_paren_lists(current, lists):
+                    sentence_final.append(clean_output(s))
+
+        for s in order_sentences_for_discourse(sentence_final):
+            s = clean_output(s)
+            if s:
+                para_simplified.append(s)
 
         final_output = "\n\n".join(output_paragraphs)
         print(f"\n=== FINAL OUTPUT ===\n{final_output}")