Spaces:
Sleeping
Sleeping
add visual verifier and fixes
Browse files- app.py +704 -696
- scripts/cr_parser.py +63 -0
- scripts/docx_helpers.py +45 -0
- scripts/etsi_client.py +1 -1
- scripts/orchestrate_cr.py +24 -0
- scripts/tests/fixtures/SETTEC(26)000048r1.docx +0 -0
- scripts/tests/fixtures/SETTEC(26)000050r1.docx +0 -0
- scripts/tests/test_lone_insertion.py +153 -0
- scripts/ts_applicator.py +84 -0
- scripts/verify_applied.py +181 -0
- skills/cr-visual-pair/SKILL.md +315 -0
- skills/cr-visual-pair/build_pairs.py +192 -0
- verify_review.py +291 -0
app.py
CHANGED
|
@@ -441,748 +441,756 @@ with st.sidebar:
|
|
| 441 |
else:
|
| 442 |
st.error("Session not found.")
|
| 443 |
|
| 444 |
-
# ββ
|
| 445 |
-
|
| 446 |
-
|
| 447 |
-
|
| 448 |
-
#
|
| 449 |
-
|
| 450 |
-
|
| 451 |
-
|
| 452 |
-
|
| 453 |
-
|
| 454 |
-
|
| 455 |
-
|
| 456 |
-
|
| 457 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 458 |
|
| 459 |
-
|
| 460 |
-
|
| 461 |
-
|
| 462 |
-
else:
|
| 463 |
-
with st.spinner("Verifying credentialsβ¦"):
|
| 464 |
-
ok = verify_eol_credentials(username, password)
|
| 465 |
-
if ok:
|
| 466 |
-
st.session_state.eol_user = username
|
| 467 |
-
st.session_state.eol_password = password
|
| 468 |
-
state["status"] = "upload"
|
| 469 |
-
save_state(sid, state)
|
| 470 |
-
st.rerun()
|
| 471 |
else:
|
| 472 |
-
st.
|
| 473 |
-
|
| 474 |
-
|
| 475 |
-
|
| 476 |
-
|
| 477 |
-
|
| 478 |
-
st.subheader("Step 1 β Upload contribution list")
|
| 479 |
-
|
| 480 |
-
mode_label = st.radio(
|
| 481 |
-
"Pipeline mode",
|
| 482 |
-
["By contributor name", "By TS (all CRs for a spec)"],
|
| 483 |
-
key="mode_radio",
|
| 484 |
-
)
|
| 485 |
-
pipeline_mode = "contributor" if mode_label.startswith("By contributor") else "ts"
|
| 486 |
-
state["mode"] = pipeline_mode
|
| 487 |
-
|
| 488 |
-
# ββ Resolve active Excel (saved from a previous run, or freshly uploaded) ββ
|
| 489 |
-
saved_name = state.get("excel_filename")
|
| 490 |
-
saved_path = session_dir(sid) / saved_name if saved_name else None
|
| 491 |
-
has_saved = saved_path is not None and saved_path.exists()
|
| 492 |
-
|
| 493 |
-
# ββ Previous meetings selector βββββββββββββββββββββββββββββββββββββββββββββ
|
| 494 |
-
past_meetings = load_meetings()
|
| 495 |
-
if past_meetings:
|
| 496 |
-
with st.expander("π Previous meetings", expanded=not has_saved):
|
| 497 |
-
meeting_options = [
|
| 498 |
-
f"{m['meeting_name']} β {m['uploaded_at']} ({m['original_filename']})"
|
| 499 |
-
for m in reversed(past_meetings)
|
| 500 |
-
]
|
| 501 |
-
chosen_label = st.selectbox("Select meeting", meeting_options, key="meeting_select")
|
| 502 |
-
if st.button("Load this meeting", key="load_meeting"):
|
| 503 |
-
chosen_idx = len(past_meetings) - 1 - meeting_options.index(chosen_label)
|
| 504 |
-
m = past_meetings[chosen_idx]
|
| 505 |
-
cached = Path(m["excel_path"])
|
| 506 |
-
if cached.exists():
|
| 507 |
-
import shutil as _sh
|
| 508 |
-
dest = session_dir(sid) / m["original_filename"]
|
| 509 |
-
_sh.copy2(str(cached), str(dest))
|
| 510 |
-
state["excel_filename"] = m["original_filename"]
|
| 511 |
-
state["excel_hash"] = m["excel_hash"]
|
| 512 |
save_state(sid, state)
|
| 513 |
st.rerun()
|
| 514 |
else:
|
| 515 |
-
st.error("
|
| 516 |
-
|
| 517 |
-
|
| 518 |
-
|
| 519 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 520 |
uploaded = st.file_uploader(
|
| 521 |
-
"
|
| 522 |
type=["xlsx", "xls"],
|
| 523 |
-
key="excel_replace",
|
| 524 |
)
|
| 525 |
-
else:
|
| 526 |
-
uploaded = st.file_uploader(
|
| 527 |
-
"Excel contribution list (.xlsx or .xls)",
|
| 528 |
-
type=["xlsx", "xls"],
|
| 529 |
-
)
|
| 530 |
|
| 531 |
-
|
| 532 |
-
|
| 533 |
-
|
| 534 |
-
|
| 535 |
-
|
| 536 |
-
|
| 537 |
-
|
| 538 |
-
|
| 539 |
-
|
| 540 |
-
|
| 541 |
-
|
| 542 |
-
|
| 543 |
-
|
| 544 |
-
|
| 545 |
-
|
| 546 |
-
|
| 547 |
-
|
| 548 |
|
| 549 |
-
|
| 550 |
-
|
| 551 |
-
|
| 552 |
-
|
| 553 |
-
|
| 554 |
|
| 555 |
-
|
| 556 |
-
|
| 557 |
-
|
| 558 |
-
|
| 559 |
-
|
| 560 |
-
|
| 561 |
-
|
| 562 |
-
|
| 563 |
-
|
| 564 |
-
|
| 565 |
-
|
| 566 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 567 |
|
| 568 |
-
|
| 569 |
-
|
| 570 |
-
|
| 571 |
-
state["excel_hash"] = excel_hash
|
| 572 |
|
| 573 |
-
|
| 574 |
-
|
| 575 |
|
| 576 |
-
|
| 577 |
-
existing = load_hf_index_cached(hf_token, hf_repo)
|
| 578 |
-
already_indexed = any(r.get("excel_hash") == excel_hash for r in existing)
|
| 579 |
|
| 580 |
-
|
| 581 |
-
|
| 582 |
-
|
| 583 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 584 |
save_state(sid, state)
|
| 585 |
st.rerun()
|
| 586 |
-
|
| 587 |
-
|
| 588 |
-
if st.button("
|
| 589 |
-
|
| 590 |
-
|
| 591 |
-
|
| 592 |
-
|
| 593 |
-
|
| 594 |
-
|
| 595 |
-
|
| 596 |
-
|
| 597 |
-
|
| 598 |
-
|
| 599 |
-
|
| 600 |
-
|
| 601 |
-
|
| 602 |
-
|
| 603 |
-
|
| 604 |
-
|
| 605 |
-
|
| 606 |
-
|
| 607 |
-
|
| 608 |
-
|
| 609 |
-
|
| 610 |
-
|
| 611 |
-
|
| 612 |
-
|
| 613 |
-
|
| 614 |
-
|
| 615 |
-
# INDEXING (build_cr_index.py running)
|
| 616 |
-
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 617 |
-
elif status == "indexing":
|
| 618 |
-
pid = state["pid"]
|
| 619 |
-
index_log = state.get("index_log", "")
|
| 620 |
-
|
| 621 |
-
proc = st.session_state.get("proc")
|
| 622 |
-
alive = False
|
| 623 |
-
if proc is not None:
|
| 624 |
-
alive = proc.poll() is None
|
| 625 |
-
else:
|
| 626 |
-
rc = read_return_code(sid)
|
| 627 |
-
if rc is None:
|
| 628 |
-
alive = is_process_alive(pid)
|
| 629 |
-
|
| 630 |
-
if alive:
|
| 631 |
-
st.subheader("β³ Building CR Indexβ¦")
|
| 632 |
-
st.info(f"PID {pid} β started {state.get('started_at', '')[:19]}")
|
| 633 |
-
log_text = tail_log(index_log, 50)
|
| 634 |
-
st.text_area("Live log (last 50 lines)", value=log_text, height=400)
|
| 635 |
-
time.sleep(2)
|
| 636 |
-
st.rerun()
|
| 637 |
-
else:
|
| 638 |
-
rc = read_return_code(sid)
|
| 639 |
-
if rc is None and proc is not None:
|
| 640 |
-
rc = proc.returncode
|
| 641 |
-
state["return_code"] = rc
|
| 642 |
-
state["completed_at"] = datetime.now().isoformat()
|
| 643 |
-
if rc == 0:
|
| 644 |
-
# Invalidate cached HF index so ts_select gets fresh data
|
| 645 |
-
st.session_state.pop(f"hf_index_{state.get('hf_repo', '')}", None)
|
| 646 |
-
state["status"] = "ts_select"
|
| 647 |
else:
|
| 648 |
-
|
| 649 |
-
|
| 650 |
-
|
| 651 |
-
|
| 652 |
-
|
| 653 |
-
|
| 654 |
-
|
| 655 |
-
|
| 656 |
-
|
| 657 |
-
|
| 658 |
-
|
| 659 |
-
|
| 660 |
-
hf_token = os.environ.get("HF_TOKEN", "")
|
| 661 |
-
hf_repo = state.get("hf_repo", "OrganizedProgrammers/CR_Index")
|
| 662 |
-
excel_hash = state.get("excel_hash", "")
|
| 663 |
-
|
| 664 |
-
with st.spinner("Loading CR index from HuggingFaceβ¦"):
|
| 665 |
-
records = load_hf_index_cached(hf_token, hf_repo)
|
| 666 |
-
|
| 667 |
-
records_for_excel = [r for r in records if r.get("excel_hash") == excel_hash]
|
| 668 |
|
| 669 |
-
|
| 670 |
-
|
| 671 |
-
if st.button("β Back to upload"):
|
| 672 |
-
state["status"] = "upload"
|
| 673 |
-
save_state(sid, state)
|
| 674 |
-
st.rerun()
|
| 675 |
-
else:
|
| 676 |
-
by_spec = defaultdict(list)
|
| 677 |
-
for r in records_for_excel:
|
| 678 |
-
by_spec[r["spec_number"]].append(r)
|
| 679 |
-
|
| 680 |
-
_ALL = "All"
|
| 681 |
-
spec_options = sorted(by_spec.keys()) + [_ALL]
|
| 682 |
-
selected_spec = st.selectbox("Select target TS spec", spec_options)
|
| 683 |
-
|
| 684 |
-
if selected_spec and selected_spec != _ALL:
|
| 685 |
-
versions = defaultdict(list)
|
| 686 |
-
for r in by_spec[selected_spec]:
|
| 687 |
-
versions[r["version"]].append(r["uid"])
|
| 688 |
-
st.write("**Versions found:**")
|
| 689 |
-
for ver, uids in sorted(versions.items()):
|
| 690 |
-
st.write(f" v{ver}: {len(uids)} CR(s) β {', '.join(uids)}")
|
| 691 |
-
elif selected_spec == _ALL:
|
| 692 |
-
total_crs = sum(len(v) for v in by_spec.values())
|
| 693 |
-
st.write(f"**All specs** β {len(by_spec)} TS(s), {total_crs} CR(s) total")
|
| 694 |
-
|
| 695 |
-
def _start_ts_run(ts_id_arg: str | None):
|
| 696 |
-
run_id = int(time.time())
|
| 697 |
-
output_dir = session_dir(sid) / f"run_{run_id}"
|
| 698 |
-
output_dir.mkdir(parents=True, exist_ok=True)
|
| 699 |
-
cr_cache_dir = session_dir(sid) / "CRs"
|
| 700 |
-
cr_cache_dir.mkdir(parents=True, exist_ok=True)
|
| 701 |
-
log_path = session_dir(sid) / f"pipeline_{run_id}.log"
|
| 702 |
-
rc_path = _rc_path(sid)
|
| 703 |
-
rc_path.unlink(missing_ok=True)
|
| 704 |
-
|
| 705 |
-
cmd = [
|
| 706 |
-
sys.executable,
|
| 707 |
-
str(SCRIPTS_DIR / "orchestrate_cr.py"),
|
| 708 |
-
"--output-dir", str(output_dir),
|
| 709 |
-
"--cr-cache-dir", str(cr_cache_dir),
|
| 710 |
-
"--ts-mode",
|
| 711 |
-
"--excel-hash", excel_hash,
|
| 712 |
-
"--hf-repo", hf_repo,
|
| 713 |
-
]
|
| 714 |
-
if ts_id_arg:
|
| 715 |
-
cmd += ["--ts-id", ts_id_arg]
|
| 716 |
-
|
| 717 |
-
env = os.environ.copy()
|
| 718 |
-
env["EOL_USER"] = st.session_state.eol_user
|
| 719 |
-
env["EOL_PASSWORD"] = st.session_state.eol_password
|
| 720 |
-
|
| 721 |
-
_launch_proc(cmd, env, log_path, sid, state, {
|
| 722 |
-
"ts_id": ts_id_arg or "all",
|
| 723 |
-
"status": "running",
|
| 724 |
-
"output_dir": str(output_dir),
|
| 725 |
-
"log_path": str(log_path),
|
| 726 |
-
"run_log_paths": [str(log_path)],
|
| 727 |
-
})
|
| 728 |
-
|
| 729 |
-
col_back, col_apply = st.columns([1, 3])
|
| 730 |
-
with col_back:
|
| 731 |
if st.button("β Back"):
|
| 732 |
state["status"] = "upload"
|
|
|
|
| 733 |
save_state(sid, state)
|
| 734 |
st.rerun()
|
| 735 |
-
with
|
| 736 |
-
|
| 737 |
-
|
| 738 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 739 |
|
| 740 |
-
#
|
|
|
|
|
|
|
| 741 |
applied_runs = state.get("applied_runs", [])
|
| 742 |
-
|
| 743 |
-
|
| 744 |
-
|
| 745 |
-
|
|
|
|
|
|
|
| 746 |
icon = "β
" if run["status"] == "done" else "β"
|
| 747 |
label = run["ts_id"] if run["ts_id"] != "all" else "All TSs"
|
| 748 |
ts_dir = Path(run["output_dir"]) / "TS"
|
| 749 |
specs = sorted(d.name for d in ts_dir.iterdir() if d.is_dir()) if ts_dir.exists() else []
|
| 750 |
specs_str = f" β {', '.join(specs)}" if specs else ""
|
| 751 |
-
st.write(f"{icon}
|
| 752 |
-
|
| 753 |
-
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 754 |
-
# PREVIEW
|
| 755 |
-
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 756 |
-
elif status == "preview":
|
| 757 |
-
cr_list = state["cr_list"]
|
| 758 |
-
st.subheader(f"Step 2 β {len(cr_list)} Accepted CR(s) found")
|
| 759 |
-
|
| 760 |
-
if cr_list:
|
| 761 |
-
import pandas as pd
|
| 762 |
-
df = pd.DataFrame(cr_list, columns=["UID", "Title"])
|
| 763 |
-
st.dataframe(df, width='stretch')
|
| 764 |
-
else:
|
| 765 |
-
st.warning(
|
| 766 |
-
f"No Accepted CRs found for **{state['person_name']}** in this file."
|
| 767 |
-
)
|
| 768 |
-
# Diagnostic: show what names are in the SubmittedBy column
|
| 769 |
-
excel_path = session_dir(sid) / state["excel_filename"]
|
| 770 |
-
found_names = peek_submitted_by(excel_path)
|
| 771 |
-
if found_names:
|
| 772 |
-
st.info(
|
| 773 |
-
"**Names found in SubmittedBy column** β copy the exact one into the field above and re-upload:\n\n"
|
| 774 |
-
+ "\n".join(f"- `{n}`" for n in found_names)
|
| 775 |
-
)
|
| 776 |
|
| 777 |
-
|
| 778 |
-
|
| 779 |
-
|
| 780 |
-
|
| 781 |
-
|
| 782 |
-
|
| 783 |
-
|
| 784 |
-
|
| 785 |
-
|
| 786 |
-
|
| 787 |
-
|
| 788 |
-
|
| 789 |
-
|
| 790 |
-
|
| 791 |
-
|
| 792 |
-
|
| 793 |
-
|
| 794 |
-
|
| 795 |
-
|
| 796 |
-
|
| 797 |
-
|
| 798 |
-
|
| 799 |
-
|
| 800 |
-
|
| 801 |
-
|
| 802 |
-
|
| 803 |
-
|
| 804 |
-
|
| 805 |
-
|
| 806 |
-
|
| 807 |
-
|
| 808 |
-
env["EOL_PASSWORD"] = st.session_state.eol_password
|
| 809 |
-
|
| 810 |
-
_launch_proc(cmd, env, log_path, sid, state, {
|
| 811 |
-
"status": "running",
|
| 812 |
-
"output_dir": str(output_dir),
|
| 813 |
-
"log_path": str(log_path),
|
| 814 |
-
"run_log_paths": [str(log_path)],
|
| 815 |
-
})
|
| 816 |
-
|
| 817 |
-
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 818 |
-
# RUNNING
|
| 819 |
-
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 820 |
-
elif status == "running":
|
| 821 |
-
pid = state["pid"]
|
| 822 |
-
log_path = state["log_path"]
|
| 823 |
-
|
| 824 |
-
# Determine whether process is still alive
|
| 825 |
-
proc = st.session_state.get("proc")
|
| 826 |
-
alive = False
|
| 827 |
-
if proc is not None:
|
| 828 |
-
alive = proc.poll() is None
|
| 829 |
-
else:
|
| 830 |
-
# Session reloaded β check returncode file, then PID
|
| 831 |
-
rc = read_return_code(sid)
|
| 832 |
-
if rc is None:
|
| 833 |
-
alive = is_process_alive(pid)
|
| 834 |
-
|
| 835 |
-
if alive:
|
| 836 |
-
st.subheader("β³ Pipeline runningβ¦")
|
| 837 |
-
st.info(f"PID {pid} β started {state.get('started_at', '')[:19]}")
|
| 838 |
-
log_text = tail_log(log_path, 100)
|
| 839 |
-
st.text_area("Live log (last 100 lines)", value=log_text, height=400)
|
| 840 |
-
time.sleep(2)
|
| 841 |
-
st.rerun()
|
| 842 |
-
else:
|
| 843 |
-
# Process finished β determine return code
|
| 844 |
-
rc = read_return_code(sid)
|
| 845 |
-
if rc is None and proc is not None:
|
| 846 |
-
rc = proc.returncode
|
| 847 |
-
state["return_code"] = rc
|
| 848 |
-
state["completed_at"] = datetime.now().isoformat()
|
| 849 |
-
fin_status = "done" if rc == 0 else "error"
|
| 850 |
-
state["status"] = fin_status
|
| 851 |
-
# Accumulate completed run for TS mode
|
| 852 |
-
if state.get("mode") == "ts":
|
| 853 |
-
run_entry = {
|
| 854 |
-
"ts_id": state.get("ts_id", ""),
|
| 855 |
-
"output_dir": state.get("output_dir", ""),
|
| 856 |
-
"log_path": state.get("log_path", ""),
|
| 857 |
-
"status": fin_status,
|
| 858 |
-
}
|
| 859 |
-
runs = state.get("applied_runs", [])
|
| 860 |
-
if not any(r["output_dir"] == run_entry["output_dir"] for r in runs):
|
| 861 |
-
runs.append(run_entry)
|
| 862 |
-
state["applied_runs"] = runs
|
| 863 |
-
save_state(sid, state)
|
| 864 |
-
st.rerun()
|
| 865 |
-
|
| 866 |
-
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 867 |
-
# DONE / ERROR
|
| 868 |
-
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 869 |
-
elif status in ("done", "error"):
|
| 870 |
-
log_path = state.get("log_path", "")
|
| 871 |
-
output_dir = Path(state.get("output_dir", ""))
|
| 872 |
-
rc = state.get("return_code")
|
| 873 |
-
|
| 874 |
-
st.subheader("Step 3 β Results")
|
| 875 |
-
if status == "done":
|
| 876 |
-
st.success("β
Pipeline completed successfully!")
|
| 877 |
-
else:
|
| 878 |
-
st.error(f"β Pipeline finished with errors (return code: {rc})")
|
| 879 |
-
|
| 880 |
-
# Per-TS results table β merge this run's logs so retry results supersede
|
| 881 |
-
# the original failed entries for the same TS key.
|
| 882 |
-
_merged: dict[str, dict] = {}
|
| 883 |
-
for _lf in state.get("run_log_paths", []):
|
| 884 |
-
for _r in parse_log_results(_lf):
|
| 885 |
-
_merged[_r["TS"]] = _r
|
| 886 |
-
results = list(_merged.values())
|
| 887 |
-
if results:
|
| 888 |
-
st.subheader("Results per TS")
|
| 889 |
-
import pandas as pd
|
| 890 |
-
|
| 891 |
-
n_warn = sum(1 for r in results if r["warnings"])
|
| 892 |
-
warn_label = f"Warnings ({n_warn})" if n_warn else "Warnings"
|
| 893 |
-
tab_summary, tab_warnings = st.tabs(["Summary", warn_label])
|
| 894 |
-
|
| 895 |
-
def _color_status(val):
|
| 896 |
-
return {
|
| 897 |
-
"OK": "background-color: #d4edda; color: #155724",
|
| 898 |
-
"WARN": "background-color: #fff3cd; color: #856404",
|
| 899 |
-
"FAIL": "background-color: #f8d7da; color: #721c24",
|
| 900 |
-
"SKIP": "background-color: #e2e3e5; color: #383d41",
|
| 901 |
-
}.get(val, "")
|
| 902 |
-
|
| 903 |
-
with tab_summary:
|
| 904 |
-
df = pd.DataFrame([{"Status": r["Status"], "TS": r["TS"]} for r in results])
|
| 905 |
-
st.dataframe(
|
| 906 |
-
df.style.map(_color_status, subset=["Status"]),
|
| 907 |
-
width='stretch',
|
| 908 |
-
)
|
| 909 |
-
|
| 910 |
-
with tab_warnings:
|
| 911 |
-
warned = [r for r in results if r["warnings"]]
|
| 912 |
-
if warned:
|
| 913 |
-
for r in warned:
|
| 914 |
-
with st.expander(f"β οΈ {r['TS']} β {len(r['warnings'])} warning(s)"):
|
| 915 |
-
for w in r["warnings"]:
|
| 916 |
-
st.text(w)
|
| 917 |
-
else:
|
| 918 |
-
st.success("No warnings.")
|
| 919 |
-
|
| 920 |
-
# Download ZIP
|
| 921 |
-
# ββ Download section ββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 922 |
-
st.subheader("Download results")
|
| 923 |
-
applied_runs = state.get("applied_runs", [])
|
| 924 |
-
valid_runs = [r for r in applied_runs if Path(r["output_dir"]).exists()
|
| 925 |
-
and any(Path(r["output_dir"]).rglob("*"))]
|
| 926 |
-
|
| 927 |
-
if valid_runs:
|
| 928 |
-
# Show a compact list of what's included
|
| 929 |
-
for run in valid_runs:
|
| 930 |
-
icon = "β
" if run["status"] == "done" else "β"
|
| 931 |
-
label = run["ts_id"] if run["ts_id"] != "all" else "All TSs"
|
| 932 |
-
ts_dir = Path(run["output_dir"]) / "TS"
|
| 933 |
-
specs = sorted(d.name for d in ts_dir.iterdir() if d.is_dir()) if ts_dir.exists() else []
|
| 934 |
-
specs_str = f" β {', '.join(specs)}" if specs else ""
|
| 935 |
-
st.write(f"{icon} {label}{specs_str}")
|
| 936 |
-
|
| 937 |
-
# Build ZIP from all accumulated runs
|
| 938 |
-
buf = io.BytesIO()
|
| 939 |
-
with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as zf:
|
| 940 |
-
for run in valid_runs:
|
| 941 |
-
out = Path(run["output_dir"])
|
| 942 |
-
for f in out.rglob("*"):
|
| 943 |
-
if f.is_file():
|
| 944 |
-
zf.write(f, f.relative_to(out.parent))
|
| 945 |
-
buf.seek(0)
|
| 946 |
-
col_dl, col_flush = st.columns([3, 1])
|
| 947 |
-
with col_dl:
|
| 948 |
st.download_button(
|
| 949 |
label="β¬ Download results ZIP",
|
| 950 |
-
data=
|
| 951 |
file_name=f"cr_results_{sid[:8]}.zip",
|
| 952 |
mime="application/zip",
|
| 953 |
type="primary",
|
| 954 |
)
|
| 955 |
-
with col_flush:
|
| 956 |
-
if st.button("π Flush"):
|
| 957 |
-
import shutil
|
| 958 |
-
for run in applied_runs:
|
| 959 |
-
out = Path(run["output_dir"])
|
| 960 |
-
if out.exists():
|
| 961 |
-
shutil.rmtree(out, ignore_errors=True)
|
| 962 |
-
state["applied_runs"] = []
|
| 963 |
-
save_state(sid, state)
|
| 964 |
-
st.rerun()
|
| 965 |
-
elif output_dir.exists() and any(output_dir.rglob("*")):
|
| 966 |
-
# Fallback for contributor mode (no applied_runs)
|
| 967 |
-
zip_bytes = make_zip(output_dir)
|
| 968 |
-
st.download_button(
|
| 969 |
-
label="β¬ Download results ZIP",
|
| 970 |
-
data=zip_bytes,
|
| 971 |
-
file_name=f"cr_results_{sid[:8]}.zip",
|
| 972 |
-
mime="application/zip",
|
| 973 |
-
type="primary",
|
| 974 |
-
)
|
| 975 |
-
else:
|
| 976 |
-
st.warning("Output directory is empty β nothing to download.")
|
| 977 |
-
|
| 978 |
-
# Full log
|
| 979 |
-
with st.expander("Full pipeline log"):
|
| 980 |
-
if log_path and Path(log_path).exists():
|
| 981 |
-
st.text(Path(log_path).read_text(errors="replace"))
|
| 982 |
else:
|
| 983 |
-
st.
|
| 984 |
-
|
| 985 |
-
# ββ TS Recovery βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 986 |
-
failed_ts_path = output_dir / "failed_ts.json"
|
| 987 |
-
if failed_ts_path.exists():
|
| 988 |
-
failed_ts_entries = json.loads(failed_ts_path.read_text())
|
| 989 |
-
if failed_ts_entries:
|
| 990 |
-
st.divider()
|
| 991 |
-
st.subheader("β οΈ Recover failed TS downloads")
|
| 992 |
-
st.info(
|
| 993 |
-
f"{len(failed_ts_entries)} TS(s) could not be downloaded. "
|
| 994 |
-
"Retry or upload each one manually, then apply the CRs."
|
| 995 |
-
)
|
| 996 |
|
| 997 |
-
|
| 998 |
-
|
| 999 |
-
|
| 1000 |
-
|
| 1001 |
-
|
| 1002 |
-
label = f"{'β
' if ready else 'β'} TS {spec_key} β CRs: {', '.join(entry['cr_uids'])}"
|
| 1003 |
-
with st.expander(label, expanded=not ready):
|
| 1004 |
-
col1, col2 = st.columns(2)
|
| 1005 |
-
|
| 1006 |
-
with col1:
|
| 1007 |
-
if st.button("π Retry download",
|
| 1008 |
-
key=f"retry_{entry['spec_compact']}_{entry['version']}"):
|
| 1009 |
-
from fetch_crs import download_ts as _dl_ts
|
| 1010 |
-
with st.spinner(f"Downloading TS {spec_key}β¦"):
|
| 1011 |
-
fn, note = _dl_ts(
|
| 1012 |
-
entry["spec_number"], entry["version"],
|
| 1013 |
-
Path(entry["spec_dir"]),
|
| 1014 |
-
st.session_state.eol_user,
|
| 1015 |
-
st.session_state.eol_password,
|
| 1016 |
-
)
|
| 1017 |
-
if fn:
|
| 1018 |
-
st.success(f"Downloaded: {fn}")
|
| 1019 |
-
st.rerun()
|
| 1020 |
-
else:
|
| 1021 |
-
st.error(f"Failed: {note}")
|
| 1022 |
-
|
| 1023 |
-
with col2:
|
| 1024 |
-
uploaded_ts = st.file_uploader(
|
| 1025 |
-
f"Or upload `{entry['expected_filename']}`",
|
| 1026 |
-
type=["docx"],
|
| 1027 |
-
key=f"upload_{entry['spec_compact']}_{entry['version']}",
|
| 1028 |
-
)
|
| 1029 |
-
if uploaded_ts is not None:
|
| 1030 |
-
Path(entry["spec_dir"]).mkdir(parents=True, exist_ok=True)
|
| 1031 |
-
dest_path.write_bytes(uploaded_ts.read())
|
| 1032 |
-
st.success("Saved β")
|
| 1033 |
-
st.rerun()
|
| 1034 |
-
|
| 1035 |
-
# Global apply button β enabled when β₯1 TS is now on disk
|
| 1036 |
-
ready_entries = [
|
| 1037 |
-
e for e in failed_ts_entries
|
| 1038 |
-
if (Path(e["spec_dir"]) / e["expected_filename"]).exists()
|
| 1039 |
-
]
|
| 1040 |
-
remaining = len(failed_ts_entries) - len(ready_entries)
|
| 1041 |
-
|
| 1042 |
-
if ready_entries:
|
| 1043 |
-
if remaining:
|
| 1044 |
-
st.warning(f"{len(ready_entries)} ready, {remaining} will be skipped.")
|
| 1045 |
-
else:
|
| 1046 |
-
st.success(f"All {len(ready_entries)} TS(s) ready.")
|
| 1047 |
-
|
| 1048 |
-
if st.button("βΆ Apply CRs to recovered TSs", type="primary"):
|
| 1049 |
-
retry_log = str(session_dir(sid) / f"pipeline_{int(time.time())}_retry.log")
|
| 1050 |
-
cmd = [
|
| 1051 |
-
sys.executable,
|
| 1052 |
-
str(SCRIPTS_DIR / "orchestrate_cr.py"),
|
| 1053 |
-
"--output-dir", state["output_dir"],
|
| 1054 |
-
"--retry-mode",
|
| 1055 |
-
]
|
| 1056 |
-
env = os.environ.copy()
|
| 1057 |
-
env["EOL_USER"] = st.session_state.eol_user
|
| 1058 |
-
env["EOL_PASSWORD"] = st.session_state.eol_password
|
| 1059 |
-
|
| 1060 |
-
_launch_proc(cmd, env, retry_log, sid, state, {
|
| 1061 |
-
"status": "running",
|
| 1062 |
-
"log_path": retry_log,
|
| 1063 |
-
"run_log_paths": state.get("run_log_paths", []) + [retry_log],
|
| 1064 |
-
})
|
| 1065 |
else:
|
| 1066 |
-
st.
|
| 1067 |
-
|
| 1068 |
-
|
| 1069 |
-
|
| 1070 |
-
|
| 1071 |
-
|
| 1072 |
-
|
| 1073 |
-
|
| 1074 |
-
|
| 1075 |
-
|
| 1076 |
-
|
| 1077 |
-
|
| 1078 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1079 |
|
| 1080 |
-
|
| 1081 |
-
|
| 1082 |
-
|
| 1083 |
-
|
| 1084 |
-
|
| 1085 |
-
|
| 1086 |
-
|
| 1087 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1088 |
)
|
| 1089 |
-
|
| 1090 |
-
|
| 1091 |
-
|
| 1092 |
-
|
| 1093 |
-
|
| 1094 |
-
|
| 1095 |
-
|
| 1096 |
-
|
| 1097 |
-
|
| 1098 |
-
|
| 1099 |
-
|
| 1100 |
-
|
| 1101 |
-
|
| 1102 |
-
|
| 1103 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1104 |
st.rerun()
|
| 1105 |
-
|
| 1106 |
-
|
| 1107 |
-
|
| 1108 |
-
|
| 1109 |
-
|
| 1110 |
-
|
| 1111 |
-
|
| 1112 |
-
|
| 1113 |
-
|
| 1114 |
-
|
| 1115 |
-
|
| 1116 |
-
|
| 1117 |
-
|
| 1118 |
-
|
| 1119 |
-
|
| 1120 |
-
|
| 1121 |
-
|
| 1122 |
-
|
| 1123 |
-
|
| 1124 |
-
|
| 1125 |
-
|
| 1126 |
-
|
| 1127 |
-
|
| 1128 |
-
|
| 1129 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1130 |
else:
|
| 1131 |
-
st.
|
| 1132 |
-
|
| 1133 |
-
|
| 1134 |
-
|
| 1135 |
-
|
| 1136 |
-
|
| 1137 |
-
|
| 1138 |
-
|
| 1139 |
-
|
| 1140 |
-
|
| 1141 |
-
|
| 1142 |
-
env["EOL_USER"] = st.session_state.eol_user
|
| 1143 |
-
env["EOL_PASSWORD"] = st.session_state.eol_password
|
| 1144 |
-
_launch_proc(cmd, env, retry_log, sid, state, {
|
| 1145 |
-
"status": "running",
|
| 1146 |
-
"log_path": retry_log,
|
| 1147 |
-
"run_log_paths": state.get("run_log_paths", []) + [retry_log],
|
| 1148 |
-
})
|
| 1149 |
else:
|
| 1150 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1151 |
|
| 1152 |
-
|
| 1153 |
-
|
| 1154 |
-
|
| 1155 |
-
with col_restart:
|
| 1156 |
-
if state.get("mode") == "ts":
|
| 1157 |
-
back_label = "β Back to TS Selection"
|
| 1158 |
-
back_status = "ts_select"
|
| 1159 |
-
clear_keys = ("pid", "output_dir", "log_path", "index_log",
|
| 1160 |
-
"started_at", "completed_at", "return_code", "ts_id")
|
| 1161 |
-
else:
|
| 1162 |
-
back_label = "β Back to CR Preview"
|
| 1163 |
-
back_status = "preview"
|
| 1164 |
-
clear_keys = ("pid", "output_dir", "log_path", "index_log",
|
| 1165 |
-
"started_at", "completed_at", "return_code")
|
| 1166 |
-
if st.button(back_label, type="primary"):
|
| 1167 |
-
for _k in clear_keys:
|
| 1168 |
-
state[_k] = None
|
| 1169 |
-
state["run_log_paths"] = []
|
| 1170 |
-
state["status"] = back_status
|
| 1171 |
-
if "proc" in st.session_state:
|
| 1172 |
-
del st.session_state.proc
|
| 1173 |
-
_rc_path(sid).unlink(missing_ok=True)
|
| 1174 |
-
save_state(sid, state)
|
| 1175 |
-
st.rerun()
|
| 1176 |
-
with col_new:
|
| 1177 |
-
if st.button("Start new session"):
|
| 1178 |
-
new_sid = str(uuid.uuid4())
|
| 1179 |
-
st.session_state.sid = new_sid
|
| 1180 |
-
st.session_state.state = new_state(new_sid)
|
| 1181 |
-
if "proc" in st.session_state:
|
| 1182 |
-
del st.session_state.proc
|
| 1183 |
-
st.query_params["sid"] = new_sid
|
| 1184 |
-
save_state(new_sid, st.session_state.state)
|
| 1185 |
-
st.rerun()
|
| 1186 |
|
| 1187 |
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 1188 |
# VERSION HISTORY (bottom of every stable page)
|
|
|
|
| 441 |
else:
|
| 442 |
st.error("Session not found.")
|
| 443 |
|
| 444 |
+
# ββ Tabs ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 445 |
+
tab_apply, tab_verify = st.tabs(["π Apply CRs", "β
Verify insertions"])
|
| 446 |
+
|
| 447 |
+
with tab_apply:
|
| 448 |
+
# ββ State machine βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 449 |
+
status: str = state["status"]
|
| 450 |
+
|
| 451 |
+
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 452 |
+
# LOGIN
|
| 453 |
+
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 454 |
+
if status == "login":
|
| 455 |
+
st.subheader("Connect with your ETSI EOL account")
|
| 456 |
+
st.info(
|
| 457 |
+
"Your credentials are used only for this session and are never stored on disk.",
|
| 458 |
+
icon="π",
|
| 459 |
+
)
|
| 460 |
+
username = st.text_input("EOL Username")
|
| 461 |
+
password = st.text_input("EOL Password", type="password")
|
| 462 |
|
| 463 |
+
if st.button("Connect", type="primary"):
|
| 464 |
+
if not username or not password:
|
| 465 |
+
st.error("Please enter both username and password.")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 466 |
else:
|
| 467 |
+
with st.spinner("Verifying credentialsβ¦"):
|
| 468 |
+
ok = verify_eol_credentials(username, password)
|
| 469 |
+
if ok:
|
| 470 |
+
st.session_state.eol_user = username
|
| 471 |
+
st.session_state.eol_password = password
|
| 472 |
+
state["status"] = "upload"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 473 |
save_state(sid, state)
|
| 474 |
st.rerun()
|
| 475 |
else:
|
| 476 |
+
st.error("Login failed β check your EOL username and password.")
|
| 477 |
+
|
| 478 |
+
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 479 |
+
# UPLOAD
|
| 480 |
+
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 481 |
+
elif status == "upload":
|
| 482 |
+
st.subheader("Step 1 β Upload contribution list")
|
| 483 |
+
|
| 484 |
+
mode_label = st.radio(
|
| 485 |
+
"Pipeline mode",
|
| 486 |
+
["By contributor name", "By TS (all CRs for a spec)"],
|
| 487 |
+
key="mode_radio",
|
| 488 |
+
)
|
| 489 |
+
pipeline_mode = "contributor" if mode_label.startswith("By contributor") else "ts"
|
| 490 |
+
state["mode"] = pipeline_mode
|
| 491 |
+
|
| 492 |
+
# ββ Resolve active Excel (saved from a previous run, or freshly uploaded) ββ
|
| 493 |
+
saved_name = state.get("excel_filename")
|
| 494 |
+
saved_path = session_dir(sid) / saved_name if saved_name else None
|
| 495 |
+
has_saved = saved_path is not None and saved_path.exists()
|
| 496 |
+
|
| 497 |
+
# ββ Previous meetings selector βββββββββββββββββββββββββββββββββββββββββββββ
|
| 498 |
+
past_meetings = load_meetings()
|
| 499 |
+
if past_meetings:
|
| 500 |
+
with st.expander("π Previous meetings", expanded=not has_saved):
|
| 501 |
+
meeting_options = [
|
| 502 |
+
f"{m['meeting_name']} β {m['uploaded_at']} ({m['original_filename']})"
|
| 503 |
+
for m in reversed(past_meetings)
|
| 504 |
+
]
|
| 505 |
+
chosen_label = st.selectbox("Select meeting", meeting_options, key="meeting_select")
|
| 506 |
+
if st.button("Load this meeting", key="load_meeting"):
|
| 507 |
+
chosen_idx = len(past_meetings) - 1 - meeting_options.index(chosen_label)
|
| 508 |
+
m = past_meetings[chosen_idx]
|
| 509 |
+
cached = Path(m["excel_path"])
|
| 510 |
+
if cached.exists():
|
| 511 |
+
import shutil as _sh
|
| 512 |
+
dest = session_dir(sid) / m["original_filename"]
|
| 513 |
+
_sh.copy2(str(cached), str(dest))
|
| 514 |
+
state["excel_filename"] = m["original_filename"]
|
| 515 |
+
state["excel_hash"] = m["excel_hash"]
|
| 516 |
+
save_state(sid, state)
|
| 517 |
+
st.rerun()
|
| 518 |
+
else:
|
| 519 |
+
st.error("Cached Excel file not found β please upload it again.")
|
| 520 |
+
|
| 521 |
+
if has_saved:
|
| 522 |
+
st.success(f"Using: **{saved_name}**")
|
| 523 |
+
with st.expander("Replace Excel file"):
|
| 524 |
+
uploaded = st.file_uploader(
|
| 525 |
+
"Upload a different Excel file (.xlsx or .xls)",
|
| 526 |
+
type=["xlsx", "xls"],
|
| 527 |
+
key="excel_replace",
|
| 528 |
+
)
|
| 529 |
+
else:
|
| 530 |
uploaded = st.file_uploader(
|
| 531 |
+
"Excel contribution list (.xlsx or .xls)",
|
| 532 |
type=["xlsx", "xls"],
|
|
|
|
| 533 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 534 |
|
| 535 |
+
# When a new file is dropped, save it immediately and update state
|
| 536 |
+
if uploaded:
|
| 537 |
+
active_path = session_dir(sid) / uploaded.name
|
| 538 |
+
active_bytes = bytes(uploaded.getbuffer())
|
| 539 |
+
active_path.write_bytes(active_bytes)
|
| 540 |
+
state["excel_filename"] = uploaded.name
|
| 541 |
+
excel_hash = hashlib.sha256(active_bytes).hexdigest()[:16]
|
| 542 |
+
state["excel_hash"] = excel_hash
|
| 543 |
+
# Save to global meeting store
|
| 544 |
+
meeting_name = peek_meeting_name(active_path)
|
| 545 |
+
if meeting_name:
|
| 546 |
+
save_meeting(meeting_name, active_path, excel_hash)
|
| 547 |
+
save_state(sid, state)
|
| 548 |
+
elif has_saved:
|
| 549 |
+
active_path = saved_path
|
| 550 |
+
else:
|
| 551 |
+
active_path = None
|
| 552 |
|
| 553 |
+
if pipeline_mode == "contributor":
|
| 554 |
+
person_name = st.text_input(
|
| 555 |
+
"Contributor name (must match SubmittedBy column)",
|
| 556 |
+
value=state.get("person_name", "Ly Thanh PHAN"),
|
| 557 |
+
)
|
| 558 |
|
| 559 |
+
if active_path and st.button("Parse CR list β", type="primary"):
|
| 560 |
+
with st.spinner("Parsing Excelβ¦"):
|
| 561 |
+
try:
|
| 562 |
+
from fetch_crs import parse_excel
|
| 563 |
+
cr_list = parse_excel(str(active_path), person_name)
|
| 564 |
+
state["status"] = "preview"
|
| 565 |
+
state["person_name"] = person_name
|
| 566 |
+
state["cr_list"] = [list(row) for row in cr_list]
|
| 567 |
+
save_state(sid, state)
|
| 568 |
+
st.rerun()
|
| 569 |
+
except Exception as exc:
|
| 570 |
+
st.error(f"Failed to parse Excel: {exc}")
|
| 571 |
+
|
| 572 |
+
else: # TS mode
|
| 573 |
+
if active_path:
|
| 574 |
+
excel_hash = state.get("excel_hash") or hashlib.sha256(active_path.read_bytes()).hexdigest()[:16]
|
| 575 |
+
state["excel_hash"] = excel_hash
|
| 576 |
+
|
| 577 |
+
hf_token = os.environ.get("HF_TOKEN", "")
|
| 578 |
+
hf_repo = state.get("hf_repo", "OrganizedProgrammers/CR_Index")
|
| 579 |
+
|
| 580 |
+
# Check whether this Excel is already indexed
|
| 581 |
+
existing = load_hf_index_cached(hf_token, hf_repo)
|
| 582 |
+
already_indexed = any(r.get("excel_hash") == excel_hash for r in existing)
|
| 583 |
+
|
| 584 |
+
if already_indexed:
|
| 585 |
+
st.success(f"This Excel (`{excel_hash}`) is already indexed in HF.")
|
| 586 |
+
if st.button("Select TS β", type="primary"):
|
| 587 |
+
state["status"] = "ts_select"
|
| 588 |
+
save_state(sid, state)
|
| 589 |
+
st.rerun()
|
| 590 |
+
else:
|
| 591 |
+
st.info(f"Excel hash: `{excel_hash}` β not yet indexed.")
|
| 592 |
+
if st.button("Build CR Index", type="primary"):
|
| 593 |
+
# CRs downloaded during indexing go to the session-level cache
|
| 594 |
+
cr_cache_dir = session_dir(sid) / "CRs"
|
| 595 |
+
cr_cache_dir.mkdir(parents=True, exist_ok=True)
|
| 596 |
+
index_log = str(session_dir(sid) / "index.log")
|
| 597 |
+
rc_path = _rc_path(sid)
|
| 598 |
+
rc_path.unlink(missing_ok=True)
|
| 599 |
+
|
| 600 |
+
cmd = [
|
| 601 |
+
sys.executable,
|
| 602 |
+
str(SCRIPTS_DIR / "build_cr_index.py"),
|
| 603 |
+
str(active_path),
|
| 604 |
+
"--output-dir", str(session_dir(sid)),
|
| 605 |
+
"--hf-repo", hf_repo,
|
| 606 |
+
]
|
| 607 |
+
env = os.environ.copy()
|
| 608 |
+
env["EOL_USER"] = st.session_state.eol_user
|
| 609 |
+
env["EOL_PASSWORD"] = st.session_state.eol_password
|
| 610 |
+
# HF_TOKEN is already in env via os.environ
|
| 611 |
+
|
| 612 |
+
_launch_proc(cmd, env, index_log, sid, state, {
|
| 613 |
+
"status": "indexing",
|
| 614 |
+
"index_log": index_log,
|
| 615 |
+
"output_dir": "", # no pipeline output yet
|
| 616 |
+
})
|
| 617 |
+
|
| 618 |
+
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 619 |
+
# INDEXING (build_cr_index.py running)
|
| 620 |
+
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 621 |
+
elif status == "indexing":
|
| 622 |
+
pid = state["pid"]
|
| 623 |
+
index_log = state.get("index_log", "")
|
| 624 |
+
|
| 625 |
+
proc = st.session_state.get("proc")
|
| 626 |
+
alive = False
|
| 627 |
+
if proc is not None:
|
| 628 |
+
alive = proc.poll() is None
|
| 629 |
+
else:
|
| 630 |
+
rc = read_return_code(sid)
|
| 631 |
+
if rc is None:
|
| 632 |
+
alive = is_process_alive(pid)
|
| 633 |
+
|
| 634 |
+
if alive:
|
| 635 |
+
st.subheader("β³ Building CR Indexβ¦")
|
| 636 |
+
st.info(f"PID {pid} β started {state.get('started_at', '')[:19]}")
|
| 637 |
+
log_text = tail_log(index_log, 50)
|
| 638 |
+
st.text_area("Live log (last 50 lines)", value=log_text, height=400)
|
| 639 |
+
time.sleep(2)
|
| 640 |
+
st.rerun()
|
| 641 |
+
else:
|
| 642 |
+
rc = read_return_code(sid)
|
| 643 |
+
if rc is None and proc is not None:
|
| 644 |
+
rc = proc.returncode
|
| 645 |
+
state["return_code"] = rc
|
| 646 |
+
state["completed_at"] = datetime.now().isoformat()
|
| 647 |
+
if rc == 0:
|
| 648 |
+
# Invalidate cached HF index so ts_select gets fresh data
|
| 649 |
+
st.session_state.pop(f"hf_index_{state.get('hf_repo', '')}", None)
|
| 650 |
+
state["status"] = "ts_select"
|
| 651 |
+
else:
|
| 652 |
+
# Expose the index log as log_path so the error state can display it
|
| 653 |
+
state["log_path"] = state.get("index_log", "")
|
| 654 |
+
state["status"] = "error"
|
| 655 |
+
save_state(sid, state)
|
| 656 |
+
st.rerun()
|
| 657 |
+
|
| 658 |
+
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 659 |
+
# TS_SELECT (index ready β pick a spec to process)
|
| 660 |
+
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 661 |
+
elif status == "ts_select":
|
| 662 |
+
st.subheader("Step 2 β Select target TS spec")
|
| 663 |
|
| 664 |
+
hf_token = os.environ.get("HF_TOKEN", "")
|
| 665 |
+
hf_repo = state.get("hf_repo", "OrganizedProgrammers/CR_Index")
|
| 666 |
+
excel_hash = state.get("excel_hash", "")
|
|
|
|
| 667 |
|
| 668 |
+
with st.spinner("Loading CR index from HuggingFaceβ¦"):
|
| 669 |
+
records = load_hf_index_cached(hf_token, hf_repo)
|
| 670 |
|
| 671 |
+
records_for_excel = [r for r in records if r.get("excel_hash") == excel_hash]
|
|
|
|
|
|
|
| 672 |
|
| 673 |
+
if not records_for_excel:
|
| 674 |
+
st.error(f"No records found for excel_hash `{excel_hash}`. Try rebuilding the index.")
|
| 675 |
+
if st.button("β Back to upload"):
|
| 676 |
+
state["status"] = "upload"
|
| 677 |
+
save_state(sid, state)
|
| 678 |
+
st.rerun()
|
| 679 |
+
else:
|
| 680 |
+
by_spec = defaultdict(list)
|
| 681 |
+
for r in records_for_excel:
|
| 682 |
+
by_spec[r["spec_number"]].append(r)
|
| 683 |
+
|
| 684 |
+
_ALL = "All"
|
| 685 |
+
spec_options = sorted(by_spec.keys()) + [_ALL]
|
| 686 |
+
selected_spec = st.selectbox("Select target TS spec", spec_options)
|
| 687 |
+
|
| 688 |
+
if selected_spec and selected_spec != _ALL:
|
| 689 |
+
versions = defaultdict(list)
|
| 690 |
+
for r in by_spec[selected_spec]:
|
| 691 |
+
versions[r["version"]].append(r["uid"])
|
| 692 |
+
st.write("**Versions found:**")
|
| 693 |
+
for ver, uids in sorted(versions.items()):
|
| 694 |
+
st.write(f" v{ver}: {len(uids)} CR(s) β {', '.join(uids)}")
|
| 695 |
+
elif selected_spec == _ALL:
|
| 696 |
+
total_crs = sum(len(v) for v in by_spec.values())
|
| 697 |
+
st.write(f"**All specs** β {len(by_spec)} TS(s), {total_crs} CR(s) total")
|
| 698 |
+
|
| 699 |
+
def _start_ts_run(ts_id_arg: str | None):
|
| 700 |
+
run_id = int(time.time())
|
| 701 |
+
output_dir = session_dir(sid) / f"run_{run_id}"
|
| 702 |
+
output_dir.mkdir(parents=True, exist_ok=True)
|
| 703 |
+
cr_cache_dir = session_dir(sid) / "CRs"
|
| 704 |
+
cr_cache_dir.mkdir(parents=True, exist_ok=True)
|
| 705 |
+
log_path = session_dir(sid) / f"pipeline_{run_id}.log"
|
| 706 |
+
rc_path = _rc_path(sid)
|
| 707 |
+
rc_path.unlink(missing_ok=True)
|
| 708 |
+
|
| 709 |
+
cmd = [
|
| 710 |
+
sys.executable,
|
| 711 |
+
str(SCRIPTS_DIR / "orchestrate_cr.py"),
|
| 712 |
+
"--output-dir", str(output_dir),
|
| 713 |
+
"--cr-cache-dir", str(cr_cache_dir),
|
| 714 |
+
"--ts-mode",
|
| 715 |
+
"--excel-hash", excel_hash,
|
| 716 |
+
"--hf-repo", hf_repo,
|
| 717 |
+
]
|
| 718 |
+
if ts_id_arg:
|
| 719 |
+
cmd += ["--ts-id", ts_id_arg]
|
| 720 |
+
|
| 721 |
+
env = os.environ.copy()
|
| 722 |
+
env["EOL_USER"] = st.session_state.eol_user
|
| 723 |
+
env["EOL_PASSWORD"] = st.session_state.eol_password
|
| 724 |
+
|
| 725 |
+
_launch_proc(cmd, env, log_path, sid, state, {
|
| 726 |
+
"ts_id": ts_id_arg or "all",
|
| 727 |
+
"status": "running",
|
| 728 |
+
"output_dir": str(output_dir),
|
| 729 |
+
"log_path": str(log_path),
|
| 730 |
+
"run_log_paths": [str(log_path)],
|
| 731 |
+
})
|
| 732 |
+
|
| 733 |
+
col_back, col_apply = st.columns([1, 3])
|
| 734 |
+
with col_back:
|
| 735 |
+
if st.button("β Back"):
|
| 736 |
+
state["status"] = "upload"
|
| 737 |
save_state(sid, state)
|
| 738 |
st.rerun()
|
| 739 |
+
with col_apply:
|
| 740 |
+
ts_arg = None if selected_spec == _ALL else selected_spec
|
| 741 |
+
if st.button("βΆ Apply CRs", type="primary"):
|
| 742 |
+
_start_ts_run(ts_arg)
|
| 743 |
+
|
| 744 |
+
# ββ Accumulated session results ββββββββββββββββββββββββββοΏ½οΏ½οΏ½ββββββββββββ
|
| 745 |
+
applied_runs = state.get("applied_runs", [])
|
| 746 |
+
if applied_runs:
|
| 747 |
+
st.divider()
|
| 748 |
+
st.subheader(f"Session results ({len(applied_runs)} run(s))")
|
| 749 |
+
for run in applied_runs:
|
| 750 |
+
icon = "β
" if run["status"] == "done" else "β"
|
| 751 |
+
label = run["ts_id"] if run["ts_id"] != "all" else "All TSs"
|
| 752 |
+
ts_dir = Path(run["output_dir"]) / "TS"
|
| 753 |
+
specs = sorted(d.name for d in ts_dir.iterdir() if d.is_dir()) if ts_dir.exists() else []
|
| 754 |
+
specs_str = f" β {', '.join(specs)}" if specs else ""
|
| 755 |
+
st.write(f"{icon} **{label}**{specs_str}")
|
| 756 |
+
|
| 757 |
+
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 758 |
+
# PREVIEW
|
| 759 |
+
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 760 |
+
elif status == "preview":
|
| 761 |
+
cr_list = state["cr_list"]
|
| 762 |
+
st.subheader(f"Step 2 β {len(cr_list)} Accepted CR(s) found")
|
| 763 |
+
|
| 764 |
+
if cr_list:
|
| 765 |
+
import pandas as pd
|
| 766 |
+
df = pd.DataFrame(cr_list, columns=["UID", "Title"])
|
| 767 |
+
st.dataframe(df, width='stretch')
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 768 |
else:
|
| 769 |
+
st.warning(
|
| 770 |
+
f"No Accepted CRs found for **{state['person_name']}** in this file."
|
| 771 |
+
)
|
| 772 |
+
# Diagnostic: show what names are in the SubmittedBy column
|
| 773 |
+
excel_path = session_dir(sid) / state["excel_filename"]
|
| 774 |
+
found_names = peek_submitted_by(excel_path)
|
| 775 |
+
if found_names:
|
| 776 |
+
st.info(
|
| 777 |
+
"**Names found in SubmittedBy column** β copy the exact one into the field above and re-upload:\n\n"
|
| 778 |
+
+ "\n".join(f"- `{n}`" for n in found_names)
|
| 779 |
+
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 780 |
|
| 781 |
+
col1, col2 = st.columns(2)
|
| 782 |
+
with col1:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 783 |
if st.button("β Back"):
|
| 784 |
state["status"] = "upload"
|
| 785 |
+
state["cr_list"] = []
|
| 786 |
save_state(sid, state)
|
| 787 |
st.rerun()
|
| 788 |
+
with col2:
|
| 789 |
+
if cr_list and st.button("βΆ Start Pipeline", type="primary"):
|
| 790 |
+
excel_path = session_dir(sid) / state["excel_filename"]
|
| 791 |
+
# Per-run directory so each pipeline's outputs are isolated
|
| 792 |
+
run_id = int(time.time())
|
| 793 |
+
output_dir = session_dir(sid) / f"run_{run_id}"
|
| 794 |
+
output_dir.mkdir(parents=True, exist_ok=True)
|
| 795 |
+
cr_cache_dir = session_dir(sid) / "CRs"
|
| 796 |
+
cr_cache_dir.mkdir(parents=True, exist_ok=True)
|
| 797 |
+
log_path = session_dir(sid) / f"pipeline_{run_id}.log"
|
| 798 |
+
rc_path = _rc_path(sid)
|
| 799 |
+
rc_path.unlink(missing_ok=True)
|
| 800 |
+
|
| 801 |
+
cmd = [
|
| 802 |
+
sys.executable,
|
| 803 |
+
str(SCRIPTS_DIR / "orchestrate_cr.py"),
|
| 804 |
+
str(excel_path),
|
| 805 |
+
state["person_name"],
|
| 806 |
+
"--output-dir", str(output_dir),
|
| 807 |
+
"--cr-cache-dir", str(cr_cache_dir),
|
| 808 |
+
]
|
| 809 |
+
|
| 810 |
+
env = os.environ.copy()
|
| 811 |
+
env["EOL_USER"] = st.session_state.eol_user
|
| 812 |
+
env["EOL_PASSWORD"] = st.session_state.eol_password
|
| 813 |
+
|
| 814 |
+
_launch_proc(cmd, env, log_path, sid, state, {
|
| 815 |
+
"status": "running",
|
| 816 |
+
"output_dir": str(output_dir),
|
| 817 |
+
"log_path": str(log_path),
|
| 818 |
+
"run_log_paths": [str(log_path)],
|
| 819 |
+
})
|
| 820 |
+
|
| 821 |
+
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 822 |
+
# RUNNING
|
| 823 |
+
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 824 |
+
elif status == "running":
|
| 825 |
+
pid = state["pid"]
|
| 826 |
+
log_path = state["log_path"]
|
| 827 |
+
|
| 828 |
+
# Determine whether process is still alive
|
| 829 |
+
proc = st.session_state.get("proc")
|
| 830 |
+
alive = False
|
| 831 |
+
if proc is not None:
|
| 832 |
+
alive = proc.poll() is None
|
| 833 |
+
else:
|
| 834 |
+
# Session reloaded β check returncode file, then PID
|
| 835 |
+
rc = read_return_code(sid)
|
| 836 |
+
if rc is None:
|
| 837 |
+
alive = is_process_alive(pid)
|
| 838 |
+
|
| 839 |
+
if alive:
|
| 840 |
+
st.subheader("β³ Pipeline runningβ¦")
|
| 841 |
+
st.info(f"PID {pid} β started {state.get('started_at', '')[:19]}")
|
| 842 |
+
log_text = tail_log(log_path, 100)
|
| 843 |
+
st.text_area("Live log (last 100 lines)", value=log_text, height=400)
|
| 844 |
+
time.sleep(2)
|
| 845 |
+
st.rerun()
|
| 846 |
+
else:
|
| 847 |
+
# Process finished β determine return code
|
| 848 |
+
rc = read_return_code(sid)
|
| 849 |
+
if rc is None and proc is not None:
|
| 850 |
+
rc = proc.returncode
|
| 851 |
+
state["return_code"] = rc
|
| 852 |
+
state["completed_at"] = datetime.now().isoformat()
|
| 853 |
+
fin_status = "done" if rc == 0 else "error"
|
| 854 |
+
state["status"] = fin_status
|
| 855 |
+
# Accumulate completed run for TS mode
|
| 856 |
+
if state.get("mode") == "ts":
|
| 857 |
+
run_entry = {
|
| 858 |
+
"ts_id": state.get("ts_id", ""),
|
| 859 |
+
"output_dir": state.get("output_dir", ""),
|
| 860 |
+
"log_path": state.get("log_path", ""),
|
| 861 |
+
"status": fin_status,
|
| 862 |
+
}
|
| 863 |
+
runs = state.get("applied_runs", [])
|
| 864 |
+
if not any(r["output_dir"] == run_entry["output_dir"] for r in runs):
|
| 865 |
+
runs.append(run_entry)
|
| 866 |
+
state["applied_runs"] = runs
|
| 867 |
+
save_state(sid, state)
|
| 868 |
+
st.rerun()
|
| 869 |
+
|
| 870 |
+
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 871 |
+
# DONE / ERROR
|
| 872 |
+
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 873 |
+
elif status in ("done", "error"):
|
| 874 |
+
log_path = state.get("log_path", "")
|
| 875 |
+
output_dir = Path(state.get("output_dir", ""))
|
| 876 |
+
rc = state.get("return_code")
|
| 877 |
+
|
| 878 |
+
st.subheader("Step 3 β Results")
|
| 879 |
+
if status == "done":
|
| 880 |
+
st.success("β
Pipeline completed successfully!")
|
| 881 |
+
else:
|
| 882 |
+
st.error(f"β Pipeline finished with errors (return code: {rc})")
|
| 883 |
+
|
| 884 |
+
# Per-TS results table β merge this run's logs so retry results supersede
|
| 885 |
+
# the original failed entries for the same TS key.
|
| 886 |
+
_merged: dict[str, dict] = {}
|
| 887 |
+
for _lf in state.get("run_log_paths", []):
|
| 888 |
+
for _r in parse_log_results(_lf):
|
| 889 |
+
_merged[_r["TS"]] = _r
|
| 890 |
+
results = list(_merged.values())
|
| 891 |
+
if results:
|
| 892 |
+
st.subheader("Results per TS")
|
| 893 |
+
import pandas as pd
|
| 894 |
+
|
| 895 |
+
n_warn = sum(1 for r in results if r["warnings"])
|
| 896 |
+
warn_label = f"Warnings ({n_warn})" if n_warn else "Warnings"
|
| 897 |
+
tab_summary, tab_warnings = st.tabs(["Summary", warn_label])
|
| 898 |
+
|
| 899 |
+
def _color_status(val):
|
| 900 |
+
return {
|
| 901 |
+
"OK": "background-color: #d4edda; color: #155724",
|
| 902 |
+
"WARN": "background-color: #fff3cd; color: #856404",
|
| 903 |
+
"FAIL": "background-color: #f8d7da; color: #721c24",
|
| 904 |
+
"SKIP": "background-color: #e2e3e5; color: #383d41",
|
| 905 |
+
}.get(val, "")
|
| 906 |
+
|
| 907 |
+
with tab_summary:
|
| 908 |
+
df = pd.DataFrame([{"Status": r["Status"], "TS": r["TS"]} for r in results])
|
| 909 |
+
st.dataframe(
|
| 910 |
+
df.style.map(_color_status, subset=["Status"]),
|
| 911 |
+
width='stretch',
|
| 912 |
+
)
|
| 913 |
+
|
| 914 |
+
with tab_warnings:
|
| 915 |
+
warned = [r for r in results if r["warnings"]]
|
| 916 |
+
if warned:
|
| 917 |
+
for r in warned:
|
| 918 |
+
with st.expander(f"β οΈ {r['TS']} β {len(r['warnings'])} warning(s)"):
|
| 919 |
+
for w in r["warnings"]:
|
| 920 |
+
st.text(w)
|
| 921 |
+
else:
|
| 922 |
+
st.success("No warnings.")
|
| 923 |
|
| 924 |
+
# Download ZIP
|
| 925 |
+
# ββ Download section ββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 926 |
+
st.subheader("Download results")
|
| 927 |
applied_runs = state.get("applied_runs", [])
|
| 928 |
+
valid_runs = [r for r in applied_runs if Path(r["output_dir"]).exists()
|
| 929 |
+
and any(Path(r["output_dir"]).rglob("*"))]
|
| 930 |
+
|
| 931 |
+
if valid_runs:
|
| 932 |
+
# Show a compact list of what's included
|
| 933 |
+
for run in valid_runs:
|
| 934 |
icon = "β
" if run["status"] == "done" else "β"
|
| 935 |
label = run["ts_id"] if run["ts_id"] != "all" else "All TSs"
|
| 936 |
ts_dir = Path(run["output_dir"]) / "TS"
|
| 937 |
specs = sorted(d.name for d in ts_dir.iterdir() if d.is_dir()) if ts_dir.exists() else []
|
| 938 |
specs_str = f" β {', '.join(specs)}" if specs else ""
|
| 939 |
+
st.write(f"{icon} {label}{specs_str}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 940 |
|
| 941 |
+
# Build ZIP from all accumulated runs
|
| 942 |
+
buf = io.BytesIO()
|
| 943 |
+
with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as zf:
|
| 944 |
+
for run in valid_runs:
|
| 945 |
+
out = Path(run["output_dir"])
|
| 946 |
+
for f in out.rglob("*"):
|
| 947 |
+
if f.is_file():
|
| 948 |
+
zf.write(f, f.relative_to(out.parent))
|
| 949 |
+
buf.seek(0)
|
| 950 |
+
col_dl, col_flush = st.columns([3, 1])
|
| 951 |
+
with col_dl:
|
| 952 |
+
st.download_button(
|
| 953 |
+
label="β¬ Download results ZIP",
|
| 954 |
+
data=buf.read(),
|
| 955 |
+
file_name=f"cr_results_{sid[:8]}.zip",
|
| 956 |
+
mime="application/zip",
|
| 957 |
+
type="primary",
|
| 958 |
+
)
|
| 959 |
+
with col_flush:
|
| 960 |
+
if st.button("π Flush"):
|
| 961 |
+
import shutil
|
| 962 |
+
for run in applied_runs:
|
| 963 |
+
out = Path(run["output_dir"])
|
| 964 |
+
if out.exists():
|
| 965 |
+
shutil.rmtree(out, ignore_errors=True)
|
| 966 |
+
state["applied_runs"] = []
|
| 967 |
+
save_state(sid, state)
|
| 968 |
+
st.rerun()
|
| 969 |
+
elif output_dir.exists() and any(output_dir.rglob("*")):
|
| 970 |
+
# Fallback for contributor mode (no applied_runs)
|
| 971 |
+
zip_bytes = make_zip(output_dir)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 972 |
st.download_button(
|
| 973 |
label="β¬ Download results ZIP",
|
| 974 |
+
data=zip_bytes,
|
| 975 |
file_name=f"cr_results_{sid[:8]}.zip",
|
| 976 |
mime="application/zip",
|
| 977 |
type="primary",
|
| 978 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 979 |
else:
|
| 980 |
+
st.warning("Output directory is empty β nothing to download.")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 981 |
|
| 982 |
+
# Full log
|
| 983 |
+
with st.expander("Full pipeline log"):
|
| 984 |
+
if log_path and Path(log_path).exists():
|
| 985 |
+
st.text(Path(log_path).read_text(errors="replace"))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 986 |
else:
|
| 987 |
+
st.text("Log not found.")
|
| 988 |
+
|
| 989 |
+
# ββ TS Recovery βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 990 |
+
failed_ts_path = output_dir / "failed_ts.json"
|
| 991 |
+
if failed_ts_path.exists():
|
| 992 |
+
failed_ts_entries = json.loads(failed_ts_path.read_text())
|
| 993 |
+
if failed_ts_entries:
|
| 994 |
+
st.divider()
|
| 995 |
+
st.subheader("β οΈ Recover failed TS downloads")
|
| 996 |
+
st.info(
|
| 997 |
+
f"{len(failed_ts_entries)} TS(s) could not be downloaded. "
|
| 998 |
+
"Retry or upload each one manually, then apply the CRs."
|
| 999 |
+
)
|
| 1000 |
+
|
| 1001 |
+
for entry in failed_ts_entries:
|
| 1002 |
+
spec_key = f"{entry['spec_number']} v{entry['version']}"
|
| 1003 |
+
dest_path = Path(entry["spec_dir"]) / entry["expected_filename"]
|
| 1004 |
+
ready = dest_path.exists()
|
| 1005 |
+
|
| 1006 |
+
label = f"{'β
' if ready else 'β'} TS {spec_key} β CRs: {', '.join(entry['cr_uids'])}"
|
| 1007 |
+
with st.expander(label, expanded=not ready):
|
| 1008 |
+
col1, col2 = st.columns(2)
|
| 1009 |
+
|
| 1010 |
+
with col1:
|
| 1011 |
+
if st.button("π Retry download",
|
| 1012 |
+
key=f"retry_{entry['spec_compact']}_{entry['version']}"):
|
| 1013 |
+
from fetch_crs import download_ts as _dl_ts
|
| 1014 |
+
with st.spinner(f"Downloading TS {spec_key}β¦"):
|
| 1015 |
+
fn, note = _dl_ts(
|
| 1016 |
+
entry["spec_number"], entry["version"],
|
| 1017 |
+
Path(entry["spec_dir"]),
|
| 1018 |
+
st.session_state.eol_user,
|
| 1019 |
+
st.session_state.eol_password,
|
| 1020 |
+
)
|
| 1021 |
+
if fn:
|
| 1022 |
+
st.success(f"Downloaded: {fn}")
|
| 1023 |
+
st.rerun()
|
| 1024 |
+
else:
|
| 1025 |
+
st.error(f"Failed: {note}")
|
| 1026 |
+
|
| 1027 |
+
with col2:
|
| 1028 |
+
uploaded_ts = st.file_uploader(
|
| 1029 |
+
f"Or upload `{entry['expected_filename']}`",
|
| 1030 |
+
type=["docx"],
|
| 1031 |
+
key=f"upload_{entry['spec_compact']}_{entry['version']}",
|
| 1032 |
+
)
|
| 1033 |
+
if uploaded_ts is not None:
|
| 1034 |
+
Path(entry["spec_dir"]).mkdir(parents=True, exist_ok=True)
|
| 1035 |
+
dest_path.write_bytes(uploaded_ts.read())
|
| 1036 |
+
st.success("Saved β")
|
| 1037 |
+
st.rerun()
|
| 1038 |
|
| 1039 |
+
# Global apply button β enabled when β₯1 TS is now on disk
|
| 1040 |
+
ready_entries = [
|
| 1041 |
+
e for e in failed_ts_entries
|
| 1042 |
+
if (Path(e["spec_dir"]) / e["expected_filename"]).exists()
|
| 1043 |
+
]
|
| 1044 |
+
remaining = len(failed_ts_entries) - len(ready_entries)
|
| 1045 |
+
|
| 1046 |
+
if ready_entries:
|
| 1047 |
+
if remaining:
|
| 1048 |
+
st.warning(f"{len(ready_entries)} ready, {remaining} will be skipped.")
|
| 1049 |
+
else:
|
| 1050 |
+
st.success(f"All {len(ready_entries)} TS(s) ready.")
|
| 1051 |
+
|
| 1052 |
+
if st.button("βΆ Apply CRs to recovered TSs", type="primary"):
|
| 1053 |
+
retry_log = str(session_dir(sid) / f"pipeline_{int(time.time())}_retry.log")
|
| 1054 |
+
cmd = [
|
| 1055 |
+
sys.executable,
|
| 1056 |
+
str(SCRIPTS_DIR / "orchestrate_cr.py"),
|
| 1057 |
+
"--output-dir", state["output_dir"],
|
| 1058 |
+
"--retry-mode",
|
| 1059 |
+
]
|
| 1060 |
+
env = os.environ.copy()
|
| 1061 |
+
env["EOL_USER"] = st.session_state.eol_user
|
| 1062 |
+
env["EOL_PASSWORD"] = st.session_state.eol_password
|
| 1063 |
+
|
| 1064 |
+
_launch_proc(cmd, env, retry_log, sid, state, {
|
| 1065 |
+
"status": "running",
|
| 1066 |
+
"log_path": retry_log,
|
| 1067 |
+
"run_log_paths": state.get("run_log_paths", []) + [retry_log],
|
| 1068 |
+
})
|
| 1069 |
+
else:
|
| 1070 |
+
st.warning("No TSs available yet β retry download or upload DOCX files above.")
|
| 1071 |
+
|
| 1072 |
+
# ββ CR Recovery βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 1073 |
+
failed_cr_path = output_dir / "failed_cr.json"
|
| 1074 |
+
if failed_cr_path.exists():
|
| 1075 |
+
failed_cr_entries = json.loads(failed_cr_path.read_text())
|
| 1076 |
+
if failed_cr_entries:
|
| 1077 |
+
st.divider()
|
| 1078 |
+
st.subheader("β οΈ Recover failed CR downloads")
|
| 1079 |
+
st.info(
|
| 1080 |
+
f"{len(failed_cr_entries)} CR(s) could not be downloaded. "
|
| 1081 |
+
"Retry or upload each one manually, then apply."
|
| 1082 |
)
|
| 1083 |
+
|
| 1084 |
+
for entry in failed_cr_entries:
|
| 1085 |
+
uid = entry["uid"]
|
| 1086 |
+
cr_dir_path = Path(entry["cr_dir"])
|
| 1087 |
+
expected = cr_dir_path / entry["expected_filename"]
|
| 1088 |
+
ready = expected.exists() or (cr_dir_path / f"{uid}_extracted.docx").exists()
|
| 1089 |
+
ts_label = (
|
| 1090 |
+
f"TS {entry['ts_spec_number']} v{entry['ts_version']}"
|
| 1091 |
+
if entry.get("ts_spec_number") else "TS unknown"
|
| 1092 |
+
)
|
| 1093 |
+
label = f"{'β
' if ready else 'β'} CR {uid} β {ts_label}"
|
| 1094 |
+
with st.expander(label, expanded=not ready):
|
| 1095 |
+
col1, col2 = st.columns(2)
|
| 1096 |
+
|
| 1097 |
+
with col1:
|
| 1098 |
+
if st.button("π Retry download", key=f"retry_cr_{uid}"):
|
| 1099 |
+
from fetch_crs import download_cr as _dl_cr
|
| 1100 |
+
with st.spinner(f"Downloading CR {uid}β¦"):
|
| 1101 |
+
fn, note = _dl_cr(
|
| 1102 |
+
uid, cr_dir_path,
|
| 1103 |
+
st.session_state.eol_user,
|
| 1104 |
+
st.session_state.eol_password,
|
| 1105 |
+
)
|
| 1106 |
+
if fn:
|
| 1107 |
+
st.success(f"Downloaded: {fn.name}")
|
| 1108 |
+
st.rerun()
|
| 1109 |
+
else:
|
| 1110 |
+
st.error(f"Failed: {note}")
|
| 1111 |
+
|
| 1112 |
+
with col2:
|
| 1113 |
+
uploaded_cr = st.file_uploader(
|
| 1114 |
+
f"Or upload `{entry['expected_filename']}`",
|
| 1115 |
+
type=["docx"],
|
| 1116 |
+
key=f"upload_cr_{uid}",
|
| 1117 |
+
)
|
| 1118 |
+
if uploaded_cr is not None:
|
| 1119 |
+
cr_dir_path.mkdir(parents=True, exist_ok=True)
|
| 1120 |
+
expected.write_bytes(uploaded_cr.read())
|
| 1121 |
+
st.success("Saved β")
|
| 1122 |
st.rerun()
|
| 1123 |
+
|
| 1124 |
+
ready_cr_entries = [
|
| 1125 |
+
e for e in failed_cr_entries
|
| 1126 |
+
if (Path(e["cr_dir"]) / e["expected_filename"]).exists()
|
| 1127 |
+
or (Path(e["cr_dir"]) / f"{e['uid']}_extracted.docx").exists()
|
| 1128 |
+
]
|
| 1129 |
+
remaining_cr = len(failed_cr_entries) - len(ready_cr_entries)
|
| 1130 |
+
|
| 1131 |
+
if ready_cr_entries:
|
| 1132 |
+
if remaining_cr:
|
| 1133 |
+
st.warning(f"{len(ready_cr_entries)} ready, {remaining_cr} still missing.")
|
| 1134 |
+
else:
|
| 1135 |
+
st.success(f"All {len(ready_cr_entries)} CR(s) ready.")
|
| 1136 |
+
|
| 1137 |
+
if st.button("βΆ Apply recovered CRs", type="primary", key="apply_recovered_crs"):
|
| 1138 |
+
retry_log = str(session_dir(sid) / f"pipeline_{int(time.time())}_retry.log")
|
| 1139 |
+
cmd = [
|
| 1140 |
+
sys.executable,
|
| 1141 |
+
str(SCRIPTS_DIR / "orchestrate_cr.py"),
|
| 1142 |
+
"--output-dir", state["output_dir"],
|
| 1143 |
+
"--retry-mode",
|
| 1144 |
+
]
|
| 1145 |
+
env = os.environ.copy()
|
| 1146 |
+
env["EOL_USER"] = st.session_state.eol_user
|
| 1147 |
+
env["EOL_PASSWORD"] = st.session_state.eol_password
|
| 1148 |
+
_launch_proc(cmd, env, retry_log, sid, state, {
|
| 1149 |
+
"status": "running",
|
| 1150 |
+
"log_path": retry_log,
|
| 1151 |
+
"run_log_paths": state.get("run_log_paths", []) + [retry_log],
|
| 1152 |
+
})
|
| 1153 |
else:
|
| 1154 |
+
st.warning("No CRs recovered yet β retry download or upload DOCX files above.")
|
| 1155 |
+
|
| 1156 |
+
# Navigation
|
| 1157 |
+
st.divider()
|
| 1158 |
+
col_restart, col_new = st.columns(2)
|
| 1159 |
+
with col_restart:
|
| 1160 |
+
if state.get("mode") == "ts":
|
| 1161 |
+
back_label = "β Back to TS Selection"
|
| 1162 |
+
back_status = "ts_select"
|
| 1163 |
+
clear_keys = ("pid", "output_dir", "log_path", "index_log",
|
| 1164 |
+
"started_at", "completed_at", "return_code", "ts_id")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1165 |
else:
|
| 1166 |
+
back_label = "β Back to CR Preview"
|
| 1167 |
+
back_status = "preview"
|
| 1168 |
+
clear_keys = ("pid", "output_dir", "log_path", "index_log",
|
| 1169 |
+
"started_at", "completed_at", "return_code")
|
| 1170 |
+
if st.button(back_label, type="primary"):
|
| 1171 |
+
for _k in clear_keys:
|
| 1172 |
+
state[_k] = None
|
| 1173 |
+
state["run_log_paths"] = []
|
| 1174 |
+
state["status"] = back_status
|
| 1175 |
+
if "proc" in st.session_state:
|
| 1176 |
+
del st.session_state.proc
|
| 1177 |
+
_rc_path(sid).unlink(missing_ok=True)
|
| 1178 |
+
save_state(sid, state)
|
| 1179 |
+
st.rerun()
|
| 1180 |
+
with col_new:
|
| 1181 |
+
if st.button("Start new session"):
|
| 1182 |
+
new_sid = str(uuid.uuid4())
|
| 1183 |
+
st.session_state.sid = new_sid
|
| 1184 |
+
st.session_state.state = new_state(new_sid)
|
| 1185 |
+
if "proc" in st.session_state:
|
| 1186 |
+
del st.session_state.proc
|
| 1187 |
+
st.query_params["sid"] = new_sid
|
| 1188 |
+
save_state(new_sid, st.session_state.state)
|
| 1189 |
+
st.rerun()
|
| 1190 |
|
| 1191 |
+
with tab_verify:
|
| 1192 |
+
from verify_review import render_verify_tab
|
| 1193 |
+
render_verify_tab(session_dir, sid)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1194 |
|
| 1195 |
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 1196 |
# VERSION HISTORY (bottom of every stable page)
|
scripts/cr_parser.py
CHANGED
|
@@ -4,6 +4,7 @@ cr_parser.py β Parse a CR DOCX's tracked changes into a JSON manifest.
|
|
| 4 |
|
| 5 |
Each entry in the manifest is one of:
|
| 6 |
{"type": "text_replace", "location": {...}, "old": "...", "new": "..."}
|
|
|
|
| 7 |
{"type": "para_insert", "location": {...}, "paragraphs": [...]}
|
| 8 |
{"type": "row_insert", "location": {...}, "cells": [...]}
|
| 9 |
|
|
@@ -246,6 +247,54 @@ def _extract_inline_replacements(p_elem):
|
|
| 246 |
return pairs
|
| 247 |
|
| 248 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 249 |
# ββ Table change extraction βββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 250 |
|
| 251 |
def _parse_table(tbl_elem, changes, section_heading='', section_number=''):
|
|
@@ -515,6 +564,20 @@ def _parse_body(body, changes):
|
|
| 515 |
'new': new_text,
|
| 516 |
})
|
| 517 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 518 |
orig = _para_orig_text(elem).strip()
|
| 519 |
if orig and not _is_skip_marker(orig):
|
| 520 |
prev_stable_text = orig
|
|
|
|
| 4 |
|
| 5 |
Each entry in the manifest is one of:
|
| 6 |
{"type": "text_replace", "location": {...}, "old": "...", "new": "..."}
|
| 7 |
+
{"type": "text_insert", "location": {...}, "before": "...", "after": "...", "text": "...", "runs_xml": [...]}
|
| 8 |
{"type": "para_insert", "location": {...}, "paragraphs": [...]}
|
| 9 |
{"type": "row_insert", "location": {...}, "cells": [...]}
|
| 10 |
|
|
|
|
| 247 |
return pairs
|
| 248 |
|
| 249 |
|
| 250 |
+
def _gather_stable_text(children, idx, direction):
|
| 251 |
+
"""Walk from idx in `direction` (-1 or +1) collecting w:t text from stable
|
| 252 |
+
siblings until hitting a w:ins/w:del boundary or the start/end of the
|
| 253 |
+
paragraph. Used to anchor a lone insertion on the text immediately
|
| 254 |
+
surrounding it."""
|
| 255 |
+
parts = []
|
| 256 |
+
j = idx + direction
|
| 257 |
+
while 0 <= j < len(children):
|
| 258 |
+
sib = children[j]
|
| 259 |
+
if sib.tag in (qn('w:ins'), qn('w:del')):
|
| 260 |
+
break
|
| 261 |
+
parts.append(''.join(t.text or '' for t in sib.findall('.//' + qn('w:t'))))
|
| 262 |
+
j += direction
|
| 263 |
+
if direction < 0:
|
| 264 |
+
parts.reverse()
|
| 265 |
+
return ''.join(parts)
|
| 266 |
+
|
| 267 |
+
|
| 268 |
+
def _extract_lone_insertions(p_elem):
|
| 269 |
+
"""
|
| 270 |
+
Return list of (before_text, after_text, ins_text, ins_elem) for standalone
|
| 271 |
+
<w:ins> children that have NO adjacent <w:del> sibling β a pure insertion
|
| 272 |
+
with nothing marked as deleted (e.g. CR SETTEC(26)000050r1's ', 3.8').
|
| 273 |
+
|
| 274 |
+
_extract_inline_replacements only ever anchors on a <w:del> element, so a
|
| 275 |
+
lone <w:ins> with no del partner is invisible to it β this covers exactly
|
| 276 |
+
that gap. Any w:ins adjacent to a w:del is skipped here since
|
| 277 |
+
_extract_inline_replacements already accounts for it as a replacement.
|
| 278 |
+
"""
|
| 279 |
+
children = list(p_elem)
|
| 280 |
+
results = []
|
| 281 |
+
for i, child in enumerate(children):
|
| 282 |
+
if child.tag != qn('w:ins') or _is_rpr_ins(child):
|
| 283 |
+
continue
|
| 284 |
+
prev_sib = children[i - 1] if i > 0 else None
|
| 285 |
+
next_sib = children[i + 1] if i + 1 < len(children) else None
|
| 286 |
+
if (prev_sib is not None and prev_sib.tag == qn('w:del')) or \
|
| 287 |
+
(next_sib is not None and next_sib.tag == qn('w:del')):
|
| 288 |
+
continue # already accounted for by _extract_inline_replacements
|
| 289 |
+
ins_text = _ins_text(child)
|
| 290 |
+
if not ins_text:
|
| 291 |
+
continue # e.g. an inserted line-break with no text content
|
| 292 |
+
before_text = _gather_stable_text(children, i, -1)
|
| 293 |
+
after_text = _gather_stable_text(children, i, +1)
|
| 294 |
+
results.append((before_text, after_text, ins_text, child))
|
| 295 |
+
return results
|
| 296 |
+
|
| 297 |
+
|
| 298 |
# ββ Table change extraction βββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 299 |
|
| 300 |
def _parse_table(tbl_elem, changes, section_heading='', section_number=''):
|
|
|
|
| 564 |
'new': new_text,
|
| 565 |
})
|
| 566 |
|
| 567 |
+
for before_text, after_text, ins_text, ins_elem in _extract_lone_insertions(elem):
|
| 568 |
+
changes.append({
|
| 569 |
+
'type': 'text_insert',
|
| 570 |
+
'location': {
|
| 571 |
+
'kind': 'body_para',
|
| 572 |
+
'para_context': _para_orig_text(elem).strip(),
|
| 573 |
+
'section_number': current_section,
|
| 574 |
+
},
|
| 575 |
+
'before': before_text,
|
| 576 |
+
'after': after_text,
|
| 577 |
+
'text': ins_text,
|
| 578 |
+
'runs_xml': _content_runs_xml(ins_elem),
|
| 579 |
+
})
|
| 580 |
+
|
| 581 |
orig = _para_orig_text(elem).strip()
|
| 582 |
if orig and not _is_skip_marker(orig):
|
| 583 |
prev_stable_text = orig
|
scripts/docx_helpers.py
CHANGED
|
@@ -344,6 +344,51 @@ def tracked_modify_para(para, old_text, new_text, rev, author=AUTHOR, date=DATE)
|
|
| 344 |
p_el.append(_make_run(after))
|
| 345 |
|
| 346 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 347 |
def _char_diff(old, new):
|
| 348 |
"""
|
| 349 |
Return a list of (op, text) tuples for a minimal character-level diff.
|
|
|
|
| 344 |
p_el.append(_make_run(after))
|
| 345 |
|
| 346 |
|
| 347 |
+
def tracked_insert_text_in_para(para, before_text, after_text, ins_text, rev,
|
| 348 |
+
author=AUTHOR, date=DATE, runs_xml=None, split_at=None):
|
| 349 |
+
"""
|
| 350 |
+
Splice a tracked INSERTION-ONLY change into para at the point where
|
| 351 |
+
before_text ends and after_text begins β a pure content insertion with
|
| 352 |
+
NO corresponding deletion (e.g. CR SETTEC(26)000050r1's ', 3.8').
|
| 353 |
+
Splits the paragraph into: [before][<w:ins>ins_text</w:ins>][after]
|
| 354 |
+
|
| 355 |
+
Can't reuse tracked_modify_para here: it locates the splice point via
|
| 356 |
+
str.partition(old_text), and old_text would be '' for a pure insertion β
|
| 357 |
+
partition('') raises "empty separator not allowed". Instead this locates
|
| 358 |
+
the splice point by finding before_text+after_text in para.text, unless
|
| 359 |
+
the caller already resolved an offset itself (e.g. via normalized
|
| 360 |
+
matching for a CR/TS Unicode variant mismatch) and passes split_at.
|
| 361 |
+
"""
|
| 362 |
+
full = para.text
|
| 363 |
+
if split_at is None:
|
| 364 |
+
combined = before_text + after_text
|
| 365 |
+
idx = full.find(combined)
|
| 366 |
+
if idx == -1:
|
| 367 |
+
raise ValueError(
|
| 368 |
+
f"Splice point not found: before={before_text!r} after={after_text!r}\nIn: {full!r}"
|
| 369 |
+
)
|
| 370 |
+
split_at = idx + len(before_text)
|
| 371 |
+
before_part = full[:split_at]
|
| 372 |
+
after_part = full[split_at:]
|
| 373 |
+
|
| 374 |
+
p_el = para._element
|
| 375 |
+
for r in p_el.findall(qn('w:r')):
|
| 376 |
+
p_el.remove(r)
|
| 377 |
+
|
| 378 |
+
if before_part:
|
| 379 |
+
p_el.append(_make_run(before_part))
|
| 380 |
+
|
| 381 |
+
ins_el = OxmlElement('w:ins')
|
| 382 |
+
for k, v in _ins_attr(rev, author, date).items():
|
| 383 |
+
ins_el.set(k, v)
|
| 384 |
+
if not _clone_runs_into(ins_el, runs_xml, rev):
|
| 385 |
+
ins_el.append(_make_run(ins_text))
|
| 386 |
+
p_el.append(ins_el)
|
| 387 |
+
|
| 388 |
+
if after_part:
|
| 389 |
+
p_el.append(_make_run(after_part))
|
| 390 |
+
|
| 391 |
+
|
| 392 |
def _char_diff(old, new):
|
| 393 |
"""
|
| 394 |
Return a list of (op, text) tuples for a minimal character-level diff.
|
scripts/etsi_client.py
CHANGED
|
@@ -272,7 +272,7 @@ class TSFetcher:
|
|
| 272 |
"title": "1",
|
| 273 |
"etsiNumber": "1",
|
| 274 |
"content": "1",
|
| 275 |
-
"version": "
|
| 276 |
"onApproval": "1",
|
| 277 |
"published": "1",
|
| 278 |
"withdrawn": "1",
|
|
|
|
| 272 |
"title": "1",
|
| 273 |
"etsiNumber": "1",
|
| 274 |
"content": "1",
|
| 275 |
+
"version": "1",
|
| 276 |
"onApproval": "1",
|
| 277 |
"published": "1",
|
| 278 |
"withdrawn": "1",
|
scripts/orchestrate_cr.py
CHANGED
|
@@ -46,6 +46,7 @@ sys.path.insert(0, str(FETCH_SCRIPTS))
|
|
| 46 |
from fetch_crs import parse_excel, download_cr, parse_cr_cover, download_ts, wsl_path
|
| 47 |
from cr_parser import parse_cr
|
| 48 |
from ts_applicator import apply_manifest
|
|
|
|
| 49 |
from finalize_ts import (
|
| 50 |
extract_cr_metadata,
|
| 51 |
compute_pub_date,
|
|
@@ -154,6 +155,21 @@ def _apply_ts_group(spec_number, version, uids, ts_paths, cr_paths, spec_dir,
|
|
| 154 |
print(f' Parsing {uid}... ', end='', flush=True)
|
| 155 |
try:
|
| 156 |
changes = parse_cr(cr_paths[uid])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 157 |
combined_manifest.extend(changes)
|
| 158 |
participating_uids.append(uid)
|
| 159 |
print(f'{len(changes)} change(s)')
|
|
@@ -190,6 +206,14 @@ def _apply_ts_group(spec_number, version, uids, ts_paths, cr_paths, spec_dir,
|
|
| 190 |
ts_final_or_applied = ts_applied # fallback if finalise raises
|
| 191 |
try:
|
| 192 |
ts_doc = docx_lib.Document(str(ts_applied))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 193 |
rev = RevCounter(ts_doc)
|
| 194 |
|
| 195 |
pub_ym, pub_month_year = compute_pub_date()
|
|
|
|
| 46 |
from fetch_crs import parse_excel, download_cr, parse_cr_cover, download_ts, wsl_path
|
| 47 |
from cr_parser import parse_cr
|
| 48 |
from ts_applicator import apply_manifest
|
| 49 |
+
from verify_applied import scan_revision_marks, verify_manifest_applied
|
| 50 |
from finalize_ts import (
|
| 51 |
extract_cr_metadata,
|
| 52 |
compute_pub_date,
|
|
|
|
| 155 |
print(f' Parsing {uid}... ', end='', flush=True)
|
| 156 |
try:
|
| 157 |
changes = parse_cr(cr_paths[uid])
|
| 158 |
+
scan = scan_revision_marks(cr_paths[uid])
|
| 159 |
+
n_marks = scan['handled']
|
| 160 |
+
if n_marks > 0 and not changes:
|
| 161 |
+
msg = (f'ERROR verify: [{uid}] CR contains {n_marks} revision mark(s)'
|
| 162 |
+
f' but parser extracted 0 changes')
|
| 163 |
+
errors.append(msg)
|
| 164 |
+
print(f'\n {msg}', end='')
|
| 165 |
+
for tag, cnt in scan['unhandled'].items():
|
| 166 |
+
msg = (f'WARN verify: [{uid}] CR contains {cnt} unhandled revision-mark '
|
| 167 |
+
f'element(s) of type <w:{tag}> β this pipeline has no apply-path '
|
| 168 |
+
f'for this mark type; it will NOT be reflected in the output TS')
|
| 169 |
+
errors.append(msg)
|
| 170 |
+
print(f'\n {msg}', end='')
|
| 171 |
+
for c in changes:
|
| 172 |
+
c['_cr_uid'] = uid
|
| 173 |
combined_manifest.extend(changes)
|
| 174 |
participating_uids.append(uid)
|
| 175 |
print(f'{len(changes)} change(s)')
|
|
|
|
| 206 |
ts_final_or_applied = ts_applied # fallback if finalise raises
|
| 207 |
try:
|
| 208 |
ts_doc = docx_lib.Document(str(ts_applied))
|
| 209 |
+
|
| 210 |
+
verify_lines = verify_manifest_applied(ts_doc, combined_manifest)
|
| 211 |
+
for line in verify_lines:
|
| 212 |
+
print(f' {line}')
|
| 213 |
+
for line in verify_lines:
|
| 214 |
+
if line.strip().startswith('ERROR'):
|
| 215 |
+
errors.append(line.strip())
|
| 216 |
+
|
| 217 |
rev = RevCounter(ts_doc)
|
| 218 |
|
| 219 |
pub_ym, pub_month_year = compute_pub_date()
|
scripts/tests/fixtures/SETTEC(26)000048r1.docx
ADDED
|
Binary file (48 kB). View file
|
|
|
scripts/tests/fixtures/SETTEC(26)000050r1.docx
ADDED
|
Binary file (47.2 kB). View file
|
|
|
scripts/tests/test_lone_insertion.py
ADDED
|
@@ -0,0 +1,153 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""
|
| 3 |
+
Regression test for the "lone w:ins" bug: a CR paragraph containing a
|
| 4 |
+
standalone <w:ins> with no adjacent <w:del> (a pure insertion, nothing marked
|
| 5 |
+
deleted) used to be silently dropped by cr_parser.py, so a CR with real
|
| 6 |
+
content parsed to "0 changes" and the TS was never updated.
|
| 7 |
+
|
| 8 |
+
Fixtures are real CRs that exhibit this exact shape:
|
| 9 |
+
SETTEC(26)000050r1.docx β 1 lone <w:ins> (", 3.8")
|
| 10 |
+
SETTEC(26)000048r1.docx β 2 lone <w:ins> occurrences (same text, two clauses)
|
| 11 |
+
|
| 12 |
+
Run: python3 -m unittest scripts/tests/test_lone_insertion.py -v
|
| 13 |
+
"""
|
| 14 |
+
|
| 15 |
+
import sys
|
| 16 |
+
import tempfile
|
| 17 |
+
import unittest
|
| 18 |
+
from pathlib import Path
|
| 19 |
+
|
| 20 |
+
SCRIPT_DIR = Path(__file__).resolve().parent.parent
|
| 21 |
+
sys.path.insert(0, str(SCRIPT_DIR))
|
| 22 |
+
|
| 23 |
+
import docx
|
| 24 |
+
from docx.oxml.ns import qn
|
| 25 |
+
|
| 26 |
+
from cr_parser import parse_cr, _extract_inline_replacements
|
| 27 |
+
from ts_applicator import apply_manifest
|
| 28 |
+
from verify_applied import scan_revision_marks, verify_manifest_applied
|
| 29 |
+
|
| 30 |
+
FIXTURES = Path(__file__).parent / 'fixtures'
|
| 31 |
+
CR_050 = FIXTURES / 'SETTEC(26)000050r1.docx'
|
| 32 |
+
CR_048 = FIXTURES / 'SETTEC(26)000048r1.docx'
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
class LoneInsertionParseTests(unittest.TestCase):
|
| 36 |
+
|
| 37 |
+
def test_cr050_yields_one_text_insert_change(self):
|
| 38 |
+
changes = parse_cr(CR_050)
|
| 39 |
+
self.assertEqual(len(changes), 1)
|
| 40 |
+
change = changes[0]
|
| 41 |
+
self.assertEqual(change['type'], 'text_insert')
|
| 42 |
+
self.assertEqual(change['text'], ', 3.8')
|
| 43 |
+
self.assertTrue(change['before'])
|
| 44 |
+
self.assertTrue(change['after'])
|
| 45 |
+
# The anchors must not themselves contain the inserted text β otherwise
|
| 46 |
+
# the splice point would be ambiguous.
|
| 47 |
+
self.assertNotIn(', 3.8', change['before'] + change['after'])
|
| 48 |
+
|
| 49 |
+
def test_cr048_yields_two_text_insert_changes(self):
|
| 50 |
+
changes = parse_cr(CR_048)
|
| 51 |
+
self.assertEqual(len(changes), 2)
|
| 52 |
+
self.assertTrue(all(c['type'] == 'text_insert' for c in changes))
|
| 53 |
+
|
| 54 |
+
def test_scan_revision_marks_reports_one_handled_no_unhandled(self):
|
| 55 |
+
scan = scan_revision_marks(CR_050)
|
| 56 |
+
self.assertEqual(scan['handled'], 1)
|
| 57 |
+
self.assertEqual(scan['unhandled'], {})
|
| 58 |
+
|
| 59 |
+
def test_old_extractor_saw_zero_changes_here(self):
|
| 60 |
+
"""Pins the exact mechanism of the original bug: a lone w:ins is
|
| 61 |
+
invisible to _extract_inline_replacements (it only pairs w:del with
|
| 62 |
+
an adjacent w:ins), which is why the parser used to report 0 changes
|
| 63 |
+
for this CR. If this assertion ever starts failing, _extract_lone_
|
| 64 |
+
insertions may have become redundant β but it should not be removed
|
| 65 |
+
without re-verifying this class of CR is still handled."""
|
| 66 |
+
doc = docx.Document(str(CR_050))
|
| 67 |
+
found_target_para = False
|
| 68 |
+
for elem in doc.element.body:
|
| 69 |
+
if elem.tag != qn('w:p'):
|
| 70 |
+
continue
|
| 71 |
+
if any(c.tag == qn('w:ins') for c in elem):
|
| 72 |
+
found_target_para = True
|
| 73 |
+
self.assertEqual(_extract_inline_replacements(elem), [])
|
| 74 |
+
self.assertTrue(found_target_para, 'fixture no longer contains the expected w:ins paragraph')
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
class LoneInsertionApplyTests(unittest.TestCase):
|
| 78 |
+
|
| 79 |
+
def _make_ts_doc(self, paragraph_text):
|
| 80 |
+
doc = docx.Document()
|
| 81 |
+
doc.add_paragraph(paragraph_text)
|
| 82 |
+
tmp = tempfile.NamedTemporaryFile(suffix='.docx', delete=False)
|
| 83 |
+
doc.save(tmp.name)
|
| 84 |
+
return Path(tmp.name)
|
| 85 |
+
|
| 86 |
+
def test_end_to_end_apply_produces_ins_with_no_del(self):
|
| 87 |
+
changes = parse_cr(CR_050)
|
| 88 |
+
change = changes[0]
|
| 89 |
+
ts_text = change['before'] + change['after']
|
| 90 |
+
|
| 91 |
+
ts_path = self._make_ts_doc(ts_text)
|
| 92 |
+
out_path = ts_path.with_name('out.docx')
|
| 93 |
+
try:
|
| 94 |
+
n_ok, n_skip, log_lines, n_parsed, n_merged = apply_manifest(
|
| 95 |
+
ts_path, changes, out_path)
|
| 96 |
+
self.assertEqual(n_ok, 1)
|
| 97 |
+
self.assertEqual(n_skip, 0)
|
| 98 |
+
|
| 99 |
+
# python-docx's Paragraph.text only reads direct-child <w:r> runs β
|
| 100 |
+
# it does not descend into <w:ins>/<w:del>, so the inserted text
|
| 101 |
+
# must be located via the raw element tree, not p.text.
|
| 102 |
+
out_doc = docx.Document(str(out_path))
|
| 103 |
+
target = next(
|
| 104 |
+
p for p in out_doc.paragraphs
|
| 105 |
+
if change['text'] in ''.join(
|
| 106 |
+
t.text or '' for t in p._element.findall('.//' + qn('w:t')))
|
| 107 |
+
)
|
| 108 |
+
p_el = target._element
|
| 109 |
+
ins_elems = p_el.findall(qn('w:ins'))
|
| 110 |
+
del_elems = p_el.findall(qn('w:del'))
|
| 111 |
+
self.assertEqual(len(ins_elems), 1)
|
| 112 |
+
self.assertEqual(len(del_elems), 0)
|
| 113 |
+
inserted_text = ''.join(t.text or '' for t in ins_elems[0].findall('.//' + qn('w:t')))
|
| 114 |
+
self.assertEqual(inserted_text, change['text'])
|
| 115 |
+
|
| 116 |
+
verify_errors = verify_manifest_applied(out_doc, changes)
|
| 117 |
+
self.assertEqual(verify_errors, [])
|
| 118 |
+
finally:
|
| 119 |
+
ts_path.unlink(missing_ok=True)
|
| 120 |
+
out_path.unlink(missing_ok=True)
|
| 121 |
+
|
| 122 |
+
def test_nbsp_variant_between_cr_and_ts_still_applies(self):
|
| 123 |
+
"""CR 48r1's second change anchors on '... and Annex A do NOT ...'
|
| 124 |
+
(plain space), but the real TS 102267 stores 'Annex\xa0A' with a non-
|
| 125 |
+
breaking space β discovered live when re-running this exact CR
|
| 126 |
+
against the real TS. The splice point must still be found via
|
| 127 |
+
normalized matching, and the TS's real NBSP must be preserved
|
| 128 |
+
(not silently overwritten with a plain space)."""
|
| 129 |
+
changes = parse_cr(CR_048)
|
| 130 |
+
change = next(c for c in changes if 'SCP82' in c['before'])
|
| 131 |
+
ts_text = change['before'] + change['after'].replace('Annex A', 'Annex\xa0A')
|
| 132 |
+
self.assertIn('\xa0A', ts_text)
|
| 133 |
+
|
| 134 |
+
ts_path = self._make_ts_doc(ts_text)
|
| 135 |
+
out_path = ts_path.with_name('out_nbsp.docx')
|
| 136 |
+
try:
|
| 137 |
+
n_ok, n_skip, log_lines, n_parsed, n_merged = apply_manifest(
|
| 138 |
+
ts_path, [change], out_path)
|
| 139 |
+
self.assertEqual(n_ok, 1)
|
| 140 |
+
self.assertEqual(n_skip, 0)
|
| 141 |
+
|
| 142 |
+
out_doc = docx.Document(str(out_path))
|
| 143 |
+
full = ''.join(
|
| 144 |
+
t.text or '' for t in out_doc.paragraphs[0]._element.findall('.//' + qn('w:t')))
|
| 145 |
+
self.assertIn(', 3.8', full)
|
| 146 |
+
self.assertIn('Annex\xa0A', full) # original NBSP preserved, not overwritten
|
| 147 |
+
finally:
|
| 148 |
+
ts_path.unlink(missing_ok=True)
|
| 149 |
+
out_path.unlink(missing_ok=True)
|
| 150 |
+
|
| 151 |
+
|
| 152 |
+
if __name__ == '__main__':
|
| 153 |
+
unittest.main()
|
scripts/ts_applicator.py
CHANGED
|
@@ -28,6 +28,7 @@ _WARN_CONF = 0.8 # confidence below this emits WARN instead of OK
|
|
| 28 |
from docx_helpers import (
|
| 29 |
RevCounter,
|
| 30 |
tracked_modify_para,
|
|
|
|
| 31 |
tracked_insert_paras_after,
|
| 32 |
tracked_insert_paras_after_el,
|
| 33 |
_clone_runs_into,
|
|
@@ -87,6 +88,36 @@ def _norm_ws(text):
|
|
| 87 |
return re.sub(r'\s+', '', text)
|
| 88 |
|
| 89 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 90 |
def _norm_alnum(text):
|
| 91 |
"""Keep only lowercase alphanumeric characters β last-resort matching.
|
| 92 |
|
|
@@ -857,6 +888,57 @@ def _apply_text_replace(doc, change, rev, author, date, log):
|
|
| 857 |
return False
|
| 858 |
|
| 859 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 860 |
def _apply_para_insert(doc, change, rev, author, date, log, last_para_inserted=None):
|
| 861 |
loc = change['location']
|
| 862 |
anchor_text = loc.get('anchor_text', '')
|
|
@@ -1029,6 +1111,8 @@ def apply_manifest(ts_path, manifest, out_path, author=DEFAULT_AUTHOR, date=DEFA
|
|
| 1029 |
last_para_inserted=last_para_inserted)
|
| 1030 |
elif ctype == 'text_replace':
|
| 1031 |
ok = _apply_text_replace(doc, change, rev, author, date, log)
|
|
|
|
|
|
|
| 1032 |
elif ctype == 'para_insert':
|
| 1033 |
ok = _apply_para_insert(doc, change, rev, author, date, log,
|
| 1034 |
last_para_inserted=last_para_inserted)
|
|
|
|
| 28 |
from docx_helpers import (
|
| 29 |
RevCounter,
|
| 30 |
tracked_modify_para,
|
| 31 |
+
tracked_insert_text_in_para,
|
| 32 |
tracked_insert_paras_after,
|
| 33 |
tracked_insert_paras_after_el,
|
| 34 |
_clone_runs_into,
|
|
|
|
| 88 |
return re.sub(r'\s+', '', text)
|
| 89 |
|
| 90 |
|
| 91 |
+
def _normalize_with_map(text):
|
| 92 |
+
"""Apply _UNICODE_REPLACEMENTS char-by-char, returning (normalized_text,
|
| 93 |
+
index_map) where index_map[i] is the index into the ORIGINAL text that
|
| 94 |
+
normalized_text[i] came from.
|
| 95 |
+
|
| 96 |
+
Used by _apply_text_insert to locate a splice point via normalized
|
| 97 |
+
matching (e.g. CR text with a plain space vs. TS text with a non-
|
| 98 |
+
breaking space) while still splicing into the ORIGINAL, unnormalized
|
| 99 |
+
paragraph text β so a real NBSP in the TS is preserved rather than
|
| 100 |
+
silently replaced with a plain space."""
|
| 101 |
+
out_chars = []
|
| 102 |
+
idx_map = []
|
| 103 |
+
for i, ch in enumerate(text):
|
| 104 |
+
replacement = ch
|
| 105 |
+
for old, new in _UNICODE_REPLACEMENTS:
|
| 106 |
+
if ch == old:
|
| 107 |
+
replacement = new
|
| 108 |
+
break
|
| 109 |
+
for c in replacement:
|
| 110 |
+
out_chars.append(c)
|
| 111 |
+
idx_map.append(i)
|
| 112 |
+
return ''.join(out_chars), idx_map
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
def _apply_unicode_replacements(text):
|
| 116 |
+
for old, new in _UNICODE_REPLACEMENTS:
|
| 117 |
+
text = text.replace(old, new)
|
| 118 |
+
return text
|
| 119 |
+
|
| 120 |
+
|
| 121 |
def _norm_alnum(text):
|
| 122 |
"""Keep only lowercase alphanumeric characters β last-resort matching.
|
| 123 |
|
|
|
|
| 888 |
return False
|
| 889 |
|
| 890 |
|
| 891 |
+
def _apply_text_insert(doc, change, rev, author, date, log):
|
| 892 |
+
loc = change['location']
|
| 893 |
+
ctx = loc.get('para_context', '')
|
| 894 |
+
section_number = loc.get('section_number', '')
|
| 895 |
+
before = change.get('before', '')
|
| 896 |
+
after = change.get('after', '')
|
| 897 |
+
ins_text = change.get('text', '')
|
| 898 |
+
runs_xml = change.get('runs_xml')
|
| 899 |
+
|
| 900 |
+
para, conf = _find_para_with_section(
|
| 901 |
+
doc, ctx, section_number, 'text_insert', log, prefer_not_in_table=True)
|
| 902 |
+
if para is None:
|
| 903 |
+
log.append(f" ERROR text_insert: paragraph not found for context {ctx[:60]!r}")
|
| 904 |
+
return False
|
| 905 |
+
|
| 906 |
+
full = para.text
|
| 907 |
+
combined = before + after
|
| 908 |
+
split_at = None
|
| 909 |
+
if combined in full:
|
| 910 |
+
split_at = full.find(combined) + len(before)
|
| 911 |
+
else:
|
| 912 |
+
# Fallback: CR text may use a plain space/quote/dash where the TS uses
|
| 913 |
+
# a Unicode variant (NBSP, curly quote, en-dash, ...) β locate the
|
| 914 |
+
# splice point via normalized matching, then map back to the
|
| 915 |
+
# ORIGINAL (unnormalized) offset so the real TS characters are kept.
|
| 916 |
+
norm_full, idx_map = _normalize_with_map(full)
|
| 917 |
+
norm_combined = _apply_unicode_replacements(combined)
|
| 918 |
+
pos = norm_full.find(norm_combined)
|
| 919 |
+
if pos != -1:
|
| 920 |
+
norm_split = pos + len(_apply_unicode_replacements(before))
|
| 921 |
+
split_at = idx_map[norm_split] if norm_split < len(idx_map) else len(full)
|
| 922 |
+
conf = min(conf, _WARN_CONF - 0.1)
|
| 923 |
+
|
| 924 |
+
if split_at is None:
|
| 925 |
+
log.append(f" ERROR text_insert: splice anchor not found β "
|
| 926 |
+
f"before={before[-30:]!r} after={after[:30]!r}")
|
| 927 |
+
return False
|
| 928 |
+
|
| 929 |
+
try:
|
| 930 |
+
tracked_insert_text_in_para(para, before, after, ins_text, rev, author, date,
|
| 931 |
+
runs_xml=runs_xml, split_at=split_at)
|
| 932 |
+
except ValueError as e:
|
| 933 |
+
log.append(f" ERROR text_insert: {e}")
|
| 934 |
+
return False
|
| 935 |
+
|
| 936 |
+
_pfx = 'WARN' if conf < _WARN_CONF else 'OK '
|
| 937 |
+
log.append(f" {_pfx} text_insert (body_para conf={conf:.1f}): "
|
| 938 |
+
f"inserted {ins_text!r} between ...{before[-20:]!r} | {after[:20]!r}...")
|
| 939 |
+
return True
|
| 940 |
+
|
| 941 |
+
|
| 942 |
def _apply_para_insert(doc, change, rev, author, date, log, last_para_inserted=None):
|
| 943 |
loc = change['location']
|
| 944 |
anchor_text = loc.get('anchor_text', '')
|
|
|
|
| 1111 |
last_para_inserted=last_para_inserted)
|
| 1112 |
elif ctype == 'text_replace':
|
| 1113 |
ok = _apply_text_replace(doc, change, rev, author, date, log)
|
| 1114 |
+
elif ctype == 'text_insert':
|
| 1115 |
+
ok = _apply_text_insert(doc, change, rev, author, date, log)
|
| 1116 |
elif ctype == 'para_insert':
|
| 1117 |
ok = _apply_para_insert(doc, change, rev, author, date, log,
|
| 1118 |
last_para_inserted=last_para_inserted)
|
scripts/verify_applied.py
ADDED
|
@@ -0,0 +1,181 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""
|
| 3 |
+
verify_applied.py β Independent content-verification pass for applied CRs.
|
| 4 |
+
|
| 5 |
+
Two checks, both operating on data already produced by the existing
|
| 6 |
+
pipeline (cr_parser.parse_cr / ts_applicator.apply_manifest). Neither
|
| 7 |
+
changes parsing or applying behaviour β this is purely additive.
|
| 8 |
+
|
| 9 |
+
Check A β scan_revision_marks(): classifies every OOXML tracked-change
|
| 10 |
+
element in a CR DOCX (w:ins/w:del content marks, plus any other
|
| 11 |
+
CT_TrackChange-family element such as w:moveFrom/w:moveTo or
|
| 12 |
+
w:*PrChange/w:cellMerge that this pipeline has no apply-path
|
| 13 |
+
for), used to flag "CR has tracked changes but parser extracted
|
| 14 |
+
0" and "CR has a revision-mark type nothing here understands".
|
| 15 |
+
|
| 16 |
+
Check B β verify_manifest_applied(): confirms each parsed change's new
|
| 17 |
+
text is actually present in the saved output TS.
|
| 18 |
+
"""
|
| 19 |
+
|
| 20 |
+
import sys
|
| 21 |
+
from pathlib import Path
|
| 22 |
+
|
| 23 |
+
import docx
|
| 24 |
+
from docx.oxml.ns import qn
|
| 25 |
+
from lxml import etree
|
| 26 |
+
|
| 27 |
+
sys.path.insert(0, str(Path(__file__).parent))
|
| 28 |
+
|
| 29 |
+
from cr_parser import (
|
| 30 |
+
_is_rpr_ins,
|
| 31 |
+
_is_inserted_para,
|
| 32 |
+
_is_fully_inserted_tbl,
|
| 33 |
+
_is_skip_marker,
|
| 34 |
+
)
|
| 35 |
+
from ts_applicator import _norm, _norm_ws, _section_range
|
| 36 |
+
|
| 37 |
+
_MIN_SNIPPET_LEN = 4 # after whitespace-stripped normalisation
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
# ββ Check A β generalized revision-mark scan βββββββββββββββββββββββββββββββββββ
|
| 41 |
+
|
| 42 |
+
def scan_revision_marks(cr_docx_path):
|
| 43 |
+
"""
|
| 44 |
+
Classify every OOXML tracked-change element in a CR DOCX.
|
| 45 |
+
|
| 46 |
+
Every element that extends CT_TrackChange (the OOXML base type for
|
| 47 |
+
revision marks) carries a w:author attribute β not just w:ins/w:del, but
|
| 48 |
+
also w:moveFrom/w:moveTo, w:rPrChange/w:pPrChange/w:tblPrChange/
|
| 49 |
+
w:tblGridChange/w:trPrChange/w:tcPrChange/w:sectPrChange, and
|
| 50 |
+
w:cellIns/w:cellDel/w:cellMerge. Walking generically by that attribute
|
| 51 |
+
(rather than hardcoding a w:ins/w:del tag lookup) means no revision-mark
|
| 52 |
+
type β including ones not named here β goes silently unnoticed.
|
| 53 |
+
|
| 54 |
+
Returns {'handled': int, 'unhandled': {tag_localname: count, ...}}.
|
| 55 |
+
- handled: w:ins/w:del content marks (excludes rPr-nested paragraph-
|
| 56 |
+
mark markers, same filter cr_parser._is_rpr_ins applies).
|
| 57 |
+
- unhandled: any other tracked-change element found. cr_parser.py /
|
| 58 |
+
ts_applicator.py have no code path for these β they will
|
| 59 |
+
NOT be reflected in the output TS.
|
| 60 |
+
"""
|
| 61 |
+
doc = docx.Document(str(cr_docx_path))
|
| 62 |
+
body = doc.element.body
|
| 63 |
+
handled = 0
|
| 64 |
+
unhandled = {}
|
| 65 |
+
for elem in body.iter():
|
| 66 |
+
if elem.get(qn('w:author')) is None:
|
| 67 |
+
continue
|
| 68 |
+
local = etree.QName(elem).localname
|
| 69 |
+
if local in ('ins', 'del'):
|
| 70 |
+
if not _is_rpr_ins(elem):
|
| 71 |
+
handled += 1
|
| 72 |
+
else:
|
| 73 |
+
unhandled[local] = unhandled.get(local, 0) + 1
|
| 74 |
+
return {'handled': handled, 'unhandled': unhandled}
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
# ββ Check B β content-presence check ββββββββββββββββββββββββββββββββββββββββββ
|
| 78 |
+
|
| 79 |
+
def _section_replace_ins_snippets(elements_xml):
|
| 80 |
+
"""From a section_replace's serialized elements, collect w:t text of the
|
| 81 |
+
ins-block elements only (skip del-block and separator elements)."""
|
| 82 |
+
snippets = []
|
| 83 |
+
for xml_str in elements_xml:
|
| 84 |
+
elem = etree.fromstring(xml_str)
|
| 85 |
+
tag = elem.tag.split('}')[-1] if '}' in elem.tag else elem.tag
|
| 86 |
+
if tag == 'p':
|
| 87 |
+
is_ins = _is_inserted_para(elem)
|
| 88 |
+
elif tag == 'tbl':
|
| 89 |
+
is_ins = _is_fully_inserted_tbl(elem)
|
| 90 |
+
else:
|
| 91 |
+
is_ins = False
|
| 92 |
+
if is_ins:
|
| 93 |
+
snippets.append(''.join(t.text or '' for t in elem.iter(qn('w:t'))))
|
| 94 |
+
return snippets
|
| 95 |
+
|
| 96 |
+
|
| 97 |
+
def _extract_snippets(change):
|
| 98 |
+
"""Return the expected inserted-text snippet(s) for one manifest change."""
|
| 99 |
+
ctype = change.get('type')
|
| 100 |
+
if ctype == 'text_replace':
|
| 101 |
+
new = change.get('new', '')
|
| 102 |
+
return [new] if new else []
|
| 103 |
+
if ctype == 'text_insert':
|
| 104 |
+
text = change.get('text', '')
|
| 105 |
+
return [text] if text else []
|
| 106 |
+
if ctype == 'para_insert':
|
| 107 |
+
return [p.get('text', '') for p in change.get('paragraphs', [])]
|
| 108 |
+
if ctype == 'row_insert':
|
| 109 |
+
return [c.get('text', '') for c in change.get('cells', [])]
|
| 110 |
+
if ctype == 'section_replace':
|
| 111 |
+
return _section_replace_ins_snippets(change.get('elements_xml', []))
|
| 112 |
+
return []
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
def _section_scope_text(doc, section_number):
|
| 116 |
+
"""Full w:t text (paragraphs + tables) of a section, or None if not found."""
|
| 117 |
+
start, end = _section_range(doc, section_number)
|
| 118 |
+
if start is None:
|
| 119 |
+
return None
|
| 120 |
+
paras = doc.paragraphs
|
| 121 |
+
start_elem = paras[start]._element
|
| 122 |
+
end_elem = paras[end]._element if end < len(paras) else None
|
| 123 |
+
parts = []
|
| 124 |
+
in_range = False
|
| 125 |
+
for child in doc.element.body:
|
| 126 |
+
if child is start_elem:
|
| 127 |
+
in_range = True
|
| 128 |
+
if in_range:
|
| 129 |
+
if end_elem is not None and child is end_elem:
|
| 130 |
+
break
|
| 131 |
+
parts.append(''.join(t.text or '' for t in child.iter(qn('w:t'))))
|
| 132 |
+
return ''.join(parts)
|
| 133 |
+
|
| 134 |
+
|
| 135 |
+
def _snippet_present(snippet, corpus):
|
| 136 |
+
if snippet in corpus:
|
| 137 |
+
return True
|
| 138 |
+
if _norm(snippet) in _norm(corpus):
|
| 139 |
+
return True
|
| 140 |
+
if _norm_ws(snippet) in _norm_ws(corpus):
|
| 141 |
+
return True
|
| 142 |
+
return False
|
| 143 |
+
|
| 144 |
+
|
| 145 |
+
def verify_manifest_applied(ts_doc, manifest):
|
| 146 |
+
"""Confirm each change's expected new-text snippet(s) are present in ts_doc.
|
| 147 |
+
|
| 148 |
+
A declared section_number is a hint, not a hard boundary: the real
|
| 149 |
+
applicator (ts_applicator.py) falls back to a global search whenever an
|
| 150 |
+
anchor/table isn't found in the declared section (see the "using global
|
| 151 |
+
match" WARN lines it emits), so content can legitimately land outside
|
| 152 |
+
its declared section. To avoid flagging those legitimate placements,
|
| 153 |
+
a snippet is only reported missing if it's absent from BOTH the
|
| 154 |
+
section-scoped corpus (when available) and the whole document.
|
| 155 |
+
|
| 156 |
+
Returns a list of 'ERROR verify: ...' lines, one per missing snippet."""
|
| 157 |
+
errors = []
|
| 158 |
+
whole_doc_corpus = None
|
| 159 |
+
|
| 160 |
+
for change in manifest:
|
| 161 |
+
ctype = change.get('type')
|
| 162 |
+
cr_uid = change.get('_cr_uid', '?')
|
| 163 |
+
section_number = (change.get('location') or {}).get('section_number', '')
|
| 164 |
+
|
| 165 |
+
scope_corpus = _section_scope_text(ts_doc, section_number) if section_number else None
|
| 166 |
+
|
| 167 |
+
for snippet in _extract_snippets(change):
|
| 168 |
+
if not snippet or _is_skip_marker(snippet):
|
| 169 |
+
continue
|
| 170 |
+
if len(_norm_ws(snippet)) < _MIN_SNIPPET_LEN:
|
| 171 |
+
continue
|
| 172 |
+
if scope_corpus is not None and _snippet_present(snippet, scope_corpus):
|
| 173 |
+
continue
|
| 174 |
+
if whole_doc_corpus is None:
|
| 175 |
+
whole_doc_corpus = ''.join(t.text or '' for t in ts_doc.element.body.iter(qn('w:t')))
|
| 176 |
+
if not _snippet_present(snippet, whole_doc_corpus):
|
| 177 |
+
errors.append(
|
| 178 |
+
f'ERROR verify: [{cr_uid}] {ctype} β expected text not found '
|
| 179 |
+
f'in output TS: {snippet[:80]!r}'
|
| 180 |
+
)
|
| 181 |
+
return errors
|
skills/cr-visual-pair/SKILL.md
ADDED
|
@@ -0,0 +1,315 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
name: cr-visual-pair
|
| 3 |
+
description: Renders each modified TS section/annex and its associated CR excerpt(s) to PNG and packages them into a review_package.zip for upload to the ApplyCRs app's "Verify insertions" tab. Does NOT modify pipeline code β export only. TRIGGER on "export review package", "prepare visual review", "pair CR and TS screenshots for the app", "generate review package", or when the user wants to upload CR/TS screenshots to the app for OK/Not OK review.
|
| 4 |
+
---
|
| 5 |
+
|
| 6 |
+
# CR Visual Pair β Export a Review Package for the App's Verify Tab
|
| 7 |
+
|
| 8 |
+
Renders every modified **TS section/annex** from an ApplyCRs run into a screenshot, next
|
| 9 |
+
to the CR excerpt(s) that produced it, and bundles them into a `review_package.zip` the
|
| 10 |
+
user uploads to the ApplyCRs app's **"β
Verify insertions"** tab. There the user clicks
|
| 11 |
+
through each pair and marks OK / Not OK / Wrong match, then downloads a log.
|
| 12 |
+
|
| 13 |
+
**A pair is one TS section or annex β never a bundle of unrelated sections.** If several
|
| 14 |
+
CRs (or several change requests inside one CR) all touch the same section, they combine
|
| 15 |
+
into that section's single pair. If one CR touches several disjoint sections/annexes,
|
| 16 |
+
each gets its own pair with only that section's own CR excerpt β never the whole CR
|
| 17 |
+
dumped alongside every section it happens to touch.
|
| 18 |
+
|
| 19 |
+
This skill only **exports images** β it never edits `cr_parser.py`, `ts_applicator.py`, or
|
| 20 |
+
any pipeline code. It does *import* `cr_parser.parse_cr()` (read-only) to get the exact
|
| 21 |
+
section/change breakdown of each CR β see Step 1. For diagnosing and fixing pipeline
|
| 22 |
+
bugs, use `cr-visual-verify` instead.
|
| 23 |
+
|
| 24 |
+
Run this skill **locally on Windows** (it needs Word COM to convert DOCX β PDF). The HF-hosted
|
| 25 |
+
app has no DOCX renderer, which is why this export step exists.
|
| 26 |
+
|
| 27 |
+
---
|
| 28 |
+
|
| 29 |
+
## Input
|
| 30 |
+
|
| 31 |
+
A **zip file** or **folder path** from an ApplyCRs run. Expected structure:
|
| 32 |
+
|
| 33 |
+
```
|
| 34 |
+
<root>/
|
| 35 |
+
CRs/<uid>.docx
|
| 36 |
+
TS/<spec>/ts_<spec>_v<old>.docx
|
| 37 |
+
TS/<spec>/ts_<spec>_v<new>_was_v<old>.docx
|
| 38 |
+
TS/<spec>/ts_<spec>_v<new>_was_v<old>.log
|
| 39 |
+
```
|
| 40 |
+
|
| 41 |
+
Windows paths: convert `C:\Users\...` to `/mnt/c/Users/...`.
|
| 42 |
+
|
| 43 |
+
You also need a local checkout of the **ApplyCRs repo** (for `scripts/cr_parser.py` and
|
| 44 |
+
this skill's `build_pairs.py`). Locate `REPO_ROOT` by checking, in order:
|
| 45 |
+
1. Three levels up from this file's own location (`<this_file>/../../..`) β true when the
|
| 46 |
+
skill is used from inside an ApplyCRs checkout.
|
| 47 |
+
2. Otherwise ask the user for the path to their ApplyCRs repo clone.
|
| 48 |
+
|
| 49 |
+
`REPO_ROOT/scripts/cr_parser.py` and `REPO_ROOT/skills/cr-visual-pair/build_pairs.py` must
|
| 50 |
+
both exist before continuing.
|
| 51 |
+
|
| 52 |
+
## Output
|
| 53 |
+
|
| 54 |
+
`review_package.zip` (default location: alongside the input, or `/tmp/cr_visual_pair_<ts>/review_package.zip`):
|
| 55 |
+
|
| 56 |
+
```
|
| 57 |
+
review_package.zip
|
| 58 |
+
manifest.json
|
| 59 |
+
pairs/001_ts_p12.png
|
| 60 |
+
pairs/001_cr_SETTEC_26_000050r1_p5.png
|
| 61 |
+
pairs/002_ts_p40.png
|
| 62 |
+
pairs/002_cr_SETTEC_26_000050r1_p3.png
|
| 63 |
+
pairs/002_cr_SETTEC_26_000051_p2.png
|
| 64 |
+
...
|
| 65 |
+
```
|
| 66 |
+
|
| 67 |
+
`manifest.json` schema β **one entry per TS section/annex**, not per CR:
|
| 68 |
+
|
| 69 |
+
```json
|
| 70 |
+
{
|
| 71 |
+
"run": "run_1783411741",
|
| 72 |
+
"generated_at": "<ISO timestamp>",
|
| 73 |
+
"pairs": [
|
| 74 |
+
{
|
| 75 |
+
"id": 1,
|
| 76 |
+
"ts_spec": "102267",
|
| 77 |
+
"ts_version": "18.0.0 -> 18.1.0",
|
| 78 |
+
"section_number": "7.5",
|
| 79 |
+
"matched": true,
|
| 80 |
+
"ts_images": ["pairs/002_ts_p40.png"],
|
| 81 |
+
"changes": [
|
| 82 |
+
{
|
| 83 |
+
"cr_uid": "SETTEC(26)000050r1",
|
| 84 |
+
"change_type": "text_replace",
|
| 85 |
+
"matched": true,
|
| 86 |
+
"summary": "short description from the CR cover page",
|
| 87 |
+
"cr_images": ["pairs/002_cr_SETTEC_26_000050r1_p3.png"]
|
| 88 |
+
},
|
| 89 |
+
{
|
| 90 |
+
"cr_uid": "SETTEC(26)000051",
|
| 91 |
+
"change_type": "para_insert",
|
| 92 |
+
"matched": true,
|
| 93 |
+
"summary": "short description from the CR cover page",
|
| 94 |
+
"cr_images": ["pairs/002_cr_SETTEC_26_000051_p2.png"]
|
| 95 |
+
}
|
| 96 |
+
]
|
| 97 |
+
}
|
| 98 |
+
]
|
| 99 |
+
}
|
| 100 |
+
```
|
| 101 |
+
|
| 102 |
+
- One pair per `(ts_spec, ts_version, section_number)`. All modification requests that
|
| 103 |
+
land inside that section β whether from one CR or several β are entries in `changes[]`
|
| 104 |
+
on that *same* pair. A different section number is always a *different* pair.
|
| 105 |
+
- `ts_images` covers every page that section occupies in the applied TS β one page for a
|
| 106 |
+
section that fits on one page, two (or more) consecutive pages for a section that
|
| 107 |
+
starts on one page and runs onto the next β never pages belonging to some other section.
|
| 108 |
+
- Pair-level `matched: false` means the section's page(s) couldn't be confidently located β
|
| 109 |
+
still include the best-guess TS image (or an empty list if none).
|
| 110 |
+
- Change-level `matched: false` means that specific change's own text couldn't be
|
| 111 |
+
confidently pinpointed within the section's page(s), even though the section itself was
|
| 112 |
+
found β the reviewer should double-check that one contributor.
|
| 113 |
+
- `id` is a stable 1-based sequence, assigned in document order (each pair's first `ts_pages`
|
| 114 |
+
entry ascending), used for the PNG filename prefix, zero-padded to 3 digits.
|
| 115 |
+
|
| 116 |
+
---
|
| 117 |
+
|
| 118 |
+
## Step 1 β Extract, enumerate, and parse each CR's changes
|
| 119 |
+
|
| 120 |
+
```bash
|
| 121 |
+
WORK_DIR="/tmp/cr_visual_pair_$(date +%s)"
|
| 122 |
+
mkdir -p "$WORK_DIR"
|
| 123 |
+
unzip -o "<zip_path>" -d "$WORK_DIR" # skip if given a folder path directly
|
| 124 |
+
RUN_ROOT=$(find "$WORK_DIR" -name "CRs" -type d -printf '%h\n' | head -1)
|
| 125 |
+
```
|
| 126 |
+
|
| 127 |
+
Enumerate `(TS log, cr_uid)` associations from every `TS/<spec>/ts_*_was_*.log` header:
|
| 128 |
+
|
| 129 |
+
```
|
| 130 |
+
TS: 102 267 v18.0.0 -> v18.1.0
|
| 131 |
+
CRs: SETTEC(26)000050r1
|
| 132 |
+
```
|
| 133 |
+
|
| 134 |
+
A log can list multiple CRs applied to the same TS β collect all of them per log. Skip
|
| 135 |
+
logs whose text says "No changes parsed" (nothing to review β no doc changed).
|
| 136 |
+
|
| 137 |
+
CR docx: `CRs/<uid>.docx`. Applied TS docx: same directory as the log,
|
| 138 |
+
`ts_<spec>_v<new>_was_v<old>.docx`.
|
| 139 |
+
|
| 140 |
+
For each **unique** `cr_uid` across the whole run, parse it once (cache by uid):
|
| 141 |
+
|
| 142 |
+
```python
|
| 143 |
+
import sys
|
| 144 |
+
sys.path.insert(0, "<REPO_ROOT>/skills/cr-visual-pair")
|
| 145 |
+
from build_pairs import changes_for_cr
|
| 146 |
+
|
| 147 |
+
changes = changes_for_cr("<RUN_ROOT>/CRs/<uid>.docx", scripts_dir="<REPO_ROOT>/scripts")
|
| 148 |
+
# -> list of {change_type, section_number, cr_search_text, ts_search_text, raw}
|
| 149 |
+
```
|
| 150 |
+
|
| 151 |
+
Also grab a one-line `summary` per `cr_uid` from its CR cover page (page 1) β same idea as
|
| 152 |
+
before, just for display context, not for matching:
|
| 153 |
+
|
| 154 |
+
```bash
|
| 155 |
+
pdftotext -f 1 -l 1 "<cr>.pdf" - | head -40 # read 'Reason for change' / title
|
| 156 |
+
```
|
| 157 |
+
|
| 158 |
+
---
|
| 159 |
+
|
| 160 |
+
## Step 2 β Screenshot pipeline (same technique as `cr-visual-verify`)
|
| 161 |
+
|
| 162 |
+
### 2a. Copy files to Windows temp
|
| 163 |
+
|
| 164 |
+
```bash
|
| 165 |
+
WIN_TEMP="/mnt/c/Users/$(ls /mnt/c/Users/ | grep -E '^[Tt][0-9]' | head -1)/Downloads/cr_visual_pair"
|
| 166 |
+
mkdir -p "$WIN_TEMP"
|
| 167 |
+
cp <docx_path> "$WIN_TEMP/<shortname>.docx"
|
| 168 |
+
```
|
| 169 |
+
|
| 170 |
+
Copy **every** unique CR docx and **every** unique TS (applied) docx involved in the run β
|
| 171 |
+
each is converted once regardless of how many sections/CRs reference it.
|
| 172 |
+
|
| 173 |
+
### 2b. Convert DOCX β PDF via Word (PowerShell)
|
| 174 |
+
|
| 175 |
+
```bash
|
| 176 |
+
powershell.exe -Command '
|
| 177 |
+
$dir = "<win_path>"
|
| 178 |
+
$word = New-Object -ComObject Word.Application
|
| 179 |
+
$word.Visible = $false
|
| 180 |
+
$files = @("<file1>", "<file2>")
|
| 181 |
+
foreach ($f in $files) {
|
| 182 |
+
$doc = $word.Documents.Open("$dir\$f.docx")
|
| 183 |
+
$doc.SaveAs([ref]"$dir\$f.pdf", [ref]17)
|
| 184 |
+
$doc.Close()
|
| 185 |
+
}
|
| 186 |
+
$word.Quit()
|
| 187 |
+
'
|
| 188 |
+
```
|
| 189 |
+
|
| 190 |
+
### 2c. Locate each change's CR page and TS page
|
| 191 |
+
|
| 192 |
+
For each TS doc, `pdftotext` it once and split into pages; do the same for every CR docx
|
| 193 |
+
involved:
|
| 194 |
+
|
| 195 |
+
```bash
|
| 196 |
+
pdftotext "<ts>.pdf" /tmp/ts_pages.txt
|
| 197 |
+
pdftotext "<cr>.pdf" /tmp/cr_pages.txt
|
| 198 |
+
```
|
| 199 |
+
|
| 200 |
+
```python
|
| 201 |
+
import sys
|
| 202 |
+
sys.path.insert(0, "<REPO_ROOT>/skills/cr-visual-pair")
|
| 203 |
+
from build_pairs import locate_page
|
| 204 |
+
|
| 205 |
+
def split_pages(path):
|
| 206 |
+
return open(path, encoding="utf-8", errors="replace").read().split("\f")
|
| 207 |
+
|
| 208 |
+
cr_pages = split_pages("/tmp/cr_pages.txt")
|
| 209 |
+
ts_pages = split_pages("/tmp/ts_pages.txt")
|
| 210 |
+
|
| 211 |
+
cr_page, cr_confident = locate_page(cr_pages, search_text=ch["cr_search_text"],
|
| 212 |
+
section_number=ch["section_number"])
|
| 213 |
+
ts_page, ts_confident = locate_page(ts_pages, search_text=ch["ts_search_text"],
|
| 214 |
+
section_number=ch["section_number"])
|
| 215 |
+
```
|
| 216 |
+
|
| 217 |
+
Do this for every change belonging to every `cr_uid` associated with this TS doc. Collect
|
| 218 |
+
one located-change record per change (see `group_by_section`'s expected shape in
|
| 219 |
+
`build_pairs.py`'s docstring): `ts_spec`, `ts_version`, `cr_uid`, `change_type`,
|
| 220 |
+
`section_number`, `summary`, `ts_page`, `ts_confident`, `cr_pages: [cr_page]`,
|
| 221 |
+
`cr_confident`.
|
| 222 |
+
|
| 223 |
+
If a change's search text isn't found and it has no `section_number` either, its
|
| 224 |
+
`ts_page` stays `None` β it will still surface as its own unmatched pair (see Step 2e),
|
| 225 |
+
never silently dropped.
|
| 226 |
+
|
| 227 |
+
### 2d. Group into section-level pairs
|
| 228 |
+
|
| 229 |
+
```python
|
| 230 |
+
from build_pairs import group_by_section
|
| 231 |
+
|
| 232 |
+
pairs = group_by_section(located_changes) # sorted by first ts page ascending
|
| 233 |
+
```
|
| 234 |
+
|
| 235 |
+
Each pair's `ts_pages` is a list, not a single page β if a section's changes were
|
| 236 |
+
confidently located on both page 1 and page 2, `ts_pages` is `[1, 2]` and **both** must be
|
| 237 |
+
rendered and included in `ts_images` (see Step 2e). Don't collapse it back to one page.
|
| 238 |
+
|
| 239 |
+
Assign `id` 1, 2, 3, β¦ in this order, per TS doc, in the order TS docs are encountered.
|
| 240 |
+
|
| 241 |
+
### 2e. Render pages to PNG
|
| 242 |
+
|
| 243 |
+
Render only the pages actually referenced by the grouped pairs β dedupe by
|
| 244 |
+
`(doc_path, page_no)` so a page shared by two changes (same section, or coincidentally
|
| 245 |
+
adjacent) is never rendered twice:
|
| 246 |
+
|
| 247 |
+
```bash
|
| 248 |
+
gs -dBATCH -dNOPAUSE -sDEVICE=png16m -r200 \
|
| 249 |
+
-dFirstPage=<N> -dLastPage=<N> \
|
| 250 |
+
-sOutputFile="<WORK_DIR>/pairs/<id>_ts_p<N>.png" "<ts>.pdf"
|
| 251 |
+
|
| 252 |
+
gs -dBATCH -dNOPAUSE -sDEVICE=png16m -r200 \
|
| 253 |
+
-dFirstPage=<N> -dLastPage=<N> \
|
| 254 |
+
-sOutputFile="<WORK_DIR>/pairs/<id>_cr_<sanitized_cr_uid>_p<N>.png" "<cr>.pdf"
|
| 255 |
+
```
|
| 256 |
+
|
| 257 |
+
Sanitize `cr_uid` for the filename (e.g. `SETTEC(26)000050r1` β `SETTEC_26_000050r1`) by
|
| 258 |
+
replacing anything that isn't `[A-Za-z0-9_-]` with `_`.
|
| 259 |
+
|
| 260 |
+
Zero-pad `<id>` to 3 digits (`001`, `002`, β¦). On the TS side, render **every** page in the
|
| 261 |
+
pair's `ts_pages` list from Step 2d (in ascending order) and add each one to `ts_images` β
|
| 262 |
+
this is how a section that starts on one page and ends on the next gets both pages shown,
|
| 263 |
+
not just the first. On the CR side, if a change's marked text visibly continues onto the
|
| 264 |
+
next page, render that page too and include it in `cr_images` β use judgment there, same
|
| 265 |
+
as the old skill's approach, just scoped to this one change/section instead of the whole CR.
|
| 266 |
+
|
| 267 |
+
### 2f. Validate alignment before finalizing each pair
|
| 268 |
+
|
| 269 |
+
Use `Read` on the TS image and every contributing change's CR image(s). Confirm the
|
| 270 |
+
section heading/number visible on the TS screenshot matches the pair's `section_number`,
|
| 271 |
+
and that each change's identifying text is visible on its own CR screenshot. If a
|
| 272 |
+
specific change's text isn't visible where expected:
|
| 273 |
+
- Try the next candidate page from Step 2c for that change only.
|
| 274 |
+
- If nothing matches after reasonable attempts, keep the best-guess image and set that
|
| 275 |
+
change's `matched: false` β don't set the whole pair to `false` just because one of
|
| 276 |
+
several contributors is uncertain.
|
| 277 |
+
|
| 278 |
+
If the section's own page can't be found at all, mark the pair `matched: false` rather
|
| 279 |
+
than guessing and reporting success.
|
| 280 |
+
|
| 281 |
+
---
|
| 282 |
+
|
| 283 |
+
## Step 3 β Write manifest and zip
|
| 284 |
+
|
| 285 |
+
Write `<WORK_DIR>/manifest.json` per the schema above, then:
|
| 286 |
+
|
| 287 |
+
```bash
|
| 288 |
+
cd "<WORK_DIR>" && zip -r review_package.zip manifest.json pairs/
|
| 289 |
+
```
|
| 290 |
+
|
| 291 |
+
Report the final path to the user, e.g.:
|
| 292 |
+
|
| 293 |
+
```
|
| 294 |
+
Review package ready: /tmp/cr_visual_pair_1783412000/review_package.zip
|
| 295 |
+
9 pair(s) (sections), 12 change(s) across CRs, 7 fully matched, 2 low-confidence
|
| 296 |
+
Upload this zip to the app's "β
Verify insertions" tab.
|
| 297 |
+
```
|
| 298 |
+
|
| 299 |
+
---
|
| 300 |
+
|
| 301 |
+
## Important rules
|
| 302 |
+
|
| 303 |
+
1. **Export only β never modify pipeline code.** `build_pairs.py`'s `changes_for_cr()`
|
| 304 |
+
only *imports* `cr_parser.parse_cr()` β it never writes to `cr_parser.py`,
|
| 305 |
+
`ts_applicator.py`, or any CR/TS document. If you spot a pipeline bug while doing this,
|
| 306 |
+
mention it to the user and suggest running `cr-visual-verify` instead; don't fix it here.
|
| 307 |
+
2. **One pair per section/annex, never a bundle of several.** A pair's CR side is scoped
|
| 308 |
+
to only the change(s) that land in that section β not the whole CR's every page.
|
| 309 |
+
3. **Multiple CRs (or multiple changes) in the same section combine into one pair** β
|
| 310 |
+
that's the whole point of grouping by `section_number` instead of by CR uid or raw
|
| 311 |
+
page number.
|
| 312 |
+
4. **Don't silently mispair** β if a section's TS page can't be confidently located, mark
|
| 313 |
+
`matched: false` instead of guessing and reporting success. Same at the per-change
|
| 314 |
+
level for CR-side matches.
|
| 315 |
+
5. **`[...]` in a CR is normal** β it means omitted unchanged content, not a missing image.
|
skills/cr-visual-pair/build_pairs.py
ADDED
|
@@ -0,0 +1,192 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
build_pairs.py β pure text/data helpers for cr-visual-pair.
|
| 3 |
+
|
| 4 |
+
Groups CR changes by the TS section/annex they land in (not by raw page
|
| 5 |
+
number, and not by CR uid), so a review "pair" is always exactly one
|
| 6 |
+
section β never a bundle of unrelated sections, and never split when
|
| 7 |
+
several CRs (or several changes within one CR) touch the same section.
|
| 8 |
+
|
| 9 |
+
Read-only consumer of scripts/cr_parser.py's parse_cr() β never imports
|
| 10 |
+
anything that writes back to CR/TS documents.
|
| 11 |
+
|
| 12 |
+
No Word/PDF/gs dependency: locate_page() and group_by_section() operate on
|
| 13 |
+
plain strings and dicts, so they can be exercised with synthetic data.
|
| 14 |
+
"""
|
| 15 |
+
|
| 16 |
+
import re
|
| 17 |
+
import sys
|
| 18 |
+
from collections import Counter
|
| 19 |
+
from pathlib import Path
|
| 20 |
+
|
| 21 |
+
_WS_RE = re.compile(r'\s+')
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
def _normalize(text):
|
| 25 |
+
return _WS_RE.sub(' ', text.strip())
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
def locate_page(pages_text, search_text=None, section_number=None, min_len=6):
|
| 29 |
+
"""
|
| 30 |
+
Find the 1-based page in `pages_text` (a list of page-text strings, in
|
| 31 |
+
document order) that corresponds to a change.
|
| 32 |
+
|
| 33 |
+
Tries, in order:
|
| 34 |
+
1. Exact whitespace-normalized substring match on `search_text` (if
|
| 35 |
+
long enough to be distinctive) -> confident.
|
| 36 |
+
2. A heading line starting with `section_number` -> confident.
|
| 37 |
+
3. A bare substring match of `section_number` anywhere on the page
|
| 38 |
+
-> not confident (could be a cross-reference, not the heading).
|
| 39 |
+
|
| 40 |
+
Returns (page_no_or_None, confident_bool).
|
| 41 |
+
"""
|
| 42 |
+
search_text = (search_text or '').strip()
|
| 43 |
+
if search_text and len(search_text) >= min_len:
|
| 44 |
+
needle = _normalize(search_text)
|
| 45 |
+
for i, page in enumerate(pages_text, start=1):
|
| 46 |
+
if needle in _normalize(page):
|
| 47 |
+
return i, True
|
| 48 |
+
|
| 49 |
+
if section_number:
|
| 50 |
+
heading_re = re.compile(r'(?m)^\s*' + re.escape(section_number) + r'(?:[ \t]|$)')
|
| 51 |
+
for i, page in enumerate(pages_text, start=1):
|
| 52 |
+
if heading_re.search(page):
|
| 53 |
+
return i, True
|
| 54 |
+
for i, page in enumerate(pages_text, start=1):
|
| 55 |
+
if section_number in page:
|
| 56 |
+
return i, False
|
| 57 |
+
|
| 58 |
+
return None, False
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
def group_by_section(located_changes):
|
| 62 |
+
"""
|
| 63 |
+
Group already-located changes into review pairs, one per
|
| 64 |
+
(ts_spec, ts_version, section_number).
|
| 65 |
+
|
| 66 |
+
Each item in `located_changes` is expected to have:
|
| 67 |
+
ts_spec, ts_version, cr_uid, change_type, section_number (may be ''),
|
| 68 |
+
summary, ts_page (int|None), ts_confident (bool),
|
| 69 |
+
cr_pages (list[int]), cr_confident (bool).
|
| 70 |
+
|
| 71 |
+
Changes with no resolvable section_number fall back to grouping by
|
| 72 |
+
their located ts_page instead (each distinct page becomes its own
|
| 73 |
+
group), so unresolved changes still never get merged across unrelated
|
| 74 |
+
locations.
|
| 75 |
+
|
| 76 |
+
A section's changes can be located on more than one page (e.g. a clause
|
| 77 |
+
that starts on page 1 and runs onto page 2) β every confidently-located
|
| 78 |
+
page is kept, not just the majority one, so the section's full page
|
| 79 |
+
range is always represented.
|
| 80 |
+
|
| 81 |
+
Returns a list of pair dicts, sorted by the first page each group
|
| 82 |
+
resolved to (unresolved groups sort last, in first-seen order):
|
| 83 |
+
{ts_spec, ts_version, section_number, ts_pages, matched, changes: [...]}
|
| 84 |
+
"""
|
| 85 |
+
groups = {}
|
| 86 |
+
order_hint = {}
|
| 87 |
+
|
| 88 |
+
for idx, ch in enumerate(located_changes):
|
| 89 |
+
section_number = ch.get('section_number') or ''
|
| 90 |
+
key_section = section_number if section_number else '__page_%r__' % (ch.get('ts_page'),)
|
| 91 |
+
key = (ch['ts_spec'], ch['ts_version'], key_section)
|
| 92 |
+
|
| 93 |
+
if key not in groups:
|
| 94 |
+
groups[key] = {
|
| 95 |
+
'ts_spec': ch['ts_spec'],
|
| 96 |
+
'ts_version': ch['ts_version'],
|
| 97 |
+
'section_number': section_number,
|
| 98 |
+
'ts_pages': [],
|
| 99 |
+
'changes': [],
|
| 100 |
+
}
|
| 101 |
+
order_hint[key] = idx
|
| 102 |
+
|
| 103 |
+
g = groups[key]
|
| 104 |
+
if ch.get('ts_page') is not None:
|
| 105 |
+
g['ts_pages'].append((ch['ts_page'], bool(ch.get('ts_confident'))))
|
| 106 |
+
g['changes'].append({
|
| 107 |
+
'cr_uid': ch['cr_uid'],
|
| 108 |
+
'change_type': ch['change_type'],
|
| 109 |
+
'summary': ch.get('summary', ''),
|
| 110 |
+
'matched': bool(ch.get('cr_confident')) and bool(ch.get('ts_confident')),
|
| 111 |
+
'cr_pages': list(ch.get('cr_pages', [])),
|
| 112 |
+
})
|
| 113 |
+
|
| 114 |
+
result = []
|
| 115 |
+
for key, g in groups.items():
|
| 116 |
+
pages = g['ts_pages']
|
| 117 |
+
confident_pages = sorted({p for p, c in pages if c})
|
| 118 |
+
if confident_pages:
|
| 119 |
+
# Span the full range between confidently-located pages, so a
|
| 120 |
+
# section that starts on one page and ends on the next always
|
| 121 |
+
# includes both β not just whichever page most changes landed on.
|
| 122 |
+
ts_pages = list(range(confident_pages[0], confident_pages[-1] + 1))
|
| 123 |
+
matched = True
|
| 124 |
+
elif pages:
|
| 125 |
+
counts = Counter(p for p, _ in pages)
|
| 126 |
+
best_page = max(counts.items(), key=lambda kv: (kv[1], -kv[0]))[0]
|
| 127 |
+
ts_pages = [best_page]
|
| 128 |
+
matched = False
|
| 129 |
+
else:
|
| 130 |
+
ts_pages = []
|
| 131 |
+
matched = False
|
| 132 |
+
result.append({
|
| 133 |
+
'ts_spec': g['ts_spec'],
|
| 134 |
+
'ts_version': g['ts_version'],
|
| 135 |
+
'section_number': g['section_number'],
|
| 136 |
+
'ts_pages': ts_pages,
|
| 137 |
+
'matched': matched,
|
| 138 |
+
'changes': g['changes'],
|
| 139 |
+
'_order_key': (ts_pages[0] if ts_pages else float('inf'), order_hint[key]),
|
| 140 |
+
})
|
| 141 |
+
|
| 142 |
+
result.sort(key=lambda r: r['_order_key'])
|
| 143 |
+
for r in result:
|
| 144 |
+
del r['_order_key']
|
| 145 |
+
return result
|
| 146 |
+
|
| 147 |
+
|
| 148 |
+
def changes_for_cr(cr_docx_path, scripts_dir):
|
| 149 |
+
"""
|
| 150 |
+
Parse one CR docx via the pipeline's own cr_parser.parse_cr() (read-only
|
| 151 |
+
import β never modifies cr_parser.py) and return one entry per discrete
|
| 152 |
+
change, each carrying the text to search for on the CR side and (where
|
| 153 |
+
the change type makes it reliable) on the applied-TS side.
|
| 154 |
+
"""
|
| 155 |
+
sys.path.insert(0, str(scripts_dir))
|
| 156 |
+
from cr_parser import parse_cr # noqa: PLC0415 β intentional lazy import
|
| 157 |
+
|
| 158 |
+
raw_changes = parse_cr(str(cr_docx_path))
|
| 159 |
+
out = []
|
| 160 |
+
for ch in raw_changes:
|
| 161 |
+
loc = ch.get('location', {})
|
| 162 |
+
ctype = ch['type']
|
| 163 |
+
section_number = loc.get('section_number') or ''
|
| 164 |
+
|
| 165 |
+
if ctype == 'text_replace':
|
| 166 |
+
cr_search_text = ch.get('old', '')
|
| 167 |
+
ts_search_text = ch.get('new', '')
|
| 168 |
+
elif ctype == 'section_replace':
|
| 169 |
+
cr_search_text = loc.get('del_heading') or loc.get('anchor_text') or ''
|
| 170 |
+
# Inserted text isn't separately extractable from elements_xml
|
| 171 |
+
# without re-parsing raw XML β rely on the section heading match
|
| 172 |
+
# instead (validated visually in the skill's step 2f).
|
| 173 |
+
ts_search_text = ''
|
| 174 |
+
elif ctype == 'para_insert':
|
| 175 |
+
first_text = next((p.get('text', '') for p in ch.get('paragraphs', []) if p.get('text')), '')
|
| 176 |
+
cr_search_text = first_text
|
| 177 |
+
ts_search_text = first_text
|
| 178 |
+
elif ctype == 'row_insert':
|
| 179 |
+
first_cell = next((c.get('text', '') for c in ch.get('cells', []) if c.get('text')), '')
|
| 180 |
+
cr_search_text = first_cell
|
| 181 |
+
ts_search_text = first_cell
|
| 182 |
+
else:
|
| 183 |
+
cr_search_text = ts_search_text = ''
|
| 184 |
+
|
| 185 |
+
out.append({
|
| 186 |
+
'change_type': ctype,
|
| 187 |
+
'section_number': section_number,
|
| 188 |
+
'cr_search_text': cr_search_text,
|
| 189 |
+
'ts_search_text': ts_search_text,
|
| 190 |
+
'raw': ch,
|
| 191 |
+
})
|
| 192 |
+
return out
|
verify_review.py
ADDED
|
@@ -0,0 +1,291 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Verify insertions tab β step through CR/TS screenshot pairs from a review
|
| 3 |
+
package (produced locally by the `cr-visual-pair` skill) and record OK /
|
| 4 |
+
Not OK / Wrong match verdicts. Verdicts live in st.session_state only; the
|
| 5 |
+
user downloads a text log at the end.
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
import base64
|
| 9 |
+
import io
|
| 10 |
+
import json
|
| 11 |
+
import mimetypes
|
| 12 |
+
import zipfile
|
| 13 |
+
from pathlib import Path
|
| 14 |
+
|
| 15 |
+
import streamlit as st
|
| 16 |
+
|
| 17 |
+
VERDICT_LABELS = {
|
| 18 |
+
"ok": "OK",
|
| 19 |
+
"not_ok": "NOT OK",
|
| 20 |
+
"wrong_match": "WRONG MATCH (page shown did not correspond)",
|
| 21 |
+
}
|
| 22 |
+
|
| 23 |
+
SKILL_PATH = Path(__file__).parent / "skills" / "cr-visual-pair" / "SKILL.md"
|
| 24 |
+
|
| 25 |
+
_ZOOM_CSS = """
|
| 26 |
+
<style>
|
| 27 |
+
.cr-zoom-thumb { cursor: zoom-in; display: block; }
|
| 28 |
+
.cr-zoom-thumb img { max-width: 100%; border-radius: 4px; display: block; }
|
| 29 |
+
.cr-zoom-overlay {
|
| 30 |
+
display: none;
|
| 31 |
+
position: fixed;
|
| 32 |
+
inset: 0;
|
| 33 |
+
background: rgba(0, 0, 0, 0.9);
|
| 34 |
+
z-index: 999999;
|
| 35 |
+
align-items: center;
|
| 36 |
+
justify-content: center;
|
| 37 |
+
}
|
| 38 |
+
.cr-zoom-overlay:target { display: flex; }
|
| 39 |
+
.cr-zoom-overlay img {
|
| 40 |
+
max-width: 95vw;
|
| 41 |
+
max-height: 95vh;
|
| 42 |
+
object-fit: contain;
|
| 43 |
+
cursor: zoom-out;
|
| 44 |
+
}
|
| 45 |
+
.cr-zoom-close {
|
| 46 |
+
position: fixed;
|
| 47 |
+
top: 16px;
|
| 48 |
+
right: 24px;
|
| 49 |
+
color: #fff;
|
| 50 |
+
font-size: 2.5rem;
|
| 51 |
+
text-decoration: none;
|
| 52 |
+
line-height: 1;
|
| 53 |
+
}
|
| 54 |
+
</style>
|
| 55 |
+
"""
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
def _img_data_uri(path: Path) -> str:
|
| 59 |
+
mime = mimetypes.guess_type(path.name)[0] or "image/png"
|
| 60 |
+
data = base64.b64encode(path.read_bytes()).decode()
|
| 61 |
+
return f"data:{mime};base64,{data}"
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
def _render_zoomable_image(img_path: Path, anchor_id: str) -> None:
|
| 65 |
+
anchor_id = "".join(c if c.isalnum() else "-" for c in anchor_id)
|
| 66 |
+
data_uri = _img_data_uri(img_path)
|
| 67 |
+
st.markdown(
|
| 68 |
+
f"""
|
| 69 |
+
<a class="cr-zoom-thumb" href="#{anchor_id}">
|
| 70 |
+
<img src="{data_uri}">
|
| 71 |
+
</a>
|
| 72 |
+
<div class="cr-zoom-overlay" id="{anchor_id}">
|
| 73 |
+
<a class="cr-zoom-close" href="#">×</a>
|
| 74 |
+
<a href="#"><img src="{data_uri}"></a>
|
| 75 |
+
</div>
|
| 76 |
+
""",
|
| 77 |
+
unsafe_allow_html=True,
|
| 78 |
+
)
|
| 79 |
+
|
| 80 |
+
|
| 81 |
+
def _render_skill_card() -> None:
|
| 82 |
+
with st.expander("π’ cr-visual-pair"):
|
| 83 |
+
if not SKILL_PATH.exists():
|
| 84 |
+
st.error(f"Skill file not found at {SKILL_PATH}")
|
| 85 |
+
return
|
| 86 |
+
content = SKILL_PATH.read_text()
|
| 87 |
+
st.caption(
|
| 88 |
+
"Run this skill locally (Windows + Word) to produce a `review_package.zip` "
|
| 89 |
+
"from an ApplyCRs run. Save it as `~/.claude/skills/cr-visual-pair/SKILL.md`, "
|
| 90 |
+
"or download it below."
|
| 91 |
+
)
|
| 92 |
+
st.download_button(
|
| 93 |
+
"β¬ Download SKILL.md",
|
| 94 |
+
data=content,
|
| 95 |
+
file_name="SKILL.md",
|
| 96 |
+
mime="text/markdown",
|
| 97 |
+
key="verify_skill_download",
|
| 98 |
+
)
|
| 99 |
+
st.code(content, language="markdown")
|
| 100 |
+
|
| 101 |
+
|
| 102 |
+
def _extract_package(uploaded, dest_dir: Path) -> dict | None:
|
| 103 |
+
dest_dir.mkdir(parents=True, exist_ok=True)
|
| 104 |
+
zip_bytes = uploaded.getbuffer()
|
| 105 |
+
with zipfile.ZipFile(io.BytesIO(zip_bytes)) as zf:
|
| 106 |
+
zf.extractall(dest_dir)
|
| 107 |
+
manifest_path = dest_dir / "manifest.json"
|
| 108 |
+
if not manifest_path.exists():
|
| 109 |
+
return None
|
| 110 |
+
return json.loads(manifest_path.read_text())
|
| 111 |
+
|
| 112 |
+
|
| 113 |
+
def _build_log(manifest: dict, verdicts: dict, review_dir: Path) -> str:
|
| 114 |
+
lines = []
|
| 115 |
+
for pair in manifest.get("pairs", []):
|
| 116 |
+
pid = pair["id"]
|
| 117 |
+
verdict = verdicts.get(pid)
|
| 118 |
+
verdict_label = VERDICT_LABELS.get(verdict, "UNREVIEWED")
|
| 119 |
+
ts_spec = pair.get("ts_spec", "?")
|
| 120 |
+
ts_version = pair.get("ts_version", "?")
|
| 121 |
+
section = pair.get("section_number", "?")
|
| 122 |
+
match_note = "" if pair.get("matched", True) else " [auto-flagged: low-confidence match]"
|
| 123 |
+
lines.append(
|
| 124 |
+
f"TS {ts_spec} v{ts_version} [section {section}] : {verdict_label}{match_note}"
|
| 125 |
+
)
|
| 126 |
+
for change in pair.get("changes", []):
|
| 127 |
+
change_note = "" if change.get("matched", True) else " [low-confidence match]"
|
| 128 |
+
lines.append(
|
| 129 |
+
f" - CR {change.get('cr_uid', '?')} ({change.get('change_type', '?')})"
|
| 130 |
+
f"{change_note}"
|
| 131 |
+
)
|
| 132 |
+
return "\n".join(lines) + "\n"
|
| 133 |
+
|
| 134 |
+
|
| 135 |
+
def render_verify_tab(session_dir_fn, sid: str) -> None:
|
| 136 |
+
st.subheader("Verify insertions")
|
| 137 |
+
st.caption(
|
| 138 |
+
"Upload a review package (generated locally with the `cr-visual-pair` skill) "
|
| 139 |
+
"and step through each CR β TS screenshot pair to confirm the insertion is correct."
|
| 140 |
+
)
|
| 141 |
+
|
| 142 |
+
_render_skill_card()
|
| 143 |
+
st.markdown(_ZOOM_CSS, unsafe_allow_html=True)
|
| 144 |
+
|
| 145 |
+
review_dir = session_dir_fn(sid) / "review"
|
| 146 |
+
manifest_key = "verify_manifest"
|
| 147 |
+
review_dir_key = "verify_review_dir"
|
| 148 |
+
|
| 149 |
+
if manifest_key not in st.session_state:
|
| 150 |
+
uploaded = st.file_uploader(
|
| 151 |
+
"Upload review package (.zip)", type=["zip"], key="verify_upload"
|
| 152 |
+
)
|
| 153 |
+
if uploaded is not None:
|
| 154 |
+
manifest = _extract_package(uploaded, review_dir)
|
| 155 |
+
if manifest is None:
|
| 156 |
+
st.error("This zip does not contain a manifest.json β not a valid review package.")
|
| 157 |
+
return
|
| 158 |
+
st.session_state[manifest_key] = manifest
|
| 159 |
+
st.session_state[review_dir_key] = str(review_dir)
|
| 160 |
+
st.session_state["verify_idx"] = 0
|
| 161 |
+
st.session_state["verify_verdicts"] = {}
|
| 162 |
+
st.rerun()
|
| 163 |
+
return
|
| 164 |
+
|
| 165 |
+
manifest = st.session_state[manifest_key]
|
| 166 |
+
pairs = manifest.get("pairs", [])
|
| 167 |
+
review_dir = Path(st.session_state[review_dir_key])
|
| 168 |
+
|
| 169 |
+
if not pairs:
|
| 170 |
+
st.warning("This review package has no pairs to review.")
|
| 171 |
+
if st.button("Upload a different package"):
|
| 172 |
+
for k in (manifest_key, review_dir_key, "verify_idx", "verify_verdicts"):
|
| 173 |
+
st.session_state.pop(k, None)
|
| 174 |
+
st.rerun()
|
| 175 |
+
return
|
| 176 |
+
|
| 177 |
+
idx = st.session_state.setdefault("verify_idx", 0)
|
| 178 |
+
verdicts = st.session_state.setdefault("verify_verdicts", {})
|
| 179 |
+
idx = max(0, min(idx, len(pairs) - 1))
|
| 180 |
+
st.session_state["verify_idx"] = idx
|
| 181 |
+
|
| 182 |
+
pair = pairs[idx]
|
| 183 |
+
pid = pair["id"]
|
| 184 |
+
|
| 185 |
+
n_ok = sum(1 for v in verdicts.values() if v == "ok")
|
| 186 |
+
n_not_ok = sum(1 for v in verdicts.values() if v == "not_ok")
|
| 187 |
+
n_wrong = sum(1 for v in verdicts.values() if v == "wrong_match")
|
| 188 |
+
n_reviewed = len(verdicts)
|
| 189 |
+
st.caption(
|
| 190 |
+
f"Pair {idx + 1} / {len(pairs)} Β· "
|
| 191 |
+
f"Reviewed: {n_reviewed}/{len(pairs)} Β· β
{n_ok} β {n_not_ok} β {n_wrong}"
|
| 192 |
+
)
|
| 193 |
+
|
| 194 |
+
st.markdown(
|
| 195 |
+
f"**TS** `{pair.get('ts_spec', '?')}` v{pair.get('ts_version', '?')} Β· "
|
| 196 |
+
f"section `{pair.get('section_number', '?')}`"
|
| 197 |
+
)
|
| 198 |
+
if not pair.get("matched", True):
|
| 199 |
+
st.warning(
|
| 200 |
+
"β The pairing tool could not confidently locate this section's page in the "
|
| 201 |
+
"TS β double-check whether it's the right page before judging the insertion."
|
| 202 |
+
)
|
| 203 |
+
|
| 204 |
+
existing_verdict = verdicts.get(pid)
|
| 205 |
+
if existing_verdict:
|
| 206 |
+
st.info(f"Current verdict: **{VERDICT_LABELS[existing_verdict]}**")
|
| 207 |
+
|
| 208 |
+
col_cr, col_ts = st.columns(2)
|
| 209 |
+
with col_cr:
|
| 210 |
+
st.markdown("**CR (source change)**")
|
| 211 |
+
changes = pair.get("changes", [])
|
| 212 |
+
if changes:
|
| 213 |
+
for change in changes:
|
| 214 |
+
label = f"`{change.get('cr_uid', '?')}` Β· {change.get('change_type', '?')}"
|
| 215 |
+
if not change.get("matched", True):
|
| 216 |
+
label += " β low-confidence"
|
| 217 |
+
st.caption(label)
|
| 218 |
+
if change.get("summary"):
|
| 219 |
+
st.caption(change["summary"])
|
| 220 |
+
cr_images = change.get("cr_images", [])
|
| 221 |
+
if cr_images:
|
| 222 |
+
for img_idx, img in enumerate(cr_images):
|
| 223 |
+
img_path = review_dir / img
|
| 224 |
+
if img_path.exists():
|
| 225 |
+
_render_zoomable_image(
|
| 226 |
+
img_path, f"cr-zoom-{pid}-{change.get('cr_uid', '?')}-{img_idx}"
|
| 227 |
+
)
|
| 228 |
+
else:
|
| 229 |
+
st.error(f"Missing image: {img}")
|
| 230 |
+
else:
|
| 231 |
+
st.warning("No CR image for this change.")
|
| 232 |
+
else:
|
| 233 |
+
st.warning("No CR changes recorded for this pair.")
|
| 234 |
+
with col_ts:
|
| 235 |
+
st.markdown("**TS (applied output)**")
|
| 236 |
+
ts_images = pair.get("ts_images", [])
|
| 237 |
+
if ts_images:
|
| 238 |
+
for img_idx, img in enumerate(ts_images):
|
| 239 |
+
img_path = review_dir / img
|
| 240 |
+
if img_path.exists():
|
| 241 |
+
_render_zoomable_image(img_path, f"ts-zoom-{pid}-{img_idx}")
|
| 242 |
+
else:
|
| 243 |
+
st.error(f"Missing image: {img}")
|
| 244 |
+
else:
|
| 245 |
+
st.warning("No TS image for this pair β pairing tool did not find a candidate page.")
|
| 246 |
+
|
| 247 |
+
def _advance():
|
| 248 |
+
if st.session_state["verify_idx"] < len(pairs) - 1:
|
| 249 |
+
st.session_state["verify_idx"] += 1
|
| 250 |
+
|
| 251 |
+
col1, col2, col3, col4, col5 = st.columns(5)
|
| 252 |
+
with col1:
|
| 253 |
+
if st.button("β
OK", type="primary", key=f"ok_{pid}"):
|
| 254 |
+
verdicts[pid] = "ok"
|
| 255 |
+
_advance()
|
| 256 |
+
st.rerun()
|
| 257 |
+
with col2:
|
| 258 |
+
if st.button("β Not OK", key=f"not_ok_{pid}"):
|
| 259 |
+
verdicts[pid] = "not_ok"
|
| 260 |
+
_advance()
|
| 261 |
+
st.rerun()
|
| 262 |
+
with col3:
|
| 263 |
+
if st.button("β Wrong match", key=f"wrong_{pid}"):
|
| 264 |
+
verdicts[pid] = "wrong_match"
|
| 265 |
+
_advance()
|
| 266 |
+
st.rerun()
|
| 267 |
+
with col4:
|
| 268 |
+
if st.button("β Prev", disabled=idx == 0, key=f"prev_{pid}"):
|
| 269 |
+
st.session_state["verify_idx"] -= 1
|
| 270 |
+
st.rerun()
|
| 271 |
+
with col5:
|
| 272 |
+
if st.button("Next β", disabled=idx == len(pairs) - 1, key=f"next_{pid}"):
|
| 273 |
+
st.session_state["verify_idx"] += 1
|
| 274 |
+
st.rerun()
|
| 275 |
+
|
| 276 |
+
if idx == len(pairs) - 1 and pid in verdicts:
|
| 277 |
+
st.success("Review complete β all pairs have a verdict. Download the log below.")
|
| 278 |
+
|
| 279 |
+
st.divider()
|
| 280 |
+
log_text = _build_log(manifest, verdicts, review_dir)
|
| 281 |
+
st.download_button(
|
| 282 |
+
"β¬ Download review log",
|
| 283 |
+
data=log_text,
|
| 284 |
+
file_name=f"cr_review_{sid[:8]}.log",
|
| 285 |
+
mime="text/plain",
|
| 286 |
+
)
|
| 287 |
+
|
| 288 |
+
if st.button("Upload a different package", key="verify_reset"):
|
| 289 |
+
for k in (manifest_key, review_dir_key, "verify_idx", "verify_verdicts"):
|
| 290 |
+
st.session_state.pop(k, None)
|
| 291 |
+
st.rerun()
|