heymenn commited on
Commit
5ecf925
Β·
1 Parent(s): e9e189e

add visual verifier and fixes

Browse files
app.py CHANGED
@@ -441,748 +441,756 @@ with st.sidebar:
441
  else:
442
  st.error("Session not found.")
443
 
444
- # ── State machine ─────────────────────────────────────────────────────────────
445
- status: str = state["status"]
446
-
447
- # ════════════════════════════════════════════════════════════════════════════
448
- # LOGIN
449
- # ════════════════════════════════════════════════════════════════════════════
450
- if status == "login":
451
- st.subheader("Connect with your ETSI EOL account")
452
- st.info(
453
- "Your credentials are used only for this session and are never stored on disk.",
454
- icon="πŸ”’",
455
- )
456
- username = st.text_input("EOL Username")
457
- password = st.text_input("EOL Password", type="password")
 
 
 
 
458
 
459
- if st.button("Connect", type="primary"):
460
- if not username or not password:
461
- st.error("Please enter both username and password.")
462
- else:
463
- with st.spinner("Verifying credentials…"):
464
- ok = verify_eol_credentials(username, password)
465
- if ok:
466
- st.session_state.eol_user = username
467
- st.session_state.eol_password = password
468
- state["status"] = "upload"
469
- save_state(sid, state)
470
- st.rerun()
471
  else:
472
- st.error("Login failed β€” check your EOL username and password.")
473
-
474
- # ════════════════════════════════════════════════════════════════════════════
475
- # UPLOAD
476
- # ════════════════════════════════════════════════════════════════════════════
477
- elif status == "upload":
478
- st.subheader("Step 1 β€” Upload contribution list")
479
-
480
- mode_label = st.radio(
481
- "Pipeline mode",
482
- ["By contributor name", "By TS (all CRs for a spec)"],
483
- key="mode_radio",
484
- )
485
- pipeline_mode = "contributor" if mode_label.startswith("By contributor") else "ts"
486
- state["mode"] = pipeline_mode
487
-
488
- # ── Resolve active Excel (saved from a previous run, or freshly uploaded) ──
489
- saved_name = state.get("excel_filename")
490
- saved_path = session_dir(sid) / saved_name if saved_name else None
491
- has_saved = saved_path is not None and saved_path.exists()
492
-
493
- # ── Previous meetings selector ─────────────────────────────────────────────
494
- past_meetings = load_meetings()
495
- if past_meetings:
496
- with st.expander("πŸ“‚ Previous meetings", expanded=not has_saved):
497
- meeting_options = [
498
- f"{m['meeting_name']} β€” {m['uploaded_at']} ({m['original_filename']})"
499
- for m in reversed(past_meetings)
500
- ]
501
- chosen_label = st.selectbox("Select meeting", meeting_options, key="meeting_select")
502
- if st.button("Load this meeting", key="load_meeting"):
503
- chosen_idx = len(past_meetings) - 1 - meeting_options.index(chosen_label)
504
- m = past_meetings[chosen_idx]
505
- cached = Path(m["excel_path"])
506
- if cached.exists():
507
- import shutil as _sh
508
- dest = session_dir(sid) / m["original_filename"]
509
- _sh.copy2(str(cached), str(dest))
510
- state["excel_filename"] = m["original_filename"]
511
- state["excel_hash"] = m["excel_hash"]
512
  save_state(sid, state)
513
  st.rerun()
514
  else:
515
- st.error("Cached Excel file not found β€” please upload it again.")
516
-
517
- if has_saved:
518
- st.success(f"Using: **{saved_name}**")
519
- with st.expander("Replace Excel file"):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
520
  uploaded = st.file_uploader(
521
- "Upload a different Excel file (.xlsx or .xls)",
522
  type=["xlsx", "xls"],
523
- key="excel_replace",
524
  )
525
- else:
526
- uploaded = st.file_uploader(
527
- "Excel contribution list (.xlsx or .xls)",
528
- type=["xlsx", "xls"],
529
- )
530
 
531
- # When a new file is dropped, save it immediately and update state
532
- if uploaded:
533
- active_path = session_dir(sid) / uploaded.name
534
- active_bytes = bytes(uploaded.getbuffer())
535
- active_path.write_bytes(active_bytes)
536
- state["excel_filename"] = uploaded.name
537
- excel_hash = hashlib.sha256(active_bytes).hexdigest()[:16]
538
- state["excel_hash"] = excel_hash
539
- # Save to global meeting store
540
- meeting_name = peek_meeting_name(active_path)
541
- if meeting_name:
542
- save_meeting(meeting_name, active_path, excel_hash)
543
- save_state(sid, state)
544
- elif has_saved:
545
- active_path = saved_path
546
- else:
547
- active_path = None
548
 
549
- if pipeline_mode == "contributor":
550
- person_name = st.text_input(
551
- "Contributor name (must match SubmittedBy column)",
552
- value=state.get("person_name", "Ly Thanh PHAN"),
553
- )
554
 
555
- if active_path and st.button("Parse CR list β†’", type="primary"):
556
- with st.spinner("Parsing Excel…"):
557
- try:
558
- from fetch_crs import parse_excel
559
- cr_list = parse_excel(str(active_path), person_name)
560
- state["status"] = "preview"
561
- state["person_name"] = person_name
562
- state["cr_list"] = [list(row) for row in cr_list]
563
- save_state(sid, state)
564
- st.rerun()
565
- except Exception as exc:
566
- st.error(f"Failed to parse Excel: {exc}")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
567
 
568
- else: # TS mode
569
- if active_path:
570
- excel_hash = state.get("excel_hash") or hashlib.sha256(active_path.read_bytes()).hexdigest()[:16]
571
- state["excel_hash"] = excel_hash
572
 
573
- hf_token = os.environ.get("HF_TOKEN", "")
574
- hf_repo = state.get("hf_repo", "OrganizedProgrammers/CR_Index")
575
 
576
- # Check whether this Excel is already indexed
577
- existing = load_hf_index_cached(hf_token, hf_repo)
578
- already_indexed = any(r.get("excel_hash") == excel_hash for r in existing)
579
 
580
- if already_indexed:
581
- st.success(f"This Excel (`{excel_hash}`) is already indexed in HF.")
582
- if st.button("Select TS β†’", type="primary"):
583
- state["status"] = "ts_select"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
584
  save_state(sid, state)
585
  st.rerun()
586
- else:
587
- st.info(f"Excel hash: `{excel_hash}` β€” not yet indexed.")
588
- if st.button("Build CR Index", type="primary"):
589
- # CRs downloaded during indexing go to the session-level cache
590
- cr_cache_dir = session_dir(sid) / "CRs"
591
- cr_cache_dir.mkdir(parents=True, exist_ok=True)
592
- index_log = str(session_dir(sid) / "index.log")
593
- rc_path = _rc_path(sid)
594
- rc_path.unlink(missing_ok=True)
595
-
596
- cmd = [
597
- sys.executable,
598
- str(SCRIPTS_DIR / "build_cr_index.py"),
599
- str(active_path),
600
- "--output-dir", str(session_dir(sid)),
601
- "--hf-repo", hf_repo,
602
- ]
603
- env = os.environ.copy()
604
- env["EOL_USER"] = st.session_state.eol_user
605
- env["EOL_PASSWORD"] = st.session_state.eol_password
606
- # HF_TOKEN is already in env via os.environ
607
-
608
- _launch_proc(cmd, env, index_log, sid, state, {
609
- "status": "indexing",
610
- "index_log": index_log,
611
- "output_dir": "", # no pipeline output yet
612
- })
613
-
614
- # ════════════════════════════════════════════════════════════════════════════
615
- # INDEXING (build_cr_index.py running)
616
- # ════════════════════════════════════════════════════════════════════════════
617
- elif status == "indexing":
618
- pid = state["pid"]
619
- index_log = state.get("index_log", "")
620
-
621
- proc = st.session_state.get("proc")
622
- alive = False
623
- if proc is not None:
624
- alive = proc.poll() is None
625
- else:
626
- rc = read_return_code(sid)
627
- if rc is None:
628
- alive = is_process_alive(pid)
629
-
630
- if alive:
631
- st.subheader("⏳ Building CR Index…")
632
- st.info(f"PID {pid} β€” started {state.get('started_at', '')[:19]}")
633
- log_text = tail_log(index_log, 50)
634
- st.text_area("Live log (last 50 lines)", value=log_text, height=400)
635
- time.sleep(2)
636
- st.rerun()
637
- else:
638
- rc = read_return_code(sid)
639
- if rc is None and proc is not None:
640
- rc = proc.returncode
641
- state["return_code"] = rc
642
- state["completed_at"] = datetime.now().isoformat()
643
- if rc == 0:
644
- # Invalidate cached HF index so ts_select gets fresh data
645
- st.session_state.pop(f"hf_index_{state.get('hf_repo', '')}", None)
646
- state["status"] = "ts_select"
647
  else:
648
- # Expose the index log as log_path so the error state can display it
649
- state["log_path"] = state.get("index_log", "")
650
- state["status"] = "error"
651
- save_state(sid, state)
652
- st.rerun()
653
-
654
- # ════════════════════════════════════════════════════════════════════════════
655
- # TS_SELECT (index ready β€” pick a spec to process)
656
- # ════════════════════════════════════════════════════════════════════════════
657
- elif status == "ts_select":
658
- st.subheader("Step 2 β€” Select target TS spec")
659
-
660
- hf_token = os.environ.get("HF_TOKEN", "")
661
- hf_repo = state.get("hf_repo", "OrganizedProgrammers/CR_Index")
662
- excel_hash = state.get("excel_hash", "")
663
-
664
- with st.spinner("Loading CR index from HuggingFace…"):
665
- records = load_hf_index_cached(hf_token, hf_repo)
666
-
667
- records_for_excel = [r for r in records if r.get("excel_hash") == excel_hash]
668
 
669
- if not records_for_excel:
670
- st.error(f"No records found for excel_hash `{excel_hash}`. Try rebuilding the index.")
671
- if st.button("← Back to upload"):
672
- state["status"] = "upload"
673
- save_state(sid, state)
674
- st.rerun()
675
- else:
676
- by_spec = defaultdict(list)
677
- for r in records_for_excel:
678
- by_spec[r["spec_number"]].append(r)
679
-
680
- _ALL = "All"
681
- spec_options = sorted(by_spec.keys()) + [_ALL]
682
- selected_spec = st.selectbox("Select target TS spec", spec_options)
683
-
684
- if selected_spec and selected_spec != _ALL:
685
- versions = defaultdict(list)
686
- for r in by_spec[selected_spec]:
687
- versions[r["version"]].append(r["uid"])
688
- st.write("**Versions found:**")
689
- for ver, uids in sorted(versions.items()):
690
- st.write(f" v{ver}: {len(uids)} CR(s) β€” {', '.join(uids)}")
691
- elif selected_spec == _ALL:
692
- total_crs = sum(len(v) for v in by_spec.values())
693
- st.write(f"**All specs** β€” {len(by_spec)} TS(s), {total_crs} CR(s) total")
694
-
695
- def _start_ts_run(ts_id_arg: str | None):
696
- run_id = int(time.time())
697
- output_dir = session_dir(sid) / f"run_{run_id}"
698
- output_dir.mkdir(parents=True, exist_ok=True)
699
- cr_cache_dir = session_dir(sid) / "CRs"
700
- cr_cache_dir.mkdir(parents=True, exist_ok=True)
701
- log_path = session_dir(sid) / f"pipeline_{run_id}.log"
702
- rc_path = _rc_path(sid)
703
- rc_path.unlink(missing_ok=True)
704
-
705
- cmd = [
706
- sys.executable,
707
- str(SCRIPTS_DIR / "orchestrate_cr.py"),
708
- "--output-dir", str(output_dir),
709
- "--cr-cache-dir", str(cr_cache_dir),
710
- "--ts-mode",
711
- "--excel-hash", excel_hash,
712
- "--hf-repo", hf_repo,
713
- ]
714
- if ts_id_arg:
715
- cmd += ["--ts-id", ts_id_arg]
716
-
717
- env = os.environ.copy()
718
- env["EOL_USER"] = st.session_state.eol_user
719
- env["EOL_PASSWORD"] = st.session_state.eol_password
720
-
721
- _launch_proc(cmd, env, log_path, sid, state, {
722
- "ts_id": ts_id_arg or "all",
723
- "status": "running",
724
- "output_dir": str(output_dir),
725
- "log_path": str(log_path),
726
- "run_log_paths": [str(log_path)],
727
- })
728
-
729
- col_back, col_apply = st.columns([1, 3])
730
- with col_back:
731
  if st.button("← Back"):
732
  state["status"] = "upload"
 
733
  save_state(sid, state)
734
  st.rerun()
735
- with col_apply:
736
- ts_arg = None if selected_spec == _ALL else selected_spec
737
- if st.button("β–Ά Apply CRs", type="primary"):
738
- _start_ts_run(ts_arg)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
739
 
740
- # ── Accumulated session results ───────────────────────────────────────
 
 
741
  applied_runs = state.get("applied_runs", [])
742
- if applied_runs:
743
- st.divider()
744
- st.subheader(f"Session results ({len(applied_runs)} run(s))")
745
- for run in applied_runs:
 
 
746
  icon = "βœ…" if run["status"] == "done" else "❌"
747
  label = run["ts_id"] if run["ts_id"] != "all" else "All TSs"
748
  ts_dir = Path(run["output_dir"]) / "TS"
749
  specs = sorted(d.name for d in ts_dir.iterdir() if d.is_dir()) if ts_dir.exists() else []
750
  specs_str = f" β€” {', '.join(specs)}" if specs else ""
751
- st.write(f"{icon} **{label}**{specs_str}")
752
-
753
- # ════════════════════════════════════════════════════════════════════════════
754
- # PREVIEW
755
- # ════════════════════════════════════════════════════════════════════════════
756
- elif status == "preview":
757
- cr_list = state["cr_list"]
758
- st.subheader(f"Step 2 β€” {len(cr_list)} Accepted CR(s) found")
759
-
760
- if cr_list:
761
- import pandas as pd
762
- df = pd.DataFrame(cr_list, columns=["UID", "Title"])
763
- st.dataframe(df, width='stretch')
764
- else:
765
- st.warning(
766
- f"No Accepted CRs found for **{state['person_name']}** in this file."
767
- )
768
- # Diagnostic: show what names are in the SubmittedBy column
769
- excel_path = session_dir(sid) / state["excel_filename"]
770
- found_names = peek_submitted_by(excel_path)
771
- if found_names:
772
- st.info(
773
- "**Names found in SubmittedBy column** β€” copy the exact one into the field above and re-upload:\n\n"
774
- + "\n".join(f"- `{n}`" for n in found_names)
775
- )
776
 
777
- col1, col2 = st.columns(2)
778
- with col1:
779
- if st.button("← Back"):
780
- state["status"] = "upload"
781
- state["cr_list"] = []
782
- save_state(sid, state)
783
- st.rerun()
784
- with col2:
785
- if cr_list and st.button("β–Ά Start Pipeline", type="primary"):
786
- excel_path = session_dir(sid) / state["excel_filename"]
787
- # Per-run directory so each pipeline's outputs are isolated
788
- run_id = int(time.time())
789
- output_dir = session_dir(sid) / f"run_{run_id}"
790
- output_dir.mkdir(parents=True, exist_ok=True)
791
- cr_cache_dir = session_dir(sid) / "CRs"
792
- cr_cache_dir.mkdir(parents=True, exist_ok=True)
793
- log_path = session_dir(sid) / f"pipeline_{run_id}.log"
794
- rc_path = _rc_path(sid)
795
- rc_path.unlink(missing_ok=True)
796
-
797
- cmd = [
798
- sys.executable,
799
- str(SCRIPTS_DIR / "orchestrate_cr.py"),
800
- str(excel_path),
801
- state["person_name"],
802
- "--output-dir", str(output_dir),
803
- "--cr-cache-dir", str(cr_cache_dir),
804
- ]
805
-
806
- env = os.environ.copy()
807
- env["EOL_USER"] = st.session_state.eol_user
808
- env["EOL_PASSWORD"] = st.session_state.eol_password
809
-
810
- _launch_proc(cmd, env, log_path, sid, state, {
811
- "status": "running",
812
- "output_dir": str(output_dir),
813
- "log_path": str(log_path),
814
- "run_log_paths": [str(log_path)],
815
- })
816
-
817
- # ════════════════════════════════════════════════════════════════════════════
818
- # RUNNING
819
- # ════════════════════════════════════════════════════════════════════════════
820
- elif status == "running":
821
- pid = state["pid"]
822
- log_path = state["log_path"]
823
-
824
- # Determine whether process is still alive
825
- proc = st.session_state.get("proc")
826
- alive = False
827
- if proc is not None:
828
- alive = proc.poll() is None
829
- else:
830
- # Session reloaded β€” check returncode file, then PID
831
- rc = read_return_code(sid)
832
- if rc is None:
833
- alive = is_process_alive(pid)
834
-
835
- if alive:
836
- st.subheader("⏳ Pipeline running…")
837
- st.info(f"PID {pid} β€” started {state.get('started_at', '')[:19]}")
838
- log_text = tail_log(log_path, 100)
839
- st.text_area("Live log (last 100 lines)", value=log_text, height=400)
840
- time.sleep(2)
841
- st.rerun()
842
- else:
843
- # Process finished β€” determine return code
844
- rc = read_return_code(sid)
845
- if rc is None and proc is not None:
846
- rc = proc.returncode
847
- state["return_code"] = rc
848
- state["completed_at"] = datetime.now().isoformat()
849
- fin_status = "done" if rc == 0 else "error"
850
- state["status"] = fin_status
851
- # Accumulate completed run for TS mode
852
- if state.get("mode") == "ts":
853
- run_entry = {
854
- "ts_id": state.get("ts_id", ""),
855
- "output_dir": state.get("output_dir", ""),
856
- "log_path": state.get("log_path", ""),
857
- "status": fin_status,
858
- }
859
- runs = state.get("applied_runs", [])
860
- if not any(r["output_dir"] == run_entry["output_dir"] for r in runs):
861
- runs.append(run_entry)
862
- state["applied_runs"] = runs
863
- save_state(sid, state)
864
- st.rerun()
865
-
866
- # ════════════════════════════════════════════════════════════════════════════
867
- # DONE / ERROR
868
- # ════════════════════════════════════════════════════════════════════════════
869
- elif status in ("done", "error"):
870
- log_path = state.get("log_path", "")
871
- output_dir = Path(state.get("output_dir", ""))
872
- rc = state.get("return_code")
873
-
874
- st.subheader("Step 3 β€” Results")
875
- if status == "done":
876
- st.success("βœ… Pipeline completed successfully!")
877
- else:
878
- st.error(f"❌ Pipeline finished with errors (return code: {rc})")
879
-
880
- # Per-TS results table β€” merge this run's logs so retry results supersede
881
- # the original failed entries for the same TS key.
882
- _merged: dict[str, dict] = {}
883
- for _lf in state.get("run_log_paths", []):
884
- for _r in parse_log_results(_lf):
885
- _merged[_r["TS"]] = _r
886
- results = list(_merged.values())
887
- if results:
888
- st.subheader("Results per TS")
889
- import pandas as pd
890
-
891
- n_warn = sum(1 for r in results if r["warnings"])
892
- warn_label = f"Warnings ({n_warn})" if n_warn else "Warnings"
893
- tab_summary, tab_warnings = st.tabs(["Summary", warn_label])
894
-
895
- def _color_status(val):
896
- return {
897
- "OK": "background-color: #d4edda; color: #155724",
898
- "WARN": "background-color: #fff3cd; color: #856404",
899
- "FAIL": "background-color: #f8d7da; color: #721c24",
900
- "SKIP": "background-color: #e2e3e5; color: #383d41",
901
- }.get(val, "")
902
-
903
- with tab_summary:
904
- df = pd.DataFrame([{"Status": r["Status"], "TS": r["TS"]} for r in results])
905
- st.dataframe(
906
- df.style.map(_color_status, subset=["Status"]),
907
- width='stretch',
908
- )
909
-
910
- with tab_warnings:
911
- warned = [r for r in results if r["warnings"]]
912
- if warned:
913
- for r in warned:
914
- with st.expander(f"⚠️ {r['TS']} β€” {len(r['warnings'])} warning(s)"):
915
- for w in r["warnings"]:
916
- st.text(w)
917
- else:
918
- st.success("No warnings.")
919
-
920
- # Download ZIP
921
- # ── Download section ──────────────────────────────────────────────────────
922
- st.subheader("Download results")
923
- applied_runs = state.get("applied_runs", [])
924
- valid_runs = [r for r in applied_runs if Path(r["output_dir"]).exists()
925
- and any(Path(r["output_dir"]).rglob("*"))]
926
-
927
- if valid_runs:
928
- # Show a compact list of what's included
929
- for run in valid_runs:
930
- icon = "βœ…" if run["status"] == "done" else "❌"
931
- label = run["ts_id"] if run["ts_id"] != "all" else "All TSs"
932
- ts_dir = Path(run["output_dir"]) / "TS"
933
- specs = sorted(d.name for d in ts_dir.iterdir() if d.is_dir()) if ts_dir.exists() else []
934
- specs_str = f" β€” {', '.join(specs)}" if specs else ""
935
- st.write(f"{icon} {label}{specs_str}")
936
-
937
- # Build ZIP from all accumulated runs
938
- buf = io.BytesIO()
939
- with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as zf:
940
- for run in valid_runs:
941
- out = Path(run["output_dir"])
942
- for f in out.rglob("*"):
943
- if f.is_file():
944
- zf.write(f, f.relative_to(out.parent))
945
- buf.seek(0)
946
- col_dl, col_flush = st.columns([3, 1])
947
- with col_dl:
948
  st.download_button(
949
  label="⬇ Download results ZIP",
950
- data=buf.read(),
951
  file_name=f"cr_results_{sid[:8]}.zip",
952
  mime="application/zip",
953
  type="primary",
954
  )
955
- with col_flush:
956
- if st.button("πŸ—‘ Flush"):
957
- import shutil
958
- for run in applied_runs:
959
- out = Path(run["output_dir"])
960
- if out.exists():
961
- shutil.rmtree(out, ignore_errors=True)
962
- state["applied_runs"] = []
963
- save_state(sid, state)
964
- st.rerun()
965
- elif output_dir.exists() and any(output_dir.rglob("*")):
966
- # Fallback for contributor mode (no applied_runs)
967
- zip_bytes = make_zip(output_dir)
968
- st.download_button(
969
- label="⬇ Download results ZIP",
970
- data=zip_bytes,
971
- file_name=f"cr_results_{sid[:8]}.zip",
972
- mime="application/zip",
973
- type="primary",
974
- )
975
- else:
976
- st.warning("Output directory is empty β€” nothing to download.")
977
-
978
- # Full log
979
- with st.expander("Full pipeline log"):
980
- if log_path and Path(log_path).exists():
981
- st.text(Path(log_path).read_text(errors="replace"))
982
  else:
983
- st.text("Log not found.")
984
-
985
- # ── TS Recovery ───────────────────────────────────────────────────────────
986
- failed_ts_path = output_dir / "failed_ts.json"
987
- if failed_ts_path.exists():
988
- failed_ts_entries = json.loads(failed_ts_path.read_text())
989
- if failed_ts_entries:
990
- st.divider()
991
- st.subheader("⚠️ Recover failed TS downloads")
992
- st.info(
993
- f"{len(failed_ts_entries)} TS(s) could not be downloaded. "
994
- "Retry or upload each one manually, then apply the CRs."
995
- )
996
 
997
- for entry in failed_ts_entries:
998
- spec_key = f"{entry['spec_number']} v{entry['version']}"
999
- dest_path = Path(entry["spec_dir"]) / entry["expected_filename"]
1000
- ready = dest_path.exists()
1001
-
1002
- label = f"{'βœ…' if ready else '❌'} TS {spec_key} β€” CRs: {', '.join(entry['cr_uids'])}"
1003
- with st.expander(label, expanded=not ready):
1004
- col1, col2 = st.columns(2)
1005
-
1006
- with col1:
1007
- if st.button("πŸ”„ Retry download",
1008
- key=f"retry_{entry['spec_compact']}_{entry['version']}"):
1009
- from fetch_crs import download_ts as _dl_ts
1010
- with st.spinner(f"Downloading TS {spec_key}…"):
1011
- fn, note = _dl_ts(
1012
- entry["spec_number"], entry["version"],
1013
- Path(entry["spec_dir"]),
1014
- st.session_state.eol_user,
1015
- st.session_state.eol_password,
1016
- )
1017
- if fn:
1018
- st.success(f"Downloaded: {fn}")
1019
- st.rerun()
1020
- else:
1021
- st.error(f"Failed: {note}")
1022
-
1023
- with col2:
1024
- uploaded_ts = st.file_uploader(
1025
- f"Or upload `{entry['expected_filename']}`",
1026
- type=["docx"],
1027
- key=f"upload_{entry['spec_compact']}_{entry['version']}",
1028
- )
1029
- if uploaded_ts is not None:
1030
- Path(entry["spec_dir"]).mkdir(parents=True, exist_ok=True)
1031
- dest_path.write_bytes(uploaded_ts.read())
1032
- st.success("Saved βœ“")
1033
- st.rerun()
1034
-
1035
- # Global apply button β€” enabled when β‰₯1 TS is now on disk
1036
- ready_entries = [
1037
- e for e in failed_ts_entries
1038
- if (Path(e["spec_dir"]) / e["expected_filename"]).exists()
1039
- ]
1040
- remaining = len(failed_ts_entries) - len(ready_entries)
1041
-
1042
- if ready_entries:
1043
- if remaining:
1044
- st.warning(f"{len(ready_entries)} ready, {remaining} will be skipped.")
1045
- else:
1046
- st.success(f"All {len(ready_entries)} TS(s) ready.")
1047
-
1048
- if st.button("β–Ά Apply CRs to recovered TSs", type="primary"):
1049
- retry_log = str(session_dir(sid) / f"pipeline_{int(time.time())}_retry.log")
1050
- cmd = [
1051
- sys.executable,
1052
- str(SCRIPTS_DIR / "orchestrate_cr.py"),
1053
- "--output-dir", state["output_dir"],
1054
- "--retry-mode",
1055
- ]
1056
- env = os.environ.copy()
1057
- env["EOL_USER"] = st.session_state.eol_user
1058
- env["EOL_PASSWORD"] = st.session_state.eol_password
1059
-
1060
- _launch_proc(cmd, env, retry_log, sid, state, {
1061
- "status": "running",
1062
- "log_path": retry_log,
1063
- "run_log_paths": state.get("run_log_paths", []) + [retry_log],
1064
- })
1065
  else:
1066
- st.warning("No TSs available yet β€” retry download or upload DOCX files above.")
1067
-
1068
- # ── CR Recovery ───────────────────────────────────────────────────────────
1069
- failed_cr_path = output_dir / "failed_cr.json"
1070
- if failed_cr_path.exists():
1071
- failed_cr_entries = json.loads(failed_cr_path.read_text())
1072
- if failed_cr_entries:
1073
- st.divider()
1074
- st.subheader("⚠️ Recover failed CR downloads")
1075
- st.info(
1076
- f"{len(failed_cr_entries)} CR(s) could not be downloaded. "
1077
- "Retry or upload each one manually, then apply."
1078
- )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1079
 
1080
- for entry in failed_cr_entries:
1081
- uid = entry["uid"]
1082
- cr_dir_path = Path(entry["cr_dir"])
1083
- expected = cr_dir_path / entry["expected_filename"]
1084
- ready = expected.exists() or (cr_dir_path / f"{uid}_extracted.docx").exists()
1085
- ts_label = (
1086
- f"TS {entry['ts_spec_number']} v{entry['ts_version']}"
1087
- if entry.get("ts_spec_number") else "TS unknown"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1088
  )
1089
- label = f"{'βœ…' if ready else '❌'} CR {uid} β†’ {ts_label}"
1090
- with st.expander(label, expanded=not ready):
1091
- col1, col2 = st.columns(2)
1092
-
1093
- with col1:
1094
- if st.button("πŸ”„ Retry download", key=f"retry_cr_{uid}"):
1095
- from fetch_crs import download_cr as _dl_cr
1096
- with st.spinner(f"Downloading CR {uid}…"):
1097
- fn, note = _dl_cr(
1098
- uid, cr_dir_path,
1099
- st.session_state.eol_user,
1100
- st.session_state.eol_password,
1101
- )
1102
- if fn:
1103
- st.success(f"Downloaded: {fn.name}")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1104
  st.rerun()
1105
- else:
1106
- st.error(f"Failed: {note}")
1107
-
1108
- with col2:
1109
- uploaded_cr = st.file_uploader(
1110
- f"Or upload `{entry['expected_filename']}`",
1111
- type=["docx"],
1112
- key=f"upload_cr_{uid}",
1113
- )
1114
- if uploaded_cr is not None:
1115
- cr_dir_path.mkdir(parents=True, exist_ok=True)
1116
- expected.write_bytes(uploaded_cr.read())
1117
- st.success("Saved βœ“")
1118
- st.rerun()
1119
-
1120
- ready_cr_entries = [
1121
- e for e in failed_cr_entries
1122
- if (Path(e["cr_dir"]) / e["expected_filename"]).exists()
1123
- or (Path(e["cr_dir"]) / f"{e['uid']}_extracted.docx").exists()
1124
- ]
1125
- remaining_cr = len(failed_cr_entries) - len(ready_cr_entries)
1126
-
1127
- if ready_cr_entries:
1128
- if remaining_cr:
1129
- st.warning(f"{len(ready_cr_entries)} ready, {remaining_cr} still missing.")
 
 
 
 
 
1130
  else:
1131
- st.success(f"All {len(ready_cr_entries)} CR(s) ready.")
1132
-
1133
- if st.button("β–Ά Apply recovered CRs", type="primary", key="apply_recovered_crs"):
1134
- retry_log = str(session_dir(sid) / f"pipeline_{int(time.time())}_retry.log")
1135
- cmd = [
1136
- sys.executable,
1137
- str(SCRIPTS_DIR / "orchestrate_cr.py"),
1138
- "--output-dir", state["output_dir"],
1139
- "--retry-mode",
1140
- ]
1141
- env = os.environ.copy()
1142
- env["EOL_USER"] = st.session_state.eol_user
1143
- env["EOL_PASSWORD"] = st.session_state.eol_password
1144
- _launch_proc(cmd, env, retry_log, sid, state, {
1145
- "status": "running",
1146
- "log_path": retry_log,
1147
- "run_log_paths": state.get("run_log_paths", []) + [retry_log],
1148
- })
1149
  else:
1150
- st.warning("No CRs recovered yet β€” retry download or upload DOCX files above.")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1151
 
1152
- # Navigation
1153
- st.divider()
1154
- col_restart, col_new = st.columns(2)
1155
- with col_restart:
1156
- if state.get("mode") == "ts":
1157
- back_label = "← Back to TS Selection"
1158
- back_status = "ts_select"
1159
- clear_keys = ("pid", "output_dir", "log_path", "index_log",
1160
- "started_at", "completed_at", "return_code", "ts_id")
1161
- else:
1162
- back_label = "← Back to CR Preview"
1163
- back_status = "preview"
1164
- clear_keys = ("pid", "output_dir", "log_path", "index_log",
1165
- "started_at", "completed_at", "return_code")
1166
- if st.button(back_label, type="primary"):
1167
- for _k in clear_keys:
1168
- state[_k] = None
1169
- state["run_log_paths"] = []
1170
- state["status"] = back_status
1171
- if "proc" in st.session_state:
1172
- del st.session_state.proc
1173
- _rc_path(sid).unlink(missing_ok=True)
1174
- save_state(sid, state)
1175
- st.rerun()
1176
- with col_new:
1177
- if st.button("Start new session"):
1178
- new_sid = str(uuid.uuid4())
1179
- st.session_state.sid = new_sid
1180
- st.session_state.state = new_state(new_sid)
1181
- if "proc" in st.session_state:
1182
- del st.session_state.proc
1183
- st.query_params["sid"] = new_sid
1184
- save_state(new_sid, st.session_state.state)
1185
- st.rerun()
1186
 
1187
  # ════════════════════════════════════════════════════════════════════════════
1188
  # VERSION HISTORY (bottom of every stable page)
 
441
  else:
442
  st.error("Session not found.")
443
 
444
+ # ── Tabs ──────────────────────────────────────────────────────────────────────
445
+ tab_apply, tab_verify = st.tabs(["πŸ“„ Apply CRs", "βœ… Verify insertions"])
446
+
447
+ with tab_apply:
448
+ # ── State machine ─────────────────────────────────────────────────────────────
449
+ status: str = state["status"]
450
+
451
+ # ════════════════════════════════════════════════════════════════════════════
452
+ # LOGIN
453
+ # ════════════════════════════════════════════════════════════════════════════
454
+ if status == "login":
455
+ st.subheader("Connect with your ETSI EOL account")
456
+ st.info(
457
+ "Your credentials are used only for this session and are never stored on disk.",
458
+ icon="πŸ”’",
459
+ )
460
+ username = st.text_input("EOL Username")
461
+ password = st.text_input("EOL Password", type="password")
462
 
463
+ if st.button("Connect", type="primary"):
464
+ if not username or not password:
465
+ st.error("Please enter both username and password.")
 
 
 
 
 
 
 
 
 
466
  else:
467
+ with st.spinner("Verifying credentials…"):
468
+ ok = verify_eol_credentials(username, password)
469
+ if ok:
470
+ st.session_state.eol_user = username
471
+ st.session_state.eol_password = password
472
+ state["status"] = "upload"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
473
  save_state(sid, state)
474
  st.rerun()
475
  else:
476
+ st.error("Login failed β€” check your EOL username and password.")
477
+
478
+ # ════════════════════════════════════════════════════════════════════════════
479
+ # UPLOAD
480
+ # ════════════════════════════════════════════════════════════════════════════
481
+ elif status == "upload":
482
+ st.subheader("Step 1 β€” Upload contribution list")
483
+
484
+ mode_label = st.radio(
485
+ "Pipeline mode",
486
+ ["By contributor name", "By TS (all CRs for a spec)"],
487
+ key="mode_radio",
488
+ )
489
+ pipeline_mode = "contributor" if mode_label.startswith("By contributor") else "ts"
490
+ state["mode"] = pipeline_mode
491
+
492
+ # ── Resolve active Excel (saved from a previous run, or freshly uploaded) ──
493
+ saved_name = state.get("excel_filename")
494
+ saved_path = session_dir(sid) / saved_name if saved_name else None
495
+ has_saved = saved_path is not None and saved_path.exists()
496
+
497
+ # ── Previous meetings selector ─────────────────────────────────────────────
498
+ past_meetings = load_meetings()
499
+ if past_meetings:
500
+ with st.expander("πŸ“‚ Previous meetings", expanded=not has_saved):
501
+ meeting_options = [
502
+ f"{m['meeting_name']} β€” {m['uploaded_at']} ({m['original_filename']})"
503
+ for m in reversed(past_meetings)
504
+ ]
505
+ chosen_label = st.selectbox("Select meeting", meeting_options, key="meeting_select")
506
+ if st.button("Load this meeting", key="load_meeting"):
507
+ chosen_idx = len(past_meetings) - 1 - meeting_options.index(chosen_label)
508
+ m = past_meetings[chosen_idx]
509
+ cached = Path(m["excel_path"])
510
+ if cached.exists():
511
+ import shutil as _sh
512
+ dest = session_dir(sid) / m["original_filename"]
513
+ _sh.copy2(str(cached), str(dest))
514
+ state["excel_filename"] = m["original_filename"]
515
+ state["excel_hash"] = m["excel_hash"]
516
+ save_state(sid, state)
517
+ st.rerun()
518
+ else:
519
+ st.error("Cached Excel file not found β€” please upload it again.")
520
+
521
+ if has_saved:
522
+ st.success(f"Using: **{saved_name}**")
523
+ with st.expander("Replace Excel file"):
524
+ uploaded = st.file_uploader(
525
+ "Upload a different Excel file (.xlsx or .xls)",
526
+ type=["xlsx", "xls"],
527
+ key="excel_replace",
528
+ )
529
+ else:
530
  uploaded = st.file_uploader(
531
+ "Excel contribution list (.xlsx or .xls)",
532
  type=["xlsx", "xls"],
 
533
  )
 
 
 
 
 
534
 
535
+ # When a new file is dropped, save it immediately and update state
536
+ if uploaded:
537
+ active_path = session_dir(sid) / uploaded.name
538
+ active_bytes = bytes(uploaded.getbuffer())
539
+ active_path.write_bytes(active_bytes)
540
+ state["excel_filename"] = uploaded.name
541
+ excel_hash = hashlib.sha256(active_bytes).hexdigest()[:16]
542
+ state["excel_hash"] = excel_hash
543
+ # Save to global meeting store
544
+ meeting_name = peek_meeting_name(active_path)
545
+ if meeting_name:
546
+ save_meeting(meeting_name, active_path, excel_hash)
547
+ save_state(sid, state)
548
+ elif has_saved:
549
+ active_path = saved_path
550
+ else:
551
+ active_path = None
552
 
553
+ if pipeline_mode == "contributor":
554
+ person_name = st.text_input(
555
+ "Contributor name (must match SubmittedBy column)",
556
+ value=state.get("person_name", "Ly Thanh PHAN"),
557
+ )
558
 
559
+ if active_path and st.button("Parse CR list β†’", type="primary"):
560
+ with st.spinner("Parsing Excel…"):
561
+ try:
562
+ from fetch_crs import parse_excel
563
+ cr_list = parse_excel(str(active_path), person_name)
564
+ state["status"] = "preview"
565
+ state["person_name"] = person_name
566
+ state["cr_list"] = [list(row) for row in cr_list]
567
+ save_state(sid, state)
568
+ st.rerun()
569
+ except Exception as exc:
570
+ st.error(f"Failed to parse Excel: {exc}")
571
+
572
+ else: # TS mode
573
+ if active_path:
574
+ excel_hash = state.get("excel_hash") or hashlib.sha256(active_path.read_bytes()).hexdigest()[:16]
575
+ state["excel_hash"] = excel_hash
576
+
577
+ hf_token = os.environ.get("HF_TOKEN", "")
578
+ hf_repo = state.get("hf_repo", "OrganizedProgrammers/CR_Index")
579
+
580
+ # Check whether this Excel is already indexed
581
+ existing = load_hf_index_cached(hf_token, hf_repo)
582
+ already_indexed = any(r.get("excel_hash") == excel_hash for r in existing)
583
+
584
+ if already_indexed:
585
+ st.success(f"This Excel (`{excel_hash}`) is already indexed in HF.")
586
+ if st.button("Select TS β†’", type="primary"):
587
+ state["status"] = "ts_select"
588
+ save_state(sid, state)
589
+ st.rerun()
590
+ else:
591
+ st.info(f"Excel hash: `{excel_hash}` β€” not yet indexed.")
592
+ if st.button("Build CR Index", type="primary"):
593
+ # CRs downloaded during indexing go to the session-level cache
594
+ cr_cache_dir = session_dir(sid) / "CRs"
595
+ cr_cache_dir.mkdir(parents=True, exist_ok=True)
596
+ index_log = str(session_dir(sid) / "index.log")
597
+ rc_path = _rc_path(sid)
598
+ rc_path.unlink(missing_ok=True)
599
+
600
+ cmd = [
601
+ sys.executable,
602
+ str(SCRIPTS_DIR / "build_cr_index.py"),
603
+ str(active_path),
604
+ "--output-dir", str(session_dir(sid)),
605
+ "--hf-repo", hf_repo,
606
+ ]
607
+ env = os.environ.copy()
608
+ env["EOL_USER"] = st.session_state.eol_user
609
+ env["EOL_PASSWORD"] = st.session_state.eol_password
610
+ # HF_TOKEN is already in env via os.environ
611
+
612
+ _launch_proc(cmd, env, index_log, sid, state, {
613
+ "status": "indexing",
614
+ "index_log": index_log,
615
+ "output_dir": "", # no pipeline output yet
616
+ })
617
+
618
+ # ════════════════════════════════════════════════════════════════════════════
619
+ # INDEXING (build_cr_index.py running)
620
+ # ════════════════════════════════════════════════════════════════════════════
621
+ elif status == "indexing":
622
+ pid = state["pid"]
623
+ index_log = state.get("index_log", "")
624
+
625
+ proc = st.session_state.get("proc")
626
+ alive = False
627
+ if proc is not None:
628
+ alive = proc.poll() is None
629
+ else:
630
+ rc = read_return_code(sid)
631
+ if rc is None:
632
+ alive = is_process_alive(pid)
633
+
634
+ if alive:
635
+ st.subheader("⏳ Building CR Index…")
636
+ st.info(f"PID {pid} β€” started {state.get('started_at', '')[:19]}")
637
+ log_text = tail_log(index_log, 50)
638
+ st.text_area("Live log (last 50 lines)", value=log_text, height=400)
639
+ time.sleep(2)
640
+ st.rerun()
641
+ else:
642
+ rc = read_return_code(sid)
643
+ if rc is None and proc is not None:
644
+ rc = proc.returncode
645
+ state["return_code"] = rc
646
+ state["completed_at"] = datetime.now().isoformat()
647
+ if rc == 0:
648
+ # Invalidate cached HF index so ts_select gets fresh data
649
+ st.session_state.pop(f"hf_index_{state.get('hf_repo', '')}", None)
650
+ state["status"] = "ts_select"
651
+ else:
652
+ # Expose the index log as log_path so the error state can display it
653
+ state["log_path"] = state.get("index_log", "")
654
+ state["status"] = "error"
655
+ save_state(sid, state)
656
+ st.rerun()
657
+
658
+ # ════════════════════════════════════════════════════════════════════════════
659
+ # TS_SELECT (index ready β€” pick a spec to process)
660
+ # ════════════════════════════════════════════════════════════════════════════
661
+ elif status == "ts_select":
662
+ st.subheader("Step 2 β€” Select target TS spec")
663
 
664
+ hf_token = os.environ.get("HF_TOKEN", "")
665
+ hf_repo = state.get("hf_repo", "OrganizedProgrammers/CR_Index")
666
+ excel_hash = state.get("excel_hash", "")
 
667
 
668
+ with st.spinner("Loading CR index from HuggingFace…"):
669
+ records = load_hf_index_cached(hf_token, hf_repo)
670
 
671
+ records_for_excel = [r for r in records if r.get("excel_hash") == excel_hash]
 
 
672
 
673
+ if not records_for_excel:
674
+ st.error(f"No records found for excel_hash `{excel_hash}`. Try rebuilding the index.")
675
+ if st.button("← Back to upload"):
676
+ state["status"] = "upload"
677
+ save_state(sid, state)
678
+ st.rerun()
679
+ else:
680
+ by_spec = defaultdict(list)
681
+ for r in records_for_excel:
682
+ by_spec[r["spec_number"]].append(r)
683
+
684
+ _ALL = "All"
685
+ spec_options = sorted(by_spec.keys()) + [_ALL]
686
+ selected_spec = st.selectbox("Select target TS spec", spec_options)
687
+
688
+ if selected_spec and selected_spec != _ALL:
689
+ versions = defaultdict(list)
690
+ for r in by_spec[selected_spec]:
691
+ versions[r["version"]].append(r["uid"])
692
+ st.write("**Versions found:**")
693
+ for ver, uids in sorted(versions.items()):
694
+ st.write(f" v{ver}: {len(uids)} CR(s) β€” {', '.join(uids)}")
695
+ elif selected_spec == _ALL:
696
+ total_crs = sum(len(v) for v in by_spec.values())
697
+ st.write(f"**All specs** β€” {len(by_spec)} TS(s), {total_crs} CR(s) total")
698
+
699
+ def _start_ts_run(ts_id_arg: str | None):
700
+ run_id = int(time.time())
701
+ output_dir = session_dir(sid) / f"run_{run_id}"
702
+ output_dir.mkdir(parents=True, exist_ok=True)
703
+ cr_cache_dir = session_dir(sid) / "CRs"
704
+ cr_cache_dir.mkdir(parents=True, exist_ok=True)
705
+ log_path = session_dir(sid) / f"pipeline_{run_id}.log"
706
+ rc_path = _rc_path(sid)
707
+ rc_path.unlink(missing_ok=True)
708
+
709
+ cmd = [
710
+ sys.executable,
711
+ str(SCRIPTS_DIR / "orchestrate_cr.py"),
712
+ "--output-dir", str(output_dir),
713
+ "--cr-cache-dir", str(cr_cache_dir),
714
+ "--ts-mode",
715
+ "--excel-hash", excel_hash,
716
+ "--hf-repo", hf_repo,
717
+ ]
718
+ if ts_id_arg:
719
+ cmd += ["--ts-id", ts_id_arg]
720
+
721
+ env = os.environ.copy()
722
+ env["EOL_USER"] = st.session_state.eol_user
723
+ env["EOL_PASSWORD"] = st.session_state.eol_password
724
+
725
+ _launch_proc(cmd, env, log_path, sid, state, {
726
+ "ts_id": ts_id_arg or "all",
727
+ "status": "running",
728
+ "output_dir": str(output_dir),
729
+ "log_path": str(log_path),
730
+ "run_log_paths": [str(log_path)],
731
+ })
732
+
733
+ col_back, col_apply = st.columns([1, 3])
734
+ with col_back:
735
+ if st.button("← Back"):
736
+ state["status"] = "upload"
737
  save_state(sid, state)
738
  st.rerun()
739
+ with col_apply:
740
+ ts_arg = None if selected_spec == _ALL else selected_spec
741
+ if st.button("β–Ά Apply CRs", type="primary"):
742
+ _start_ts_run(ts_arg)
743
+
744
+ # ── Accumulated session results ──────────────────────────���────────────
745
+ applied_runs = state.get("applied_runs", [])
746
+ if applied_runs:
747
+ st.divider()
748
+ st.subheader(f"Session results ({len(applied_runs)} run(s))")
749
+ for run in applied_runs:
750
+ icon = "βœ…" if run["status"] == "done" else "❌"
751
+ label = run["ts_id"] if run["ts_id"] != "all" else "All TSs"
752
+ ts_dir = Path(run["output_dir"]) / "TS"
753
+ specs = sorted(d.name for d in ts_dir.iterdir() if d.is_dir()) if ts_dir.exists() else []
754
+ specs_str = f" β€” {', '.join(specs)}" if specs else ""
755
+ st.write(f"{icon} **{label}**{specs_str}")
756
+
757
+ # ════════════════════════════════════════════════════════════════════════════
758
+ # PREVIEW
759
+ # ════════════════════════════════════════════════════════════════════════════
760
+ elif status == "preview":
761
+ cr_list = state["cr_list"]
762
+ st.subheader(f"Step 2 β€” {len(cr_list)} Accepted CR(s) found")
763
+
764
+ if cr_list:
765
+ import pandas as pd
766
+ df = pd.DataFrame(cr_list, columns=["UID", "Title"])
767
+ st.dataframe(df, width='stretch')
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
768
  else:
769
+ st.warning(
770
+ f"No Accepted CRs found for **{state['person_name']}** in this file."
771
+ )
772
+ # Diagnostic: show what names are in the SubmittedBy column
773
+ excel_path = session_dir(sid) / state["excel_filename"]
774
+ found_names = peek_submitted_by(excel_path)
775
+ if found_names:
776
+ st.info(
777
+ "**Names found in SubmittedBy column** β€” copy the exact one into the field above and re-upload:\n\n"
778
+ + "\n".join(f"- `{n}`" for n in found_names)
779
+ )
 
 
 
 
 
 
 
 
 
780
 
781
+ col1, col2 = st.columns(2)
782
+ with col1:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
783
  if st.button("← Back"):
784
  state["status"] = "upload"
785
+ state["cr_list"] = []
786
  save_state(sid, state)
787
  st.rerun()
788
+ with col2:
789
+ if cr_list and st.button("β–Ά Start Pipeline", type="primary"):
790
+ excel_path = session_dir(sid) / state["excel_filename"]
791
+ # Per-run directory so each pipeline's outputs are isolated
792
+ run_id = int(time.time())
793
+ output_dir = session_dir(sid) / f"run_{run_id}"
794
+ output_dir.mkdir(parents=True, exist_ok=True)
795
+ cr_cache_dir = session_dir(sid) / "CRs"
796
+ cr_cache_dir.mkdir(parents=True, exist_ok=True)
797
+ log_path = session_dir(sid) / f"pipeline_{run_id}.log"
798
+ rc_path = _rc_path(sid)
799
+ rc_path.unlink(missing_ok=True)
800
+
801
+ cmd = [
802
+ sys.executable,
803
+ str(SCRIPTS_DIR / "orchestrate_cr.py"),
804
+ str(excel_path),
805
+ state["person_name"],
806
+ "--output-dir", str(output_dir),
807
+ "--cr-cache-dir", str(cr_cache_dir),
808
+ ]
809
+
810
+ env = os.environ.copy()
811
+ env["EOL_USER"] = st.session_state.eol_user
812
+ env["EOL_PASSWORD"] = st.session_state.eol_password
813
+
814
+ _launch_proc(cmd, env, log_path, sid, state, {
815
+ "status": "running",
816
+ "output_dir": str(output_dir),
817
+ "log_path": str(log_path),
818
+ "run_log_paths": [str(log_path)],
819
+ })
820
+
821
+ # ════════════════════════════════════════════════════════════════════════════
822
+ # RUNNING
823
+ # ════════════════════════════════════════════════════════════════════════════
824
+ elif status == "running":
825
+ pid = state["pid"]
826
+ log_path = state["log_path"]
827
+
828
+ # Determine whether process is still alive
829
+ proc = st.session_state.get("proc")
830
+ alive = False
831
+ if proc is not None:
832
+ alive = proc.poll() is None
833
+ else:
834
+ # Session reloaded β€” check returncode file, then PID
835
+ rc = read_return_code(sid)
836
+ if rc is None:
837
+ alive = is_process_alive(pid)
838
+
839
+ if alive:
840
+ st.subheader("⏳ Pipeline running…")
841
+ st.info(f"PID {pid} β€” started {state.get('started_at', '')[:19]}")
842
+ log_text = tail_log(log_path, 100)
843
+ st.text_area("Live log (last 100 lines)", value=log_text, height=400)
844
+ time.sleep(2)
845
+ st.rerun()
846
+ else:
847
+ # Process finished β€” determine return code
848
+ rc = read_return_code(sid)
849
+ if rc is None and proc is not None:
850
+ rc = proc.returncode
851
+ state["return_code"] = rc
852
+ state["completed_at"] = datetime.now().isoformat()
853
+ fin_status = "done" if rc == 0 else "error"
854
+ state["status"] = fin_status
855
+ # Accumulate completed run for TS mode
856
+ if state.get("mode") == "ts":
857
+ run_entry = {
858
+ "ts_id": state.get("ts_id", ""),
859
+ "output_dir": state.get("output_dir", ""),
860
+ "log_path": state.get("log_path", ""),
861
+ "status": fin_status,
862
+ }
863
+ runs = state.get("applied_runs", [])
864
+ if not any(r["output_dir"] == run_entry["output_dir"] for r in runs):
865
+ runs.append(run_entry)
866
+ state["applied_runs"] = runs
867
+ save_state(sid, state)
868
+ st.rerun()
869
+
870
+ # ════════════════════════════════════════════════════════════════════════════
871
+ # DONE / ERROR
872
+ # ════════════════════════════════════════════════════════════════════════════
873
+ elif status in ("done", "error"):
874
+ log_path = state.get("log_path", "")
875
+ output_dir = Path(state.get("output_dir", ""))
876
+ rc = state.get("return_code")
877
+
878
+ st.subheader("Step 3 β€” Results")
879
+ if status == "done":
880
+ st.success("βœ… Pipeline completed successfully!")
881
+ else:
882
+ st.error(f"❌ Pipeline finished with errors (return code: {rc})")
883
+
884
+ # Per-TS results table β€” merge this run's logs so retry results supersede
885
+ # the original failed entries for the same TS key.
886
+ _merged: dict[str, dict] = {}
887
+ for _lf in state.get("run_log_paths", []):
888
+ for _r in parse_log_results(_lf):
889
+ _merged[_r["TS"]] = _r
890
+ results = list(_merged.values())
891
+ if results:
892
+ st.subheader("Results per TS")
893
+ import pandas as pd
894
+
895
+ n_warn = sum(1 for r in results if r["warnings"])
896
+ warn_label = f"Warnings ({n_warn})" if n_warn else "Warnings"
897
+ tab_summary, tab_warnings = st.tabs(["Summary", warn_label])
898
+
899
+ def _color_status(val):
900
+ return {
901
+ "OK": "background-color: #d4edda; color: #155724",
902
+ "WARN": "background-color: #fff3cd; color: #856404",
903
+ "FAIL": "background-color: #f8d7da; color: #721c24",
904
+ "SKIP": "background-color: #e2e3e5; color: #383d41",
905
+ }.get(val, "")
906
+
907
+ with tab_summary:
908
+ df = pd.DataFrame([{"Status": r["Status"], "TS": r["TS"]} for r in results])
909
+ st.dataframe(
910
+ df.style.map(_color_status, subset=["Status"]),
911
+ width='stretch',
912
+ )
913
+
914
+ with tab_warnings:
915
+ warned = [r for r in results if r["warnings"]]
916
+ if warned:
917
+ for r in warned:
918
+ with st.expander(f"⚠️ {r['TS']} β€” {len(r['warnings'])} warning(s)"):
919
+ for w in r["warnings"]:
920
+ st.text(w)
921
+ else:
922
+ st.success("No warnings.")
923
 
924
+ # Download ZIP
925
+ # ── Download section ──────────────────────────────────────────────────────
926
+ st.subheader("Download results")
927
  applied_runs = state.get("applied_runs", [])
928
+ valid_runs = [r for r in applied_runs if Path(r["output_dir"]).exists()
929
+ and any(Path(r["output_dir"]).rglob("*"))]
930
+
931
+ if valid_runs:
932
+ # Show a compact list of what's included
933
+ for run in valid_runs:
934
  icon = "βœ…" if run["status"] == "done" else "❌"
935
  label = run["ts_id"] if run["ts_id"] != "all" else "All TSs"
936
  ts_dir = Path(run["output_dir"]) / "TS"
937
  specs = sorted(d.name for d in ts_dir.iterdir() if d.is_dir()) if ts_dir.exists() else []
938
  specs_str = f" β€” {', '.join(specs)}" if specs else ""
939
+ st.write(f"{icon} {label}{specs_str}")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
940
 
941
+ # Build ZIP from all accumulated runs
942
+ buf = io.BytesIO()
943
+ with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as zf:
944
+ for run in valid_runs:
945
+ out = Path(run["output_dir"])
946
+ for f in out.rglob("*"):
947
+ if f.is_file():
948
+ zf.write(f, f.relative_to(out.parent))
949
+ buf.seek(0)
950
+ col_dl, col_flush = st.columns([3, 1])
951
+ with col_dl:
952
+ st.download_button(
953
+ label="⬇ Download results ZIP",
954
+ data=buf.read(),
955
+ file_name=f"cr_results_{sid[:8]}.zip",
956
+ mime="application/zip",
957
+ type="primary",
958
+ )
959
+ with col_flush:
960
+ if st.button("πŸ—‘ Flush"):
961
+ import shutil
962
+ for run in applied_runs:
963
+ out = Path(run["output_dir"])
964
+ if out.exists():
965
+ shutil.rmtree(out, ignore_errors=True)
966
+ state["applied_runs"] = []
967
+ save_state(sid, state)
968
+ st.rerun()
969
+ elif output_dir.exists() and any(output_dir.rglob("*")):
970
+ # Fallback for contributor mode (no applied_runs)
971
+ zip_bytes = make_zip(output_dir)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
972
  st.download_button(
973
  label="⬇ Download results ZIP",
974
+ data=zip_bytes,
975
  file_name=f"cr_results_{sid[:8]}.zip",
976
  mime="application/zip",
977
  type="primary",
978
  )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
979
  else:
980
+ st.warning("Output directory is empty β€” nothing to download.")
 
 
 
 
 
 
 
 
 
 
 
 
981
 
982
+ # Full log
983
+ with st.expander("Full pipeline log"):
984
+ if log_path and Path(log_path).exists():
985
+ st.text(Path(log_path).read_text(errors="replace"))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
986
  else:
987
+ st.text("Log not found.")
988
+
989
+ # ── TS Recovery ───────────────────────────────────────────────────────────
990
+ failed_ts_path = output_dir / "failed_ts.json"
991
+ if failed_ts_path.exists():
992
+ failed_ts_entries = json.loads(failed_ts_path.read_text())
993
+ if failed_ts_entries:
994
+ st.divider()
995
+ st.subheader("⚠️ Recover failed TS downloads")
996
+ st.info(
997
+ f"{len(failed_ts_entries)} TS(s) could not be downloaded. "
998
+ "Retry or upload each one manually, then apply the CRs."
999
+ )
1000
+
1001
+ for entry in failed_ts_entries:
1002
+ spec_key = f"{entry['spec_number']} v{entry['version']}"
1003
+ dest_path = Path(entry["spec_dir"]) / entry["expected_filename"]
1004
+ ready = dest_path.exists()
1005
+
1006
+ label = f"{'βœ…' if ready else '❌'} TS {spec_key} β€” CRs: {', '.join(entry['cr_uids'])}"
1007
+ with st.expander(label, expanded=not ready):
1008
+ col1, col2 = st.columns(2)
1009
+
1010
+ with col1:
1011
+ if st.button("πŸ”„ Retry download",
1012
+ key=f"retry_{entry['spec_compact']}_{entry['version']}"):
1013
+ from fetch_crs import download_ts as _dl_ts
1014
+ with st.spinner(f"Downloading TS {spec_key}…"):
1015
+ fn, note = _dl_ts(
1016
+ entry["spec_number"], entry["version"],
1017
+ Path(entry["spec_dir"]),
1018
+ st.session_state.eol_user,
1019
+ st.session_state.eol_password,
1020
+ )
1021
+ if fn:
1022
+ st.success(f"Downloaded: {fn}")
1023
+ st.rerun()
1024
+ else:
1025
+ st.error(f"Failed: {note}")
1026
+
1027
+ with col2:
1028
+ uploaded_ts = st.file_uploader(
1029
+ f"Or upload `{entry['expected_filename']}`",
1030
+ type=["docx"],
1031
+ key=f"upload_{entry['spec_compact']}_{entry['version']}",
1032
+ )
1033
+ if uploaded_ts is not None:
1034
+ Path(entry["spec_dir"]).mkdir(parents=True, exist_ok=True)
1035
+ dest_path.write_bytes(uploaded_ts.read())
1036
+ st.success("Saved βœ“")
1037
+ st.rerun()
1038
 
1039
+ # Global apply button β€” enabled when β‰₯1 TS is now on disk
1040
+ ready_entries = [
1041
+ e for e in failed_ts_entries
1042
+ if (Path(e["spec_dir"]) / e["expected_filename"]).exists()
1043
+ ]
1044
+ remaining = len(failed_ts_entries) - len(ready_entries)
1045
+
1046
+ if ready_entries:
1047
+ if remaining:
1048
+ st.warning(f"{len(ready_entries)} ready, {remaining} will be skipped.")
1049
+ else:
1050
+ st.success(f"All {len(ready_entries)} TS(s) ready.")
1051
+
1052
+ if st.button("β–Ά Apply CRs to recovered TSs", type="primary"):
1053
+ retry_log = str(session_dir(sid) / f"pipeline_{int(time.time())}_retry.log")
1054
+ cmd = [
1055
+ sys.executable,
1056
+ str(SCRIPTS_DIR / "orchestrate_cr.py"),
1057
+ "--output-dir", state["output_dir"],
1058
+ "--retry-mode",
1059
+ ]
1060
+ env = os.environ.copy()
1061
+ env["EOL_USER"] = st.session_state.eol_user
1062
+ env["EOL_PASSWORD"] = st.session_state.eol_password
1063
+
1064
+ _launch_proc(cmd, env, retry_log, sid, state, {
1065
+ "status": "running",
1066
+ "log_path": retry_log,
1067
+ "run_log_paths": state.get("run_log_paths", []) + [retry_log],
1068
+ })
1069
+ else:
1070
+ st.warning("No TSs available yet β€” retry download or upload DOCX files above.")
1071
+
1072
+ # ── CR Recovery ───────────────────────────────────────────────────────────
1073
+ failed_cr_path = output_dir / "failed_cr.json"
1074
+ if failed_cr_path.exists():
1075
+ failed_cr_entries = json.loads(failed_cr_path.read_text())
1076
+ if failed_cr_entries:
1077
+ st.divider()
1078
+ st.subheader("⚠️ Recover failed CR downloads")
1079
+ st.info(
1080
+ f"{len(failed_cr_entries)} CR(s) could not be downloaded. "
1081
+ "Retry or upload each one manually, then apply."
1082
  )
1083
+
1084
+ for entry in failed_cr_entries:
1085
+ uid = entry["uid"]
1086
+ cr_dir_path = Path(entry["cr_dir"])
1087
+ expected = cr_dir_path / entry["expected_filename"]
1088
+ ready = expected.exists() or (cr_dir_path / f"{uid}_extracted.docx").exists()
1089
+ ts_label = (
1090
+ f"TS {entry['ts_spec_number']} v{entry['ts_version']}"
1091
+ if entry.get("ts_spec_number") else "TS unknown"
1092
+ )
1093
+ label = f"{'βœ…' if ready else '❌'} CR {uid} β†’ {ts_label}"
1094
+ with st.expander(label, expanded=not ready):
1095
+ col1, col2 = st.columns(2)
1096
+
1097
+ with col1:
1098
+ if st.button("πŸ”„ Retry download", key=f"retry_cr_{uid}"):
1099
+ from fetch_crs import download_cr as _dl_cr
1100
+ with st.spinner(f"Downloading CR {uid}…"):
1101
+ fn, note = _dl_cr(
1102
+ uid, cr_dir_path,
1103
+ st.session_state.eol_user,
1104
+ st.session_state.eol_password,
1105
+ )
1106
+ if fn:
1107
+ st.success(f"Downloaded: {fn.name}")
1108
+ st.rerun()
1109
+ else:
1110
+ st.error(f"Failed: {note}")
1111
+
1112
+ with col2:
1113
+ uploaded_cr = st.file_uploader(
1114
+ f"Or upload `{entry['expected_filename']}`",
1115
+ type=["docx"],
1116
+ key=f"upload_cr_{uid}",
1117
+ )
1118
+ if uploaded_cr is not None:
1119
+ cr_dir_path.mkdir(parents=True, exist_ok=True)
1120
+ expected.write_bytes(uploaded_cr.read())
1121
+ st.success("Saved βœ“")
1122
  st.rerun()
1123
+
1124
+ ready_cr_entries = [
1125
+ e for e in failed_cr_entries
1126
+ if (Path(e["cr_dir"]) / e["expected_filename"]).exists()
1127
+ or (Path(e["cr_dir"]) / f"{e['uid']}_extracted.docx").exists()
1128
+ ]
1129
+ remaining_cr = len(failed_cr_entries) - len(ready_cr_entries)
1130
+
1131
+ if ready_cr_entries:
1132
+ if remaining_cr:
1133
+ st.warning(f"{len(ready_cr_entries)} ready, {remaining_cr} still missing.")
1134
+ else:
1135
+ st.success(f"All {len(ready_cr_entries)} CR(s) ready.")
1136
+
1137
+ if st.button("β–Ά Apply recovered CRs", type="primary", key="apply_recovered_crs"):
1138
+ retry_log = str(session_dir(sid) / f"pipeline_{int(time.time())}_retry.log")
1139
+ cmd = [
1140
+ sys.executable,
1141
+ str(SCRIPTS_DIR / "orchestrate_cr.py"),
1142
+ "--output-dir", state["output_dir"],
1143
+ "--retry-mode",
1144
+ ]
1145
+ env = os.environ.copy()
1146
+ env["EOL_USER"] = st.session_state.eol_user
1147
+ env["EOL_PASSWORD"] = st.session_state.eol_password
1148
+ _launch_proc(cmd, env, retry_log, sid, state, {
1149
+ "status": "running",
1150
+ "log_path": retry_log,
1151
+ "run_log_paths": state.get("run_log_paths", []) + [retry_log],
1152
+ })
1153
  else:
1154
+ st.warning("No CRs recovered yet β€” retry download or upload DOCX files above.")
1155
+
1156
+ # Navigation
1157
+ st.divider()
1158
+ col_restart, col_new = st.columns(2)
1159
+ with col_restart:
1160
+ if state.get("mode") == "ts":
1161
+ back_label = "← Back to TS Selection"
1162
+ back_status = "ts_select"
1163
+ clear_keys = ("pid", "output_dir", "log_path", "index_log",
1164
+ "started_at", "completed_at", "return_code", "ts_id")
 
 
 
 
 
 
 
1165
  else:
1166
+ back_label = "← Back to CR Preview"
1167
+ back_status = "preview"
1168
+ clear_keys = ("pid", "output_dir", "log_path", "index_log",
1169
+ "started_at", "completed_at", "return_code")
1170
+ if st.button(back_label, type="primary"):
1171
+ for _k in clear_keys:
1172
+ state[_k] = None
1173
+ state["run_log_paths"] = []
1174
+ state["status"] = back_status
1175
+ if "proc" in st.session_state:
1176
+ del st.session_state.proc
1177
+ _rc_path(sid).unlink(missing_ok=True)
1178
+ save_state(sid, state)
1179
+ st.rerun()
1180
+ with col_new:
1181
+ if st.button("Start new session"):
1182
+ new_sid = str(uuid.uuid4())
1183
+ st.session_state.sid = new_sid
1184
+ st.session_state.state = new_state(new_sid)
1185
+ if "proc" in st.session_state:
1186
+ del st.session_state.proc
1187
+ st.query_params["sid"] = new_sid
1188
+ save_state(new_sid, st.session_state.state)
1189
+ st.rerun()
1190
 
1191
+ with tab_verify:
1192
+ from verify_review import render_verify_tab
1193
+ render_verify_tab(session_dir, sid)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1194
 
1195
  # ════════════════════════════════════════════════════════════════════════════
1196
  # VERSION HISTORY (bottom of every stable page)
scripts/cr_parser.py CHANGED
@@ -4,6 +4,7 @@ cr_parser.py β€” Parse a CR DOCX's tracked changes into a JSON manifest.
4
 
5
  Each entry in the manifest is one of:
6
  {"type": "text_replace", "location": {...}, "old": "...", "new": "..."}
 
7
  {"type": "para_insert", "location": {...}, "paragraphs": [...]}
8
  {"type": "row_insert", "location": {...}, "cells": [...]}
9
 
@@ -246,6 +247,54 @@ def _extract_inline_replacements(p_elem):
246
  return pairs
247
 
248
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
249
  # ── Table change extraction ───────────────────────────────────────────────────
250
 
251
  def _parse_table(tbl_elem, changes, section_heading='', section_number=''):
@@ -515,6 +564,20 @@ def _parse_body(body, changes):
515
  'new': new_text,
516
  })
517
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
518
  orig = _para_orig_text(elem).strip()
519
  if orig and not _is_skip_marker(orig):
520
  prev_stable_text = orig
 
4
 
5
  Each entry in the manifest is one of:
6
  {"type": "text_replace", "location": {...}, "old": "...", "new": "..."}
7
+ {"type": "text_insert", "location": {...}, "before": "...", "after": "...", "text": "...", "runs_xml": [...]}
8
  {"type": "para_insert", "location": {...}, "paragraphs": [...]}
9
  {"type": "row_insert", "location": {...}, "cells": [...]}
10
 
 
247
  return pairs
248
 
249
 
250
+ def _gather_stable_text(children, idx, direction):
251
+ """Walk from idx in `direction` (-1 or +1) collecting w:t text from stable
252
+ siblings until hitting a w:ins/w:del boundary or the start/end of the
253
+ paragraph. Used to anchor a lone insertion on the text immediately
254
+ surrounding it."""
255
+ parts = []
256
+ j = idx + direction
257
+ while 0 <= j < len(children):
258
+ sib = children[j]
259
+ if sib.tag in (qn('w:ins'), qn('w:del')):
260
+ break
261
+ parts.append(''.join(t.text or '' for t in sib.findall('.//' + qn('w:t'))))
262
+ j += direction
263
+ if direction < 0:
264
+ parts.reverse()
265
+ return ''.join(parts)
266
+
267
+
268
+ def _extract_lone_insertions(p_elem):
269
+ """
270
+ Return list of (before_text, after_text, ins_text, ins_elem) for standalone
271
+ <w:ins> children that have NO adjacent <w:del> sibling β€” a pure insertion
272
+ with nothing marked as deleted (e.g. CR SETTEC(26)000050r1's ', 3.8').
273
+
274
+ _extract_inline_replacements only ever anchors on a <w:del> element, so a
275
+ lone <w:ins> with no del partner is invisible to it β€” this covers exactly
276
+ that gap. Any w:ins adjacent to a w:del is skipped here since
277
+ _extract_inline_replacements already accounts for it as a replacement.
278
+ """
279
+ children = list(p_elem)
280
+ results = []
281
+ for i, child in enumerate(children):
282
+ if child.tag != qn('w:ins') or _is_rpr_ins(child):
283
+ continue
284
+ prev_sib = children[i - 1] if i > 0 else None
285
+ next_sib = children[i + 1] if i + 1 < len(children) else None
286
+ if (prev_sib is not None and prev_sib.tag == qn('w:del')) or \
287
+ (next_sib is not None and next_sib.tag == qn('w:del')):
288
+ continue # already accounted for by _extract_inline_replacements
289
+ ins_text = _ins_text(child)
290
+ if not ins_text:
291
+ continue # e.g. an inserted line-break with no text content
292
+ before_text = _gather_stable_text(children, i, -1)
293
+ after_text = _gather_stable_text(children, i, +1)
294
+ results.append((before_text, after_text, ins_text, child))
295
+ return results
296
+
297
+
298
  # ── Table change extraction ───────────────────────────────────────────────────
299
 
300
  def _parse_table(tbl_elem, changes, section_heading='', section_number=''):
 
564
  'new': new_text,
565
  })
566
 
567
+ for before_text, after_text, ins_text, ins_elem in _extract_lone_insertions(elem):
568
+ changes.append({
569
+ 'type': 'text_insert',
570
+ 'location': {
571
+ 'kind': 'body_para',
572
+ 'para_context': _para_orig_text(elem).strip(),
573
+ 'section_number': current_section,
574
+ },
575
+ 'before': before_text,
576
+ 'after': after_text,
577
+ 'text': ins_text,
578
+ 'runs_xml': _content_runs_xml(ins_elem),
579
+ })
580
+
581
  orig = _para_orig_text(elem).strip()
582
  if orig and not _is_skip_marker(orig):
583
  prev_stable_text = orig
scripts/docx_helpers.py CHANGED
@@ -344,6 +344,51 @@ def tracked_modify_para(para, old_text, new_text, rev, author=AUTHOR, date=DATE)
344
  p_el.append(_make_run(after))
345
 
346
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
347
  def _char_diff(old, new):
348
  """
349
  Return a list of (op, text) tuples for a minimal character-level diff.
 
344
  p_el.append(_make_run(after))
345
 
346
 
347
+ def tracked_insert_text_in_para(para, before_text, after_text, ins_text, rev,
348
+ author=AUTHOR, date=DATE, runs_xml=None, split_at=None):
349
+ """
350
+ Splice a tracked INSERTION-ONLY change into para at the point where
351
+ before_text ends and after_text begins β€” a pure content insertion with
352
+ NO corresponding deletion (e.g. CR SETTEC(26)000050r1's ', 3.8').
353
+ Splits the paragraph into: [before][<w:ins>ins_text</w:ins>][after]
354
+
355
+ Can't reuse tracked_modify_para here: it locates the splice point via
356
+ str.partition(old_text), and old_text would be '' for a pure insertion β€”
357
+ partition('') raises "empty separator not allowed". Instead this locates
358
+ the splice point by finding before_text+after_text in para.text, unless
359
+ the caller already resolved an offset itself (e.g. via normalized
360
+ matching for a CR/TS Unicode variant mismatch) and passes split_at.
361
+ """
362
+ full = para.text
363
+ if split_at is None:
364
+ combined = before_text + after_text
365
+ idx = full.find(combined)
366
+ if idx == -1:
367
+ raise ValueError(
368
+ f"Splice point not found: before={before_text!r} after={after_text!r}\nIn: {full!r}"
369
+ )
370
+ split_at = idx + len(before_text)
371
+ before_part = full[:split_at]
372
+ after_part = full[split_at:]
373
+
374
+ p_el = para._element
375
+ for r in p_el.findall(qn('w:r')):
376
+ p_el.remove(r)
377
+
378
+ if before_part:
379
+ p_el.append(_make_run(before_part))
380
+
381
+ ins_el = OxmlElement('w:ins')
382
+ for k, v in _ins_attr(rev, author, date).items():
383
+ ins_el.set(k, v)
384
+ if not _clone_runs_into(ins_el, runs_xml, rev):
385
+ ins_el.append(_make_run(ins_text))
386
+ p_el.append(ins_el)
387
+
388
+ if after_part:
389
+ p_el.append(_make_run(after_part))
390
+
391
+
392
  def _char_diff(old, new):
393
  """
394
  Return a list of (op, text) tuples for a minimal character-level diff.
scripts/etsi_client.py CHANGED
@@ -272,7 +272,7 @@ class TSFetcher:
272
  "title": "1",
273
  "etsiNumber": "1",
274
  "content": "1",
275
- "version": "0",
276
  "onApproval": "1",
277
  "published": "1",
278
  "withdrawn": "1",
 
272
  "title": "1",
273
  "etsiNumber": "1",
274
  "content": "1",
275
+ "version": "1",
276
  "onApproval": "1",
277
  "published": "1",
278
  "withdrawn": "1",
scripts/orchestrate_cr.py CHANGED
@@ -46,6 +46,7 @@ sys.path.insert(0, str(FETCH_SCRIPTS))
46
  from fetch_crs import parse_excel, download_cr, parse_cr_cover, download_ts, wsl_path
47
  from cr_parser import parse_cr
48
  from ts_applicator import apply_manifest
 
49
  from finalize_ts import (
50
  extract_cr_metadata,
51
  compute_pub_date,
@@ -154,6 +155,21 @@ def _apply_ts_group(spec_number, version, uids, ts_paths, cr_paths, spec_dir,
154
  print(f' Parsing {uid}... ', end='', flush=True)
155
  try:
156
  changes = parse_cr(cr_paths[uid])
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
157
  combined_manifest.extend(changes)
158
  participating_uids.append(uid)
159
  print(f'{len(changes)} change(s)')
@@ -190,6 +206,14 @@ def _apply_ts_group(spec_number, version, uids, ts_paths, cr_paths, spec_dir,
190
  ts_final_or_applied = ts_applied # fallback if finalise raises
191
  try:
192
  ts_doc = docx_lib.Document(str(ts_applied))
 
 
 
 
 
 
 
 
193
  rev = RevCounter(ts_doc)
194
 
195
  pub_ym, pub_month_year = compute_pub_date()
 
46
  from fetch_crs import parse_excel, download_cr, parse_cr_cover, download_ts, wsl_path
47
  from cr_parser import parse_cr
48
  from ts_applicator import apply_manifest
49
+ from verify_applied import scan_revision_marks, verify_manifest_applied
50
  from finalize_ts import (
51
  extract_cr_metadata,
52
  compute_pub_date,
 
155
  print(f' Parsing {uid}... ', end='', flush=True)
156
  try:
157
  changes = parse_cr(cr_paths[uid])
158
+ scan = scan_revision_marks(cr_paths[uid])
159
+ n_marks = scan['handled']
160
+ if n_marks > 0 and not changes:
161
+ msg = (f'ERROR verify: [{uid}] CR contains {n_marks} revision mark(s)'
162
+ f' but parser extracted 0 changes')
163
+ errors.append(msg)
164
+ print(f'\n {msg}', end='')
165
+ for tag, cnt in scan['unhandled'].items():
166
+ msg = (f'WARN verify: [{uid}] CR contains {cnt} unhandled revision-mark '
167
+ f'element(s) of type <w:{tag}> β€” this pipeline has no apply-path '
168
+ f'for this mark type; it will NOT be reflected in the output TS')
169
+ errors.append(msg)
170
+ print(f'\n {msg}', end='')
171
+ for c in changes:
172
+ c['_cr_uid'] = uid
173
  combined_manifest.extend(changes)
174
  participating_uids.append(uid)
175
  print(f'{len(changes)} change(s)')
 
206
  ts_final_or_applied = ts_applied # fallback if finalise raises
207
  try:
208
  ts_doc = docx_lib.Document(str(ts_applied))
209
+
210
+ verify_lines = verify_manifest_applied(ts_doc, combined_manifest)
211
+ for line in verify_lines:
212
+ print(f' {line}')
213
+ for line in verify_lines:
214
+ if line.strip().startswith('ERROR'):
215
+ errors.append(line.strip())
216
+
217
  rev = RevCounter(ts_doc)
218
 
219
  pub_ym, pub_month_year = compute_pub_date()
scripts/tests/fixtures/SETTEC(26)000048r1.docx ADDED
Binary file (48 kB). View file
 
scripts/tests/fixtures/SETTEC(26)000050r1.docx ADDED
Binary file (47.2 kB). View file
 
scripts/tests/test_lone_insertion.py ADDED
@@ -0,0 +1,153 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """
3
+ Regression test for the "lone w:ins" bug: a CR paragraph containing a
4
+ standalone <w:ins> with no adjacent <w:del> (a pure insertion, nothing marked
5
+ deleted) used to be silently dropped by cr_parser.py, so a CR with real
6
+ content parsed to "0 changes" and the TS was never updated.
7
+
8
+ Fixtures are real CRs that exhibit this exact shape:
9
+ SETTEC(26)000050r1.docx β€” 1 lone <w:ins> (", 3.8")
10
+ SETTEC(26)000048r1.docx β€” 2 lone <w:ins> occurrences (same text, two clauses)
11
+
12
+ Run: python3 -m unittest scripts/tests/test_lone_insertion.py -v
13
+ """
14
+
15
+ import sys
16
+ import tempfile
17
+ import unittest
18
+ from pathlib import Path
19
+
20
+ SCRIPT_DIR = Path(__file__).resolve().parent.parent
21
+ sys.path.insert(0, str(SCRIPT_DIR))
22
+
23
+ import docx
24
+ from docx.oxml.ns import qn
25
+
26
+ from cr_parser import parse_cr, _extract_inline_replacements
27
+ from ts_applicator import apply_manifest
28
+ from verify_applied import scan_revision_marks, verify_manifest_applied
29
+
30
+ FIXTURES = Path(__file__).parent / 'fixtures'
31
+ CR_050 = FIXTURES / 'SETTEC(26)000050r1.docx'
32
+ CR_048 = FIXTURES / 'SETTEC(26)000048r1.docx'
33
+
34
+
35
+ class LoneInsertionParseTests(unittest.TestCase):
36
+
37
+ def test_cr050_yields_one_text_insert_change(self):
38
+ changes = parse_cr(CR_050)
39
+ self.assertEqual(len(changes), 1)
40
+ change = changes[0]
41
+ self.assertEqual(change['type'], 'text_insert')
42
+ self.assertEqual(change['text'], ', 3.8')
43
+ self.assertTrue(change['before'])
44
+ self.assertTrue(change['after'])
45
+ # The anchors must not themselves contain the inserted text β€” otherwise
46
+ # the splice point would be ambiguous.
47
+ self.assertNotIn(', 3.8', change['before'] + change['after'])
48
+
49
+ def test_cr048_yields_two_text_insert_changes(self):
50
+ changes = parse_cr(CR_048)
51
+ self.assertEqual(len(changes), 2)
52
+ self.assertTrue(all(c['type'] == 'text_insert' for c in changes))
53
+
54
+ def test_scan_revision_marks_reports_one_handled_no_unhandled(self):
55
+ scan = scan_revision_marks(CR_050)
56
+ self.assertEqual(scan['handled'], 1)
57
+ self.assertEqual(scan['unhandled'], {})
58
+
59
+ def test_old_extractor_saw_zero_changes_here(self):
60
+ """Pins the exact mechanism of the original bug: a lone w:ins is
61
+ invisible to _extract_inline_replacements (it only pairs w:del with
62
+ an adjacent w:ins), which is why the parser used to report 0 changes
63
+ for this CR. If this assertion ever starts failing, _extract_lone_
64
+ insertions may have become redundant β€” but it should not be removed
65
+ without re-verifying this class of CR is still handled."""
66
+ doc = docx.Document(str(CR_050))
67
+ found_target_para = False
68
+ for elem in doc.element.body:
69
+ if elem.tag != qn('w:p'):
70
+ continue
71
+ if any(c.tag == qn('w:ins') for c in elem):
72
+ found_target_para = True
73
+ self.assertEqual(_extract_inline_replacements(elem), [])
74
+ self.assertTrue(found_target_para, 'fixture no longer contains the expected w:ins paragraph')
75
+
76
+
77
+ class LoneInsertionApplyTests(unittest.TestCase):
78
+
79
+ def _make_ts_doc(self, paragraph_text):
80
+ doc = docx.Document()
81
+ doc.add_paragraph(paragraph_text)
82
+ tmp = tempfile.NamedTemporaryFile(suffix='.docx', delete=False)
83
+ doc.save(tmp.name)
84
+ return Path(tmp.name)
85
+
86
+ def test_end_to_end_apply_produces_ins_with_no_del(self):
87
+ changes = parse_cr(CR_050)
88
+ change = changes[0]
89
+ ts_text = change['before'] + change['after']
90
+
91
+ ts_path = self._make_ts_doc(ts_text)
92
+ out_path = ts_path.with_name('out.docx')
93
+ try:
94
+ n_ok, n_skip, log_lines, n_parsed, n_merged = apply_manifest(
95
+ ts_path, changes, out_path)
96
+ self.assertEqual(n_ok, 1)
97
+ self.assertEqual(n_skip, 0)
98
+
99
+ # python-docx's Paragraph.text only reads direct-child <w:r> runs β€”
100
+ # it does not descend into <w:ins>/<w:del>, so the inserted text
101
+ # must be located via the raw element tree, not p.text.
102
+ out_doc = docx.Document(str(out_path))
103
+ target = next(
104
+ p for p in out_doc.paragraphs
105
+ if change['text'] in ''.join(
106
+ t.text or '' for t in p._element.findall('.//' + qn('w:t')))
107
+ )
108
+ p_el = target._element
109
+ ins_elems = p_el.findall(qn('w:ins'))
110
+ del_elems = p_el.findall(qn('w:del'))
111
+ self.assertEqual(len(ins_elems), 1)
112
+ self.assertEqual(len(del_elems), 0)
113
+ inserted_text = ''.join(t.text or '' for t in ins_elems[0].findall('.//' + qn('w:t')))
114
+ self.assertEqual(inserted_text, change['text'])
115
+
116
+ verify_errors = verify_manifest_applied(out_doc, changes)
117
+ self.assertEqual(verify_errors, [])
118
+ finally:
119
+ ts_path.unlink(missing_ok=True)
120
+ out_path.unlink(missing_ok=True)
121
+
122
+ def test_nbsp_variant_between_cr_and_ts_still_applies(self):
123
+ """CR 48r1's second change anchors on '... and Annex A do NOT ...'
124
+ (plain space), but the real TS 102267 stores 'Annex\xa0A' with a non-
125
+ breaking space β€” discovered live when re-running this exact CR
126
+ against the real TS. The splice point must still be found via
127
+ normalized matching, and the TS's real NBSP must be preserved
128
+ (not silently overwritten with a plain space)."""
129
+ changes = parse_cr(CR_048)
130
+ change = next(c for c in changes if 'SCP82' in c['before'])
131
+ ts_text = change['before'] + change['after'].replace('Annex A', 'Annex\xa0A')
132
+ self.assertIn('\xa0A', ts_text)
133
+
134
+ ts_path = self._make_ts_doc(ts_text)
135
+ out_path = ts_path.with_name('out_nbsp.docx')
136
+ try:
137
+ n_ok, n_skip, log_lines, n_parsed, n_merged = apply_manifest(
138
+ ts_path, [change], out_path)
139
+ self.assertEqual(n_ok, 1)
140
+ self.assertEqual(n_skip, 0)
141
+
142
+ out_doc = docx.Document(str(out_path))
143
+ full = ''.join(
144
+ t.text or '' for t in out_doc.paragraphs[0]._element.findall('.//' + qn('w:t')))
145
+ self.assertIn(', 3.8', full)
146
+ self.assertIn('Annex\xa0A', full) # original NBSP preserved, not overwritten
147
+ finally:
148
+ ts_path.unlink(missing_ok=True)
149
+ out_path.unlink(missing_ok=True)
150
+
151
+
152
+ if __name__ == '__main__':
153
+ unittest.main()
scripts/ts_applicator.py CHANGED
@@ -28,6 +28,7 @@ _WARN_CONF = 0.8 # confidence below this emits WARN instead of OK
28
  from docx_helpers import (
29
  RevCounter,
30
  tracked_modify_para,
 
31
  tracked_insert_paras_after,
32
  tracked_insert_paras_after_el,
33
  _clone_runs_into,
@@ -87,6 +88,36 @@ def _norm_ws(text):
87
  return re.sub(r'\s+', '', text)
88
 
89
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
90
  def _norm_alnum(text):
91
  """Keep only lowercase alphanumeric characters β€” last-resort matching.
92
 
@@ -857,6 +888,57 @@ def _apply_text_replace(doc, change, rev, author, date, log):
857
  return False
858
 
859
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
860
  def _apply_para_insert(doc, change, rev, author, date, log, last_para_inserted=None):
861
  loc = change['location']
862
  anchor_text = loc.get('anchor_text', '')
@@ -1029,6 +1111,8 @@ def apply_manifest(ts_path, manifest, out_path, author=DEFAULT_AUTHOR, date=DEFA
1029
  last_para_inserted=last_para_inserted)
1030
  elif ctype == 'text_replace':
1031
  ok = _apply_text_replace(doc, change, rev, author, date, log)
 
 
1032
  elif ctype == 'para_insert':
1033
  ok = _apply_para_insert(doc, change, rev, author, date, log,
1034
  last_para_inserted=last_para_inserted)
 
28
  from docx_helpers import (
29
  RevCounter,
30
  tracked_modify_para,
31
+ tracked_insert_text_in_para,
32
  tracked_insert_paras_after,
33
  tracked_insert_paras_after_el,
34
  _clone_runs_into,
 
88
  return re.sub(r'\s+', '', text)
89
 
90
 
91
+ def _normalize_with_map(text):
92
+ """Apply _UNICODE_REPLACEMENTS char-by-char, returning (normalized_text,
93
+ index_map) where index_map[i] is the index into the ORIGINAL text that
94
+ normalized_text[i] came from.
95
+
96
+ Used by _apply_text_insert to locate a splice point via normalized
97
+ matching (e.g. CR text with a plain space vs. TS text with a non-
98
+ breaking space) while still splicing into the ORIGINAL, unnormalized
99
+ paragraph text β€” so a real NBSP in the TS is preserved rather than
100
+ silently replaced with a plain space."""
101
+ out_chars = []
102
+ idx_map = []
103
+ for i, ch in enumerate(text):
104
+ replacement = ch
105
+ for old, new in _UNICODE_REPLACEMENTS:
106
+ if ch == old:
107
+ replacement = new
108
+ break
109
+ for c in replacement:
110
+ out_chars.append(c)
111
+ idx_map.append(i)
112
+ return ''.join(out_chars), idx_map
113
+
114
+
115
+ def _apply_unicode_replacements(text):
116
+ for old, new in _UNICODE_REPLACEMENTS:
117
+ text = text.replace(old, new)
118
+ return text
119
+
120
+
121
  def _norm_alnum(text):
122
  """Keep only lowercase alphanumeric characters β€” last-resort matching.
123
 
 
888
  return False
889
 
890
 
891
+ def _apply_text_insert(doc, change, rev, author, date, log):
892
+ loc = change['location']
893
+ ctx = loc.get('para_context', '')
894
+ section_number = loc.get('section_number', '')
895
+ before = change.get('before', '')
896
+ after = change.get('after', '')
897
+ ins_text = change.get('text', '')
898
+ runs_xml = change.get('runs_xml')
899
+
900
+ para, conf = _find_para_with_section(
901
+ doc, ctx, section_number, 'text_insert', log, prefer_not_in_table=True)
902
+ if para is None:
903
+ log.append(f" ERROR text_insert: paragraph not found for context {ctx[:60]!r}")
904
+ return False
905
+
906
+ full = para.text
907
+ combined = before + after
908
+ split_at = None
909
+ if combined in full:
910
+ split_at = full.find(combined) + len(before)
911
+ else:
912
+ # Fallback: CR text may use a plain space/quote/dash where the TS uses
913
+ # a Unicode variant (NBSP, curly quote, en-dash, ...) β€” locate the
914
+ # splice point via normalized matching, then map back to the
915
+ # ORIGINAL (unnormalized) offset so the real TS characters are kept.
916
+ norm_full, idx_map = _normalize_with_map(full)
917
+ norm_combined = _apply_unicode_replacements(combined)
918
+ pos = norm_full.find(norm_combined)
919
+ if pos != -1:
920
+ norm_split = pos + len(_apply_unicode_replacements(before))
921
+ split_at = idx_map[norm_split] if norm_split < len(idx_map) else len(full)
922
+ conf = min(conf, _WARN_CONF - 0.1)
923
+
924
+ if split_at is None:
925
+ log.append(f" ERROR text_insert: splice anchor not found β€” "
926
+ f"before={before[-30:]!r} after={after[:30]!r}")
927
+ return False
928
+
929
+ try:
930
+ tracked_insert_text_in_para(para, before, after, ins_text, rev, author, date,
931
+ runs_xml=runs_xml, split_at=split_at)
932
+ except ValueError as e:
933
+ log.append(f" ERROR text_insert: {e}")
934
+ return False
935
+
936
+ _pfx = 'WARN' if conf < _WARN_CONF else 'OK '
937
+ log.append(f" {_pfx} text_insert (body_para conf={conf:.1f}): "
938
+ f"inserted {ins_text!r} between ...{before[-20:]!r} | {after[:20]!r}...")
939
+ return True
940
+
941
+
942
  def _apply_para_insert(doc, change, rev, author, date, log, last_para_inserted=None):
943
  loc = change['location']
944
  anchor_text = loc.get('anchor_text', '')
 
1111
  last_para_inserted=last_para_inserted)
1112
  elif ctype == 'text_replace':
1113
  ok = _apply_text_replace(doc, change, rev, author, date, log)
1114
+ elif ctype == 'text_insert':
1115
+ ok = _apply_text_insert(doc, change, rev, author, date, log)
1116
  elif ctype == 'para_insert':
1117
  ok = _apply_para_insert(doc, change, rev, author, date, log,
1118
  last_para_inserted=last_para_inserted)
scripts/verify_applied.py ADDED
@@ -0,0 +1,181 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """
3
+ verify_applied.py β€” Independent content-verification pass for applied CRs.
4
+
5
+ Two checks, both operating on data already produced by the existing
6
+ pipeline (cr_parser.parse_cr / ts_applicator.apply_manifest). Neither
7
+ changes parsing or applying behaviour β€” this is purely additive.
8
+
9
+ Check A β€” scan_revision_marks(): classifies every OOXML tracked-change
10
+ element in a CR DOCX (w:ins/w:del content marks, plus any other
11
+ CT_TrackChange-family element such as w:moveFrom/w:moveTo or
12
+ w:*PrChange/w:cellMerge that this pipeline has no apply-path
13
+ for), used to flag "CR has tracked changes but parser extracted
14
+ 0" and "CR has a revision-mark type nothing here understands".
15
+
16
+ Check B β€” verify_manifest_applied(): confirms each parsed change's new
17
+ text is actually present in the saved output TS.
18
+ """
19
+
20
+ import sys
21
+ from pathlib import Path
22
+
23
+ import docx
24
+ from docx.oxml.ns import qn
25
+ from lxml import etree
26
+
27
+ sys.path.insert(0, str(Path(__file__).parent))
28
+
29
+ from cr_parser import (
30
+ _is_rpr_ins,
31
+ _is_inserted_para,
32
+ _is_fully_inserted_tbl,
33
+ _is_skip_marker,
34
+ )
35
+ from ts_applicator import _norm, _norm_ws, _section_range
36
+
37
+ _MIN_SNIPPET_LEN = 4 # after whitespace-stripped normalisation
38
+
39
+
40
+ # ── Check A β€” generalized revision-mark scan ───────────────────────────────────
41
+
42
+ def scan_revision_marks(cr_docx_path):
43
+ """
44
+ Classify every OOXML tracked-change element in a CR DOCX.
45
+
46
+ Every element that extends CT_TrackChange (the OOXML base type for
47
+ revision marks) carries a w:author attribute β€” not just w:ins/w:del, but
48
+ also w:moveFrom/w:moveTo, w:rPrChange/w:pPrChange/w:tblPrChange/
49
+ w:tblGridChange/w:trPrChange/w:tcPrChange/w:sectPrChange, and
50
+ w:cellIns/w:cellDel/w:cellMerge. Walking generically by that attribute
51
+ (rather than hardcoding a w:ins/w:del tag lookup) means no revision-mark
52
+ type β€” including ones not named here β€” goes silently unnoticed.
53
+
54
+ Returns {'handled': int, 'unhandled': {tag_localname: count, ...}}.
55
+ - handled: w:ins/w:del content marks (excludes rPr-nested paragraph-
56
+ mark markers, same filter cr_parser._is_rpr_ins applies).
57
+ - unhandled: any other tracked-change element found. cr_parser.py /
58
+ ts_applicator.py have no code path for these β€” they will
59
+ NOT be reflected in the output TS.
60
+ """
61
+ doc = docx.Document(str(cr_docx_path))
62
+ body = doc.element.body
63
+ handled = 0
64
+ unhandled = {}
65
+ for elem in body.iter():
66
+ if elem.get(qn('w:author')) is None:
67
+ continue
68
+ local = etree.QName(elem).localname
69
+ if local in ('ins', 'del'):
70
+ if not _is_rpr_ins(elem):
71
+ handled += 1
72
+ else:
73
+ unhandled[local] = unhandled.get(local, 0) + 1
74
+ return {'handled': handled, 'unhandled': unhandled}
75
+
76
+
77
+ # ── Check B β€” content-presence check ──────────────────────────────────────────
78
+
79
+ def _section_replace_ins_snippets(elements_xml):
80
+ """From a section_replace's serialized elements, collect w:t text of the
81
+ ins-block elements only (skip del-block and separator elements)."""
82
+ snippets = []
83
+ for xml_str in elements_xml:
84
+ elem = etree.fromstring(xml_str)
85
+ tag = elem.tag.split('}')[-1] if '}' in elem.tag else elem.tag
86
+ if tag == 'p':
87
+ is_ins = _is_inserted_para(elem)
88
+ elif tag == 'tbl':
89
+ is_ins = _is_fully_inserted_tbl(elem)
90
+ else:
91
+ is_ins = False
92
+ if is_ins:
93
+ snippets.append(''.join(t.text or '' for t in elem.iter(qn('w:t'))))
94
+ return snippets
95
+
96
+
97
+ def _extract_snippets(change):
98
+ """Return the expected inserted-text snippet(s) for one manifest change."""
99
+ ctype = change.get('type')
100
+ if ctype == 'text_replace':
101
+ new = change.get('new', '')
102
+ return [new] if new else []
103
+ if ctype == 'text_insert':
104
+ text = change.get('text', '')
105
+ return [text] if text else []
106
+ if ctype == 'para_insert':
107
+ return [p.get('text', '') for p in change.get('paragraphs', [])]
108
+ if ctype == 'row_insert':
109
+ return [c.get('text', '') for c in change.get('cells', [])]
110
+ if ctype == 'section_replace':
111
+ return _section_replace_ins_snippets(change.get('elements_xml', []))
112
+ return []
113
+
114
+
115
+ def _section_scope_text(doc, section_number):
116
+ """Full w:t text (paragraphs + tables) of a section, or None if not found."""
117
+ start, end = _section_range(doc, section_number)
118
+ if start is None:
119
+ return None
120
+ paras = doc.paragraphs
121
+ start_elem = paras[start]._element
122
+ end_elem = paras[end]._element if end < len(paras) else None
123
+ parts = []
124
+ in_range = False
125
+ for child in doc.element.body:
126
+ if child is start_elem:
127
+ in_range = True
128
+ if in_range:
129
+ if end_elem is not None and child is end_elem:
130
+ break
131
+ parts.append(''.join(t.text or '' for t in child.iter(qn('w:t'))))
132
+ return ''.join(parts)
133
+
134
+
135
+ def _snippet_present(snippet, corpus):
136
+ if snippet in corpus:
137
+ return True
138
+ if _norm(snippet) in _norm(corpus):
139
+ return True
140
+ if _norm_ws(snippet) in _norm_ws(corpus):
141
+ return True
142
+ return False
143
+
144
+
145
+ def verify_manifest_applied(ts_doc, manifest):
146
+ """Confirm each change's expected new-text snippet(s) are present in ts_doc.
147
+
148
+ A declared section_number is a hint, not a hard boundary: the real
149
+ applicator (ts_applicator.py) falls back to a global search whenever an
150
+ anchor/table isn't found in the declared section (see the "using global
151
+ match" WARN lines it emits), so content can legitimately land outside
152
+ its declared section. To avoid flagging those legitimate placements,
153
+ a snippet is only reported missing if it's absent from BOTH the
154
+ section-scoped corpus (when available) and the whole document.
155
+
156
+ Returns a list of 'ERROR verify: ...' lines, one per missing snippet."""
157
+ errors = []
158
+ whole_doc_corpus = None
159
+
160
+ for change in manifest:
161
+ ctype = change.get('type')
162
+ cr_uid = change.get('_cr_uid', '?')
163
+ section_number = (change.get('location') or {}).get('section_number', '')
164
+
165
+ scope_corpus = _section_scope_text(ts_doc, section_number) if section_number else None
166
+
167
+ for snippet in _extract_snippets(change):
168
+ if not snippet or _is_skip_marker(snippet):
169
+ continue
170
+ if len(_norm_ws(snippet)) < _MIN_SNIPPET_LEN:
171
+ continue
172
+ if scope_corpus is not None and _snippet_present(snippet, scope_corpus):
173
+ continue
174
+ if whole_doc_corpus is None:
175
+ whole_doc_corpus = ''.join(t.text or '' for t in ts_doc.element.body.iter(qn('w:t')))
176
+ if not _snippet_present(snippet, whole_doc_corpus):
177
+ errors.append(
178
+ f'ERROR verify: [{cr_uid}] {ctype} β€” expected text not found '
179
+ f'in output TS: {snippet[:80]!r}'
180
+ )
181
+ return errors
skills/cr-visual-pair/SKILL.md ADDED
@@ -0,0 +1,315 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ name: cr-visual-pair
3
+ description: Renders each modified TS section/annex and its associated CR excerpt(s) to PNG and packages them into a review_package.zip for upload to the ApplyCRs app's "Verify insertions" tab. Does NOT modify pipeline code β€” export only. TRIGGER on "export review package", "prepare visual review", "pair CR and TS screenshots for the app", "generate review package", or when the user wants to upload CR/TS screenshots to the app for OK/Not OK review.
4
+ ---
5
+
6
+ # CR Visual Pair β€” Export a Review Package for the App's Verify Tab
7
+
8
+ Renders every modified **TS section/annex** from an ApplyCRs run into a screenshot, next
9
+ to the CR excerpt(s) that produced it, and bundles them into a `review_package.zip` the
10
+ user uploads to the ApplyCRs app's **"βœ… Verify insertions"** tab. There the user clicks
11
+ through each pair and marks OK / Not OK / Wrong match, then downloads a log.
12
+
13
+ **A pair is one TS section or annex β€” never a bundle of unrelated sections.** If several
14
+ CRs (or several change requests inside one CR) all touch the same section, they combine
15
+ into that section's single pair. If one CR touches several disjoint sections/annexes,
16
+ each gets its own pair with only that section's own CR excerpt β€” never the whole CR
17
+ dumped alongside every section it happens to touch.
18
+
19
+ This skill only **exports images** β€” it never edits `cr_parser.py`, `ts_applicator.py`, or
20
+ any pipeline code. It does *import* `cr_parser.parse_cr()` (read-only) to get the exact
21
+ section/change breakdown of each CR β€” see Step 1. For diagnosing and fixing pipeline
22
+ bugs, use `cr-visual-verify` instead.
23
+
24
+ Run this skill **locally on Windows** (it needs Word COM to convert DOCX β†’ PDF). The HF-hosted
25
+ app has no DOCX renderer, which is why this export step exists.
26
+
27
+ ---
28
+
29
+ ## Input
30
+
31
+ A **zip file** or **folder path** from an ApplyCRs run. Expected structure:
32
+
33
+ ```
34
+ <root>/
35
+ CRs/<uid>.docx
36
+ TS/<spec>/ts_<spec>_v<old>.docx
37
+ TS/<spec>/ts_<spec>_v<new>_was_v<old>.docx
38
+ TS/<spec>/ts_<spec>_v<new>_was_v<old>.log
39
+ ```
40
+
41
+ Windows paths: convert `C:\Users\...` to `/mnt/c/Users/...`.
42
+
43
+ You also need a local checkout of the **ApplyCRs repo** (for `scripts/cr_parser.py` and
44
+ this skill's `build_pairs.py`). Locate `REPO_ROOT` by checking, in order:
45
+ 1. Three levels up from this file's own location (`<this_file>/../../..`) β€” true when the
46
+ skill is used from inside an ApplyCRs checkout.
47
+ 2. Otherwise ask the user for the path to their ApplyCRs repo clone.
48
+
49
+ `REPO_ROOT/scripts/cr_parser.py` and `REPO_ROOT/skills/cr-visual-pair/build_pairs.py` must
50
+ both exist before continuing.
51
+
52
+ ## Output
53
+
54
+ `review_package.zip` (default location: alongside the input, or `/tmp/cr_visual_pair_<ts>/review_package.zip`):
55
+
56
+ ```
57
+ review_package.zip
58
+ manifest.json
59
+ pairs/001_ts_p12.png
60
+ pairs/001_cr_SETTEC_26_000050r1_p5.png
61
+ pairs/002_ts_p40.png
62
+ pairs/002_cr_SETTEC_26_000050r1_p3.png
63
+ pairs/002_cr_SETTEC_26_000051_p2.png
64
+ ...
65
+ ```
66
+
67
+ `manifest.json` schema β€” **one entry per TS section/annex**, not per CR:
68
+
69
+ ```json
70
+ {
71
+ "run": "run_1783411741",
72
+ "generated_at": "<ISO timestamp>",
73
+ "pairs": [
74
+ {
75
+ "id": 1,
76
+ "ts_spec": "102267",
77
+ "ts_version": "18.0.0 -> 18.1.0",
78
+ "section_number": "7.5",
79
+ "matched": true,
80
+ "ts_images": ["pairs/002_ts_p40.png"],
81
+ "changes": [
82
+ {
83
+ "cr_uid": "SETTEC(26)000050r1",
84
+ "change_type": "text_replace",
85
+ "matched": true,
86
+ "summary": "short description from the CR cover page",
87
+ "cr_images": ["pairs/002_cr_SETTEC_26_000050r1_p3.png"]
88
+ },
89
+ {
90
+ "cr_uid": "SETTEC(26)000051",
91
+ "change_type": "para_insert",
92
+ "matched": true,
93
+ "summary": "short description from the CR cover page",
94
+ "cr_images": ["pairs/002_cr_SETTEC_26_000051_p2.png"]
95
+ }
96
+ ]
97
+ }
98
+ ]
99
+ }
100
+ ```
101
+
102
+ - One pair per `(ts_spec, ts_version, section_number)`. All modification requests that
103
+ land inside that section β€” whether from one CR or several β€” are entries in `changes[]`
104
+ on that *same* pair. A different section number is always a *different* pair.
105
+ - `ts_images` covers every page that section occupies in the applied TS β€” one page for a
106
+ section that fits on one page, two (or more) consecutive pages for a section that
107
+ starts on one page and runs onto the next β€” never pages belonging to some other section.
108
+ - Pair-level `matched: false` means the section's page(s) couldn't be confidently located β€”
109
+ still include the best-guess TS image (or an empty list if none).
110
+ - Change-level `matched: false` means that specific change's own text couldn't be
111
+ confidently pinpointed within the section's page(s), even though the section itself was
112
+ found β€” the reviewer should double-check that one contributor.
113
+ - `id` is a stable 1-based sequence, assigned in document order (each pair's first `ts_pages`
114
+ entry ascending), used for the PNG filename prefix, zero-padded to 3 digits.
115
+
116
+ ---
117
+
118
+ ## Step 1 β€” Extract, enumerate, and parse each CR's changes
119
+
120
+ ```bash
121
+ WORK_DIR="/tmp/cr_visual_pair_$(date +%s)"
122
+ mkdir -p "$WORK_DIR"
123
+ unzip -o "<zip_path>" -d "$WORK_DIR" # skip if given a folder path directly
124
+ RUN_ROOT=$(find "$WORK_DIR" -name "CRs" -type d -printf '%h\n' | head -1)
125
+ ```
126
+
127
+ Enumerate `(TS log, cr_uid)` associations from every `TS/<spec>/ts_*_was_*.log` header:
128
+
129
+ ```
130
+ TS: 102 267 v18.0.0 -> v18.1.0
131
+ CRs: SETTEC(26)000050r1
132
+ ```
133
+
134
+ A log can list multiple CRs applied to the same TS β€” collect all of them per log. Skip
135
+ logs whose text says "No changes parsed" (nothing to review β€” no doc changed).
136
+
137
+ CR docx: `CRs/<uid>.docx`. Applied TS docx: same directory as the log,
138
+ `ts_<spec>_v<new>_was_v<old>.docx`.
139
+
140
+ For each **unique** `cr_uid` across the whole run, parse it once (cache by uid):
141
+
142
+ ```python
143
+ import sys
144
+ sys.path.insert(0, "<REPO_ROOT>/skills/cr-visual-pair")
145
+ from build_pairs import changes_for_cr
146
+
147
+ changes = changes_for_cr("<RUN_ROOT>/CRs/<uid>.docx", scripts_dir="<REPO_ROOT>/scripts")
148
+ # -> list of {change_type, section_number, cr_search_text, ts_search_text, raw}
149
+ ```
150
+
151
+ Also grab a one-line `summary` per `cr_uid` from its CR cover page (page 1) β€” same idea as
152
+ before, just for display context, not for matching:
153
+
154
+ ```bash
155
+ pdftotext -f 1 -l 1 "<cr>.pdf" - | head -40 # read 'Reason for change' / title
156
+ ```
157
+
158
+ ---
159
+
160
+ ## Step 2 β€” Screenshot pipeline (same technique as `cr-visual-verify`)
161
+
162
+ ### 2a. Copy files to Windows temp
163
+
164
+ ```bash
165
+ WIN_TEMP="/mnt/c/Users/$(ls /mnt/c/Users/ | grep -E '^[Tt][0-9]' | head -1)/Downloads/cr_visual_pair"
166
+ mkdir -p "$WIN_TEMP"
167
+ cp <docx_path> "$WIN_TEMP/<shortname>.docx"
168
+ ```
169
+
170
+ Copy **every** unique CR docx and **every** unique TS (applied) docx involved in the run β€”
171
+ each is converted once regardless of how many sections/CRs reference it.
172
+
173
+ ### 2b. Convert DOCX β†’ PDF via Word (PowerShell)
174
+
175
+ ```bash
176
+ powershell.exe -Command '
177
+ $dir = "<win_path>"
178
+ $word = New-Object -ComObject Word.Application
179
+ $word.Visible = $false
180
+ $files = @("<file1>", "<file2>")
181
+ foreach ($f in $files) {
182
+ $doc = $word.Documents.Open("$dir\$f.docx")
183
+ $doc.SaveAs([ref]"$dir\$f.pdf", [ref]17)
184
+ $doc.Close()
185
+ }
186
+ $word.Quit()
187
+ '
188
+ ```
189
+
190
+ ### 2c. Locate each change's CR page and TS page
191
+
192
+ For each TS doc, `pdftotext` it once and split into pages; do the same for every CR docx
193
+ involved:
194
+
195
+ ```bash
196
+ pdftotext "<ts>.pdf" /tmp/ts_pages.txt
197
+ pdftotext "<cr>.pdf" /tmp/cr_pages.txt
198
+ ```
199
+
200
+ ```python
201
+ import sys
202
+ sys.path.insert(0, "<REPO_ROOT>/skills/cr-visual-pair")
203
+ from build_pairs import locate_page
204
+
205
+ def split_pages(path):
206
+ return open(path, encoding="utf-8", errors="replace").read().split("\f")
207
+
208
+ cr_pages = split_pages("/tmp/cr_pages.txt")
209
+ ts_pages = split_pages("/tmp/ts_pages.txt")
210
+
211
+ cr_page, cr_confident = locate_page(cr_pages, search_text=ch["cr_search_text"],
212
+ section_number=ch["section_number"])
213
+ ts_page, ts_confident = locate_page(ts_pages, search_text=ch["ts_search_text"],
214
+ section_number=ch["section_number"])
215
+ ```
216
+
217
+ Do this for every change belonging to every `cr_uid` associated with this TS doc. Collect
218
+ one located-change record per change (see `group_by_section`'s expected shape in
219
+ `build_pairs.py`'s docstring): `ts_spec`, `ts_version`, `cr_uid`, `change_type`,
220
+ `section_number`, `summary`, `ts_page`, `ts_confident`, `cr_pages: [cr_page]`,
221
+ `cr_confident`.
222
+
223
+ If a change's search text isn't found and it has no `section_number` either, its
224
+ `ts_page` stays `None` β€” it will still surface as its own unmatched pair (see Step 2e),
225
+ never silently dropped.
226
+
227
+ ### 2d. Group into section-level pairs
228
+
229
+ ```python
230
+ from build_pairs import group_by_section
231
+
232
+ pairs = group_by_section(located_changes) # sorted by first ts page ascending
233
+ ```
234
+
235
+ Each pair's `ts_pages` is a list, not a single page β€” if a section's changes were
236
+ confidently located on both page 1 and page 2, `ts_pages` is `[1, 2]` and **both** must be
237
+ rendered and included in `ts_images` (see Step 2e). Don't collapse it back to one page.
238
+
239
+ Assign `id` 1, 2, 3, … in this order, per TS doc, in the order TS docs are encountered.
240
+
241
+ ### 2e. Render pages to PNG
242
+
243
+ Render only the pages actually referenced by the grouped pairs β€” dedupe by
244
+ `(doc_path, page_no)` so a page shared by two changes (same section, or coincidentally
245
+ adjacent) is never rendered twice:
246
+
247
+ ```bash
248
+ gs -dBATCH -dNOPAUSE -sDEVICE=png16m -r200 \
249
+ -dFirstPage=<N> -dLastPage=<N> \
250
+ -sOutputFile="<WORK_DIR>/pairs/<id>_ts_p<N>.png" "<ts>.pdf"
251
+
252
+ gs -dBATCH -dNOPAUSE -sDEVICE=png16m -r200 \
253
+ -dFirstPage=<N> -dLastPage=<N> \
254
+ -sOutputFile="<WORK_DIR>/pairs/<id>_cr_<sanitized_cr_uid>_p<N>.png" "<cr>.pdf"
255
+ ```
256
+
257
+ Sanitize `cr_uid` for the filename (e.g. `SETTEC(26)000050r1` β†’ `SETTEC_26_000050r1`) by
258
+ replacing anything that isn't `[A-Za-z0-9_-]` with `_`.
259
+
260
+ Zero-pad `<id>` to 3 digits (`001`, `002`, …). On the TS side, render **every** page in the
261
+ pair's `ts_pages` list from Step 2d (in ascending order) and add each one to `ts_images` β€”
262
+ this is how a section that starts on one page and ends on the next gets both pages shown,
263
+ not just the first. On the CR side, if a change's marked text visibly continues onto the
264
+ next page, render that page too and include it in `cr_images` β€” use judgment there, same
265
+ as the old skill's approach, just scoped to this one change/section instead of the whole CR.
266
+
267
+ ### 2f. Validate alignment before finalizing each pair
268
+
269
+ Use `Read` on the TS image and every contributing change's CR image(s). Confirm the
270
+ section heading/number visible on the TS screenshot matches the pair's `section_number`,
271
+ and that each change's identifying text is visible on its own CR screenshot. If a
272
+ specific change's text isn't visible where expected:
273
+ - Try the next candidate page from Step 2c for that change only.
274
+ - If nothing matches after reasonable attempts, keep the best-guess image and set that
275
+ change's `matched: false` β€” don't set the whole pair to `false` just because one of
276
+ several contributors is uncertain.
277
+
278
+ If the section's own page can't be found at all, mark the pair `matched: false` rather
279
+ than guessing and reporting success.
280
+
281
+ ---
282
+
283
+ ## Step 3 β€” Write manifest and zip
284
+
285
+ Write `<WORK_DIR>/manifest.json` per the schema above, then:
286
+
287
+ ```bash
288
+ cd "<WORK_DIR>" && zip -r review_package.zip manifest.json pairs/
289
+ ```
290
+
291
+ Report the final path to the user, e.g.:
292
+
293
+ ```
294
+ Review package ready: /tmp/cr_visual_pair_1783412000/review_package.zip
295
+ 9 pair(s) (sections), 12 change(s) across CRs, 7 fully matched, 2 low-confidence
296
+ Upload this zip to the app's "βœ… Verify insertions" tab.
297
+ ```
298
+
299
+ ---
300
+
301
+ ## Important rules
302
+
303
+ 1. **Export only β€” never modify pipeline code.** `build_pairs.py`'s `changes_for_cr()`
304
+ only *imports* `cr_parser.parse_cr()` β€” it never writes to `cr_parser.py`,
305
+ `ts_applicator.py`, or any CR/TS document. If you spot a pipeline bug while doing this,
306
+ mention it to the user and suggest running `cr-visual-verify` instead; don't fix it here.
307
+ 2. **One pair per section/annex, never a bundle of several.** A pair's CR side is scoped
308
+ to only the change(s) that land in that section β€” not the whole CR's every page.
309
+ 3. **Multiple CRs (or multiple changes) in the same section combine into one pair** β€”
310
+ that's the whole point of grouping by `section_number` instead of by CR uid or raw
311
+ page number.
312
+ 4. **Don't silently mispair** β€” if a section's TS page can't be confidently located, mark
313
+ `matched: false` instead of guessing and reporting success. Same at the per-change
314
+ level for CR-side matches.
315
+ 5. **`[...]` in a CR is normal** β€” it means omitted unchanged content, not a missing image.
skills/cr-visual-pair/build_pairs.py ADDED
@@ -0,0 +1,192 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ build_pairs.py β€” pure text/data helpers for cr-visual-pair.
3
+
4
+ Groups CR changes by the TS section/annex they land in (not by raw page
5
+ number, and not by CR uid), so a review "pair" is always exactly one
6
+ section β€” never a bundle of unrelated sections, and never split when
7
+ several CRs (or several changes within one CR) touch the same section.
8
+
9
+ Read-only consumer of scripts/cr_parser.py's parse_cr() β€” never imports
10
+ anything that writes back to CR/TS documents.
11
+
12
+ No Word/PDF/gs dependency: locate_page() and group_by_section() operate on
13
+ plain strings and dicts, so they can be exercised with synthetic data.
14
+ """
15
+
16
+ import re
17
+ import sys
18
+ from collections import Counter
19
+ from pathlib import Path
20
+
21
+ _WS_RE = re.compile(r'\s+')
22
+
23
+
24
+ def _normalize(text):
25
+ return _WS_RE.sub(' ', text.strip())
26
+
27
+
28
+ def locate_page(pages_text, search_text=None, section_number=None, min_len=6):
29
+ """
30
+ Find the 1-based page in `pages_text` (a list of page-text strings, in
31
+ document order) that corresponds to a change.
32
+
33
+ Tries, in order:
34
+ 1. Exact whitespace-normalized substring match on `search_text` (if
35
+ long enough to be distinctive) -> confident.
36
+ 2. A heading line starting with `section_number` -> confident.
37
+ 3. A bare substring match of `section_number` anywhere on the page
38
+ -> not confident (could be a cross-reference, not the heading).
39
+
40
+ Returns (page_no_or_None, confident_bool).
41
+ """
42
+ search_text = (search_text or '').strip()
43
+ if search_text and len(search_text) >= min_len:
44
+ needle = _normalize(search_text)
45
+ for i, page in enumerate(pages_text, start=1):
46
+ if needle in _normalize(page):
47
+ return i, True
48
+
49
+ if section_number:
50
+ heading_re = re.compile(r'(?m)^\s*' + re.escape(section_number) + r'(?:[ \t]|$)')
51
+ for i, page in enumerate(pages_text, start=1):
52
+ if heading_re.search(page):
53
+ return i, True
54
+ for i, page in enumerate(pages_text, start=1):
55
+ if section_number in page:
56
+ return i, False
57
+
58
+ return None, False
59
+
60
+
61
+ def group_by_section(located_changes):
62
+ """
63
+ Group already-located changes into review pairs, one per
64
+ (ts_spec, ts_version, section_number).
65
+
66
+ Each item in `located_changes` is expected to have:
67
+ ts_spec, ts_version, cr_uid, change_type, section_number (may be ''),
68
+ summary, ts_page (int|None), ts_confident (bool),
69
+ cr_pages (list[int]), cr_confident (bool).
70
+
71
+ Changes with no resolvable section_number fall back to grouping by
72
+ their located ts_page instead (each distinct page becomes its own
73
+ group), so unresolved changes still never get merged across unrelated
74
+ locations.
75
+
76
+ A section's changes can be located on more than one page (e.g. a clause
77
+ that starts on page 1 and runs onto page 2) β€” every confidently-located
78
+ page is kept, not just the majority one, so the section's full page
79
+ range is always represented.
80
+
81
+ Returns a list of pair dicts, sorted by the first page each group
82
+ resolved to (unresolved groups sort last, in first-seen order):
83
+ {ts_spec, ts_version, section_number, ts_pages, matched, changes: [...]}
84
+ """
85
+ groups = {}
86
+ order_hint = {}
87
+
88
+ for idx, ch in enumerate(located_changes):
89
+ section_number = ch.get('section_number') or ''
90
+ key_section = section_number if section_number else '__page_%r__' % (ch.get('ts_page'),)
91
+ key = (ch['ts_spec'], ch['ts_version'], key_section)
92
+
93
+ if key not in groups:
94
+ groups[key] = {
95
+ 'ts_spec': ch['ts_spec'],
96
+ 'ts_version': ch['ts_version'],
97
+ 'section_number': section_number,
98
+ 'ts_pages': [],
99
+ 'changes': [],
100
+ }
101
+ order_hint[key] = idx
102
+
103
+ g = groups[key]
104
+ if ch.get('ts_page') is not None:
105
+ g['ts_pages'].append((ch['ts_page'], bool(ch.get('ts_confident'))))
106
+ g['changes'].append({
107
+ 'cr_uid': ch['cr_uid'],
108
+ 'change_type': ch['change_type'],
109
+ 'summary': ch.get('summary', ''),
110
+ 'matched': bool(ch.get('cr_confident')) and bool(ch.get('ts_confident')),
111
+ 'cr_pages': list(ch.get('cr_pages', [])),
112
+ })
113
+
114
+ result = []
115
+ for key, g in groups.items():
116
+ pages = g['ts_pages']
117
+ confident_pages = sorted({p for p, c in pages if c})
118
+ if confident_pages:
119
+ # Span the full range between confidently-located pages, so a
120
+ # section that starts on one page and ends on the next always
121
+ # includes both β€” not just whichever page most changes landed on.
122
+ ts_pages = list(range(confident_pages[0], confident_pages[-1] + 1))
123
+ matched = True
124
+ elif pages:
125
+ counts = Counter(p for p, _ in pages)
126
+ best_page = max(counts.items(), key=lambda kv: (kv[1], -kv[0]))[0]
127
+ ts_pages = [best_page]
128
+ matched = False
129
+ else:
130
+ ts_pages = []
131
+ matched = False
132
+ result.append({
133
+ 'ts_spec': g['ts_spec'],
134
+ 'ts_version': g['ts_version'],
135
+ 'section_number': g['section_number'],
136
+ 'ts_pages': ts_pages,
137
+ 'matched': matched,
138
+ 'changes': g['changes'],
139
+ '_order_key': (ts_pages[0] if ts_pages else float('inf'), order_hint[key]),
140
+ })
141
+
142
+ result.sort(key=lambda r: r['_order_key'])
143
+ for r in result:
144
+ del r['_order_key']
145
+ return result
146
+
147
+
148
+ def changes_for_cr(cr_docx_path, scripts_dir):
149
+ """
150
+ Parse one CR docx via the pipeline's own cr_parser.parse_cr() (read-only
151
+ import β€” never modifies cr_parser.py) and return one entry per discrete
152
+ change, each carrying the text to search for on the CR side and (where
153
+ the change type makes it reliable) on the applied-TS side.
154
+ """
155
+ sys.path.insert(0, str(scripts_dir))
156
+ from cr_parser import parse_cr # noqa: PLC0415 β€” intentional lazy import
157
+
158
+ raw_changes = parse_cr(str(cr_docx_path))
159
+ out = []
160
+ for ch in raw_changes:
161
+ loc = ch.get('location', {})
162
+ ctype = ch['type']
163
+ section_number = loc.get('section_number') or ''
164
+
165
+ if ctype == 'text_replace':
166
+ cr_search_text = ch.get('old', '')
167
+ ts_search_text = ch.get('new', '')
168
+ elif ctype == 'section_replace':
169
+ cr_search_text = loc.get('del_heading') or loc.get('anchor_text') or ''
170
+ # Inserted text isn't separately extractable from elements_xml
171
+ # without re-parsing raw XML β€” rely on the section heading match
172
+ # instead (validated visually in the skill's step 2f).
173
+ ts_search_text = ''
174
+ elif ctype == 'para_insert':
175
+ first_text = next((p.get('text', '') for p in ch.get('paragraphs', []) if p.get('text')), '')
176
+ cr_search_text = first_text
177
+ ts_search_text = first_text
178
+ elif ctype == 'row_insert':
179
+ first_cell = next((c.get('text', '') for c in ch.get('cells', []) if c.get('text')), '')
180
+ cr_search_text = first_cell
181
+ ts_search_text = first_cell
182
+ else:
183
+ cr_search_text = ts_search_text = ''
184
+
185
+ out.append({
186
+ 'change_type': ctype,
187
+ 'section_number': section_number,
188
+ 'cr_search_text': cr_search_text,
189
+ 'ts_search_text': ts_search_text,
190
+ 'raw': ch,
191
+ })
192
+ return out
verify_review.py ADDED
@@ -0,0 +1,291 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Verify insertions tab β€” step through CR/TS screenshot pairs from a review
3
+ package (produced locally by the `cr-visual-pair` skill) and record OK /
4
+ Not OK / Wrong match verdicts. Verdicts live in st.session_state only; the
5
+ user downloads a text log at the end.
6
+ """
7
+
8
+ import base64
9
+ import io
10
+ import json
11
+ import mimetypes
12
+ import zipfile
13
+ from pathlib import Path
14
+
15
+ import streamlit as st
16
+
17
+ VERDICT_LABELS = {
18
+ "ok": "OK",
19
+ "not_ok": "NOT OK",
20
+ "wrong_match": "WRONG MATCH (page shown did not correspond)",
21
+ }
22
+
23
+ SKILL_PATH = Path(__file__).parent / "skills" / "cr-visual-pair" / "SKILL.md"
24
+
25
+ _ZOOM_CSS = """
26
+ <style>
27
+ .cr-zoom-thumb { cursor: zoom-in; display: block; }
28
+ .cr-zoom-thumb img { max-width: 100%; border-radius: 4px; display: block; }
29
+ .cr-zoom-overlay {
30
+ display: none;
31
+ position: fixed;
32
+ inset: 0;
33
+ background: rgba(0, 0, 0, 0.9);
34
+ z-index: 999999;
35
+ align-items: center;
36
+ justify-content: center;
37
+ }
38
+ .cr-zoom-overlay:target { display: flex; }
39
+ .cr-zoom-overlay img {
40
+ max-width: 95vw;
41
+ max-height: 95vh;
42
+ object-fit: contain;
43
+ cursor: zoom-out;
44
+ }
45
+ .cr-zoom-close {
46
+ position: fixed;
47
+ top: 16px;
48
+ right: 24px;
49
+ color: #fff;
50
+ font-size: 2.5rem;
51
+ text-decoration: none;
52
+ line-height: 1;
53
+ }
54
+ </style>
55
+ """
56
+
57
+
58
+ def _img_data_uri(path: Path) -> str:
59
+ mime = mimetypes.guess_type(path.name)[0] or "image/png"
60
+ data = base64.b64encode(path.read_bytes()).decode()
61
+ return f"data:{mime};base64,{data}"
62
+
63
+
64
+ def _render_zoomable_image(img_path: Path, anchor_id: str) -> None:
65
+ anchor_id = "".join(c if c.isalnum() else "-" for c in anchor_id)
66
+ data_uri = _img_data_uri(img_path)
67
+ st.markdown(
68
+ f"""
69
+ <a class="cr-zoom-thumb" href="#{anchor_id}">
70
+ <img src="{data_uri}">
71
+ </a>
72
+ <div class="cr-zoom-overlay" id="{anchor_id}">
73
+ <a class="cr-zoom-close" href="#">&times;</a>
74
+ <a href="#"><img src="{data_uri}"></a>
75
+ </div>
76
+ """,
77
+ unsafe_allow_html=True,
78
+ )
79
+
80
+
81
+ def _render_skill_card() -> None:
82
+ with st.expander("🟒 cr-visual-pair"):
83
+ if not SKILL_PATH.exists():
84
+ st.error(f"Skill file not found at {SKILL_PATH}")
85
+ return
86
+ content = SKILL_PATH.read_text()
87
+ st.caption(
88
+ "Run this skill locally (Windows + Word) to produce a `review_package.zip` "
89
+ "from an ApplyCRs run. Save it as `~/.claude/skills/cr-visual-pair/SKILL.md`, "
90
+ "or download it below."
91
+ )
92
+ st.download_button(
93
+ "⬇ Download SKILL.md",
94
+ data=content,
95
+ file_name="SKILL.md",
96
+ mime="text/markdown",
97
+ key="verify_skill_download",
98
+ )
99
+ st.code(content, language="markdown")
100
+
101
+
102
+ def _extract_package(uploaded, dest_dir: Path) -> dict | None:
103
+ dest_dir.mkdir(parents=True, exist_ok=True)
104
+ zip_bytes = uploaded.getbuffer()
105
+ with zipfile.ZipFile(io.BytesIO(zip_bytes)) as zf:
106
+ zf.extractall(dest_dir)
107
+ manifest_path = dest_dir / "manifest.json"
108
+ if not manifest_path.exists():
109
+ return None
110
+ return json.loads(manifest_path.read_text())
111
+
112
+
113
+ def _build_log(manifest: dict, verdicts: dict, review_dir: Path) -> str:
114
+ lines = []
115
+ for pair in manifest.get("pairs", []):
116
+ pid = pair["id"]
117
+ verdict = verdicts.get(pid)
118
+ verdict_label = VERDICT_LABELS.get(verdict, "UNREVIEWED")
119
+ ts_spec = pair.get("ts_spec", "?")
120
+ ts_version = pair.get("ts_version", "?")
121
+ section = pair.get("section_number", "?")
122
+ match_note = "" if pair.get("matched", True) else " [auto-flagged: low-confidence match]"
123
+ lines.append(
124
+ f"TS {ts_spec} v{ts_version} [section {section}] : {verdict_label}{match_note}"
125
+ )
126
+ for change in pair.get("changes", []):
127
+ change_note = "" if change.get("matched", True) else " [low-confidence match]"
128
+ lines.append(
129
+ f" - CR {change.get('cr_uid', '?')} ({change.get('change_type', '?')})"
130
+ f"{change_note}"
131
+ )
132
+ return "\n".join(lines) + "\n"
133
+
134
+
135
+ def render_verify_tab(session_dir_fn, sid: str) -> None:
136
+ st.subheader("Verify insertions")
137
+ st.caption(
138
+ "Upload a review package (generated locally with the `cr-visual-pair` skill) "
139
+ "and step through each CR ↔ TS screenshot pair to confirm the insertion is correct."
140
+ )
141
+
142
+ _render_skill_card()
143
+ st.markdown(_ZOOM_CSS, unsafe_allow_html=True)
144
+
145
+ review_dir = session_dir_fn(sid) / "review"
146
+ manifest_key = "verify_manifest"
147
+ review_dir_key = "verify_review_dir"
148
+
149
+ if manifest_key not in st.session_state:
150
+ uploaded = st.file_uploader(
151
+ "Upload review package (.zip)", type=["zip"], key="verify_upload"
152
+ )
153
+ if uploaded is not None:
154
+ manifest = _extract_package(uploaded, review_dir)
155
+ if manifest is None:
156
+ st.error("This zip does not contain a manifest.json β€” not a valid review package.")
157
+ return
158
+ st.session_state[manifest_key] = manifest
159
+ st.session_state[review_dir_key] = str(review_dir)
160
+ st.session_state["verify_idx"] = 0
161
+ st.session_state["verify_verdicts"] = {}
162
+ st.rerun()
163
+ return
164
+
165
+ manifest = st.session_state[manifest_key]
166
+ pairs = manifest.get("pairs", [])
167
+ review_dir = Path(st.session_state[review_dir_key])
168
+
169
+ if not pairs:
170
+ st.warning("This review package has no pairs to review.")
171
+ if st.button("Upload a different package"):
172
+ for k in (manifest_key, review_dir_key, "verify_idx", "verify_verdicts"):
173
+ st.session_state.pop(k, None)
174
+ st.rerun()
175
+ return
176
+
177
+ idx = st.session_state.setdefault("verify_idx", 0)
178
+ verdicts = st.session_state.setdefault("verify_verdicts", {})
179
+ idx = max(0, min(idx, len(pairs) - 1))
180
+ st.session_state["verify_idx"] = idx
181
+
182
+ pair = pairs[idx]
183
+ pid = pair["id"]
184
+
185
+ n_ok = sum(1 for v in verdicts.values() if v == "ok")
186
+ n_not_ok = sum(1 for v in verdicts.values() if v == "not_ok")
187
+ n_wrong = sum(1 for v in verdicts.values() if v == "wrong_match")
188
+ n_reviewed = len(verdicts)
189
+ st.caption(
190
+ f"Pair {idx + 1} / {len(pairs)} Β· "
191
+ f"Reviewed: {n_reviewed}/{len(pairs)} Β· βœ… {n_ok} ❌ {n_not_ok} ⚠ {n_wrong}"
192
+ )
193
+
194
+ st.markdown(
195
+ f"**TS** `{pair.get('ts_spec', '?')}` v{pair.get('ts_version', '?')} Β· "
196
+ f"section `{pair.get('section_number', '?')}`"
197
+ )
198
+ if not pair.get("matched", True):
199
+ st.warning(
200
+ "⚠ The pairing tool could not confidently locate this section's page in the "
201
+ "TS β€” double-check whether it's the right page before judging the insertion."
202
+ )
203
+
204
+ existing_verdict = verdicts.get(pid)
205
+ if existing_verdict:
206
+ st.info(f"Current verdict: **{VERDICT_LABELS[existing_verdict]}**")
207
+
208
+ col_cr, col_ts = st.columns(2)
209
+ with col_cr:
210
+ st.markdown("**CR (source change)**")
211
+ changes = pair.get("changes", [])
212
+ if changes:
213
+ for change in changes:
214
+ label = f"`{change.get('cr_uid', '?')}` Β· {change.get('change_type', '?')}"
215
+ if not change.get("matched", True):
216
+ label += " ⚠ low-confidence"
217
+ st.caption(label)
218
+ if change.get("summary"):
219
+ st.caption(change["summary"])
220
+ cr_images = change.get("cr_images", [])
221
+ if cr_images:
222
+ for img_idx, img in enumerate(cr_images):
223
+ img_path = review_dir / img
224
+ if img_path.exists():
225
+ _render_zoomable_image(
226
+ img_path, f"cr-zoom-{pid}-{change.get('cr_uid', '?')}-{img_idx}"
227
+ )
228
+ else:
229
+ st.error(f"Missing image: {img}")
230
+ else:
231
+ st.warning("No CR image for this change.")
232
+ else:
233
+ st.warning("No CR changes recorded for this pair.")
234
+ with col_ts:
235
+ st.markdown("**TS (applied output)**")
236
+ ts_images = pair.get("ts_images", [])
237
+ if ts_images:
238
+ for img_idx, img in enumerate(ts_images):
239
+ img_path = review_dir / img
240
+ if img_path.exists():
241
+ _render_zoomable_image(img_path, f"ts-zoom-{pid}-{img_idx}")
242
+ else:
243
+ st.error(f"Missing image: {img}")
244
+ else:
245
+ st.warning("No TS image for this pair β€” pairing tool did not find a candidate page.")
246
+
247
+ def _advance():
248
+ if st.session_state["verify_idx"] < len(pairs) - 1:
249
+ st.session_state["verify_idx"] += 1
250
+
251
+ col1, col2, col3, col4, col5 = st.columns(5)
252
+ with col1:
253
+ if st.button("βœ… OK", type="primary", key=f"ok_{pid}"):
254
+ verdicts[pid] = "ok"
255
+ _advance()
256
+ st.rerun()
257
+ with col2:
258
+ if st.button("❌ Not OK", key=f"not_ok_{pid}"):
259
+ verdicts[pid] = "not_ok"
260
+ _advance()
261
+ st.rerun()
262
+ with col3:
263
+ if st.button("⚠ Wrong match", key=f"wrong_{pid}"):
264
+ verdicts[pid] = "wrong_match"
265
+ _advance()
266
+ st.rerun()
267
+ with col4:
268
+ if st.button("← Prev", disabled=idx == 0, key=f"prev_{pid}"):
269
+ st.session_state["verify_idx"] -= 1
270
+ st.rerun()
271
+ with col5:
272
+ if st.button("Next β†’", disabled=idx == len(pairs) - 1, key=f"next_{pid}"):
273
+ st.session_state["verify_idx"] += 1
274
+ st.rerun()
275
+
276
+ if idx == len(pairs) - 1 and pid in verdicts:
277
+ st.success("Review complete β€” all pairs have a verdict. Download the log below.")
278
+
279
+ st.divider()
280
+ log_text = _build_log(manifest, verdicts, review_dir)
281
+ st.download_button(
282
+ "⬇ Download review log",
283
+ data=log_text,
284
+ file_name=f"cr_review_{sid[:8]}.log",
285
+ mime="text/plain",
286
+ )
287
+
288
+ if st.button("Upload a different package", key="verify_reset"):
289
+ for k in (manifest_key, review_dir_key, "verify_idx", "verify_verdicts"):
290
+ st.session_state.pop(k, None)
291
+ st.rerun()