""" End-to-end workflow: parse -> store -> calculate -> QA -> report -> VBA -> drawing review -> submission. Sequential only (no LangGraph) so the pipeline always loads. """ from datetime import datetime from pathlib import Path from typing import Any, Dict import pandas as pd from core.xml_parser import parse_station_base, parse_geometry_report, detected_summary from core.engineering import ( compute_track_displacements, run_compliance_checks, track_data_from_dict, NR_LIMITS, ) from core.database import ( insert_chainages, insert_survey_features, insert_drawing_metadata, clear_project_data, ) from core.excel_export import write_workbook from core.drawing_reviewer import DrawingReviewer from core.submission_checker import check_submission_readiness from core.survey_processor import SurveyImporter from agent.code_tools import generate_all_vba_scripts def _safe_index(project_name: str): try: from core.chroma_project import index_project_data index_project_data(project_name) except Exception as exc: return str(exc) return None def build_lift_slue_df(calculations: Dict, chainages, track_names: Dict) -> pd.DataFrame: rows = [] for track_no, data in (calculations or {}).items(): name = track_names.get(track_no, f"Track {track_no}") for ch in chainages: rows.append( { "Track": name, "Chainage": ch, "Lift (mm)": data.get("lift", {}).get(ch), "Slue (mm)": data.get("slue", {}).get(ch), "Cant (mm)": data.get("pr_cant", {}).get(ch), "Radius (m)": data.get("pr_radius", {}).get(ch), "Source": "existing vs proposed XML", "Standard clause": "NR/L2/TRK/2500 Section 4.2", } ) return pd.DataFrame(rows) def build_alignment_comparison(ex_df: pd.DataFrame, pr_df: pd.DataFrame) -> pd.DataFrame: if ex_df is None or pr_df is None or ex_df.empty or pr_df.empty: return pd.DataFrame() cols = [ c for c in ["datum_chainage", "offset_alignment", "level_m", "first_offset_m", "offset_radius_m"] if c in ex_df.columns ] left = ex_df[cols].copy() if cols else ex_df.copy() right = pr_df[cols].copy() if cols else pr_df.copy() left = left.rename(columns={c: f"ex_{c}" for c in left.columns if c != "datum_chainage"}) right = right.rename(columns={c: f"pr_{c}" for c in right.columns if c != "datum_chainage"}) if "datum_chainage" not in left.columns or "datum_chainage" not in right.columns: return pd.DataFrame() merged = pd.merge(left, right, on="datum_chainage", how="outer") merged["source"] = "alignment XML comparison" merged["standard_clause"] = "NR/L2/TRK/2500" return merged class OrchestratorState: def __init__(self): self.project_name = "" self.ex_xml = None self.pr_xml = None self.geom_xml = None self.dxf_data = None self.pdf_data = None self.drawing_metadata = {} self.track_category = "C" self.design_speed = 75.0 self.ex_name = "existing.xml" self.pr_name = "proposed.xml" self.limits = dict(NR_LIMITS) self.mapd = {} self.track_names = {} self.chainages = [] self.skip_clear = False self.ex_df = pd.DataFrame() self.pr_df = pd.DataFrame() self.H_df = pd.DataFrame() self.V_df = pd.DataFrame() self.E_df = pd.DataFrame() self.calculations = {} self.qa_results = {} self.vba_scripts = {} self.drawing_issues = None self.report_path = None self.errors = [] self.step_results = {} self.submission_status = None class Orchestrator: steps = ( "clear_project", "parse_xml", "store", "import_survey", "calculate", "run_qa", "generate_report", "generate_vba", "drawing_review", "index_chroma", "submission_check", ) def __init__(self): self.state = OrchestratorState() def run_full_pipeline( self, ex_xml: bytes = None, pr_xml: bytes = None, geom_xml: bytes = None, dxf_data: bytes = None, pdf_data: bytes = None, drawing_metadata: Dict = None, project_name: str = "Project", track_category: str = "C", design_speed: float = 75.0, ex_name: str = "existing.xml", pr_name: str = "proposed.xml", limits: Dict = None, mapd: Dict = None, track_names: Dict = None, chainages=None, skip_clear: bool = False, **_kwargs, ) -> Dict: self.state.project_name = project_name self.state.ex_xml = ex_xml self.state.pr_xml = pr_xml self.state.geom_xml = geom_xml self.state.dxf_data = dxf_data self.state.pdf_data = pdf_data self.state.drawing_metadata = drawing_metadata or {} self.state.track_category = track_category self.state.design_speed = design_speed self.state.ex_name = ex_name self.state.pr_name = pr_name self.state.limits = dict(limits or NR_LIMITS) self.state.mapd = mapd or {} self.state.track_names = track_names or {} self.state.chainages = list(chainages or []) self.state.skip_clear = skip_clear return self._run_sequential() def _run_sequential(self): results = {} for step in self.steps: method = getattr(self, f"_step_{step}") try: result = method() results[step] = {"status": "success", "result": result} self.state.step_results[step] = result except Exception as e: results[step] = {"status": "error", "error": str(e)} self.state.errors.append(f"{step}: {e}") status = "complete" if not self.state.errors else "partial" return { "status": status, "results": results, "errors": self.state.errors, "report_path": self.state.report_path, "vba_scripts": self.state.vba_scripts, "drawing_issues": self.state.drawing_issues, "submission": self.state.submission_status, "calculations": self.state.calculations, "qa_results": self.state.qa_results, } def _step_clear_project(self): if self.state.skip_clear: return {"cleared": False, "skipped": True} clear_project_data() return {"cleared": True} def _step_parse_xml(self): if self.state.ex_xml: self.state.ex_df = parse_station_base(self.state.ex_xml) if self.state.pr_xml: self.state.pr_df = parse_station_base(self.state.pr_xml) if self.state.geom_xml: self.state.H_df, self.state.V_df, self.state.E_df = parse_geometry_report(self.state.geom_xml) src = self.state.pr_df if not self.state.pr_df.empty else self.state.ex_df if not src.empty and "datum_chainage" in src.columns and not self.state.chainages: self.state.chainages = sorted(src["datum_chainage"].dropna().unique().tolist()) return {"existing_rows": len(self.state.ex_df), "proposed_rows": len(self.state.pr_df)} def _step_store(self): if not self.state.ex_df.empty: insert_chainages(self.state.ex_df, "existing", source_file=self.state.ex_name) if not self.state.pr_df.empty: insert_chainages(self.state.pr_df, "proposed", source_file=self.state.pr_name) return {"stored": True} def _step_import_survey(self): if not self.state.dxf_data: return {"skipped": True} survey = SurveyImporter().import_dxf(self.state.dxf_data) if survey.get("status") == "success": insert_survey_features(survey.get("features", []), source_file="survey.dxf") return survey def _step_calculate(self): if self.state.ex_df.empty or self.state.pr_df.empty: raise ValueError("Existing and proposed XML are required for calculations") mapd = self.state.mapd or { 1: { "ex_cl": None, "ex_lh": None, "ex_rh": None, "pr_cl": None, } } if not self.state.track_names: self.state.track_names = {1: "Track 1"} calculations = {} for track_no, mapping in mapd.items(): track_data = { "ex_left_level": {}, "ex_right_level": {}, "pr_left_level": {}, "pr_right_level": {}, "ex_offset": {}, "pr_offset": {}, "pr_radius": {}, } for ch in self.state.chainages: ex_rows = self.state.ex_df[self.state.ex_df["datum_chainage"].round(6) == round(ch, 6)] pr_rows = self.state.pr_df[self.state.pr_df["datum_chainage"].round(6) == round(ch, 6)] if mapping.get("ex_lh"): ex_lh = ex_rows[ex_rows["offset_alignment"] == mapping["ex_lh"]] else: ex_lh = ex_rows if mapping.get("ex_rh"): ex_rh = ex_rows[ex_rows["offset_alignment"] == mapping["ex_rh"]] else: ex_rh = ex_rows if mapping.get("ex_cl"): ex_cl = ex_rows[ex_rows["offset_alignment"] == mapping["ex_cl"]] else: ex_cl = ex_rows if mapping.get("pr_cl"): pr_cl = pr_rows[pr_rows["offset_alignment"] == mapping["pr_cl"]] else: pr_cl = pr_rows if not ex_lh.empty: track_data["ex_left_level"][ch] = ex_lh.iloc[0].get("level_m") if not ex_rh.empty: track_data["ex_right_level"][ch] = ex_rh.iloc[0].get("level_m") if not ex_cl.empty: track_data["ex_offset"][ch] = ex_cl.iloc[0].get("first_offset_m") if ch not in track_data["ex_left_level"]: track_data["ex_left_level"][ch] = ex_cl.iloc[0].get("level_m") if ch not in track_data["ex_right_level"]: track_data["ex_right_level"][ch] = ex_cl.iloc[0].get("level_m") if not pr_cl.empty: track_data["pr_left_level"][ch] = pr_cl.iloc[0].get("level_m") track_data["pr_right_level"][ch] = pr_cl.iloc[0].get("level_m") track_data["pr_offset"][ch] = pr_cl.iloc[0].get("first_offset_m") track_data["pr_radius"][ch] = pr_cl.iloc[0].get("offset_smoothing_radius_m") or pr_cl.iloc[0].get( "offset_radius_m" ) calculations[track_no] = compute_track_displacements( track_data, track_no, self.state.chainages ).to_dict() self.state.calculations = calculations return {"tracks": len(calculations)} def _step_run_qa(self): qa_results = {} for track_no, data in self.state.calculations.items(): td = track_data_from_dict(data, track_no, self.state.chainages) qa_results[track_no] = run_compliance_checks( td, self.state.limits, self.state.track_category, self.state.design_speed, self.state.track_names.get(track_no, f"Track {track_no}"), ) self.state.qa_results = qa_results return {"tracks_checked": len(qa_results)} def _step_generate_report(self): lift_df = build_lift_slue_df(self.state.calculations, self.state.chainages, self.state.track_names) align_summary = pd.DataFrame() if not self.state.ex_df.empty: parts = [detected_summary(self.state.ex_df).assign(Source="Existing")] if not self.state.pr_df.empty: parts.append(detected_summary(self.state.pr_df).assign(Source="Proposed")) align_summary = pd.concat(parts, ignore_index=True) findings = [] summary = {} if self.state.qa_results: first = next(iter(self.state.qa_results.values())) findings = first.get("findings", []) summary = first.get("summary", {}) out_dir = Path(__file__).resolve().parent.parent / "data" / "exports" out_dir.mkdir(parents=True, exist_ok=True) path = out_dir / f"{self.state.project_name}_report.xlsx" write_workbook( path, {"project_name": self.state.project_name, "date": datetime.now().isoformat()}, "Single Datum Alignment", self.state.track_names or {1: "Track 1"}, self.state.calculations, lift_df, self.state.H_df, self.state.V_df, self.state.E_df, align_summary, pd.DataFrame(findings), summary, chainages=self.state.chainages, timestamp=datetime.now().isoformat(), ) self.state.report_path = str(path) return {"report_path": str(path)} def _step_generate_vba(self): self.state.vba_scripts = generate_all_vba_scripts( self.state.chainages, self.state.project_name, self.state.calculations ) return {"scripts": list(self.state.vba_scripts.keys())} def _step_drawing_review(self): if not self.state.pdf_data: return {"skipped": True} if self.state.drawing_metadata: insert_drawing_metadata(self.state.drawing_metadata) self.state.drawing_issues = DrawingReviewer().review_pdf( self.state.pdf_data, self.state.drawing_metadata ) return {"issues": len(self.state.drawing_issues or [])} def _step_index_chroma(self): err = _safe_index(self.state.project_name) if err: return {"skipped": True, "error": err} return {"indexed": True} def _step_submission_check(self): status = check_submission_readiness( project_name=self.state.project_name, ex_df=self.state.ex_df, pr_df=self.state.pr_df, calculations=self.state.calculations, qa_results=self.state.qa_results, drawing_issues=self.state.drawing_issues, vba_script="\n".join(self.state.vba_scripts.values()) if self.state.vba_scripts else None, ) self.state.submission_status = status return status def run_pipeline(**kwargs) -> Dict[str, Any]: """Compatibility wrapper used by core.__init__ and older callers.""" return Orchestrator().run_full_pipeline(**kwargs) def run_orchestrator(session_state) -> Dict: info = session_state.get("project_info", {}) if hasattr(session_state, "get") else {} limits = session_state.get("limits", {}) if hasattr(session_state, "get") else {} return Orchestrator().run_full_pipeline( ex_xml=session_state.get("ex_xml_bytes"), pr_xml=session_state.get("pr_xml_bytes"), geom_xml=session_state.get("geom_xml_bytes"), dxf_data=session_state.get("dxf_bytes") or session_state.get("dxf_data"), pdf_data=session_state.get("pdf_bytes") or session_state.get("pdf_data"), drawing_metadata=session_state.get("drawing_metadata"), project_name=info.get("project_name", "Project"), track_category=limits.get("track_category", "C"), design_speed=limits.get("design_speed", 75), limits=limits or None, mapd=session_state.get("mapd"), track_names=session_state.get("track_names"), chainages=session_state.get("chainages"), skip_clear=True, ex_name=session_state.get("ex_name", "existing.xml"), pr_name=session_state.get("pr_name", "proposed.xml"), )