flowmarg / core /orchestrator.py
akash
Deploy to Hugging Face (source code only, no binary files)
2379054
Raw History Blame Contribute Delete
16.2 kB
"""
End-to-end workflow: parse -> store -> calculate -> QA -> report -> VBA -> drawing review -> submission.
Sequential only (no LangGraph) so the pipeline always loads.
"""
from datetime import datetime
from pathlib import Path
from typing import Any, Dict
import pandas as pd
from core.xml_parser import parse_station_base, parse_geometry_report, detected_summary
from core.engineering import (
compute_track_displacements,
run_compliance_checks,
track_data_from_dict,
NR_LIMITS,
)
from core.database import (
insert_chainages,
insert_survey_features,
insert_drawing_metadata,
clear_project_data,
)
from core.excel_export import write_workbook
from core.drawing_reviewer import DrawingReviewer
from core.submission_checker import check_submission_readiness
from core.survey_processor import SurveyImporter
from agent.code_tools import generate_all_vba_scripts
def _safe_index(project_name: str):
try:
from core.chroma_project import index_project_data
index_project_data(project_name)
except Exception as exc:
return str(exc)
return None
def build_lift_slue_df(calculations: Dict, chainages, track_names: Dict) -> pd.DataFrame:
rows = []
for track_no, data in (calculations or {}).items():
name = track_names.get(track_no, f"Track {track_no}")
for ch in chainages:
rows.append(
{
"Track": name,
"Chainage": ch,
"Lift (mm)": data.get("lift", {}).get(ch),
"Slue (mm)": data.get("slue", {}).get(ch),
"Cant (mm)": data.get("pr_cant", {}).get(ch),
"Radius (m)": data.get("pr_radius", {}).get(ch),
"Source": "existing vs proposed XML",
"Standard clause": "NR/L2/TRK/2500 Section 4.2",
}
)
return pd.DataFrame(rows)
def build_alignment_comparison(ex_df: pd.DataFrame, pr_df: pd.DataFrame) -> pd.DataFrame:
if ex_df is None or pr_df is None or ex_df.empty or pr_df.empty:
return pd.DataFrame()
cols = [
c
for c in ["datum_chainage", "offset_alignment", "level_m", "first_offset_m", "offset_radius_m"]
if c in ex_df.columns
]
left = ex_df[cols].copy() if cols else ex_df.copy()
right = pr_df[cols].copy() if cols else pr_df.copy()
left = left.rename(columns={c: f"ex_{c}" for c in left.columns if c != "datum_chainage"})
right = right.rename(columns={c: f"pr_{c}" for c in right.columns if c != "datum_chainage"})
if "datum_chainage" not in left.columns or "datum_chainage" not in right.columns:
return pd.DataFrame()
merged = pd.merge(left, right, on="datum_chainage", how="outer")
merged["source"] = "alignment XML comparison"
merged["standard_clause"] = "NR/L2/TRK/2500"
return merged
class OrchestratorState:
def __init__(self):
self.project_name = ""
self.ex_xml = None
self.pr_xml = None
self.geom_xml = None
self.dxf_data = None
self.pdf_data = None
self.drawing_metadata = {}
self.track_category = "C"
self.design_speed = 75.0
self.ex_name = "existing.xml"
self.pr_name = "proposed.xml"
self.limits = dict(NR_LIMITS)
self.mapd = {}
self.track_names = {}
self.chainages = []
self.skip_clear = False
self.ex_df = pd.DataFrame()
self.pr_df = pd.DataFrame()
self.H_df = pd.DataFrame()
self.V_df = pd.DataFrame()
self.E_df = pd.DataFrame()
self.calculations = {}
self.qa_results = {}
self.vba_scripts = {}
self.drawing_issues = None
self.report_path = None
self.errors = []
self.step_results = {}
self.submission_status = None
class Orchestrator:
steps = (
"clear_project",
"parse_xml",
"store",
"import_survey",
"calculate",
"run_qa",
"generate_report",
"generate_vba",
"drawing_review",
"index_chroma",
"submission_check",
)
def __init__(self):
self.state = OrchestratorState()
def run_full_pipeline(
self,
ex_xml: bytes = None,
pr_xml: bytes = None,
geom_xml: bytes = None,
dxf_data: bytes = None,
pdf_data: bytes = None,
drawing_metadata: Dict = None,
project_name: str = "Project",
track_category: str = "C",
design_speed: float = 75.0,
ex_name: str = "existing.xml",
pr_name: str = "proposed.xml",
limits: Dict = None,
mapd: Dict = None,
track_names: Dict = None,
chainages=None,
skip_clear: bool = False,
**_kwargs,
) -> Dict:
self.state.project_name = project_name
self.state.ex_xml = ex_xml
self.state.pr_xml = pr_xml
self.state.geom_xml = geom_xml
self.state.dxf_data = dxf_data
self.state.pdf_data = pdf_data
self.state.drawing_metadata = drawing_metadata or {}
self.state.track_category = track_category
self.state.design_speed = design_speed
self.state.ex_name = ex_name
self.state.pr_name = pr_name
self.state.limits = dict(limits or NR_LIMITS)
self.state.mapd = mapd or {}
self.state.track_names = track_names or {}
self.state.chainages = list(chainages or [])
self.state.skip_clear = skip_clear
return self._run_sequential()
def _run_sequential(self):
results = {}
for step in self.steps:
method = getattr(self, f"_step_{step}")
try:
result = method()
results[step] = {"status": "success", "result": result}
self.state.step_results[step] = result
except Exception as e:
results[step] = {"status": "error", "error": str(e)}
self.state.errors.append(f"{step}: {e}")
status = "complete" if not self.state.errors else "partial"
return {
"status": status,
"results": results,
"errors": self.state.errors,
"report_path": self.state.report_path,
"vba_scripts": self.state.vba_scripts,
"drawing_issues": self.state.drawing_issues,
"submission": self.state.submission_status,
"calculations": self.state.calculations,
"qa_results": self.state.qa_results,
}
def _step_clear_project(self):
if self.state.skip_clear:
return {"cleared": False, "skipped": True}
clear_project_data()
return {"cleared": True}
def _step_parse_xml(self):
if self.state.ex_xml:
self.state.ex_df = parse_station_base(self.state.ex_xml)
if self.state.pr_xml:
self.state.pr_df = parse_station_base(self.state.pr_xml)
if self.state.geom_xml:
self.state.H_df, self.state.V_df, self.state.E_df = parse_geometry_report(self.state.geom_xml)
src = self.state.pr_df if not self.state.pr_df.empty else self.state.ex_df
if not src.empty and "datum_chainage" in src.columns and not self.state.chainages:
self.state.chainages = sorted(src["datum_chainage"].dropna().unique().tolist())
return {"existing_rows": len(self.state.ex_df), "proposed_rows": len(self.state.pr_df)}
def _step_store(self):
if not self.state.ex_df.empty:
insert_chainages(self.state.ex_df, "existing", source_file=self.state.ex_name)
if not self.state.pr_df.empty:
insert_chainages(self.state.pr_df, "proposed", source_file=self.state.pr_name)
return {"stored": True}
def _step_import_survey(self):
if not self.state.dxf_data:
return {"skipped": True}
survey = SurveyImporter().import_dxf(self.state.dxf_data)
if survey.get("status") == "success":
insert_survey_features(survey.get("features", []), source_file="survey.dxf")
return survey
def _step_calculate(self):
if self.state.ex_df.empty or self.state.pr_df.empty:
raise ValueError("Existing and proposed XML are required for calculations")
mapd = self.state.mapd or {
1: {
"ex_cl": None,
"ex_lh": None,
"ex_rh": None,
"pr_cl": None,
}
}
if not self.state.track_names:
self.state.track_names = {1: "Track 1"}
calculations = {}
for track_no, mapping in mapd.items():
track_data = {
"ex_left_level": {},
"ex_right_level": {},
"pr_left_level": {},
"pr_right_level": {},
"ex_offset": {},
"pr_offset": {},
"pr_radius": {},
}
for ch in self.state.chainages:
ex_rows = self.state.ex_df[self.state.ex_df["datum_chainage"].round(6) == round(ch, 6)]
pr_rows = self.state.pr_df[self.state.pr_df["datum_chainage"].round(6) == round(ch, 6)]
if mapping.get("ex_lh"):
ex_lh = ex_rows[ex_rows["offset_alignment"] == mapping["ex_lh"]]
else:
ex_lh = ex_rows
if mapping.get("ex_rh"):
ex_rh = ex_rows[ex_rows["offset_alignment"] == mapping["ex_rh"]]
else:
ex_rh = ex_rows
if mapping.get("ex_cl"):
ex_cl = ex_rows[ex_rows["offset_alignment"] == mapping["ex_cl"]]
else:
ex_cl = ex_rows
if mapping.get("pr_cl"):
pr_cl = pr_rows[pr_rows["offset_alignment"] == mapping["pr_cl"]]
else:
pr_cl = pr_rows
if not ex_lh.empty:
track_data["ex_left_level"][ch] = ex_lh.iloc[0].get("level_m")
if not ex_rh.empty:
track_data["ex_right_level"][ch] = ex_rh.iloc[0].get("level_m")
if not ex_cl.empty:
track_data["ex_offset"][ch] = ex_cl.iloc[0].get("first_offset_m")
if ch not in track_data["ex_left_level"]:
track_data["ex_left_level"][ch] = ex_cl.iloc[0].get("level_m")
if ch not in track_data["ex_right_level"]:
track_data["ex_right_level"][ch] = ex_cl.iloc[0].get("level_m")
if not pr_cl.empty:
track_data["pr_left_level"][ch] = pr_cl.iloc[0].get("level_m")
track_data["pr_right_level"][ch] = pr_cl.iloc[0].get("level_m")
track_data["pr_offset"][ch] = pr_cl.iloc[0].get("first_offset_m")
track_data["pr_radius"][ch] = pr_cl.iloc[0].get("offset_smoothing_radius_m") or pr_cl.iloc[0].get(
"offset_radius_m"
)
calculations[track_no] = compute_track_displacements(
track_data, track_no, self.state.chainages
).to_dict()
self.state.calculations = calculations
return {"tracks": len(calculations)}
def _step_run_qa(self):
qa_results = {}
for track_no, data in self.state.calculations.items():
td = track_data_from_dict(data, track_no, self.state.chainages)
qa_results[track_no] = run_compliance_checks(
td,
self.state.limits,
self.state.track_category,
self.state.design_speed,
self.state.track_names.get(track_no, f"Track {track_no}"),
)
self.state.qa_results = qa_results
return {"tracks_checked": len(qa_results)}
def _step_generate_report(self):
lift_df = build_lift_slue_df(self.state.calculations, self.state.chainages, self.state.track_names)
align_summary = pd.DataFrame()
if not self.state.ex_df.empty:
parts = [detected_summary(self.state.ex_df).assign(Source="Existing")]
if not self.state.pr_df.empty:
parts.append(detected_summary(self.state.pr_df).assign(Source="Proposed"))
align_summary = pd.concat(parts, ignore_index=True)
findings = []
summary = {}
if self.state.qa_results:
first = next(iter(self.state.qa_results.values()))
findings = first.get("findings", [])
summary = first.get("summary", {})
out_dir = Path(__file__).resolve().parent.parent / "data" / "exports"
out_dir.mkdir(parents=True, exist_ok=True)
path = out_dir / f"{self.state.project_name}_report.xlsx"
write_workbook(
path,
{"project_name": self.state.project_name, "date": datetime.now().isoformat()},
"Single Datum Alignment",
self.state.track_names or {1: "Track 1"},
self.state.calculations,
lift_df,
self.state.H_df,
self.state.V_df,
self.state.E_df,
align_summary,
pd.DataFrame(findings),
summary,
chainages=self.state.chainages,
timestamp=datetime.now().isoformat(),
)
self.state.report_path = str(path)
return {"report_path": str(path)}
def _step_generate_vba(self):
self.state.vba_scripts = generate_all_vba_scripts(
self.state.chainages, self.state.project_name, self.state.calculations
)
return {"scripts": list(self.state.vba_scripts.keys())}
def _step_drawing_review(self):
if not self.state.pdf_data:
return {"skipped": True}
if self.state.drawing_metadata:
insert_drawing_metadata(self.state.drawing_metadata)
self.state.drawing_issues = DrawingReviewer().review_pdf(
self.state.pdf_data, self.state.drawing_metadata
)
return {"issues": len(self.state.drawing_issues or [])}
def _step_index_chroma(self):
err = _safe_index(self.state.project_name)
if err:
return {"skipped": True, "error": err}
return {"indexed": True}
def _step_submission_check(self):
status = check_submission_readiness(
project_name=self.state.project_name,
ex_df=self.state.ex_df,
pr_df=self.state.pr_df,
calculations=self.state.calculations,
qa_results=self.state.qa_results,
drawing_issues=self.state.drawing_issues,
vba_script="\n".join(self.state.vba_scripts.values()) if self.state.vba_scripts else None,
)
self.state.submission_status = status
return status
def run_pipeline(**kwargs) -> Dict[str, Any]:
"""Compatibility wrapper used by core.__init__ and older callers."""
return Orchestrator().run_full_pipeline(**kwargs)
def run_orchestrator(session_state) -> Dict:
info = session_state.get("project_info", {}) if hasattr(session_state, "get") else {}
limits = session_state.get("limits", {}) if hasattr(session_state, "get") else {}
return Orchestrator().run_full_pipeline(
ex_xml=session_state.get("ex_xml_bytes"),
pr_xml=session_state.get("pr_xml_bytes"),
geom_xml=session_state.get("geom_xml_bytes"),
dxf_data=session_state.get("dxf_bytes") or session_state.get("dxf_data"),
pdf_data=session_state.get("pdf_bytes") or session_state.get("pdf_data"),
drawing_metadata=session_state.get("drawing_metadata"),
project_name=info.get("project_name", "Project"),
track_category=limits.get("track_category", "C"),
design_speed=limits.get("design_speed", 75),
limits=limits or None,
mapd=session_state.get("mapd"),
track_names=session_state.get("track_names"),
chainages=session_state.get("chainages"),
skip_clear=True,
ex_name=session_state.get("ex_name", "existing.xml"),
pr_name=session_state.get("pr_name", "proposed.xml"),
)