Download core/orchestrator.py from Surajnitp/flowmarg: direct link, hf CLI and curl.
- Browser
- Download file 16.2 kB
-
https://huggingface.co/spaces/Surajnitp/flowmarg/resolve/main/core/orchestrator.py
- Command line
-
hf download hf://spaces/Surajnitp/flowmarg/core/orchestrator.py
-
curl -L -o orchestrator.py https://huggingface.co/spaces/Surajnitp/flowmarg/resolve/main/core/orchestrator.py
16.2 kB
| """ | |
| End-to-end workflow: parse -> store -> calculate -> QA -> report -> VBA -> drawing review -> submission. | |
| Sequential only (no LangGraph) so the pipeline always loads. | |
| """ | |
| from datetime import datetime | |
| from pathlib import Path | |
| from typing import Any, Dict | |
| import pandas as pd | |
| from core.xml_parser import parse_station_base, parse_geometry_report, detected_summary | |
| from core.engineering import ( | |
| compute_track_displacements, | |
| run_compliance_checks, | |
| track_data_from_dict, | |
| NR_LIMITS, | |
| ) | |
| from core.database import ( | |
| insert_chainages, | |
| insert_survey_features, | |
| insert_drawing_metadata, | |
| clear_project_data, | |
| ) | |
| from core.excel_export import write_workbook | |
| from core.drawing_reviewer import DrawingReviewer | |
| from core.submission_checker import check_submission_readiness | |
| from core.survey_processor import SurveyImporter | |
| from agent.code_tools import generate_all_vba_scripts | |
| def _safe_index(project_name: str): | |
| try: | |
| from core.chroma_project import index_project_data | |
| index_project_data(project_name) | |
| except Exception as exc: | |
| return str(exc) | |
| return None | |
| def build_lift_slue_df(calculations: Dict, chainages, track_names: Dict) -> pd.DataFrame: | |
| rows = [] | |
| for track_no, data in (calculations or {}).items(): | |
| name = track_names.get(track_no, f"Track {track_no}") | |
| for ch in chainages: | |
| rows.append( | |
| { | |
| "Track": name, | |
| "Chainage": ch, | |
| "Lift (mm)": data.get("lift", {}).get(ch), | |
| "Slue (mm)": data.get("slue", {}).get(ch), | |
| "Cant (mm)": data.get("pr_cant", {}).get(ch), | |
| "Radius (m)": data.get("pr_radius", {}).get(ch), | |
| "Source": "existing vs proposed XML", | |
| "Standard clause": "NR/L2/TRK/2500 Section 4.2", | |
| } | |
| ) | |
| return pd.DataFrame(rows) | |
| def build_alignment_comparison(ex_df: pd.DataFrame, pr_df: pd.DataFrame) -> pd.DataFrame: | |
| if ex_df is None or pr_df is None or ex_df.empty or pr_df.empty: | |
| return pd.DataFrame() | |
| cols = [ | |
| c | |
| for c in ["datum_chainage", "offset_alignment", "level_m", "first_offset_m", "offset_radius_m"] | |
| if c in ex_df.columns | |
| ] | |
| left = ex_df[cols].copy() if cols else ex_df.copy() | |
| right = pr_df[cols].copy() if cols else pr_df.copy() | |
| left = left.rename(columns={c: f"ex_{c}" for c in left.columns if c != "datum_chainage"}) | |
| right = right.rename(columns={c: f"pr_{c}" for c in right.columns if c != "datum_chainage"}) | |
| if "datum_chainage" not in left.columns or "datum_chainage" not in right.columns: | |
| return pd.DataFrame() | |
| merged = pd.merge(left, right, on="datum_chainage", how="outer") | |
| merged["source"] = "alignment XML comparison" | |
| merged["standard_clause"] = "NR/L2/TRK/2500" | |
| return merged | |
| class OrchestratorState: | |
| def __init__(self): | |
| self.project_name = "" | |
| self.ex_xml = None | |
| self.pr_xml = None | |
| self.geom_xml = None | |
| self.dxf_data = None | |
| self.pdf_data = None | |
| self.drawing_metadata = {} | |
| self.track_category = "C" | |
| self.design_speed = 75.0 | |
| self.ex_name = "existing.xml" | |
| self.pr_name = "proposed.xml" | |
| self.limits = dict(NR_LIMITS) | |
| self.mapd = {} | |
| self.track_names = {} | |
| self.chainages = [] | |
| self.skip_clear = False | |
| self.ex_df = pd.DataFrame() | |
| self.pr_df = pd.DataFrame() | |
| self.H_df = pd.DataFrame() | |
| self.V_df = pd.DataFrame() | |
| self.E_df = pd.DataFrame() | |
| self.calculations = {} | |
| self.qa_results = {} | |
| self.vba_scripts = {} | |
| self.drawing_issues = None | |
| self.report_path = None | |
| self.errors = [] | |
| self.step_results = {} | |
| self.submission_status = None | |
| class Orchestrator: | |
| steps = ( | |
| "clear_project", | |
| "parse_xml", | |
| "store", | |
| "import_survey", | |
| "calculate", | |
| "run_qa", | |
| "generate_report", | |
| "generate_vba", | |
| "drawing_review", | |
| "index_chroma", | |
| "submission_check", | |
| ) | |
| def __init__(self): | |
| self.state = OrchestratorState() | |
| def run_full_pipeline( | |
| self, | |
| ex_xml: bytes = None, | |
| pr_xml: bytes = None, | |
| geom_xml: bytes = None, | |
| dxf_data: bytes = None, | |
| pdf_data: bytes = None, | |
| drawing_metadata: Dict = None, | |
| project_name: str = "Project", | |
| track_category: str = "C", | |
| design_speed: float = 75.0, | |
| ex_name: str = "existing.xml", | |
| pr_name: str = "proposed.xml", | |
| limits: Dict = None, | |
| mapd: Dict = None, | |
| track_names: Dict = None, | |
| chainages=None, | |
| skip_clear: bool = False, | |
| **_kwargs, | |
| ) -> Dict: | |
| self.state.project_name = project_name | |
| self.state.ex_xml = ex_xml | |
| self.state.pr_xml = pr_xml | |
| self.state.geom_xml = geom_xml | |
| self.state.dxf_data = dxf_data | |
| self.state.pdf_data = pdf_data | |
| self.state.drawing_metadata = drawing_metadata or {} | |
| self.state.track_category = track_category | |
| self.state.design_speed = design_speed | |
| self.state.ex_name = ex_name | |
| self.state.pr_name = pr_name | |
| self.state.limits = dict(limits or NR_LIMITS) | |
| self.state.mapd = mapd or {} | |
| self.state.track_names = track_names or {} | |
| self.state.chainages = list(chainages or []) | |
| self.state.skip_clear = skip_clear | |
| return self._run_sequential() | |
| def _run_sequential(self): | |
| results = {} | |
| for step in self.steps: | |
| method = getattr(self, f"_step_{step}") | |
| try: | |
| result = method() | |
| results[step] = {"status": "success", "result": result} | |
| self.state.step_results[step] = result | |
| except Exception as e: | |
| results[step] = {"status": "error", "error": str(e)} | |
| self.state.errors.append(f"{step}: {e}") | |
| status = "complete" if not self.state.errors else "partial" | |
| return { | |
| "status": status, | |
| "results": results, | |
| "errors": self.state.errors, | |
| "report_path": self.state.report_path, | |
| "vba_scripts": self.state.vba_scripts, | |
| "drawing_issues": self.state.drawing_issues, | |
| "submission": self.state.submission_status, | |
| "calculations": self.state.calculations, | |
| "qa_results": self.state.qa_results, | |
| } | |
| def _step_clear_project(self): | |
| if self.state.skip_clear: | |
| return {"cleared": False, "skipped": True} | |
| clear_project_data() | |
| return {"cleared": True} | |
| def _step_parse_xml(self): | |
| if self.state.ex_xml: | |
| self.state.ex_df = parse_station_base(self.state.ex_xml) | |
| if self.state.pr_xml: | |
| self.state.pr_df = parse_station_base(self.state.pr_xml) | |
| if self.state.geom_xml: | |
| self.state.H_df, self.state.V_df, self.state.E_df = parse_geometry_report(self.state.geom_xml) | |
| src = self.state.pr_df if not self.state.pr_df.empty else self.state.ex_df | |
| if not src.empty and "datum_chainage" in src.columns and not self.state.chainages: | |
| self.state.chainages = sorted(src["datum_chainage"].dropna().unique().tolist()) | |
| return {"existing_rows": len(self.state.ex_df), "proposed_rows": len(self.state.pr_df)} | |
| def _step_store(self): | |
| if not self.state.ex_df.empty: | |
| insert_chainages(self.state.ex_df, "existing", source_file=self.state.ex_name) | |
| if not self.state.pr_df.empty: | |
| insert_chainages(self.state.pr_df, "proposed", source_file=self.state.pr_name) | |
| return {"stored": True} | |
| def _step_import_survey(self): | |
| if not self.state.dxf_data: | |
| return {"skipped": True} | |
| survey = SurveyImporter().import_dxf(self.state.dxf_data) | |
| if survey.get("status") == "success": | |
| insert_survey_features(survey.get("features", []), source_file="survey.dxf") | |
| return survey | |
| def _step_calculate(self): | |
| if self.state.ex_df.empty or self.state.pr_df.empty: | |
| raise ValueError("Existing and proposed XML are required for calculations") | |
| mapd = self.state.mapd or { | |
| 1: { | |
| "ex_cl": None, | |
| "ex_lh": None, | |
| "ex_rh": None, | |
| "pr_cl": None, | |
| } | |
| } | |
| if not self.state.track_names: | |
| self.state.track_names = {1: "Track 1"} | |
| calculations = {} | |
| for track_no, mapping in mapd.items(): | |
| track_data = { | |
| "ex_left_level": {}, | |
| "ex_right_level": {}, | |
| "pr_left_level": {}, | |
| "pr_right_level": {}, | |
| "ex_offset": {}, | |
| "pr_offset": {}, | |
| "pr_radius": {}, | |
| } | |
| for ch in self.state.chainages: | |
| ex_rows = self.state.ex_df[self.state.ex_df["datum_chainage"].round(6) == round(ch, 6)] | |
| pr_rows = self.state.pr_df[self.state.pr_df["datum_chainage"].round(6) == round(ch, 6)] | |
| if mapping.get("ex_lh"): | |
| ex_lh = ex_rows[ex_rows["offset_alignment"] == mapping["ex_lh"]] | |
| else: | |
| ex_lh = ex_rows | |
| if mapping.get("ex_rh"): | |
| ex_rh = ex_rows[ex_rows["offset_alignment"] == mapping["ex_rh"]] | |
| else: | |
| ex_rh = ex_rows | |
| if mapping.get("ex_cl"): | |
| ex_cl = ex_rows[ex_rows["offset_alignment"] == mapping["ex_cl"]] | |
| else: | |
| ex_cl = ex_rows | |
| if mapping.get("pr_cl"): | |
| pr_cl = pr_rows[pr_rows["offset_alignment"] == mapping["pr_cl"]] | |
| else: | |
| pr_cl = pr_rows | |
| if not ex_lh.empty: | |
| track_data["ex_left_level"][ch] = ex_lh.iloc[0].get("level_m") | |
| if not ex_rh.empty: | |
| track_data["ex_right_level"][ch] = ex_rh.iloc[0].get("level_m") | |
| if not ex_cl.empty: | |
| track_data["ex_offset"][ch] = ex_cl.iloc[0].get("first_offset_m") | |
| if ch not in track_data["ex_left_level"]: | |
| track_data["ex_left_level"][ch] = ex_cl.iloc[0].get("level_m") | |
| if ch not in track_data["ex_right_level"]: | |
| track_data["ex_right_level"][ch] = ex_cl.iloc[0].get("level_m") | |
| if not pr_cl.empty: | |
| track_data["pr_left_level"][ch] = pr_cl.iloc[0].get("level_m") | |
| track_data["pr_right_level"][ch] = pr_cl.iloc[0].get("level_m") | |
| track_data["pr_offset"][ch] = pr_cl.iloc[0].get("first_offset_m") | |
| track_data["pr_radius"][ch] = pr_cl.iloc[0].get("offset_smoothing_radius_m") or pr_cl.iloc[0].get( | |
| "offset_radius_m" | |
| ) | |
| calculations[track_no] = compute_track_displacements( | |
| track_data, track_no, self.state.chainages | |
| ).to_dict() | |
| self.state.calculations = calculations | |
| return {"tracks": len(calculations)} | |
| def _step_run_qa(self): | |
| qa_results = {} | |
| for track_no, data in self.state.calculations.items(): | |
| td = track_data_from_dict(data, track_no, self.state.chainages) | |
| qa_results[track_no] = run_compliance_checks( | |
| td, | |
| self.state.limits, | |
| self.state.track_category, | |
| self.state.design_speed, | |
| self.state.track_names.get(track_no, f"Track {track_no}"), | |
| ) | |
| self.state.qa_results = qa_results | |
| return {"tracks_checked": len(qa_results)} | |
| def _step_generate_report(self): | |
| lift_df = build_lift_slue_df(self.state.calculations, self.state.chainages, self.state.track_names) | |
| align_summary = pd.DataFrame() | |
| if not self.state.ex_df.empty: | |
| parts = [detected_summary(self.state.ex_df).assign(Source="Existing")] | |
| if not self.state.pr_df.empty: | |
| parts.append(detected_summary(self.state.pr_df).assign(Source="Proposed")) | |
| align_summary = pd.concat(parts, ignore_index=True) | |
| findings = [] | |
| summary = {} | |
| if self.state.qa_results: | |
| first = next(iter(self.state.qa_results.values())) | |
| findings = first.get("findings", []) | |
| summary = first.get("summary", {}) | |
| out_dir = Path(__file__).resolve().parent.parent / "data" / "exports" | |
| out_dir.mkdir(parents=True, exist_ok=True) | |
| path = out_dir / f"{self.state.project_name}_report.xlsx" | |
| write_workbook( | |
| path, | |
| {"project_name": self.state.project_name, "date": datetime.now().isoformat()}, | |
| "Single Datum Alignment", | |
| self.state.track_names or {1: "Track 1"}, | |
| self.state.calculations, | |
| lift_df, | |
| self.state.H_df, | |
| self.state.V_df, | |
| self.state.E_df, | |
| align_summary, | |
| pd.DataFrame(findings), | |
| summary, | |
| chainages=self.state.chainages, | |
| timestamp=datetime.now().isoformat(), | |
| ) | |
| self.state.report_path = str(path) | |
| return {"report_path": str(path)} | |
| def _step_generate_vba(self): | |
| self.state.vba_scripts = generate_all_vba_scripts( | |
| self.state.chainages, self.state.project_name, self.state.calculations | |
| ) | |
| return {"scripts": list(self.state.vba_scripts.keys())} | |
| def _step_drawing_review(self): | |
| if not self.state.pdf_data: | |
| return {"skipped": True} | |
| if self.state.drawing_metadata: | |
| insert_drawing_metadata(self.state.drawing_metadata) | |
| self.state.drawing_issues = DrawingReviewer().review_pdf( | |
| self.state.pdf_data, self.state.drawing_metadata | |
| ) | |
| return {"issues": len(self.state.drawing_issues or [])} | |
| def _step_index_chroma(self): | |
| err = _safe_index(self.state.project_name) | |
| if err: | |
| return {"skipped": True, "error": err} | |
| return {"indexed": True} | |
| def _step_submission_check(self): | |
| status = check_submission_readiness( | |
| project_name=self.state.project_name, | |
| ex_df=self.state.ex_df, | |
| pr_df=self.state.pr_df, | |
| calculations=self.state.calculations, | |
| qa_results=self.state.qa_results, | |
| drawing_issues=self.state.drawing_issues, | |
| vba_script="\n".join(self.state.vba_scripts.values()) if self.state.vba_scripts else None, | |
| ) | |
| self.state.submission_status = status | |
| return status | |
| def run_pipeline(**kwargs) -> Dict[str, Any]: | |
| """Compatibility wrapper used by core.__init__ and older callers.""" | |
| return Orchestrator().run_full_pipeline(**kwargs) | |
| def run_orchestrator(session_state) -> Dict: | |
| info = session_state.get("project_info", {}) if hasattr(session_state, "get") else {} | |
| limits = session_state.get("limits", {}) if hasattr(session_state, "get") else {} | |
| return Orchestrator().run_full_pipeline( | |
| ex_xml=session_state.get("ex_xml_bytes"), | |
| pr_xml=session_state.get("pr_xml_bytes"), | |
| geom_xml=session_state.get("geom_xml_bytes"), | |
| dxf_data=session_state.get("dxf_bytes") or session_state.get("dxf_data"), | |
| pdf_data=session_state.get("pdf_bytes") or session_state.get("pdf_data"), | |
| drawing_metadata=session_state.get("drawing_metadata"), | |
| project_name=info.get("project_name", "Project"), | |
| track_category=limits.get("track_category", "C"), | |
| design_speed=limits.get("design_speed", 75), | |
| limits=limits or None, | |
| mapd=session_state.get("mapd"), | |
| track_names=session_state.get("track_names"), | |
| chainages=session_state.get("chainages"), | |
| skip_clear=True, | |
| ex_name=session_state.get("ex_name", "existing.xml"), | |
| pr_name=session_state.get("pr_name", "proposed.xml"), | |
| ) | |