basel00omar commited on
Commit
55bfa94
·
1 Parent(s): 8cd11ac

fix: silent bugs including empty pdf, vector measure bounds, tie beam mismatch, lazy imports, and dead code

Browse files
api/routers/boq.py CHANGED
@@ -210,6 +210,8 @@ async def export_pdf(project_id: int, current_user: dict = Depends(get_current_u
210
  validate_critical_inputs_for_export(state_data)
211
 
212
  items = state_data.get("boq_items") or []
 
 
213
 
214
  # Map items to PDF builder expectations
215
  boq_items = []
 
210
  validate_critical_inputs_for_export(state_data)
211
 
212
  items = state_data.get("boq_items") or []
213
+ if not items:
214
+ raise HTTPException(status_code=400, detail="No BOQ items calculated yet for this project. Please calculate first.")
215
 
216
  # Map items to PDF builder expectations
217
  boq_items = []
api/routers/extraction.py CHANGED
@@ -12,6 +12,8 @@ from utils.db import safe_query, safe_execute
12
  from workflow.step3_extract import extract_page, merge_schedule_json_to_results
13
  from engine.sanity_checker import sanity_check
14
  from api.routers.data_review import reconstruct_project_inputs
 
 
15
 
16
  SETTINGS_FILE = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "user_settings.json")
17
  def get_user_settings():
@@ -208,7 +210,6 @@ async def run_extraction(req: RunExtractionReq, current_user: dict = Depends(get
208
  for item in data.get(lst_type, []):
209
  mark = str(item.get("mark", "")).strip().upper()
210
  if mark and len(mark) > 1:
211
- from _pdf_utils import normalize_mark
212
  m_clean = normalize_mark(mark)
213
  match_pattern = "".join(c + r"[\s\-_./]*" if c.isalpha() else c for c in m_clean).rstrip(r"[\s\-_./]*")
214
  n = len(re.findall(rf"\b{match_pattern}\b", full_text))
 
12
  from workflow.step3_extract import extract_page, merge_schedule_json_to_results
13
  from engine.sanity_checker import sanity_check
14
  from api.routers.data_review import reconstruct_project_inputs
15
+ from utils.garbage_collection import background_cleanup_project
16
+ from _pdf_utils import normalize_mark
17
 
18
  SETTINGS_FILE = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "user_settings.json")
19
  def get_user_settings():
 
210
  for item in data.get(lst_type, []):
211
  mark = str(item.get("mark", "")).strip().upper()
212
  if mark and len(mark) > 1:
 
213
  m_clean = normalize_mark(mark)
214
  match_pattern = "".join(c + r"[\s\-_./]*" if c.isalpha() else c for c in m_clean).rstrip(r"[\s\-_./]*")
215
  n = len(re.findall(rf"\b{match_pattern}\b", full_text))
engine/project_boq_bridge.py CHANGED
@@ -646,179 +646,3 @@ BOQ_ITEM_DEFAULT_RATES = {
646
  def _get_item_rate(item_key: str, emirate: str) -> float:
647
  # Pricing disabled by request — quantity-only BOQ (all rates & totals are zero).
648
  return 0.0
649
- default_rate = BOQ_ITEM_DEFAULT_RATES.get(item_key, 0.0)
650
-
651
- # Try fetching from DB
652
- from utils.db import safe_query
653
- import pandas as pd
654
-
655
- em_display = "Dubai"
656
- em_clean = emirate.strip().lower()
657
- if "abu" in em_clean:
658
- em_display = "Abu Dhabi"
659
- elif "sharjah" in em_clean:
660
- em_display = "Sharjah"
661
- elif "ajman" in em_clean:
662
- em_display = "Ajman"
663
-
664
- mapping = {
665
- "foundation_pcc": "Ready Mix C20/25",
666
- "tie_beam_pcc": "Ready Mix C20/25",
667
- "foundation_concrete": "Ready Mix C30/37",
668
- "neck_column_concrete": "Ready Mix C30/37",
669
- "tie_beam_concrete": "Ready Mix C30/37",
670
- "slab_on_grade": "Ready Mix C30/37",
671
- "col_gf_to_1st": "Ready Mix C30/37",
672
- "col_1st_to_2nd": "Ready Mix C30/37",
673
- "col_roof": "Ready Mix C30/37",
674
- "beam_concrete_1st": "Ready Mix C30/37",
675
- "beam_concrete_2nd": "Ready Mix C30/37",
676
- "beam_concrete_roof": "Ready Mix C30/37",
677
- "slab_concrete_1st": "Ready Mix C30/37",
678
- "slab_concrete_2nd": "Ready Mix C30/37",
679
- "slab_concrete_roof": "Ready Mix C30/37",
680
- "parapet_concrete": "Ready Mix C30/37",
681
- "staircase_1st": "Ready Mix C30/37",
682
-
683
- "solid_block_work": "Hollow Block 8\"",
684
- "parapet_block_work": "Hollow Block 8\"",
685
- "thermal_block_gf": "Hollow Block 8\"",
686
- "thermal_block_1f": "Hollow Block 8\"",
687
- "thermal_block_2f": "Hollow Block 8\"",
688
- "internal_block_gf": "Hollow Block 6\"",
689
- "internal_block_1f": "Hollow Block 6\"",
690
- "internal_block_2f": "Hollow Block 6\"",
691
-
692
- "foundation_bitumen": "Bitumen Membrane 4mm",
693
- "neck_column_bitumen": "Bitumen Membrane 4mm",
694
- "tie_beam_bitumen": "Bitumen Membrane 4mm",
695
- "block_work_bitumen": "Bitumen Membrane 4mm",
696
- "roof_waterproofing": "Bitumen Membrane 4mm",
697
- "wet_wp_1f": "Bitumen Membrane 4mm",
698
- "wet_wp_2f": "Bitumen Membrane 4mm",
699
- "balcony_wp_gf": "Bitumen Membrane 4mm",
700
- "balcony_wp_1f": "Bitumen Membrane 4mm",
701
-
702
- "dry_floor_gf": "Porcelain Tiles 60x60",
703
- "dry_floor_1f": "Porcelain Tiles 60x60",
704
- "dry_floor_2f": "Porcelain Tiles 60x60",
705
-
706
- "wet_floor_gf": "Ceramic Tiles 60x60",
707
- "wet_floor_1f": "Ceramic Tiles 60x60",
708
- "wet_floor_2f": "Ceramic Tiles 60x60",
709
- "wall_tiles_gf": "Ceramic Tiles 60x60",
710
- "wall_tiles_1f": "Ceramic Tiles 60x60",
711
- "wall_tiles_2f": "Ceramic Tiles 60x60",
712
-
713
- "int_paint_gf": "Emulsion Paint",
714
- "int_paint_1f": "Emulsion Paint",
715
- "int_paint_2f": "Emulsion Paint",
716
- "external_finishes": "External Paint",
717
-
718
- "door_openings": "Wooden Door (Standard)",
719
- "window_openings": "Aluminum Window",
720
- }
721
-
722
- prefix = mapping.get(item_key)
723
- if prefix:
724
- db_name = f"{prefix} ({em_display})"
725
- try:
726
- df = safe_query("SELECT rate_aed FROM qto_market_prices WHERE item_name = %s", (db_name,))
727
- if not df.empty and pd.notnull(df.iloc[0]["rate_aed"]):
728
- db_rate = float(df.iloc[0]["rate_aed"])
729
- if "block" in item_key:
730
- return round(db_rate * 12.5, 2)
731
- return db_rate
732
- except Exception:
733
- pass
734
-
735
- factors = {"dubai": 1.0, "abudhabi": 1.05, "sharjah": 0.95, "ajman": 0.90}
736
- factor = factors.get(em_clean, 1.0)
737
- return round(default_rate * factor, 2)
738
-
739
- def build_boq_dataframe_from_project(project: dict) -> tuple[pd.DataFrame, dict]:
740
- """
741
- Build the export-ready BOQ DataFrame straight from _project_data.json.
742
- Returns (df, meta). df columns match engine/boq_builder output so the
743
- existing Excel exporter & UI table work unchanged.
744
- """
745
- results, meta = compute_all_quantities(project)
746
- emirate = project.get("emirate") or "Dubai"
747
-
748
- rows = []
749
- item_num = 1
750
-
751
- def add_section(title: str):
752
- rows.append({
753
- "#": "", "Description (English)": f"▌ {title}", "البيان": "",
754
- "Unit": "", "Quantity": "", "_is_header": True,
755
- })
756
-
757
- for page_type, page in DRAWING_ITEMS_MAP.items():
758
- if page_type in ["slab_1st", "slab_2nd", "first_floor_plan", "second_floor_plan"]:
759
- total_qty = sum(results.get(item["key"], 0.0) for item in page["items"])
760
- if total_qty == 0:
761
- continue
762
-
763
- section_title = f"{page['label_en']} | {page['label_ar']}"
764
- add_section(section_title)
765
- for item in page["items"]:
766
- qty = results.get(item["key"], 0.0)
767
- nrm_code = POMI_NRM_MAP.get(item["key"], str(item_num))
768
- rate = _get_item_rate(item["key"], emirate)
769
- total = qty * rate if qty is not None and rate is not None else 0.0
770
-
771
- en_name = item["name_en"]
772
- ar_name = item["name_ar"]
773
- # Removed dynamic concrete grade appending as per user request
774
- if "block" in item["key"]:
775
- t = "200mm"
776
- if "solid" in item["key"]:
777
- t = project.get("solid_block_thickness", "200mm")
778
- elif "thermal" in item["key"] or "parapet" in item["key"]:
779
- t = project.get("thermal_block_thickness", "200mm")
780
- elif "internal" in item["key"]:
781
- t = project.get("hollow_block_thickness", "100mm")
782
- en_name += f" ({t} Thickness)"
783
- ar_name += f" (سماكة {t})"
784
- rows.append({
785
- "#": nrm_code,
786
- "Description (English)": en_name,
787
- "البيان": ar_name,
788
- "Unit": item["unit"],
789
- "Quantity": round(float(qty), 2) if qty is not None else 0.0,
790
- "has_error": qty is None,
791
- "_is_header": False,
792
- })
793
- item_num += 1
794
-
795
- df = pd.DataFrame(rows)
796
- df = df.where(pd.notnull(df), None)
797
- return df, meta
798
-
799
-
800
- def load_project(path: str = PROJECT_JSON) -> dict:
801
- with open(path, encoding="utf-8") as f:
802
- return json.load(f)
803
-
804
-
805
- # ── CLI smoke test ─────────────────────────────────────────────────────────────
806
- if __name__ == "__main__":
807
- import sys
808
- sys.stdout.reconfigure(encoding="utf-8")
809
- p = sys.argv[1] if len(sys.argv) > 1 else PROJECT_JSON
810
- proj = load_project(p)
811
- df, meta = build_boq_dataframe_from_project(proj)
812
- data = df[df["_is_header"] == False]
813
- print(f"\nComputed {len(data)} items.")
814
- nonzero = data[data["Quantity"] > 0]
815
- print(f"Non-zero: {len(nonzero)} / {len(data)}")
816
- print(df.drop(columns=["_is_header"]).to_string(index=False))
817
- if meta["estimates"]:
818
- print("\nESTIMATED (confirm in Schedule Review):")
819
- for e in meta["estimates"]:
820
- print(f" - {e}")
821
- if meta["needs_input"]:
822
- print("\nNEEDS INPUT:")
823
- for n in meta["needs_input"]:
824
- print(f" - {n}")
 
646
  def _get_item_rate(item_key: str, emirate: str) -> float:
647
  # Pricing disabled by request — quantity-only BOQ (all rates & totals are zero).
648
  return 0.0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
workflow/sanity_checks.py CHANGED
@@ -60,14 +60,14 @@ def perform_structural_sanity_checks(data: dict, drawing_type: str) -> list[str]
60
  """
61
  warnings = []
62
 
63
- if drawing_type in ["foundation_plan", "tie_beam_plan"]:
64
  l = data.get("longest_length", 0) or 0
65
  w = data.get("longest_width", 0) or 0
66
  if l > 0 and w > 0:
67
  if l < 5 or w < 5:
68
  warnings.append(f"Overall dimensions ({l}m x {w}m) seem too small for a villa. Are they in meters?")
69
 
70
- if drawing_type == "tie_beam_plan":
71
  tb_len = data.get("tb_total_length", 0) or 0
72
  if tb_len > 0 and l > 0 and w > 0:
73
  approx_perim = (l + w) * 2
 
60
  """
61
  warnings = []
62
 
63
+ if drawing_type in ["foundation_plan", "tie_beam"]:
64
  l = data.get("longest_length", 0) or 0
65
  w = data.get("longest_width", 0) or 0
66
  if l > 0 and w > 0:
67
  if l < 5 or w < 5:
68
  warnings.append(f"Overall dimensions ({l}m x {w}m) seem too small for a villa. Are they in meters?")
69
 
70
+ if drawing_type == "tie_beam":
71
  tb_len = data.get("tb_total_length", 0) or 0
72
  if tb_len > 0 and l > 0 and w > 0:
73
  approx_perim = (l + w) * 2
workflow/step3_extract.py CHANGED
@@ -15,6 +15,8 @@ from engine.dimension_filter import (
15
  get_ai_prompt_for_drawing,
16
  filter_ocr_by_drawing_inputs,
17
  )
 
 
18
  from workflow.workflow_state import mark_step_done
19
 
20
 
@@ -677,7 +679,6 @@ def extract_page(page_arr: np.ndarray, drawing_type: str, page_texts: str, user_
677
  merged["int_walls_10cm_m"] = wall_data.get("int_walls_10cm_m", 0.0)
678
  merged["int_walls_20cm_m"] = wall_data.get("int_walls_20cm_m", 0.0)
679
  return merged
680
- return merged
681
 
682
  data = asyncio.run(run_dedicated_extractors())
683
  data = _finishes_from_rooms(data)
@@ -1223,7 +1224,6 @@ def render_step3() -> bool:
1223
  for item in data.get(lst_type, []):
1224
  mark = str(item.get("mark", "")).strip().upper()
1225
  if mark and len(mark) > 1: # avoid matching single letters like 'D' everywhere
1226
- from _pdf_utils import normalize_mark
1227
  m_clean = normalize_mark(mark)
1228
  match_pattern = ""
1229
  for char in m_clean:
 
15
  get_ai_prompt_for_drawing,
16
  filter_ocr_by_drawing_inputs,
17
  )
18
+ from workflow.workflow_state import get_current_step, set_step, step_done, STEPS
19
+ from _pdf_utils import normalize_mark
20
  from workflow.workflow_state import mark_step_done
21
 
22
 
 
679
  merged["int_walls_10cm_m"] = wall_data.get("int_walls_10cm_m", 0.0)
680
  merged["int_walls_20cm_m"] = wall_data.get("int_walls_20cm_m", 0.0)
681
  return merged
 
682
 
683
  data = asyncio.run(run_dedicated_extractors())
684
  data = _finishes_from_rooms(data)
 
1224
  for item in data.get(lst_type, []):
1225
  mark = str(item.get("mark", "")).strip().upper()
1226
  if mark and len(mark) > 1: # avoid matching single letters like 'D' everywhere
 
1227
  m_clean = normalize_mark(mark)
1228
  match_pattern = ""
1229
  for char in m_clean:
workflow/step6_review.py CHANGED
@@ -31,7 +31,7 @@ def get_section_confidence(section: str) -> str:
31
  """Heuristic to determine confidence based on source pages."""
32
  res = st.session_state.get("extraction_results", {})
33
  deps = []
34
- if section == "sub": deps = ["foundation_plan", "tie_beam_plan"]
35
  elif section == "sup": deps = ["ground_columns", "upper_columns", "ground_floor_plan", "first_floor_plan", "roof_plan"]
36
  elif section == "fin": deps = ["ground_floor_plan", "first_floor_plan"]
37
  elif section == "open": deps = ["window_schedule", "door_schedule"]
 
31
  """Heuristic to determine confidence based on source pages."""
32
  res = st.session_state.get("extraction_results", {})
33
  deps = []
34
+ if section == "sub": deps = ["foundation_plan", "tie_beam"]
35
  elif section == "sup": deps = ["ground_columns", "upper_columns", "ground_floor_plan", "first_floor_plan", "roof_plan"]
36
  elif section == "fin": deps = ["ground_floor_plan", "first_floor_plan"]
37
  elif section == "open": deps = ["window_schedule", "door_schedule"]
workflow/vector_measure.py CHANGED
@@ -102,6 +102,9 @@ def measure_page(pdf_path: str, page_index: int,
102
  'matched': int, 'confidence': float } }
103
  """
104
  doc = fitz.open(pdf_path)
 
 
 
105
  page = doc[page_index]
106
  ratio = scale_ratio or detect_page_scale(page)
107
  pt_to_m = _pt_to_m(ratio)
@@ -161,6 +164,9 @@ def measure_tie_beams(pdf_path: str, page_index: int,
161
  confidence, scale} }.
162
  """
163
  doc = fitz.open(pdf_path)
 
 
 
164
  page = doc[page_index]
165
  ratio = scale_ratio or detect_page_scale(page)
166
  pt_to_m = _pt_to_m(ratio)
 
102
  'matched': int, 'confidence': float } }
103
  """
104
  doc = fitz.open(pdf_path)
105
+ if page_index >= len(doc):
106
+ doc.close()
107
+ return {}
108
  page = doc[page_index]
109
  ratio = scale_ratio or detect_page_scale(page)
110
  pt_to_m = _pt_to_m(ratio)
 
164
  confidence, scale} }.
165
  """
166
  doc = fitz.open(pdf_path)
167
+ if page_index >= len(doc):
168
+ doc.close()
169
+ return {}
170
  page = doc[page_index]
171
  ratio = scale_ratio or detect_page_scale(page)
172
  pt_to_m = _pt_to_m(ratio)