Update data_handler.py

data_handler.py

@@ -1,126 +1,173 @@
 import pandas as pd
+import numpy as np
 import plotly.express as px
 import plotly.graph_objects as go
 from datetime import datetime
-import prediction_engine as engine # Kita panggil engine untuk prediksi
+import prediction_engine as engine
 import re
 
 # ==============================================================================
-# 1. SMART COLUMN MAPPING (AUTO-ADJUST)
+# 1. SMART COLUMN MAPPING (Kamus Sinonim)
 # ==============================================================================
-# Ini kamus sinonim. Sistem akan mencari kolom yg mirip dengan kata-kata ini.
 COLUMN_SYNONYMS = {
-    'Town': ['town', 'city', 'location', 'kota', 'lokasi', 'area'],
-    'Property_Residential': ['property_residential', 'property type', 'type', 'tipe', 'jenis properti', 'category'],
-    'List Year': ['list year', 'list_year', 'year', 'tahun', 'tahun daftar'],
-    'Assessed Value': ['assessed value', 'assessed_value', 'value', 'price', 'assessed', 'nilai', 'harga', 'taksiran']
+    'Town': ['town', 'city', 'location', 'kota', 'lokasi', 'area', 'wilayah', 'daerah'],
+    'Property_Residential': ['property_residential', 'property', 'type', 'tipe', 'jenis', 'kategori', 'category', 'jenis properti'],
+    'List Year': ['list year', 'year', 'tahun', 'thn', 'tahun daftar', 'listing year'],
+    'Assessed Value': ['assessed value', 'value', 'price', 'assessed', 'nilai', 'harga', 'taksiran', 'harga taksiran', 'amount', 'rp', 'idr', 'usd']
 }
 
-def normalize_columns(df):
+def find_header_row(df_raw):
     """
-    Mengubah nama kolom user yang aneh-aneh menjadi standar sistem.
-    Contoh: User upload 'Kota' -> Sistem ubah jadi 'Town'
+    Mencari baris mana yang mengandung Header. 
+    Berguna jika header Excel ada di baris ke-2 atau ke-3.
     """
-    df.columns = df.columns.str.strip() # Hapus spasi di nama kolom
-    
-    # Buat map dari nama kolom asli ke nama kolom standar
+    # Cek 10 baris pertama
+    for i in range(min(10, len(df_raw))):
+        row_values = df_raw.iloc[i].astype(str).str.lower().tolist()
+        # Hitung berapa banyak kata kunci 'wajib' yang muncul di baris ini
+        matches = 0
+        for key, synonyms in COLUMN_SYNONYMS.items():
+            if any(syn in " ".join(row_values) for syn in synonyms):
+                matches += 1
+        
+        # Jika minimal 3 kolom wajib ditemukan di baris ini, ini adalah Header!
+        if matches >= 3:
+            print(f"✅ Header ditemukan di baris ke-{i}")
+            # Set baris ini sebagai header
+            df_new = df_raw.iloc[i+1:].copy()
+            df_new.columns = df_raw.iloc[i]
+            return df_new
+            
+    # Jika tidak ketemu, kembalikan apa adanya (asumsi baris 0 adalah header)
+    return df_raw
+
+def normalize_columns(df):
+    """Mengubah nama kolom user menjadi standar sistem."""
+    df.columns = df.columns.astype(str).str.strip()
     rename_map = {}
+    
+    # Loop setiap kolom di Dataframe user
     for col in df.columns:
-        col_lower = col.lower().replace('_', ' ')
+        col_lower = col.lower().replace('_', ' ').replace('.', ' ')
         
+        # Cari kecocokan di kamus sinonim
         for standard_col, synonyms in COLUMN_SYNONYMS.items():
-            # Cek jika nama kolom mengandung salah satu sinonim
-            if any(syn in col_lower for syn in synonyms):
-                # Prioritas: Jangan timpa jika sudah ada yang pas
+            # Cek Exact Match atau Partial Match
+            if any(syn == col_lower or syn in col_lower.split() for syn in synonyms):
                 if standard_col not in rename_map.values():
                     rename_map[col] = standard_col
     
-    # Lakukan rename
     if rename_map:
-        print(f"🔄 Auto-Adjusting Columns: {rename_map}")
+        print(f"🔄 Rename Kolom: {rename_map}")
         df = df.rename(columns=rename_map)
         
     return df
 
-def clean_currency(x):
-    """Membersihkan data angka yang terformat string (misal: '$ 150,000')"""
-    if isinstance(x, str):
-        # Hapus simbol $ , dan spasi
-        clean = re.sub(r'[^\d.]', '', x)
-        try:
-            return float(clean)
-        except:
-            return 0.0
-    return x
+def clean_currency_aggressive(x):
+    """
+    Pembersih angka SUPER KUAT.
+    Bisa baca: 'Rp 1.500.000', '$ 150,000.00', '150.000', '150,000'
+    """
+    if pd.isna(x) or x == "":
+        return 0.0
+    
+    s = str(x).strip()
+    
+    try:
+        # Jika formatnya sudah float/int murni
+        if isinstance(x, (int, float)):
+            return float(x)
+
+        # 1. Buang Simbol Mata Uang & Huruf (Rp, $, USD, dll)
+        # Hanya sisakan angka, titik, koma, dan minus
+        s = re.sub(r'[^\d.,-]', '', s)
+        
+        if not s: return 0.0
+
+        # 2. Deteksi Format Indonesia (Titik sebagai ribuan) vs US (Koma sebagai ribuan)
+        # Jika ada titik DAN koma (misal: 150.000,00 atau 150,000.00)
+        if '.' in s and ',' in s:
+            if s.rfind('.') < s.rfind(','): 
+                # Format Indo: 150.000,00 -> Titik dihapus, Koma jadi Titik
+                s = s.replace('.', '').replace(',', '.')
+            else:
+                # Format US: 150,000.00 -> Koma dihapus
+                s = s.replace(',', '')
+        
+        # Jika hanya ada Titik (misal: 150.000 atau 150.55)
+        elif '.' in s:
+            # Jika titik muncul lebih dari sekali (1.000.000), itu pasti ribuan -> Hapus
+            if s.count('.') > 1:
+                s = s.replace('.', '')
+            # Jika titik cuma satu tapi di akhir (100. -> 100)
+            elif s.endswith('.'):
+                s = s.replace('.', '')
+            # Cek panjang desimal. Jika 3 digit persis setelah titik (150.000), kemungkinan besar ribuan
+            # TAPI ini ambigu. Kita asumsikan user properti input angka bulat besar.
+            # Amannya: Jika hasilnya nanti < 1000 dan input aslinya panjang, berarti salah baca.
+            # Mari gunakan pendekatan 'pd.to_numeric' yang lebih pintar nanti.
+            # Untuk sekarang: Hapus titik jika terlihat seperti ribuan
+            elif len(s.split('.')[-1]) == 3: 
+                s = s.replace('.', '')
+                
+        # Jika hanya ada Koma (misal: 150,000) -> Hapus koma
+        elif ',' in s:
+            s = s.replace(',', '')
+
+        return float(s)
+    except:
+        return 0.0
 
 # ==============================================================================
-# 2. ROBUST PLOTTING (ANTI ERROR)
+# 2. ROBUST PLOTTING (Agar tidak Error "NoneType")
 # ==============================================================================
 def safe_generate_plots(df):
-    """
-    Membuat grafik dengan pengecekan data kosong agar tidak error TypeError.
-    """
-    # Cek Data Kosong
-    if df is None or df.empty or len(df) == 0:
-        # Return grafik kosong dengan tulisan "No Data"
-        empty_fig = go.Figure()
-        empty_fig.update_layout(
-            xaxis={"visible": False}, yaxis={"visible": False},
-            annotations=[{"text": "Data Kosong / Tidak Valid", "showarrow": False, "font": {"size": 20}}]
+    if df is None or df.empty:
+        empty = go.Figure().update_layout(
+            title="Data Kosong / Gagal Membaca Angka", 
+            xaxis={"visible":False}, yaxis={"visible":False},
+            annotations=[{"text": "Cek Format Angka Excel Anda", "showarrow":False, "font":{"size":20}}]
         )
-        return empty_fig, empty_fig, empty_fig, empty_fig
+        return empty, empty, empty, empty
 
-    # 1. Scatter Plot
+    # 1. Scatter
     try:
         fig1 = px.scatter(
             df, x="Assessed Value", y="Predicted_Sale_Amount", color="Property_Residential",
-            hover_data=['Town', 'List Year'], title="📈 Korelasi: Nilai Taksiran vs Prediksi",
+            hover_data=['Town', 'List Year'], title="📈 Prediksi vs Nilai Taksiran",
             template="plotly_white", opacity=0.7
         )
-        max_val = max(df["Assessed Value"].max(), df["Predicted_Sale_Amount"].max())
-        fig1.add_shape(type="line", x0=0, y0=0, x1=max_val, y1=max_val, line=dict(color="Red", width=2, dash="dash"))
-    except:
-        fig1 = go.Figure().update_layout(title="Gagal Render Scatter")
+    except: fig1 = go.Figure()
 
-    # 2. Box Plot
+    # 2. Box
     try:
-        top_towns = df['Town'].value_counts().nlargest(15).index
+        top = df['Town'].value_counts().nlargest(10).index
         fig2 = px.box(
-            df[df['Town'].isin(top_towns)], x="Town", y="Predicted_Sale_Amount", color="Town",
-            title="🏙️ Sebaran Harga Kota", template="plotly_white"
+            df[df['Town'].isin(top)], x="Town", y="Predicted_Sale_Amount", color="Town",
+            title="🏙️ Sebaran Harga", template="plotly_white"
         )
         fig2.update_layout(showlegend=False)
-    except:
-        fig2 = go.Figure().update_layout(title="Gagal Render Boxplot")
+    except: fig2 = go.Figure()
 
-    # 3. Histogram Ratio (DIPERBAIKI DARI KODE SEBELUMNYA)
+    # 3. Histogram
     try:
-        df_valid = df[df['Assessed Value'] > 0].copy()
-        if not df_valid.empty:
-            df_valid['Ratio'] = df_valid['Predicted_Sale_Amount'] / df_valid['Assessed Value']
+        df_ok = df[df['Assessed Value'] > 100] # Hanya data valid
+        if not df_ok.empty:
+            df_ok['Ratio'] = df_ok['Predicted_Sale_Amount'] / df_ok['Assessed Value']
             fig3 = px.histogram(
-                df_valid, x="Ratio", nbins=30, title="📊 Distribusi Rasio",
+                df_ok, x="Ratio", nbins=30, title="📊 Rasio Kenaikan",
                 color_discrete_sequence=['#00CC96'], template="plotly_white"
             )
             fig3.add_vline(x=1.0, line_dash="dash", line_color="red")
         else:
-            # FIX: Jangan pakai px.histogram tanpa data. Pakai go.Figure kosong.
-            fig3 = go.Figure()
-            fig3.update_layout(title="Assessed Value 0 semua", xaxis={"visible":False}, yaxis={"visible":False})
-            fig3.add_annotation(text="Tidak bisa hitung rasio (Assessed Value = 0)", showarrow=False)
-    except Exception as e:
-        print(f"Plot Error: {e}")
-        fig3 = go.Figure().update_layout(title="Error Histogram")
+            fig3 = go.Figure().update_layout(title="Data Value Kosong")
+    except: fig3 = go.Figure()
 
-    # 4. Bar Chart
+    # 4. Bar
     try:
-        val_by_type = df.groupby("Property_Residential")["Predicted_Sale_Amount"].sum().reset_index()
-        fig4 = px.bar(
-            val_by_type, x="Predicted_Sale_Amount", y="Property_Residential", orientation='h',
-            title="💰 Valuasi per Tipe", text_auto='.2s', color="Predicted_Sale_Amount", template="plotly_white"
-        )
-    except:
-        fig4 = go.Figure().update_layout(title="Gagal Render Bar Chart")
+        val = df.groupby("Property_Residential")["Predicted_Sale_Amount"].sum().reset_index()
+        fig4 = px.bar(val, x="Predicted_Sale_Amount", y="Property_Residential", orientation='h', title="💰 Total Aset")
+    except: fig4 = go.Figure()
 
     return fig1, fig2, fig3, fig4
 
@@ -131,35 +178,48 @@ def process_file_pipeline(file_obj):
     if file_obj is None: return None
     
     try:
-        # A. Baca File (Flexible Excel/CSV)
+        # A. BACA FILE (Header=None dulu supaya bisa cari header manual)
         filename = file_obj.name.lower()
         if filename.endswith('.csv'):
-            df = pd.read_csv(file_obj.name)
+            df = pd.read_csv(file_obj.name, header=None)
         elif filename.endswith(('.xlsx', '.xls')):
-            df = pd.read_excel(file_obj.name)
+            df = pd.read_excel(file_obj.name, header=None)
         else:
-            return "ERROR_FORMAT", "Format harus .csv atau .xlsx", None, None, None, None, None
+            return "ERROR_FORMAT", "Format salah", None, None, None, None, None
+
+        # B. CARI HEADER ASLI
+        df = find_header_row(df)
 
-        # B. Normalisasi Kolom (Fitur Auto-Adjust)
+        # C. RENAMING KOLOM
         df = normalize_columns(df)
 
-        # C. Validasi Kolom Standar
+        # D. CEK KOLOM WAJIB
         req_cols = ['Town', 'Property_Residential', 'List Year', 'Assessed Value']
         missing = [c for c in req_cols if c not in df.columns]
-        
         if missing:
-            # Pesan error lebih ramah
-            return "ERROR_COLS", f"Sistem tidak mengenali kolom: {', '.join(missing)}. \n(Coba ganti nama header di Excel Anda)", None, None, None, None, None
+            col_list = ", ".join(df.columns.tolist())
+            return "ERROR_COLS", f"Kolom tidak ditemukan.\nKolom terbaca: {col_list}.\nKurang: {missing}", None, None, None, None, None
+
+        # E. DATA CLEANING & TYPE CONVERSION
+        # Bersihkan Assessed Value dengan Agresif
+        df['Assessed Value'] = df['Assessed Value'].apply(clean_currency_aggressive)
         
-        # D. Cleaning Data
-        df['Assessed Value'] = df['Assessed Value'].apply(clean_currency)
-        df = df.dropna(subset=['Assessed Value']) # Hapus baris kosong
+        # Bersihkan List Year
+        df['List Year'] = pd.to_numeric(df['List Year'], errors='coerce').fillna(2021).astype(int).astype(str)
         
-        # Tambah tanggal default
+        # Filter: Hapus jika Value 0 atau NaN
+        initial_len = len(df)
+        df = df[df['Assessed Value'] > 0].copy()
+        final_len = len(df)
+        
+        if df.empty:
+            return "ERROR_COLS", f"Data terbaca tapi nilainya 0 semua. Cek format angka Excel. (Baris awal: {initial_len}, Baris valid: 0)", None, None, None, None, None
+
+        # Tambah Tanggal Default
         if 'Prediction Date' not in df.columns:
             df['Prediction Date'] = datetime.now().strftime('%Y-%m-%d')
 
-        # E. Loop Prediksi (Panggil Engine)
+        # F. LOOP PREDIKSI
         preds = []
         for _, row in df.iterrows():
             val = engine.predict_single(
@@ -173,12 +233,9 @@ def process_file_pipeline(file_obj):
         
         df['Predicted_Sale_Amount'] = preds
         
-        # F. Generate Metrics & Plots
-        # Pakai engine untuk hitung metrics, tapi plot pakai fungsi safe di sini
-        metrics = engine.calculate_key_metrics(df) 
+        # G. HASIL
+        metrics = engine.calculate_key_metrics(df)
         fig1, fig2, fig3, fig4 = safe_generate_plots(df)
-        
-        # G. Save & Return
         out_csv = "analysis_result.csv"
         df.to_csv(out_csv, index=False)