Update app.py

app.py

@@ -2,23 +2,27 @@ import os, glob, json, requests
 import numpy as np
 import pandas as pd
 import gradio as gr
-from sklearn.preprocessing import StandardScaler
 
+# ---- optional deps with safe fallbacks ----
 try:
     from sentence_transformers import SentenceTransformer
 except Exception:
     SentenceTransformer = None
+try:
+    from sklearn.preprocessing import StandardScaler
+except Exception:
+    StandardScaler = None
 try:
     from sklearn.feature_extraction.text import HashingVectorizer
 except Exception:
     HashingVectorizer = None
 
-CSV_MAIN = "RideSearch_dataset.csv"
-CSV_PARTS_GLOB = "RideSearch_part*_small.csv"
-EMB_TEXT_NPY = "emb_text.npy"
-EMB_NUM_NPY  = "emb_num.npy"
+CSV_MAIN        = "RideSearch_dataset.csv"
+CSV_PARTS_GLOB  = "RideSearch_part*_small.csv"
+EMB_TEXT_NPY    = "emb_text.npy"
+EMB_NUM_NPY     = "emb_num.npy"
 TRIMS_OVERRIDES = "trims_overrides.json"
-EMBED_MODEL = os.environ.get("EMBED_MODEL", "sentence-transformers/all-MiniLM-L6-v2")
+EMBED_MODEL     = os.environ.get("EMBED_MODEL", "sentence-transformers/all-MiniLM-L6-v2")
 
 NUM_COLS_CANON = [
     "horsepower","zero_to_100_kmh_s","seats","cargo_liters","price_usd",
@@ -308,6 +312,7 @@ REAL_TRIMS = {
 def _norm(x): return str(x or "").strip().lower()
 REAL_TRIMS_N = {(_norm(mk), _norm(md)): trims for (mk, md), trims in REAL_TRIMS.items()}
 
+# --------- data loading / unification ----------
 def load_df():
     if os.path.exists(CSV_MAIN):
         return pd.read_csv(CSV_MAIN)
@@ -316,26 +321,33 @@ def load_df():
         df = pd.concat([pd.read_csv(p) for p in parts], ignore_index=True)
         df.to_csv(CSV_MAIN, index=False)
         return df
-    raise FileNotFoundError("Upload RideSearch_dataset.csv or the parts RideSearch_part*_small.csv.")
+    raise FileNotFoundError("Upload RideSearch_dataset.csv or RideSearch_part*_small.csv files.")
 
 def unify_columns(df: pd.DataFrame) -> pd.DataFrame:
     df = df.copy()
+
     if "popularity" in df.columns and "popularity_score" not in df.columns: df["popularity_score"] = df["popularity"]
-    if "comfort" in df.columns and "comfort_score" not in df.columns: df["comfort_score"] = df["comfort"]
+    if "comfort"    in df.columns and "comfort_score"    not in df.columns: df["comfort_score"]    = df["comfort"]
     if "reliability" in df.columns and "reliability_score" not in df.columns: df["reliability_score"] = df["reliability"]
-    if "safety" in df.columns and "safety_rating" not in df.columns: df["safety_rating"] = df["safety"]
-    if "trim_display" not in df.columns: df["trim_display"] = df["trim"] if "trim" in df.columns else ""
+    if "safety"     in df.columns and "safety_rating"    not in df.columns: df["safety_rating"]    = df["safety"]
+
+    if "trim_display" not in df.columns:
+        df["trim_display"] = df["trim"] if "trim" in df.columns else ""
+
     for c in NUM_COLS_CANON:
         if c not in df.columns: df[c] = np.nan
-    for c in NUM_COLS_CANON: df[c] = pd.to_numeric(df[c], errors="coerce")
+        df[c] = pd.to_numeric(df[c], errors="coerce")
+
     if "text_record" not in df.columns:
-        cols = ["make","model","trim_display","body_type","fuel","engine_type"]
-        cols = [c for c in cols if c in df.columns]
+        cols = [c for c in ["make","model","trim_display","body_type","fuel","engine_type"] if c in df.columns]
         df["text_record"] = df[cols].fillna("").astype(str).agg(" ".join, axis=1)
-    for col in ["make","model","body_type","fuel","engine_type","name"]:
-        if col not in df.columns: df[col] = ""
+
+    for c in ["make","model","body_type","fuel","engine_type","name"]:
+        if c not in df.columns: df[c] = ""
+
     return df
 
+# --------- quality fix for 0–100 ----------
 def suspicious_zero_to_100(series: pd.Series) -> bool:
     s = series.dropna()
     if len(s) < 20: return False
@@ -347,136 +359,185 @@ def estimate_0_100(row):
     body = str(row.get("body_type","")).lower()
     fuel = str(row.get("fuel","")).lower()
     trim = f"{row.get('model','')} {row.get('trim_display','')}".lower()
+
     base = 26.0 - 3.2*np.log(max(hp, 60.0))
     if any(k in body for k in ["suv","crossover","pickup","truck","van"]): base += 0.7
     if any(k in body for k in ["coupe","roadster"]): base -= 0.4
     if "electric" in fuel or "ev" in fuel: base -= 0.8
     if "hybrid" in fuel: base -= 0.3
     if "diesel" in fuel: base += 0.2
-    if any(x in trim for x in [" m "," amg","rs","type r","sti","gts","gt3","hellcat","svr","cupra","john cooper works"]): base -= 0.6
+    if any(x in trim for x in [" m "," amg","rs","type r","sti","gts","gt3","hellcat","svr","cupra","john cooper works"]):
+        base -= 0.6
     return float(np.clip(round(base,2), 2.8, 14.5))
 
+# --------- embeddings ----------
 def ensure_embeddings(df):
-    txt_ok = os.path.exists(EMB_TEXT_NPY)
-    num_ok = os.path.exists(EMB_NUM_NPY)
-    if txt_ok and num_ok:
+    if os.path.exists(EMB_TEXT_NPY) and os.path.exists(EMB_NUM_NPY):
         return np.load(EMB_TEXT_NPY), np.load(EMB_NUM_NPY)
+
+    # text
     if SentenceTransformer is not None:
         model = SentenceTransformer(EMBED_MODEL)
-        Etext = model.encode(df["text_record"].astype(str).tolist(), batch_size=256, show_progress_bar=False, normalize_embeddings=True).astype("float32")
+        Etext = model.encode(df["text_record"].astype(str).tolist(),
+                             batch_size=256, show_progress_bar=False,
+                             normalize_embeddings=True).astype("float32")
     else:
-        if HashingVectorizer is None: raise RuntimeError("Install sentence-transformers or scikit-learn for text embeddings.")
+        if HashingVectorizer is None:
+            raise RuntimeError("Need sentence-transformers or scikit-learn for text embeddings.")
         hv = HashingVectorizer(n_features=512, alternate_sign=False, norm="l2")
         Etext = hv.transform(df["text_record"].fillna("").tolist()).toarray().astype("float32")
     np.save(EMB_TEXT_NPY, Etext)
-    X = df[["horsepower","zero_to_100_kmh_s","seats","cargo_liters","price_usd","popularity_score","comfort_score","reliability_score","tech_score","ownership_cost_score","safety_rating"]].copy()
+
+    # numeric
+    X = df[["horsepower","zero_to_100_kmh_s","seats","cargo_liters","price_usd",
+            "popularity_score","comfort_score","reliability_score","tech_score",
+            "ownership_cost_score","safety_rating"]].copy()
     X = X.fillna(X.mean(numeric_only=True))
+    if StandardScaler is None:
+        raise RuntimeError("scikit-learn missing: add it to requirements.txt")
     scaler = StandardScaler()
     Enum = scaler.fit_transform(X.values.astype("float32")).astype("float32")
     np.save(EMB_NUM_NPY, Enum)
     return Etext, Enum
 
+# --------- trims (learned + overrides + curated) ----------
 def load_overrides():
     if os.path.exists(TRIMS_OVERRIDES):
         try:
-            with open(TRIMS_OVERRIDES, "r", encoding="utf-8") as f: return json.load(f)
-        except Exception: return {}
+            with open(TRIMS_OVERRIDES, "r", encoding="utf-8") as f:
+                return json.load(f)
+        except Exception:
+            return {}
     return {}
 
 def save_overrides(data):
-    with open(TRIMS_OVERRIDES, "w", encoding="utf-8") as f: json.dump(data, f, indent=2, ensure_ascii=False)
+    with open(TRIMS_OVERRIDES, "w", encoding="utf-8") as f:
+        json.dump(data, f, indent=2, ensure_ascii=False)
 
 def learned_trims_from_csv(df):
     out = {}
-    for (mk, md), sub in df.groupby(["make", "model"]):
-        vals = (sub["trim_display"].dropna().astype(str).map(str.strip).replace({"": np.nan}).dropna().value_counts().index.tolist())
-        if vals: out[(_norm(mk), _norm(md))] = vals[:20]
+    for (mk, md), sub in df.groupby(["make","model"]):
+        vals = (sub["trim_display"].dropna().astype(str).map(str.strip))
+        vals = [v for v in vals if v and v.lower() not in ("nan","none")]
+        if vals:
+            top = pd.Series(vals).value_counts().index.tolist()[:20]
+            out[(_norm(mk), _norm(md))] = top
     return out
 
 def get_trims_for(make, model, learned, overrides):
-    if overrides.get(make, {}).get(model): return overrides[make][model]
-    mk, md = _norm(make), _norm(model)
-    if (mk, md) in REAL_TRIMS_N: return REAL_TRIMS_N[(mk, md)]
-    if (mk, md) in learned: return learned[(mk, md)]
+    if overrides.get(make, {}).get(model):
+        return overrides[make][model]
+    key = (_norm(make), _norm(model))
+    if key in learned: return learned[key]
+    if key in REAL_TRIMS_N: return REAL_TRIMS_N[key]
     return ["Base"]
 
+# --------- images ----------
 def wiki_image(make, model, year=None):
     q = f"{year} {make} {model}" if year else f"{make} {model}"
     try:
-        r = requests.get("https://en.wikipedia.org/w/api.php", params={"action":"query","format":"json","prop":"pageimages","piprop":"thumbnail","pithumbsize":600,"generator":"search","gsrsearch":q,"gsrlimit":1,"gsrnamespace":0}, timeout=6).json()
+        r = requests.get(
+            "https://en.wikipedia.org/w/api.php",
+            params={"action":"query","format":"json","prop":"pageimages",
+                    "piprop":"thumbnail","pithumbsize":600,
+                    "generator":"search","gsrsearch":q,"gsrlimit":1,"gsrnamespace":0},
+            timeout=6
+        ).json()
         pages = r.get("query",{}).get("pages",{})
         if pages:
-            page = next(iter(pages.values()))
-            return page.get("thumbnail",{}).get("source")
+            return next(iter(pages.values())).get("thumbnail",{}).get("source")
     except Exception:
         pass
     return None
 
-def cosine_sim_row_vs_mat(mat, vec):
+# --------- recommend ----------
+def _cosine(mat, vec):
     mat = mat / (np.linalg.norm(mat, axis=1, keepdims=True)+1e-9)
     v = vec / (np.linalg.norm(vec)+1e-9)
     return mat @ v
 
-def recommend(df, Etext, Enum, make, model, trim, year, topk, alpha, body, fuel, y_min, y_max, p_min, p_max, safety, reliab, cross_brand_only=True, exclude_same_model=True, unique_brand=True):
+def recommend(df, Etext, Enum, make, model, trim, year, topk, alpha,
+              body, fuel, y_min, y_max, p_min, p_max, safety, reliab,
+              cross_brand_only=True, exclude_same_model=True, unique_brand=True):
+
     sub = df[(df["make"]==make) & (df["model"]==model)].copy()
     if trim: sub = sub[sub["trim_display"].astype(str).str.lower()==str(trim).lower()]
     if year: sub = sub[pd.to_numeric(sub["year"], errors="coerce")==int(year)]
     if sub.empty:
         sub = df[(df["make"]==make) & (df["model"]==model)].copy()
         if sub.empty: return "No such make/model in dataset.", None, "", []
+
     a = sub.sort_values("popularity_score", ascending=False).iloc[0]
     a_idx = int(a.name)
+
     pool = df.copy()
-    if cross_brand_only: pool = pool[pool["make"] != a["make"]]
+    if cross_brand_only:  pool = pool[pool["make"] != a["make"]]
     if exclude_same_model: pool = pool[~((pool["make"]==a["make"]) & (pool["model"]==a["model"]))]
+
     if body and body!="Any": pool = pool[pool["body_type"]==body]
     if fuel and fuel!="Any": pool = pool[pool["fuel"]==fuel]
     pool = pool[(pool["year"]>=y_min) & (pool["year"]<=y_max)]
     pool = pool[(pool["price_usd"]>=p_min) & (pool["price_usd"]<=p_max)]
     pool = pool[(pool["safety_rating"]>=safety) & (pool["reliability_score"]>=reliab)]
     if pool.empty: return "No cars after your filters. Loosen price/year/safety.", None, "", []
+
     cand_idx = pool.index.values
-    st = cosine_sim_row_vs_mat(Etext[cand_idx], Etext[a_idx])
-    sn = cosine_sim_row_vs_mat(Enum[cand_idx], Enum[a_idx])
-    s = float(alpha)*st + (1-float(alpha))*sn
+    st = _cosine(Etext[cand_idx], Etext[a_idx])
+    sn = _cosine(Enum[cand_idx],  Enum[a_idx])
+    s  = float(alpha)*st + (1-float(alpha))*sn
     order = np.argsort(-s)
+
     chosen, seen = [], set()
     for j in order:
         k = cand_idx[j]
         brand = str(df.loc[k,"make"]).lower()
         if unique_brand and brand in seen: continue
-        seen.add(brand)
-        chosen.append(k)
+        seen.add(brand); chosen.append(k)
         if len(chosen) >= int(topk): break
     if not chosen: return "No recommendations after constraints.", None, "", []
+
     out = df.loc[chosen].copy()
     sim_lookup = {cand_idx[i]: float(s[i]) for i in range(len(cand_idx))}
     out["similarity_%"] = [round(sim_lookup[k]*100,1) for k in chosen]
-    cols = ["name","make","model","trim_display","year","body_type","fuel","engine_type","price_usd","horsepower","zero_to_100_kmh_s","popularity_score","comfort_score","reliability_score","tech_score","ownership_cost_score","safety_rating","similarity_%"]
+
+    cols = ["name","make","model","trim_display","year","body_type","fuel","engine_type",
+            "price_usd","horsepower","zero_to_100_kmh_s","popularity_score","comfort_score",
+            "reliability_score","tech_score","ownership_cost_score","safety_rating","similarity_%"]
     table = out[[c for c in cols if c in out.columns]].reset_index(drop=True)
+
     disp_trim = str(a.get("trim_display","")).strip()
-    anchor_md = (f"**{a['make']} {a['model']} {disp_trim} {int(a['year']) if pd.notna(a['year']) else ''}**  \n"
-                 f"Body: {a.get('body_type','')} • Fuel: {a.get('fuel','')} • Engine: {a.get('engine_type','')}  \n"
-                 f"HP: {int(a['horsepower']) if pd.notna(a['horsepower']) else '—'} • 0–100: {a.get('zero_to_100_kmh_s','—')}s "
-                 f"• Price: ${int(a['price_usd']) if pd.notna(a['price_usd']) else '—'}  \n"
-                 f"Popularity {int(a.get('popularity_score',0))}/10 • Comfort {int(a.get('comfort_score',0))}/10 • "
-                 f"Reliability {int(a.get('reliability_score',0))}/100 • Safety {int(a.get('safety_rating',0))}★")
+    anchor_md = (
+        f"**{a['make']} {a['model']} {disp_trim} {int(a['year']) if pd.notna(a['year']) else ''}**  \n"
+        f"Body: {a.get('body_type','')} • Fuel: {a.get('fuel','')} • Engine: {a.get('engine_type','')}  \n"
+        f"HP: {int(a['horsepower']) if pd.notna(a['horsepower']) else '—'} • 0–100: {a.get('zero_to_100_kmh_s','—')}s "
+        f"• Price: ${int(a['price_usd']) if pd.notna(a['price_usd']) else '—'}  \n"
+        f"Popularity {int(a.get('popularity_score',0))}/10 • Comfort {int(a.get('comfort_score',0))}/10 • "
+        f"Reliability {int(a.get('reliability_score',0))}/100 • Safety {int(a.get('safety_rating',0))}★"
+    )
+
     cards = []
     for _, r in out.head(3).iterrows():
         img = wiki_image(r.get("make",""), r.get("model",""), r.get("year",None))
         title = f"{r.get('make','')} {r.get('model','')} {str(r.get('trim_display','')).strip()}"
         cards.append((img, title))
+
     return None, table, anchor_md, cards
 
+# --------- UI ----------
 def build_app():
     df = unify_columns(load_df())
-    if suspicious_zero_to_100(df["zero_to_100_kmh_s"]): df["zero_to_100_kmh_s"] = df.apply(estimate_0_100, axis=1)
+    if suspicious_zero_to_100(df["zero_to_100_kmh_s"]):
+        df["zero_to_100_kmh_s"] = df.apply(estimate_0_100, axis=1)
+
     Etext, Enum = ensure_embeddings(df)
-    learned = learned_trims_from_csv(df)
+
+    learned   = learned_trims_from_csv(df)
     overrides = load_overrides()
+
     makes = sorted(df["make"].dropna().astype(str).unique().tolist())
     body_choices = ["Any"] + sorted([b for b in df["body_type"].dropna().astype(str).unique().tolist() if b])
     fuel_choices = ["Any"] + sorted([f for f in df["fuel"].dropna().astype(str).unique().tolist() if f])
+
     y_lo, y_hi = int(df["year"].min(skipna=True)), int(df["year"].max(skipna=True))
     p_lo, p_hi = int(df["price_usd"].min(skipna=True)), int(df["price_usd"].max(skipna=True))
 
@@ -486,13 +547,16 @@ def build_app():
         return gr.update(choices=opts, value=None)
 
     def trims_years_for(make, model):
-        if not make or not model: return gr.update(choices=[], value=None), gr.update(choices=[], value=None)
+        if not make or not model:
+            return gr.update(choices=[], value=None), gr.update(choices=[], value=None)
         trims = get_trims_for(make, model, learned, overrides)
-        years = sorted(pd.to_numeric(df[(df["make"]==make)&(df["model"]==model)]["year"], errors="coerce").dropna().astype(int).unique().tolist())
+        years = sorted(pd.to_numeric(df[(df["make"]==make)&(df["model"]==model)]["year"],
+                                     errors="coerce").dropna().astype(int).unique().tolist())
         return gr.update(choices=trims, value=None), gr.update(choices=[None]+years, value=None)
 
     with gr.Blocks(theme=gr.themes.Soft(), title="RideSearch") as demo:
         gr.Markdown("## RideSearch — cross-brand recommendations with realistic trims & photos")
+
         with gr.Tab("Pick & Recommend"):
             with gr.Row():
                 mk = gr.Dropdown(makes, label="Make")
@@ -501,6 +565,7 @@ def build_app():
                 yr = gr.Dropdown([], label="Year (optional)")
             mk.change(models_for, mk, md)
             md.change(lambda a,b: trims_years_for(a,b), [mk, md], [tr, yr])
+
             with gr.Row():
                 body = gr.Dropdown(body_choices, value="Any", label="Body")
                 fuel = gr.Dropdown(fuel_choices, value="Any", label="Fuel")
@@ -517,30 +582,38 @@ def build_app():
                 topk = gr.Slider(1, 10, value=5, step=1, label="Recommendations")
                 alpha = gr.Slider(0, 1, value=0.7, step=0.05, label="α — Text vs Numeric")
             with gr.Row():
-                cross = gr.Checkbox(True, label="Cross-brand only")
+                cross  = gr.Checkbox(True, label="Cross-brand only")
                 xmodel = gr.Checkbox(True, label="Exclude same model family")
-                uniqb = gr.Checkbox(True, label="Unique brands (no repeats)")
+                uniqb  = gr.Checkbox(True, label="Unique brands (no repeats)")
+
             run = gr.Button("Recommend", variant="primary")
             err = gr.Markdown()
             anchor_md = gr.Markdown()
             table = gr.Dataframe(interactive=False, wrap=True, label="Recommendations")
             gallery = gr.Gallery(label="Photos", height=220, columns=[3])
 
-            def on_click(mk_, md_, tr_, yr_, topk_, alpha_, body_, fuel_, y_min_, y_max_, p_min_, p_max_, safety_, reliab_, cross_, xmodel_, uniqb_):
+            def on_click(mk_, md_, tr_, yr_, topk_, alpha_,
+                         body_, fuel_, y_min_, y_max_, p_min_, p_max_, safety_, reliab_,
+                         cross_, xmodel_, uniqb_):
                 msg, tbl, atext, cards = recommend(
-                    df, Etext, Enum, mk_, md_, tr_, yr_, int(topk_), float(alpha_),
-                    body_, fuel_, int(y_min_), int(y_max_), int(p_min_), int(p_max_), int(safety_), int(reliab_),
+                    df, Etext, Enum, mk_, md_, tr_, yr_,
+                    int(topk_), float(alpha_),
+                    body_, fuel_, int(y_min_), int(y_max_),
+                    int(p_min_), int(p_max_), int(safety_), int(reliab_),
                     cross_brand_only=bool(cross_), exclude_same_model=bool(xmodel_), unique_brand=bool(uniqb_)
                 )
-                if msg: return gr.update(value=f"**{msg}**"), gr.update(value=""), pd.DataFrame(), []
+                if msg:
+                    return gr.update(value=f"**{msg}**"), gr.update(value=""), pd.DataFrame(), []
                 return gr.update(value=""), gr.update(value=atext), tbl, cards
 
-            run.click(on_click,
-                      [mk, md, tr, yr, topk, alpha, body, fuel, y_min, y_max, p_min, p_max, safety, reliab, cross, xmodel, uniqb],
-                      [err, anchor_md, table, gallery])
+            run.click(
+                on_click,
+                [mk, md, tr, yr, topk, alpha, body, fuel, y_min, y_max, p_min, p_max, safety, reliab, cross, xmodel, uniqb],
+                [err, anchor_md, table, gallery]
+            )
 
         with gr.Tab("Admin • Trim Overrides"):
-            gr.Markdown("Paste correct trims (one per line) for a model. Saved to trims_overrides.json.")
+            gr.Markdown("Paste correct trims per line for a model. Saved to trims_overrides.json.")
             a_mk = gr.Dropdown(makes, label="Make")
             a_md = gr.Dropdown([], label="Model")
             a_txt = gr.Textbox(lines=8, label="Display trims (one per line)")
@@ -551,9 +624,9 @@ def build_app():
             def do_save(make, model, txt):
                 nonlocal overrides
                 trims = [t.strip() for t in str(txt).splitlines() if t.strip()]
-                if not make or not model or not trims: return "⚠️ Provide make, model, and at least one trim."
-                if make not in overrides: overrides[make] = {}
-                overrides[make][model] = trims
+                if not make or not model or not trims:
+                    return "⚠️ Provide make, model, and at least one trim."
+                overrides.setdefault(make, {})[model] = trims
                 save_overrides(overrides)
                 return f"✅ Saved {len(trims)} trims for {make} {model}. Refresh the Pick tab."
             save_btn.click(do_save, [a_mk, a_md, a_txt], save_msg)