okiel21 commited on
Commit
f5ea5a9
·
verified ·
1 Parent(s): 502d1bb

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +330 -43
app.py CHANGED
@@ -1,3 +1,13 @@
 
 
 
 
 
 
 
 
 
 
1
  import os
2
  import json
3
  import re
@@ -7,7 +17,7 @@ import pandas as pd
7
  import numpy as np
8
  import streamlit as st
9
 
10
- # Optional persistence
11
  HAS_DUCKDB = False
12
  try:
13
  import duckdb
@@ -15,13 +25,51 @@ try:
15
  except Exception:
16
  HAS_DUCKDB = False
17
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
18
  st.set_page_config(page_title="Unified Data Catalog — MVP", layout="wide")
19
 
20
  EMAIL_RE = r'[^@\s]+@[^@\s]+\.[^@\s]+'
21
  PHONE_RE_BR = r'^\+?\d{10,14}$'
22
  CPF_RE = r'^\d{11}$'
23
 
24
- KNOWN_ID_HINTS = {"user_id","customer_id","client_id","id","contact_id","order_id","email","cpf","phone","document"}
 
 
 
 
 
25
 
26
  def infer_simple_type(s: pd.Series) -> str:
27
  dt = str(s.dtype)
@@ -29,11 +77,10 @@ def infer_simple_type(s: pd.Series) -> str:
29
  if dt.startswith(("float","Float")): return "float"
30
  if dt == "bool": return "boolean"
31
  try:
32
- parsed = pd.to_datetime(s.dropna().head(20), errors="raise", utc=True)
33
- if len(parsed) > 0: return "datetime"
34
  except Exception:
35
- pass
36
- return "string"
37
 
38
  def detect_pii_values(vals: List[str]):
39
  flags = {"email": False, "phone": False, "cpf": False}
@@ -70,9 +117,11 @@ def profile_dataframe(table_name: str, df: pd.DataFrame, headers_only: bool) ->
70
  distinct = None
71
  null_pct = None
72
  examples = []
73
- pii_flags = {"email": "email" in col.lower(),
74
- "phone": "phone" in col.lower() or "telefone" in col.lower(),
75
- "cpf": "cpf" in col.lower() or "document" in col.lower()}
 
 
76
  else:
77
  inferred = infer_simple_type(series)
78
  distinct = int(series.nunique(dropna=True))
@@ -96,42 +145,102 @@ def profile_dataframe(table_name: str, df: pd.DataFrame, headers_only: bool) ->
96
  })
97
  return pd.DataFrame(cols)
98
 
99
- def find_relationships(tables: Dict[str, pd.DataFrame], headers_only: bool, threshold: float = 0.4) -> pd.DataFrame:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
100
  if headers_only:
101
  return pd.DataFrame()
102
  rows = []
103
- tab_names = list(tables.keys())
104
- for i in range(len(tab_names)):
105
- for j in range(i+1, len(tab_names)):
106
- t1, t2 = tab_names[i], tab_names[j]
107
  df1, df2 = tables[t1], tables[t2]
108
- cands1 = [c for c in df1.columns if c.lower() in KNOWN_ID_HINTS or "email" in c.lower() or "order" in c.lower()]
109
- cands2 = [c for c in df2.columns if c.lower() in KNOWN_ID_HINTS or "email" in c.lower() or "order" in c.lower()]
110
- cands1 = cands1 or list(df1.columns[:3])
111
- cands2 = cands2 or list(df2.columns[:3])
112
- for c1 in cands1:
113
- s1 = set(df1[c1].dropna().astype(str).unique()[:5000])
114
- if not s1: continue
115
- for c2 in cands2:
116
- s2 = set(df2[c2].dropna().astype(str).unique()[:5000])
117
- if not s2: continue
118
- inter = len(s1 & s2)
119
- denom = max(1, min(len(s1), len(s2)))
120
- overlap = inter / denom
121
- if overlap >= threshold:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
122
  rows.append({
123
  "left_table": t1, "left_key": c1,
124
  "right_table": t2, "right_key": c2,
125
- "overlap_ratio": round(overlap, 3),
 
 
 
 
 
126
  "relationship": "potential join"
127
  })
128
- return pd.DataFrame(rows)
 
129
 
130
  def keyword_search(glossary_df: pd.DataFrame, query: str, top_k: int = 12) -> pd.DataFrame:
131
  if glossary_df is None or glossary_df.empty or not query.strip():
132
  return pd.DataFrame()
133
  q = query.lower()
134
- mask = (glossary_df["column"].str.lower().str.contains(q)) | (glossary_df["description_suggestion"].str.lower().str.contains(q))
 
 
135
  df = glossary_df[mask].copy()
136
  if df.empty:
137
  tokens = [t for t in q.split() if len(t) > 2]
@@ -166,17 +275,25 @@ def save_to_duckdb(glossary_df: pd.DataFrame, db_path: str = "catalog.duckdb"):
166
  con.execute("INSERT INTO catalog.glossary SELECT * FROM glossary_df")
167
  con.close()
168
 
 
 
169
  st.title("Unified Data Catalog — CSV/Excel MVP")
170
- st.caption("Upload files → auto-generate glossary → discover relationships → ask questions (keyword-based)")
171
 
172
  with st.sidebar:
173
  st.header("1) Upload files")
174
  uploaded = st.file_uploader("Upload CSV/Excel files", type=["csv","xlsx","xls"], accept_multiple_files=True)
 
175
  st.header("2) Options")
176
  headers_only = st.toggle("Header-only mode (no data profiling)", value=False)
177
  role = st.selectbox("Viewer role (PII masking)", ["Marketing/CS (masked)", "Data Steward (full)"])
178
  persist = st.toggle("Persist glossary to DuckDB", value=False, help="Saves to catalog.duckdb (requires duckdb)")
179
 
 
 
 
 
 
180
  tables: Dict[str, pd.DataFrame] = {}
181
 
182
  if uploaded:
@@ -186,6 +303,7 @@ if uploaded:
186
  df = pd.read_csv(file) if not headers_only else pd.read_csv(file, nrows=0)
187
  tables[name.replace(".csv","")] = df
188
  else:
 
189
  xls = pd.ExcelFile(file)
190
  first = xls.sheet_names[0] if xls.sheet_names else None
191
  if first:
@@ -196,6 +314,29 @@ if not tables:
196
  st.info("Upload at least one CSV/Excel to begin.")
197
  st.stop()
198
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
199
  st.subheader("Detected Tables")
200
  tabs = st.tabs([t for t in tables.keys()])
201
  for i, tname in enumerate(tables.keys()):
@@ -204,7 +345,8 @@ for i, tname in enumerate(tables.keys()):
204
  st.write(f"**{tname}** — rows: {len(df):,} (headers only: {headers_only}), columns: {len(df.columns)}")
205
  st.dataframe(df.head(10) if not headers_only else pd.DataFrame(columns=df.columns))
206
 
207
- # Build unified glossary
 
208
  st.subheader("Unified Glossary")
209
  profiles = []
210
  for t, df in tables.items():
@@ -213,14 +355,14 @@ for t, df in tables.items():
213
  profiles.append(prof)
214
  glossary_df = pd.concat(profiles, ignore_index=True)
215
 
216
- # Mask examples for Marketing/CS viewers
217
  if role.startswith("Marketing"):
218
  def mask_examples(lst):
219
  out = []
220
  for v in lst or []:
221
  if isinstance(v, str) and (re.search(EMAIL_RE, v) or re.match(PHONE_RE_BR, v) or re.match(CPF_RE, v)):
222
- if len(v) <= 4: out.append("*" * len(v))
223
- else: out.append(v[:2] + "*" * (len(v)-4) + v[-2:])
224
  else:
225
  out.append(v)
226
  return out
@@ -237,8 +379,6 @@ with right:
237
  # Downloads
238
  st.download_button("Download glossary (CSV)", glossary_df.to_csv(index=False).encode("utf-8"), file_name="glossary.csv")
239
  st.download_button("Download glossary (JSON)", glossary_df.to_json(orient="records").encode("utf-8"), file_name="glossary.json")
240
-
241
- # Markdown export
242
  md = export_markdown(glossary_df)
243
  st.download_button("Export data dictionary (Markdown)", md.encode("utf-8"), file_name="data_dictionary.md")
244
 
@@ -247,15 +387,23 @@ if persist:
247
  save_to_duckdb(glossary_df, db_path="catalog.duckdb")
248
  st.success("Glossary saved to DuckDB (catalog.duckdb).")
249
 
250
- # Relationship discovery
251
- st.subheader("Potential Relationships")
252
- rels = find_relationships(tables, headers_only=headers_only, threshold=0.4)
 
 
 
 
 
 
253
  if rels.empty:
254
  st.info("No strong relationships detected (or disabled in header-only mode).")
255
  else:
256
- st.dataframe(rels, use_container_width=True, height=260)
 
 
 
257
 
258
- # Q&A
259
  st.subheader("Ask the Catalog")
260
  q = st.text_input("Type a keyword or short question (e.g., 'delivery', 'utm campaign', 'order id')")
261
  if st.button("Search"):
@@ -265,3 +413,142 @@ if st.button("Search"):
265
  else:
266
  st.write(f"Top matches for: **{q}**")
267
  st.dataframe(res[["table","column","inferred_type","null_pct","pii","description_suggestion","examples"]])
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # app.py — Unified Data Catalog (Streamlit, Hugging Face friendly)
2
+ # Features:
3
+ # - Upload CSV/XLSX (header-only mode supported)
4
+ # - Auto glossary: type, null%, PII flags, examples
5
+ # - Role-based PII masking (Marketing/CS vs Data Steward)
6
+ # - Manual key tagging + scored relationship inference (name/type/overlap)
7
+ # - Exports: CSV/JSON/Markdown for glossary; key_hints.json; relationships.csv
8
+ # - Optional DuckDB persistence
9
+ # - LLM Enrichment (OpenAI): embeddings -> clustering -> cluster naming -> interactive graph (pyvis)
10
+
11
  import os
12
  import json
13
  import re
 
17
  import numpy as np
18
  import streamlit as st
19
 
20
+ # ---------------- Optional deps (graceful fallbacks) ----------------
21
  HAS_DUCKDB = False
22
  try:
23
  import duckdb
 
25
  except Exception:
26
  HAS_DUCKDB = False
27
 
28
+ HAS_RAPIDFUZZ = False
29
+ try:
30
+ from rapidfuzz import fuzz
31
+ HAS_RAPIDFUZZ = True
32
+ except Exception:
33
+ HAS_RAPIDFUZZ = False
34
+
35
+ HAS_SKLEARN = False
36
+ try:
37
+ from sklearn.decomposition import PCA
38
+ from sklearn.cluster import KMeans
39
+ HAS_SKLEARN = True
40
+ except Exception:
41
+ HAS_SKLEARN = False
42
+
43
+ HAS_PYVIS = False
44
+ try:
45
+ from pyvis.network import Network
46
+ import streamlit.components.v1 as components
47
+ import networkx as nx
48
+ HAS_PYVIS = True
49
+ except Exception:
50
+ HAS_PYVIS = False
51
+
52
+ # OpenAI (new SDK)
53
+ HAS_OPENAI = False
54
+ try:
55
+ from openai import OpenAI
56
+ HAS_OPENAI = True
57
+ except Exception:
58
+ HAS_OPENAI = False
59
+
60
+ # ---------------- App config ----------------
61
  st.set_page_config(page_title="Unified Data Catalog — MVP", layout="wide")
62
 
63
  EMAIL_RE = r'[^@\s]+@[^@\s]+\.[^@\s]+'
64
  PHONE_RE_BR = r'^\+?\d{10,14}$'
65
  CPF_RE = r'^\d{11}$'
66
 
67
+ KNOWN_ID_HINTS = {
68
+ "user_id","customer_id","client_id","id","contact_id",
69
+ "order_id","order","email","cpf","phone","document"
70
+ }
71
+
72
+ # ====================== Core helpers ======================
73
 
74
  def infer_simple_type(s: pd.Series) -> str:
75
  dt = str(s.dtype)
 
77
  if dt.startswith(("float","Float")): return "float"
78
  if dt == "bool": return "boolean"
79
  try:
80
+ pd.to_datetime(s.dropna().head(20), errors="raise", utc=True)
81
+ return "datetime"
82
  except Exception:
83
+ return "string"
 
84
 
85
  def detect_pii_values(vals: List[str]):
86
  flags = {"email": False, "phone": False, "cpf": False}
 
117
  distinct = None
118
  null_pct = None
119
  examples = []
120
+ pii_flags = {
121
+ "email": "email" in col.lower(),
122
+ "phone": "phone" in col.lower() or "telefone" in col.lower(),
123
+ "cpf": "cpf" in col.lower() or "document" in col.lower()
124
+ }
125
  else:
126
  inferred = infer_simple_type(series)
127
  distinct = int(series.nunique(dropna=True))
 
145
  })
146
  return pd.DataFrame(cols)
147
 
148
+ # ================= Relationships (scored) =================
149
+
150
+ def _type_bucket(s: pd.Series) -> str:
151
+ dt = str(s.dtype)
152
+ if dt.startswith(("int","Int")): return "int"
153
+ if dt.startswith(("float","Float")): return "float"
154
+ if dt == "bool": return "bool"
155
+ try:
156
+ pd.to_datetime(s.dropna().head(10), errors="raise")
157
+ return "date"
158
+ except Exception:
159
+ return "str"
160
+
161
+ def _name_similarity(a: str, b: str) -> float:
162
+ a, b = a.lower(), b.lower()
163
+ if HAS_RAPIDFUZZ:
164
+ return max(fuzz.token_sort_ratio(a,b), fuzz.partial_ratio(a,b))/100.0
165
+ # simple fallback
166
+ return 1.0 if a == b else (min(len(a), len(b))/max(len(a), len(b)) if a in b or b in a else 0.0)
167
+
168
+ def _uniqueness(series: pd.Series) -> float:
169
+ s = series.dropna()
170
+ if len(s)==0: return 0.0
171
+ return s.nunique()/len(s)
172
+
173
+ def infer_relationships_scored(
174
+ tables: Dict[str, pd.DataFrame],
175
+ headers_only: bool,
176
+ hints: Dict[str, list],
177
+ overlap_sample:int=5000,
178
+ base_thresh:float=0.55
179
+ ) -> pd.DataFrame:
180
  if headers_only:
181
  return pd.DataFrame()
182
  rows = []
183
+ names = list(tables.keys())
184
+ for i in range(len(names)):
185
+ for j in range(i+1, len(names)):
186
+ t1, t2 = names[i], names[j]
187
  df1, df2 = tables[t1], tables[t2]
188
+ # candidate columns: manual hints first, then heuristics
189
+ cand1 = hints.get(t1) or [c for c in df1.columns if any(k in c.lower() for k in ["id","order","email","cpf","user","customer"])]
190
+ cand2 = hints.get(t2) or [c for c in df2.columns if any(k in c.lower() for k in ["id","order","email","cpf","user","customer"])]
191
+ cand1 = cand1 or list(df1.columns[:5])
192
+ cand2 = cand2 or list(df2.columns[:5])
193
+
194
+ for c1 in cand1:
195
+ s1 = df1[c1]
196
+ type1 = _type_bucket(s1)
197
+ uniq1 = _uniqueness(s1)
198
+ vals1 = set(s1.dropna().astype(str).unique()[:overlap_sample])
199
+ if not vals1:
200
+ continue
201
+ for c2 in cand2:
202
+ s2 = df2[c2]
203
+ type2 = _type_bucket(s2)
204
+ uniq2 = _uniqueness(s2)
205
+ vals2 = set(s2.dropna().astype(str).unique()[:overlap_sample])
206
+ if not vals2:
207
+ continue
208
+
209
+ name_sim = _name_similarity(c1, c2) # 0..1
210
+ type_match = 1.0 if (type1==type2 or {"int","float"}=={type1,type2}) else 0.0
211
+ inter = len(vals1 & vals2)
212
+ denom = max(1, min(len(vals1), len(vals2)))
213
+ value_overlap = inter/denom # 0..1
214
+
215
+ score = 0.40*name_sim + 0.30*type_match + 0.30*value_overlap
216
+
217
+ # bonus for likely keys
218
+ null1 = float(s1.isna().mean()); null2 = float(s2.isna().mean())
219
+ if uniq1>0.7 and uniq2>0.7 and null1<0.3 and null2<0.3:
220
+ score += 0.05
221
+
222
+ if score >= base_thresh:
223
  rows.append({
224
  "left_table": t1, "left_key": c1,
225
  "right_table": t2, "right_key": c2,
226
+ "name_similarity": round(name_sim,3),
227
+ "type_match": int(type_match),
228
+ "value_overlap": round(value_overlap,3),
229
+ "left_uniqueness": round(uniq1,3),
230
+ "right_uniqueness": round(uniq2,3),
231
+ "score": round(min(score,1.0),3),
232
  "relationship": "potential join"
233
  })
234
+ df = pd.DataFrame(rows).sort_values("score", ascending=False)
235
+ return df
236
 
237
  def keyword_search(glossary_df: pd.DataFrame, query: str, top_k: int = 12) -> pd.DataFrame:
238
  if glossary_df is None or glossary_df.empty or not query.strip():
239
  return pd.DataFrame()
240
  q = query.lower()
241
+ mask = (glossary_df["column"].str.lower().str.contains(q)) | (
242
+ glossary_df["description_suggestion"].str.lower().str.contains(q)
243
+ )
244
  df = glossary_df[mask].copy()
245
  if df.empty:
246
  tokens = [t for t in q.split() if len(t) > 2]
 
275
  con.execute("INSERT INTO catalog.glossary SELECT * FROM glossary_df")
276
  con.close()
277
 
278
+ # ====================== UI: Header, Sidebar ======================
279
+
280
  st.title("Unified Data Catalog — CSV/Excel MVP")
281
+ st.caption("Upload → Glossary → Relationships → Q&A → (optional) LLM Enrichment & Graph")
282
 
283
  with st.sidebar:
284
  st.header("1) Upload files")
285
  uploaded = st.file_uploader("Upload CSV/Excel files", type=["csv","xlsx","xls"], accept_multiple_files=True)
286
+
287
  st.header("2) Options")
288
  headers_only = st.toggle("Header-only mode (no data profiling)", value=False)
289
  role = st.selectbox("Viewer role (PII masking)", ["Marketing/CS (masked)", "Data Steward (full)"])
290
  persist = st.toggle("Persist glossary to DuckDB", value=False, help="Saves to catalog.duckdb (requires duckdb)")
291
 
292
+ st.header("3) LLM Enrichment")
293
+ use_llm = st.toggle("Enrich with LLM (clusters + graph)", value=False)
294
+ openai_key = st.text_input("OPENAI_API_KEY", type="password") if use_llm else None
295
+ k_groups = st.slider("Number of semantic groups", 4, 12, 6) if use_llm else None
296
+
297
  tables: Dict[str, pd.DataFrame] = {}
298
 
299
  if uploaded:
 
303
  df = pd.read_csv(file) if not headers_only else pd.read_csv(file, nrows=0)
304
  tables[name.replace(".csv","")] = df
305
  else:
306
+ # Excel (first sheet)
307
  xls = pd.ExcelFile(file)
308
  first = xls.sheet_names[0] if xls.sheet_names else None
309
  if first:
 
314
  st.info("Upload at least one CSV/Excel to begin.")
315
  st.stop()
316
 
317
+ # ====================== Manual key tagging ======================
318
+
319
+ if "key_hints" not in st.session_state:
320
+ st.session_state.key_hints = {}
321
+
322
+ st.subheader("Key Fields (optional manual tagging)")
323
+ for tname, df in tables.items():
324
+ cols = list(df.columns)
325
+ default = [c for c in cols if c.lower() in {"id","order","order_id","email","cpf","user_id","customer_id"}]
326
+ picked = st.multiselect(
327
+ f"Select likely key columns in **{tname}**",
328
+ cols, default=default, key=f"keys_{tname}"
329
+ )
330
+ st.session_state.key_hints[tname] = picked
331
+
332
+ st.download_button(
333
+ "Download key hints (JSON)",
334
+ json.dumps(st.session_state.key_hints, indent=2).encode("utf-8"),
335
+ file_name="key_hints.json"
336
+ )
337
+
338
+ # ====================== Detected tables ======================
339
+
340
  st.subheader("Detected Tables")
341
  tabs = st.tabs([t for t in tables.keys()])
342
  for i, tname in enumerate(tables.keys()):
 
345
  st.write(f"**{tname}** — rows: {len(df):,} (headers only: {headers_only}), columns: {len(df.columns)}")
346
  st.dataframe(df.head(10) if not headers_only else pd.DataFrame(columns=df.columns))
347
 
348
+ # ====================== Glossary ======================
349
+
350
  st.subheader("Unified Glossary")
351
  profiles = []
352
  for t, df in tables.items():
 
355
  profiles.append(prof)
356
  glossary_df = pd.concat(profiles, ignore_index=True)
357
 
358
+ # Role-based masking of examples
359
  if role.startswith("Marketing"):
360
  def mask_examples(lst):
361
  out = []
362
  for v in lst or []:
363
  if isinstance(v, str) and (re.search(EMAIL_RE, v) or re.match(PHONE_RE_BR, v) or re.match(CPF_RE, v)):
364
+ if len(v) <= 4: out.append("*"*len(v))
365
+ else: out.append(v[:2] + "*"*(len(v)-4) + v[-2:])
366
  else:
367
  out.append(v)
368
  return out
 
379
  # Downloads
380
  st.download_button("Download glossary (CSV)", glossary_df.to_csv(index=False).encode("utf-8"), file_name="glossary.csv")
381
  st.download_button("Download glossary (JSON)", glossary_df.to_json(orient="records").encode("utf-8"), file_name="glossary.json")
 
 
382
  md = export_markdown(glossary_df)
383
  st.download_button("Export data dictionary (Markdown)", md.encode("utf-8"), file_name="data_dictionary.md")
384
 
 
387
  save_to_duckdb(glossary_df, db_path="catalog.duckdb")
388
  st.success("Glossary saved to DuckDB (catalog.duckdb).")
389
 
390
+ # ====================== Relationships ======================
391
+
392
+ st.subheader("Potential Relationships (scored)")
393
+ rels = infer_relationships_scored(
394
+ tables,
395
+ headers_only=headers_only,
396
+ hints=st.session_state.key_hints,
397
+ base_thresh=0.55
398
+ )
399
  if rels.empty:
400
  st.info("No strong relationships detected (or disabled in header-only mode).")
401
  else:
402
+ st.dataframe(rels, use_container_width=True, height=320)
403
+ st.download_button("Download relationships (CSV)", rels.to_csv(index=False).encode("utf-8"), file_name="relationships.csv")
404
+
405
+ # ====================== Q&A ======================
406
 
 
407
  st.subheader("Ask the Catalog")
408
  q = st.text_input("Type a keyword or short question (e.g., 'delivery', 'utm campaign', 'order id')")
409
  if st.button("Search"):
 
413
  else:
414
  st.write(f"Top matches for: **{q}**")
415
  st.dataframe(res[["table","column","inferred_type","null_pct","pii","description_suggestion","examples"]])
416
+
417
+ # ====================== LLM Enrichment (optional) ======================
418
+
419
+ def build_text_repr(row):
420
+ ex = ", ".join([str(x) for x in (row.get("examples") or [])])
421
+ return f"{row['table']}.{row['column']} | {row['inferred_type']} | PII={row['pii']} | {row['description_suggestion']} | ex: {ex}"
422
+
423
+ def get_embeddings_openai(texts, api_key):
424
+ # Keep batches small to avoid timeouts in Spaces
425
+ client = OpenAI(api_key=api_key)
426
+ vectors = []
427
+ for t in texts:
428
+ resp = client.embeddings.create(model="text-embedding-3-small", input=t)
429
+ vectors.append(resp.data[0].embedding)
430
+ return np.array(vectors, dtype=np.float32)
431
+
432
+ def cluster_columns(embeddings, k:int=6):
433
+ if not HAS_SKLEARN:
434
+ raise RuntimeError("scikit-learn not installed")
435
+ X = PCA(n_components=min(50, embeddings.shape[1])).fit_transform(embeddings)
436
+ km = KMeans(n_clusters=k, n_init="auto", random_state=42)
437
+ labels = km.fit_predict(X)
438
+ return labels
439
+
440
+ def label_cluster_via_llm(api_key, cluster_samples:list):
441
+ client = OpenAI(api_key=api_key)
442
+ prompt = f"""
443
+ You are a data catalog taxonomist. Name and describe this group of fields in 3-8 words (business-oriented), then list 3 bullets of what analysts can do with it.
444
+ Fields:
445
+ {chr(10).join(cluster_samples[:12])}
446
+ Return JSON with keys: name, description, bullets.
447
+ """
448
+ resp = client.chat.completions.create(
449
+ model="gpt-4o-mini",
450
+ messages=[{"role":"user","content":prompt}],
451
+ temperature=0.2,
452
+ response_format={"type":"json_object"},
453
+ max_tokens=250
454
+ )
455
+ return json.loads(resp.choices[0].message.content)
456
+
457
+ def build_graph(glossary_df, rels_df, cluster_labels, cluster_meta):
458
+ if not HAS_PYVIS:
459
+ raise RuntimeError("pyvis/networkx not installed")
460
+ G = nx.Graph()
461
+ # cluster nodes
462
+ for cid, name in cluster_labels.items():
463
+ G.add_node(f"cluster::{cid}", label=name, color="#6C5CE7", shape="ellipse")
464
+ # table nodes
465
+ for t in sorted(glossary_df["table"].unique()):
466
+ G.add_node(f"table::{t}", label=t, color="#0984E3", shape="box")
467
+ # column nodes (can be heavy for very wide tables; keep for now)
468
+ for _, r in glossary_df.iterrows():
469
+ cid = int(r["cluster_id"])
470
+ node_id = f"col::{r['table']}.{r['column']}"
471
+ G.add_node(node_id, label=r["column"], color="#74B9FF", shape="dot")
472
+ G.add_edge(f"table::{r['table']}", node_id, weight=1)
473
+ G.add_edge(f"cluster::{cid}", node_id, weight=1)
474
+ # relationship edges between tables
475
+ for _, e in rels_df.iterrows():
476
+ lt = e["left_table"]; rt = e["right_table"]
477
+ w = float(e.get("score", 0.6))
478
+ G.add_edge(f"table::{lt}", f"table::{rt}", weight=5*w, color="#00B894")
479
+ return G
480
+
481
+ def render_pyvis(G, height="680px"):
482
+ net = Network(height=height, width="100%", notebook=False, cdn_resources="in_line")
483
+ net.from_nx(G)
484
+ net.toggle_physics(True)
485
+ html = net.generate_html()
486
+ components.html(html, height=700, scrolling=True)
487
+
488
+ if use_llm:
489
+ # Dependency/key checks
490
+ if not HAS_OPENAI:
491
+ st.warning("LLM enrichment unavailable: OpenAI SDK not installed.")
492
+ elif not openai_key:
493
+ st.info("Enter OPENAI_API_KEY in the sidebar to enable enrichment.")
494
+ elif not HAS_SKLEARN:
495
+ st.warning("LLM enrichment unavailable: scikit-learn not installed.")
496
+ else:
497
+ st.subheader("Semantic Groups (LLM-enriched)")
498
+ try:
499
+ # 1) texts & embeddings
500
+ texts = [build_text_repr(r) for _, r in glossary_df.iterrows()]
501
+ with st.spinner("Computing embeddings..."):
502
+ embs = get_embeddings_openai(texts, openai_key)
503
+
504
+ # 2) clustering
505
+ with st.spinner("Clustering columns..."):
506
+ labels = cluster_columns(embs, k=k_groups or 6)
507
+ glossary_df["cluster_id"] = labels
508
+
509
+ # 3) label clusters
510
+ cluster_labels, cluster_meta = {}, {}
511
+ for cid in sorted(set(labels)):
512
+ samples = [texts[i] for i in np.where(labels==cid)[0][:15]]
513
+ try:
514
+ meta = label_cluster_via_llm(openai_key, samples)
515
+ cluster_labels[cid] = meta.get("name") or f"Group {cid}"
516
+ cluster_meta[cid] = meta
517
+ except Exception:
518
+ cluster_labels[cid] = f"Group {cid}"
519
+ cluster_meta[cid] = {"description":"", "bullets":[]}
520
+
521
+ # show groups
522
+ for cid in sorted(cluster_labels.keys()):
523
+ sub = glossary_df[glossary_df["cluster_id"]==cid]
524
+ st.markdown(f"**{cluster_labels[cid]}** — {len(sub)} fields")
525
+ if cluster_meta[cid].get("description"):
526
+ st.caption(cluster_meta[cid]["description"])
527
+ cols_preview = ", ".join([f"{r['table']}.{r['column']}" for _, r in sub.head(8).iterrows()])
528
+ if cols_preview:
529
+ st.write(cols_preview)
530
+
531
+ # 4) graph
532
+ st.subheader("Visual Map — Data Ecosystem")
533
+ if not rels.empty and HAS_PYVIS:
534
+ G = build_graph(glossary_df, rels, cluster_labels, cluster_meta)
535
+ render_pyvis(G)
536
+ elif not HAS_PYVIS:
537
+ st.info("Install pyvis & networkx to render the interactive graph.")
538
+ else:
539
+ st.info("No relationships available to render between tables.")
540
+
541
+ # 5) downloads
542
+ payload = {
543
+ "cluster_labels": cluster_labels,
544
+ "cluster_meta": cluster_meta,
545
+ "columns": glossary_df[["table","column","cluster_id"]].to_dict(orient="records")
546
+ }
547
+ st.download_button(
548
+ "Download semantic groups (JSON)",
549
+ json.dumps(payload, indent=2).encode("utf-8"),
550
+ file_name="semantic_groups.json"
551
+ )
552
+
553
+ except Exception as e:
554
+ st.error(f"LLM enrichment failed: {e}")