sofhiaazzhr Claude Opus 5.5 commited on
Commit
cf34801
·
1 Parent(s): 0fb7af8

[NOTICKET] feat(cluster): name example members per cluster

Browse files

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

Files changed (1) hide show
  1. src/tools/analytics/cluster.py +71 -9
src/tools/analytics/cluster.py CHANGED
@@ -47,6 +47,9 @@ _RANDOM_STATE = 42
47
  _WEAK_SILHOUETTE = 0.25
48
  # Chart column name (internal; never collides with user data).
49
  _CLUSTER_COL = "cluster"
 
 
 
50
 
51
 
52
  class NotEnoughDataError(ValueError):
@@ -68,6 +71,11 @@ class ClusterInput(BaseModel):
68
  description=f"Number of clusters; omit to choose automatically between "
69
  f"{_K_MIN} and {_K_MAX} by silhouette score.",
70
  )
 
 
 
 
 
71
 
72
 
73
  DESCRIPTION = build_description(
@@ -82,7 +90,8 @@ DESCRIPTION = build_description(
82
  "the user wants totals per an EXISTING category column -> analyze_aggregate",
83
  ],
84
  output="the chosen k with its silhouette score, one profile row per cluster "
85
- "(size, share, per-column means), and a scatter coloured by cluster.",
 
86
  examples=[
87
  "ada berapa kelompok pelanggan berdasarkan demografi?",
88
  "what natural groups are in this data?",
@@ -128,19 +137,63 @@ def _fit_kmeans(x: np.ndarray, k_values: list[int], random_state: int):
128
  return best[0], best[1], best[2], scores
129
 
130
 
131
- def _profiles(frame: pd.DataFrame, labels: np.ndarray, column_ids: list[str]) -> list[dict]:
132
- """One row per cluster: size, share of rows, and the mean of each column."""
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
133
  total = len(labels)
134
  out: list[dict] = []
135
  for label in sorted(set(labels.tolist())):
136
- members = frame[labels == label]
137
  profile: dict[str, object] = {
138
  "cluster": int(label),
139
- "size": int(len(members)),
140
- "share": round(len(members) / total, 4) if total else 0.0,
141
  }
142
  for col in column_ids:
143
- profile[col] = _clean(members[col].mean())
 
 
 
 
 
 
144
  out.append(profile)
145
  return out
146
 
@@ -149,9 +202,14 @@ async def analyze_cluster(
149
  df: pd.DataFrame,
150
  column_ids: list[str],
151
  k: int | None = None,
 
152
  ) -> dict[str, object]:
153
  """Group rows by `column_ids` with KMeans, choosing k by silhouette when unset.
154
 
 
 
 
 
155
  Async ONLY so the fit search can be offloaded via `run_fit`; everything else is
156
  synchronous compute. `data` is materialised upstream (Pattern A) and arrives as
157
  `df`.
@@ -167,7 +225,8 @@ async def analyze_cluster(
167
  """
168
  if not column_ids:
169
  raise EmptyColumnSelectionError("column_ids must name at least one numeric column")
170
- missing = [c for c in column_ids if c not in df.columns]
 
171
  if missing:
172
  raise ColumnNotFoundError(f"columns not found: {missing}")
173
 
@@ -242,7 +301,10 @@ async def analyze_cluster(
242
  "weak": silhouette <= _WEAK_SILHOUETTE,
243
  },
244
  "silhouette_by_k": by_k,
245
- "clusters": _profiles(frame, labels, capped),
 
 
 
246
  "caveats": caveats,
247
  "charts": [chart],
248
  }
 
47
  _WEAK_SILHOUETTE = 0.25
48
  # Chart column name (internal; never collides with user data).
49
  _CLUSTER_COL = "cluster"
50
+ # Members named per cluster, most typical (nearest its centre) first. The test demo
51
+ # answer said "cluster 1: 67 units" and could not say WHICH units.
52
+ _MAX_MEMBERS = 15
53
 
54
 
55
  class NotEnoughDataError(ValueError):
 
71
  description=f"Number of clusters; omit to choose automatically between "
72
  f"{_K_MIN} and {_K_MAX} by silhouette score.",
73
  )
74
+ label_columns: list[str] | None = Field(
75
+ None, description="Identifier columns (e.g. unit/equipment no.) used to NAME each "
76
+ "cluster's members. Never fed to the model; omit to use the table's most varied "
77
+ "text column."
78
+ )
79
 
80
 
81
  DESCRIPTION = build_description(
 
90
  "the user wants totals per an EXISTING category column -> analyze_aggregate",
91
  ],
92
  output="the chosen k with its silhouette score, one profile row per cluster "
93
+ "(size, share, per-column means, its most typical members by name), and a scatter "
94
+ "coloured by cluster.",
95
  examples=[
96
  "ada berapa kelompok pelanggan berdasarkan demografi?",
97
  "what natural groups are in this data?",
 
137
  return best[0], best[1], best[2], scores
138
 
139
 
140
+ def _default_labels(df: pd.DataFrame, column_ids: list[str]) -> list[str]:
141
+ """The most varied text column not being clustered (e.g. `eqm_no`), or none."""
142
+ text = [
143
+ c for c in df.columns
144
+ if c not in column_ids
145
+ and (pd.api.types.is_object_dtype(df[c]) or pd.api.types.is_string_dtype(df[c]))
146
+ ]
147
+ return [max(text, key=lambda c: df[c].nunique())] if text else []
148
+
149
+
150
+ def _member_names(
151
+ df: pd.DataFrame, frame: pd.DataFrame, labels: np.ndarray, label_columns: list[str]
152
+ ) -> dict[int, list[str]]:
153
+ """Per cluster, up to _MAX_MEMBERS distinct row names, nearest the centre first.
154
+
155
+ Distance is measured on standardised columns (as the fit saw them), so a column
156
+ with large units does not decide who is "typical". Names repeat when rows are
157
+ unit-months (local run 2 Nov); each is listed once, at its most typical row.
158
+ """
159
+ x = frame.to_numpy(dtype=float)
160
+ std = x.std(axis=0)
161
+ x = (x - x.mean(axis=0)) / np.where(std == 0, 1.0, std)
162
+ names = df.loc[frame.index, label_columns].astype(str).agg(" · ".join, axis=1).to_numpy()
163
+ out: dict[int, list[str]] = {}
164
+ for label in sorted(set(labels.tolist())):
165
+ mask = labels == label
166
+ dist = np.linalg.norm(x[mask] - x[mask].mean(axis=0), axis=1)
167
+ ordered = dict.fromkeys(names[mask][np.argsort(dist, kind="stable")].tolist())
168
+ out[int(label)] = list(ordered)[:_MAX_MEMBERS]
169
+ return out
170
+
171
+
172
+ def _profiles(
173
+ frame: pd.DataFrame,
174
+ labels: np.ndarray,
175
+ column_ids: list[str],
176
+ members: dict[int, list[str]] | None = None,
177
+ ) -> list[dict]:
178
+ """One row per cluster: size, share of rows, the mean of each column and, when
179
+ rows can be named, its most typical members."""
180
  total = len(labels)
181
  out: list[dict] = []
182
  for label in sorted(set(labels.tolist())):
183
+ rows = frame[labels == label]
184
  profile: dict[str, object] = {
185
  "cluster": int(label),
186
+ "size": int(len(rows)),
187
+ "share": round(len(rows) / total, 4) if total else 0.0,
188
  }
189
  for col in column_ids:
190
+ profile[col] = _clean(rows[col].mean())
191
+ if members is not None:
192
+ # A bounded, most-typical-first SAMPLE, named so (local run 2 Nov: a bare
193
+ # `members` list of 15 was presented as the whole group of 109). `size`
194
+ # stays the true row count.
195
+ profile["example_members"] = members[int(label)]
196
+ profile["members_shown"] = len(members[int(label)])
197
  out.append(profile)
198
  return out
199
 
 
202
  df: pd.DataFrame,
203
  column_ids: list[str],
204
  k: int | None = None,
205
+ label_columns: list[str] | None = None,
206
  ) -> dict[str, object]:
207
  """Group rows by `column_ids` with KMeans, choosing k by silhouette when unset.
208
 
209
+ `label_columns` name each cluster's members (never fed to the model); when omitted
210
+ the table's most varied text column is used, and with none the profiles carry no
211
+ `example_members`.
212
+
213
  Async ONLY so the fit search can be offloaded via `run_fit`; everything else is
214
  synchronous compute. `data` is materialised upstream (Pattern A) and arrives as
215
  `df`.
 
225
  """
226
  if not column_ids:
227
  raise EmptyColumnSelectionError("column_ids must name at least one numeric column")
228
+ names_by = list(label_columns) if label_columns else _default_labels(df, column_ids)
229
+ missing = [c for c in column_ids + names_by if c not in df.columns]
230
  if missing:
231
  raise ColumnNotFoundError(f"columns not found: {missing}")
232
 
 
301
  "weak": silhouette <= _WEAK_SILHOUETTE,
302
  },
303
  "silhouette_by_k": by_k,
304
+ "clusters": _profiles(
305
+ frame, labels, capped,
306
+ _member_names(df, frame, labels, names_by) if names_by else None,
307
+ ),
308
  "caveats": caveats,
309
  "charts": [chart],
310
  }