Commit ·
cf34801
1
Parent(s): 0fb7af8
[NOTICKET] feat(cluster): name example members per cluster
Browse filesCo-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
src/tools/analytics/cluster.py
CHANGED
|
@@ -47,6 +47,9 @@ _RANDOM_STATE = 42
|
|
| 47 |
_WEAK_SILHOUETTE = 0.25
|
| 48 |
# Chart column name (internal; never collides with user data).
|
| 49 |
_CLUSTER_COL = "cluster"
|
|
|
|
|
|
|
|
|
|
| 50 |
|
| 51 |
|
| 52 |
class NotEnoughDataError(ValueError):
|
|
@@ -68,6 +71,11 @@ class ClusterInput(BaseModel):
|
|
| 68 |
description=f"Number of clusters; omit to choose automatically between "
|
| 69 |
f"{_K_MIN} and {_K_MAX} by silhouette score.",
|
| 70 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 71 |
|
| 72 |
|
| 73 |
DESCRIPTION = build_description(
|
|
@@ -82,7 +90,8 @@ DESCRIPTION = build_description(
|
|
| 82 |
"the user wants totals per an EXISTING category column -> analyze_aggregate",
|
| 83 |
],
|
| 84 |
output="the chosen k with its silhouette score, one profile row per cluster "
|
| 85 |
-
"(size, share, per-column means), and a scatter
|
|
|
|
| 86 |
examples=[
|
| 87 |
"ada berapa kelompok pelanggan berdasarkan demografi?",
|
| 88 |
"what natural groups are in this data?",
|
|
@@ -128,19 +137,63 @@ def _fit_kmeans(x: np.ndarray, k_values: list[int], random_state: int):
|
|
| 128 |
return best[0], best[1], best[2], scores
|
| 129 |
|
| 130 |
|
| 131 |
-
def
|
| 132 |
-
"""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 133 |
total = len(labels)
|
| 134 |
out: list[dict] = []
|
| 135 |
for label in sorted(set(labels.tolist())):
|
| 136 |
-
|
| 137 |
profile: dict[str, object] = {
|
| 138 |
"cluster": int(label),
|
| 139 |
-
"size": int(len(
|
| 140 |
-
"share": round(len(
|
| 141 |
}
|
| 142 |
for col in column_ids:
|
| 143 |
-
profile[col] = _clean(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 144 |
out.append(profile)
|
| 145 |
return out
|
| 146 |
|
|
@@ -149,9 +202,14 @@ async def analyze_cluster(
|
|
| 149 |
df: pd.DataFrame,
|
| 150 |
column_ids: list[str],
|
| 151 |
k: int | None = None,
|
|
|
|
| 152 |
) -> dict[str, object]:
|
| 153 |
"""Group rows by `column_ids` with KMeans, choosing k by silhouette when unset.
|
| 154 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 155 |
Async ONLY so the fit search can be offloaded via `run_fit`; everything else is
|
| 156 |
synchronous compute. `data` is materialised upstream (Pattern A) and arrives as
|
| 157 |
`df`.
|
|
@@ -167,7 +225,8 @@ async def analyze_cluster(
|
|
| 167 |
"""
|
| 168 |
if not column_ids:
|
| 169 |
raise EmptyColumnSelectionError("column_ids must name at least one numeric column")
|
| 170 |
-
|
|
|
|
| 171 |
if missing:
|
| 172 |
raise ColumnNotFoundError(f"columns not found: {missing}")
|
| 173 |
|
|
@@ -242,7 +301,10 @@ async def analyze_cluster(
|
|
| 242 |
"weak": silhouette <= _WEAK_SILHOUETTE,
|
| 243 |
},
|
| 244 |
"silhouette_by_k": by_k,
|
| 245 |
-
"clusters": _profiles(
|
|
|
|
|
|
|
|
|
|
| 246 |
"caveats": caveats,
|
| 247 |
"charts": [chart],
|
| 248 |
}
|
|
|
|
| 47 |
_WEAK_SILHOUETTE = 0.25
|
| 48 |
# Chart column name (internal; never collides with user data).
|
| 49 |
_CLUSTER_COL = "cluster"
|
| 50 |
+
# Members named per cluster, most typical (nearest its centre) first. The test demo
|
| 51 |
+
# answer said "cluster 1: 67 units" and could not say WHICH units.
|
| 52 |
+
_MAX_MEMBERS = 15
|
| 53 |
|
| 54 |
|
| 55 |
class NotEnoughDataError(ValueError):
|
|
|
|
| 71 |
description=f"Number of clusters; omit to choose automatically between "
|
| 72 |
f"{_K_MIN} and {_K_MAX} by silhouette score.",
|
| 73 |
)
|
| 74 |
+
label_columns: list[str] | None = Field(
|
| 75 |
+
None, description="Identifier columns (e.g. unit/equipment no.) used to NAME each "
|
| 76 |
+
"cluster's members. Never fed to the model; omit to use the table's most varied "
|
| 77 |
+
"text column."
|
| 78 |
+
)
|
| 79 |
|
| 80 |
|
| 81 |
DESCRIPTION = build_description(
|
|
|
|
| 90 |
"the user wants totals per an EXISTING category column -> analyze_aggregate",
|
| 91 |
],
|
| 92 |
output="the chosen k with its silhouette score, one profile row per cluster "
|
| 93 |
+
"(size, share, per-column means, its most typical members by name), and a scatter "
|
| 94 |
+
"coloured by cluster.",
|
| 95 |
examples=[
|
| 96 |
"ada berapa kelompok pelanggan berdasarkan demografi?",
|
| 97 |
"what natural groups are in this data?",
|
|
|
|
| 137 |
return best[0], best[1], best[2], scores
|
| 138 |
|
| 139 |
|
| 140 |
+
def _default_labels(df: pd.DataFrame, column_ids: list[str]) -> list[str]:
|
| 141 |
+
"""The most varied text column not being clustered (e.g. `eqm_no`), or none."""
|
| 142 |
+
text = [
|
| 143 |
+
c for c in df.columns
|
| 144 |
+
if c not in column_ids
|
| 145 |
+
and (pd.api.types.is_object_dtype(df[c]) or pd.api.types.is_string_dtype(df[c]))
|
| 146 |
+
]
|
| 147 |
+
return [max(text, key=lambda c: df[c].nunique())] if text else []
|
| 148 |
+
|
| 149 |
+
|
| 150 |
+
def _member_names(
|
| 151 |
+
df: pd.DataFrame, frame: pd.DataFrame, labels: np.ndarray, label_columns: list[str]
|
| 152 |
+
) -> dict[int, list[str]]:
|
| 153 |
+
"""Per cluster, up to _MAX_MEMBERS distinct row names, nearest the centre first.
|
| 154 |
+
|
| 155 |
+
Distance is measured on standardised columns (as the fit saw them), so a column
|
| 156 |
+
with large units does not decide who is "typical". Names repeat when rows are
|
| 157 |
+
unit-months (local run 2 Nov); each is listed once, at its most typical row.
|
| 158 |
+
"""
|
| 159 |
+
x = frame.to_numpy(dtype=float)
|
| 160 |
+
std = x.std(axis=0)
|
| 161 |
+
x = (x - x.mean(axis=0)) / np.where(std == 0, 1.0, std)
|
| 162 |
+
names = df.loc[frame.index, label_columns].astype(str).agg(" · ".join, axis=1).to_numpy()
|
| 163 |
+
out: dict[int, list[str]] = {}
|
| 164 |
+
for label in sorted(set(labels.tolist())):
|
| 165 |
+
mask = labels == label
|
| 166 |
+
dist = np.linalg.norm(x[mask] - x[mask].mean(axis=0), axis=1)
|
| 167 |
+
ordered = dict.fromkeys(names[mask][np.argsort(dist, kind="stable")].tolist())
|
| 168 |
+
out[int(label)] = list(ordered)[:_MAX_MEMBERS]
|
| 169 |
+
return out
|
| 170 |
+
|
| 171 |
+
|
| 172 |
+
def _profiles(
|
| 173 |
+
frame: pd.DataFrame,
|
| 174 |
+
labels: np.ndarray,
|
| 175 |
+
column_ids: list[str],
|
| 176 |
+
members: dict[int, list[str]] | None = None,
|
| 177 |
+
) -> list[dict]:
|
| 178 |
+
"""One row per cluster: size, share of rows, the mean of each column and, when
|
| 179 |
+
rows can be named, its most typical members."""
|
| 180 |
total = len(labels)
|
| 181 |
out: list[dict] = []
|
| 182 |
for label in sorted(set(labels.tolist())):
|
| 183 |
+
rows = frame[labels == label]
|
| 184 |
profile: dict[str, object] = {
|
| 185 |
"cluster": int(label),
|
| 186 |
+
"size": int(len(rows)),
|
| 187 |
+
"share": round(len(rows) / total, 4) if total else 0.0,
|
| 188 |
}
|
| 189 |
for col in column_ids:
|
| 190 |
+
profile[col] = _clean(rows[col].mean())
|
| 191 |
+
if members is not None:
|
| 192 |
+
# A bounded, most-typical-first SAMPLE, named so (local run 2 Nov: a bare
|
| 193 |
+
# `members` list of 15 was presented as the whole group of 109). `size`
|
| 194 |
+
# stays the true row count.
|
| 195 |
+
profile["example_members"] = members[int(label)]
|
| 196 |
+
profile["members_shown"] = len(members[int(label)])
|
| 197 |
out.append(profile)
|
| 198 |
return out
|
| 199 |
|
|
|
|
| 202 |
df: pd.DataFrame,
|
| 203 |
column_ids: list[str],
|
| 204 |
k: int | None = None,
|
| 205 |
+
label_columns: list[str] | None = None,
|
| 206 |
) -> dict[str, object]:
|
| 207 |
"""Group rows by `column_ids` with KMeans, choosing k by silhouette when unset.
|
| 208 |
|
| 209 |
+
`label_columns` name each cluster's members (never fed to the model); when omitted
|
| 210 |
+
the table's most varied text column is used, and with none the profiles carry no
|
| 211 |
+
`example_members`.
|
| 212 |
+
|
| 213 |
Async ONLY so the fit search can be offloaded via `run_fit`; everything else is
|
| 214 |
synchronous compute. `data` is materialised upstream (Pattern A) and arrives as
|
| 215 |
`df`.
|
|
|
|
| 225 |
"""
|
| 226 |
if not column_ids:
|
| 227 |
raise EmptyColumnSelectionError("column_ids must name at least one numeric column")
|
| 228 |
+
names_by = list(label_columns) if label_columns else _default_labels(df, column_ids)
|
| 229 |
+
missing = [c for c in column_ids + names_by if c not in df.columns]
|
| 230 |
if missing:
|
| 231 |
raise ColumnNotFoundError(f"columns not found: {missing}")
|
| 232 |
|
|
|
|
| 301 |
"weak": silhouette <= _WEAK_SILHOUETTE,
|
| 302 |
},
|
| 303 |
"silhouette_by_k": by_k,
|
| 304 |
+
"clusters": _profiles(
|
| 305 |
+
frame, labels, capped,
|
| 306 |
+
_member_names(df, frame, labels, names_by) if names_by else None,
|
| 307 |
+
),
|
| 308 |
"caveats": caveats,
|
| 309 |
"charts": [chart],
|
| 310 |
}
|