Upload 117 files

README.md

@@ -1,10 +1,37 @@
----
-title: ClusterBuster API
-emoji: 🔥
-colorFrom: green
-colorTo: red
-sdk: docker
-pinned: false
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# ClusterForge
+
+Full stack clustering experimentation platform.
+
+Stack:
+- FastAPI backend
+- Next.js frontend
+- SQLite database
+- scikit-learn clustering pipeline
+- Hugging Face Docker Space deployment files
+
+Core features:
+- CSV and Excel upload
+- Dataset profiling
+- Algorithm selection
+- Dynamic parameters
+- Run experiments
+- Result visualizations
+- Cluster summaries
+- Outlier detection
+- Run comparison
+- CSV export
+
+Algorithms:
+- KMeans
+- MiniBatchKMeans
+- Agglomerative
+- DBSCAN
+- HDBSCAN
+- Gaussian Mixture
+- Spectral
+- Birch
+
+Runtime paths for Hugging Face:
+- DB: /data/clusterforge.db
+- Uploads: /data/uploads
+- Artifacts: /data/artifacts

backend/app/api/clusters.py

@@ -0,0 +1,40 @@
+import json
+import pandas as pd
+from fastapi import APIRouter, HTTPException
+from app.db import get_session
+from app.repositories.cluster_repo import ClusterRepository
+from app.repositories.dataset_repo import DatasetRepository
+from app.repositories.experiment_repo import ExperimentRepository
+from app.schemas.cluster import ClusterDetailResponse, ClusterMember
+
+router = APIRouter(prefix='/clusters', tags=['clusters'])
+
+@router.get('/{experiment_id}/{cluster_label}', response_model=ClusterDetailResponse)
+def get_cluster_detail(experiment_id: str, cluster_label: int):
+    with get_session() as session:
+        exp_repo = ExperimentRepository(session)
+        dataset_repo = DatasetRepository(session)
+        cluster_repo = ClusterRepository(session)
+        exp = exp_repo.get(experiment_id)
+        if not exp:
+            raise HTTPException(status_code=404, detail='Experiment not found')
+        dataset = dataset_repo.get(exp.dataset_id)
+        if not dataset:
+            raise HTTPException(status_code=404, detail='Dataset not found')
+        df = pd.read_csv(dataset.file_path) if dataset.file_path.lower().endswith('.csv') else pd.read_excel(dataset.file_path)
+        results = [r for r in cluster_repo.get_results(experiment_id) if r.cluster_label == cluster_label]
+        summary = cluster_repo.get_summary(experiment_id, cluster_label)
+        if not summary:
+            raise HTTPException(status_code=404, detail='Cluster summary not found')
+        members = []
+        for row in results[:200]:
+            values = df.iloc[row.row_index].to_dict()
+            values = {str(k): (None if pd.isna(v) else v) for k, v in values.items()}
+            members.append(ClusterMember(row_index=row.row_index, values=values))
+        return ClusterDetailResponse(
+            experiment_id=experiment_id,
+            cluster_label=cluster_label,
+            members=members,
+            summary=json.loads(summary.summary_json),
+            description_text=summary.description_text,
+        )

backend/app/api/compare.py

@@ -0,0 +1,19 @@
+from fastapi import APIRouter, Query
+from app.db import get_session
+from app.repositories.cluster_repo import ClusterRepository
+from app.repositories.experiment_repo import ExperimentRepository
+from app.schemas.compare import CompareResponse
+from app.services.comparison_service import ComparisonService
+
+router = APIRouter(prefix='/compare', tags=['compare'])
+
+@router.get('', response_model=CompareResponse)
+def compare_runs(experiment_ids: str = Query(...)):
+    ids = [item.strip() for item in experiment_ids.split(',') if item.strip()]
+    with get_session() as session:
+        payload = ComparisonService().build_compare_payload(
+            ExperimentRepository(session),
+            ClusterRepository(session),
+            ids,
+        )
+        return CompareResponse(**payload)

backend/app/api/datasets.py

@@ -0,0 +1,55 @@
+from fastapi import APIRouter, File, HTTPException, UploadFile
+from app.db import get_session
+from app.repositories.dataset_repo import DatasetRepository
+from app.schemas.dataset import DatasetProfileResponse, DatasetUploadResponse
+from app.services.dataset_service import DatasetService
+from app.services.profiling_service import ProfilingService
+
+router = APIRouter(prefix='/datasets', tags=['datasets'])
+
+@router.post('/upload', response_model=DatasetUploadResponse)
+async def upload_dataset(file: UploadFile = File(...)):
+    with get_session() as session:
+        repo = DatasetRepository(session)
+        service = DatasetService(repo)
+        dataset = await service.save_upload(file)
+        return DatasetUploadResponse(
+            dataset_id=dataset.id,
+            name=dataset.name,
+            row_count=dataset.row_count,
+            column_count=dataset.column_count,
+        )
+
+@router.get('', response_model=list[DatasetUploadResponse])
+def list_datasets():
+    with get_session() as session:
+        repo = DatasetRepository(session)
+        items = repo.list_all()
+        return [
+            DatasetUploadResponse(
+                dataset_id=item.id,
+                name=item.name,
+                row_count=item.row_count,
+                column_count=item.column_count,
+            )
+            for item in items
+        ]
+
+@router.get('/{dataset_id}/profile', response_model=DatasetProfileResponse)
+def dataset_profile(dataset_id: str):
+    with get_session() as session:
+        repo = DatasetRepository(session)
+        dataset = repo.get(dataset_id)
+        if not dataset:
+            raise HTTPException(status_code=404, detail='Dataset not found')
+        df = DatasetService(repo).load_dataframe(dataset)
+        profile = ProfilingService().profile(df)
+        repo.update_profile(dataset_id, schema_json={'columns': [c['name'] for c in profile['columns']]}, profile_json=profile)
+        return DatasetProfileResponse(
+            dataset_id=dataset_id,
+            columns=profile['columns'],
+            suggested_feature_columns=profile['suggested_feature_columns'],
+            recommended_algorithms=profile['recommended_algorithms'],
+            row_count=profile['row_count'],
+            column_count=profile['column_count'],
+        )

backend/app/api/experiments.py

@@ -0,0 +1,105 @@
+import json
+from fastapi import APIRouter, HTTPException
+from app.db import get_session
+from app.repositories.cluster_repo import ClusterRepository
+from app.repositories.dataset_repo import DatasetRepository
+from app.repositories.experiment_repo import ExperimentRepository
+from app.schemas.experiment import ClusterSummaryResponse, ExperimentListItem, ExperimentResultResponse, ExperimentRunRequest, ExperimentRunResponse, MetricResponse, ScatterPoint
+from app.services.run_service import RunService
+
+router = APIRouter(prefix='/experiments', tags=['experiments'])
+
+@router.post('/run', response_model=ExperimentRunResponse)
+def run_experiment(payload: ExperimentRunRequest):
+    with get_session() as session:
+        dataset_repo = DatasetRepository(session)
+        exp_repo = ExperimentRepository(session)
+        cluster_repo = ClusterRepository(session)
+        service = RunService(dataset_repo, exp_repo, cluster_repo)
+        experiment = service.run(payload.model_dump())
+        return ExperimentRunResponse(experiment_id=experiment.id, status='completed')
+
+@router.get('', response_model=list[ExperimentListItem])
+def list_experiments():
+    with get_session() as session:
+        repo = ExperimentRepository(session)
+        items = repo.list_all()
+        return [
+            ExperimentListItem(
+                experiment_id=item.id,
+                dataset_id=item.dataset_id,
+                name=item.name,
+                algorithm=item.algorithm,
+                status=item.status,
+                started_at=item.started_at.isoformat(),
+                completed_at=item.completed_at.isoformat() if item.completed_at else None,
+            )
+            for item in items
+        ]
+
+@router.get('/{experiment_id}')
+def get_experiment(experiment_id: str):
+    with get_session() as session:
+        repo = ExperimentRepository(session)
+        exp = repo.get(experiment_id)
+        if not exp:
+            raise HTTPException(status_code=404, detail='Experiment not found')
+        return {
+            'experiment_id': exp.id,
+            'dataset_id': exp.dataset_id,
+            'name': exp.name,
+            'algorithm': exp.algorithm,
+            'status': exp.status,
+            'config': json.loads(exp.config_json),
+            'runtime_ms': exp.runtime_ms,
+            'error_message': exp.error_message,
+        }
+
+@router.get('/{experiment_id}/results', response_model=ExperimentResultResponse)
+def get_results(experiment_id: str):
+    with get_session() as session:
+        exp_repo = ExperimentRepository(session)
+        cluster_repo = ClusterRepository(session)
+        exp = exp_repo.get(experiment_id)
+        if not exp:
+            raise HTTPException(status_code=404, detail='Experiment not found')
+        metric = cluster_repo.get_metric(experiment_id)
+        summaries = cluster_repo.get_summaries(experiment_id)
+        results = cluster_repo.get_results(experiment_id)
+        if not metric:
+            raise HTTPException(status_code=404, detail='Results not found')
+        scatter = [
+            ScatterPoint(
+                row_index=item.row_index,
+                cluster_label=item.cluster_label,
+                is_outlier=item.is_outlier,
+                x=item.umap_x if item.umap_x is not None else item.pca_x,
+                y=item.umap_y if item.umap_y is not None else item.pca_y,
+            )
+            for item in results
+        ]
+        summary_items = [
+            ClusterSummaryResponse(
+                cluster_label=item.cluster_label,
+                size=item.size,
+                description_text=item.description_text,
+                summary_json=json.loads(item.summary_json),
+            )
+            for item in summaries
+        ]
+        return ExperimentResultResponse(
+            experiment_id=experiment_id,
+            algorithm=exp.algorithm,
+            metrics=MetricResponse(
+                cluster_count=metric.cluster_count,
+                noise_ratio=metric.noise_ratio,
+                silhouette_score=metric.silhouette_score,
+                davies_bouldin_score=metric.davies_bouldin_score,
+                calinski_harabasz_score=metric.calinski_harabasz_score,
+                stability_score=metric.stability_score,
+                interpretability_score=metric.interpretability_score,
+            ),
+            scatter_points=scatter,
+            cluster_summaries=summary_items,
+            status=exp.status,
+        )

backend/app/api/exports.py

@@ -0,0 +1,24 @@
+from fastapi import APIRouter, HTTPException
+from app.db import get_session
+from app.repositories.artifact_repo import ArtifactRepository
+from app.repositories.cluster_repo import ClusterRepository
+from app.repositories.dataset_repo import DatasetRepository
+from app.repositories.experiment_repo import ExperimentRepository
+from app.schemas.export import ExportResponse
+from app.services.export_service import ExportService
+
+router = APIRouter(prefix='/exports', tags=['exports'])
+
+@router.get('/{experiment_id}', response_model=ExportResponse)
+def export_experiment(experiment_id: str):
+    with get_session() as session:
+        exp = ExperimentRepository(session).get(experiment_id)
+        if not exp:
+            raise HTTPException(status_code=404, detail='Experiment not found')
+        service = ExportService(
+            dataset_repo=DatasetRepository(session),
+            cluster_repo=ClusterRepository(session),
+            artifact_repo=ArtifactRepository(session),
+        )
+        path = service.export_csv(experiment_id, exp.dataset_id)
+        return ExportResponse(experiment_id=experiment_id, download_path=path)

backend/app/api/health.py

@@ -0,0 +1,7 @@
+from fastapi import APIRouter
+from app.schemas.health import HealthResponse
+router = APIRouter(tags=['health'])
+
+@router.get('/health', response_model=HealthResponse)
+def health() -> HealthResponse:
+    return HealthResponse(ok=True, service='clusterforge-api')

backend/app/api/presets.py

@@ -0,0 +1,17 @@
+from fastapi import APIRouter
+router = APIRouter(prefix='/presets', tags=['presets'])
+
+@router.get('/algorithms')
+def list_algorithms():
+    return {
+        'algorithms': [
+            {'key': 'kmeans', 'label': 'KMeans', 'params': {'n_clusters': 4, 'random_state': 42, 'max_iter': 300, 'init': 'k-means++'}},
+            {'key': 'minibatch_kmeans', 'label': 'MiniBatch KMeans', 'params': {'n_clusters': 4, 'batch_size': 256, 'max_iter': 200}},
+            {'key': 'agglomerative', 'label': 'Agglomerative', 'params': {'n_clusters': 4, 'linkage': 'ward', 'metric': 'euclidean'}},
+            {'key': 'dbscan', 'label': 'DBSCAN', 'params': {'eps': 0.7, 'min_samples': 5, 'metric': 'euclidean'}},
+            {'key': 'hdbscan', 'label': 'HDBSCAN', 'params': {'min_cluster_size': 15, 'min_samples': 5, 'cluster_selection_method': 'eom'}},
+            {'key': 'gmm', 'label': 'Gaussian Mixture', 'params': {'n_components': 4, 'covariance_type': 'full'}},
+            {'key': 'spectral', 'label': 'Spectral', 'params': {'n_clusters': 4, 'affinity': 'nearest_neighbors', 'n_neighbors': 10}},
+            {'key': 'birch', 'label': 'Birch', 'params': {'n_clusters': 4, 'threshold': 0.5, 'branching_factor': 50}},
+        ]
+    }

backend/app/config.py

@@ -0,0 +1,23 @@
+from functools import lru_cache
+from typing import List
+from pydantic import Field
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+class Settings(BaseSettings):
+    model_config = SettingsConfigDict(env_file='.env', env_file_encoding='utf-8', extra='ignore')
+    app_name: str = Field(default='ClusterForge', alias='APP_NAME')
+    app_env: str = Field(default='dev', alias='APP_ENV')
+    api_prefix: str = Field(default='/api', alias='API_PREFIX')
+    data_dir: str = Field(default='/data', alias='DATA_DIR')
+    db_path: str = Field(default='/data/clusterforge.db', alias='DB_PATH')
+    upload_dir: str = Field(default='/data/uploads', alias='UPLOAD_DIR')
+    artifact_dir: str = Field(default='/data/artifacts', alias='ARTIFACT_DIR')
+    allowed_origins_raw: str = Field(default='http://localhost:3000,http://127.0.0.1:3000', alias='ALLOWED_ORIGINS')
+
+    @property
+    def allowed_origins(self) -> List[str]:
+        return [x.strip() for x in self.allowed_origins_raw.split(',') if x.strip()]
+
+@lru_cache
+def get_settings() -> Settings:
+    return Settings()

backend/app/db.py

@@ -0,0 +1,36 @@
+from contextlib import contextmanager
+from sqlalchemy import create_engine
+from sqlalchemy.orm import declarative_base, sessionmaker
+from app.config import get_settings
+
+settings = get_settings()
+engine = create_engine(
+    f"sqlite:///{settings.db_path}",
+    connect_args={'check_same_thread': False},
+    future=True,
+)
+SessionLocal = sessionmaker(bind=engine, autoflush=False, autocommit=False, future=True)
+Base = declarative_base()
+
+def init_storage_dirs() -> None:
+    import os
+    os.makedirs(settings.data_dir, exist_ok=True)
+    os.makedirs(settings.upload_dir, exist_ok=True)
+    os.makedirs(settings.artifact_dir, exist_ok=True)
+
+def init_db() -> None:
+    init_storage_dirs()
+    from app.models import dataset, experiment, cluster_result, artifact  # noqa
+    Base.metadata.create_all(bind=engine)
+
+@contextmanager
+def get_session():
+    session = SessionLocal()
+    try:
+        yield session
+        session.commit()
+    except Exception:
+        session.rollback()
+        raise
+    finally:
+        session.close()

backend/app/logging_config.py

@@ -0,0 +1,7 @@
+import logging
+
+def setup_logging() -> None:
+    logging.basicConfig(
+        level=logging.INFO,
+        format='%(asctime)s | %(levelname)s | %(name)s | %(message)s'
+    )

backend/app/main.py

@@ -0,0 +1,31 @@
+from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+from app.api.clusters import router as clusters_router
+from app.api.compare import router as compare_router
+from app.api.datasets import router as datasets_router
+from app.api.experiments import router as experiments_router
+from app.api.exports import router as exports_router
+from app.api.health import router as health_router
+from app.api.presets import router as presets_router
+from app.config import get_settings
+from app.db import init_db
+from app.logging_config import setup_logging
+
+setup_logging()
+settings = get_settings()
+init_db()
+app = FastAPI(title=settings.app_name)
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=settings.allowed_origins,
+    allow_credentials=True,
+    allow_methods=['*'],
+    allow_headers=['*'],
+)
+app.include_router(health_router, prefix=settings.api_prefix)
+app.include_router(datasets_router, prefix=settings.api_prefix)
+app.include_router(experiments_router, prefix=settings.api_prefix)
+app.include_router(clusters_router, prefix=settings.api_prefix)
+app.include_router(compare_router, prefix=settings.api_prefix)
+app.include_router(exports_router, prefix=settings.api_prefix)
+app.include_router(presets_router, prefix=settings.api_prefix)

backend/app/ml/algorithms/agglomerative_runner.py

@@ -0,0 +1,14 @@
+from sklearn.cluster import AgglomerativeClustering
+from app.ml.algorithms.base import BaseClusterRunner, ClusterRunOutput
+class Runner(BaseClusterRunner):
+    name = 'agglomerative'
+    def default_params(self) -> dict:
+        return {'n_clusters': 4, 'linkage': 'ward', 'metric': 'euclidean'}
+    def fit_predict(self, X, params: dict) -> ClusterRunOutput:
+        cfg = self.default_params() | params
+        kwargs = {'n_clusters': int(cfg['n_clusters']), 'linkage': str(cfg.get('linkage', 'ward'))}
+        if kwargs['linkage'] != 'ward':
+            kwargs['metric'] = str(cfg.get('metric', 'euclidean'))
+        model = AgglomerativeClustering(**kwargs)
+        labels = model.fit_predict(X)
+        return ClusterRunOutput(labels=labels.tolist())

backend/app/ml/algorithms/base.py

@@ -0,0 +1,17 @@
+from dataclasses import dataclass
+from typing import Any
+
+@dataclass
+class ClusterRunOutput:
+    labels: list[int]
+    cluster_centers: list[list[float]] | None = None
+    extra: dict[str, Any] | None = None
+
+class BaseClusterRunner:
+    name: str = 'base'
+    def fit_predict(self, X, params: dict) -> ClusterRunOutput:
+        raise NotImplementedError
+    def supports_noise(self) -> bool:
+        return False
+    def default_params(self) -> dict:
+        return {}

backend/app/ml/algorithms/birch_runner.py

@@ -0,0 +1,15 @@
+from sklearn.cluster import Birch
+from app.ml.algorithms.base import BaseClusterRunner, ClusterRunOutput
+class Runner(BaseClusterRunner):
+    name = 'birch'
+    def default_params(self) -> dict:
+        return {'n_clusters': 4, 'threshold': 0.5, 'branching_factor': 50}
+    def fit_predict(self, X, params: dict) -> ClusterRunOutput:
+        cfg = self.default_params() | params
+        model = Birch(
+            n_clusters=int(cfg.get('n_clusters', 4)),
+            threshold=float(cfg.get('threshold', 0.5)),
+            branching_factor=int(cfg.get('branching_factor', 50)),
+        )
+        labels = model.fit_predict(X)
+        return ClusterRunOutput(labels=labels.tolist())

backend/app/ml/algorithms/dbscan_runner.py

@@ -0,0 +1,17 @@
+from sklearn.cluster import DBSCAN
+from app.ml.algorithms.base import BaseClusterRunner, ClusterRunOutput
+class Runner(BaseClusterRunner):
+    name = 'dbscan'
+    def default_params(self) -> dict:
+        return {'eps': 0.7, 'min_samples': 5, 'metric': 'euclidean'}
+    def supports_noise(self) -> bool:
+        return True
+    def fit_predict(self, X, params: dict) -> ClusterRunOutput:
+        cfg = self.default_params() | params
+        model = DBSCAN(
+            eps=float(cfg.get('eps', 0.7)),
+            min_samples=int(cfg.get('min_samples', 5)),
+            metric=str(cfg.get('metric', 'euclidean')),
+        )
+        labels = model.fit_predict(X)
+        return ClusterRunOutput(labels=labels.tolist())

backend/app/ml/algorithms/gmm_runner.py

@@ -0,0 +1,15 @@
+from sklearn.mixture import GaussianMixture
+from app.ml.algorithms.base import BaseClusterRunner, ClusterRunOutput
+class Runner(BaseClusterRunner):
+    name = 'gmm'
+    def default_params(self) -> dict:
+        return {'n_components': 4, 'covariance_type': 'full', 'random_state': 42}
+    def fit_predict(self, X, params: dict) -> ClusterRunOutput:
+        cfg = self.default_params() | params
+        model = GaussianMixture(
+            n_components=int(cfg.get('n_components', 4)),
+            covariance_type=str(cfg.get('covariance_type', 'full')),
+            random_state=int(cfg.get('random_state', 42)),
+        )
+        labels = model.fit_predict(X)
+        return ClusterRunOutput(labels=labels.tolist(), cluster_centers=model.means_.tolist())

backend/app/ml/algorithms/hdbscan_runner.py

@@ -0,0 +1,20 @@
+from app.ml.algorithms.base import BaseClusterRunner, ClusterRunOutput
+class Runner(BaseClusterRunner):
+    name = 'hdbscan'
+    def default_params(self) -> dict:
+        return {'min_cluster_size': 15, 'min_samples': 5, 'cluster_selection_method': 'eom'}
+    def supports_noise(self) -> bool:
+        return True
+    def fit_predict(self, X, params: dict) -> ClusterRunOutput:
+        try:
+            import hdbscan
+        except Exception as exc:
+            raise ValueError('HDBSCAN is not installed in this environment') from exc
+        cfg = self.default_params() | params
+        model = hdbscan.HDBSCAN(
+            min_cluster_size=int(cfg.get('min_cluster_size', 15)),
+            min_samples=int(cfg.get('min_samples', 5)),
+            cluster_selection_method=str(cfg.get('cluster_selection_method', 'eom')),
+        )
+        labels = model.fit_predict(X)
+        return ClusterRunOutput(labels=labels.tolist())

backend/app/ml/algorithms/kmeans_runner.py

@@ -0,0 +1,17 @@
+from sklearn.cluster import KMeans
+from app.ml.algorithms.base import BaseClusterRunner, ClusterRunOutput
+class Runner(BaseClusterRunner):
+    name = 'kmeans'
+    def default_params(self) -> dict:
+        return {'n_clusters': 4, 'random_state': 42, 'max_iter': 300, 'init': 'k-means++'}
+    def fit_predict(self, X, params: dict) -> ClusterRunOutput:
+        cfg = self.default_params() | params
+        model = KMeans(
+            n_clusters=int(cfg['n_clusters']),
+            random_state=int(cfg.get('random_state', 42)),
+            max_iter=int(cfg.get('max_iter', 300)),
+            init=str(cfg.get('init', 'k-means++')),
+            n_init=10,
+        )
+        labels = model.fit_predict(X)
+        return ClusterRunOutput(labels=labels.tolist(), cluster_centers=model.cluster_centers_.tolist())

backend/app/ml/algorithms/minibatch_kmeans_runner.py

@@ -0,0 +1,17 @@
+from sklearn.cluster import MiniBatchKMeans
+from app.ml.algorithms.base import BaseClusterRunner, ClusterRunOutput
+class Runner(BaseClusterRunner):
+    name = 'minibatch_kmeans'
+    def default_params(self) -> dict:
+        return {'n_clusters': 4, 'random_state': 42, 'max_iter': 200, 'batch_size': 256}
+    def fit_predict(self, X, params: dict) -> ClusterRunOutput:
+        cfg = self.default_params() | params
+        model = MiniBatchKMeans(
+            n_clusters=int(cfg['n_clusters']),
+            random_state=int(cfg.get('random_state', 42)),
+            max_iter=int(cfg.get('max_iter', 200)),
+            batch_size=int(cfg.get('batch_size', 256)),
+            n_init=5,
+        )
+        labels = model.fit_predict(X)
+        return ClusterRunOutput(labels=labels.tolist(), cluster_centers=model.cluster_centers_.tolist())

backend/app/ml/algorithms/spectral_runner.py

@@ -0,0 +1,17 @@
+from sklearn.cluster import SpectralClustering
+from app.ml.algorithms.base import BaseClusterRunner, ClusterRunOutput
+class Runner(BaseClusterRunner):
+    name = 'spectral'
+    def default_params(self) -> dict:
+        return {'n_clusters': 4, 'affinity': 'nearest_neighbors', 'n_neighbors': 10, 'random_state': 42}
+    def fit_predict(self, X, params: dict) -> ClusterRunOutput:
+        cfg = self.default_params() | params
+        model = SpectralClustering(
+            n_clusters=int(cfg.get('n_clusters', 4)),
+            affinity=str(cfg.get('affinity', 'nearest_neighbors')),
+            n_neighbors=int(cfg.get('n_neighbors', 10)),
+            random_state=int(cfg.get('random_state', 42)),
+            assign_labels='kmeans',
+        )
+        labels = model.fit_predict(X)
+        return ClusterRunOutput(labels=labels.tolist())

backend/app/ml/explain/cluster_profiles.py

@@ -0,0 +1,22 @@
+import pandas as pd
+
+def build_cluster_profiles(df: pd.DataFrame, labels: list[int], feature_columns: list[str]) -> dict[int, dict]:
+    temp = df[feature_columns].copy()
+    temp['_cluster'] = labels
+    profiles = {}
+    for cluster_label, cluster_df in temp.groupby('_cluster'):
+        numeric_profile = {}
+        categorical_profile = {}
+        for col in feature_columns:
+            if pd.api.types.is_numeric_dtype(temp[col]):
+                numeric_profile[col] = round(float(cluster_df[col].mean()), 4)
+            else:
+                vc = cluster_df[col].astype(str).value_counts(normalize=True)
+                if len(vc):
+                    categorical_profile[col] = {'top_value': vc.index[0], 'share': round(float(vc.iloc[0]), 4)}
+        profiles[int(cluster_label)] = {
+            'size': int(len(cluster_df)),
+            'numeric_profile': numeric_profile,
+            'categorical_profile': categorical_profile,
+        }
+    return profiles

backend/app/ml/explain/feature_lift.py

@@ -0,0 +1,24 @@
+import pandas as pd
+
+def compute_feature_lift(df: pd.DataFrame, labels: list[int], feature_columns: list[str]) -> dict[int, dict]:
+    temp = df[feature_columns].copy()
+    temp['_cluster'] = labels
+    out = {}
+    global_numeric = {c: float(temp[c].mean()) for c in feature_columns if pd.api.types.is_numeric_dtype(temp[c])}
+    for cluster_label, cluster_df in temp.groupby('_cluster'):
+        info = {'top_numeric_lifts': [], 'top_categorical_modes': []}
+        numeric_lifts = []
+        for col in feature_columns:
+            if pd.api.types.is_numeric_dtype(temp[col]):
+                base = global_numeric.get(col, 0.0)
+                mean = float(cluster_df[col].mean())
+                lift = round(mean - base, 4)
+                numeric_lifts.append({'feature': col, 'cluster_mean': round(mean, 4), 'global_mean': round(base, 4), 'lift': lift})
+            else:
+                mode_series = cluster_df[col].astype(str).value_counts(normalize=True)
+                if len(mode_series):
+                    info['top_categorical_modes'].append({'feature': col, 'value': mode_series.index[0], 'share': round(float(mode_series.iloc[0]), 4)})
+        info['top_numeric_lifts'] = sorted(numeric_lifts, key=lambda x: abs(x['lift']), reverse=True)[:5]
+        info['top_categorical_modes'] = info['top_categorical_modes'][:5]
+        out[int(cluster_label)] = info
+    return out

backend/app/ml/explain/summary_writer.py

@@ -0,0 +1,10 @@
+def build_cluster_description(cluster_label: int, profile: dict, lift_info: dict) -> str:
+    numeric = lift_info.get('top_numeric_lifts', [])[:3]
+    categorical = lift_info.get('top_categorical_modes', [])[:2]
+    parts = []
+    if numeric:
+        parts.append(', '.join([('high ' if item['lift'] >= 0 else 'low ') + item['feature'] for item in numeric]))
+    if categorical:
+        parts.append(', '.join([f"{item['feature']} is mostly {item['value']}" for item in categorical]))
+    core = '. '.join([p for p in parts if p]).strip() or 'This cluster is defined by a mixed feature pattern.'
+    return f"Cluster {cluster_label} has {profile.get('size', 0)} rows. {core}."

backend/app/ml/metrics/clustering_metrics.py

@@ -0,0 +1,33 @@
+import numpy as np
+from sklearn.metrics import calinski_harabasz_score, davies_bouldin_score, silhouette_score
+
+def compute_metrics(X, labels: list[int]) -> dict:
+    arr = np.array(labels)
+    unique = set(arr.tolist())
+    non_noise = [x for x in unique if x != -1]
+    cluster_count = len(non_noise)
+    noise_ratio = float((arr == -1).mean()) if len(arr) else 0.0
+    metrics = {
+        'cluster_count': cluster_count,
+        'noise_ratio': round(noise_ratio, 4),
+        'silhouette_score': None,
+        'davies_bouldin_score': None,
+        'calinski_harabasz_score': None,
+    }
+    mask = arr != -1 if -1 in unique else np.ones(len(arr), dtype=bool)
+    X_eval = X[mask]
+    y_eval = arr[mask]
+    if len(set(y_eval.tolist())) >= 2 and len(X_eval) > len(set(y_eval.tolist())):
+        try:
+            metrics['silhouette_score'] = round(float(silhouette_score(X_eval, y_eval)), 4)
+        except Exception:
+            pass
+        try:
+            metrics['davies_bouldin_score'] = round(float(davies_bouldin_score(X_eval, y_eval)), 4)
+        except Exception:
+            pass
+        try:
+            metrics['calinski_harabasz_score'] = round(float(calinski_harabasz_score(X_eval, y_eval)), 4)
+        except Exception:
+            pass
+    return metrics

backend/app/ml/metrics/interpretability.py

@@ -0,0 +1,19 @@
+import numpy as np
+import pandas as pd
+
+def compute_interpretability_score(df: pd.DataFrame, labels: list[int], feature_columns: list[str]) -> float:
+    if len(set(labels)) <= 1:
+        return 0.0
+    scored = []
+    temp = df[feature_columns].copy()
+    temp['_cluster'] = labels
+    for col in feature_columns:
+        if pd.api.types.is_numeric_dtype(temp[col]):
+            global_std = temp[col].std() or 1.0
+            diffs = temp.groupby('_cluster')[col].mean().diff().abs().fillna(0)
+            lift = float(diffs.mean() / global_std)
+        else:
+            top_share = temp.groupby('_cluster')[col].agg(lambda s: s.astype(str).value_counts(normalize=True).iloc[0] if len(s) else 0).mean()
+            lift = float(top_share)
+        scored.append(min(lift, 1.0))
+    return round(float(np.mean(scored)), 4) if scored else 0.0

backend/app/ml/metrics/stability.py

@@ -0,0 +1,10 @@
+import numpy as np
+from sklearn.metrics import normalized_mutual_info_score
+
+def compute_stability_score(labels: list[int]) -> float:
+    arr = np.array(labels)
+    if len(set(arr.tolist())) <= 1:
+        return 0.0
+    rotated = np.roll(arr, 1)
+    score = normalized_mutual_info_score(arr, rotated)
+    return round(float(score), 4)

backend/app/ml/outliers/centroid_distance.py

@@ -0,0 +1,18 @@
+import numpy as np
+
+def detect_centroid_outliers(X, labels: list[int], threshold_quantile: float = 0.95) -> list[bool]:
+    arr = np.array(labels)
+    outliers = np.zeros(len(arr), dtype=bool)
+    for cluster_label in sorted(set(arr.tolist())):
+        if cluster_label == -1:
+            outliers[arr == cluster_label] = True
+            continue
+        idx = np.where(arr == cluster_label)[0]
+        if len(idx) <= 2:
+            continue
+        cluster_points = X[idx]
+        centroid = cluster_points.mean(axis=0)
+        dists = np.linalg.norm(cluster_points - centroid, axis=1)
+        cut = np.quantile(dists, threshold_quantile)
+        outliers[idx] = dists >= cut
+    return outliers.tolist()

backend/app/ml/outliers/density_noise.py

@@ -0,0 +1,2 @@
+def detect_noise_outliers(labels: list[int]) -> list[bool]:
+    return [label == -1 for label in labels]

backend/app/ml/outliers/isolation_forest.py

@@ -0,0 +1,6 @@
+from sklearn.ensemble import IsolationForest
+
+def detect_isolation_outliers(X) -> list[bool]:
+    model = IsolationForest(random_state=42, contamination='auto')
+    flags = model.fit_predict(X)
+    return [item == -1 for item in flags.tolist()]

backend/app/ml/preprocess/column_inference.py

@@ -0,0 +1,33 @@
+from dataclasses import dataclass
+import pandas as pd
+
+@dataclass
+class InferredColumn:
+    name: str
+    inferred_type: str
+    missing_pct: float
+    cardinality: int
+    sample_values: list
+
+def infer_columns(df: pd.DataFrame) -> list[InferredColumn]:
+    cols = []
+    for col in df.columns:
+        series = df[col]
+        missing_pct = float(series.isna().mean() * 100)
+        cardinality = int(series.nunique(dropna=True))
+        sample_values = series.dropna().astype(str).head(5).tolist()
+        if pd.api.types.is_bool_dtype(series):
+            inferred = 'boolean'
+        elif pd.api.types.is_numeric_dtype(series):
+            inferred = 'numeric'
+        else:
+            unique_ratio = cardinality / max(len(series), 1)
+            inferred = 'high_cardinality' if cardinality > 50 and unique_ratio > 0.3 else 'categorical'
+        cols.append(InferredColumn(
+            name=col,
+            inferred_type=inferred,
+            missing_pct=round(missing_pct, 2),
+            cardinality=cardinality,
+            sample_values=sample_values,
+        ))
+    return cols

backend/app/ml/preprocess/encoders.py

@@ -0,0 +1,7 @@
+from sklearn.preprocessing import OneHotEncoder
+
+def get_onehot_encoder():
+    try:
+        return OneHotEncoder(handle_unknown='ignore', sparse_output=False)
+    except TypeError:
+        return OneHotEncoder(handle_unknown='ignore', sparse=False)

backend/app/ml/preprocess/imputers.py

@@ -0,0 +1,7 @@
+from sklearn.impute import SimpleImputer
+
+def numeric_imputer():
+    return SimpleImputer(strategy='median')
+
+def categorical_imputer():
+    return SimpleImputer(strategy='most_frequent')

backend/app/ml/preprocess/matrix_builder.py

@@ -0,0 +1,32 @@
+import pandas as pd
+from sklearn.compose import ColumnTransformer
+from sklearn.pipeline import Pipeline
+from app.ml.preprocess.encoders import get_onehot_encoder
+from app.ml.preprocess.imputers import categorical_imputer, numeric_imputer
+from app.ml.preprocess.scalers import get_scaler
+
+def build_feature_matrix(df: pd.DataFrame, feature_columns: list[str], numeric_scaler: str = 'standard', categorical_encoder: str = 'onehot'):
+    selected = df[feature_columns].copy()
+    numeric_cols = [c for c in selected.columns if pd.api.types.is_numeric_dtype(selected[c]) or pd.api.types.is_bool_dtype(selected[c])]
+    categorical_cols = [c for c in selected.columns if c not in numeric_cols]
+
+    numeric_pipeline = Pipeline(steps=[('imputer', numeric_imputer()), ('scaler', get_scaler(numeric_scaler))])
+    categorical_pipeline = Pipeline(steps=[('imputer', categorical_imputer()), ('encoder', get_onehot_encoder())])
+
+    transformer = ColumnTransformer(
+        transformers=[('num', numeric_pipeline, numeric_cols), ('cat', categorical_pipeline, categorical_cols)],
+        remainder='drop',
+    )
+    X = transformer.fit_transform(selected)
+
+    feature_names = []
+    feature_names.extend(numeric_cols)
+    if categorical_cols:
+        enc = transformer.named_transformers_['cat'].named_steps['encoder']
+        try:
+            cat_names = enc.get_feature_names_out(categorical_cols).tolist()
+        except Exception:
+            cat_names = categorical_cols
+        feature_names.extend(cat_names)
+
+    return X, feature_names, {'numeric_cols': numeric_cols, 'categorical_cols': categorical_cols}

backend/app/ml/preprocess/reducers.py

@@ -0,0 +1,16 @@
+from sklearn.decomposition import PCA
+
+def pca_2d(X):
+    reducer = PCA(n_components=2, random_state=42)
+    coords = reducer.fit_transform(X)
+    return coords[:, 0], coords[:, 1]
+
+def umap_2d(X):
+    try:
+        import umap
+        reducer = umap.UMAP(n_components=2, random_state=42)
+        coords = reducer.fit_transform(X)
+    except Exception:
+        reducer = PCA(n_components=2, random_state=42)
+        coords = reducer.fit_transform(X)
+    return coords[:, 0], coords[:, 1]

backend/app/ml/preprocess/scalers.py

@@ -0,0 +1,9 @@
+from sklearn.preprocessing import MinMaxScaler, RobustScaler, StandardScaler
+
+def get_scaler(name: str):
+    name = (name or 'standard').lower()
+    if name == 'minmax':
+        return MinMaxScaler()
+    if name == 'robust':
+        return RobustScaler()
+    return StandardScaler()

backend/app/models/artifact.py

@@ -0,0 +1,12 @@
+from datetime import datetime
+from sqlalchemy import DateTime, ForeignKey, Integer, String, Text
+from sqlalchemy.orm import Mapped, mapped_column
+from app.db import Base
+
+class Artifact(Base):
+    __tablename__ = 'artifacts'
+    id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
+    experiment_id: Mapped[str] = mapped_column(ForeignKey('experiments.id'), index=True, nullable=False)
+    artifact_type: Mapped[str] = mapped_column(String(64), nullable=False)
+    path: Mapped[str] = mapped_column(Text, nullable=False)
+    created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow, nullable=False)

backend/app/models/cluster_result.py

@@ -0,0 +1,36 @@
+from sqlalchemy import Boolean, Float, ForeignKey, Integer, String, Text
+from sqlalchemy.orm import Mapped, mapped_column
+from app.db import Base
+
+class ExperimentMetric(Base):
+    __tablename__ = 'experiment_metrics'
+    id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
+    experiment_id: Mapped[str] = mapped_column(ForeignKey('experiments.id'), index=True, nullable=False)
+    cluster_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
+    noise_ratio: Mapped[float] = mapped_column(Float, default=0.0, nullable=False)
+    silhouette_score: Mapped[float | None] = mapped_column(Float, nullable=True)
+    davies_bouldin_score: Mapped[float | None] = mapped_column(Float, nullable=True)
+    calinski_harabasz_score: Mapped[float | None] = mapped_column(Float, nullable=True)
+    stability_score: Mapped[float | None] = mapped_column(Float, nullable=True)
+    interpretability_score: Mapped[float | None] = mapped_column(Float, nullable=True)
+
+class ClusterResult(Base):
+    __tablename__ = 'cluster_results'
+    id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
+    experiment_id: Mapped[str] = mapped_column(ForeignKey('experiments.id'), index=True, nullable=False)
+    row_index: Mapped[int] = mapped_column(Integer, nullable=False)
+    cluster_label: Mapped[int] = mapped_column(Integer, nullable=False, index=True)
+    is_outlier: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False)
+    pca_x: Mapped[float | None] = mapped_column(Float, nullable=True)
+    pca_y: Mapped[float | None] = mapped_column(Float, nullable=True)
+    umap_x: Mapped[float | None] = mapped_column(Float, nullable=True)
+    umap_y: Mapped[float | None] = mapped_column(Float, nullable=True)
+
+class ClusterSummary(Base):
+    __tablename__ = 'cluster_summaries'
+    id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
+    experiment_id: Mapped[str] = mapped_column(ForeignKey('experiments.id'), index=True, nullable=False)
+    cluster_label: Mapped[int] = mapped_column(Integer, nullable=False, index=True)
+    size: Mapped[int] = mapped_column(Integer, nullable=False)
+    summary_json: Mapped[str] = mapped_column(Text, default='{}', nullable=False)
+    description_text: Mapped[str] = mapped_column(Text, nullable=False)

backend/app/models/dataset.py

@@ -0,0 +1,16 @@
+from datetime import datetime
+from sqlalchemy import DateTime, Integer, String, Text
+from sqlalchemy.orm import Mapped, mapped_column
+from app.db import Base
+
+class Dataset(Base):
+    __tablename__ = 'datasets'
+    id: Mapped[str] = mapped_column(String(32), primary_key=True)
+    name: Mapped[str] = mapped_column(String(255), nullable=False)
+    file_path: Mapped[str] = mapped_column(Text, nullable=False)
+    sha256: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
+    uploaded_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow, nullable=False)
+    row_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
+    column_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
+    schema_json: Mapped[str] = mapped_column(Text, default='{}', nullable=False)
+    profile_json: Mapped[str] = mapped_column(Text, default='{}', nullable=False)

backend/app/models/experiment.py

@@ -0,0 +1,17 @@
+from datetime import datetime
+from sqlalchemy import DateTime, ForeignKey, Integer, String, Text
+from sqlalchemy.orm import Mapped, mapped_column
+from app.db import Base
+
+class Experiment(Base):
+    __tablename__ = 'experiments'
+    id: Mapped[str] = mapped_column(String(32), primary_key=True)
+    dataset_id: Mapped[str] = mapped_column(ForeignKey('datasets.id'), index=True, nullable=False)
+    name: Mapped[str] = mapped_column(String(255), nullable=False)
+    algorithm: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
+    status: Mapped[str] = mapped_column(String(32), nullable=False, default='pending')
+    config_json: Mapped[str] = mapped_column(Text, default='{}', nullable=False)
+    started_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow, nullable=False)
+    completed_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
+    runtime_ms: Mapped[int | None] = mapped_column(Integer, nullable=True)
+    error_message: Mapped[str | None] = mapped_column(Text, nullable=True)