Delete backend

backend/app/api/clusters.py

@@ -1,40 +0,0 @@
-import json
-import pandas as pd
-from fastapi import APIRouter, HTTPException
-from app.db import get_session
-from app.repositories.cluster_repo import ClusterRepository
-from app.repositories.dataset_repo import DatasetRepository
-from app.repositories.experiment_repo import ExperimentRepository
-from app.schemas.cluster import ClusterDetailResponse, ClusterMember
-
-router = APIRouter(prefix='/clusters', tags=['clusters'])
-
-@router.get('/{experiment_id}/{cluster_label}', response_model=ClusterDetailResponse)
-def get_cluster_detail(experiment_id: str, cluster_label: int):
-    with get_session() as session:
-        exp_repo = ExperimentRepository(session)
-        dataset_repo = DatasetRepository(session)
-        cluster_repo = ClusterRepository(session)
-        exp = exp_repo.get(experiment_id)
-        if not exp:
-            raise HTTPException(status_code=404, detail='Experiment not found')
-        dataset = dataset_repo.get(exp.dataset_id)
-        if not dataset:
-            raise HTTPException(status_code=404, detail='Dataset not found')
-        df = pd.read_csv(dataset.file_path) if dataset.file_path.lower().endswith('.csv') else pd.read_excel(dataset.file_path)
-        results = [r for r in cluster_repo.get_results(experiment_id) if r.cluster_label == cluster_label]
-        summary = cluster_repo.get_summary(experiment_id, cluster_label)
-        if not summary:
-            raise HTTPException(status_code=404, detail='Cluster summary not found')
-        members = []
-        for row in results[:200]:
-            values = df.iloc[row.row_index].to_dict()
-            values = {str(k): (None if pd.isna(v) else v) for k, v in values.items()}
-            members.append(ClusterMember(row_index=row.row_index, values=values))
-        return ClusterDetailResponse(
-            experiment_id=experiment_id,
-            cluster_label=cluster_label,
-            members=members,
-            summary=json.loads(summary.summary_json),
-            description_text=summary.description_text,
-        )

backend/app/api/compare.py

@@ -1,19 +0,0 @@
-from fastapi import APIRouter, Query
-from app.db import get_session
-from app.repositories.cluster_repo import ClusterRepository
-from app.repositories.experiment_repo import ExperimentRepository
-from app.schemas.compare import CompareResponse
-from app.services.comparison_service import ComparisonService
-
-router = APIRouter(prefix='/compare', tags=['compare'])
-
-@router.get('', response_model=CompareResponse)
-def compare_runs(experiment_ids: str = Query(...)):
-    ids = [item.strip() for item in experiment_ids.split(',') if item.strip()]
-    with get_session() as session:
-        payload = ComparisonService().build_compare_payload(
-            ExperimentRepository(session),
-            ClusterRepository(session),
-            ids,
-        )
-        return CompareResponse(**payload)

backend/app/api/datasets.py

@@ -1,55 +0,0 @@
-from fastapi import APIRouter, File, HTTPException, UploadFile
-from app.db import get_session
-from app.repositories.dataset_repo import DatasetRepository
-from app.schemas.dataset import DatasetProfileResponse, DatasetUploadResponse
-from app.services.dataset_service import DatasetService
-from app.services.profiling_service import ProfilingService
-
-router = APIRouter(prefix='/datasets', tags=['datasets'])
-
-@router.post('/upload', response_model=DatasetUploadResponse)
-async def upload_dataset(file: UploadFile = File(...)):
-    with get_session() as session:
-        repo = DatasetRepository(session)
-        service = DatasetService(repo)
-        dataset = await service.save_upload(file)
-        return DatasetUploadResponse(
-            dataset_id=dataset.id,
-            name=dataset.name,
-            row_count=dataset.row_count,
-            column_count=dataset.column_count,
-        )
-
-@router.get('', response_model=list[DatasetUploadResponse])
-def list_datasets():
-    with get_session() as session:
-        repo = DatasetRepository(session)
-        items = repo.list_all()
-        return [
-            DatasetUploadResponse(
-                dataset_id=item.id,
-                name=item.name,
-                row_count=item.row_count,
-                column_count=item.column_count,
-            )
-            for item in items
-        ]
-
-@router.get('/{dataset_id}/profile', response_model=DatasetProfileResponse)
-def dataset_profile(dataset_id: str):
-    with get_session() as session:
-        repo = DatasetRepository(session)
-        dataset = repo.get(dataset_id)
-        if not dataset:
-            raise HTTPException(status_code=404, detail='Dataset not found')
-        df = DatasetService(repo).load_dataframe(dataset)
-        profile = ProfilingService().profile(df)
-        repo.update_profile(dataset_id, schema_json={'columns': [c['name'] for c in profile['columns']]}, profile_json=profile)
-        return DatasetProfileResponse(
-            dataset_id=dataset_id,
-            columns=profile['columns'],
-            suggested_feature_columns=profile['suggested_feature_columns'],
-            recommended_algorithms=profile['recommended_algorithms'],
-            row_count=profile['row_count'],
-            column_count=profile['column_count'],
-        )

backend/app/api/experiments.py

@@ -1,105 +0,0 @@
-import json
-from fastapi import APIRouter, HTTPException
-from app.db import get_session
-from app.repositories.cluster_repo import ClusterRepository
-from app.repositories.dataset_repo import DatasetRepository
-from app.repositories.experiment_repo import ExperimentRepository
-from app.schemas.experiment import ClusterSummaryResponse, ExperimentListItem, ExperimentResultResponse, ExperimentRunRequest, ExperimentRunResponse, MetricResponse, ScatterPoint
-from app.services.run_service import RunService
-
-router = APIRouter(prefix='/experiments', tags=['experiments'])
-
-@router.post('/run', response_model=ExperimentRunResponse)
-def run_experiment(payload: ExperimentRunRequest):
-    with get_session() as session:
-        dataset_repo = DatasetRepository(session)
-        exp_repo = ExperimentRepository(session)
-        cluster_repo = ClusterRepository(session)
-        service = RunService(dataset_repo, exp_repo, cluster_repo)
-        experiment = service.run(payload.model_dump())
-        return ExperimentRunResponse(experiment_id=experiment.id, status='completed')
-
-@router.get('', response_model=list[ExperimentListItem])
-def list_experiments():
-    with get_session() as session:
-        repo = ExperimentRepository(session)
-        items = repo.list_all()
-        return [
-            ExperimentListItem(
-                experiment_id=item.id,
-                dataset_id=item.dataset_id,
-                name=item.name,
-                algorithm=item.algorithm,
-                status=item.status,
-                started_at=item.started_at.isoformat(),
-                completed_at=item.completed_at.isoformat() if item.completed_at else None,
-            )
-            for item in items
-        ]
-
-@router.get('/{experiment_id}')
-def get_experiment(experiment_id: str):
-    with get_session() as session:
-        repo = ExperimentRepository(session)
-        exp = repo.get(experiment_id)
-        if not exp:
-            raise HTTPException(status_code=404, detail='Experiment not found')
-        return {
-            'experiment_id': exp.id,
-            'dataset_id': exp.dataset_id,
-            'name': exp.name,
-            'algorithm': exp.algorithm,
-            'status': exp.status,
-            'config': json.loads(exp.config_json),
-            'runtime_ms': exp.runtime_ms,
-            'error_message': exp.error_message,
-        }
-
-@router.get('/{experiment_id}/results', response_model=ExperimentResultResponse)
-def get_results(experiment_id: str):
-    with get_session() as session:
-        exp_repo = ExperimentRepository(session)
-        cluster_repo = ClusterRepository(session)
-        exp = exp_repo.get(experiment_id)
-        if not exp:
-            raise HTTPException(status_code=404, detail='Experiment not found')
-        metric = cluster_repo.get_metric(experiment_id)
-        summaries = cluster_repo.get_summaries(experiment_id)
-        results = cluster_repo.get_results(experiment_id)
-        if not metric:
-            raise HTTPException(status_code=404, detail='Results not found')
-        scatter = [
-            ScatterPoint(
-                row_index=item.row_index,
-                cluster_label=item.cluster_label,
-                is_outlier=item.is_outlier,
-                x=item.umap_x if item.umap_x is not None else item.pca_x,
-                y=item.umap_y if item.umap_y is not None else item.pca_y,
-            )
-            for item in results
-        ]
-        summary_items = [
-            ClusterSummaryResponse(
-                cluster_label=item.cluster_label,
-                size=item.size,
-                description_text=item.description_text,
-                summary_json=json.loads(item.summary_json),
-            )
-            for item in summaries
-        ]
-        return ExperimentResultResponse(
-            experiment_id=experiment_id,
-            algorithm=exp.algorithm,
-            metrics=MetricResponse(
-                cluster_count=metric.cluster_count,
-                noise_ratio=metric.noise_ratio,
-                silhouette_score=metric.silhouette_score,
-                davies_bouldin_score=metric.davies_bouldin_score,
-                calinski_harabasz_score=metric.calinski_harabasz_score,
-                stability_score=metric.stability_score,
-                interpretability_score=metric.interpretability_score,
-            ),
-            scatter_points=scatter,
-            cluster_summaries=summary_items,
-            status=exp.status,
-        )

backend/app/api/exports.py

@@ -1,24 +0,0 @@
-from fastapi import APIRouter, HTTPException
-from app.db import get_session
-from app.repositories.artifact_repo import ArtifactRepository
-from app.repositories.cluster_repo import ClusterRepository
-from app.repositories.dataset_repo import DatasetRepository
-from app.repositories.experiment_repo import ExperimentRepository
-from app.schemas.export import ExportResponse
-from app.services.export_service import ExportService
-
-router = APIRouter(prefix='/exports', tags=['exports'])
-
-@router.get('/{experiment_id}', response_model=ExportResponse)
-def export_experiment(experiment_id: str):
-    with get_session() as session:
-        exp = ExperimentRepository(session).get(experiment_id)
-        if not exp:
-            raise HTTPException(status_code=404, detail='Experiment not found')
-        service = ExportService(
-            dataset_repo=DatasetRepository(session),
-            cluster_repo=ClusterRepository(session),
-            artifact_repo=ArtifactRepository(session),
-        )
-        path = service.export_csv(experiment_id, exp.dataset_id)
-        return ExportResponse(experiment_id=experiment_id, download_path=path)

backend/app/api/health.py

@@ -1,7 +0,0 @@
-from fastapi import APIRouter
-from app.schemas.health import HealthResponse
-router = APIRouter(tags=['health'])
-
-@router.get('/health', response_model=HealthResponse)
-def health() -> HealthResponse:
-    return HealthResponse(ok=True, service='clusterforge-api')

backend/app/api/presets.py

@@ -1,17 +0,0 @@
-from fastapi import APIRouter
-router = APIRouter(prefix='/presets', tags=['presets'])
-
-@router.get('/algorithms')
-def list_algorithms():
-    return {
-        'algorithms': [
-            {'key': 'kmeans', 'label': 'KMeans', 'params': {'n_clusters': 4, 'random_state': 42, 'max_iter': 300, 'init': 'k-means++'}},
-            {'key': 'minibatch_kmeans', 'label': 'MiniBatch KMeans', 'params': {'n_clusters': 4, 'batch_size': 256, 'max_iter': 200}},
-            {'key': 'agglomerative', 'label': 'Agglomerative', 'params': {'n_clusters': 4, 'linkage': 'ward', 'metric': 'euclidean'}},
-            {'key': 'dbscan', 'label': 'DBSCAN', 'params': {'eps': 0.7, 'min_samples': 5, 'metric': 'euclidean'}},
-            {'key': 'hdbscan', 'label': 'HDBSCAN', 'params': {'min_cluster_size': 15, 'min_samples': 5, 'cluster_selection_method': 'eom'}},
-            {'key': 'gmm', 'label': 'Gaussian Mixture', 'params': {'n_components': 4, 'covariance_type': 'full'}},
-            {'key': 'spectral', 'label': 'Spectral', 'params': {'n_clusters': 4, 'affinity': 'nearest_neighbors', 'n_neighbors': 10}},
-            {'key': 'birch', 'label': 'Birch', 'params': {'n_clusters': 4, 'threshold': 0.5, 'branching_factor': 50}},
-        ]
-    }

backend/app/config.py

@@ -1,23 +0,0 @@
-from functools import lru_cache
-from typing import List
-from pydantic import Field
-from pydantic_settings import BaseSettings, SettingsConfigDict
-
-class Settings(BaseSettings):
-    model_config = SettingsConfigDict(env_file='.env', env_file_encoding='utf-8', extra='ignore')
-    app_name: str = Field(default='ClusterForge', alias='APP_NAME')
-    app_env: str = Field(default='dev', alias='APP_ENV')
-    api_prefix: str = Field(default='/api', alias='API_PREFIX')
-    data_dir: str = Field(default='/data', alias='DATA_DIR')
-    db_path: str = Field(default='/data/clusterforge.db', alias='DB_PATH')
-    upload_dir: str = Field(default='/data/uploads', alias='UPLOAD_DIR')
-    artifact_dir: str = Field(default='/data/artifacts', alias='ARTIFACT_DIR')
-    allowed_origins_raw: str = Field(default='http://localhost:3000,http://127.0.0.1:3000', alias='ALLOWED_ORIGINS')
-
-    @property
-    def allowed_origins(self) -> List[str]:
-        return [x.strip() for x in self.allowed_origins_raw.split(',') if x.strip()]
-
-@lru_cache
-def get_settings() -> Settings:
-    return Settings()

backend/app/db.py

@@ -1,36 +0,0 @@
-from contextlib import contextmanager
-from sqlalchemy import create_engine
-from sqlalchemy.orm import declarative_base, sessionmaker
-from app.config import get_settings
-
-settings = get_settings()
-engine = create_engine(
-    f"sqlite:///{settings.db_path}",
-    connect_args={'check_same_thread': False},
-    future=True,
-)
-SessionLocal = sessionmaker(bind=engine, autoflush=False, autocommit=False, future=True)
-Base = declarative_base()
-
-def init_storage_dirs() -> None:
-    import os
-    os.makedirs(settings.data_dir, exist_ok=True)
-    os.makedirs(settings.upload_dir, exist_ok=True)
-    os.makedirs(settings.artifact_dir, exist_ok=True)
-
-def init_db() -> None:
-    init_storage_dirs()
-    from app.models import dataset, experiment, cluster_result, artifact  # noqa
-    Base.metadata.create_all(bind=engine)
-
-@contextmanager
-def get_session():
-    session = SessionLocal()
-    try:
-        yield session
-        session.commit()
-    except Exception:
-        session.rollback()
-        raise
-    finally:
-        session.close()

backend/app/logging_config.py

@@ -1,7 +0,0 @@
-import logging
-
-def setup_logging() -> None:
-    logging.basicConfig(
-        level=logging.INFO,
-        format='%(asctime)s | %(levelname)s | %(name)s | %(message)s'
-    )

backend/app/main.py

@@ -1,31 +0,0 @@
-from fastapi import FastAPI
-from fastapi.middleware.cors import CORSMiddleware
-from app.api.clusters import router as clusters_router
-from app.api.compare import router as compare_router
-from app.api.datasets import router as datasets_router
-from app.api.experiments import router as experiments_router
-from app.api.exports import router as exports_router
-from app.api.health import router as health_router
-from app.api.presets import router as presets_router
-from app.config import get_settings
-from app.db import init_db
-from app.logging_config import setup_logging
-
-setup_logging()
-settings = get_settings()
-init_db()
-app = FastAPI(title=settings.app_name)
-app.add_middleware(
-    CORSMiddleware,
-    allow_origins=settings.allowed_origins,
-    allow_credentials=True,
-    allow_methods=['*'],
-    allow_headers=['*'],
-)
-app.include_router(health_router, prefix=settings.api_prefix)
-app.include_router(datasets_router, prefix=settings.api_prefix)
-app.include_router(experiments_router, prefix=settings.api_prefix)
-app.include_router(clusters_router, prefix=settings.api_prefix)
-app.include_router(compare_router, prefix=settings.api_prefix)
-app.include_router(exports_router, prefix=settings.api_prefix)
-app.include_router(presets_router, prefix=settings.api_prefix)

backend/app/ml/algorithms/agglomerative_runner.py

@@ -1,14 +0,0 @@
-from sklearn.cluster import AgglomerativeClustering
-from app.ml.algorithms.base import BaseClusterRunner, ClusterRunOutput
-class Runner(BaseClusterRunner):
-    name = 'agglomerative'
-    def default_params(self) -> dict:
-        return {'n_clusters': 4, 'linkage': 'ward', 'metric': 'euclidean'}
-    def fit_predict(self, X, params: dict) -> ClusterRunOutput:
-        cfg = self.default_params() | params
-        kwargs = {'n_clusters': int(cfg['n_clusters']), 'linkage': str(cfg.get('linkage', 'ward'))}
-        if kwargs['linkage'] != 'ward':
-            kwargs['metric'] = str(cfg.get('metric', 'euclidean'))
-        model = AgglomerativeClustering(**kwargs)
-        labels = model.fit_predict(X)
-        return ClusterRunOutput(labels=labels.tolist())

backend/app/ml/algorithms/base.py

@@ -1,17 +0,0 @@
-from dataclasses import dataclass
-from typing import Any
-
-@dataclass
-class ClusterRunOutput:
-    labels: list[int]
-    cluster_centers: list[list[float]] | None = None
-    extra: dict[str, Any] | None = None
-
-class BaseClusterRunner:
-    name: str = 'base'
-    def fit_predict(self, X, params: dict) -> ClusterRunOutput:
-        raise NotImplementedError
-    def supports_noise(self) -> bool:
-        return False
-    def default_params(self) -> dict:
-        return {}

backend/app/ml/algorithms/birch_runner.py

@@ -1,15 +0,0 @@
-from sklearn.cluster import Birch
-from app.ml.algorithms.base import BaseClusterRunner, ClusterRunOutput
-class Runner(BaseClusterRunner):
-    name = 'birch'
-    def default_params(self) -> dict:
-        return {'n_clusters': 4, 'threshold': 0.5, 'branching_factor': 50}
-    def fit_predict(self, X, params: dict) -> ClusterRunOutput:
-        cfg = self.default_params() | params
-        model = Birch(
-            n_clusters=int(cfg.get('n_clusters', 4)),
-            threshold=float(cfg.get('threshold', 0.5)),
-            branching_factor=int(cfg.get('branching_factor', 50)),
-        )
-        labels = model.fit_predict(X)
-        return ClusterRunOutput(labels=labels.tolist())

backend/app/ml/algorithms/dbscan_runner.py

@@ -1,17 +0,0 @@
-from sklearn.cluster import DBSCAN
-from app.ml.algorithms.base import BaseClusterRunner, ClusterRunOutput
-class Runner(BaseClusterRunner):
-    name = 'dbscan'
-    def default_params(self) -> dict:
-        return {'eps': 0.7, 'min_samples': 5, 'metric': 'euclidean'}
-    def supports_noise(self) -> bool:
-        return True
-    def fit_predict(self, X, params: dict) -> ClusterRunOutput:
-        cfg = self.default_params() | params
-        model = DBSCAN(
-            eps=float(cfg.get('eps', 0.7)),
-            min_samples=int(cfg.get('min_samples', 5)),
-            metric=str(cfg.get('metric', 'euclidean')),
-        )
-        labels = model.fit_predict(X)
-        return ClusterRunOutput(labels=labels.tolist())

backend/app/ml/algorithms/gmm_runner.py

@@ -1,15 +0,0 @@
-from sklearn.mixture import GaussianMixture
-from app.ml.algorithms.base import BaseClusterRunner, ClusterRunOutput
-class Runner(BaseClusterRunner):
-    name = 'gmm'
-    def default_params(self) -> dict:
-        return {'n_components': 4, 'covariance_type': 'full', 'random_state': 42}
-    def fit_predict(self, X, params: dict) -> ClusterRunOutput:
-        cfg = self.default_params() | params
-        model = GaussianMixture(
-            n_components=int(cfg.get('n_components', 4)),
-            covariance_type=str(cfg.get('covariance_type', 'full')),
-            random_state=int(cfg.get('random_state', 42)),
-        )
-        labels = model.fit_predict(X)
-        return ClusterRunOutput(labels=labels.tolist(), cluster_centers=model.means_.tolist())

backend/app/ml/algorithms/hdbscan_runner.py

@@ -1,20 +0,0 @@
-from app.ml.algorithms.base import BaseClusterRunner, ClusterRunOutput
-class Runner(BaseClusterRunner):
-    name = 'hdbscan'
-    def default_params(self) -> dict:
-        return {'min_cluster_size': 15, 'min_samples': 5, 'cluster_selection_method': 'eom'}
-    def supports_noise(self) -> bool:
-        return True
-    def fit_predict(self, X, params: dict) -> ClusterRunOutput:
-        try:
-            import hdbscan
-        except Exception as exc:
-            raise ValueError('HDBSCAN is not installed in this environment') from exc
-        cfg = self.default_params() | params
-        model = hdbscan.HDBSCAN(
-            min_cluster_size=int(cfg.get('min_cluster_size', 15)),
-            min_samples=int(cfg.get('min_samples', 5)),
-            cluster_selection_method=str(cfg.get('cluster_selection_method', 'eom')),
-        )
-        labels = model.fit_predict(X)
-        return ClusterRunOutput(labels=labels.tolist())

backend/app/ml/algorithms/kmeans_runner.py

@@ -1,17 +0,0 @@
-from sklearn.cluster import KMeans
-from app.ml.algorithms.base import BaseClusterRunner, ClusterRunOutput
-class Runner(BaseClusterRunner):
-    name = 'kmeans'
-    def default_params(self) -> dict:
-        return {'n_clusters': 4, 'random_state': 42, 'max_iter': 300, 'init': 'k-means++'}
-    def fit_predict(self, X, params: dict) -> ClusterRunOutput:
-        cfg = self.default_params() | params
-        model = KMeans(
-            n_clusters=int(cfg['n_clusters']),
-            random_state=int(cfg.get('random_state', 42)),
-            max_iter=int(cfg.get('max_iter', 300)),
-            init=str(cfg.get('init', 'k-means++')),
-            n_init=10,
-        )
-        labels = model.fit_predict(X)
-        return ClusterRunOutput(labels=labels.tolist(), cluster_centers=model.cluster_centers_.tolist())

backend/app/ml/algorithms/minibatch_kmeans_runner.py

@@ -1,17 +0,0 @@
-from sklearn.cluster import MiniBatchKMeans
-from app.ml.algorithms.base import BaseClusterRunner, ClusterRunOutput
-class Runner(BaseClusterRunner):
-    name = 'minibatch_kmeans'
-    def default_params(self) -> dict:
-        return {'n_clusters': 4, 'random_state': 42, 'max_iter': 200, 'batch_size': 256}
-    def fit_predict(self, X, params: dict) -> ClusterRunOutput:
-        cfg = self.default_params() | params
-        model = MiniBatchKMeans(
-            n_clusters=int(cfg['n_clusters']),
-            random_state=int(cfg.get('random_state', 42)),
-            max_iter=int(cfg.get('max_iter', 200)),
-            batch_size=int(cfg.get('batch_size', 256)),
-            n_init=5,
-        )
-        labels = model.fit_predict(X)
-        return ClusterRunOutput(labels=labels.tolist(), cluster_centers=model.cluster_centers_.tolist())

backend/app/ml/algorithms/spectral_runner.py

@@ -1,17 +0,0 @@
-from sklearn.cluster import SpectralClustering
-from app.ml.algorithms.base import BaseClusterRunner, ClusterRunOutput
-class Runner(BaseClusterRunner):
-    name = 'spectral'
-    def default_params(self) -> dict:
-        return {'n_clusters': 4, 'affinity': 'nearest_neighbors', 'n_neighbors': 10, 'random_state': 42}
-    def fit_predict(self, X, params: dict) -> ClusterRunOutput:
-        cfg = self.default_params() | params
-        model = SpectralClustering(
-            n_clusters=int(cfg.get('n_clusters', 4)),
-            affinity=str(cfg.get('affinity', 'nearest_neighbors')),
-            n_neighbors=int(cfg.get('n_neighbors', 10)),
-            random_state=int(cfg.get('random_state', 42)),
-            assign_labels='kmeans',
-        )
-        labels = model.fit_predict(X)
-        return ClusterRunOutput(labels=labels.tolist())

backend/app/ml/explain/cluster_profiles.py

@@ -1,22 +0,0 @@
-import pandas as pd
-
-def build_cluster_profiles(df: pd.DataFrame, labels: list[int], feature_columns: list[str]) -> dict[int, dict]:
-    temp = df[feature_columns].copy()
-    temp['_cluster'] = labels
-    profiles = {}
-    for cluster_label, cluster_df in temp.groupby('_cluster'):
-        numeric_profile = {}
-        categorical_profile = {}
-        for col in feature_columns:
-            if pd.api.types.is_numeric_dtype(temp[col]):
-                numeric_profile[col] = round(float(cluster_df[col].mean()), 4)
-            else:
-                vc = cluster_df[col].astype(str).value_counts(normalize=True)
-                if len(vc):
-                    categorical_profile[col] = {'top_value': vc.index[0], 'share': round(float(vc.iloc[0]), 4)}
-        profiles[int(cluster_label)] = {
-            'size': int(len(cluster_df)),
-            'numeric_profile': numeric_profile,
-            'categorical_profile': categorical_profile,
-        }
-    return profiles

backend/app/ml/explain/feature_lift.py

@@ -1,24 +0,0 @@
-import pandas as pd
-
-def compute_feature_lift(df: pd.DataFrame, labels: list[int], feature_columns: list[str]) -> dict[int, dict]:
-    temp = df[feature_columns].copy()
-    temp['_cluster'] = labels
-    out = {}
-    global_numeric = {c: float(temp[c].mean()) for c in feature_columns if pd.api.types.is_numeric_dtype(temp[c])}
-    for cluster_label, cluster_df in temp.groupby('_cluster'):
-        info = {'top_numeric_lifts': [], 'top_categorical_modes': []}
-        numeric_lifts = []
-        for col in feature_columns:
-            if pd.api.types.is_numeric_dtype(temp[col]):
-                base = global_numeric.get(col, 0.0)
-                mean = float(cluster_df[col].mean())
-                lift = round(mean - base, 4)
-                numeric_lifts.append({'feature': col, 'cluster_mean': round(mean, 4), 'global_mean': round(base, 4), 'lift': lift})
-            else:
-                mode_series = cluster_df[col].astype(str).value_counts(normalize=True)
-                if len(mode_series):
-                    info['top_categorical_modes'].append({'feature': col, 'value': mode_series.index[0], 'share': round(float(mode_series.iloc[0]), 4)})
-        info['top_numeric_lifts'] = sorted(numeric_lifts, key=lambda x: abs(x['lift']), reverse=True)[:5]
-        info['top_categorical_modes'] = info['top_categorical_modes'][:5]
-        out[int(cluster_label)] = info
-    return out

backend/app/ml/explain/summary_writer.py

@@ -1,10 +0,0 @@
-def build_cluster_description(cluster_label: int, profile: dict, lift_info: dict) -> str:
-    numeric = lift_info.get('top_numeric_lifts', [])[:3]
-    categorical = lift_info.get('top_categorical_modes', [])[:2]
-    parts = []
-    if numeric:
-        parts.append(', '.join([('high ' if item['lift'] >= 0 else 'low ') + item['feature'] for item in numeric]))
-    if categorical:
-        parts.append(', '.join([f"{item['feature']} is mostly {item['value']}" for item in categorical]))
-    core = '. '.join([p for p in parts if p]).strip() or 'This cluster is defined by a mixed feature pattern.'
-    return f"Cluster {cluster_label} has {profile.get('size', 0)} rows. {core}."

backend/app/ml/metrics/clustering_metrics.py

@@ -1,33 +0,0 @@
-import numpy as np
-from sklearn.metrics import calinski_harabasz_score, davies_bouldin_score, silhouette_score
-
-def compute_metrics(X, labels: list[int]) -> dict:
-    arr = np.array(labels)
-    unique = set(arr.tolist())
-    non_noise = [x for x in unique if x != -1]
-    cluster_count = len(non_noise)
-    noise_ratio = float((arr == -1).mean()) if len(arr) else 0.0
-    metrics = {
-        'cluster_count': cluster_count,
-        'noise_ratio': round(noise_ratio, 4),
-        'silhouette_score': None,
-        'davies_bouldin_score': None,
-        'calinski_harabasz_score': None,
-    }
-    mask = arr != -1 if -1 in unique else np.ones(len(arr), dtype=bool)
-    X_eval = X[mask]
-    y_eval = arr[mask]
-    if len(set(y_eval.tolist())) >= 2 and len(X_eval) > len(set(y_eval.tolist())):
-        try:
-            metrics['silhouette_score'] = round(float(silhouette_score(X_eval, y_eval)), 4)
-        except Exception:
-            pass
-        try:
-            metrics['davies_bouldin_score'] = round(float(davies_bouldin_score(X_eval, y_eval)), 4)
-        except Exception:
-            pass
-        try:
-            metrics['calinski_harabasz_score'] = round(float(calinski_harabasz_score(X_eval, y_eval)), 4)
-        except Exception:
-            pass
-    return metrics

backend/app/ml/metrics/interpretability.py

@@ -1,19 +0,0 @@
-import numpy as np
-import pandas as pd
-
-def compute_interpretability_score(df: pd.DataFrame, labels: list[int], feature_columns: list[str]) -> float:
-    if len(set(labels)) <= 1:
-        return 0.0
-    scored = []
-    temp = df[feature_columns].copy()
-    temp['_cluster'] = labels
-    for col in feature_columns:
-        if pd.api.types.is_numeric_dtype(temp[col]):
-            global_std = temp[col].std() or 1.0
-            diffs = temp.groupby('_cluster')[col].mean().diff().abs().fillna(0)
-            lift = float(diffs.mean() / global_std)
-        else:
-            top_share = temp.groupby('_cluster')[col].agg(lambda s: s.astype(str).value_counts(normalize=True).iloc[0] if len(s) else 0).mean()
-            lift = float(top_share)
-        scored.append(min(lift, 1.0))
-    return round(float(np.mean(scored)), 4) if scored else 0.0

backend/app/ml/metrics/stability.py

@@ -1,10 +0,0 @@
-import numpy as np
-from sklearn.metrics import normalized_mutual_info_score
-
-def compute_stability_score(labels: list[int]) -> float:
-    arr = np.array(labels)
-    if len(set(arr.tolist())) <= 1:
-        return 0.0
-    rotated = np.roll(arr, 1)
-    score = normalized_mutual_info_score(arr, rotated)
-    return round(float(score), 4)

backend/app/ml/outliers/centroid_distance.py

@@ -1,18 +0,0 @@
-import numpy as np
-
-def detect_centroid_outliers(X, labels: list[int], threshold_quantile: float = 0.95) -> list[bool]:
-    arr = np.array(labels)
-    outliers = np.zeros(len(arr), dtype=bool)
-    for cluster_label in sorted(set(arr.tolist())):
-        if cluster_label == -1:
-            outliers[arr == cluster_label] = True
-            continue
-        idx = np.where(arr == cluster_label)[0]
-        if len(idx) <= 2:
-            continue
-        cluster_points = X[idx]
-        centroid = cluster_points.mean(axis=0)
-        dists = np.linalg.norm(cluster_points - centroid, axis=1)
-        cut = np.quantile(dists, threshold_quantile)
-        outliers[idx] = dists >= cut
-    return outliers.tolist()

backend/app/ml/outliers/density_noise.py

@@ -1,2 +0,0 @@
-def detect_noise_outliers(labels: list[int]) -> list[bool]:
-    return [label == -1 for label in labels]

backend/app/ml/outliers/isolation_forest.py

@@ -1,6 +0,0 @@
-from sklearn.ensemble import IsolationForest
-
-def detect_isolation_outliers(X) -> list[bool]:
-    model = IsolationForest(random_state=42, contamination='auto')
-    flags = model.fit_predict(X)
-    return [item == -1 for item in flags.tolist()]

backend/app/ml/preprocess/column_inference.py

@@ -1,33 +0,0 @@
-from dataclasses import dataclass
-import pandas as pd
-
-@dataclass
-class InferredColumn:
-    name: str
-    inferred_type: str
-    missing_pct: float
-    cardinality: int
-    sample_values: list
-
-def infer_columns(df: pd.DataFrame) -> list[InferredColumn]:
-    cols = []
-    for col in df.columns:
-        series = df[col]
-        missing_pct = float(series.isna().mean() * 100)
-        cardinality = int(series.nunique(dropna=True))
-        sample_values = series.dropna().astype(str).head(5).tolist()
-        if pd.api.types.is_bool_dtype(series):
-            inferred = 'boolean'
-        elif pd.api.types.is_numeric_dtype(series):
-            inferred = 'numeric'
-        else:
-            unique_ratio = cardinality / max(len(series), 1)
-            inferred = 'high_cardinality' if cardinality > 50 and unique_ratio > 0.3 else 'categorical'
-        cols.append(InferredColumn(
-            name=col,
-            inferred_type=inferred,
-            missing_pct=round(missing_pct, 2),
-            cardinality=cardinality,
-            sample_values=sample_values,
-        ))
-    return cols

backend/app/ml/preprocess/encoders.py

@@ -1,7 +0,0 @@
-from sklearn.preprocessing import OneHotEncoder
-
-def get_onehot_encoder():
-    try:
-        return OneHotEncoder(handle_unknown='ignore', sparse_output=False)
-    except TypeError:
-        return OneHotEncoder(handle_unknown='ignore', sparse=False)

backend/app/ml/preprocess/imputers.py

@@ -1,7 +0,0 @@
-from sklearn.impute import SimpleImputer
-
-def numeric_imputer():
-    return SimpleImputer(strategy='median')
-
-def categorical_imputer():
-    return SimpleImputer(strategy='most_frequent')

backend/app/ml/preprocess/matrix_builder.py

@@ -1,32 +0,0 @@
-import pandas as pd
-from sklearn.compose import ColumnTransformer
-from sklearn.pipeline import Pipeline
-from app.ml.preprocess.encoders import get_onehot_encoder
-from app.ml.preprocess.imputers import categorical_imputer, numeric_imputer
-from app.ml.preprocess.scalers import get_scaler
-
-def build_feature_matrix(df: pd.DataFrame, feature_columns: list[str], numeric_scaler: str = 'standard', categorical_encoder: str = 'onehot'):
-    selected = df[feature_columns].copy()
-    numeric_cols = [c for c in selected.columns if pd.api.types.is_numeric_dtype(selected[c]) or pd.api.types.is_bool_dtype(selected[c])]
-    categorical_cols = [c for c in selected.columns if c not in numeric_cols]
-
-    numeric_pipeline = Pipeline(steps=[('imputer', numeric_imputer()), ('scaler', get_scaler(numeric_scaler))])
-    categorical_pipeline = Pipeline(steps=[('imputer', categorical_imputer()), ('encoder', get_onehot_encoder())])
-
-    transformer = ColumnTransformer(
-        transformers=[('num', numeric_pipeline, numeric_cols), ('cat', categorical_pipeline, categorical_cols)],
-        remainder='drop',
-    )
-    X = transformer.fit_transform(selected)
-
-    feature_names = []
-    feature_names.extend(numeric_cols)
-    if categorical_cols:
-        enc = transformer.named_transformers_['cat'].named_steps['encoder']
-        try:
-            cat_names = enc.get_feature_names_out(categorical_cols).tolist()
-        except Exception:
-            cat_names = categorical_cols
-        feature_names.extend(cat_names)
-
-    return X, feature_names, {'numeric_cols': numeric_cols, 'categorical_cols': categorical_cols}

backend/app/ml/preprocess/reducers.py

@@ -1,16 +0,0 @@
-from sklearn.decomposition import PCA
-
-def pca_2d(X):
-    reducer = PCA(n_components=2, random_state=42)
-    coords = reducer.fit_transform(X)
-    return coords[:, 0], coords[:, 1]
-
-def umap_2d(X):
-    try:
-        import umap
-        reducer = umap.UMAP(n_components=2, random_state=42)
-        coords = reducer.fit_transform(X)
-    except Exception:
-        reducer = PCA(n_components=2, random_state=42)
-        coords = reducer.fit_transform(X)
-    return coords[:, 0], coords[:, 1]

backend/app/ml/preprocess/scalers.py

@@ -1,9 +0,0 @@
-from sklearn.preprocessing import MinMaxScaler, RobustScaler, StandardScaler
-
-def get_scaler(name: str):
-    name = (name or 'standard').lower()
-    if name == 'minmax':
-        return MinMaxScaler()
-    if name == 'robust':
-        return RobustScaler()
-    return StandardScaler()

backend/app/models/artifact.py

@@ -1,12 +0,0 @@
-from datetime import datetime
-from sqlalchemy import DateTime, ForeignKey, Integer, String, Text
-from sqlalchemy.orm import Mapped, mapped_column
-from app.db import Base
-
-class Artifact(Base):
-    __tablename__ = 'artifacts'
-    id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
-    experiment_id: Mapped[str] = mapped_column(ForeignKey('experiments.id'), index=True, nullable=False)
-    artifact_type: Mapped[str] = mapped_column(String(64), nullable=False)
-    path: Mapped[str] = mapped_column(Text, nullable=False)
-    created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow, nullable=False)

backend/app/models/cluster_result.py

@@ -1,36 +0,0 @@
-from sqlalchemy import Boolean, Float, ForeignKey, Integer, String, Text
-from sqlalchemy.orm import Mapped, mapped_column
-from app.db import Base
-
-class ExperimentMetric(Base):
-    __tablename__ = 'experiment_metrics'
-    id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
-    experiment_id: Mapped[str] = mapped_column(ForeignKey('experiments.id'), index=True, nullable=False)
-    cluster_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
-    noise_ratio: Mapped[float] = mapped_column(Float, default=0.0, nullable=False)
-    silhouette_score: Mapped[float | None] = mapped_column(Float, nullable=True)
-    davies_bouldin_score: Mapped[float | None] = mapped_column(Float, nullable=True)
-    calinski_harabasz_score: Mapped[float | None] = mapped_column(Float, nullable=True)
-    stability_score: Mapped[float | None] = mapped_column(Float, nullable=True)
-    interpretability_score: Mapped[float | None] = mapped_column(Float, nullable=True)
-
-class ClusterResult(Base):
-    __tablename__ = 'cluster_results'
-    id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
-    experiment_id: Mapped[str] = mapped_column(ForeignKey('experiments.id'), index=True, nullable=False)
-    row_index: Mapped[int] = mapped_column(Integer, nullable=False)
-    cluster_label: Mapped[int] = mapped_column(Integer, nullable=False, index=True)
-    is_outlier: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False)
-    pca_x: Mapped[float | None] = mapped_column(Float, nullable=True)
-    pca_y: Mapped[float | None] = mapped_column(Float, nullable=True)
-    umap_x: Mapped[float | None] = mapped_column(Float, nullable=True)
-    umap_y: Mapped[float | None] = mapped_column(Float, nullable=True)
-
-class ClusterSummary(Base):
-    __tablename__ = 'cluster_summaries'
-    id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
-    experiment_id: Mapped[str] = mapped_column(ForeignKey('experiments.id'), index=True, nullable=False)
-    cluster_label: Mapped[int] = mapped_column(Integer, nullable=False, index=True)
-    size: Mapped[int] = mapped_column(Integer, nullable=False)
-    summary_json: Mapped[str] = mapped_column(Text, default='{}', nullable=False)
-    description_text: Mapped[str] = mapped_column(Text, nullable=False)

backend/app/models/dataset.py

@@ -1,16 +0,0 @@
-from datetime import datetime
-from sqlalchemy import DateTime, Integer, String, Text
-from sqlalchemy.orm import Mapped, mapped_column
-from app.db import Base
-
-class Dataset(Base):
-    __tablename__ = 'datasets'
-    id: Mapped[str] = mapped_column(String(32), primary_key=True)
-    name: Mapped[str] = mapped_column(String(255), nullable=False)
-    file_path: Mapped[str] = mapped_column(Text, nullable=False)
-    sha256: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
-    uploaded_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow, nullable=False)
-    row_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
-    column_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
-    schema_json: Mapped[str] = mapped_column(Text, default='{}', nullable=False)
-    profile_json: Mapped[str] = mapped_column(Text, default='{}', nullable=False)

backend/app/models/experiment.py

@@ -1,17 +0,0 @@
-from datetime import datetime
-from sqlalchemy import DateTime, ForeignKey, Integer, String, Text
-from sqlalchemy.orm import Mapped, mapped_column
-from app.db import Base
-
-class Experiment(Base):
-    __tablename__ = 'experiments'
-    id: Mapped[str] = mapped_column(String(32), primary_key=True)
-    dataset_id: Mapped[str] = mapped_column(ForeignKey('datasets.id'), index=True, nullable=False)
-    name: Mapped[str] = mapped_column(String(255), nullable=False)
-    algorithm: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
-    status: Mapped[str] = mapped_column(String(32), nullable=False, default='pending')
-    config_json: Mapped[str] = mapped_column(Text, default='{}', nullable=False)
-    started_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow, nullable=False)
-    completed_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
-    runtime_ms: Mapped[int | None] = mapped_column(Integer, nullable=True)
-    error_message: Mapped[str | None] = mapped_column(Text, nullable=True)