diff --git a/backend/.DS_Store b/backend/.DS_Store deleted file mode 100644 index ba27f483746bc38feb7c61e939fd716c22fcac3e..0000000000000000000000000000000000000000 Binary files a/backend/.DS_Store and /dev/null differ diff --git a/backend/app/__init__.py b/backend/app/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/backend/app/api/__init__.py b/backend/app/api/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/backend/app/api/clusters.py b/backend/app/api/clusters.py deleted file mode 100644 index 63a58ec708d6c407d2ee9cf8263b9d45fa8b7267..0000000000000000000000000000000000000000 --- a/backend/app/api/clusters.py +++ /dev/null @@ -1,40 +0,0 @@ -import json -import pandas as pd -from fastapi import APIRouter, HTTPException -from app.db import get_session -from app.repositories.cluster_repo import ClusterRepository -from app.repositories.dataset_repo import DatasetRepository -from app.repositories.experiment_repo import ExperimentRepository -from app.schemas.cluster import ClusterDetailResponse, ClusterMember - -router = APIRouter(prefix='/clusters', tags=['clusters']) - -@router.get('/{experiment_id}/{cluster_label}', response_model=ClusterDetailResponse) -def get_cluster_detail(experiment_id: str, cluster_label: int): - with get_session() as session: - exp_repo = ExperimentRepository(session) - dataset_repo = DatasetRepository(session) - cluster_repo = ClusterRepository(session) - exp = exp_repo.get(experiment_id) - if not exp: - raise HTTPException(status_code=404, detail='Experiment not found') - dataset = dataset_repo.get(exp.dataset_id) - if not dataset: - raise HTTPException(status_code=404, detail='Dataset not found') - df = pd.read_csv(dataset.file_path) if dataset.file_path.lower().endswith('.csv') else pd.read_excel(dataset.file_path) - results = [r for r in cluster_repo.get_results(experiment_id) if r.cluster_label == cluster_label] - summary = cluster_repo.get_summary(experiment_id, cluster_label) - if not summary: - raise HTTPException(status_code=404, detail='Cluster summary not found') - members = [] - for row in results[:200]: - values = df.iloc[row.row_index].to_dict() - values = {str(k): (None if pd.isna(v) else v) for k, v in values.items()} - members.append(ClusterMember(row_index=row.row_index, values=values)) - return ClusterDetailResponse( - experiment_id=experiment_id, - cluster_label=cluster_label, - members=members, - summary=json.loads(summary.summary_json), - description_text=summary.description_text, - ) diff --git a/backend/app/api/compare.py b/backend/app/api/compare.py deleted file mode 100644 index 04f3d721148d5a4361d06d0200eb20274aa71558..0000000000000000000000000000000000000000 --- a/backend/app/api/compare.py +++ /dev/null @@ -1,19 +0,0 @@ -from fastapi import APIRouter, Query -from app.db import get_session -from app.repositories.cluster_repo import ClusterRepository -from app.repositories.experiment_repo import ExperimentRepository -from app.schemas.compare import CompareResponse -from app.services.comparison_service import ComparisonService - -router = APIRouter(prefix='/compare', tags=['compare']) - -@router.get('', response_model=CompareResponse) -def compare_runs(experiment_ids: str = Query(...)): - ids = [item.strip() for item in experiment_ids.split(',') if item.strip()] - with get_session() as session: - payload = ComparisonService().build_compare_payload( - ExperimentRepository(session), - ClusterRepository(session), - ids, - ) - return CompareResponse(**payload) diff --git a/backend/app/api/datasets.py b/backend/app/api/datasets.py deleted file mode 100644 index da17b521d3131716cee3a5166f8fd421f5cb0977..0000000000000000000000000000000000000000 --- a/backend/app/api/datasets.py +++ /dev/null @@ -1,55 +0,0 @@ -from fastapi import APIRouter, File, HTTPException, UploadFile -from app.db import get_session -from app.repositories.dataset_repo import DatasetRepository -from app.schemas.dataset import DatasetProfileResponse, DatasetUploadResponse -from app.services.dataset_service import DatasetService -from app.services.profiling_service import ProfilingService - -router = APIRouter(prefix='/datasets', tags=['datasets']) - -@router.post('/upload', response_model=DatasetUploadResponse) -async def upload_dataset(file: UploadFile = File(...)): - with get_session() as session: - repo = DatasetRepository(session) - service = DatasetService(repo) - dataset = await service.save_upload(file) - return DatasetUploadResponse( - dataset_id=dataset.id, - name=dataset.name, - row_count=dataset.row_count, - column_count=dataset.column_count, - ) - -@router.get('', response_model=list[DatasetUploadResponse]) -def list_datasets(): - with get_session() as session: - repo = DatasetRepository(session) - items = repo.list_all() - return [ - DatasetUploadResponse( - dataset_id=item.id, - name=item.name, - row_count=item.row_count, - column_count=item.column_count, - ) - for item in items - ] - -@router.get('/{dataset_id}/profile', response_model=DatasetProfileResponse) -def dataset_profile(dataset_id: str): - with get_session() as session: - repo = DatasetRepository(session) - dataset = repo.get(dataset_id) - if not dataset: - raise HTTPException(status_code=404, detail='Dataset not found') - df = DatasetService(repo).load_dataframe(dataset) - profile = ProfilingService().profile(df) - repo.update_profile(dataset_id, schema_json={'columns': [c['name'] for c in profile['columns']]}, profile_json=profile) - return DatasetProfileResponse( - dataset_id=dataset_id, - columns=profile['columns'], - suggested_feature_columns=profile['suggested_feature_columns'], - recommended_algorithms=profile['recommended_algorithms'], - row_count=profile['row_count'], - column_count=profile['column_count'], - ) diff --git a/backend/app/api/experiments.py b/backend/app/api/experiments.py deleted file mode 100644 index f7c5bdc21744cde43675abbb4db33d54243c2700..0000000000000000000000000000000000000000 --- a/backend/app/api/experiments.py +++ /dev/null @@ -1,105 +0,0 @@ -import json -from fastapi import APIRouter, HTTPException -from app.db import get_session -from app.repositories.cluster_repo import ClusterRepository -from app.repositories.dataset_repo import DatasetRepository -from app.repositories.experiment_repo import ExperimentRepository -from app.schemas.experiment import ClusterSummaryResponse, ExperimentListItem, ExperimentResultResponse, ExperimentRunRequest, ExperimentRunResponse, MetricResponse, ScatterPoint -from app.services.run_service import RunService - -router = APIRouter(prefix='/experiments', tags=['experiments']) - -@router.post('/run', response_model=ExperimentRunResponse) -def run_experiment(payload: ExperimentRunRequest): - with get_session() as session: - dataset_repo = DatasetRepository(session) - exp_repo = ExperimentRepository(session) - cluster_repo = ClusterRepository(session) - service = RunService(dataset_repo, exp_repo, cluster_repo) - experiment = service.run(payload.model_dump()) - return ExperimentRunResponse(experiment_id=experiment.id, status='completed') - -@router.get('', response_model=list[ExperimentListItem]) -def list_experiments(): - with get_session() as session: - repo = ExperimentRepository(session) - items = repo.list_all() - return [ - ExperimentListItem( - experiment_id=item.id, - dataset_id=item.dataset_id, - name=item.name, - algorithm=item.algorithm, - status=item.status, - started_at=item.started_at.isoformat(), - completed_at=item.completed_at.isoformat() if item.completed_at else None, - ) - for item in items - ] - -@router.get('/{experiment_id}') -def get_experiment(experiment_id: str): - with get_session() as session: - repo = ExperimentRepository(session) - exp = repo.get(experiment_id) - if not exp: - raise HTTPException(status_code=404, detail='Experiment not found') - return { - 'experiment_id': exp.id, - 'dataset_id': exp.dataset_id, - 'name': exp.name, - 'algorithm': exp.algorithm, - 'status': exp.status, - 'config': json.loads(exp.config_json), - 'runtime_ms': exp.runtime_ms, - 'error_message': exp.error_message, - } - -@router.get('/{experiment_id}/results', response_model=ExperimentResultResponse) -def get_results(experiment_id: str): - with get_session() as session: - exp_repo = ExperimentRepository(session) - cluster_repo = ClusterRepository(session) - exp = exp_repo.get(experiment_id) - if not exp: - raise HTTPException(status_code=404, detail='Experiment not found') - metric = cluster_repo.get_metric(experiment_id) - summaries = cluster_repo.get_summaries(experiment_id) - results = cluster_repo.get_results(experiment_id) - if not metric: - raise HTTPException(status_code=404, detail='Results not found') - scatter = [ - ScatterPoint( - row_index=item.row_index, - cluster_label=item.cluster_label, - is_outlier=item.is_outlier, - x=item.umap_x if item.umap_x is not None else item.pca_x, - y=item.umap_y if item.umap_y is not None else item.pca_y, - ) - for item in results - ] - summary_items = [ - ClusterSummaryResponse( - cluster_label=item.cluster_label, - size=item.size, - description_text=item.description_text, - summary_json=json.loads(item.summary_json), - ) - for item in summaries - ] - return ExperimentResultResponse( - experiment_id=experiment_id, - algorithm=exp.algorithm, - metrics=MetricResponse( - cluster_count=metric.cluster_count, - noise_ratio=metric.noise_ratio, - silhouette_score=metric.silhouette_score, - davies_bouldin_score=metric.davies_bouldin_score, - calinski_harabasz_score=metric.calinski_harabasz_score, - stability_score=metric.stability_score, - interpretability_score=metric.interpretability_score, - ), - scatter_points=scatter, - cluster_summaries=summary_items, - status=exp.status, - ) diff --git a/backend/app/api/exports.py b/backend/app/api/exports.py deleted file mode 100644 index 661b73407685c38bba37114849a5eddaf99fea97..0000000000000000000000000000000000000000 --- a/backend/app/api/exports.py +++ /dev/null @@ -1,24 +0,0 @@ -from fastapi import APIRouter, HTTPException -from app.db import get_session -from app.repositories.artifact_repo import ArtifactRepository -from app.repositories.cluster_repo import ClusterRepository -from app.repositories.dataset_repo import DatasetRepository -from app.repositories.experiment_repo import ExperimentRepository -from app.schemas.export import ExportResponse -from app.services.export_service import ExportService - -router = APIRouter(prefix='/exports', tags=['exports']) - -@router.get('/{experiment_id}', response_model=ExportResponse) -def export_experiment(experiment_id: str): - with get_session() as session: - exp = ExperimentRepository(session).get(experiment_id) - if not exp: - raise HTTPException(status_code=404, detail='Experiment not found') - service = ExportService( - dataset_repo=DatasetRepository(session), - cluster_repo=ClusterRepository(session), - artifact_repo=ArtifactRepository(session), - ) - path = service.export_csv(experiment_id, exp.dataset_id) - return ExportResponse(experiment_id=experiment_id, download_path=path) diff --git a/backend/app/api/health.py b/backend/app/api/health.py deleted file mode 100644 index 6648e86867ac1d4494b50d744e47635650dfc7de..0000000000000000000000000000000000000000 --- a/backend/app/api/health.py +++ /dev/null @@ -1,7 +0,0 @@ -from fastapi import APIRouter -from app.schemas.health import HealthResponse -router = APIRouter(tags=['health']) - -@router.get('/health', response_model=HealthResponse) -def health() -> HealthResponse: - return HealthResponse(ok=True, service='clusterforge-api') diff --git a/backend/app/api/presets.py b/backend/app/api/presets.py deleted file mode 100644 index c934338397631cd80143ad5c1ab5638f79d3ce6c..0000000000000000000000000000000000000000 --- a/backend/app/api/presets.py +++ /dev/null @@ -1,17 +0,0 @@ -from fastapi import APIRouter -router = APIRouter(prefix='/presets', tags=['presets']) - -@router.get('/algorithms') -def list_algorithms(): - return { - 'algorithms': [ - {'key': 'kmeans', 'label': 'KMeans', 'params': {'n_clusters': 4, 'random_state': 42, 'max_iter': 300, 'init': 'k-means++'}}, - {'key': 'minibatch_kmeans', 'label': 'MiniBatch KMeans', 'params': {'n_clusters': 4, 'batch_size': 256, 'max_iter': 200}}, - {'key': 'agglomerative', 'label': 'Agglomerative', 'params': {'n_clusters': 4, 'linkage': 'ward', 'metric': 'euclidean'}}, - {'key': 'dbscan', 'label': 'DBSCAN', 'params': {'eps': 0.7, 'min_samples': 5, 'metric': 'euclidean'}}, - {'key': 'hdbscan', 'label': 'HDBSCAN', 'params': {'min_cluster_size': 15, 'min_samples': 5, 'cluster_selection_method': 'eom'}}, - {'key': 'gmm', 'label': 'Gaussian Mixture', 'params': {'n_components': 4, 'covariance_type': 'full'}}, - {'key': 'spectral', 'label': 'Spectral', 'params': {'n_clusters': 4, 'affinity': 'nearest_neighbors', 'n_neighbors': 10}}, - {'key': 'birch', 'label': 'Birch', 'params': {'n_clusters': 4, 'threshold': 0.5, 'branching_factor': 50}}, - ] - } diff --git a/backend/app/config.py b/backend/app/config.py deleted file mode 100644 index 78bfe8ecfdadb89cc5336f1cbc7288107364d6ef..0000000000000000000000000000000000000000 --- a/backend/app/config.py +++ /dev/null @@ -1,23 +0,0 @@ -from functools import lru_cache -from typing import List -from pydantic import Field -from pydantic_settings import BaseSettings, SettingsConfigDict - -class Settings(BaseSettings): - model_config = SettingsConfigDict(env_file='.env', env_file_encoding='utf-8', extra='ignore') - app_name: str = Field(default='ClusterForge', alias='APP_NAME') - app_env: str = Field(default='dev', alias='APP_ENV') - api_prefix: str = Field(default='/api', alias='API_PREFIX') - data_dir: str = Field(default='/data', alias='DATA_DIR') - db_path: str = Field(default='/data/clusterforge.db', alias='DB_PATH') - upload_dir: str = Field(default='/data/uploads', alias='UPLOAD_DIR') - artifact_dir: str = Field(default='/data/artifacts', alias='ARTIFACT_DIR') - allowed_origins_raw: str = Field(default='http://localhost:3000,http://127.0.0.1:3000', alias='ALLOWED_ORIGINS') - - @property - def allowed_origins(self) -> List[str]: - return [x.strip() for x in self.allowed_origins_raw.split(',') if x.strip()] - -@lru_cache -def get_settings() -> Settings: - return Settings() diff --git a/backend/app/db.py b/backend/app/db.py deleted file mode 100644 index 66f387b7298c772a6cd13109c46473121545921d..0000000000000000000000000000000000000000 --- a/backend/app/db.py +++ /dev/null @@ -1,36 +0,0 @@ -from contextlib import contextmanager -from sqlalchemy import create_engine -from sqlalchemy.orm import declarative_base, sessionmaker -from app.config import get_settings - -settings = get_settings() -engine = create_engine( - f"sqlite:///{settings.db_path}", - connect_args={'check_same_thread': False}, - future=True, -) -SessionLocal = sessionmaker(bind=engine, autoflush=False, autocommit=False, future=True) -Base = declarative_base() - -def init_storage_dirs() -> None: - import os - os.makedirs(settings.data_dir, exist_ok=True) - os.makedirs(settings.upload_dir, exist_ok=True) - os.makedirs(settings.artifact_dir, exist_ok=True) - -def init_db() -> None: - init_storage_dirs() - from app.models import dataset, experiment, cluster_result, artifact # noqa - Base.metadata.create_all(bind=engine) - -@contextmanager -def get_session(): - session = SessionLocal() - try: - yield session - session.commit() - except Exception: - session.rollback() - raise - finally: - session.close() diff --git a/backend/app/logging_config.py b/backend/app/logging_config.py deleted file mode 100644 index 730062ef056da664fb9780de25ee49512b6341c1..0000000000000000000000000000000000000000 --- a/backend/app/logging_config.py +++ /dev/null @@ -1,7 +0,0 @@ -import logging - -def setup_logging() -> None: - logging.basicConfig( - level=logging.INFO, - format='%(asctime)s | %(levelname)s | %(name)s | %(message)s' - ) diff --git a/backend/app/main.py b/backend/app/main.py deleted file mode 100644 index 92eee7ee5fe1db6fef986b6375fb08cf9cb2a88c..0000000000000000000000000000000000000000 --- a/backend/app/main.py +++ /dev/null @@ -1,31 +0,0 @@ -from fastapi import FastAPI -from fastapi.middleware.cors import CORSMiddleware -from app.api.clusters import router as clusters_router -from app.api.compare import router as compare_router -from app.api.datasets import router as datasets_router -from app.api.experiments import router as experiments_router -from app.api.exports import router as exports_router -from app.api.health import router as health_router -from app.api.presets import router as presets_router -from app.config import get_settings -from app.db import init_db -from app.logging_config import setup_logging - -setup_logging() -settings = get_settings() -init_db() -app = FastAPI(title=settings.app_name) -app.add_middleware( - CORSMiddleware, - allow_origins=settings.allowed_origins, - allow_credentials=True, - allow_methods=['*'], - allow_headers=['*'], -) -app.include_router(health_router, prefix=settings.api_prefix) -app.include_router(datasets_router, prefix=settings.api_prefix) -app.include_router(experiments_router, prefix=settings.api_prefix) -app.include_router(clusters_router, prefix=settings.api_prefix) -app.include_router(compare_router, prefix=settings.api_prefix) -app.include_router(exports_router, prefix=settings.api_prefix) -app.include_router(presets_router, prefix=settings.api_prefix) diff --git a/backend/app/ml/__init__.py b/backend/app/ml/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/backend/app/ml/algorithms/__init__.py b/backend/app/ml/algorithms/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/backend/app/ml/algorithms/agglomerative_runner.py b/backend/app/ml/algorithms/agglomerative_runner.py deleted file mode 100644 index dcf000cad61f44e82cb96fa32d64380dc172338c..0000000000000000000000000000000000000000 --- a/backend/app/ml/algorithms/agglomerative_runner.py +++ /dev/null @@ -1,14 +0,0 @@ -from sklearn.cluster import AgglomerativeClustering -from app.ml.algorithms.base import BaseClusterRunner, ClusterRunOutput -class Runner(BaseClusterRunner): - name = 'agglomerative' - def default_params(self) -> dict: - return {'n_clusters': 4, 'linkage': 'ward', 'metric': 'euclidean'} - def fit_predict(self, X, params: dict) -> ClusterRunOutput: - cfg = self.default_params() | params - kwargs = {'n_clusters': int(cfg['n_clusters']), 'linkage': str(cfg.get('linkage', 'ward'))} - if kwargs['linkage'] != 'ward': - kwargs['metric'] = str(cfg.get('metric', 'euclidean')) - model = AgglomerativeClustering(**kwargs) - labels = model.fit_predict(X) - return ClusterRunOutput(labels=labels.tolist()) diff --git a/backend/app/ml/algorithms/base.py b/backend/app/ml/algorithms/base.py deleted file mode 100644 index 0499ce20d20fff75852ac1a980e5f12f25b9f6cf..0000000000000000000000000000000000000000 --- a/backend/app/ml/algorithms/base.py +++ /dev/null @@ -1,17 +0,0 @@ -from dataclasses import dataclass -from typing import Any - -@dataclass -class ClusterRunOutput: - labels: list[int] - cluster_centers: list[list[float]] | None = None - extra: dict[str, Any] | None = None - -class BaseClusterRunner: - name: str = 'base' - def fit_predict(self, X, params: dict) -> ClusterRunOutput: - raise NotImplementedError - def supports_noise(self) -> bool: - return False - def default_params(self) -> dict: - return {} diff --git a/backend/app/ml/algorithms/birch_runner.py b/backend/app/ml/algorithms/birch_runner.py deleted file mode 100644 index ad4a73fd10d486aebff8e242f7cf62410de2046f..0000000000000000000000000000000000000000 --- a/backend/app/ml/algorithms/birch_runner.py +++ /dev/null @@ -1,15 +0,0 @@ -from sklearn.cluster import Birch -from app.ml.algorithms.base import BaseClusterRunner, ClusterRunOutput -class Runner(BaseClusterRunner): - name = 'birch' - def default_params(self) -> dict: - return {'n_clusters': 4, 'threshold': 0.5, 'branching_factor': 50} - def fit_predict(self, X, params: dict) -> ClusterRunOutput: - cfg = self.default_params() | params - model = Birch( - n_clusters=int(cfg.get('n_clusters', 4)), - threshold=float(cfg.get('threshold', 0.5)), - branching_factor=int(cfg.get('branching_factor', 50)), - ) - labels = model.fit_predict(X) - return ClusterRunOutput(labels=labels.tolist()) diff --git a/backend/app/ml/algorithms/dbscan_runner.py b/backend/app/ml/algorithms/dbscan_runner.py deleted file mode 100644 index 436769364172b75fb481f539b9f9bf9a18ae1bb0..0000000000000000000000000000000000000000 --- a/backend/app/ml/algorithms/dbscan_runner.py +++ /dev/null @@ -1,17 +0,0 @@ -from sklearn.cluster import DBSCAN -from app.ml.algorithms.base import BaseClusterRunner, ClusterRunOutput -class Runner(BaseClusterRunner): - name = 'dbscan' - def default_params(self) -> dict: - return {'eps': 0.7, 'min_samples': 5, 'metric': 'euclidean'} - def supports_noise(self) -> bool: - return True - def fit_predict(self, X, params: dict) -> ClusterRunOutput: - cfg = self.default_params() | params - model = DBSCAN( - eps=float(cfg.get('eps', 0.7)), - min_samples=int(cfg.get('min_samples', 5)), - metric=str(cfg.get('metric', 'euclidean')), - ) - labels = model.fit_predict(X) - return ClusterRunOutput(labels=labels.tolist()) diff --git a/backend/app/ml/algorithms/gmm_runner.py b/backend/app/ml/algorithms/gmm_runner.py deleted file mode 100644 index 293537b7483f4c8975da03ac354de31ea5e7caea..0000000000000000000000000000000000000000 --- a/backend/app/ml/algorithms/gmm_runner.py +++ /dev/null @@ -1,15 +0,0 @@ -from sklearn.mixture import GaussianMixture -from app.ml.algorithms.base import BaseClusterRunner, ClusterRunOutput -class Runner(BaseClusterRunner): - name = 'gmm' - def default_params(self) -> dict: - return {'n_components': 4, 'covariance_type': 'full', 'random_state': 42} - def fit_predict(self, X, params: dict) -> ClusterRunOutput: - cfg = self.default_params() | params - model = GaussianMixture( - n_components=int(cfg.get('n_components', 4)), - covariance_type=str(cfg.get('covariance_type', 'full')), - random_state=int(cfg.get('random_state', 42)), - ) - labels = model.fit_predict(X) - return ClusterRunOutput(labels=labels.tolist(), cluster_centers=model.means_.tolist()) diff --git a/backend/app/ml/algorithms/hdbscan_runner.py b/backend/app/ml/algorithms/hdbscan_runner.py deleted file mode 100644 index d5dbb7423c7c947853f137b45895e7194c44c40d..0000000000000000000000000000000000000000 --- a/backend/app/ml/algorithms/hdbscan_runner.py +++ /dev/null @@ -1,20 +0,0 @@ -from app.ml.algorithms.base import BaseClusterRunner, ClusterRunOutput -class Runner(BaseClusterRunner): - name = 'hdbscan' - def default_params(self) -> dict: - return {'min_cluster_size': 15, 'min_samples': 5, 'cluster_selection_method': 'eom'} - def supports_noise(self) -> bool: - return True - def fit_predict(self, X, params: dict) -> ClusterRunOutput: - try: - import hdbscan - except Exception as exc: - raise ValueError('HDBSCAN is not installed in this environment') from exc - cfg = self.default_params() | params - model = hdbscan.HDBSCAN( - min_cluster_size=int(cfg.get('min_cluster_size', 15)), - min_samples=int(cfg.get('min_samples', 5)), - cluster_selection_method=str(cfg.get('cluster_selection_method', 'eom')), - ) - labels = model.fit_predict(X) - return ClusterRunOutput(labels=labels.tolist()) diff --git a/backend/app/ml/algorithms/kmeans_runner.py b/backend/app/ml/algorithms/kmeans_runner.py deleted file mode 100644 index 5bd84143940130dbfed15e1c8240b099d455156b..0000000000000000000000000000000000000000 --- a/backend/app/ml/algorithms/kmeans_runner.py +++ /dev/null @@ -1,17 +0,0 @@ -from sklearn.cluster import KMeans -from app.ml.algorithms.base import BaseClusterRunner, ClusterRunOutput -class Runner(BaseClusterRunner): - name = 'kmeans' - def default_params(self) -> dict: - return {'n_clusters': 4, 'random_state': 42, 'max_iter': 300, 'init': 'k-means++'} - def fit_predict(self, X, params: dict) -> ClusterRunOutput: - cfg = self.default_params() | params - model = KMeans( - n_clusters=int(cfg['n_clusters']), - random_state=int(cfg.get('random_state', 42)), - max_iter=int(cfg.get('max_iter', 300)), - init=str(cfg.get('init', 'k-means++')), - n_init=10, - ) - labels = model.fit_predict(X) - return ClusterRunOutput(labels=labels.tolist(), cluster_centers=model.cluster_centers_.tolist()) diff --git a/backend/app/ml/algorithms/minibatch_kmeans_runner.py b/backend/app/ml/algorithms/minibatch_kmeans_runner.py deleted file mode 100644 index 2f34f9619c40e6d88a1f1f8879b79063c9d10596..0000000000000000000000000000000000000000 --- a/backend/app/ml/algorithms/minibatch_kmeans_runner.py +++ /dev/null @@ -1,17 +0,0 @@ -from sklearn.cluster import MiniBatchKMeans -from app.ml.algorithms.base import BaseClusterRunner, ClusterRunOutput -class Runner(BaseClusterRunner): - name = 'minibatch_kmeans' - def default_params(self) -> dict: - return {'n_clusters': 4, 'random_state': 42, 'max_iter': 200, 'batch_size': 256} - def fit_predict(self, X, params: dict) -> ClusterRunOutput: - cfg = self.default_params() | params - model = MiniBatchKMeans( - n_clusters=int(cfg['n_clusters']), - random_state=int(cfg.get('random_state', 42)), - max_iter=int(cfg.get('max_iter', 200)), - batch_size=int(cfg.get('batch_size', 256)), - n_init=5, - ) - labels = model.fit_predict(X) - return ClusterRunOutput(labels=labels.tolist(), cluster_centers=model.cluster_centers_.tolist()) diff --git a/backend/app/ml/algorithms/spectral_runner.py b/backend/app/ml/algorithms/spectral_runner.py deleted file mode 100644 index c0b0c25e356a983879b77e7fcbfbc329ff822948..0000000000000000000000000000000000000000 --- a/backend/app/ml/algorithms/spectral_runner.py +++ /dev/null @@ -1,17 +0,0 @@ -from sklearn.cluster import SpectralClustering -from app.ml.algorithms.base import BaseClusterRunner, ClusterRunOutput -class Runner(BaseClusterRunner): - name = 'spectral' - def default_params(self) -> dict: - return {'n_clusters': 4, 'affinity': 'nearest_neighbors', 'n_neighbors': 10, 'random_state': 42} - def fit_predict(self, X, params: dict) -> ClusterRunOutput: - cfg = self.default_params() | params - model = SpectralClustering( - n_clusters=int(cfg.get('n_clusters', 4)), - affinity=str(cfg.get('affinity', 'nearest_neighbors')), - n_neighbors=int(cfg.get('n_neighbors', 10)), - random_state=int(cfg.get('random_state', 42)), - assign_labels='kmeans', - ) - labels = model.fit_predict(X) - return ClusterRunOutput(labels=labels.tolist()) diff --git a/backend/app/ml/explain/__init__.py b/backend/app/ml/explain/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/backend/app/ml/explain/cluster_profiles.py b/backend/app/ml/explain/cluster_profiles.py deleted file mode 100644 index ef47f6d4dff098acd819ef9eba5ea65f1db3b531..0000000000000000000000000000000000000000 --- a/backend/app/ml/explain/cluster_profiles.py +++ /dev/null @@ -1,22 +0,0 @@ -import pandas as pd - -def build_cluster_profiles(df: pd.DataFrame, labels: list[int], feature_columns: list[str]) -> dict[int, dict]: - temp = df[feature_columns].copy() - temp['_cluster'] = labels - profiles = {} - for cluster_label, cluster_df in temp.groupby('_cluster'): - numeric_profile = {} - categorical_profile = {} - for col in feature_columns: - if pd.api.types.is_numeric_dtype(temp[col]): - numeric_profile[col] = round(float(cluster_df[col].mean()), 4) - else: - vc = cluster_df[col].astype(str).value_counts(normalize=True) - if len(vc): - categorical_profile[col] = {'top_value': vc.index[0], 'share': round(float(vc.iloc[0]), 4)} - profiles[int(cluster_label)] = { - 'size': int(len(cluster_df)), - 'numeric_profile': numeric_profile, - 'categorical_profile': categorical_profile, - } - return profiles diff --git a/backend/app/ml/explain/feature_lift.py b/backend/app/ml/explain/feature_lift.py deleted file mode 100644 index d96c693ee0c58099f4f6885f61e7d2134cf026fb..0000000000000000000000000000000000000000 --- a/backend/app/ml/explain/feature_lift.py +++ /dev/null @@ -1,24 +0,0 @@ -import pandas as pd - -def compute_feature_lift(df: pd.DataFrame, labels: list[int], feature_columns: list[str]) -> dict[int, dict]: - temp = df[feature_columns].copy() - temp['_cluster'] = labels - out = {} - global_numeric = {c: float(temp[c].mean()) for c in feature_columns if pd.api.types.is_numeric_dtype(temp[c])} - for cluster_label, cluster_df in temp.groupby('_cluster'): - info = {'top_numeric_lifts': [], 'top_categorical_modes': []} - numeric_lifts = [] - for col in feature_columns: - if pd.api.types.is_numeric_dtype(temp[col]): - base = global_numeric.get(col, 0.0) - mean = float(cluster_df[col].mean()) - lift = round(mean - base, 4) - numeric_lifts.append({'feature': col, 'cluster_mean': round(mean, 4), 'global_mean': round(base, 4), 'lift': lift}) - else: - mode_series = cluster_df[col].astype(str).value_counts(normalize=True) - if len(mode_series): - info['top_categorical_modes'].append({'feature': col, 'value': mode_series.index[0], 'share': round(float(mode_series.iloc[0]), 4)}) - info['top_numeric_lifts'] = sorted(numeric_lifts, key=lambda x: abs(x['lift']), reverse=True)[:5] - info['top_categorical_modes'] = info['top_categorical_modes'][:5] - out[int(cluster_label)] = info - return out diff --git a/backend/app/ml/explain/summary_writer.py b/backend/app/ml/explain/summary_writer.py deleted file mode 100644 index 95729cdf08dc6caf7d91932f25b47cc180de2f21..0000000000000000000000000000000000000000 --- a/backend/app/ml/explain/summary_writer.py +++ /dev/null @@ -1,10 +0,0 @@ -def build_cluster_description(cluster_label: int, profile: dict, lift_info: dict) -> str: - numeric = lift_info.get('top_numeric_lifts', [])[:3] - categorical = lift_info.get('top_categorical_modes', [])[:2] - parts = [] - if numeric: - parts.append(', '.join([('high ' if item['lift'] >= 0 else 'low ') + item['feature'] for item in numeric])) - if categorical: - parts.append(', '.join([f"{item['feature']} is mostly {item['value']}" for item in categorical])) - core = '. '.join([p for p in parts if p]).strip() or 'This cluster is defined by a mixed feature pattern.' - return f"Cluster {cluster_label} has {profile.get('size', 0)} rows. {core}." diff --git a/backend/app/ml/metrics/__init__.py b/backend/app/ml/metrics/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/backend/app/ml/metrics/clustering_metrics.py b/backend/app/ml/metrics/clustering_metrics.py deleted file mode 100644 index c6a1fd46ec6fc4ed16aa4c2103ec8bd58acc3a59..0000000000000000000000000000000000000000 --- a/backend/app/ml/metrics/clustering_metrics.py +++ /dev/null @@ -1,33 +0,0 @@ -import numpy as np -from sklearn.metrics import calinski_harabasz_score, davies_bouldin_score, silhouette_score - -def compute_metrics(X, labels: list[int]) -> dict: - arr = np.array(labels) - unique = set(arr.tolist()) - non_noise = [x for x in unique if x != -1] - cluster_count = len(non_noise) - noise_ratio = float((arr == -1).mean()) if len(arr) else 0.0 - metrics = { - 'cluster_count': cluster_count, - 'noise_ratio': round(noise_ratio, 4), - 'silhouette_score': None, - 'davies_bouldin_score': None, - 'calinski_harabasz_score': None, - } - mask = arr != -1 if -1 in unique else np.ones(len(arr), dtype=bool) - X_eval = X[mask] - y_eval = arr[mask] - if len(set(y_eval.tolist())) >= 2 and len(X_eval) > len(set(y_eval.tolist())): - try: - metrics['silhouette_score'] = round(float(silhouette_score(X_eval, y_eval)), 4) - except Exception: - pass - try: - metrics['davies_bouldin_score'] = round(float(davies_bouldin_score(X_eval, y_eval)), 4) - except Exception: - pass - try: - metrics['calinski_harabasz_score'] = round(float(calinski_harabasz_score(X_eval, y_eval)), 4) - except Exception: - pass - return metrics diff --git a/backend/app/ml/metrics/interpretability.py b/backend/app/ml/metrics/interpretability.py deleted file mode 100644 index c011dd25f1dd625717e94c5b98035c5a21434da8..0000000000000000000000000000000000000000 --- a/backend/app/ml/metrics/interpretability.py +++ /dev/null @@ -1,19 +0,0 @@ -import numpy as np -import pandas as pd - -def compute_interpretability_score(df: pd.DataFrame, labels: list[int], feature_columns: list[str]) -> float: - if len(set(labels)) <= 1: - return 0.0 - scored = [] - temp = df[feature_columns].copy() - temp['_cluster'] = labels - for col in feature_columns: - if pd.api.types.is_numeric_dtype(temp[col]): - global_std = temp[col].std() or 1.0 - diffs = temp.groupby('_cluster')[col].mean().diff().abs().fillna(0) - lift = float(diffs.mean() / global_std) - else: - top_share = temp.groupby('_cluster')[col].agg(lambda s: s.astype(str).value_counts(normalize=True).iloc[0] if len(s) else 0).mean() - lift = float(top_share) - scored.append(min(lift, 1.0)) - return round(float(np.mean(scored)), 4) if scored else 0.0 diff --git a/backend/app/ml/metrics/stability.py b/backend/app/ml/metrics/stability.py deleted file mode 100644 index bafd00b4e18751924fe7ff7151f21f3cef2dab9d..0000000000000000000000000000000000000000 --- a/backend/app/ml/metrics/stability.py +++ /dev/null @@ -1,10 +0,0 @@ -import numpy as np -from sklearn.metrics import normalized_mutual_info_score - -def compute_stability_score(labels: list[int]) -> float: - arr = np.array(labels) - if len(set(arr.tolist())) <= 1: - return 0.0 - rotated = np.roll(arr, 1) - score = normalized_mutual_info_score(arr, rotated) - return round(float(score), 4) diff --git a/backend/app/ml/outliers/__init__.py b/backend/app/ml/outliers/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/backend/app/ml/outliers/centroid_distance.py b/backend/app/ml/outliers/centroid_distance.py deleted file mode 100644 index e61b71ada7dc3b5afdcc189b96c39637f0d860cb..0000000000000000000000000000000000000000 --- a/backend/app/ml/outliers/centroid_distance.py +++ /dev/null @@ -1,18 +0,0 @@ -import numpy as np - -def detect_centroid_outliers(X, labels: list[int], threshold_quantile: float = 0.95) -> list[bool]: - arr = np.array(labels) - outliers = np.zeros(len(arr), dtype=bool) - for cluster_label in sorted(set(arr.tolist())): - if cluster_label == -1: - outliers[arr == cluster_label] = True - continue - idx = np.where(arr == cluster_label)[0] - if len(idx) <= 2: - continue - cluster_points = X[idx] - centroid = cluster_points.mean(axis=0) - dists = np.linalg.norm(cluster_points - centroid, axis=1) - cut = np.quantile(dists, threshold_quantile) - outliers[idx] = dists >= cut - return outliers.tolist() diff --git a/backend/app/ml/outliers/density_noise.py b/backend/app/ml/outliers/density_noise.py deleted file mode 100644 index e018098f0d501671cab4c3a70857227d1f3c74fb..0000000000000000000000000000000000000000 --- a/backend/app/ml/outliers/density_noise.py +++ /dev/null @@ -1,2 +0,0 @@ -def detect_noise_outliers(labels: list[int]) -> list[bool]: - return [label == -1 for label in labels] diff --git a/backend/app/ml/outliers/isolation_forest.py b/backend/app/ml/outliers/isolation_forest.py deleted file mode 100644 index 82909e682f4cacffe3b1f2ac648800114697eb51..0000000000000000000000000000000000000000 --- a/backend/app/ml/outliers/isolation_forest.py +++ /dev/null @@ -1,6 +0,0 @@ -from sklearn.ensemble import IsolationForest - -def detect_isolation_outliers(X) -> list[bool]: - model = IsolationForest(random_state=42, contamination='auto') - flags = model.fit_predict(X) - return [item == -1 for item in flags.tolist()] diff --git a/backend/app/ml/preprocess/__init__.py b/backend/app/ml/preprocess/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/backend/app/ml/preprocess/column_inference.py b/backend/app/ml/preprocess/column_inference.py deleted file mode 100644 index 3a00437aeb00b32ec3bdd44604d94f2f62e4b898..0000000000000000000000000000000000000000 --- a/backend/app/ml/preprocess/column_inference.py +++ /dev/null @@ -1,33 +0,0 @@ -from dataclasses import dataclass -import pandas as pd - -@dataclass -class InferredColumn: - name: str - inferred_type: str - missing_pct: float - cardinality: int - sample_values: list - -def infer_columns(df: pd.DataFrame) -> list[InferredColumn]: - cols = [] - for col in df.columns: - series = df[col] - missing_pct = float(series.isna().mean() * 100) - cardinality = int(series.nunique(dropna=True)) - sample_values = series.dropna().astype(str).head(5).tolist() - if pd.api.types.is_bool_dtype(series): - inferred = 'boolean' - elif pd.api.types.is_numeric_dtype(series): - inferred = 'numeric' - else: - unique_ratio = cardinality / max(len(series), 1) - inferred = 'high_cardinality' if cardinality > 50 and unique_ratio > 0.3 else 'categorical' - cols.append(InferredColumn( - name=col, - inferred_type=inferred, - missing_pct=round(missing_pct, 2), - cardinality=cardinality, - sample_values=sample_values, - )) - return cols diff --git a/backend/app/ml/preprocess/encoders.py b/backend/app/ml/preprocess/encoders.py deleted file mode 100644 index a06fab6892e453f6a8d6995f19cbf39b5a9e0395..0000000000000000000000000000000000000000 --- a/backend/app/ml/preprocess/encoders.py +++ /dev/null @@ -1,7 +0,0 @@ -from sklearn.preprocessing import OneHotEncoder - -def get_onehot_encoder(): - try: - return OneHotEncoder(handle_unknown='ignore', sparse_output=False) - except TypeError: - return OneHotEncoder(handle_unknown='ignore', sparse=False) diff --git a/backend/app/ml/preprocess/imputers.py b/backend/app/ml/preprocess/imputers.py deleted file mode 100644 index ea787e7e1ccfa46ed7a919e2e67873017f26b5e6..0000000000000000000000000000000000000000 --- a/backend/app/ml/preprocess/imputers.py +++ /dev/null @@ -1,7 +0,0 @@ -from sklearn.impute import SimpleImputer - -def numeric_imputer(): - return SimpleImputer(strategy='median') - -def categorical_imputer(): - return SimpleImputer(strategy='most_frequent') diff --git a/backend/app/ml/preprocess/matrix_builder.py b/backend/app/ml/preprocess/matrix_builder.py deleted file mode 100644 index eef599d94946addccc2a0977a30d8a260d2c045e..0000000000000000000000000000000000000000 --- a/backend/app/ml/preprocess/matrix_builder.py +++ /dev/null @@ -1,32 +0,0 @@ -import pandas as pd -from sklearn.compose import ColumnTransformer -from sklearn.pipeline import Pipeline -from app.ml.preprocess.encoders import get_onehot_encoder -from app.ml.preprocess.imputers import categorical_imputer, numeric_imputer -from app.ml.preprocess.scalers import get_scaler - -def build_feature_matrix(df: pd.DataFrame, feature_columns: list[str], numeric_scaler: str = 'standard', categorical_encoder: str = 'onehot'): - selected = df[feature_columns].copy() - numeric_cols = [c for c in selected.columns if pd.api.types.is_numeric_dtype(selected[c]) or pd.api.types.is_bool_dtype(selected[c])] - categorical_cols = [c for c in selected.columns if c not in numeric_cols] - - numeric_pipeline = Pipeline(steps=[('imputer', numeric_imputer()), ('scaler', get_scaler(numeric_scaler))]) - categorical_pipeline = Pipeline(steps=[('imputer', categorical_imputer()), ('encoder', get_onehot_encoder())]) - - transformer = ColumnTransformer( - transformers=[('num', numeric_pipeline, numeric_cols), ('cat', categorical_pipeline, categorical_cols)], - remainder='drop', - ) - X = transformer.fit_transform(selected) - - feature_names = [] - feature_names.extend(numeric_cols) - if categorical_cols: - enc = transformer.named_transformers_['cat'].named_steps['encoder'] - try: - cat_names = enc.get_feature_names_out(categorical_cols).tolist() - except Exception: - cat_names = categorical_cols - feature_names.extend(cat_names) - - return X, feature_names, {'numeric_cols': numeric_cols, 'categorical_cols': categorical_cols} diff --git a/backend/app/ml/preprocess/reducers.py b/backend/app/ml/preprocess/reducers.py deleted file mode 100644 index e1d95b6e83203bdec7a1c205e7436a577007f5a0..0000000000000000000000000000000000000000 --- a/backend/app/ml/preprocess/reducers.py +++ /dev/null @@ -1,16 +0,0 @@ -from sklearn.decomposition import PCA - -def pca_2d(X): - reducer = PCA(n_components=2, random_state=42) - coords = reducer.fit_transform(X) - return coords[:, 0], coords[:, 1] - -def umap_2d(X): - try: - import umap - reducer = umap.UMAP(n_components=2, random_state=42) - coords = reducer.fit_transform(X) - except Exception: - reducer = PCA(n_components=2, random_state=42) - coords = reducer.fit_transform(X) - return coords[:, 0], coords[:, 1] diff --git a/backend/app/ml/preprocess/scalers.py b/backend/app/ml/preprocess/scalers.py deleted file mode 100644 index 20b1c371d9ce3174665d8360df27a64c0dec707a..0000000000000000000000000000000000000000 --- a/backend/app/ml/preprocess/scalers.py +++ /dev/null @@ -1,9 +0,0 @@ -from sklearn.preprocessing import MinMaxScaler, RobustScaler, StandardScaler - -def get_scaler(name: str): - name = (name or 'standard').lower() - if name == 'minmax': - return MinMaxScaler() - if name == 'robust': - return RobustScaler() - return StandardScaler() diff --git a/backend/app/models/__init__.py b/backend/app/models/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/backend/app/models/artifact.py b/backend/app/models/artifact.py deleted file mode 100644 index 22708953fd729da8bfe79ea466d243ee40ec79f2..0000000000000000000000000000000000000000 --- a/backend/app/models/artifact.py +++ /dev/null @@ -1,12 +0,0 @@ -from datetime import datetime -from sqlalchemy import DateTime, ForeignKey, Integer, String, Text -from sqlalchemy.orm import Mapped, mapped_column -from app.db import Base - -class Artifact(Base): - __tablename__ = 'artifacts' - id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) - experiment_id: Mapped[str] = mapped_column(ForeignKey('experiments.id'), index=True, nullable=False) - artifact_type: Mapped[str] = mapped_column(String(64), nullable=False) - path: Mapped[str] = mapped_column(Text, nullable=False) - created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow, nullable=False) diff --git a/backend/app/models/cluster_result.py b/backend/app/models/cluster_result.py deleted file mode 100644 index 51a7e7b1e5f33209c1b1740bdd927707d9b6aed8..0000000000000000000000000000000000000000 --- a/backend/app/models/cluster_result.py +++ /dev/null @@ -1,36 +0,0 @@ -from sqlalchemy import Boolean, Float, ForeignKey, Integer, String, Text -from sqlalchemy.orm import Mapped, mapped_column -from app.db import Base - -class ExperimentMetric(Base): - __tablename__ = 'experiment_metrics' - id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) - experiment_id: Mapped[str] = mapped_column(ForeignKey('experiments.id'), index=True, nullable=False) - cluster_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False) - noise_ratio: Mapped[float] = mapped_column(Float, default=0.0, nullable=False) - silhouette_score: Mapped[float | None] = mapped_column(Float, nullable=True) - davies_bouldin_score: Mapped[float | None] = mapped_column(Float, nullable=True) - calinski_harabasz_score: Mapped[float | None] = mapped_column(Float, nullable=True) - stability_score: Mapped[float | None] = mapped_column(Float, nullable=True) - interpretability_score: Mapped[float | None] = mapped_column(Float, nullable=True) - -class ClusterResult(Base): - __tablename__ = 'cluster_results' - id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) - experiment_id: Mapped[str] = mapped_column(ForeignKey('experiments.id'), index=True, nullable=False) - row_index: Mapped[int] = mapped_column(Integer, nullable=False) - cluster_label: Mapped[int] = mapped_column(Integer, nullable=False, index=True) - is_outlier: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False) - pca_x: Mapped[float | None] = mapped_column(Float, nullable=True) - pca_y: Mapped[float | None] = mapped_column(Float, nullable=True) - umap_x: Mapped[float | None] = mapped_column(Float, nullable=True) - umap_y: Mapped[float | None] = mapped_column(Float, nullable=True) - -class ClusterSummary(Base): - __tablename__ = 'cluster_summaries' - id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) - experiment_id: Mapped[str] = mapped_column(ForeignKey('experiments.id'), index=True, nullable=False) - cluster_label: Mapped[int] = mapped_column(Integer, nullable=False, index=True) - size: Mapped[int] = mapped_column(Integer, nullable=False) - summary_json: Mapped[str] = mapped_column(Text, default='{}', nullable=False) - description_text: Mapped[str] = mapped_column(Text, nullable=False) diff --git a/backend/app/models/dataset.py b/backend/app/models/dataset.py deleted file mode 100644 index d4dd96387cf328e1370ce3e375f9a60c85ee035b..0000000000000000000000000000000000000000 --- a/backend/app/models/dataset.py +++ /dev/null @@ -1,16 +0,0 @@ -from datetime import datetime -from sqlalchemy import DateTime, Integer, String, Text -from sqlalchemy.orm import Mapped, mapped_column -from app.db import Base - -class Dataset(Base): - __tablename__ = 'datasets' - id: Mapped[str] = mapped_column(String(32), primary_key=True) - name: Mapped[str] = mapped_column(String(255), nullable=False) - file_path: Mapped[str] = mapped_column(Text, nullable=False) - sha256: Mapped[str] = mapped_column(String(64), nullable=False, index=True) - uploaded_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow, nullable=False) - row_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False) - column_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False) - schema_json: Mapped[str] = mapped_column(Text, default='{}', nullable=False) - profile_json: Mapped[str] = mapped_column(Text, default='{}', nullable=False) diff --git a/backend/app/models/experiment.py b/backend/app/models/experiment.py deleted file mode 100644 index 2f305fb8c4a80a5c1dfea9344915b4ebd804d960..0000000000000000000000000000000000000000 --- a/backend/app/models/experiment.py +++ /dev/null @@ -1,17 +0,0 @@ -from datetime import datetime -from sqlalchemy import DateTime, ForeignKey, Integer, String, Text -from sqlalchemy.orm import Mapped, mapped_column -from app.db import Base - -class Experiment(Base): - __tablename__ = 'experiments' - id: Mapped[str] = mapped_column(String(32), primary_key=True) - dataset_id: Mapped[str] = mapped_column(ForeignKey('datasets.id'), index=True, nullable=False) - name: Mapped[str] = mapped_column(String(255), nullable=False) - algorithm: Mapped[str] = mapped_column(String(64), nullable=False, index=True) - status: Mapped[str] = mapped_column(String(32), nullable=False, default='pending') - config_json: Mapped[str] = mapped_column(Text, default='{}', nullable=False) - started_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow, nullable=False) - completed_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True) - runtime_ms: Mapped[int | None] = mapped_column(Integer, nullable=True) - error_message: Mapped[str | None] = mapped_column(Text, nullable=True) diff --git a/backend/app/repositories/__init__.py b/backend/app/repositories/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/backend/app/repositories/artifact_repo.py b/backend/app/repositories/artifact_repo.py deleted file mode 100644 index 906565a98d1f842ded51cc937c438d8b82f16875..0000000000000000000000000000000000000000 --- a/backend/app/repositories/artifact_repo.py +++ /dev/null @@ -1,16 +0,0 @@ -from sqlalchemy import select -from sqlalchemy.orm import Session -from app.models.artifact import Artifact - -class ArtifactRepository: - def __init__(self, session: Session): - self.session = session - - def create(self, artifact: Artifact) -> Artifact: - self.session.add(artifact) - self.session.flush() - return artifact - - def list_for_experiment(self, experiment_id: str) -> list[Artifact]: - stmt = select(Artifact).where(Artifact.experiment_id == experiment_id).order_by(Artifact.created_at.desc()) - return list(self.session.scalars(stmt)) diff --git a/backend/app/repositories/cluster_repo.py b/backend/app/repositories/cluster_repo.py deleted file mode 100644 index 0de41e3b9cc45749225321dab65fc6c764cb0576..0000000000000000000000000000000000000000 --- a/backend/app/repositories/cluster_repo.py +++ /dev/null @@ -1,41 +0,0 @@ -from sqlalchemy import delete, select -from sqlalchemy.orm import Session -from app.models.cluster_result import ClusterResult, ClusterSummary, ExperimentMetric - -class ClusterRepository: - def __init__(self, session: Session): - self.session = session - - def replace_results(self, experiment_id: str, results: list[ClusterResult]) -> None: - self.session.execute(delete(ClusterResult).where(ClusterResult.experiment_id == experiment_id)) - self.session.add_all(results) - self.session.flush() - - def replace_summaries(self, experiment_id: str, summaries: list[ClusterSummary]) -> None: - self.session.execute(delete(ClusterSummary).where(ClusterSummary.experiment_id == experiment_id)) - self.session.add_all(summaries) - self.session.flush() - - def replace_metric(self, experiment_id: str, metric: ExperimentMetric) -> None: - self.session.execute(delete(ExperimentMetric).where(ExperimentMetric.experiment_id == experiment_id)) - self.session.add(metric) - self.session.flush() - - def get_results(self, experiment_id: str) -> list[ClusterResult]: - stmt = select(ClusterResult).where(ClusterResult.experiment_id == experiment_id).order_by(ClusterResult.row_index.asc()) - return list(self.session.scalars(stmt)) - - def get_metric(self, experiment_id: str) -> ExperimentMetric | None: - stmt = select(ExperimentMetric).where(ExperimentMetric.experiment_id == experiment_id) - return self.session.scalar(stmt) - - def get_summaries(self, experiment_id: str) -> list[ClusterSummary]: - stmt = select(ClusterSummary).where(ClusterSummary.experiment_id == experiment_id).order_by(ClusterSummary.cluster_label.asc()) - return list(self.session.scalars(stmt)) - - def get_summary(self, experiment_id: str, cluster_label: int) -> ClusterSummary | None: - stmt = select(ClusterSummary).where( - ClusterSummary.experiment_id == experiment_id, - ClusterSummary.cluster_label == cluster_label, - ) - return self.session.scalar(stmt) diff --git a/backend/app/repositories/dataset_repo.py b/backend/app/repositories/dataset_repo.py deleted file mode 100644 index e811d1f3bd34d4d3c0cbc0d30975a243984047d4..0000000000000000000000000000000000000000 --- a/backend/app/repositories/dataset_repo.py +++ /dev/null @@ -1,27 +0,0 @@ -import json -from sqlalchemy import select -from sqlalchemy.orm import Session -from app.models.dataset import Dataset - -class DatasetRepository: - def __init__(self, session: Session): - self.session = session - - def create(self, dataset: Dataset) -> Dataset: - self.session.add(dataset) - self.session.flush() - return dataset - - def get(self, dataset_id: str) -> Dataset | None: - return self.session.get(Dataset, dataset_id) - - def list_all(self) -> list[Dataset]: - return list(self.session.scalars(select(Dataset).order_by(Dataset.uploaded_at.desc()))) - - def update_profile(self, dataset_id: str, schema_json: dict, profile_json: dict) -> None: - dataset = self.get(dataset_id) - if not dataset: - raise ValueError('Dataset not found') - dataset.schema_json = json.dumps(schema_json) - dataset.profile_json = json.dumps(profile_json) - self.session.flush() diff --git a/backend/app/repositories/experiment_repo.py b/backend/app/repositories/experiment_repo.py deleted file mode 100644 index 31b97996aa656ab348ed05e1f7b5efe69139cba7..0000000000000000000000000000000000000000 --- a/backend/app/repositories/experiment_repo.py +++ /dev/null @@ -1,28 +0,0 @@ -from sqlalchemy import select -from sqlalchemy.orm import Session -from app.models.experiment import Experiment - -class ExperimentRepository: - def __init__(self, session: Session): - self.session = session - - def create(self, experiment: Experiment) -> Experiment: - self.session.add(experiment) - self.session.flush() - return experiment - - def get(self, experiment_id: str) -> Experiment | None: - return self.session.get(Experiment, experiment_id) - - def list_all(self) -> list[Experiment]: - return list(self.session.scalars(select(Experiment).order_by(Experiment.started_at.desc()))) - - def update_status(self, experiment_id: str, status: str, runtime_ms: int | None = None, error_message: str | None = None, completed_at=None) -> None: - exp = self.get(experiment_id) - if not exp: - raise ValueError('Experiment not found') - exp.status = status - exp.runtime_ms = runtime_ms - exp.error_message = error_message - exp.completed_at = completed_at - self.session.flush() diff --git a/backend/app/schemas/__init__.py b/backend/app/schemas/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/backend/app/schemas/cluster.py b/backend/app/schemas/cluster.py deleted file mode 100644 index ec10002eef02aa024035de5d7dd44abf40ca0edc..0000000000000000000000000000000000000000 --- a/backend/app/schemas/cluster.py +++ /dev/null @@ -1,13 +0,0 @@ -from typing import Any, Dict, List -from pydantic import BaseModel - -class ClusterMember(BaseModel): - row_index: int - values: Dict[str, Any] - -class ClusterDetailResponse(BaseModel): - experiment_id: str - cluster_label: int - members: List[ClusterMember] - summary: Dict[str, Any] - description_text: str diff --git a/backend/app/schemas/compare.py b/backend/app/schemas/compare.py deleted file mode 100644 index d80a1975a27ea5f1eb3112c8d8cc5a539b0f1146..0000000000000000000000000000000000000000 --- a/backend/app/schemas/compare.py +++ /dev/null @@ -1,17 +0,0 @@ -from typing import List -from pydantic import BaseModel - -class CompareRunItem(BaseModel): - experiment_id: str - name: str - algorithm: str - cluster_count: int - noise_ratio: float - silhouette_score: float | None = None - davies_bouldin_score: float | None = None - calinski_harabasz_score: float | None = None - stability_score: float | None = None - interpretability_score: float | None = None - -class CompareResponse(BaseModel): - runs: List[CompareRunItem] diff --git a/backend/app/schemas/dataset.py b/backend/app/schemas/dataset.py deleted file mode 100644 index 33edce1cc0926500e93477f9fa9ee7230e99df21..0000000000000000000000000000000000000000 --- a/backend/app/schemas/dataset.py +++ /dev/null @@ -1,23 +0,0 @@ -from typing import Any, List -from pydantic import BaseModel - -class DatasetUploadResponse(BaseModel): - dataset_id: str - name: str - row_count: int - column_count: int - -class ColumnProfile(BaseModel): - name: str - inferred_type: str - missing_pct: float - cardinality: int - sample_values: List[Any] - -class DatasetProfileResponse(BaseModel): - dataset_id: str - columns: List[ColumnProfile] - suggested_feature_columns: List[str] - recommended_algorithms: List[str] - row_count: int - column_count: int diff --git a/backend/app/schemas/experiment.py b/backend/app/schemas/experiment.py deleted file mode 100644 index 9b2f30753011f98c1907bac54db9e8c2b9783c85..0000000000000000000000000000000000000000 --- a/backend/app/schemas/experiment.py +++ /dev/null @@ -1,59 +0,0 @@ -from typing import Any, Dict, List, Optional -from pydantic import BaseModel, Field - -class PreprocessingConfig(BaseModel): - numeric_scaler: str = 'standard' - categorical_encoder: str = 'onehot' - dimensionality_reduction: str = 'pca' - -class ExperimentRunRequest(BaseModel): - dataset_id: str - name: str = Field(default='Untitled Experiment') - algorithm: str - feature_columns: List[str] - preprocessing: PreprocessingConfig = PreprocessingConfig() - algorithm_params: Dict[str, Any] = Field(default_factory=dict) - outlier_detection_enabled: bool = True - -class ExperimentRunResponse(BaseModel): - experiment_id: str - status: str - -class MetricResponse(BaseModel): - cluster_count: int - noise_ratio: float - silhouette_score: Optional[float] = None - davies_bouldin_score: Optional[float] = None - calinski_harabasz_score: Optional[float] = None - stability_score: Optional[float] = None - interpretability_score: Optional[float] = None - -class ScatterPoint(BaseModel): - row_index: int - cluster_label: int - is_outlier: bool - x: float - y: float - -class ClusterSummaryResponse(BaseModel): - cluster_label: int - size: int - description_text: str - summary_json: Dict[str, Any] - -class ExperimentResultResponse(BaseModel): - experiment_id: str - algorithm: str - metrics: MetricResponse - scatter_points: List[ScatterPoint] - cluster_summaries: List[ClusterSummaryResponse] - status: str - -class ExperimentListItem(BaseModel): - experiment_id: str - dataset_id: str - name: str - algorithm: str - status: str - started_at: str - completed_at: Optional[str] = None diff --git a/backend/app/schemas/export.py b/backend/app/schemas/export.py deleted file mode 100644 index b042d0d59dac19b273bdc9412a8ac41dc03187d1..0000000000000000000000000000000000000000 --- a/backend/app/schemas/export.py +++ /dev/null @@ -1,5 +0,0 @@ -from pydantic import BaseModel - -class ExportResponse(BaseModel): - experiment_id: str - download_path: str diff --git a/backend/app/schemas/health.py b/backend/app/schemas/health.py deleted file mode 100644 index 9e5f219724512af25e26bc882116cb4c011c4545..0000000000000000000000000000000000000000 --- a/backend/app/schemas/health.py +++ /dev/null @@ -1,5 +0,0 @@ -from pydantic import BaseModel - -class HealthResponse(BaseModel): - ok: bool - service: str diff --git a/backend/app/services/__init__.py b/backend/app/services/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/backend/app/services/clustering_service.py b/backend/app/services/clustering_service.py deleted file mode 100644 index 18fabeb6c60ad4b9a58ea8f824e0cfb7da5df719..0000000000000000000000000000000000000000 --- a/backend/app/services/clustering_service.py +++ /dev/null @@ -1,24 +0,0 @@ -from importlib import import_module -from app.utils.validators import SUPPORTED_ALGORITHMS - -class ClusteringService: - def get_runner(self, algorithm: str): - if algorithm not in SUPPORTED_ALGORITHMS: - raise ValueError(f'Unsupported algorithm: {algorithm}') - module_name = { - 'kmeans': 'kmeans_runner', - 'minibatch_kmeans': 'minibatch_kmeans_runner', - 'agglomerative': 'agglomerative_runner', - 'dbscan': 'dbscan_runner', - 'hdbscan': 'hdbscan_runner', - 'gmm': 'gmm_runner', - 'spectral': 'spectral_runner', - 'birch': 'birch_runner', - }[algorithm] - module = import_module(f'app.ml.algorithms.{module_name}') - return module.Runner() - - def run(self, X, algorithm: str, algorithm_params: dict): - runner = self.get_runner(algorithm) - result = runner.fit_predict(X, algorithm_params) - return runner, result diff --git a/backend/app/services/comparison_service.py b/backend/app/services/comparison_service.py deleted file mode 100644 index fbf3271c1fd617952482d819d61985a38725f3cd..0000000000000000000000000000000000000000 --- a/backend/app/services/comparison_service.py +++ /dev/null @@ -1,21 +0,0 @@ -class ComparisonService: - def build_compare_payload(self, experiment_repo, cluster_repo, experiment_ids: list[str]): - runs = [] - for experiment_id in experiment_ids: - exp = experiment_repo.get(experiment_id) - metric = cluster_repo.get_metric(experiment_id) - if not exp or not metric: - continue - runs.append({ - 'experiment_id': exp.id, - 'name': exp.name, - 'algorithm': exp.algorithm, - 'cluster_count': metric.cluster_count, - 'noise_ratio': metric.noise_ratio, - 'silhouette_score': metric.silhouette_score, - 'davies_bouldin_score': metric.davies_bouldin_score, - 'calinski_harabasz_score': metric.calinski_harabasz_score, - 'stability_score': metric.stability_score, - 'interpretability_score': metric.interpretability_score, - }) - return {'runs': runs} diff --git a/backend/app/services/dataset_service.py b/backend/app/services/dataset_service.py deleted file mode 100644 index 08be86598289782f37f3f821a33640084026548d..0000000000000000000000000000000000000000 --- a/backend/app/services/dataset_service.py +++ /dev/null @@ -1,43 +0,0 @@ -import io -from pathlib import Path -import pandas as pd -from fastapi import UploadFile -from app.models.dataset import Dataset -from app.repositories.dataset_repo import DatasetRepository -from app.services.storage_service import StorageService -from app.utils.file_hash import sha256_file -from app.utils.ids import generate_id - -class DatasetService: - def __init__(self, repo: DatasetRepository): - self.repo = repo - self.storage = StorageService() - - def load_dataframe(self, dataset: Dataset) -> pd.DataFrame: - path = Path(dataset.file_path) - if path.suffix.lower() in {'.xlsx', '.xls'}: - return pd.read_excel(path) - return pd.read_csv(path) - - async def save_upload(self, upload: UploadFile) -> Dataset: - dataset_id = generate_id('ds') - suffix = Path(upload.filename or 'dataset.csv').suffix or '.csv' - file_path = self.storage.uploads_dir() / f"{dataset_id}{suffix}" - content = await upload.read() - with open(file_path, 'wb') as f: - f.write(content) - if suffix.lower() in {'.xlsx', '.xls'}: - df = pd.read_excel(io.BytesIO(content)) - else: - df = pd.read_csv(io.BytesIO(content)) - dataset = Dataset( - id=dataset_id, - name=upload.filename or f'{dataset_id}.csv', - file_path=str(file_path), - sha256=sha256_file(file_path), - row_count=int(len(df)), - column_count=int(len(df.columns)), - schema_json='{}', - profile_json='{}', - ) - return self.repo.create(dataset) diff --git a/backend/app/services/evaluation_service.py b/backend/app/services/evaluation_service.py deleted file mode 100644 index c59013dfaed535fd2243056a9331a498f5872084..0000000000000000000000000000000000000000 --- a/backend/app/services/evaluation_service.py +++ /dev/null @@ -1,10 +0,0 @@ -from app.ml.metrics.clustering_metrics import compute_metrics -from app.ml.metrics.interpretability import compute_interpretability_score -from app.ml.metrics.stability import compute_stability_score - -class EvaluationService: - def evaluate(self, X, df, labels: list[int], feature_columns: list[str]) -> dict: - metrics = compute_metrics(X, labels) - metrics['stability_score'] = compute_stability_score(labels) - metrics['interpretability_score'] = compute_interpretability_score(df, labels, feature_columns) - return metrics diff --git a/backend/app/services/explainability_service.py b/backend/app/services/explainability_service.py deleted file mode 100644 index 739bec1e922f6b249feb710f850863647f64bd8d..0000000000000000000000000000000000000000 --- a/backend/app/services/explainability_service.py +++ /dev/null @@ -1,23 +0,0 @@ -import json -from app.ml.explain.cluster_profiles import build_cluster_profiles -from app.ml.explain.feature_lift import compute_feature_lift -from app.ml.explain.summary_writer import build_cluster_description -from app.models.cluster_result import ClusterSummary - -class ExplainabilityService: - def build_summaries(self, experiment_id: str, df, labels: list[int], feature_columns: list[str]) -> list[ClusterSummary]: - profiles = build_cluster_profiles(df, labels, feature_columns) - lifts = compute_feature_lift(df, labels, feature_columns) - summaries = [] - for cluster_label, profile in sorted(profiles.items(), key=lambda x: x[0]): - lift_info = lifts.get(cluster_label, {}) - summary_json = {'profile': profile, 'feature_lift': lift_info} - description = build_cluster_description(cluster_label, profile, lift_info) - summaries.append(ClusterSummary( - experiment_id=experiment_id, - cluster_label=int(cluster_label), - size=int(profile['size']), - summary_json=json.dumps(summary_json), - description_text=description, - )) - return summaries diff --git a/backend/app/services/export_service.py b/backend/app/services/export_service.py deleted file mode 100644 index 162a83c9a1f61defff294a87950e3a9fdccf9f70..0000000000000000000000000000000000000000 --- a/backend/app/services/export_service.py +++ /dev/null @@ -1,32 +0,0 @@ -from pathlib import Path -import pandas as pd -from app.models.artifact import Artifact -from app.services.storage_service import StorageService - -class ExportService: - def __init__(self, dataset_repo, cluster_repo, artifact_repo): - self.dataset_repo = dataset_repo - self.cluster_repo = cluster_repo - self.artifact_repo = artifact_repo - self.storage = StorageService() - - def export_csv(self, experiment_id: str, dataset_id: str) -> str: - dataset = self.dataset_repo.get(dataset_id) - if not dataset: - raise ValueError('Dataset not found') - results = self.cluster_repo.get_results(experiment_id) - df = pd.read_csv(dataset.file_path) if dataset.file_path.lower().endswith('.csv') else pd.read_excel(dataset.file_path) - enrich = pd.DataFrame({ - 'row_index': [r.row_index for r in results], - 'cluster_label': [r.cluster_label for r in results], - 'is_outlier': [r.is_outlier for r in results], - 'pca_x': [r.pca_x for r in results], - 'pca_y': [r.pca_y for r in results], - 'umap_x': [r.umap_x for r in results], - 'umap_y': [r.umap_y for r in results], - }) - merged = df.reset_index().rename(columns={'index': 'row_index'}).merge(enrich, on='row_index', how='left') - out_path = Path(self.storage.artifacts_dir()) / f'{experiment_id}_labeled.csv' - merged.to_csv(out_path, index=False) - self.artifact_repo.create(Artifact(experiment_id=experiment_id, artifact_type='labeled_csv', path=str(out_path))) - return str(out_path) diff --git a/backend/app/services/outlier_service.py b/backend/app/services/outlier_service.py deleted file mode 100644 index ab34a42bc749c98f499d4496ccb4ef391dbb10da..0000000000000000000000000000000000000000 --- a/backend/app/services/outlier_service.py +++ /dev/null @@ -1,11 +0,0 @@ -from app.ml.outliers.centroid_distance import detect_centroid_outliers -from app.ml.outliers.density_noise import detect_noise_outliers -from app.ml.outliers.isolation_forest import detect_isolation_outliers - -class OutlierService: - def detect(self, X, labels: list[int], algorithm: str) -> list[bool]: - if algorithm in {'dbscan', 'hdbscan'}: - return detect_noise_outliers(labels) - centroid_flags = detect_centroid_outliers(X, labels) - isolation_flags = detect_isolation_outliers(X) - return [bool(a or b) for a, b in zip(centroid_flags, isolation_flags)] diff --git a/backend/app/services/preprocessing_service.py b/backend/app/services/preprocessing_service.py deleted file mode 100644 index 88cd525b2a640cb07b42b92fa01ef801ce52d0c3..0000000000000000000000000000000000000000 --- a/backend/app/services/preprocessing_service.py +++ /dev/null @@ -1,10 +0,0 @@ -from app.ml.preprocess.matrix_builder import build_feature_matrix - -class PreprocessingService: - def transform(self, df, feature_columns: list[str], preprocessing: dict): - return build_feature_matrix( - df=df, - feature_columns=feature_columns, - numeric_scaler=preprocessing.get('numeric_scaler', 'standard'), - categorical_encoder=preprocessing.get('categorical_encoder', 'onehot'), - ) diff --git a/backend/app/services/profiling_service.py b/backend/app/services/profiling_service.py deleted file mode 100644 index f2ed1c4cc28a31ecd214e396c2de40a6bd5ed256..0000000000000000000000000000000000000000 --- a/backend/app/services/profiling_service.py +++ /dev/null @@ -1,35 +0,0 @@ -from app.ml.preprocess.column_inference import infer_columns - -class ProfilingService: - def profile(self, df): - inferred = infer_columns(df) - columns = [{ - 'name': c.name, - 'inferred_type': c.inferred_type, - 'missing_pct': c.missing_pct, - 'cardinality': c.cardinality, - 'sample_values': c.sample_values, - } for c in inferred] - feature_columns = [c['name'] for c in columns if c['inferred_type'] in {'numeric', 'boolean', 'categorical'}] - recommended = self.recommend_algorithms(df, columns) - return { - 'columns': columns, - 'suggested_feature_columns': feature_columns[:20], - 'recommended_algorithms': recommended, - 'row_count': int(len(df)), - 'column_count': int(len(df.columns)), - } - - def recommend_algorithms(self, df, columns): - row_count = len(df) - numeric_count = sum(1 for c in columns if c['inferred_type'] in {'numeric', 'boolean'}) - cat_count = sum(1 for c in columns if c['inferred_type'] == 'categorical') - recs = [] - if row_count > 5000: - recs.extend(['minibatch_kmeans', 'birch']) - else: - recs.append('kmeans') - if cat_count > numeric_count: - recs.append('agglomerative') - recs.extend(['dbscan', 'gmm', 'spectral']) - return list(dict.fromkeys(recs))[:5] diff --git a/backend/app/services/run_service.py b/backend/app/services/run_service.py deleted file mode 100644 index eb0627590fc1ded875d786edc520887073a1fe6c..0000000000000000000000000000000000000000 --- a/backend/app/services/run_service.py +++ /dev/null @@ -1,92 +0,0 @@ -import json -import time -import pandas as pd -from app.models.cluster_result import ClusterResult, ExperimentMetric -from app.models.experiment import Experiment -from app.services.preprocessing_service import PreprocessingService -from app.services.clustering_service import ClusteringService -from app.services.evaluation_service import EvaluationService -from app.services.visualization_service import VisualizationService -from app.services.explainability_service import ExplainabilityService -from app.services.outlier_service import OutlierService -from app.utils.ids import generate_id -from app.utils.time_utils import utc_now - -class RunService: - def __init__(self, dataset_repo, experiment_repo, cluster_repo): - self.dataset_repo = dataset_repo - self.experiment_repo = experiment_repo - self.cluster_repo = cluster_repo - self.preprocess = PreprocessingService() - self.clusterer = ClusteringService() - self.evaluate_service = EvaluationService() - self.visualize = VisualizationService() - self.explain = ExplainabilityService() - self.outliers = OutlierService() - - def run(self, payload: dict): - dataset = self.dataset_repo.get(payload['dataset_id']) - if not dataset: - raise ValueError('Dataset not found') - experiment = Experiment( - id=generate_id('exp'), - dataset_id=dataset.id, - name=payload['name'], - algorithm=payload['algorithm'], - status='running', - config_json=json.dumps(payload), - started_at=utc_now(), - ) - self.experiment_repo.create(experiment) - start = time.perf_counter() - try: - df = pd.read_csv(dataset.file_path) if dataset.file_path.lower().endswith('.csv') else pd.read_excel(dataset.file_path) - X, feature_names, metadata = self.preprocess.transform( - df=df, - feature_columns=payload['feature_columns'], - preprocessing=payload['preprocessing'], - ) - runner, run_output = self.clusterer.run( - X=X, - algorithm=payload['algorithm'], - algorithm_params=payload['algorithm_params'], - ) - labels = run_output.labels - coords = self.visualize.build_coords(X) - outlier_flags = self.outliers.detect(X, labels, payload['algorithm']) if payload.get('outlier_detection_enabled', True) else [False] * len(labels) - metrics = self.evaluate_service.evaluate(X, df, labels, payload['feature_columns']) - - rows = [] - for i, label in enumerate(labels): - rows.append(ClusterResult( - experiment_id=experiment.id, - row_index=i, - cluster_label=int(label), - is_outlier=bool(outlier_flags[i]), - pca_x=float(coords['pca_x'][i]), - pca_y=float(coords['pca_y'][i]), - umap_x=float(coords['umap_x'][i]), - umap_y=float(coords['umap_y'][i]), - )) - self.cluster_repo.replace_results(experiment.id, rows) - metric_row = ExperimentMetric( - experiment_id=experiment.id, - cluster_count=int(metrics['cluster_count']), - noise_ratio=float(metrics['noise_ratio']), - silhouette_score=metrics['silhouette_score'], - davies_bouldin_score=metrics['davies_bouldin_score'], - calinski_harabasz_score=metrics['calinski_harabasz_score'], - stability_score=metrics['stability_score'], - interpretability_score=metrics['interpretability_score'], - ) - self.cluster_repo.replace_metric(experiment.id, metric_row) - summaries = self.explain.build_summaries(experiment.id, df, labels, payload['feature_columns']) - self.cluster_repo.replace_summaries(experiment.id, summaries) - - runtime_ms = int((time.perf_counter() - start) * 1000) - self.experiment_repo.update_status(experiment.id, 'completed', runtime_ms=runtime_ms, completed_at=utc_now()) - except Exception as exc: - runtime_ms = int((time.perf_counter() - start) * 1000) - self.experiment_repo.update_status(experiment.id, 'failed', runtime_ms=runtime_ms, error_message=str(exc), completed_at=utc_now()) - raise - return experiment diff --git a/backend/app/services/storage_service.py b/backend/app/services/storage_service.py deleted file mode 100644 index 85b31873b54bf3b7f3ba249c1c8f7a512bc6a095..0000000000000000000000000000000000000000 --- a/backend/app/services/storage_service.py +++ /dev/null @@ -1,16 +0,0 @@ -from pathlib import Path -from app.config import get_settings - -class StorageService: - def __init__(self): - self.settings = get_settings() - - def uploads_dir(self) -> Path: - path = Path(self.settings.upload_dir) - path.mkdir(parents=True, exist_ok=True) - return path - - def artifacts_dir(self) -> Path: - path = Path(self.settings.artifact_dir) - path.mkdir(parents=True, exist_ok=True) - return path diff --git a/backend/app/services/visualization_service.py b/backend/app/services/visualization_service.py deleted file mode 100644 index 0cc33a8e4ff08155665295ce7d3edc4b3352a640..0000000000000000000000000000000000000000 --- a/backend/app/services/visualization_service.py +++ /dev/null @@ -1,12 +0,0 @@ -from app.ml.preprocess.reducers import pca_2d, umap_2d - -class VisualizationService: - def build_coords(self, X): - pca_x, pca_y = pca_2d(X) - umap_x, umap_y = umap_2d(X) - return { - 'pca_x': pca_x.tolist(), - 'pca_y': pca_y.tolist(), - 'umap_x': umap_x.tolist(), - 'umap_y': umap_y.tolist(), - } diff --git a/backend/app/utils/__init__.py b/backend/app/utils/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/backend/app/utils/file_hash.py b/backend/app/utils/file_hash.py deleted file mode 100644 index 31dc4c7aa2a0b39292bd0181d87eedf7e3ad28e5..0000000000000000000000000000000000000000 --- a/backend/app/utils/file_hash.py +++ /dev/null @@ -1,9 +0,0 @@ -import hashlib -from pathlib import Path - -def sha256_file(path: str | Path) -> str: - h = hashlib.sha256() - with open(path, 'rb') as f: - for chunk in iter(lambda: f.read(8192), b''): - h.update(chunk) - return h.hexdigest() diff --git a/backend/app/utils/ids.py b/backend/app/utils/ids.py deleted file mode 100644 index 89653419348aebf77c54da1c41b08b585adbf722..0000000000000000000000000000000000000000 --- a/backend/app/utils/ids.py +++ /dev/null @@ -1,4 +0,0 @@ -import secrets - -def generate_id(prefix: str) -> str: - return f"{prefix}_{secrets.token_hex(8)}" diff --git a/backend/app/utils/time_utils.py b/backend/app/utils/time_utils.py deleted file mode 100644 index 233f02321de2ae961593931806bb1dfdeb919534..0000000000000000000000000000000000000000 --- a/backend/app/utils/time_utils.py +++ /dev/null @@ -1,4 +0,0 @@ -from datetime import datetime - -def utc_now() -> datetime: - return datetime.utcnow() diff --git a/backend/app/utils/validators.py b/backend/app/utils/validators.py deleted file mode 100644 index fbc34f038e8a2ac8c05ff9ed024c34dbf26561c5..0000000000000000000000000000000000000000 --- a/backend/app/utils/validators.py +++ /dev/null @@ -1,10 +0,0 @@ -SUPPORTED_ALGORITHMS = { - 'kmeans', - 'minibatch_kmeans', - 'agglomerative', - 'dbscan', - 'hdbscan', - 'gmm', - 'spectral', - 'birch', -} diff --git a/backend/requirements.txt b/backend/requirements.txt deleted file mode 100644 index 27b837977e98d595cb6069f6ae8ade259dda189a..0000000000000000000000000000000000000000 --- a/backend/requirements.txt +++ /dev/null @@ -1,13 +0,0 @@ -fastapi==0.115.12 -uvicorn[standard]==0.34.2 -sqlalchemy==2.0.40 -pydantic==2.11.3 -pydantic-settings==2.8.1 -python-multipart==0.0.20 -pandas==2.2.3 -numpy==2.2.5 -scikit-learn==1.6.1 -joblib==1.4.2 -umap-learn==0.5.7 -hdbscan==0.8.40 -openpyxl==3.1.5 diff --git a/backend/tests/test_health.py b/backend/tests/test_health.py deleted file mode 100644 index fd66ec0efba24fc13dfb7502e6f0037197525fc9..0000000000000000000000000000000000000000 --- a/backend/tests/test_health.py +++ /dev/null @@ -1,8 +0,0 @@ -from fastapi.testclient import TestClient -from app.main import app - -def test_health(): - client = TestClient(app) - resp = client.get('/api/health') - assert resp.status_code == 200 - assert resp.json()['ok'] is True