diff --git a/README.md b/README.md index 0fa0ee7ad325c1fc5bf5707c32f71144e8cb6333..3f9bc50b2eeb5874061f15c7c444daf20a62e60d 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,37 @@ ---- -title: ClusterBuster API -emoji: 🔥 -colorFrom: green -colorTo: red -sdk: docker -pinned: false ---- - -Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference +# ClusterForge + +Full stack clustering experimentation platform. + +Stack: +- FastAPI backend +- Next.js frontend +- SQLite database +- scikit-learn clustering pipeline +- Hugging Face Docker Space deployment files + +Core features: +- CSV and Excel upload +- Dataset profiling +- Algorithm selection +- Dynamic parameters +- Run experiments +- Result visualizations +- Cluster summaries +- Outlier detection +- Run comparison +- CSV export + +Algorithms: +- KMeans +- MiniBatchKMeans +- Agglomerative +- DBSCAN +- HDBSCAN +- Gaussian Mixture +- Spectral +- Birch + +Runtime paths for Hugging Face: +- DB: /data/clusterforge.db +- Uploads: /data/uploads +- Artifacts: /data/artifacts diff --git a/backend/.DS_Store b/backend/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..ba27f483746bc38feb7c61e939fd716c22fcac3e Binary files /dev/null and b/backend/.DS_Store differ diff --git a/backend/app/__init__.py b/backend/app/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/backend/app/api/__init__.py b/backend/app/api/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/backend/app/api/clusters.py b/backend/app/api/clusters.py new file mode 100644 index 0000000000000000000000000000000000000000..63a58ec708d6c407d2ee9cf8263b9d45fa8b7267 --- /dev/null +++ b/backend/app/api/clusters.py @@ -0,0 +1,40 @@ +import json +import pandas as pd +from fastapi import APIRouter, HTTPException +from app.db import get_session +from app.repositories.cluster_repo import ClusterRepository +from app.repositories.dataset_repo import DatasetRepository +from app.repositories.experiment_repo import ExperimentRepository +from app.schemas.cluster import ClusterDetailResponse, ClusterMember + +router = APIRouter(prefix='/clusters', tags=['clusters']) + +@router.get('/{experiment_id}/{cluster_label}', response_model=ClusterDetailResponse) +def get_cluster_detail(experiment_id: str, cluster_label: int): + with get_session() as session: + exp_repo = ExperimentRepository(session) + dataset_repo = DatasetRepository(session) + cluster_repo = ClusterRepository(session) + exp = exp_repo.get(experiment_id) + if not exp: + raise HTTPException(status_code=404, detail='Experiment not found') + dataset = dataset_repo.get(exp.dataset_id) + if not dataset: + raise HTTPException(status_code=404, detail='Dataset not found') + df = pd.read_csv(dataset.file_path) if dataset.file_path.lower().endswith('.csv') else pd.read_excel(dataset.file_path) + results = [r for r in cluster_repo.get_results(experiment_id) if r.cluster_label == cluster_label] + summary = cluster_repo.get_summary(experiment_id, cluster_label) + if not summary: + raise HTTPException(status_code=404, detail='Cluster summary not found') + members = [] + for row in results[:200]: + values = df.iloc[row.row_index].to_dict() + values = {str(k): (None if pd.isna(v) else v) for k, v in values.items()} + members.append(ClusterMember(row_index=row.row_index, values=values)) + return ClusterDetailResponse( + experiment_id=experiment_id, + cluster_label=cluster_label, + members=members, + summary=json.loads(summary.summary_json), + description_text=summary.description_text, + ) diff --git a/backend/app/api/compare.py b/backend/app/api/compare.py new file mode 100644 index 0000000000000000000000000000000000000000..04f3d721148d5a4361d06d0200eb20274aa71558 --- /dev/null +++ b/backend/app/api/compare.py @@ -0,0 +1,19 @@ +from fastapi import APIRouter, Query +from app.db import get_session +from app.repositories.cluster_repo import ClusterRepository +from app.repositories.experiment_repo import ExperimentRepository +from app.schemas.compare import CompareResponse +from app.services.comparison_service import ComparisonService + +router = APIRouter(prefix='/compare', tags=['compare']) + +@router.get('', response_model=CompareResponse) +def compare_runs(experiment_ids: str = Query(...)): + ids = [item.strip() for item in experiment_ids.split(',') if item.strip()] + with get_session() as session: + payload = ComparisonService().build_compare_payload( + ExperimentRepository(session), + ClusterRepository(session), + ids, + ) + return CompareResponse(**payload) diff --git a/backend/app/api/datasets.py b/backend/app/api/datasets.py new file mode 100644 index 0000000000000000000000000000000000000000..da17b521d3131716cee3a5166f8fd421f5cb0977 --- /dev/null +++ b/backend/app/api/datasets.py @@ -0,0 +1,55 @@ +from fastapi import APIRouter, File, HTTPException, UploadFile +from app.db import get_session +from app.repositories.dataset_repo import DatasetRepository +from app.schemas.dataset import DatasetProfileResponse, DatasetUploadResponse +from app.services.dataset_service import DatasetService +from app.services.profiling_service import ProfilingService + +router = APIRouter(prefix='/datasets', tags=['datasets']) + +@router.post('/upload', response_model=DatasetUploadResponse) +async def upload_dataset(file: UploadFile = File(...)): + with get_session() as session: + repo = DatasetRepository(session) + service = DatasetService(repo) + dataset = await service.save_upload(file) + return DatasetUploadResponse( + dataset_id=dataset.id, + name=dataset.name, + row_count=dataset.row_count, + column_count=dataset.column_count, + ) + +@router.get('', response_model=list[DatasetUploadResponse]) +def list_datasets(): + with get_session() as session: + repo = DatasetRepository(session) + items = repo.list_all() + return [ + DatasetUploadResponse( + dataset_id=item.id, + name=item.name, + row_count=item.row_count, + column_count=item.column_count, + ) + for item in items + ] + +@router.get('/{dataset_id}/profile', response_model=DatasetProfileResponse) +def dataset_profile(dataset_id: str): + with get_session() as session: + repo = DatasetRepository(session) + dataset = repo.get(dataset_id) + if not dataset: + raise HTTPException(status_code=404, detail='Dataset not found') + df = DatasetService(repo).load_dataframe(dataset) + profile = ProfilingService().profile(df) + repo.update_profile(dataset_id, schema_json={'columns': [c['name'] for c in profile['columns']]}, profile_json=profile) + return DatasetProfileResponse( + dataset_id=dataset_id, + columns=profile['columns'], + suggested_feature_columns=profile['suggested_feature_columns'], + recommended_algorithms=profile['recommended_algorithms'], + row_count=profile['row_count'], + column_count=profile['column_count'], + ) diff --git a/backend/app/api/experiments.py b/backend/app/api/experiments.py new file mode 100644 index 0000000000000000000000000000000000000000..f7c5bdc21744cde43675abbb4db33d54243c2700 --- /dev/null +++ b/backend/app/api/experiments.py @@ -0,0 +1,105 @@ +import json +from fastapi import APIRouter, HTTPException +from app.db import get_session +from app.repositories.cluster_repo import ClusterRepository +from app.repositories.dataset_repo import DatasetRepository +from app.repositories.experiment_repo import ExperimentRepository +from app.schemas.experiment import ClusterSummaryResponse, ExperimentListItem, ExperimentResultResponse, ExperimentRunRequest, ExperimentRunResponse, MetricResponse, ScatterPoint +from app.services.run_service import RunService + +router = APIRouter(prefix='/experiments', tags=['experiments']) + +@router.post('/run', response_model=ExperimentRunResponse) +def run_experiment(payload: ExperimentRunRequest): + with get_session() as session: + dataset_repo = DatasetRepository(session) + exp_repo = ExperimentRepository(session) + cluster_repo = ClusterRepository(session) + service = RunService(dataset_repo, exp_repo, cluster_repo) + experiment = service.run(payload.model_dump()) + return ExperimentRunResponse(experiment_id=experiment.id, status='completed') + +@router.get('', response_model=list[ExperimentListItem]) +def list_experiments(): + with get_session() as session: + repo = ExperimentRepository(session) + items = repo.list_all() + return [ + ExperimentListItem( + experiment_id=item.id, + dataset_id=item.dataset_id, + name=item.name, + algorithm=item.algorithm, + status=item.status, + started_at=item.started_at.isoformat(), + completed_at=item.completed_at.isoformat() if item.completed_at else None, + ) + for item in items + ] + +@router.get('/{experiment_id}') +def get_experiment(experiment_id: str): + with get_session() as session: + repo = ExperimentRepository(session) + exp = repo.get(experiment_id) + if not exp: + raise HTTPException(status_code=404, detail='Experiment not found') + return { + 'experiment_id': exp.id, + 'dataset_id': exp.dataset_id, + 'name': exp.name, + 'algorithm': exp.algorithm, + 'status': exp.status, + 'config': json.loads(exp.config_json), + 'runtime_ms': exp.runtime_ms, + 'error_message': exp.error_message, + } + +@router.get('/{experiment_id}/results', response_model=ExperimentResultResponse) +def get_results(experiment_id: str): + with get_session() as session: + exp_repo = ExperimentRepository(session) + cluster_repo = ClusterRepository(session) + exp = exp_repo.get(experiment_id) + if not exp: + raise HTTPException(status_code=404, detail='Experiment not found') + metric = cluster_repo.get_metric(experiment_id) + summaries = cluster_repo.get_summaries(experiment_id) + results = cluster_repo.get_results(experiment_id) + if not metric: + raise HTTPException(status_code=404, detail='Results not found') + scatter = [ + ScatterPoint( + row_index=item.row_index, + cluster_label=item.cluster_label, + is_outlier=item.is_outlier, + x=item.umap_x if item.umap_x is not None else item.pca_x, + y=item.umap_y if item.umap_y is not None else item.pca_y, + ) + for item in results + ] + summary_items = [ + ClusterSummaryResponse( + cluster_label=item.cluster_label, + size=item.size, + description_text=item.description_text, + summary_json=json.loads(item.summary_json), + ) + for item in summaries + ] + return ExperimentResultResponse( + experiment_id=experiment_id, + algorithm=exp.algorithm, + metrics=MetricResponse( + cluster_count=metric.cluster_count, + noise_ratio=metric.noise_ratio, + silhouette_score=metric.silhouette_score, + davies_bouldin_score=metric.davies_bouldin_score, + calinski_harabasz_score=metric.calinski_harabasz_score, + stability_score=metric.stability_score, + interpretability_score=metric.interpretability_score, + ), + scatter_points=scatter, + cluster_summaries=summary_items, + status=exp.status, + ) diff --git a/backend/app/api/exports.py b/backend/app/api/exports.py new file mode 100644 index 0000000000000000000000000000000000000000..661b73407685c38bba37114849a5eddaf99fea97 --- /dev/null +++ b/backend/app/api/exports.py @@ -0,0 +1,24 @@ +from fastapi import APIRouter, HTTPException +from app.db import get_session +from app.repositories.artifact_repo import ArtifactRepository +from app.repositories.cluster_repo import ClusterRepository +from app.repositories.dataset_repo import DatasetRepository +from app.repositories.experiment_repo import ExperimentRepository +from app.schemas.export import ExportResponse +from app.services.export_service import ExportService + +router = APIRouter(prefix='/exports', tags=['exports']) + +@router.get('/{experiment_id}', response_model=ExportResponse) +def export_experiment(experiment_id: str): + with get_session() as session: + exp = ExperimentRepository(session).get(experiment_id) + if not exp: + raise HTTPException(status_code=404, detail='Experiment not found') + service = ExportService( + dataset_repo=DatasetRepository(session), + cluster_repo=ClusterRepository(session), + artifact_repo=ArtifactRepository(session), + ) + path = service.export_csv(experiment_id, exp.dataset_id) + return ExportResponse(experiment_id=experiment_id, download_path=path) diff --git a/backend/app/api/health.py b/backend/app/api/health.py new file mode 100644 index 0000000000000000000000000000000000000000..6648e86867ac1d4494b50d744e47635650dfc7de --- /dev/null +++ b/backend/app/api/health.py @@ -0,0 +1,7 @@ +from fastapi import APIRouter +from app.schemas.health import HealthResponse +router = APIRouter(tags=['health']) + +@router.get('/health', response_model=HealthResponse) +def health() -> HealthResponse: + return HealthResponse(ok=True, service='clusterforge-api') diff --git a/backend/app/api/presets.py b/backend/app/api/presets.py new file mode 100644 index 0000000000000000000000000000000000000000..c934338397631cd80143ad5c1ab5638f79d3ce6c --- /dev/null +++ b/backend/app/api/presets.py @@ -0,0 +1,17 @@ +from fastapi import APIRouter +router = APIRouter(prefix='/presets', tags=['presets']) + +@router.get('/algorithms') +def list_algorithms(): + return { + 'algorithms': [ + {'key': 'kmeans', 'label': 'KMeans', 'params': {'n_clusters': 4, 'random_state': 42, 'max_iter': 300, 'init': 'k-means++'}}, + {'key': 'minibatch_kmeans', 'label': 'MiniBatch KMeans', 'params': {'n_clusters': 4, 'batch_size': 256, 'max_iter': 200}}, + {'key': 'agglomerative', 'label': 'Agglomerative', 'params': {'n_clusters': 4, 'linkage': 'ward', 'metric': 'euclidean'}}, + {'key': 'dbscan', 'label': 'DBSCAN', 'params': {'eps': 0.7, 'min_samples': 5, 'metric': 'euclidean'}}, + {'key': 'hdbscan', 'label': 'HDBSCAN', 'params': {'min_cluster_size': 15, 'min_samples': 5, 'cluster_selection_method': 'eom'}}, + {'key': 'gmm', 'label': 'Gaussian Mixture', 'params': {'n_components': 4, 'covariance_type': 'full'}}, + {'key': 'spectral', 'label': 'Spectral', 'params': {'n_clusters': 4, 'affinity': 'nearest_neighbors', 'n_neighbors': 10}}, + {'key': 'birch', 'label': 'Birch', 'params': {'n_clusters': 4, 'threshold': 0.5, 'branching_factor': 50}}, + ] + } diff --git a/backend/app/config.py b/backend/app/config.py new file mode 100644 index 0000000000000000000000000000000000000000..78bfe8ecfdadb89cc5336f1cbc7288107364d6ef --- /dev/null +++ b/backend/app/config.py @@ -0,0 +1,23 @@ +from functools import lru_cache +from typing import List +from pydantic import Field +from pydantic_settings import BaseSettings, SettingsConfigDict + +class Settings(BaseSettings): + model_config = SettingsConfigDict(env_file='.env', env_file_encoding='utf-8', extra='ignore') + app_name: str = Field(default='ClusterForge', alias='APP_NAME') + app_env: str = Field(default='dev', alias='APP_ENV') + api_prefix: str = Field(default='/api', alias='API_PREFIX') + data_dir: str = Field(default='/data', alias='DATA_DIR') + db_path: str = Field(default='/data/clusterforge.db', alias='DB_PATH') + upload_dir: str = Field(default='/data/uploads', alias='UPLOAD_DIR') + artifact_dir: str = Field(default='/data/artifacts', alias='ARTIFACT_DIR') + allowed_origins_raw: str = Field(default='http://localhost:3000,http://127.0.0.1:3000', alias='ALLOWED_ORIGINS') + + @property + def allowed_origins(self) -> List[str]: + return [x.strip() for x in self.allowed_origins_raw.split(',') if x.strip()] + +@lru_cache +def get_settings() -> Settings: + return Settings() diff --git a/backend/app/db.py b/backend/app/db.py new file mode 100644 index 0000000000000000000000000000000000000000..66f387b7298c772a6cd13109c46473121545921d --- /dev/null +++ b/backend/app/db.py @@ -0,0 +1,36 @@ +from contextlib import contextmanager +from sqlalchemy import create_engine +from sqlalchemy.orm import declarative_base, sessionmaker +from app.config import get_settings + +settings = get_settings() +engine = create_engine( + f"sqlite:///{settings.db_path}", + connect_args={'check_same_thread': False}, + future=True, +) +SessionLocal = sessionmaker(bind=engine, autoflush=False, autocommit=False, future=True) +Base = declarative_base() + +def init_storage_dirs() -> None: + import os + os.makedirs(settings.data_dir, exist_ok=True) + os.makedirs(settings.upload_dir, exist_ok=True) + os.makedirs(settings.artifact_dir, exist_ok=True) + +def init_db() -> None: + init_storage_dirs() + from app.models import dataset, experiment, cluster_result, artifact # noqa + Base.metadata.create_all(bind=engine) + +@contextmanager +def get_session(): + session = SessionLocal() + try: + yield session + session.commit() + except Exception: + session.rollback() + raise + finally: + session.close() diff --git a/backend/app/logging_config.py b/backend/app/logging_config.py new file mode 100644 index 0000000000000000000000000000000000000000..730062ef056da664fb9780de25ee49512b6341c1 --- /dev/null +++ b/backend/app/logging_config.py @@ -0,0 +1,7 @@ +import logging + +def setup_logging() -> None: + logging.basicConfig( + level=logging.INFO, + format='%(asctime)s | %(levelname)s | %(name)s | %(message)s' + ) diff --git a/backend/app/main.py b/backend/app/main.py new file mode 100644 index 0000000000000000000000000000000000000000..92eee7ee5fe1db6fef986b6375fb08cf9cb2a88c --- /dev/null +++ b/backend/app/main.py @@ -0,0 +1,31 @@ +from fastapi import FastAPI +from fastapi.middleware.cors import CORSMiddleware +from app.api.clusters import router as clusters_router +from app.api.compare import router as compare_router +from app.api.datasets import router as datasets_router +from app.api.experiments import router as experiments_router +from app.api.exports import router as exports_router +from app.api.health import router as health_router +from app.api.presets import router as presets_router +from app.config import get_settings +from app.db import init_db +from app.logging_config import setup_logging + +setup_logging() +settings = get_settings() +init_db() +app = FastAPI(title=settings.app_name) +app.add_middleware( + CORSMiddleware, + allow_origins=settings.allowed_origins, + allow_credentials=True, + allow_methods=['*'], + allow_headers=['*'], +) +app.include_router(health_router, prefix=settings.api_prefix) +app.include_router(datasets_router, prefix=settings.api_prefix) +app.include_router(experiments_router, prefix=settings.api_prefix) +app.include_router(clusters_router, prefix=settings.api_prefix) +app.include_router(compare_router, prefix=settings.api_prefix) +app.include_router(exports_router, prefix=settings.api_prefix) +app.include_router(presets_router, prefix=settings.api_prefix) diff --git a/backend/app/ml/__init__.py b/backend/app/ml/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/backend/app/ml/algorithms/__init__.py b/backend/app/ml/algorithms/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/backend/app/ml/algorithms/agglomerative_runner.py b/backend/app/ml/algorithms/agglomerative_runner.py new file mode 100644 index 0000000000000000000000000000000000000000..dcf000cad61f44e82cb96fa32d64380dc172338c --- /dev/null +++ b/backend/app/ml/algorithms/agglomerative_runner.py @@ -0,0 +1,14 @@ +from sklearn.cluster import AgglomerativeClustering +from app.ml.algorithms.base import BaseClusterRunner, ClusterRunOutput +class Runner(BaseClusterRunner): + name = 'agglomerative' + def default_params(self) -> dict: + return {'n_clusters': 4, 'linkage': 'ward', 'metric': 'euclidean'} + def fit_predict(self, X, params: dict) -> ClusterRunOutput: + cfg = self.default_params() | params + kwargs = {'n_clusters': int(cfg['n_clusters']), 'linkage': str(cfg.get('linkage', 'ward'))} + if kwargs['linkage'] != 'ward': + kwargs['metric'] = str(cfg.get('metric', 'euclidean')) + model = AgglomerativeClustering(**kwargs) + labels = model.fit_predict(X) + return ClusterRunOutput(labels=labels.tolist()) diff --git a/backend/app/ml/algorithms/base.py b/backend/app/ml/algorithms/base.py new file mode 100644 index 0000000000000000000000000000000000000000..0499ce20d20fff75852ac1a980e5f12f25b9f6cf --- /dev/null +++ b/backend/app/ml/algorithms/base.py @@ -0,0 +1,17 @@ +from dataclasses import dataclass +from typing import Any + +@dataclass +class ClusterRunOutput: + labels: list[int] + cluster_centers: list[list[float]] | None = None + extra: dict[str, Any] | None = None + +class BaseClusterRunner: + name: str = 'base' + def fit_predict(self, X, params: dict) -> ClusterRunOutput: + raise NotImplementedError + def supports_noise(self) -> bool: + return False + def default_params(self) -> dict: + return {} diff --git a/backend/app/ml/algorithms/birch_runner.py b/backend/app/ml/algorithms/birch_runner.py new file mode 100644 index 0000000000000000000000000000000000000000..ad4a73fd10d486aebff8e242f7cf62410de2046f --- /dev/null +++ b/backend/app/ml/algorithms/birch_runner.py @@ -0,0 +1,15 @@ +from sklearn.cluster import Birch +from app.ml.algorithms.base import BaseClusterRunner, ClusterRunOutput +class Runner(BaseClusterRunner): + name = 'birch' + def default_params(self) -> dict: + return {'n_clusters': 4, 'threshold': 0.5, 'branching_factor': 50} + def fit_predict(self, X, params: dict) -> ClusterRunOutput: + cfg = self.default_params() | params + model = Birch( + n_clusters=int(cfg.get('n_clusters', 4)), + threshold=float(cfg.get('threshold', 0.5)), + branching_factor=int(cfg.get('branching_factor', 50)), + ) + labels = model.fit_predict(X) + return ClusterRunOutput(labels=labels.tolist()) diff --git a/backend/app/ml/algorithms/dbscan_runner.py b/backend/app/ml/algorithms/dbscan_runner.py new file mode 100644 index 0000000000000000000000000000000000000000..436769364172b75fb481f539b9f9bf9a18ae1bb0 --- /dev/null +++ b/backend/app/ml/algorithms/dbscan_runner.py @@ -0,0 +1,17 @@ +from sklearn.cluster import DBSCAN +from app.ml.algorithms.base import BaseClusterRunner, ClusterRunOutput +class Runner(BaseClusterRunner): + name = 'dbscan' + def default_params(self) -> dict: + return {'eps': 0.7, 'min_samples': 5, 'metric': 'euclidean'} + def supports_noise(self) -> bool: + return True + def fit_predict(self, X, params: dict) -> ClusterRunOutput: + cfg = self.default_params() | params + model = DBSCAN( + eps=float(cfg.get('eps', 0.7)), + min_samples=int(cfg.get('min_samples', 5)), + metric=str(cfg.get('metric', 'euclidean')), + ) + labels = model.fit_predict(X) + return ClusterRunOutput(labels=labels.tolist()) diff --git a/backend/app/ml/algorithms/gmm_runner.py b/backend/app/ml/algorithms/gmm_runner.py new file mode 100644 index 0000000000000000000000000000000000000000..293537b7483f4c8975da03ac354de31ea5e7caea --- /dev/null +++ b/backend/app/ml/algorithms/gmm_runner.py @@ -0,0 +1,15 @@ +from sklearn.mixture import GaussianMixture +from app.ml.algorithms.base import BaseClusterRunner, ClusterRunOutput +class Runner(BaseClusterRunner): + name = 'gmm' + def default_params(self) -> dict: + return {'n_components': 4, 'covariance_type': 'full', 'random_state': 42} + def fit_predict(self, X, params: dict) -> ClusterRunOutput: + cfg = self.default_params() | params + model = GaussianMixture( + n_components=int(cfg.get('n_components', 4)), + covariance_type=str(cfg.get('covariance_type', 'full')), + random_state=int(cfg.get('random_state', 42)), + ) + labels = model.fit_predict(X) + return ClusterRunOutput(labels=labels.tolist(), cluster_centers=model.means_.tolist()) diff --git a/backend/app/ml/algorithms/hdbscan_runner.py b/backend/app/ml/algorithms/hdbscan_runner.py new file mode 100644 index 0000000000000000000000000000000000000000..d5dbb7423c7c947853f137b45895e7194c44c40d --- /dev/null +++ b/backend/app/ml/algorithms/hdbscan_runner.py @@ -0,0 +1,20 @@ +from app.ml.algorithms.base import BaseClusterRunner, ClusterRunOutput +class Runner(BaseClusterRunner): + name = 'hdbscan' + def default_params(self) -> dict: + return {'min_cluster_size': 15, 'min_samples': 5, 'cluster_selection_method': 'eom'} + def supports_noise(self) -> bool: + return True + def fit_predict(self, X, params: dict) -> ClusterRunOutput: + try: + import hdbscan + except Exception as exc: + raise ValueError('HDBSCAN is not installed in this environment') from exc + cfg = self.default_params() | params + model = hdbscan.HDBSCAN( + min_cluster_size=int(cfg.get('min_cluster_size', 15)), + min_samples=int(cfg.get('min_samples', 5)), + cluster_selection_method=str(cfg.get('cluster_selection_method', 'eom')), + ) + labels = model.fit_predict(X) + return ClusterRunOutput(labels=labels.tolist()) diff --git a/backend/app/ml/algorithms/kmeans_runner.py b/backend/app/ml/algorithms/kmeans_runner.py new file mode 100644 index 0000000000000000000000000000000000000000..5bd84143940130dbfed15e1c8240b099d455156b --- /dev/null +++ b/backend/app/ml/algorithms/kmeans_runner.py @@ -0,0 +1,17 @@ +from sklearn.cluster import KMeans +from app.ml.algorithms.base import BaseClusterRunner, ClusterRunOutput +class Runner(BaseClusterRunner): + name = 'kmeans' + def default_params(self) -> dict: + return {'n_clusters': 4, 'random_state': 42, 'max_iter': 300, 'init': 'k-means++'} + def fit_predict(self, X, params: dict) -> ClusterRunOutput: + cfg = self.default_params() | params + model = KMeans( + n_clusters=int(cfg['n_clusters']), + random_state=int(cfg.get('random_state', 42)), + max_iter=int(cfg.get('max_iter', 300)), + init=str(cfg.get('init', 'k-means++')), + n_init=10, + ) + labels = model.fit_predict(X) + return ClusterRunOutput(labels=labels.tolist(), cluster_centers=model.cluster_centers_.tolist()) diff --git a/backend/app/ml/algorithms/minibatch_kmeans_runner.py b/backend/app/ml/algorithms/minibatch_kmeans_runner.py new file mode 100644 index 0000000000000000000000000000000000000000..2f34f9619c40e6d88a1f1f8879b79063c9d10596 --- /dev/null +++ b/backend/app/ml/algorithms/minibatch_kmeans_runner.py @@ -0,0 +1,17 @@ +from sklearn.cluster import MiniBatchKMeans +from app.ml.algorithms.base import BaseClusterRunner, ClusterRunOutput +class Runner(BaseClusterRunner): + name = 'minibatch_kmeans' + def default_params(self) -> dict: + return {'n_clusters': 4, 'random_state': 42, 'max_iter': 200, 'batch_size': 256} + def fit_predict(self, X, params: dict) -> ClusterRunOutput: + cfg = self.default_params() | params + model = MiniBatchKMeans( + n_clusters=int(cfg['n_clusters']), + random_state=int(cfg.get('random_state', 42)), + max_iter=int(cfg.get('max_iter', 200)), + batch_size=int(cfg.get('batch_size', 256)), + n_init=5, + ) + labels = model.fit_predict(X) + return ClusterRunOutput(labels=labels.tolist(), cluster_centers=model.cluster_centers_.tolist()) diff --git a/backend/app/ml/algorithms/spectral_runner.py b/backend/app/ml/algorithms/spectral_runner.py new file mode 100644 index 0000000000000000000000000000000000000000..c0b0c25e356a983879b77e7fcbfbc329ff822948 --- /dev/null +++ b/backend/app/ml/algorithms/spectral_runner.py @@ -0,0 +1,17 @@ +from sklearn.cluster import SpectralClustering +from app.ml.algorithms.base import BaseClusterRunner, ClusterRunOutput +class Runner(BaseClusterRunner): + name = 'spectral' + def default_params(self) -> dict: + return {'n_clusters': 4, 'affinity': 'nearest_neighbors', 'n_neighbors': 10, 'random_state': 42} + def fit_predict(self, X, params: dict) -> ClusterRunOutput: + cfg = self.default_params() | params + model = SpectralClustering( + n_clusters=int(cfg.get('n_clusters', 4)), + affinity=str(cfg.get('affinity', 'nearest_neighbors')), + n_neighbors=int(cfg.get('n_neighbors', 10)), + random_state=int(cfg.get('random_state', 42)), + assign_labels='kmeans', + ) + labels = model.fit_predict(X) + return ClusterRunOutput(labels=labels.tolist()) diff --git a/backend/app/ml/explain/__init__.py b/backend/app/ml/explain/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/backend/app/ml/explain/cluster_profiles.py b/backend/app/ml/explain/cluster_profiles.py new file mode 100644 index 0000000000000000000000000000000000000000..ef47f6d4dff098acd819ef9eba5ea65f1db3b531 --- /dev/null +++ b/backend/app/ml/explain/cluster_profiles.py @@ -0,0 +1,22 @@ +import pandas as pd + +def build_cluster_profiles(df: pd.DataFrame, labels: list[int], feature_columns: list[str]) -> dict[int, dict]: + temp = df[feature_columns].copy() + temp['_cluster'] = labels + profiles = {} + for cluster_label, cluster_df in temp.groupby('_cluster'): + numeric_profile = {} + categorical_profile = {} + for col in feature_columns: + if pd.api.types.is_numeric_dtype(temp[col]): + numeric_profile[col] = round(float(cluster_df[col].mean()), 4) + else: + vc = cluster_df[col].astype(str).value_counts(normalize=True) + if len(vc): + categorical_profile[col] = {'top_value': vc.index[0], 'share': round(float(vc.iloc[0]), 4)} + profiles[int(cluster_label)] = { + 'size': int(len(cluster_df)), + 'numeric_profile': numeric_profile, + 'categorical_profile': categorical_profile, + } + return profiles diff --git a/backend/app/ml/explain/feature_lift.py b/backend/app/ml/explain/feature_lift.py new file mode 100644 index 0000000000000000000000000000000000000000..d96c693ee0c58099f4f6885f61e7d2134cf026fb --- /dev/null +++ b/backend/app/ml/explain/feature_lift.py @@ -0,0 +1,24 @@ +import pandas as pd + +def compute_feature_lift(df: pd.DataFrame, labels: list[int], feature_columns: list[str]) -> dict[int, dict]: + temp = df[feature_columns].copy() + temp['_cluster'] = labels + out = {} + global_numeric = {c: float(temp[c].mean()) for c in feature_columns if pd.api.types.is_numeric_dtype(temp[c])} + for cluster_label, cluster_df in temp.groupby('_cluster'): + info = {'top_numeric_lifts': [], 'top_categorical_modes': []} + numeric_lifts = [] + for col in feature_columns: + if pd.api.types.is_numeric_dtype(temp[col]): + base = global_numeric.get(col, 0.0) + mean = float(cluster_df[col].mean()) + lift = round(mean - base, 4) + numeric_lifts.append({'feature': col, 'cluster_mean': round(mean, 4), 'global_mean': round(base, 4), 'lift': lift}) + else: + mode_series = cluster_df[col].astype(str).value_counts(normalize=True) + if len(mode_series): + info['top_categorical_modes'].append({'feature': col, 'value': mode_series.index[0], 'share': round(float(mode_series.iloc[0]), 4)}) + info['top_numeric_lifts'] = sorted(numeric_lifts, key=lambda x: abs(x['lift']), reverse=True)[:5] + info['top_categorical_modes'] = info['top_categorical_modes'][:5] + out[int(cluster_label)] = info + return out diff --git a/backend/app/ml/explain/summary_writer.py b/backend/app/ml/explain/summary_writer.py new file mode 100644 index 0000000000000000000000000000000000000000..95729cdf08dc6caf7d91932f25b47cc180de2f21 --- /dev/null +++ b/backend/app/ml/explain/summary_writer.py @@ -0,0 +1,10 @@ +def build_cluster_description(cluster_label: int, profile: dict, lift_info: dict) -> str: + numeric = lift_info.get('top_numeric_lifts', [])[:3] + categorical = lift_info.get('top_categorical_modes', [])[:2] + parts = [] + if numeric: + parts.append(', '.join([('high ' if item['lift'] >= 0 else 'low ') + item['feature'] for item in numeric])) + if categorical: + parts.append(', '.join([f"{item['feature']} is mostly {item['value']}" for item in categorical])) + core = '. '.join([p for p in parts if p]).strip() or 'This cluster is defined by a mixed feature pattern.' + return f"Cluster {cluster_label} has {profile.get('size', 0)} rows. {core}." diff --git a/backend/app/ml/metrics/__init__.py b/backend/app/ml/metrics/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/backend/app/ml/metrics/clustering_metrics.py b/backend/app/ml/metrics/clustering_metrics.py new file mode 100644 index 0000000000000000000000000000000000000000..c6a1fd46ec6fc4ed16aa4c2103ec8bd58acc3a59 --- /dev/null +++ b/backend/app/ml/metrics/clustering_metrics.py @@ -0,0 +1,33 @@ +import numpy as np +from sklearn.metrics import calinski_harabasz_score, davies_bouldin_score, silhouette_score + +def compute_metrics(X, labels: list[int]) -> dict: + arr = np.array(labels) + unique = set(arr.tolist()) + non_noise = [x for x in unique if x != -1] + cluster_count = len(non_noise) + noise_ratio = float((arr == -1).mean()) if len(arr) else 0.0 + metrics = { + 'cluster_count': cluster_count, + 'noise_ratio': round(noise_ratio, 4), + 'silhouette_score': None, + 'davies_bouldin_score': None, + 'calinski_harabasz_score': None, + } + mask = arr != -1 if -1 in unique else np.ones(len(arr), dtype=bool) + X_eval = X[mask] + y_eval = arr[mask] + if len(set(y_eval.tolist())) >= 2 and len(X_eval) > len(set(y_eval.tolist())): + try: + metrics['silhouette_score'] = round(float(silhouette_score(X_eval, y_eval)), 4) + except Exception: + pass + try: + metrics['davies_bouldin_score'] = round(float(davies_bouldin_score(X_eval, y_eval)), 4) + except Exception: + pass + try: + metrics['calinski_harabasz_score'] = round(float(calinski_harabasz_score(X_eval, y_eval)), 4) + except Exception: + pass + return metrics diff --git a/backend/app/ml/metrics/interpretability.py b/backend/app/ml/metrics/interpretability.py new file mode 100644 index 0000000000000000000000000000000000000000..c011dd25f1dd625717e94c5b98035c5a21434da8 --- /dev/null +++ b/backend/app/ml/metrics/interpretability.py @@ -0,0 +1,19 @@ +import numpy as np +import pandas as pd + +def compute_interpretability_score(df: pd.DataFrame, labels: list[int], feature_columns: list[str]) -> float: + if len(set(labels)) <= 1: + return 0.0 + scored = [] + temp = df[feature_columns].copy() + temp['_cluster'] = labels + for col in feature_columns: + if pd.api.types.is_numeric_dtype(temp[col]): + global_std = temp[col].std() or 1.0 + diffs = temp.groupby('_cluster')[col].mean().diff().abs().fillna(0) + lift = float(diffs.mean() / global_std) + else: + top_share = temp.groupby('_cluster')[col].agg(lambda s: s.astype(str).value_counts(normalize=True).iloc[0] if len(s) else 0).mean() + lift = float(top_share) + scored.append(min(lift, 1.0)) + return round(float(np.mean(scored)), 4) if scored else 0.0 diff --git a/backend/app/ml/metrics/stability.py b/backend/app/ml/metrics/stability.py new file mode 100644 index 0000000000000000000000000000000000000000..bafd00b4e18751924fe7ff7151f21f3cef2dab9d --- /dev/null +++ b/backend/app/ml/metrics/stability.py @@ -0,0 +1,10 @@ +import numpy as np +from sklearn.metrics import normalized_mutual_info_score + +def compute_stability_score(labels: list[int]) -> float: + arr = np.array(labels) + if len(set(arr.tolist())) <= 1: + return 0.0 + rotated = np.roll(arr, 1) + score = normalized_mutual_info_score(arr, rotated) + return round(float(score), 4) diff --git a/backend/app/ml/outliers/__init__.py b/backend/app/ml/outliers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/backend/app/ml/outliers/centroid_distance.py b/backend/app/ml/outliers/centroid_distance.py new file mode 100644 index 0000000000000000000000000000000000000000..e61b71ada7dc3b5afdcc189b96c39637f0d860cb --- /dev/null +++ b/backend/app/ml/outliers/centroid_distance.py @@ -0,0 +1,18 @@ +import numpy as np + +def detect_centroid_outliers(X, labels: list[int], threshold_quantile: float = 0.95) -> list[bool]: + arr = np.array(labels) + outliers = np.zeros(len(arr), dtype=bool) + for cluster_label in sorted(set(arr.tolist())): + if cluster_label == -1: + outliers[arr == cluster_label] = True + continue + idx = np.where(arr == cluster_label)[0] + if len(idx) <= 2: + continue + cluster_points = X[idx] + centroid = cluster_points.mean(axis=0) + dists = np.linalg.norm(cluster_points - centroid, axis=1) + cut = np.quantile(dists, threshold_quantile) + outliers[idx] = dists >= cut + return outliers.tolist() diff --git a/backend/app/ml/outliers/density_noise.py b/backend/app/ml/outliers/density_noise.py new file mode 100644 index 0000000000000000000000000000000000000000..e018098f0d501671cab4c3a70857227d1f3c74fb --- /dev/null +++ b/backend/app/ml/outliers/density_noise.py @@ -0,0 +1,2 @@ +def detect_noise_outliers(labels: list[int]) -> list[bool]: + return [label == -1 for label in labels] diff --git a/backend/app/ml/outliers/isolation_forest.py b/backend/app/ml/outliers/isolation_forest.py new file mode 100644 index 0000000000000000000000000000000000000000..82909e682f4cacffe3b1f2ac648800114697eb51 --- /dev/null +++ b/backend/app/ml/outliers/isolation_forest.py @@ -0,0 +1,6 @@ +from sklearn.ensemble import IsolationForest + +def detect_isolation_outliers(X) -> list[bool]: + model = IsolationForest(random_state=42, contamination='auto') + flags = model.fit_predict(X) + return [item == -1 for item in flags.tolist()] diff --git a/backend/app/ml/preprocess/__init__.py b/backend/app/ml/preprocess/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/backend/app/ml/preprocess/column_inference.py b/backend/app/ml/preprocess/column_inference.py new file mode 100644 index 0000000000000000000000000000000000000000..3a00437aeb00b32ec3bdd44604d94f2f62e4b898 --- /dev/null +++ b/backend/app/ml/preprocess/column_inference.py @@ -0,0 +1,33 @@ +from dataclasses import dataclass +import pandas as pd + +@dataclass +class InferredColumn: + name: str + inferred_type: str + missing_pct: float + cardinality: int + sample_values: list + +def infer_columns(df: pd.DataFrame) -> list[InferredColumn]: + cols = [] + for col in df.columns: + series = df[col] + missing_pct = float(series.isna().mean() * 100) + cardinality = int(series.nunique(dropna=True)) + sample_values = series.dropna().astype(str).head(5).tolist() + if pd.api.types.is_bool_dtype(series): + inferred = 'boolean' + elif pd.api.types.is_numeric_dtype(series): + inferred = 'numeric' + else: + unique_ratio = cardinality / max(len(series), 1) + inferred = 'high_cardinality' if cardinality > 50 and unique_ratio > 0.3 else 'categorical' + cols.append(InferredColumn( + name=col, + inferred_type=inferred, + missing_pct=round(missing_pct, 2), + cardinality=cardinality, + sample_values=sample_values, + )) + return cols diff --git a/backend/app/ml/preprocess/encoders.py b/backend/app/ml/preprocess/encoders.py new file mode 100644 index 0000000000000000000000000000000000000000..a06fab6892e453f6a8d6995f19cbf39b5a9e0395 --- /dev/null +++ b/backend/app/ml/preprocess/encoders.py @@ -0,0 +1,7 @@ +from sklearn.preprocessing import OneHotEncoder + +def get_onehot_encoder(): + try: + return OneHotEncoder(handle_unknown='ignore', sparse_output=False) + except TypeError: + return OneHotEncoder(handle_unknown='ignore', sparse=False) diff --git a/backend/app/ml/preprocess/imputers.py b/backend/app/ml/preprocess/imputers.py new file mode 100644 index 0000000000000000000000000000000000000000..ea787e7e1ccfa46ed7a919e2e67873017f26b5e6 --- /dev/null +++ b/backend/app/ml/preprocess/imputers.py @@ -0,0 +1,7 @@ +from sklearn.impute import SimpleImputer + +def numeric_imputer(): + return SimpleImputer(strategy='median') + +def categorical_imputer(): + return SimpleImputer(strategy='most_frequent') diff --git a/backend/app/ml/preprocess/matrix_builder.py b/backend/app/ml/preprocess/matrix_builder.py new file mode 100644 index 0000000000000000000000000000000000000000..eef599d94946addccc2a0977a30d8a260d2c045e --- /dev/null +++ b/backend/app/ml/preprocess/matrix_builder.py @@ -0,0 +1,32 @@ +import pandas as pd +from sklearn.compose import ColumnTransformer +from sklearn.pipeline import Pipeline +from app.ml.preprocess.encoders import get_onehot_encoder +from app.ml.preprocess.imputers import categorical_imputer, numeric_imputer +from app.ml.preprocess.scalers import get_scaler + +def build_feature_matrix(df: pd.DataFrame, feature_columns: list[str], numeric_scaler: str = 'standard', categorical_encoder: str = 'onehot'): + selected = df[feature_columns].copy() + numeric_cols = [c for c in selected.columns if pd.api.types.is_numeric_dtype(selected[c]) or pd.api.types.is_bool_dtype(selected[c])] + categorical_cols = [c for c in selected.columns if c not in numeric_cols] + + numeric_pipeline = Pipeline(steps=[('imputer', numeric_imputer()), ('scaler', get_scaler(numeric_scaler))]) + categorical_pipeline = Pipeline(steps=[('imputer', categorical_imputer()), ('encoder', get_onehot_encoder())]) + + transformer = ColumnTransformer( + transformers=[('num', numeric_pipeline, numeric_cols), ('cat', categorical_pipeline, categorical_cols)], + remainder='drop', + ) + X = transformer.fit_transform(selected) + + feature_names = [] + feature_names.extend(numeric_cols) + if categorical_cols: + enc = transformer.named_transformers_['cat'].named_steps['encoder'] + try: + cat_names = enc.get_feature_names_out(categorical_cols).tolist() + except Exception: + cat_names = categorical_cols + feature_names.extend(cat_names) + + return X, feature_names, {'numeric_cols': numeric_cols, 'categorical_cols': categorical_cols} diff --git a/backend/app/ml/preprocess/reducers.py b/backend/app/ml/preprocess/reducers.py new file mode 100644 index 0000000000000000000000000000000000000000..e1d95b6e83203bdec7a1c205e7436a577007f5a0 --- /dev/null +++ b/backend/app/ml/preprocess/reducers.py @@ -0,0 +1,16 @@ +from sklearn.decomposition import PCA + +def pca_2d(X): + reducer = PCA(n_components=2, random_state=42) + coords = reducer.fit_transform(X) + return coords[:, 0], coords[:, 1] + +def umap_2d(X): + try: + import umap + reducer = umap.UMAP(n_components=2, random_state=42) + coords = reducer.fit_transform(X) + except Exception: + reducer = PCA(n_components=2, random_state=42) + coords = reducer.fit_transform(X) + return coords[:, 0], coords[:, 1] diff --git a/backend/app/ml/preprocess/scalers.py b/backend/app/ml/preprocess/scalers.py new file mode 100644 index 0000000000000000000000000000000000000000..20b1c371d9ce3174665d8360df27a64c0dec707a --- /dev/null +++ b/backend/app/ml/preprocess/scalers.py @@ -0,0 +1,9 @@ +from sklearn.preprocessing import MinMaxScaler, RobustScaler, StandardScaler + +def get_scaler(name: str): + name = (name or 'standard').lower() + if name == 'minmax': + return MinMaxScaler() + if name == 'robust': + return RobustScaler() + return StandardScaler() diff --git a/backend/app/models/__init__.py b/backend/app/models/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/backend/app/models/artifact.py b/backend/app/models/artifact.py new file mode 100644 index 0000000000000000000000000000000000000000..22708953fd729da8bfe79ea466d243ee40ec79f2 --- /dev/null +++ b/backend/app/models/artifact.py @@ -0,0 +1,12 @@ +from datetime import datetime +from sqlalchemy import DateTime, ForeignKey, Integer, String, Text +from sqlalchemy.orm import Mapped, mapped_column +from app.db import Base + +class Artifact(Base): + __tablename__ = 'artifacts' + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + experiment_id: Mapped[str] = mapped_column(ForeignKey('experiments.id'), index=True, nullable=False) + artifact_type: Mapped[str] = mapped_column(String(64), nullable=False) + path: Mapped[str] = mapped_column(Text, nullable=False) + created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow, nullable=False) diff --git a/backend/app/models/cluster_result.py b/backend/app/models/cluster_result.py new file mode 100644 index 0000000000000000000000000000000000000000..51a7e7b1e5f33209c1b1740bdd927707d9b6aed8 --- /dev/null +++ b/backend/app/models/cluster_result.py @@ -0,0 +1,36 @@ +from sqlalchemy import Boolean, Float, ForeignKey, Integer, String, Text +from sqlalchemy.orm import Mapped, mapped_column +from app.db import Base + +class ExperimentMetric(Base): + __tablename__ = 'experiment_metrics' + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + experiment_id: Mapped[str] = mapped_column(ForeignKey('experiments.id'), index=True, nullable=False) + cluster_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False) + noise_ratio: Mapped[float] = mapped_column(Float, default=0.0, nullable=False) + silhouette_score: Mapped[float | None] = mapped_column(Float, nullable=True) + davies_bouldin_score: Mapped[float | None] = mapped_column(Float, nullable=True) + calinski_harabasz_score: Mapped[float | None] = mapped_column(Float, nullable=True) + stability_score: Mapped[float | None] = mapped_column(Float, nullable=True) + interpretability_score: Mapped[float | None] = mapped_column(Float, nullable=True) + +class ClusterResult(Base): + __tablename__ = 'cluster_results' + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + experiment_id: Mapped[str] = mapped_column(ForeignKey('experiments.id'), index=True, nullable=False) + row_index: Mapped[int] = mapped_column(Integer, nullable=False) + cluster_label: Mapped[int] = mapped_column(Integer, nullable=False, index=True) + is_outlier: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False) + pca_x: Mapped[float | None] = mapped_column(Float, nullable=True) + pca_y: Mapped[float | None] = mapped_column(Float, nullable=True) + umap_x: Mapped[float | None] = mapped_column(Float, nullable=True) + umap_y: Mapped[float | None] = mapped_column(Float, nullable=True) + +class ClusterSummary(Base): + __tablename__ = 'cluster_summaries' + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + experiment_id: Mapped[str] = mapped_column(ForeignKey('experiments.id'), index=True, nullable=False) + cluster_label: Mapped[int] = mapped_column(Integer, nullable=False, index=True) + size: Mapped[int] = mapped_column(Integer, nullable=False) + summary_json: Mapped[str] = mapped_column(Text, default='{}', nullable=False) + description_text: Mapped[str] = mapped_column(Text, nullable=False) diff --git a/backend/app/models/dataset.py b/backend/app/models/dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..d4dd96387cf328e1370ce3e375f9a60c85ee035b --- /dev/null +++ b/backend/app/models/dataset.py @@ -0,0 +1,16 @@ +from datetime import datetime +from sqlalchemy import DateTime, Integer, String, Text +from sqlalchemy.orm import Mapped, mapped_column +from app.db import Base + +class Dataset(Base): + __tablename__ = 'datasets' + id: Mapped[str] = mapped_column(String(32), primary_key=True) + name: Mapped[str] = mapped_column(String(255), nullable=False) + file_path: Mapped[str] = mapped_column(Text, nullable=False) + sha256: Mapped[str] = mapped_column(String(64), nullable=False, index=True) + uploaded_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow, nullable=False) + row_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False) + column_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False) + schema_json: Mapped[str] = mapped_column(Text, default='{}', nullable=False) + profile_json: Mapped[str] = mapped_column(Text, default='{}', nullable=False) diff --git a/backend/app/models/experiment.py b/backend/app/models/experiment.py new file mode 100644 index 0000000000000000000000000000000000000000..2f305fb8c4a80a5c1dfea9344915b4ebd804d960 --- /dev/null +++ b/backend/app/models/experiment.py @@ -0,0 +1,17 @@ +from datetime import datetime +from sqlalchemy import DateTime, ForeignKey, Integer, String, Text +from sqlalchemy.orm import Mapped, mapped_column +from app.db import Base + +class Experiment(Base): + __tablename__ = 'experiments' + id: Mapped[str] = mapped_column(String(32), primary_key=True) + dataset_id: Mapped[str] = mapped_column(ForeignKey('datasets.id'), index=True, nullable=False) + name: Mapped[str] = mapped_column(String(255), nullable=False) + algorithm: Mapped[str] = mapped_column(String(64), nullable=False, index=True) + status: Mapped[str] = mapped_column(String(32), nullable=False, default='pending') + config_json: Mapped[str] = mapped_column(Text, default='{}', nullable=False) + started_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow, nullable=False) + completed_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True) + runtime_ms: Mapped[int | None] = mapped_column(Integer, nullable=True) + error_message: Mapped[str | None] = mapped_column(Text, nullable=True) diff --git a/backend/app/repositories/__init__.py b/backend/app/repositories/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/backend/app/repositories/artifact_repo.py b/backend/app/repositories/artifact_repo.py new file mode 100644 index 0000000000000000000000000000000000000000..906565a98d1f842ded51cc937c438d8b82f16875 --- /dev/null +++ b/backend/app/repositories/artifact_repo.py @@ -0,0 +1,16 @@ +from sqlalchemy import select +from sqlalchemy.orm import Session +from app.models.artifact import Artifact + +class ArtifactRepository: + def __init__(self, session: Session): + self.session = session + + def create(self, artifact: Artifact) -> Artifact: + self.session.add(artifact) + self.session.flush() + return artifact + + def list_for_experiment(self, experiment_id: str) -> list[Artifact]: + stmt = select(Artifact).where(Artifact.experiment_id == experiment_id).order_by(Artifact.created_at.desc()) + return list(self.session.scalars(stmt)) diff --git a/backend/app/repositories/cluster_repo.py b/backend/app/repositories/cluster_repo.py new file mode 100644 index 0000000000000000000000000000000000000000..0de41e3b9cc45749225321dab65fc6c764cb0576 --- /dev/null +++ b/backend/app/repositories/cluster_repo.py @@ -0,0 +1,41 @@ +from sqlalchemy import delete, select +from sqlalchemy.orm import Session +from app.models.cluster_result import ClusterResult, ClusterSummary, ExperimentMetric + +class ClusterRepository: + def __init__(self, session: Session): + self.session = session + + def replace_results(self, experiment_id: str, results: list[ClusterResult]) -> None: + self.session.execute(delete(ClusterResult).where(ClusterResult.experiment_id == experiment_id)) + self.session.add_all(results) + self.session.flush() + + def replace_summaries(self, experiment_id: str, summaries: list[ClusterSummary]) -> None: + self.session.execute(delete(ClusterSummary).where(ClusterSummary.experiment_id == experiment_id)) + self.session.add_all(summaries) + self.session.flush() + + def replace_metric(self, experiment_id: str, metric: ExperimentMetric) -> None: + self.session.execute(delete(ExperimentMetric).where(ExperimentMetric.experiment_id == experiment_id)) + self.session.add(metric) + self.session.flush() + + def get_results(self, experiment_id: str) -> list[ClusterResult]: + stmt = select(ClusterResult).where(ClusterResult.experiment_id == experiment_id).order_by(ClusterResult.row_index.asc()) + return list(self.session.scalars(stmt)) + + def get_metric(self, experiment_id: str) -> ExperimentMetric | None: + stmt = select(ExperimentMetric).where(ExperimentMetric.experiment_id == experiment_id) + return self.session.scalar(stmt) + + def get_summaries(self, experiment_id: str) -> list[ClusterSummary]: + stmt = select(ClusterSummary).where(ClusterSummary.experiment_id == experiment_id).order_by(ClusterSummary.cluster_label.asc()) + return list(self.session.scalars(stmt)) + + def get_summary(self, experiment_id: str, cluster_label: int) -> ClusterSummary | None: + stmt = select(ClusterSummary).where( + ClusterSummary.experiment_id == experiment_id, + ClusterSummary.cluster_label == cluster_label, + ) + return self.session.scalar(stmt) diff --git a/backend/app/repositories/dataset_repo.py b/backend/app/repositories/dataset_repo.py new file mode 100644 index 0000000000000000000000000000000000000000..e811d1f3bd34d4d3c0cbc0d30975a243984047d4 --- /dev/null +++ b/backend/app/repositories/dataset_repo.py @@ -0,0 +1,27 @@ +import json +from sqlalchemy import select +from sqlalchemy.orm import Session +from app.models.dataset import Dataset + +class DatasetRepository: + def __init__(self, session: Session): + self.session = session + + def create(self, dataset: Dataset) -> Dataset: + self.session.add(dataset) + self.session.flush() + return dataset + + def get(self, dataset_id: str) -> Dataset | None: + return self.session.get(Dataset, dataset_id) + + def list_all(self) -> list[Dataset]: + return list(self.session.scalars(select(Dataset).order_by(Dataset.uploaded_at.desc()))) + + def update_profile(self, dataset_id: str, schema_json: dict, profile_json: dict) -> None: + dataset = self.get(dataset_id) + if not dataset: + raise ValueError('Dataset not found') + dataset.schema_json = json.dumps(schema_json) + dataset.profile_json = json.dumps(profile_json) + self.session.flush() diff --git a/backend/app/repositories/experiment_repo.py b/backend/app/repositories/experiment_repo.py new file mode 100644 index 0000000000000000000000000000000000000000..31b97996aa656ab348ed05e1f7b5efe69139cba7 --- /dev/null +++ b/backend/app/repositories/experiment_repo.py @@ -0,0 +1,28 @@ +from sqlalchemy import select +from sqlalchemy.orm import Session +from app.models.experiment import Experiment + +class ExperimentRepository: + def __init__(self, session: Session): + self.session = session + + def create(self, experiment: Experiment) -> Experiment: + self.session.add(experiment) + self.session.flush() + return experiment + + def get(self, experiment_id: str) -> Experiment | None: + return self.session.get(Experiment, experiment_id) + + def list_all(self) -> list[Experiment]: + return list(self.session.scalars(select(Experiment).order_by(Experiment.started_at.desc()))) + + def update_status(self, experiment_id: str, status: str, runtime_ms: int | None = None, error_message: str | None = None, completed_at=None) -> None: + exp = self.get(experiment_id) + if not exp: + raise ValueError('Experiment not found') + exp.status = status + exp.runtime_ms = runtime_ms + exp.error_message = error_message + exp.completed_at = completed_at + self.session.flush() diff --git a/backend/app/schemas/__init__.py b/backend/app/schemas/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/backend/app/schemas/cluster.py b/backend/app/schemas/cluster.py new file mode 100644 index 0000000000000000000000000000000000000000..ec10002eef02aa024035de5d7dd44abf40ca0edc --- /dev/null +++ b/backend/app/schemas/cluster.py @@ -0,0 +1,13 @@ +from typing import Any, Dict, List +from pydantic import BaseModel + +class ClusterMember(BaseModel): + row_index: int + values: Dict[str, Any] + +class ClusterDetailResponse(BaseModel): + experiment_id: str + cluster_label: int + members: List[ClusterMember] + summary: Dict[str, Any] + description_text: str diff --git a/backend/app/schemas/compare.py b/backend/app/schemas/compare.py new file mode 100644 index 0000000000000000000000000000000000000000..d80a1975a27ea5f1eb3112c8d8cc5a539b0f1146 --- /dev/null +++ b/backend/app/schemas/compare.py @@ -0,0 +1,17 @@ +from typing import List +from pydantic import BaseModel + +class CompareRunItem(BaseModel): + experiment_id: str + name: str + algorithm: str + cluster_count: int + noise_ratio: float + silhouette_score: float | None = None + davies_bouldin_score: float | None = None + calinski_harabasz_score: float | None = None + stability_score: float | None = None + interpretability_score: float | None = None + +class CompareResponse(BaseModel): + runs: List[CompareRunItem] diff --git a/backend/app/schemas/dataset.py b/backend/app/schemas/dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..33edce1cc0926500e93477f9fa9ee7230e99df21 --- /dev/null +++ b/backend/app/schemas/dataset.py @@ -0,0 +1,23 @@ +from typing import Any, List +from pydantic import BaseModel + +class DatasetUploadResponse(BaseModel): + dataset_id: str + name: str + row_count: int + column_count: int + +class ColumnProfile(BaseModel): + name: str + inferred_type: str + missing_pct: float + cardinality: int + sample_values: List[Any] + +class DatasetProfileResponse(BaseModel): + dataset_id: str + columns: List[ColumnProfile] + suggested_feature_columns: List[str] + recommended_algorithms: List[str] + row_count: int + column_count: int diff --git a/backend/app/schemas/experiment.py b/backend/app/schemas/experiment.py new file mode 100644 index 0000000000000000000000000000000000000000..9b2f30753011f98c1907bac54db9e8c2b9783c85 --- /dev/null +++ b/backend/app/schemas/experiment.py @@ -0,0 +1,59 @@ +from typing import Any, Dict, List, Optional +from pydantic import BaseModel, Field + +class PreprocessingConfig(BaseModel): + numeric_scaler: str = 'standard' + categorical_encoder: str = 'onehot' + dimensionality_reduction: str = 'pca' + +class ExperimentRunRequest(BaseModel): + dataset_id: str + name: str = Field(default='Untitled Experiment') + algorithm: str + feature_columns: List[str] + preprocessing: PreprocessingConfig = PreprocessingConfig() + algorithm_params: Dict[str, Any] = Field(default_factory=dict) + outlier_detection_enabled: bool = True + +class ExperimentRunResponse(BaseModel): + experiment_id: str + status: str + +class MetricResponse(BaseModel): + cluster_count: int + noise_ratio: float + silhouette_score: Optional[float] = None + davies_bouldin_score: Optional[float] = None + calinski_harabasz_score: Optional[float] = None + stability_score: Optional[float] = None + interpretability_score: Optional[float] = None + +class ScatterPoint(BaseModel): + row_index: int + cluster_label: int + is_outlier: bool + x: float + y: float + +class ClusterSummaryResponse(BaseModel): + cluster_label: int + size: int + description_text: str + summary_json: Dict[str, Any] + +class ExperimentResultResponse(BaseModel): + experiment_id: str + algorithm: str + metrics: MetricResponse + scatter_points: List[ScatterPoint] + cluster_summaries: List[ClusterSummaryResponse] + status: str + +class ExperimentListItem(BaseModel): + experiment_id: str + dataset_id: str + name: str + algorithm: str + status: str + started_at: str + completed_at: Optional[str] = None diff --git a/backend/app/schemas/export.py b/backend/app/schemas/export.py new file mode 100644 index 0000000000000000000000000000000000000000..b042d0d59dac19b273bdc9412a8ac41dc03187d1 --- /dev/null +++ b/backend/app/schemas/export.py @@ -0,0 +1,5 @@ +from pydantic import BaseModel + +class ExportResponse(BaseModel): + experiment_id: str + download_path: str diff --git a/backend/app/schemas/health.py b/backend/app/schemas/health.py new file mode 100644 index 0000000000000000000000000000000000000000..9e5f219724512af25e26bc882116cb4c011c4545 --- /dev/null +++ b/backend/app/schemas/health.py @@ -0,0 +1,5 @@ +from pydantic import BaseModel + +class HealthResponse(BaseModel): + ok: bool + service: str diff --git a/backend/app/services/__init__.py b/backend/app/services/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/backend/app/services/clustering_service.py b/backend/app/services/clustering_service.py new file mode 100644 index 0000000000000000000000000000000000000000..18fabeb6c60ad4b9a58ea8f824e0cfb7da5df719 --- /dev/null +++ b/backend/app/services/clustering_service.py @@ -0,0 +1,24 @@ +from importlib import import_module +from app.utils.validators import SUPPORTED_ALGORITHMS + +class ClusteringService: + def get_runner(self, algorithm: str): + if algorithm not in SUPPORTED_ALGORITHMS: + raise ValueError(f'Unsupported algorithm: {algorithm}') + module_name = { + 'kmeans': 'kmeans_runner', + 'minibatch_kmeans': 'minibatch_kmeans_runner', + 'agglomerative': 'agglomerative_runner', + 'dbscan': 'dbscan_runner', + 'hdbscan': 'hdbscan_runner', + 'gmm': 'gmm_runner', + 'spectral': 'spectral_runner', + 'birch': 'birch_runner', + }[algorithm] + module = import_module(f'app.ml.algorithms.{module_name}') + return module.Runner() + + def run(self, X, algorithm: str, algorithm_params: dict): + runner = self.get_runner(algorithm) + result = runner.fit_predict(X, algorithm_params) + return runner, result diff --git a/backend/app/services/comparison_service.py b/backend/app/services/comparison_service.py new file mode 100644 index 0000000000000000000000000000000000000000..fbf3271c1fd617952482d819d61985a38725f3cd --- /dev/null +++ b/backend/app/services/comparison_service.py @@ -0,0 +1,21 @@ +class ComparisonService: + def build_compare_payload(self, experiment_repo, cluster_repo, experiment_ids: list[str]): + runs = [] + for experiment_id in experiment_ids: + exp = experiment_repo.get(experiment_id) + metric = cluster_repo.get_metric(experiment_id) + if not exp or not metric: + continue + runs.append({ + 'experiment_id': exp.id, + 'name': exp.name, + 'algorithm': exp.algorithm, + 'cluster_count': metric.cluster_count, + 'noise_ratio': metric.noise_ratio, + 'silhouette_score': metric.silhouette_score, + 'davies_bouldin_score': metric.davies_bouldin_score, + 'calinski_harabasz_score': metric.calinski_harabasz_score, + 'stability_score': metric.stability_score, + 'interpretability_score': metric.interpretability_score, + }) + return {'runs': runs} diff --git a/backend/app/services/dataset_service.py b/backend/app/services/dataset_service.py new file mode 100644 index 0000000000000000000000000000000000000000..08be86598289782f37f3f821a33640084026548d --- /dev/null +++ b/backend/app/services/dataset_service.py @@ -0,0 +1,43 @@ +import io +from pathlib import Path +import pandas as pd +from fastapi import UploadFile +from app.models.dataset import Dataset +from app.repositories.dataset_repo import DatasetRepository +from app.services.storage_service import StorageService +from app.utils.file_hash import sha256_file +from app.utils.ids import generate_id + +class DatasetService: + def __init__(self, repo: DatasetRepository): + self.repo = repo + self.storage = StorageService() + + def load_dataframe(self, dataset: Dataset) -> pd.DataFrame: + path = Path(dataset.file_path) + if path.suffix.lower() in {'.xlsx', '.xls'}: + return pd.read_excel(path) + return pd.read_csv(path) + + async def save_upload(self, upload: UploadFile) -> Dataset: + dataset_id = generate_id('ds') + suffix = Path(upload.filename or 'dataset.csv').suffix or '.csv' + file_path = self.storage.uploads_dir() / f"{dataset_id}{suffix}" + content = await upload.read() + with open(file_path, 'wb') as f: + f.write(content) + if suffix.lower() in {'.xlsx', '.xls'}: + df = pd.read_excel(io.BytesIO(content)) + else: + df = pd.read_csv(io.BytesIO(content)) + dataset = Dataset( + id=dataset_id, + name=upload.filename or f'{dataset_id}.csv', + file_path=str(file_path), + sha256=sha256_file(file_path), + row_count=int(len(df)), + column_count=int(len(df.columns)), + schema_json='{}', + profile_json='{}', + ) + return self.repo.create(dataset) diff --git a/backend/app/services/evaluation_service.py b/backend/app/services/evaluation_service.py new file mode 100644 index 0000000000000000000000000000000000000000..c59013dfaed535fd2243056a9331a498f5872084 --- /dev/null +++ b/backend/app/services/evaluation_service.py @@ -0,0 +1,10 @@ +from app.ml.metrics.clustering_metrics import compute_metrics +from app.ml.metrics.interpretability import compute_interpretability_score +from app.ml.metrics.stability import compute_stability_score + +class EvaluationService: + def evaluate(self, X, df, labels: list[int], feature_columns: list[str]) -> dict: + metrics = compute_metrics(X, labels) + metrics['stability_score'] = compute_stability_score(labels) + metrics['interpretability_score'] = compute_interpretability_score(df, labels, feature_columns) + return metrics diff --git a/backend/app/services/explainability_service.py b/backend/app/services/explainability_service.py new file mode 100644 index 0000000000000000000000000000000000000000..739bec1e922f6b249feb710f850863647f64bd8d --- /dev/null +++ b/backend/app/services/explainability_service.py @@ -0,0 +1,23 @@ +import json +from app.ml.explain.cluster_profiles import build_cluster_profiles +from app.ml.explain.feature_lift import compute_feature_lift +from app.ml.explain.summary_writer import build_cluster_description +from app.models.cluster_result import ClusterSummary + +class ExplainabilityService: + def build_summaries(self, experiment_id: str, df, labels: list[int], feature_columns: list[str]) -> list[ClusterSummary]: + profiles = build_cluster_profiles(df, labels, feature_columns) + lifts = compute_feature_lift(df, labels, feature_columns) + summaries = [] + for cluster_label, profile in sorted(profiles.items(), key=lambda x: x[0]): + lift_info = lifts.get(cluster_label, {}) + summary_json = {'profile': profile, 'feature_lift': lift_info} + description = build_cluster_description(cluster_label, profile, lift_info) + summaries.append(ClusterSummary( + experiment_id=experiment_id, + cluster_label=int(cluster_label), + size=int(profile['size']), + summary_json=json.dumps(summary_json), + description_text=description, + )) + return summaries diff --git a/backend/app/services/export_service.py b/backend/app/services/export_service.py new file mode 100644 index 0000000000000000000000000000000000000000..162a83c9a1f61defff294a87950e3a9fdccf9f70 --- /dev/null +++ b/backend/app/services/export_service.py @@ -0,0 +1,32 @@ +from pathlib import Path +import pandas as pd +from app.models.artifact import Artifact +from app.services.storage_service import StorageService + +class ExportService: + def __init__(self, dataset_repo, cluster_repo, artifact_repo): + self.dataset_repo = dataset_repo + self.cluster_repo = cluster_repo + self.artifact_repo = artifact_repo + self.storage = StorageService() + + def export_csv(self, experiment_id: str, dataset_id: str) -> str: + dataset = self.dataset_repo.get(dataset_id) + if not dataset: + raise ValueError('Dataset not found') + results = self.cluster_repo.get_results(experiment_id) + df = pd.read_csv(dataset.file_path) if dataset.file_path.lower().endswith('.csv') else pd.read_excel(dataset.file_path) + enrich = pd.DataFrame({ + 'row_index': [r.row_index for r in results], + 'cluster_label': [r.cluster_label for r in results], + 'is_outlier': [r.is_outlier for r in results], + 'pca_x': [r.pca_x for r in results], + 'pca_y': [r.pca_y for r in results], + 'umap_x': [r.umap_x for r in results], + 'umap_y': [r.umap_y for r in results], + }) + merged = df.reset_index().rename(columns={'index': 'row_index'}).merge(enrich, on='row_index', how='left') + out_path = Path(self.storage.artifacts_dir()) / f'{experiment_id}_labeled.csv' + merged.to_csv(out_path, index=False) + self.artifact_repo.create(Artifact(experiment_id=experiment_id, artifact_type='labeled_csv', path=str(out_path))) + return str(out_path) diff --git a/backend/app/services/outlier_service.py b/backend/app/services/outlier_service.py new file mode 100644 index 0000000000000000000000000000000000000000..ab34a42bc749c98f499d4496ccb4ef391dbb10da --- /dev/null +++ b/backend/app/services/outlier_service.py @@ -0,0 +1,11 @@ +from app.ml.outliers.centroid_distance import detect_centroid_outliers +from app.ml.outliers.density_noise import detect_noise_outliers +from app.ml.outliers.isolation_forest import detect_isolation_outliers + +class OutlierService: + def detect(self, X, labels: list[int], algorithm: str) -> list[bool]: + if algorithm in {'dbscan', 'hdbscan'}: + return detect_noise_outliers(labels) + centroid_flags = detect_centroid_outliers(X, labels) + isolation_flags = detect_isolation_outliers(X) + return [bool(a or b) for a, b in zip(centroid_flags, isolation_flags)] diff --git a/backend/app/services/preprocessing_service.py b/backend/app/services/preprocessing_service.py new file mode 100644 index 0000000000000000000000000000000000000000..88cd525b2a640cb07b42b92fa01ef801ce52d0c3 --- /dev/null +++ b/backend/app/services/preprocessing_service.py @@ -0,0 +1,10 @@ +from app.ml.preprocess.matrix_builder import build_feature_matrix + +class PreprocessingService: + def transform(self, df, feature_columns: list[str], preprocessing: dict): + return build_feature_matrix( + df=df, + feature_columns=feature_columns, + numeric_scaler=preprocessing.get('numeric_scaler', 'standard'), + categorical_encoder=preprocessing.get('categorical_encoder', 'onehot'), + ) diff --git a/backend/app/services/profiling_service.py b/backend/app/services/profiling_service.py new file mode 100644 index 0000000000000000000000000000000000000000..f2ed1c4cc28a31ecd214e396c2de40a6bd5ed256 --- /dev/null +++ b/backend/app/services/profiling_service.py @@ -0,0 +1,35 @@ +from app.ml.preprocess.column_inference import infer_columns + +class ProfilingService: + def profile(self, df): + inferred = infer_columns(df) + columns = [{ + 'name': c.name, + 'inferred_type': c.inferred_type, + 'missing_pct': c.missing_pct, + 'cardinality': c.cardinality, + 'sample_values': c.sample_values, + } for c in inferred] + feature_columns = [c['name'] for c in columns if c['inferred_type'] in {'numeric', 'boolean', 'categorical'}] + recommended = self.recommend_algorithms(df, columns) + return { + 'columns': columns, + 'suggested_feature_columns': feature_columns[:20], + 'recommended_algorithms': recommended, + 'row_count': int(len(df)), + 'column_count': int(len(df.columns)), + } + + def recommend_algorithms(self, df, columns): + row_count = len(df) + numeric_count = sum(1 for c in columns if c['inferred_type'] in {'numeric', 'boolean'}) + cat_count = sum(1 for c in columns if c['inferred_type'] == 'categorical') + recs = [] + if row_count > 5000: + recs.extend(['minibatch_kmeans', 'birch']) + else: + recs.append('kmeans') + if cat_count > numeric_count: + recs.append('agglomerative') + recs.extend(['dbscan', 'gmm', 'spectral']) + return list(dict.fromkeys(recs))[:5] diff --git a/backend/app/services/run_service.py b/backend/app/services/run_service.py new file mode 100644 index 0000000000000000000000000000000000000000..eb0627590fc1ded875d786edc520887073a1fe6c --- /dev/null +++ b/backend/app/services/run_service.py @@ -0,0 +1,92 @@ +import json +import time +import pandas as pd +from app.models.cluster_result import ClusterResult, ExperimentMetric +from app.models.experiment import Experiment +from app.services.preprocessing_service import PreprocessingService +from app.services.clustering_service import ClusteringService +from app.services.evaluation_service import EvaluationService +from app.services.visualization_service import VisualizationService +from app.services.explainability_service import ExplainabilityService +from app.services.outlier_service import OutlierService +from app.utils.ids import generate_id +from app.utils.time_utils import utc_now + +class RunService: + def __init__(self, dataset_repo, experiment_repo, cluster_repo): + self.dataset_repo = dataset_repo + self.experiment_repo = experiment_repo + self.cluster_repo = cluster_repo + self.preprocess = PreprocessingService() + self.clusterer = ClusteringService() + self.evaluate_service = EvaluationService() + self.visualize = VisualizationService() + self.explain = ExplainabilityService() + self.outliers = OutlierService() + + def run(self, payload: dict): + dataset = self.dataset_repo.get(payload['dataset_id']) + if not dataset: + raise ValueError('Dataset not found') + experiment = Experiment( + id=generate_id('exp'), + dataset_id=dataset.id, + name=payload['name'], + algorithm=payload['algorithm'], + status='running', + config_json=json.dumps(payload), + started_at=utc_now(), + ) + self.experiment_repo.create(experiment) + start = time.perf_counter() + try: + df = pd.read_csv(dataset.file_path) if dataset.file_path.lower().endswith('.csv') else pd.read_excel(dataset.file_path) + X, feature_names, metadata = self.preprocess.transform( + df=df, + feature_columns=payload['feature_columns'], + preprocessing=payload['preprocessing'], + ) + runner, run_output = self.clusterer.run( + X=X, + algorithm=payload['algorithm'], + algorithm_params=payload['algorithm_params'], + ) + labels = run_output.labels + coords = self.visualize.build_coords(X) + outlier_flags = self.outliers.detect(X, labels, payload['algorithm']) if payload.get('outlier_detection_enabled', True) else [False] * len(labels) + metrics = self.evaluate_service.evaluate(X, df, labels, payload['feature_columns']) + + rows = [] + for i, label in enumerate(labels): + rows.append(ClusterResult( + experiment_id=experiment.id, + row_index=i, + cluster_label=int(label), + is_outlier=bool(outlier_flags[i]), + pca_x=float(coords['pca_x'][i]), + pca_y=float(coords['pca_y'][i]), + umap_x=float(coords['umap_x'][i]), + umap_y=float(coords['umap_y'][i]), + )) + self.cluster_repo.replace_results(experiment.id, rows) + metric_row = ExperimentMetric( + experiment_id=experiment.id, + cluster_count=int(metrics['cluster_count']), + noise_ratio=float(metrics['noise_ratio']), + silhouette_score=metrics['silhouette_score'], + davies_bouldin_score=metrics['davies_bouldin_score'], + calinski_harabasz_score=metrics['calinski_harabasz_score'], + stability_score=metrics['stability_score'], + interpretability_score=metrics['interpretability_score'], + ) + self.cluster_repo.replace_metric(experiment.id, metric_row) + summaries = self.explain.build_summaries(experiment.id, df, labels, payload['feature_columns']) + self.cluster_repo.replace_summaries(experiment.id, summaries) + + runtime_ms = int((time.perf_counter() - start) * 1000) + self.experiment_repo.update_status(experiment.id, 'completed', runtime_ms=runtime_ms, completed_at=utc_now()) + except Exception as exc: + runtime_ms = int((time.perf_counter() - start) * 1000) + self.experiment_repo.update_status(experiment.id, 'failed', runtime_ms=runtime_ms, error_message=str(exc), completed_at=utc_now()) + raise + return experiment diff --git a/backend/app/services/storage_service.py b/backend/app/services/storage_service.py new file mode 100644 index 0000000000000000000000000000000000000000..85b31873b54bf3b7f3ba249c1c8f7a512bc6a095 --- /dev/null +++ b/backend/app/services/storage_service.py @@ -0,0 +1,16 @@ +from pathlib import Path +from app.config import get_settings + +class StorageService: + def __init__(self): + self.settings = get_settings() + + def uploads_dir(self) -> Path: + path = Path(self.settings.upload_dir) + path.mkdir(parents=True, exist_ok=True) + return path + + def artifacts_dir(self) -> Path: + path = Path(self.settings.artifact_dir) + path.mkdir(parents=True, exist_ok=True) + return path diff --git a/backend/app/services/visualization_service.py b/backend/app/services/visualization_service.py new file mode 100644 index 0000000000000000000000000000000000000000..0cc33a8e4ff08155665295ce7d3edc4b3352a640 --- /dev/null +++ b/backend/app/services/visualization_service.py @@ -0,0 +1,12 @@ +from app.ml.preprocess.reducers import pca_2d, umap_2d + +class VisualizationService: + def build_coords(self, X): + pca_x, pca_y = pca_2d(X) + umap_x, umap_y = umap_2d(X) + return { + 'pca_x': pca_x.tolist(), + 'pca_y': pca_y.tolist(), + 'umap_x': umap_x.tolist(), + 'umap_y': umap_y.tolist(), + } diff --git a/backend/app/utils/__init__.py b/backend/app/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/backend/app/utils/file_hash.py b/backend/app/utils/file_hash.py new file mode 100644 index 0000000000000000000000000000000000000000..31dc4c7aa2a0b39292bd0181d87eedf7e3ad28e5 --- /dev/null +++ b/backend/app/utils/file_hash.py @@ -0,0 +1,9 @@ +import hashlib +from pathlib import Path + +def sha256_file(path: str | Path) -> str: + h = hashlib.sha256() + with open(path, 'rb') as f: + for chunk in iter(lambda: f.read(8192), b''): + h.update(chunk) + return h.hexdigest() diff --git a/backend/app/utils/ids.py b/backend/app/utils/ids.py new file mode 100644 index 0000000000000000000000000000000000000000..89653419348aebf77c54da1c41b08b585adbf722 --- /dev/null +++ b/backend/app/utils/ids.py @@ -0,0 +1,4 @@ +import secrets + +def generate_id(prefix: str) -> str: + return f"{prefix}_{secrets.token_hex(8)}" diff --git a/backend/app/utils/time_utils.py b/backend/app/utils/time_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..233f02321de2ae961593931806bb1dfdeb919534 --- /dev/null +++ b/backend/app/utils/time_utils.py @@ -0,0 +1,4 @@ +from datetime import datetime + +def utc_now() -> datetime: + return datetime.utcnow() diff --git a/backend/app/utils/validators.py b/backend/app/utils/validators.py new file mode 100644 index 0000000000000000000000000000000000000000..fbc34f038e8a2ac8c05ff9ed024c34dbf26561c5 --- /dev/null +++ b/backend/app/utils/validators.py @@ -0,0 +1,10 @@ +SUPPORTED_ALGORITHMS = { + 'kmeans', + 'minibatch_kmeans', + 'agglomerative', + 'dbscan', + 'hdbscan', + 'gmm', + 'spectral', + 'birch', +} diff --git a/backend/requirements.txt b/backend/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..27b837977e98d595cb6069f6ae8ade259dda189a --- /dev/null +++ b/backend/requirements.txt @@ -0,0 +1,13 @@ +fastapi==0.115.12 +uvicorn[standard]==0.34.2 +sqlalchemy==2.0.40 +pydantic==2.11.3 +pydantic-settings==2.8.1 +python-multipart==0.0.20 +pandas==2.2.3 +numpy==2.2.5 +scikit-learn==1.6.1 +joblib==1.4.2 +umap-learn==0.5.7 +hdbscan==0.8.40 +openpyxl==3.1.5 diff --git a/backend/tests/test_health.py b/backend/tests/test_health.py new file mode 100644 index 0000000000000000000000000000000000000000..fd66ec0efba24fc13dfb7502e6f0037197525fc9 --- /dev/null +++ b/backend/tests/test_health.py @@ -0,0 +1,8 @@ +from fastapi.testclient import TestClient +from app.main import app + +def test_health(): + client = TestClient(app) + resp = client.get('/api/health') + assert resp.status_code == 200 + assert resp.json()['ok'] is True diff --git a/data_samples/access_demo.csv b/data_samples/access_demo.csv new file mode 100644 index 0000000000000000000000000000000000000000..a7f8924638291a91b87aec4bcfdb18f35b70cd3f --- /dev/null +++ b/data_samples/access_demo.csv @@ -0,0 +1,801 @@ +user_id,department,title,location,application_name,entitlement_count,privileged_access_flag,mfa_enabled,risk_score,peer_access_overlap_score,last_login_days_ago,granted_count_6m,revoked_count_6m,review_count_6m,sensitive_access_flag +user_00000,Support,Analyst,NY,IAM,13,0,1,0.689,0.708,139,2,6,1,0 +user_00001,Finance,Analyst,SF,Payroll,5,0,1,0.62,0.609,56,14,4,1,0 +user_00002,HR,Admin,London,HCM,8,0,1,0.387,0.458,67,1,7,9,0 +user_00003,Finance,Engineer,NY,IAM,16,0,1,0.256,0.696,74,2,3,2,1 +user_00004,Sales,Director,London,HCM,20,1,1,0.786,0.726,174,20,1,10,0 +user_00005,Support,Manager,Bangalore,IAM,12,0,1,0.316,0.894,176,17,3,6,0 +user_00006,Finance,Manager,NY,DataLake,13,0,1,0.954,0.857,80,6,7,7,1 +user_00007,Support,Engineer,SF,HCM,9,0,1,0.531,0.76,109,18,6,6,0 +user_00008,HR,Manager,Bangalore,ERP,8,0,1,0.834,0.517,16,12,6,10,1 +user_00009,Sales,Contractor,Austin,Payroll,5,0,1,0.645,0.603,68,20,5,2,0 +user_00010,Engineering,Engineer,SF,DataLake,5,0,1,0.71,0.581,27,20,4,9,0 +user_00011,Legal,Manager,SF,CRM,5,0,1,0.216,0.897,78,7,0,4,1 +user_00012,Legal,Analyst,NY,ERP,9,0,1,0.957,0.324,135,19,6,4,1 +user_00013,Legal,Admin,SF,IAM,28,1,1,0.711,0.386,16,10,0,10,1 +user_00014,Legal,Manager,Bangalore,ERP,6,0,0,0.125,0.448,131,7,4,8,0 +user_00015,HR,Contractor,SF,Payroll,20,0,1,0.846,0.343,24,13,5,7,1 +user_00016,Sales,Engineer,NY,IAM,8,0,1,0.371,0.846,63,6,3,9,0 +user_00017,Sales,Manager,London,CRM,13,0,1,0.839,0.532,140,3,0,9,0 +user_00018,Finance,Analyst,SF,DataLake,20,0,1,0.822,0.323,0,12,4,8,0 +user_00019,Engineering,Engineer,Bangalore,IAM,20,0,1,0.274,0.244,138,1,5,1,0 +user_00020,Finance,Contractor,London,Payroll,10,0,1,0.264,0.339,152,2,3,7,1 +user_00021,Finance,Contractor,SF,ERP,7,0,1,0.652,0.437,66,6,5,4,1 +user_00022,Engineering,Engineer,SF,IAM,14,0,1,0.701,0.254,117,19,1,2,0 +user_00023,Legal,Manager,Bangalore,HCM,7,0,1,0.226,0.825,180,9,8,1,0 +user_00024,Support,Contractor,Austin,IAM,8,0,1,0.812,0.757,39,8,4,10,0 +user_00025,HR,Admin,Austin,HCM,20,0,1,0.877,0.269,108,8,0,1,1 +user_00026,Engineering,Manager,Austin,CRM,19,0,1,0.549,0.284,176,4,8,1,0 +user_00027,Engineering,Contractor,Bangalore,CRM,18,0,0,0.392,0.9,10,11,3,4,0 +user_00028,Support,Analyst,Austin,Payroll,18,0,1,0.841,0.378,41,5,6,1,0 +user_00029,HR,Admin,Austin,IAM,21,1,1,0.911,0.487,9,15,3,4,1 +user_00030,Sales,Director,Austin,CRM,17,1,1,0.569,0.409,17,8,5,9,0 +user_00031,Sales,Admin,Bangalore,ERP,8,0,1,0.87,0.399,27,19,6,6,0 +user_00032,Support,Director,London,Payroll,19,1,1,0.402,0.233,111,0,8,9,0 +user_00033,Support,Admin,SF,ERP,15,0,1,0.778,0.74,76,16,4,7,0 +user_00034,Engineering,Engineer,Austin,Payroll,9,0,1,0.852,0.708,44,19,4,7,0 +user_00035,Legal,Analyst,Austin,DataLake,15,0,1,0.641,0.583,43,2,4,9,0 +user_00036,Support,Admin,Bangalore,CRM,14,0,1,0.218,0.235,121,19,1,8,0 +user_00037,Sales,Admin,Bangalore,IAM,17,0,1,0.218,0.716,27,13,3,3,0 +user_00038,Support,Contractor,London,ERP,12,0,1,0.741,0.701,143,19,5,8,0 +user_00039,Legal,Admin,Bangalore,DataLake,22,1,1,0.951,0.83,70,16,7,4,1 +user_00040,Engineering,Engineer,NY,IAM,14,0,1,0.356,0.605,35,4,3,7,0 +user_00041,Support,Manager,SF,ERP,18,0,1,0.473,0.247,107,12,0,10,0 +user_00042,Sales,Engineer,NY,HCM,14,0,1,0.863,0.514,139,19,3,8,0 +user_00043,HR,Director,London,IAM,24,1,1,1.086,0.936,136,0,6,10,1 +user_00044,Legal,Admin,NY,DataLake,10,0,1,0.462,0.541,86,12,4,7,1 +user_00045,Engineering,Analyst,London,ERP,6,0,1,0.155,0.918,10,0,3,4,0 +user_00046,Finance,Contractor,SF,DataLake,8,0,1,0.472,0.392,94,5,1,3,0 +user_00047,Engineering,Analyst,Bangalore,ERP,14,0,1,0.867,0.497,50,2,3,2,0 +user_00048,Support,Director,Bangalore,Payroll,16,1,1,0.779,0.252,165,10,0,7,0 +user_00049,Sales,Analyst,London,HCM,19,0,1,0.687,0.924,69,19,8,8,0 +user_00050,Sales,Engineer,Bangalore,HCM,15,0,1,0.169,0.861,62,14,6,6,0 +user_00051,Finance,Engineer,Austin,CRM,16,0,1,0.804,0.726,70,17,0,9,0 +user_00052,HR,Analyst,SF,DataLake,12,0,1,0.593,0.795,23,9,3,7,1 +user_00053,Support,Manager,Austin,IAM,16,0,1,0.375,0.947,140,10,5,8,0 +user_00054,Engineering,Director,Austin,IAM,11,0,1,0.529,0.772,47,6,3,8,0 +user_00055,Engineering,Admin,Bangalore,Payroll,20,1,1,0.487,0.471,77,0,8,3,0 +user_00056,Engineering,Analyst,NY,Payroll,14,0,1,0.702,0.277,3,18,4,8,0 +user_00057,Sales,Engineer,Austin,CRM,6,0,1,0.482,0.817,102,15,1,10,0 +user_00058,Support,Admin,NY,Payroll,14,0,1,0.195,0.773,155,19,3,9,0 +user_00059,Sales,Engineer,London,HCM,18,0,1,0.148,0.92,25,6,3,5,0 +user_00060,Support,Analyst,SF,CRM,10,0,0,0.102,0.538,152,15,4,1,0 +user_00061,HR,Director,Austin,IAM,21,1,1,0.85,0.696,50,13,1,9,1 +user_00062,HR,Admin,SF,ERP,13,1,1,0.826,0.818,73,14,1,8,1 +user_00063,Support,Director,London,Payroll,32,1,0,0.282,0.524,82,19,4,1,0 +user_00064,Finance,Manager,Bangalore,ERP,13,0,0,0.805,0.553,166,14,4,3,1 +user_00065,Legal,Engineer,London,DataLake,16,0,1,0.736,0.843,84,13,7,5,1 +user_00066,Support,Engineer,Bangalore,ERP,19,0,1,0.359,0.928,103,16,0,9,0 +user_00067,Sales,Engineer,NY,CRM,16,0,1,0.6,0.77,52,8,8,3,0 +user_00068,Engineering,Engineer,London,ERP,5,0,1,0.291,0.319,141,0,8,7,0 +user_00069,Finance,Manager,NY,ERP,9,0,1,0.433,0.729,106,15,7,4,1 +user_00070,Sales,Contractor,SF,DataLake,11,0,1,0.803,0.848,70,13,5,9,0 +user_00071,Engineering,Analyst,Austin,IAM,14,0,1,0.628,0.849,114,17,7,6,0 +user_00072,Engineering,Contractor,Bangalore,DataLake,19,0,1,0.885,0.379,98,7,6,1,0 +user_00073,Engineering,Admin,London,DataLake,32,1,1,0.772,0.314,9,4,8,10,0 +user_00074,Engineering,Analyst,London,ERP,19,0,1,0.428,0.691,39,2,7,5,0 +user_00075,Engineering,Contractor,London,IAM,7,0,1,0.787,0.485,81,20,7,9,0 +user_00076,Finance,Contractor,NY,IAM,14,0,1,0.447,0.274,162,3,7,3,0 +user_00077,Support,Director,NY,ERP,14,0,1,0.216,0.598,144,5,2,3,0 +user_00078,Finance,Contractor,London,CRM,20,0,1,0.569,0.39,65,0,7,5,1 +user_00079,Support,Contractor,SF,ERP,19,0,1,0.339,0.922,176,8,7,5,0 +user_00080,HR,Engineer,London,DataLake,16,0,1,0.659,0.216,168,12,4,1,0 +user_00081,Legal,Admin,NY,HCM,26,1,1,0.525,0.343,64,4,1,1,1 +user_00082,Engineering,Engineer,NY,Payroll,16,0,0,0.336,0.761,44,6,2,9,0 +user_00083,Engineering,Contractor,Bangalore,HCM,10,0,1,0.855,0.926,75,10,1,8,0 +user_00084,Finance,Manager,SF,IAM,17,0,1,0.833,0.21,137,3,7,6,1 +user_00085,Support,Admin,Austin,IAM,22,1,1,0.627,0.665,143,10,3,2,0 +user_00086,Support,Engineer,Austin,IAM,18,0,1,0.857,0.428,126,3,1,4,0 +user_00087,Legal,Manager,London,IAM,18,0,1,0.324,0.511,25,15,6,5,1 +user_00088,Finance,Admin,Austin,CRM,16,0,1,0.494,0.876,165,11,0,7,1 +user_00089,Engineering,Manager,NY,DataLake,7,0,1,0.612,0.928,12,10,3,3,0 +user_00090,Legal,Manager,NY,Payroll,11,0,1,0.896,0.446,37,19,0,5,1 +user_00091,HR,Manager,Bangalore,CRM,8,0,1,0.205,0.469,60,18,5,1,0 +user_00092,HR,Director,NY,ERP,7,0,1,0.49,0.645,115,16,3,10,1 +user_00093,Finance,Admin,Bangalore,ERP,6,0,1,0.541,0.281,113,2,1,6,1 +user_00094,Legal,Manager,NY,Payroll,15,0,1,0.524,0.54,154,13,1,2,0 +user_00095,Support,Admin,Bangalore,CRM,30,1,1,0.581,0.821,102,13,1,6,0 +user_00096,Sales,Director,Austin,CRM,20,0,0,0.168,0.524,95,4,8,1,0 +user_00097,Legal,Contractor,Bangalore,ERP,18,0,1,0.853,0.517,13,9,4,6,0 +user_00098,Finance,Contractor,Bangalore,IAM,20,0,1,0.38,0.617,29,8,3,7,0 +user_00099,Legal,Contractor,Bangalore,Payroll,5,0,1,0.864,0.401,46,8,4,6,1 +user_00100,Engineering,Analyst,SF,CRM,17,0,1,0.607,0.223,135,6,6,7,0 +user_00101,Sales,Director,SF,IAM,15,0,1,0.807,0.317,158,1,1,5,0 +user_00102,Sales,Admin,London,DataLake,27,1,1,0.66,0.459,28,9,7,9,0 +user_00103,Support,Director,NY,Payroll,6,0,1,0.856,0.776,65,9,5,2,0 +user_00104,Finance,Engineer,London,DataLake,12,0,1,0.634,0.466,101,1,6,1,0 +user_00105,Sales,Analyst,Austin,Payroll,18,0,1,0.612,0.417,103,0,5,3,0 +user_00106,Legal,Engineer,Austin,ERP,12,0,1,0.519,0.497,79,10,3,6,0 +user_00107,HR,Analyst,Bangalore,Payroll,11,0,1,0.967,0.684,37,7,1,3,1 +user_00108,Engineering,Manager,SF,Payroll,9,0,0,0.861,0.671,118,18,7,10,0 +user_00109,Support,Admin,Bangalore,IAM,15,0,1,0.475,0.674,70,18,0,6,0 +user_00110,Legal,Analyst,Austin,ERP,6,0,1,0.261,0.848,23,19,8,7,1 +user_00111,Sales,Contractor,Bangalore,IAM,6,0,1,0.557,0.341,154,15,8,3,0 +user_00112,Finance,Engineer,NY,HCM,7,0,1,0.231,0.731,112,16,8,10,1 +user_00113,HR,Director,Austin,HCM,21,1,1,0.403,0.271,173,12,4,5,1 +user_00114,Support,Admin,Bangalore,HCM,21,1,1,0.983,0.433,167,12,2,10,0 +user_00115,Support,Analyst,Austin,Payroll,17,0,1,0.751,0.703,175,16,1,7,0 +user_00116,Legal,Director,NY,CRM,15,0,1,0.9,0.316,75,3,8,9,0 +user_00117,Support,Contractor,NY,IAM,15,0,1,0.401,0.322,177,19,2,8,0 +user_00118,Finance,Engineer,Austin,CRM,19,0,1,0.799,0.537,136,7,4,8,1 +user_00119,HR,Director,Bangalore,HCM,30,1,1,1.123,0.813,154,4,4,1,1 +user_00120,Support,Engineer,Austin,Payroll,8,0,1,0.328,0.772,69,14,8,3,0 +user_00121,Sales,Analyst,SF,DataLake,16,0,1,0.417,0.48,98,2,5,4,0 +user_00122,Finance,Director,NY,CRM,14,1,1,1.014,0.722,35,15,7,10,1 +user_00123,Finance,Analyst,NY,CRM,18,0,1,0.29,0.291,61,13,7,2,0 +user_00124,Finance,Engineer,Bangalore,IAM,12,0,1,0.201,0.465,146,13,2,2,1 +user_00125,Legal,Director,NY,Payroll,16,1,1,0.385,0.49,95,8,0,6,1 +user_00126,Finance,Director,SF,Payroll,22,1,0,0.787,0.81,44,14,7,3,1 +user_00127,HR,Analyst,London,CRM,6,0,1,0.352,0.433,101,17,7,5,0 +user_00128,Finance,Admin,SF,ERP,15,0,1,0.494,0.867,112,12,5,3,1 +user_00129,Sales,Admin,London,Payroll,13,0,1,0.163,0.652,46,17,4,6,0 +user_00130,Finance,Analyst,Austin,HCM,19,0,1,0.333,0.533,114,1,5,10,1 +user_00131,Sales,Director,NY,IAM,17,0,1,0.7,0.32,7,4,7,1,0 +user_00132,HR,Analyst,SF,HCM,16,0,1,0.226,0.315,115,11,5,8,1 +user_00133,Finance,Contractor,SF,DataLake,15,0,1,0.968,0.219,47,15,8,7,1 +user_00134,Legal,Analyst,Austin,IAM,19,0,1,0.428,0.887,125,6,1,3,1 +user_00135,Finance,Engineer,SF,DataLake,7,0,1,0.989,0.701,16,17,8,5,1 +user_00136,Engineering,Manager,SF,HCM,12,0,1,0.734,0.378,126,0,5,9,0 +user_00137,Legal,Director,London,Payroll,13,1,1,1.15,0.739,134,13,6,10,1 +user_00138,Finance,Manager,Austin,DataLake,19,0,1,0.303,0.822,141,20,2,3,1 +user_00139,Sales,Contractor,NY,ERP,9,0,1,0.358,0.732,88,16,4,2,0 +user_00140,Engineering,Manager,Bangalore,HCM,9,0,1,0.527,0.577,43,18,2,3,0 +user_00141,Support,Contractor,Bangalore,HCM,6,0,0,0.859,0.778,67,20,3,10,0 +user_00142,Sales,Contractor,NY,DataLake,14,0,1,0.486,0.804,175,12,4,8,0 +user_00143,Finance,Admin,NY,DataLake,11,0,1,0.45,0.739,88,12,2,6,1 +user_00144,Legal,Contractor,NY,DataLake,8,0,1,0.213,0.238,98,19,6,4,0 +user_00145,HR,Director,Bangalore,CRM,33,1,1,0.953,0.573,23,12,8,8,0 +user_00146,HR,Manager,Bangalore,ERP,14,0,1,0.874,0.704,73,17,0,2,1 +user_00147,Sales,Manager,Austin,IAM,16,0,1,0.42,0.628,49,11,8,5,0 +user_00148,Finance,Engineer,Bangalore,Payroll,13,0,1,0.295,0.931,100,11,5,9,1 +user_00149,Engineering,Manager,SF,Payroll,17,0,0,0.131,0.735,121,16,7,3,0 +user_00150,Legal,Contractor,SF,Payroll,15,0,1,0.593,0.832,151,10,8,9,0 +user_00151,Legal,Engineer,Bangalore,ERP,16,0,1,0.884,0.638,176,20,0,10,0 +user_00152,Sales,Director,Bangalore,IAM,20,1,1,0.669,0.741,97,4,3,1,0 +user_00153,Legal,Admin,NY,DataLake,9,0,1,0.528,0.781,120,1,2,9,1 +user_00154,HR,Contractor,Austin,DataLake,17,0,1,0.568,0.6,139,11,4,10,1 +user_00155,Sales,Manager,SF,HCM,14,0,1,0.886,0.778,180,6,7,6,0 +user_00156,Sales,Director,Bangalore,IAM,22,1,1,0.791,0.485,101,11,2,5,0 +user_00157,Finance,Director,NY,HCM,20,0,1,0.631,0.9,178,8,2,2,1 +user_00158,Legal,Admin,Bangalore,Payroll,12,0,1,0.18,0.448,120,3,2,1,0 +user_00159,Legal,Manager,London,DataLake,20,0,1,0.971,0.441,165,1,1,10,1 +user_00160,HR,Contractor,NY,CRM,18,0,1,0.229,0.827,126,5,4,1,1 +user_00161,Finance,Director,Bangalore,HCM,26,1,1,0.785,0.57,34,16,7,5,1 +user_00162,HR,Analyst,Austin,DataLake,13,0,0,0.37,0.421,172,6,2,10,0 +user_00163,Support,Engineer,London,Payroll,15,0,1,0.176,0.936,122,10,3,1,0 +user_00164,Engineering,Engineer,SF,DataLake,13,0,1,0.678,0.209,167,11,3,1,0 +user_00165,Support,Analyst,London,HCM,10,0,1,0.838,0.866,79,3,4,6,0 +user_00166,Legal,Manager,SF,DataLake,9,0,1,0.685,0.512,140,15,8,4,1 +user_00167,HR,Admin,Bangalore,DataLake,22,1,1,0.34,0.822,129,6,8,3,0 +user_00168,HR,Director,Bangalore,IAM,26,1,1,0.706,0.242,33,16,6,8,1 +user_00169,Legal,Analyst,Bangalore,HCM,5,0,1,0.202,0.363,18,1,6,6,1 +user_00170,Support,Analyst,Bangalore,ERP,7,0,0,0.427,0.777,164,20,6,6,0 +user_00171,Sales,Engineer,London,DataLake,7,0,1,0.8,0.3,89,3,2,9,0 +user_00172,Sales,Contractor,SF,IAM,12,0,1,0.883,0.547,139,13,8,7,0 +user_00173,HR,Manager,London,HCM,11,0,0,0.628,0.661,4,7,3,2,0 +user_00174,Finance,Contractor,NY,IAM,6,0,1,0.522,0.376,55,1,2,9,1 +user_00175,Engineering,Manager,Bangalore,HCM,15,0,1,0.215,0.695,56,13,4,5,0 +user_00176,Finance,Contractor,Bangalore,CRM,18,0,1,0.978,0.906,171,12,8,6,1 +user_00177,Support,Engineer,London,CRM,14,0,1,0.53,0.381,57,9,2,8,0 +user_00178,Finance,Contractor,London,Payroll,9,0,1,0.363,0.685,115,11,1,9,1 +user_00179,Support,Manager,NY,HCM,17,0,1,0.132,0.255,150,17,3,8,0 +user_00180,HR,Director,Austin,CRM,20,1,1,0.807,0.728,101,7,8,6,0 +user_00181,Engineering,Engineer,London,Payroll,16,0,1,0.51,0.573,25,15,5,4,0 +user_00182,Engineering,Director,London,CRM,9,0,1,0.848,0.634,106,9,2,4,0 +user_00183,Engineering,Manager,London,Payroll,12,0,1,0.855,0.453,125,20,7,8,0 +user_00184,HR,Admin,Austin,DataLake,10,0,1,0.966,0.593,19,1,0,7,1 +user_00185,HR,Admin,SF,CRM,19,0,0,0.61,0.9,123,15,0,1,0 +user_00186,Legal,Contractor,London,Payroll,13,0,1,0.114,0.81,172,13,2,2,0 +user_00187,Finance,Contractor,SF,Payroll,13,0,1,0.418,0.48,103,14,3,4,0 +user_00188,Engineering,Admin,NY,IAM,12,1,1,0.552,0.557,163,0,2,2,0 +user_00189,Sales,Engineer,Austin,Payroll,8,0,1,0.284,0.876,177,18,7,5,0 +user_00190,Finance,Analyst,Austin,Payroll,6,0,1,0.452,0.921,150,4,6,2,1 +user_00191,Engineering,Manager,Bangalore,CRM,17,0,1,0.365,0.767,35,2,8,6,0 +user_00192,Finance,Analyst,London,ERP,15,0,1,0.864,0.498,83,0,4,8,0 +user_00193,Sales,Manager,Austin,Payroll,17,0,1,0.569,0.485,158,9,3,2,0 +user_00194,Finance,Director,SF,CRM,17,0,1,0.273,0.907,113,11,4,2,1 +user_00195,HR,Analyst,SF,Payroll,18,0,0,0.483,0.27,153,10,6,1,1 +user_00196,Engineering,Engineer,London,ERP,12,0,1,0.648,0.327,68,9,4,8,0 +user_00197,HR,Analyst,SF,DataLake,14,0,1,0.489,0.908,160,15,3,8,1 +user_00198,Finance,Manager,Austin,DataLake,15,0,1,0.79,0.53,110,20,2,5,0 +user_00199,Engineering,Contractor,SF,DataLake,15,0,1,0.875,0.419,177,20,7,10,0 +user_00200,HR,Director,London,HCM,8,0,1,0.998,0.335,140,0,7,4,1 +user_00201,Sales,Director,NY,DataLake,27,1,1,0.444,0.694,108,12,1,8,0 +user_00202,Legal,Engineer,Bangalore,Payroll,18,0,0,0.975,0.726,137,19,1,2,1 +user_00203,HR,Admin,Austin,Payroll,17,0,1,0.544,0.208,66,7,8,9,1 +user_00204,Legal,Contractor,Bangalore,HCM,17,0,1,0.571,0.575,88,0,7,2,0 +user_00205,Engineering,Engineer,NY,ERP,9,0,1,0.464,0.355,123,11,7,2,0 +user_00206,Sales,Admin,London,HCM,6,0,0,0.374,0.282,42,7,8,3,0 +user_00207,Legal,Manager,Austin,DataLake,12,0,1,0.349,0.693,101,0,3,8,1 +user_00208,Legal,Engineer,London,CRM,11,0,1,0.747,0.247,26,17,2,6,0 +user_00209,Engineering,Manager,London,ERP,13,0,1,0.522,0.679,152,12,6,10,0 +user_00210,Finance,Director,Austin,IAM,28,1,1,0.888,0.435,61,9,1,3,1 +user_00211,Engineering,Admin,SF,DataLake,23,1,1,0.399,0.677,177,20,2,9,0 +user_00212,HR,Engineer,Austin,HCM,19,0,0,0.387,0.207,34,6,6,9,0 +user_00213,Legal,Admin,London,IAM,31,1,1,0.802,0.797,56,9,0,5,1 +user_00214,HR,Contractor,Austin,DataLake,20,0,1,0.558,0.401,139,11,8,1,0 +user_00215,Support,Engineer,Bangalore,CRM,18,0,1,0.623,0.386,8,14,4,6,0 +user_00216,Finance,Engineer,NY,CRM,11,0,1,0.889,0.658,109,5,2,4,1 +user_00217,HR,Analyst,Bangalore,HCM,10,0,1,0.434,0.63,131,3,2,7,1 +user_00218,Finance,Director,SF,CRM,23,1,1,0.791,0.567,147,20,4,7,0 +user_00219,Sales,Engineer,NY,IAM,7,0,1,0.323,0.902,94,14,7,6,0 +user_00220,Legal,Analyst,NY,DataLake,16,0,1,0.518,0.926,54,15,4,6,1 +user_00221,Engineering,Director,Bangalore,Payroll,30,1,1,0.86,0.234,160,14,0,2,0 +user_00222,HR,Engineer,London,DataLake,11,0,1,0.341,0.856,70,2,5,5,0 +user_00223,Finance,Director,SF,IAM,12,0,1,0.763,0.201,122,2,2,10,1 +user_00224,HR,Contractor,London,IAM,15,0,1,0.438,0.299,18,7,6,2,0 +user_00225,Support,Analyst,NY,HCM,16,0,1,0.719,0.765,34,20,2,2,0 +user_00226,Legal,Contractor,NY,CRM,19,0,1,0.702,0.314,105,19,7,4,1 +user_00227,Finance,Analyst,SF,Payroll,17,0,1,0.452,0.305,70,20,1,4,1 +user_00228,Legal,Contractor,Bangalore,HCM,5,0,1,0.364,0.655,48,12,4,1,1 +user_00229,HR,Engineer,London,DataLake,7,0,1,0.11,0.471,81,4,6,10,0 +user_00230,Sales,Director,Bangalore,DataLake,12,1,1,0.261,0.362,15,12,8,5,0 +user_00231,Support,Admin,Bangalore,DataLake,23,1,1,0.68,0.315,21,9,1,9,0 +user_00232,HR,Manager,Bangalore,Payroll,7,0,1,0.684,0.835,14,7,1,7,0 +user_00233,Finance,Engineer,Bangalore,HCM,10,0,0,0.31,0.209,127,11,8,9,1 +user_00234,Support,Director,SF,IAM,13,0,1,0.123,0.805,69,16,1,5,0 +user_00235,Support,Contractor,NY,DataLake,19,0,1,0.5,0.625,94,5,4,2,0 +user_00236,Legal,Admin,NY,ERP,5,0,0,0.491,0.89,45,1,8,7,1 +user_00237,HR,Director,SF,HCM,21,1,1,0.752,0.876,45,7,8,1,1 +user_00238,Sales,Analyst,London,HCM,14,0,0,0.465,0.477,74,18,4,10,0 +user_00239,HR,Engineer,Austin,DataLake,11,0,1,0.396,0.205,104,0,3,9,1 +user_00240,Engineering,Admin,NY,ERP,26,1,1,0.674,0.379,125,1,2,5,0 +user_00241,Finance,Analyst,SF,Payroll,18,0,1,0.798,0.636,129,4,6,2,1 +user_00242,Legal,Contractor,NY,CRM,12,0,1,0.452,0.801,179,10,5,8,1 +user_00243,Engineering,Manager,NY,CRM,9,0,1,0.185,0.537,80,13,1,9,0 +user_00244,Engineering,Manager,London,HCM,19,0,1,0.172,0.806,77,19,0,4,0 +user_00245,Engineering,Manager,NY,IAM,12,0,1,0.139,0.424,44,0,6,8,0 +user_00246,Legal,Admin,Bangalore,CRM,8,0,0,0.825,0.367,50,12,5,2,1 +user_00247,Legal,Contractor,Austin,ERP,5,0,1,0.717,0.807,23,10,1,1,1 +user_00248,Sales,Analyst,SF,Payroll,18,0,1,0.406,0.52,90,6,2,5,0 +user_00249,Engineering,Engineer,SF,ERP,12,0,1,0.43,0.613,15,2,4,7,0 +user_00250,HR,Engineer,SF,Payroll,12,0,1,0.635,0.215,97,11,7,9,1 +user_00251,Sales,Director,Bangalore,DataLake,20,1,0,0.728,0.222,71,1,7,9,0 +user_00252,Engineering,Contractor,SF,CRM,8,0,1,0.313,0.812,14,7,6,6,0 +user_00253,HR,Manager,SF,IAM,16,0,0,0.76,0.911,144,4,3,8,1 +user_00254,Legal,Director,SF,CRM,17,1,1,0.251,0.888,51,4,5,3,0 +user_00255,Engineering,Analyst,NY,CRM,9,0,1,0.789,0.92,95,12,1,6,0 +user_00256,Engineering,Director,SF,IAM,33,1,1,0.769,0.331,143,19,5,4,0 +user_00257,Support,Contractor,SF,DataLake,14,0,1,0.683,0.912,146,12,0,3,0 +user_00258,Legal,Director,Bangalore,DataLake,9,0,1,0.39,0.937,42,20,1,6,1 +user_00259,Sales,Analyst,Austin,CRM,12,0,1,0.73,0.74,12,12,7,7,0 +user_00260,HR,Engineer,London,HCM,17,0,1,0.831,0.634,175,7,2,8,1 +user_00261,Finance,Analyst,Austin,HCM,8,0,1,0.81,0.321,19,17,1,6,0 +user_00262,Legal,Contractor,London,DataLake,10,0,1,0.326,0.276,52,6,6,9,1 +user_00263,Support,Admin,Bangalore,HCM,14,0,0,0.414,0.612,125,4,0,6,0 +user_00264,Finance,Engineer,NY,IAM,20,0,0,0.266,0.679,20,15,1,5,0 +user_00265,Sales,Engineer,London,IAM,13,0,1,0.41,0.478,97,11,2,8,0 +user_00266,Finance,Director,London,HCM,13,1,1,0.784,0.264,88,17,6,10,0 +user_00267,HR,Contractor,NY,Payroll,18,0,1,0.949,0.63,60,15,4,2,1 +user_00268,Sales,Admin,NY,Payroll,15,1,1,0.609,0.912,42,6,3,3,0 +user_00269,Finance,Engineer,NY,DataLake,11,0,1,0.676,0.44,17,2,6,9,1 +user_00270,Sales,Contractor,Austin,HCM,19,0,1,0.598,0.592,107,3,6,3,0 +user_00271,HR,Contractor,Bangalore,ERP,8,0,1,0.829,0.851,88,13,4,7,1 +user_00272,Support,Engineer,Bangalore,Payroll,20,0,1,0.422,0.664,174,12,0,7,0 +user_00273,Engineering,Admin,SF,DataLake,13,0,0,0.754,0.427,71,15,1,4,0 +user_00274,HR,Director,London,IAM,32,1,1,0.4,0.359,104,15,8,6,0 +user_00275,Finance,Contractor,SF,IAM,17,0,1,0.519,0.627,44,6,3,1,0 +user_00276,HR,Analyst,London,HCM,11,0,1,0.471,0.498,31,0,3,6,0 +user_00277,Sales,Contractor,London,CRM,20,0,1,0.375,0.828,66,2,4,1,0 +user_00278,Sales,Analyst,SF,Payroll,11,0,1,0.642,0.402,104,16,0,1,0 +user_00279,Sales,Manager,SF,Payroll,5,0,1,0.744,0.429,164,8,6,7,0 +user_00280,Engineering,Engineer,Austin,CRM,19,0,1,0.759,0.498,26,0,8,6,0 +user_00281,Legal,Admin,Bangalore,ERP,26,1,1,0.471,0.444,70,8,4,4,0 +user_00282,HR,Contractor,SF,Payroll,17,0,1,0.312,0.219,127,9,4,7,1 +user_00283,Sales,Engineer,NY,Payroll,11,0,1,0.278,0.601,121,11,8,5,0 +user_00284,HR,Admin,Bangalore,CRM,13,0,1,0.246,0.71,85,20,1,4,0 +user_00285,Engineering,Manager,Austin,DataLake,15,0,1,0.335,0.838,120,1,1,8,0 +user_00286,HR,Contractor,NY,DataLake,14,0,0,0.883,0.303,51,10,6,10,1 +user_00287,Sales,Manager,Austin,IAM,10,0,1,0.605,0.333,162,1,0,8,0 +user_00288,Engineering,Manager,SF,Payroll,10,0,1,0.724,0.297,111,20,6,7,0 +user_00289,Sales,Engineer,London,ERP,6,0,1,0.398,0.211,82,16,3,1,0 +user_00290,Support,Analyst,SF,CRM,16,0,1,0.407,0.642,42,2,0,5,0 +user_00291,Engineering,Engineer,Bangalore,Payroll,15,0,1,0.896,0.457,125,11,3,3,0 +user_00292,Sales,Analyst,SF,CRM,9,0,1,0.572,0.327,93,20,0,7,0 +user_00293,Support,Director,Bangalore,Payroll,21,1,1,0.919,0.236,29,13,7,7,0 +user_00294,Finance,Contractor,Austin,CRM,11,0,1,0.441,0.696,169,16,4,10,1 +user_00295,Legal,Admin,SF,Payroll,17,1,1,0.435,0.349,51,8,8,4,1 +user_00296,Engineering,Admin,London,ERP,20,0,1,0.485,0.376,3,2,1,10,0 +user_00297,Support,Engineer,SF,ERP,7,0,1,0.615,0.543,75,8,7,1,0 +user_00298,Finance,Manager,NY,HCM,15,0,1,0.276,0.769,11,0,2,3,1 +user_00299,Engineering,Director,London,HCM,17,1,1,0.396,0.501,116,8,6,8,0 +user_00300,Sales,Admin,SF,IAM,6,0,1,0.829,0.562,145,14,2,10,0 +user_00301,Sales,Director,Bangalore,IAM,9,0,1,0.19,0.508,11,2,4,6,0 +user_00302,Finance,Admin,Bangalore,Payroll,27,1,1,0.765,0.716,158,3,6,4,1 +user_00303,Sales,Director,London,Payroll,6,0,1,0.226,0.837,126,5,1,8,0 +user_00304,Engineering,Manager,Austin,HCM,6,0,1,0.615,0.502,60,2,7,8,0 +user_00305,Legal,Engineer,NY,DataLake,15,0,1,0.796,0.213,103,13,0,9,1 +user_00306,Legal,Analyst,NY,Payroll,14,0,1,0.724,0.542,87,11,0,4,1 +user_00307,Sales,Director,Bangalore,Payroll,35,1,1,0.612,0.273,168,7,0,6,0 +user_00308,Engineering,Admin,Austin,HCM,13,1,1,0.644,0.816,104,3,4,10,0 +user_00309,HR,Manager,NY,DataLake,11,0,1,0.352,0.453,106,1,1,8,1 +user_00310,Sales,Admin,Bangalore,CRM,25,1,1,0.289,0.901,149,15,8,3,0 +user_00311,Support,Contractor,SF,IAM,9,0,1,0.58,0.862,129,12,6,10,0 +user_00312,Support,Manager,Austin,DataLake,15,0,1,0.207,0.51,179,16,4,9,0 +user_00313,Engineering,Admin,Bangalore,CRM,25,1,1,0.799,0.891,33,20,2,9,0 +user_00314,Legal,Admin,Austin,Payroll,34,1,1,0.884,0.513,136,9,6,1,1 +user_00315,Legal,Analyst,NY,ERP,6,0,1,0.828,0.938,174,8,2,1,1 +user_00316,Support,Director,SF,ERP,28,1,1,0.944,0.755,57,20,8,7,0 +user_00317,HR,Contractor,Austin,ERP,17,0,1,0.83,0.714,141,1,0,5,1 +user_00318,HR,Contractor,London,IAM,16,0,1,0.683,0.782,49,9,7,3,1 +user_00319,Support,Analyst,SF,CRM,8,0,0,0.323,0.398,41,18,4,1,0 +user_00320,Engineering,Engineer,SF,ERP,9,0,1,0.719,0.669,169,9,7,10,0 +user_00321,Legal,Director,NY,CRM,23,1,0,0.654,0.512,137,12,1,6,0 +user_00322,HR,Manager,Austin,HCM,11,0,1,0.897,0.818,166,0,1,6,0 +user_00323,Sales,Manager,Bangalore,Payroll,12,0,1,0.357,0.887,29,12,4,9,0 +user_00324,Support,Director,NY,HCM,31,1,0,0.969,0.344,131,2,1,3,0 +user_00325,Legal,Contractor,NY,CRM,11,0,1,0.283,0.603,179,13,1,3,1 +user_00326,Finance,Engineer,Austin,ERP,6,0,1,0.546,0.631,58,8,6,7,0 +user_00327,Sales,Admin,London,ERP,32,1,1,0.255,0.74,16,18,0,2,0 +user_00328,Engineering,Contractor,NY,HCM,14,0,0,0.712,0.528,101,0,7,3,0 +user_00329,Legal,Manager,SF,Payroll,14,0,1,0.742,0.886,165,11,0,9,1 +user_00330,Legal,Manager,NY,Payroll,17,0,1,0.168,0.358,88,6,6,9,0 +user_00331,HR,Manager,SF,HCM,11,0,1,0.74,0.839,140,14,8,2,0 +user_00332,Engineering,Admin,NY,CRM,20,0,1,0.637,0.376,72,19,1,4,0 +user_00333,Engineering,Engineer,London,HCM,14,0,1,0.553,0.806,109,5,0,6,0 +user_00334,HR,Manager,London,HCM,9,0,1,0.802,0.547,91,9,7,9,0 +user_00335,Finance,Manager,Bangalore,HCM,9,0,1,0.687,0.67,19,0,8,1,0 +user_00336,Legal,Contractor,London,CRM,6,0,1,0.915,0.605,167,4,5,7,1 +user_00337,HR,Contractor,Austin,Payroll,7,0,1,0.108,0.379,157,5,8,3,0 +user_00338,Sales,Manager,SF,Payroll,9,0,1,0.132,0.771,59,2,4,6,0 +user_00339,HR,Analyst,SF,HCM,17,0,1,0.905,0.23,166,10,1,9,1 +user_00340,Support,Director,Austin,Payroll,21,1,1,0.542,0.69,19,16,4,7,0 +user_00341,HR,Contractor,London,HCM,20,0,1,0.205,0.862,112,5,3,2,1 +user_00342,Legal,Admin,Austin,HCM,8,0,1,0.749,0.74,123,15,7,4,0 +user_00343,Legal,Manager,London,ERP,9,0,1,0.208,0.584,146,4,0,3,0 +user_00344,Engineering,Manager,Bangalore,DataLake,5,0,1,0.858,0.362,155,15,7,3,0 +user_00345,HR,Engineer,Bangalore,Payroll,10,0,1,0.312,0.743,87,14,1,4,0 +user_00346,Sales,Engineer,SF,ERP,5,0,1,0.827,0.713,38,19,1,3,0 +user_00347,Sales,Contractor,Bangalore,ERP,8,0,1,0.204,0.653,0,20,6,9,0 +user_00348,Sales,Director,NY,DataLake,24,1,1,0.555,0.642,178,18,1,7,0 +user_00349,HR,Director,London,ERP,28,1,1,0.677,0.342,1,18,8,2,1 +user_00350,Legal,Analyst,London,IAM,15,0,1,0.324,0.689,64,8,8,7,1 +user_00351,Legal,Director,Bangalore,CRM,18,0,1,0.586,0.478,142,15,4,5,1 +user_00352,Legal,Director,London,IAM,16,1,1,0.909,0.321,54,16,6,6,1 +user_00353,Legal,Contractor,NY,IAM,5,0,1,0.477,0.65,22,2,4,4,0 +user_00354,Engineering,Analyst,Bangalore,ERP,17,0,1,0.517,0.734,152,1,2,8,0 +user_00355,Sales,Analyst,SF,IAM,14,0,0,0.58,0.587,59,19,2,1,0 +user_00356,Engineering,Manager,NY,CRM,7,0,1,0.383,0.496,150,6,8,1,0 +user_00357,Support,Admin,NY,Payroll,27,1,1,0.528,0.922,70,7,3,8,0 +user_00358,Sales,Contractor,NY,HCM,10,0,1,0.547,0.577,87,3,8,5,0 +user_00359,Legal,Contractor,Austin,DataLake,20,0,1,0.969,0.544,117,18,0,8,1 +user_00360,Finance,Director,NY,CRM,12,0,1,0.482,0.602,179,8,3,5,1 +user_00361,Finance,Engineer,Austin,CRM,20,0,1,0.815,0.402,174,6,7,4,0 +user_00362,HR,Admin,SF,CRM,26,1,1,0.873,0.29,20,15,0,9,0 +user_00363,Finance,Engineer,Austin,DataLake,13,0,1,0.811,0.583,101,7,2,1,1 +user_00364,Legal,Engineer,London,ERP,16,0,1,0.821,0.543,30,16,7,6,1 +user_00365,Support,Analyst,NY,IAM,11,0,1,0.36,0.66,66,5,8,5,0 +user_00366,HR,Director,London,CRM,31,1,1,0.811,0.479,102,8,7,3,0 +user_00367,Legal,Manager,Bangalore,ERP,13,0,1,0.885,0.347,38,15,0,3,1 +user_00368,Sales,Engineer,SF,IAM,15,0,1,0.55,0.626,101,3,2,1,0 +user_00369,HR,Director,London,IAM,24,1,1,0.698,0.899,65,11,0,7,0 +user_00370,Finance,Director,Bangalore,IAM,10,0,1,0.739,0.566,114,2,6,10,1 +user_00371,Engineering,Engineer,SF,CRM,15,0,1,0.557,0.837,18,2,2,6,0 +user_00372,HR,Director,Austin,Payroll,26,1,1,0.975,0.904,150,16,7,7,1 +user_00373,HR,Manager,NY,CRM,13,0,1,0.418,0.272,121,17,0,1,0 +user_00374,Sales,Contractor,NY,ERP,13,0,1,0.624,0.864,79,7,3,5,0 +user_00375,Sales,Admin,SF,CRM,5,0,1,0.788,0.444,121,14,8,5,0 +user_00376,Legal,Engineer,SF,IAM,10,0,1,0.705,0.677,144,13,8,9,1 +user_00377,Engineering,Director,London,ERP,28,1,1,0.458,0.428,0,16,2,6,0 +user_00378,Engineering,Engineer,Bangalore,ERP,20,0,1,0.713,0.798,152,7,0,9,0 +user_00379,Engineering,Manager,London,Payroll,18,0,1,0.672,0.684,96,7,4,2,0 +user_00380,Sales,Admin,SF,Payroll,22,1,1,0.676,0.402,48,2,2,3,0 +user_00381,Engineering,Analyst,London,Payroll,19,0,1,0.494,0.44,38,1,8,1,0 +user_00382,HR,Analyst,Austin,IAM,19,0,1,0.221,0.57,87,13,4,8,0 +user_00383,Engineering,Engineer,NY,HCM,7,0,0,0.721,0.254,120,5,3,5,0 +user_00384,Sales,Analyst,NY,IAM,12,0,1,0.454,0.753,8,6,7,2,0 +user_00385,Legal,Admin,SF,HCM,22,1,1,0.384,0.903,133,3,4,8,1 +user_00386,Support,Director,SF,ERP,11,0,1,0.447,0.211,70,18,1,7,0 +user_00387,HR,Analyst,NY,IAM,20,0,1,0.817,0.387,127,2,5,1,1 +user_00388,Finance,Director,SF,CRM,28,1,1,0.797,0.683,88,14,3,1,1 +user_00389,Support,Engineer,Austin,IAM,9,0,1,0.488,0.528,75,17,7,4,0 +user_00390,Legal,Manager,Bangalore,IAM,19,0,1,0.579,0.845,41,2,8,3,0 +user_00391,Sales,Director,Austin,DataLake,20,1,1,0.837,0.862,158,11,3,6,0 +user_00392,Support,Analyst,Bangalore,DataLake,11,0,1,0.713,0.316,72,10,6,6,0 +user_00393,Finance,Analyst,Bangalore,ERP,9,0,1,0.362,0.564,25,1,6,8,1 +user_00394,Legal,Contractor,Bangalore,HCM,12,0,1,0.11,0.891,167,15,1,8,0 +user_00395,Support,Analyst,Austin,Payroll,18,0,1,0.147,0.663,135,1,1,5,0 +user_00396,Finance,Manager,Bangalore,IAM,12,0,1,0.254,0.693,145,20,2,9,0 +user_00397,Support,Contractor,London,IAM,8,0,1,0.64,0.727,109,3,2,3,0 +user_00398,Legal,Engineer,Bangalore,CRM,19,0,1,0.54,0.948,95,13,6,6,0 +user_00399,Legal,Manager,Bangalore,DataLake,13,0,1,0.558,0.498,8,9,0,9,1 +user_00400,HR,Director,NY,HCM,5,0,1,0.509,0.628,146,4,1,1,0 +user_00401,Sales,Director,SF,HCM,12,0,1,0.34,0.628,145,6,1,6,0 +user_00402,Sales,Manager,London,IAM,8,0,1,0.13,0.806,97,2,0,6,0 +user_00403,Engineering,Engineer,London,ERP,12,0,1,0.843,0.675,161,12,0,5,0 +user_00404,Legal,Analyst,Austin,CRM,15,0,1,0.323,0.705,100,14,8,2,1 +user_00405,HR,Engineer,Austin,CRM,9,0,0,0.657,0.237,102,8,1,7,1 +user_00406,Legal,Engineer,SF,DataLake,11,0,1,0.657,0.621,134,7,3,2,1 +user_00407,Engineering,Engineer,Austin,IAM,7,0,1,0.439,0.461,82,11,0,5,0 +user_00408,HR,Director,Bangalore,Payroll,21,1,0,1.005,0.35,95,4,1,2,1 +user_00409,Support,Analyst,SF,Payroll,9,0,0,0.655,0.283,115,15,0,6,0 +user_00410,HR,Admin,Bangalore,HCM,20,0,1,0.313,0.724,4,12,8,1,1 +user_00411,Engineering,Manager,NY,HCM,11,0,1,0.472,0.909,17,12,4,6,0 +user_00412,Finance,Analyst,Austin,IAM,17,0,1,0.607,0.732,80,15,8,2,1 +user_00413,HR,Manager,Bangalore,Payroll,5,0,1,0.624,0.45,100,5,0,6,1 +user_00414,Support,Director,Austin,CRM,18,1,1,0.953,0.868,7,17,4,4,0 +user_00415,HR,Director,London,DataLake,23,1,1,0.372,0.928,154,10,7,9,0 +user_00416,HR,Manager,SF,IAM,11,0,1,0.764,0.593,34,13,2,10,1 +user_00417,HR,Director,NY,ERP,20,1,1,0.466,0.222,81,13,6,5,1 +user_00418,Sales,Admin,Austin,CRM,8,0,1,0.71,0.659,68,19,3,10,0 +user_00419,HR,Manager,NY,IAM,20,0,1,0.684,0.734,95,5,6,8,1 +user_00420,Sales,Manager,SF,CRM,12,0,1,0.875,0.894,105,7,3,4,0 +user_00421,Finance,Manager,London,DataLake,15,0,1,0.542,0.782,155,12,5,7,1 +user_00422,Legal,Admin,Bangalore,IAM,19,1,1,0.716,0.858,62,16,8,7,1 +user_00423,Support,Analyst,Bangalore,DataLake,6,0,1,0.418,0.428,73,14,2,7,0 +user_00424,Engineering,Contractor,NY,IAM,9,0,1,0.461,0.742,28,5,8,3,0 +user_00425,Sales,Engineer,Austin,DataLake,5,0,1,0.817,0.872,116,2,4,6,0 +user_00426,Support,Admin,SF,ERP,13,1,1,0.757,0.264,98,16,2,7,0 +user_00427,Legal,Engineer,NY,HCM,9,0,1,0.343,0.734,107,9,4,8,1 +user_00428,Engineering,Admin,SF,CRM,11,0,1,0.302,0.544,127,17,0,10,0 +user_00429,Sales,Manager,SF,Payroll,20,0,1,0.164,0.65,62,17,4,10,0 +user_00430,Support,Contractor,SF,ERP,10,0,0,0.379,0.309,8,18,4,6,0 +user_00431,Finance,Analyst,NY,CRM,10,0,1,0.729,0.725,146,17,2,8,1 +user_00432,Support,Contractor,Austin,ERP,7,0,1,0.818,0.815,109,17,1,5,0 +user_00433,Sales,Manager,NY,IAM,13,0,1,0.52,0.722,156,1,1,10,0 +user_00434,Support,Admin,Bangalore,DataLake,25,1,1,0.503,0.594,159,11,2,10,0 +user_00435,Legal,Engineer,Bangalore,DataLake,20,0,1,0.386,0.418,49,0,1,1,1 +user_00436,Sales,Admin,NY,CRM,29,1,0,0.53,0.805,54,0,3,5,0 +user_00437,Engineering,Analyst,Austin,HCM,15,0,1,0.367,0.497,145,19,0,3,0 +user_00438,HR,Director,Austin,DataLake,21,1,1,0.474,0.545,137,10,7,7,1 +user_00439,Support,Analyst,Bangalore,HCM,15,0,1,0.267,0.714,90,2,2,5,0 +user_00440,Finance,Director,London,Payroll,24,1,1,1.042,0.764,180,10,0,5,1 +user_00441,Sales,Analyst,London,CRM,16,0,1,0.342,0.594,44,3,5,1,0 +user_00442,HR,Analyst,London,DataLake,9,0,1,0.587,0.855,79,10,2,2,1 +user_00443,Legal,Admin,SF,DataLake,8,0,1,0.135,0.836,22,10,7,1,0 +user_00444,Finance,Contractor,Austin,HCM,11,0,1,0.329,0.599,112,3,4,3,0 +user_00445,HR,Director,NY,Payroll,8,0,1,0.639,0.403,180,9,3,3,1 +user_00446,Finance,Director,Bangalore,ERP,14,1,1,0.89,0.279,13,2,6,7,1 +user_00447,HR,Admin,Bangalore,IAM,33,1,1,0.616,0.349,171,17,1,8,1 +user_00448,Legal,Manager,NY,IAM,19,0,1,0.748,0.733,119,15,3,4,1 +user_00449,HR,Contractor,Austin,DataLake,10,0,1,0.889,0.444,124,0,1,8,1 +user_00450,Finance,Director,London,HCM,22,1,1,0.904,0.798,83,17,1,5,0 +user_00451,Legal,Engineer,London,CRM,12,0,1,0.586,0.25,53,18,6,7,1 +user_00452,Legal,Director,Bangalore,IAM,21,1,1,0.341,0.647,83,17,1,8,0 +user_00453,HR,Director,London,Payroll,17,0,1,0.628,0.268,35,18,0,3,0 +user_00454,Support,Manager,London,Payroll,14,0,1,0.458,0.354,81,9,4,7,0 +user_00455,Support,Director,NY,Payroll,24,1,1,0.3,0.876,13,0,8,6,0 +user_00456,Finance,Director,NY,DataLake,15,0,1,0.548,0.358,44,15,2,4,1 +user_00457,Legal,Engineer,Austin,CRM,14,0,1,0.209,0.686,36,15,6,3,1 +user_00458,Finance,Manager,London,IAM,18,0,1,0.608,0.614,115,12,5,4,1 +user_00459,HR,Engineer,Bangalore,DataLake,12,0,1,0.841,0.632,8,1,7,6,0 +user_00460,Finance,Contractor,London,ERP,9,0,1,0.987,0.784,119,16,1,9,1 +user_00461,Legal,Analyst,SF,IAM,7,0,1,0.838,0.434,13,10,8,1,1 +user_00462,Sales,Engineer,Bangalore,DataLake,15,0,1,0.882,0.421,42,1,4,10,0 +user_00463,Engineering,Analyst,London,Payroll,10,0,0,0.892,0.807,159,0,4,8,0 +user_00464,Finance,Manager,Bangalore,HCM,13,0,1,0.32,0.393,28,19,2,10,0 +user_00465,Support,Manager,London,HCM,10,0,1,0.289,0.336,135,1,5,2,0 +user_00466,HR,Contractor,SF,DataLake,17,0,1,0.781,0.325,147,11,0,5,1 +user_00467,Finance,Analyst,SF,DataLake,14,0,1,0.802,0.599,97,12,7,5,1 +user_00468,HR,Contractor,London,DataLake,17,0,1,0.524,0.917,24,17,5,7,0 +user_00469,Legal,Admin,Austin,ERP,13,0,1,0.469,0.706,61,18,3,9,0 +user_00470,Engineering,Admin,SF,ERP,5,0,0,0.178,0.278,114,11,3,5,0 +user_00471,Engineering,Manager,NY,ERP,7,0,1,0.625,0.827,153,20,1,2,0 +user_00472,Finance,Contractor,SF,IAM,10,0,1,0.403,0.55,19,20,7,5,1 +user_00473,Sales,Admin,Austin,ERP,13,0,1,0.366,0.245,67,5,7,2,0 +user_00474,Sales,Manager,Bangalore,DataLake,10,0,1,0.215,0.777,55,19,1,8,0 +user_00475,Support,Director,NY,ERP,13,0,0,0.498,0.69,0,1,6,5,0 +user_00476,HR,Admin,Austin,Payroll,32,1,1,1.025,0.786,17,17,3,1,1 +user_00477,Sales,Admin,London,Payroll,16,0,1,0.42,0.674,165,2,5,9,0 +user_00478,Engineering,Analyst,London,HCM,11,0,1,0.874,0.696,76,12,1,6,0 +user_00479,Sales,Analyst,London,IAM,11,0,0,0.12,0.947,61,15,6,2,0 +user_00480,Legal,Admin,Austin,DataLake,10,1,1,0.62,0.741,103,3,3,9,1 +user_00481,Sales,Admin,Bangalore,Payroll,12,0,0,0.213,0.69,120,19,0,4,0 +user_00482,Support,Admin,SF,IAM,6,0,1,0.178,0.658,151,2,1,2,0 +user_00483,Sales,Contractor,Bangalore,CRM,7,0,1,0.749,0.937,173,12,5,2,0 +user_00484,Support,Contractor,Bangalore,CRM,15,0,1,0.574,0.452,30,8,2,7,0 +user_00485,HR,Admin,Austin,Payroll,13,1,0,1.028,0.659,40,0,7,2,1 +user_00486,Finance,Analyst,SF,IAM,14,0,1,0.834,0.557,32,6,2,5,1 +user_00487,Legal,Engineer,London,CRM,6,0,1,0.5,0.913,160,9,5,7,0 +user_00488,Finance,Contractor,SF,Payroll,8,0,1,0.709,0.605,56,14,3,5,0 +user_00489,Finance,Manager,NY,IAM,20,0,1,0.736,0.232,165,11,4,4,1 +user_00490,Sales,Director,Austin,DataLake,12,1,1,0.99,0.332,7,15,8,2,0 +user_00491,Finance,Manager,London,HCM,18,0,1,0.919,0.373,28,10,3,2,1 +user_00492,HR,Admin,London,CRM,19,1,1,0.393,0.527,55,19,7,3,1 +user_00493,Sales,Director,SF,CRM,25,1,1,0.352,0.719,15,8,2,2,0 +user_00494,Sales,Engineer,London,CRM,15,0,1,0.583,0.792,163,19,2,10,0 +user_00495,Legal,Manager,NY,Payroll,20,0,1,0.87,0.409,109,7,3,6,1 +user_00496,Sales,Contractor,SF,Payroll,9,0,1,0.434,0.233,120,16,0,7,0 +user_00497,Legal,Engineer,London,IAM,8,0,1,0.266,0.765,177,13,3,3,1 +user_00498,Engineering,Director,SF,ERP,17,1,1,0.379,0.544,146,18,0,8,0 +user_00499,Sales,Manager,London,CRM,16,0,1,0.607,0.845,18,12,5,9,0 +user_00500,Finance,Director,London,Payroll,26,1,0,1.055,0.752,14,16,5,1,1 +user_00501,Finance,Engineer,Austin,ERP,16,0,1,0.647,0.25,52,4,5,4,1 +user_00502,Legal,Analyst,London,Payroll,11,0,1,0.918,0.578,108,8,0,2,1 +user_00503,Engineering,Director,London,Payroll,20,1,1,0.793,0.6,74,14,5,2,0 +user_00504,HR,Engineer,NY,CRM,14,0,1,0.321,0.723,127,4,3,9,0 +user_00505,Support,Analyst,Austin,DataLake,9,0,1,0.455,0.684,74,6,1,1,0 +user_00506,HR,Contractor,Austin,HCM,15,0,1,0.757,0.831,113,14,4,10,1 +user_00507,HR,Director,SF,HCM,19,1,1,0.478,0.917,158,5,8,5,0 +user_00508,HR,Manager,SF,ERP,6,0,1,0.453,0.477,52,17,0,6,0 +user_00509,Engineering,Manager,Austin,DataLake,10,0,1,0.836,0.503,127,4,7,6,0 +user_00510,HR,Analyst,Bangalore,HCM,14,0,1,0.976,0.766,146,4,7,6,1 +user_00511,Support,Analyst,London,ERP,9,0,1,0.346,0.501,56,14,5,2,0 +user_00512,Engineering,Analyst,Bangalore,IAM,11,0,1,0.725,0.775,50,0,1,9,0 +user_00513,Sales,Admin,NY,DataLake,23,1,1,0.734,0.908,170,11,6,3,0 +user_00514,Support,Engineer,Austin,DataLake,16,0,1,0.381,0.933,143,2,3,8,0 +user_00515,Support,Engineer,London,HCM,10,0,1,0.431,0.827,153,2,2,4,0 +user_00516,Engineering,Engineer,Bangalore,HCM,7,0,1,0.171,0.507,69,13,2,9,0 +user_00517,Sales,Admin,Bangalore,CRM,18,1,1,0.482,0.568,77,18,4,8,0 +user_00518,Legal,Admin,London,CRM,13,0,1,0.867,0.554,167,9,6,3,1 +user_00519,Sales,Admin,SF,HCM,20,0,1,0.389,0.56,111,8,7,2,0 +user_00520,Engineering,Contractor,Bangalore,ERP,13,0,1,0.217,0.546,177,4,5,3,0 +user_00521,Finance,Director,Austin,HCM,15,1,1,0.861,0.258,49,20,2,2,1 +user_00522,Legal,Manager,Bangalore,HCM,14,0,1,0.651,0.272,148,11,2,1,0 +user_00523,HR,Contractor,Austin,DataLake,18,0,1,0.33,0.363,128,7,3,9,1 +user_00524,Legal,Admin,Bangalore,IAM,19,1,1,0.411,0.814,4,15,5,5,1 +user_00525,Support,Director,SF,Payroll,22,1,1,0.273,0.811,56,4,2,4,0 +user_00526,Engineering,Manager,SF,CRM,7,0,1,0.727,0.258,123,13,0,4,0 +user_00527,Finance,Analyst,Bangalore,DataLake,8,0,0,0.441,0.208,24,3,3,10,1 +user_00528,HR,Manager,Austin,CRM,13,0,1,0.228,0.575,156,7,8,10,0 +user_00529,Finance,Analyst,Bangalore,Payroll,11,0,1,0.337,0.766,1,3,3,9,1 +user_00530,Support,Engineer,Austin,IAM,10,0,1,0.186,0.826,97,13,3,9,0 +user_00531,HR,Engineer,NY,HCM,8,0,1,0.309,0.772,180,9,8,1,1 +user_00532,Finance,Director,London,Payroll,14,0,1,0.455,0.615,74,13,3,2,0 +user_00533,Sales,Admin,Austin,IAM,20,0,1,0.45,0.694,21,3,2,7,0 +user_00534,Sales,Admin,Bangalore,IAM,16,1,1,0.409,0.35,103,3,3,7,0 +user_00535,Support,Engineer,Austin,IAM,13,0,1,0.385,0.7,103,15,8,5,0 +user_00536,HR,Engineer,NY,DataLake,20,0,1,0.208,0.275,13,2,7,1,0 +user_00537,HR,Engineer,Bangalore,CRM,14,0,1,0.808,0.696,133,15,7,4,1 +user_00538,Engineering,Admin,London,HCM,28,1,1,1.012,0.224,110,3,6,8,0 +user_00539,Engineering,Engineer,London,DataLake,6,0,1,0.628,0.785,180,5,7,7,0 +user_00540,HR,Director,London,IAM,35,1,1,0.682,0.552,143,5,8,1,0 +user_00541,Sales,Director,London,DataLake,25,1,1,0.273,0.234,93,11,0,6,0 +user_00542,Sales,Analyst,London,HCM,16,0,1,0.629,0.386,165,9,1,2,0 +user_00543,Engineering,Analyst,Austin,CRM,5,0,1,0.469,0.848,145,16,6,5,0 +user_00544,Finance,Admin,London,DataLake,27,1,1,0.997,0.275,172,6,5,8,1 +user_00545,Legal,Admin,Austin,DataLake,11,0,1,0.879,0.256,137,6,4,5,0 +user_00546,Finance,Director,Bangalore,IAM,21,1,0,0.519,0.25,135,15,6,4,0 +user_00547,Engineering,Analyst,Austin,Payroll,14,0,1,0.351,0.511,64,19,8,8,0 +user_00548,Support,Contractor,SF,ERP,10,0,1,0.629,0.818,119,8,8,10,0 +user_00549,Sales,Contractor,NY,Payroll,11,0,1,0.292,0.76,76,13,6,6,0 +user_00550,Finance,Admin,SF,HCM,19,1,1,0.439,0.379,38,6,0,4,0 +user_00551,Engineering,Engineer,London,Payroll,18,0,1,0.13,0.32,34,0,4,5,0 +user_00552,Support,Contractor,SF,IAM,6,0,1,0.144,0.872,80,19,4,2,0 +user_00553,Sales,Director,NY,IAM,9,0,1,0.688,0.9,133,18,8,5,0 +user_00554,Sales,Analyst,SF,Payroll,8,0,1,0.567,0.492,60,13,3,3,0 +user_00555,Finance,Manager,SF,DataLake,18,0,1,0.245,0.6,179,3,5,9,1 +user_00556,Sales,Engineer,SF,HCM,9,0,1,0.394,0.431,165,19,0,1,0 +user_00557,Support,Admin,SF,DataLake,14,0,1,0.39,0.809,31,17,3,8,0 +user_00558,Sales,Analyst,London,HCM,14,0,1,0.705,0.423,41,14,5,7,0 +user_00559,Finance,Director,NY,DataLake,28,1,1,1.051,0.815,41,8,8,2,1 +user_00560,Engineering,Engineer,Bangalore,IAM,15,0,1,0.71,0.681,47,20,7,3,0 +user_00561,Finance,Engineer,SF,HCM,7,0,1,0.613,0.927,166,4,6,6,1 +user_00562,Sales,Engineer,SF,HCM,10,0,1,0.362,0.411,21,13,5,3,0 +user_00563,Sales,Engineer,Bangalore,CRM,5,0,1,0.618,0.67,22,14,3,5,0 +user_00564,Support,Manager,NY,DataLake,18,0,1,0.171,0.476,145,3,0,9,0 +user_00565,Sales,Engineer,London,ERP,12,0,1,0.79,0.787,73,15,6,3,0 +user_00566,Sales,Manager,Bangalore,CRM,14,0,1,0.244,0.794,7,9,4,4,0 +user_00567,Legal,Engineer,SF,IAM,19,0,0,0.307,0.442,177,12,3,7,1 +user_00568,Support,Engineer,Bangalore,HCM,7,0,1,0.142,0.898,120,11,0,1,0 +user_00569,Engineering,Contractor,Bangalore,DataLake,14,0,1,0.206,0.869,10,18,2,5,0 +user_00570,HR,Contractor,London,DataLake,9,0,1,0.624,0.306,29,10,0,3,1 +user_00571,Sales,Director,Austin,HCM,27,1,1,0.991,0.588,111,5,5,8,0 +user_00572,HR,Contractor,London,CRM,6,0,1,0.476,0.323,119,13,1,8,0 +user_00573,Finance,Engineer,Austin,Payroll,20,0,1,0.877,0.85,31,11,1,4,1 +user_00574,Engineering,Manager,SF,IAM,13,0,1,0.569,0.638,1,16,2,7,0 +user_00575,Finance,Manager,London,HCM,5,0,1,0.745,0.496,151,3,5,3,1 +user_00576,Finance,Analyst,London,IAM,20,0,1,0.415,0.694,84,1,7,6,1 +user_00577,Support,Admin,Bangalore,ERP,14,0,1,0.695,0.61,154,12,1,7,0 +user_00578,Engineering,Manager,SF,DataLake,7,0,1,0.23,0.785,72,20,2,2,0 +user_00579,Engineering,Engineer,London,IAM,14,0,1,0.387,0.272,94,16,3,8,0 +user_00580,Sales,Engineer,London,HCM,18,0,1,0.749,0.66,111,20,6,2,0 +user_00581,Finance,Contractor,Austin,IAM,12,0,1,0.324,0.571,161,20,8,2,1 +user_00582,Engineering,Engineer,London,ERP,20,0,1,0.202,0.877,63,16,6,5,0 +user_00583,Legal,Analyst,Bangalore,ERP,8,0,1,0.176,0.696,145,8,5,5,0 +user_00584,Legal,Analyst,Bangalore,IAM,19,0,1,0.774,0.234,106,9,5,10,0 +user_00585,Support,Engineer,London,Payroll,13,0,1,0.797,0.34,47,2,4,2,0 +user_00586,Sales,Contractor,NY,ERP,17,0,0,0.625,0.545,103,19,7,5,0 +user_00587,Legal,Contractor,Austin,IAM,17,0,0,0.308,0.343,4,6,0,4,1 +user_00588,Sales,Admin,SF,DataLake,17,1,1,0.621,0.555,5,3,2,3,0 +user_00589,Support,Admin,London,DataLake,17,1,1,0.411,0.627,100,1,5,1,0 +user_00590,Finance,Admin,London,IAM,19,1,1,0.739,0.643,156,15,3,1,1 +user_00591,Legal,Analyst,London,Payroll,7,0,1,0.534,0.555,126,19,5,8,1 +user_00592,Support,Engineer,NY,CRM,14,0,1,0.45,0.581,150,10,7,3,0 +user_00593,Engineering,Admin,London,IAM,28,1,1,0.926,0.666,72,2,7,3,0 +user_00594,HR,Manager,SF,ERP,18,0,1,0.368,0.235,4,11,4,6,1 +user_00595,Support,Manager,SF,Payroll,6,0,1,0.191,0.37,143,7,7,5,0 +user_00596,Engineering,Director,NY,Payroll,20,1,1,0.949,0.653,26,1,7,8,0 +user_00597,Finance,Analyst,Bangalore,ERP,6,0,1,0.704,0.853,92,5,4,8,1 +user_00598,Legal,Engineer,NY,HCM,15,0,1,0.262,0.927,1,7,7,6,0 +user_00599,Engineering,Contractor,SF,ERP,13,0,1,0.432,0.672,62,4,8,7,0 +user_00600,HR,Manager,London,DataLake,19,0,1,0.489,0.222,37,0,8,2,1 +user_00601,Finance,Director,Austin,DataLake,17,1,1,0.878,0.681,147,1,8,5,1 +user_00602,Sales,Engineer,Bangalore,HCM,15,0,0,0.324,0.596,37,12,7,4,0 +user_00603,Legal,Contractor,Bangalore,IAM,16,0,1,0.358,0.426,9,17,5,7,1 +user_00604,Support,Admin,SF,Payroll,22,1,0,0.691,0.591,162,5,0,1,0 +user_00605,Legal,Contractor,Austin,HCM,18,0,1,0.148,0.48,160,16,5,2,0 +user_00606,Finance,Admin,NY,ERP,29,1,1,0.994,0.942,144,20,8,10,1 +user_00607,HR,Director,Austin,IAM,9,0,1,0.398,0.446,149,6,5,8,1 +user_00608,Finance,Manager,London,ERP,7,0,1,0.304,0.361,19,11,8,7,1 +user_00609,Legal,Contractor,Bangalore,HCM,20,0,1,0.552,0.898,125,13,2,3,1 +user_00610,Legal,Manager,London,CRM,8,0,1,0.226,0.318,143,11,1,10,1 +user_00611,Sales,Analyst,NY,HCM,14,0,1,0.194,0.429,95,9,4,8,0 +user_00612,Finance,Manager,SF,DataLake,6,0,1,0.372,0.814,125,0,1,9,1 +user_00613,Engineering,Analyst,NY,DataLake,20,0,1,0.462,0.453,96,15,2,3,0 +user_00614,Legal,Admin,London,CRM,21,1,1,0.882,0.507,59,12,5,9,1 +user_00615,Finance,Admin,London,CRM,19,1,1,0.967,0.432,165,3,8,2,1 +user_00616,HR,Contractor,Bangalore,ERP,11,0,1,0.215,0.694,136,5,0,5,1 +user_00617,HR,Manager,Bangalore,Payroll,15,0,1,0.593,0.9,87,13,2,7,1 +user_00618,Finance,Contractor,Austin,ERP,8,0,1,0.428,0.337,22,4,4,7,1 +user_00619,Support,Manager,NY,Payroll,15,0,0,0.147,0.492,81,4,6,6,0 +user_00620,Engineering,Engineer,NY,ERP,11,0,1,0.569,0.57,129,12,1,5,0 +user_00621,Finance,Director,London,ERP,33,1,1,0.959,0.402,150,3,3,9,0 +user_00622,Support,Manager,Austin,Payroll,18,0,0,0.609,0.353,36,11,8,3,0 +user_00623,HR,Contractor,London,Payroll,19,0,0,0.572,0.822,74,20,2,1,1 +user_00624,Finance,Engineer,Austin,IAM,17,0,1,0.37,0.831,72,1,2,9,1 +user_00625,Legal,Analyst,Austin,IAM,8,0,1,0.899,0.859,53,19,8,6,0 +user_00626,Support,Director,Austin,IAM,28,1,1,1.036,0.456,117,19,5,1,0 +user_00627,Support,Manager,NY,Payroll,20,0,1,0.638,0.51,178,15,6,9,0 +user_00628,Finance,Admin,NY,IAM,34,1,1,1.128,0.482,153,11,6,3,1 +user_00629,Support,Analyst,NY,DataLake,16,0,1,0.277,0.215,71,11,3,2,0 +user_00630,Legal,Admin,Austin,DataLake,21,1,1,0.974,0.73,53,0,3,7,0 +user_00631,Finance,Director,Bangalore,CRM,16,1,1,0.847,0.296,145,7,8,5,0 +user_00632,HR,Admin,NY,Payroll,28,1,1,0.713,0.903,164,20,0,6,1 +user_00633,Sales,Engineer,Austin,ERP,19,0,1,0.707,0.863,57,1,4,7,0 +user_00634,Legal,Engineer,Austin,CRM,15,0,1,0.746,0.943,57,13,1,4,0 +user_00635,HR,Manager,Austin,IAM,8,0,1,0.746,0.644,160,10,4,4,0 +user_00636,Finance,Engineer,SF,ERP,15,0,1,0.955,0.782,158,14,1,8,1 +user_00637,Legal,Analyst,NY,ERP,17,0,1,0.755,0.787,15,14,0,5,1 +user_00638,Legal,Admin,London,Payroll,6,0,1,0.521,0.663,105,7,0,8,1 +user_00639,Sales,Engineer,London,Payroll,8,0,1,0.641,0.219,132,10,3,2,0 +user_00640,Finance,Analyst,Bangalore,Payroll,11,0,1,0.607,0.467,123,19,7,3,1 +user_00641,HR,Manager,NY,HCM,18,0,1,0.703,0.433,138,4,2,8,0 +user_00642,Engineering,Contractor,Austin,HCM,16,0,1,0.626,0.308,60,16,8,1,0 +user_00643,Sales,Director,SF,IAM,17,1,1,0.784,0.721,119,19,5,10,0 +user_00644,Legal,Director,Bangalore,ERP,19,0,1,0.836,0.679,177,7,7,10,0 +user_00645,HR,Manager,London,HCM,17,0,1,0.861,0.205,2,2,5,1,1 +user_00646,Legal,Analyst,SF,CRM,16,0,1,0.259,0.413,56,8,2,6,0 +user_00647,Legal,Manager,SF,CRM,13,0,0,0.642,0.484,28,10,3,5,0 +user_00648,Sales,Manager,Austin,HCM,10,0,1,0.898,0.861,16,3,2,3,0 +user_00649,HR,Manager,Bangalore,HCM,15,0,1,0.709,0.891,157,18,3,9,1 +user_00650,Engineering,Manager,Austin,CRM,6,0,1,0.798,0.839,149,15,3,2,0 +user_00651,Sales,Admin,Bangalore,Payroll,17,0,1,0.399,0.589,45,10,5,9,0 +user_00652,Support,Engineer,London,DataLake,13,0,1,0.837,0.638,9,6,3,4,0 +user_00653,Sales,Manager,London,CRM,14,0,1,0.557,0.902,95,8,8,2,0 +user_00654,HR,Analyst,Austin,HCM,12,0,0,0.61,0.452,115,14,6,7,1 +user_00655,Engineering,Engineer,Austin,HCM,17,0,1,0.287,0.684,91,2,4,9,0 +user_00656,Legal,Contractor,NY,CRM,8,0,0,0.231,0.35,50,10,4,5,1 +user_00657,Support,Admin,Austin,Payroll,9,0,1,0.107,0.425,88,12,6,10,0 +user_00658,Support,Analyst,London,IAM,9,0,1,0.679,0.357,59,16,8,8,0 +user_00659,Support,Engineer,Austin,ERP,15,0,1,0.854,0.513,112,14,8,1,0 +user_00660,Engineering,Manager,Bangalore,CRM,14,0,1,0.627,0.466,57,13,6,7,0 +user_00661,Support,Contractor,Austin,HCM,5,0,1,0.838,0.315,84,18,6,7,0 +user_00662,HR,Engineer,NY,CRM,20,0,1,0.97,0.679,77,2,2,2,1 +user_00663,Support,Contractor,SF,ERP,20,0,1,0.156,0.922,72,6,3,5,0 +user_00664,Support,Contractor,NY,DataLake,6,0,1,0.237,0.845,105,12,2,2,0 +user_00665,HR,Contractor,London,ERP,17,0,1,0.365,0.524,170,15,2,10,1 +user_00666,Legal,Director,Bangalore,DataLake,13,0,0,0.314,0.248,102,19,3,9,1 +user_00667,Engineering,Contractor,London,DataLake,19,0,0,0.173,0.685,116,11,4,7,0 +user_00668,Legal,Manager,London,Payroll,18,0,1,0.627,0.724,24,12,0,4,0 +user_00669,HR,Director,Bangalore,IAM,11,0,1,0.809,0.291,94,2,1,9,1 +user_00670,Engineering,Engineer,London,CRM,13,0,1,0.47,0.255,177,14,6,6,0 +user_00671,Support,Engineer,SF,DataLake,6,0,1,0.667,0.302,171,14,7,8,0 +user_00672,Finance,Analyst,Bangalore,IAM,19,0,1,0.423,0.321,4,19,1,3,1 +user_00673,Engineering,Analyst,Bangalore,IAM,17,0,0,0.711,0.708,39,14,5,3,0 +user_00674,HR,Admin,NY,DataLake,21,1,1,0.863,0.737,26,12,4,8,1 +user_00675,Finance,Contractor,NY,DataLake,11,0,1,0.291,0.269,107,0,0,6,0 +user_00676,HR,Admin,NY,IAM,10,0,0,0.61,0.812,82,17,3,10,1 +user_00677,HR,Contractor,London,ERP,12,0,1,0.459,0.211,167,10,0,7,1 +user_00678,HR,Engineer,SF,HCM,16,0,0,0.275,0.426,153,15,1,9,0 +user_00679,Legal,Manager,Austin,Payroll,15,0,1,0.742,0.929,177,4,5,3,1 +user_00680,Support,Admin,Austin,IAM,26,1,1,0.737,0.639,97,16,3,9,0 +user_00681,Legal,Admin,NY,ERP,14,0,1,0.806,0.665,15,4,5,8,1 +user_00682,Engineering,Manager,SF,DataLake,11,0,0,0.839,0.649,113,10,2,5,0 +user_00683,Engineering,Admin,NY,DataLake,19,1,1,0.836,0.457,52,17,0,4,0 +user_00684,Support,Admin,SF,DataLake,22,1,1,0.255,0.852,17,6,0,3,0 +user_00685,HR,Manager,NY,Payroll,11,0,1,0.441,0.55,42,6,3,7,0 +user_00686,HR,Contractor,SF,Payroll,19,0,1,0.79,0.561,92,12,6,3,0 +user_00687,Sales,Engineer,London,Payroll,18,0,1,0.735,0.215,97,13,1,5,0 +user_00688,Legal,Contractor,SF,CRM,13,0,1,0.563,0.481,155,15,1,6,1 +user_00689,HR,Admin,London,ERP,5,0,1,0.499,0.503,106,12,5,7,1 +user_00690,Sales,Director,Bangalore,ERP,8,0,1,0.23,0.772,65,11,8,8,0 +user_00691,Support,Admin,Austin,IAM,28,1,1,0.444,0.226,92,20,0,3,0 +user_00692,Engineering,Contractor,London,IAM,9,0,1,0.563,0.697,41,0,1,2,0 +user_00693,Support,Admin,London,Payroll,7,0,1,0.801,0.324,109,8,4,9,0 +user_00694,Legal,Admin,London,IAM,13,1,1,0.971,0.458,36,8,3,7,1 +user_00695,Support,Director,NY,DataLake,11,1,0,0.807,0.594,167,9,6,3,0 +user_00696,HR,Director,SF,CRM,20,1,1,0.42,0.351,46,6,4,7,0 +user_00697,Sales,Admin,Bangalore,IAM,10,0,1,0.402,0.408,87,0,3,8,0 +user_00698,Sales,Admin,NY,HCM,5,0,1,0.89,0.273,133,0,6,10,0 +user_00699,Legal,Director,London,DataLake,25,1,1,0.524,0.436,61,11,6,7,0 +user_00700,Legal,Analyst,Austin,ERP,5,0,1,0.521,0.827,35,0,1,9,1 +user_00701,Engineering,Admin,SF,Payroll,20,0,1,0.25,0.94,82,0,5,1,0 +user_00702,Support,Director,London,DataLake,17,0,1,0.379,0.904,121,3,6,5,0 +user_00703,Legal,Manager,SF,Payroll,6,0,1,0.27,0.824,140,14,2,8,1 +user_00704,Finance,Engineer,Bangalore,CRM,10,0,1,0.184,0.945,7,5,5,7,0 +user_00705,Engineering,Manager,Austin,IAM,20,0,1,0.209,0.601,125,2,5,4,0 +user_00706,Legal,Contractor,NY,DataLake,17,0,1,0.65,0.842,59,7,4,2,1 +user_00707,Engineering,Analyst,NY,ERP,15,0,1,0.696,0.287,94,4,5,7,0 +user_00708,Legal,Admin,NY,CRM,17,0,1,0.492,0.709,32,16,3,2,1 +user_00709,Sales,Analyst,Bangalore,DataLake,15,0,1,0.337,0.364,174,14,2,10,0 +user_00710,Engineering,Manager,NY,Payroll,15,0,1,0.651,0.405,180,18,7,4,0 +user_00711,Sales,Director,London,HCM,9,0,1,0.848,0.369,13,5,7,7,0 +user_00712,Engineering,Admin,SF,Payroll,27,1,1,0.629,0.461,100,12,1,9,0 +user_00713,Support,Director,Austin,CRM,23,1,1,0.852,0.635,13,9,7,3,0 +user_00714,Legal,Manager,London,HCM,9,0,1,0.799,0.751,2,11,1,3,1 +user_00715,Sales,Engineer,SF,ERP,8,0,1,0.244,0.239,131,20,5,10,0 +user_00716,Sales,Manager,London,CRM,6,0,1,0.491,0.716,44,20,6,2,0 +user_00717,Sales,Admin,London,IAM,19,0,0,0.193,0.292,164,16,3,1,0 +user_00718,Legal,Contractor,Bangalore,IAM,8,0,1,0.587,0.683,111,9,2,2,1 +user_00719,Engineering,Analyst,Bangalore,CRM,17,0,1,0.754,0.788,135,15,1,9,0 +user_00720,Legal,Contractor,NY,CRM,8,0,1,0.756,0.204,114,10,1,1,1 +user_00721,HR,Contractor,SF,ERP,13,0,0,0.3,0.424,139,5,6,1,1 +user_00722,HR,Director,NY,HCM,26,1,1,0.906,0.592,90,19,8,9,1 +user_00723,HR,Admin,NY,ERP,18,1,1,0.916,0.918,154,17,3,10,0 +user_00724,Engineering,Director,SF,DataLake,20,1,1,0.555,0.878,102,14,1,5,0 +user_00725,Support,Engineer,SF,Payroll,11,0,1,0.316,0.675,165,6,5,10,0 +user_00726,Support,Admin,Bangalore,DataLake,19,1,1,0.597,0.219,154,7,1,5,0 +user_00727,Finance,Contractor,NY,DataLake,7,0,1,0.49,0.45,69,4,3,5,1 +user_00728,Finance,Analyst,London,CRM,17,0,1,0.299,0.595,78,18,7,2,1 +user_00729,Legal,Director,Bangalore,Payroll,13,1,1,1.107,0.492,88,17,8,10,1 +user_00730,Sales,Engineer,Bangalore,CRM,13,0,1,0.671,0.693,36,12,4,1,0 +user_00731,Support,Admin,Austin,CRM,30,1,1,1.039,0.799,128,9,4,7,0 +user_00732,Legal,Admin,SF,HCM,18,0,1,0.708,0.258,61,2,2,10,1 +user_00733,Finance,Analyst,Austin,DataLake,5,0,1,0.946,0.302,127,16,8,2,1 +user_00734,Sales,Analyst,Bangalore,CRM,5,0,1,0.7,0.383,169,16,0,10,0 +user_00735,Sales,Manager,Bangalore,CRM,10,0,1,0.207,0.465,13,13,7,7,0 +user_00736,HR,Director,NY,Payroll,8,0,1,0.435,0.36,161,18,6,4,1 +user_00737,Engineering,Admin,Bangalore,ERP,33,1,1,0.296,0.204,117,10,4,6,0 +user_00738,Finance,Analyst,Austin,CRM,17,0,0,0.545,0.348,135,16,2,7,1 +user_00739,Finance,Engineer,NY,IAM,13,0,1,0.399,0.473,46,17,4,2,0 +user_00740,Sales,Admin,London,HCM,18,0,1,0.796,0.349,48,9,0,3,0 +user_00741,Sales,Engineer,Austin,ERP,10,0,1,0.483,0.699,68,3,6,8,0 +user_00742,Legal,Engineer,SF,DataLake,17,0,1,0.186,0.216,39,11,8,1,0 +user_00743,Support,Manager,NY,IAM,15,0,1,0.69,0.526,74,9,7,2,0 +user_00744,Sales,Manager,Bangalore,CRM,10,0,1,0.192,0.398,179,7,6,4,0 +user_00745,Sales,Admin,NY,Payroll,18,1,1,0.723,0.676,58,3,2,7,0 +user_00746,Finance,Director,SF,Payroll,17,1,1,0.654,0.594,179,20,3,4,1 +user_00747,Engineering,Engineer,London,ERP,18,0,0,0.188,0.473,89,13,3,6,0 +user_00748,Finance,Manager,NY,IAM,15,0,1,0.324,0.655,110,5,4,8,0 +user_00749,Engineering,Director,Austin,IAM,5,0,1,0.79,0.448,92,18,8,10,0 +user_00750,Engineering,Manager,Austin,CRM,17,0,1,0.204,0.757,30,1,3,4,0 +user_00751,Support,Engineer,NY,ERP,15,0,1,0.892,0.351,39,1,2,4,0 +user_00752,Sales,Contractor,Austin,Payroll,16,0,1,0.674,0.883,17,15,6,9,0 +user_00753,Legal,Analyst,SF,IAM,9,0,0,0.653,0.796,177,6,6,8,1 +user_00754,Support,Contractor,Bangalore,ERP,10,0,1,0.407,0.785,102,19,4,4,0 +user_00755,HR,Analyst,SF,Payroll,5,0,1,0.766,0.594,77,7,4,5,1 +user_00756,Legal,Analyst,SF,Payroll,12,0,0,0.219,0.863,34,8,0,4,0 +user_00757,HR,Contractor,NY,HCM,16,0,0,0.664,0.612,34,13,4,6,0 +user_00758,Finance,Admin,NY,HCM,19,1,1,0.68,0.934,149,3,6,5,0 +user_00759,Support,Admin,Bangalore,DataLake,22,1,1,0.715,0.806,101,4,7,4,0 +user_00760,Support,Admin,London,DataLake,7,0,1,0.248,0.847,48,18,7,3,0 +user_00761,Support,Director,NY,CRM,17,0,1,0.156,0.254,62,14,6,2,0 +user_00762,Support,Contractor,SF,IAM,5,0,0,0.407,0.283,3,20,0,3,0 +user_00763,HR,Analyst,Austin,IAM,5,0,1,0.185,0.752,86,2,3,3,0 +user_00764,Sales,Contractor,London,CRM,14,0,1,0.295,0.608,68,20,8,2,0 +user_00765,Finance,Analyst,NY,ERP,18,0,0,0.379,0.37,98,4,5,5,1 +user_00766,HR,Admin,London,ERP,27,1,1,0.53,0.917,72,11,1,10,1 +user_00767,Engineering,Analyst,Austin,ERP,6,0,1,0.443,0.451,46,13,2,5,0 +user_00768,Legal,Director,Austin,DataLake,25,1,1,0.819,0.257,0,4,2,6,0 +user_00769,HR,Engineer,London,DataLake,7,0,1,0.968,0.409,23,10,2,3,1 +user_00770,Sales,Director,Austin,DataLake,32,1,1,0.517,0.802,167,7,6,7,0 +user_00771,Legal,Engineer,Bangalore,Payroll,12,0,0,0.772,0.595,177,7,1,4,1 +user_00772,Engineering,Analyst,SF,ERP,16,0,1,0.207,0.283,89,11,2,5,0 +user_00773,Sales,Manager,Bangalore,DataLake,12,0,1,0.145,0.303,178,4,6,1,0 +user_00774,Support,Engineer,Austin,ERP,8,0,1,0.481,0.645,40,0,2,6,0 +user_00775,HR,Analyst,SF,Payroll,18,0,1,0.476,0.305,135,9,7,7,0 +user_00776,Support,Director,London,IAM,14,1,1,0.298,0.682,154,5,8,3,0 +user_00777,Finance,Engineer,London,DataLake,13,0,1,0.767,0.518,40,8,1,9,1 +user_00778,Sales,Manager,SF,DataLake,6,0,1,0.758,0.932,154,2,7,8,0 +user_00779,Engineering,Admin,London,CRM,16,1,1,0.406,0.281,19,11,5,1,0 +user_00780,HR,Director,SF,Payroll,14,1,1,0.411,0.897,61,8,1,10,0 +user_00781,Legal,Analyst,Bangalore,DataLake,6,0,1,0.955,0.206,91,9,6,9,1 +user_00782,HR,Director,London,IAM,18,1,1,0.943,0.276,38,0,6,8,1 +user_00783,Legal,Analyst,Austin,IAM,19,0,1,0.175,0.739,55,20,2,3,0 +user_00784,Legal,Contractor,Bangalore,HCM,19,0,1,0.488,0.842,110,1,6,4,1 +user_00785,HR,Contractor,Bangalore,CRM,20,0,1,0.214,0.853,140,2,7,2,0 +user_00786,Sales,Director,SF,CRM,16,1,1,0.533,0.253,117,8,7,5,0 +user_00787,Sales,Admin,SF,Payroll,25,1,1,0.479,0.507,158,4,7,6,0 +user_00788,Legal,Contractor,Austin,IAM,18,0,1,0.902,0.943,10,19,0,4,1 +user_00789,Support,Contractor,NY,CRM,8,0,1,0.556,0.84,57,12,7,2,0 +user_00790,Engineering,Engineer,London,IAM,19,0,0,0.685,0.888,101,13,3,7,0 +user_00791,Engineering,Analyst,Austin,HCM,20,0,1,0.481,0.409,9,17,2,8,0 +user_00792,HR,Admin,SF,Payroll,17,0,1,0.389,0.523,121,6,0,4,0 +user_00793,Finance,Admin,London,ERP,9,0,1,0.265,0.855,115,13,8,4,1 +user_00794,Support,Director,SF,ERP,28,1,1,0.488,0.349,107,15,7,9,0 +user_00795,Support,Manager,London,HCM,17,0,1,0.815,0.349,170,18,2,6,0 +user_00796,Support,Admin,Austin,DataLake,9,0,1,0.191,0.927,104,8,8,1,0 +user_00797,Sales,Manager,London,IAM,8,0,1,0.883,0.515,139,5,6,4,0 +user_00798,Engineering,Manager,SF,IAM,5,0,1,0.113,0.872,128,3,7,8,0 +user_00799,Sales,Engineer,Austin,HCM,18,0,1,0.369,0.895,117,13,5,3,0 diff --git a/frontend/.DS_Store b/frontend/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..8cbc2fbff59a842aa2a2e44c2398ced5511037be Binary files /dev/null and b/frontend/.DS_Store differ diff --git a/frontend/app/compare/page.tsx b/frontend/app/compare/page.tsx new file mode 100644 index 0000000000000000000000000000000000000000..88c47a38bc4f05de24b23a1221535a69265a1d26 --- /dev/null +++ b/frontend/app/compare/page.tsx @@ -0,0 +1,26 @@ +'use client'; +import { useState } from 'react'; +import RunComparisonTable from '../../components/RunComparisonTable'; +import { apiGet } from '../../lib/api'; +import type { CompareRunItem } from '../../lib/types'; + +export default function ComparePage() { + const [ids, setIds] = useState(''); + const [runs, setRuns] = useState([]); + async function compare() { + const data = await apiGet<{ runs: CompareRunItem[] }>(`/compare?experiment_ids=${encodeURIComponent(ids)}`); + setRuns(data.runs); + } + return ( +
+
+

Compare Runs

+

Enter experiment ids separated by commas.

+ setIds(e.target.value)} placeholder='exp_x,exp_y,exp_z' /> +
+ +
+ {runs.length > 0 ? : null} +
+ ); +} diff --git a/frontend/app/datasets/[id]/page.tsx b/frontend/app/datasets/[id]/page.tsx new file mode 100644 index 0000000000000000000000000000000000000000..88ec2da36c34b0cf33ebefae77903f5b7b91811f --- /dev/null +++ b/frontend/app/datasets/[id]/page.tsx @@ -0,0 +1,9 @@ +import DatasetProfileCard from '../../../components/DatasetProfileCard'; +import { apiGet } from '../../../lib/api'; +import type { DatasetProfile } from '../../../lib/types'; + +export default async function DatasetProfilePage({ params }: { params: Promise<{ id: string }> }) { + const { id } = await params; + const profile = await apiGet(`/datasets/${id}/profile`); + return ; +} diff --git a/frontend/app/experiments/[id]/page.tsx b/frontend/app/experiments/[id]/page.tsx new file mode 100644 index 0000000000000000000000000000000000000000..01736c64e5e5cf86085baec1e1b131fd0c297e61 --- /dev/null +++ b/frontend/app/experiments/[id]/page.tsx @@ -0,0 +1,32 @@ +import ClusterBarChart from '../../../components/ClusterBarChart'; +import ClusterScatterPlot from '../../../components/ClusterScatterPlot'; +import ClusterSummaryCard from '../../../components/ClusterSummaryCard'; +import ClusterTable from '../../../components/ClusterTable'; +import MetricsGrid from '../../../components/MetricsGrid'; +import OutlierTable from '../../../components/OutlierTable'; +import { apiGet } from '../../../lib/api'; +import type { ExperimentResult } from '../../../lib/types'; + +export default async function ExperimentResultPage({ params }: { params: Promise<{ id: string }> }) { + const { id } = await params; + const data = await apiGet(`/experiments/${id}/results`); + return ( +
+
+
+

Experiment {data.experiment_id}

+
{data.algorithm}
+
+

Status: {data.status}

+
+ + + + + +
+ {data.cluster_summaries.map((item) => )} +
+
+ ); +} diff --git a/frontend/app/experiments/new/page.tsx b/frontend/app/experiments/new/page.tsx new file mode 100644 index 0000000000000000000000000000000000000000..84a639fb9b584487f09fa1c44899e75620df2937 --- /dev/null +++ b/frontend/app/experiments/new/page.tsx @@ -0,0 +1,96 @@ +'use client'; +import { useEffect, useState } from 'react'; +import AlgorithmSelector from '../../../components/AlgorithmSelector'; +import DynamicParamForm from '../../../components/DynamicParamForm'; +import { apiGet, apiPost } from '../../../lib/api'; +import type { AlgorithmPreset, DatasetItem, DatasetProfile } from '../../../lib/types'; + +export default function NewExperimentPage() { + const [datasets, setDatasets] = useState([]); + const [algorithms, setAlgorithms] = useState([]); + const [datasetId, setDatasetId] = useState(''); + const [profile, setProfile] = useState(null); + const [algorithm, setAlgorithm] = useState('kmeans'); + const [params, setParams] = useState>({}); + const [featureColumns, setFeatureColumns] = useState([]); + const [name, setName] = useState('Experiment'); + const [message, setMessage] = useState(''); + + useEffect(() => { + apiGet('/datasets').then(setDatasets).catch(() => {}); + apiGet<{ algorithms: AlgorithmPreset[] }>('/presets/algorithms').then((d) => { + setAlgorithms(d.algorithms); + if (d.algorithms.length > 0) { + setAlgorithm(d.algorithms[0].key); + setParams(d.algorithms[0].params); + } + }).catch(() => {}); + }, []); + + useEffect(() => { + if (!datasetId) return; + apiGet(`/datasets/${datasetId}/profile`).then((data) => { + setProfile(data); + setFeatureColumns(data.suggested_feature_columns); + }).catch(() => {}); + }, [datasetId]); + + useEffect(() => { + const match = algorithms.find((a) => a.key === algorithm); + if (match) setParams(match.params); + }, [algorithm, algorithms]); + + async function run() { + setMessage(''); + const result = await apiPost<{ experiment_id: string; status: string }>('/experiments/run', { + dataset_id: datasetId, + name, + algorithm, + feature_columns: featureColumns, + preprocessing: { numeric_scaler: 'standard', categorical_encoder: 'onehot', dimensionality_reduction: 'umap' }, + algorithm_params: params, + outlier_detection_enabled: true + }); + setMessage(`Completed: ${result.experiment_id}`); + } + + return ( +
+
+

New Experiment

+
+
setName(e.target.value)} />
+
+ + +
+ +
+ +