from schemas import DatasetSchema from validator import ColumnValidator FileFormat = Literal["csv", "xlsx", "json"] def _infer_format(path: str | Path) -> FileFormat: ext = Path(path).suffix.lower() if ext == ".csv": return "csv" if ext in {".xlsx", ".xls"}: return "xlsx" if ext == ".json": return "json" raise ValueError(f"Unsupported file extension: {ext}") def read_dataset( path: str | Path, *, fmt: Optional[FileFormat] = None, schema: Optional[DatasetSchema] = None, dataset_name: Optional[str] = None, **read_kwargs, ) -> pd.DataFrame: path = Path(path) file_format = fmt or _infer_format(path) dataset_name = dataset_name or path.name if file_format == "csv": df = pd.read_csv(path, **read_kwargs) elif file_format == "xlsx": df = pd.read_excel(path, **read_kwargs) elif file_format == "json": df = pd.read_json(path, **read_kwargs) else: raise ValueError(f"Unsupported file format: {file_format}") if schema is not None: validator = ColumnValidator(schema) validator.validate(df, dataset_name=dataset_name) df = validator.align_columns(df) return df def write_dataset( df: pd.DataFrame, path: str | Path, *, fmt: Optional[FileFormat] = None, schema: Optional[DatasetSchema] = None, dataset_name: Optional[str] = None, index: bool = False, **write_kwargs, ) -> None: path = Path(path) file_format = fmt or _infer_format(path) dataset_name = dataset_name or path.name if schema is not None: validator = ColumnValidator(schema) validator.validate(df, dataset_name=dataset_name) df = validator.align_columns(df) if file_format == "csv": df.to_csv(path, index=index, **write_kwargs) elif file_format == "xlsx": df.to_excel(path, index=index, **write_kwargs) elif file_format == "json": df.to_json(path, **write_kwargs) else: raise ValueError(f"Unsupported file format: {file_format}")