Spaces:
Sleeping
Sleeping
Download data_io.py from jonels1988/Agile_Lab: direct link, hf CLI and curl.
- Browser
- Download file 2.08 kB
-
https://huggingface.co/spaces/jonels1988/Agile_Lab/resolve/main/data_io.py
- Command line
-
hf download hf://spaces/jonels1988/Agile_Lab/data_io.py
-
curl -L -o data_io.py https://huggingface.co/spaces/jonels1988/Agile_Lab/resolve/main/data_io.py
2.08 kB
| from schemas import DatasetSchema | |
| from validator import ColumnValidator | |
| FileFormat = Literal["csv", "xlsx", "json"] | |
| def _infer_format(path: str | Path) -> FileFormat: | |
| ext = Path(path).suffix.lower() | |
| if ext == ".csv": | |
| return "csv" | |
| if ext in {".xlsx", ".xls"}: | |
| return "xlsx" | |
| if ext == ".json": | |
| return "json" | |
| raise ValueError(f"Unsupported file extension: {ext}") | |
| def read_dataset( | |
| path: str | Path, | |
| *, | |
| fmt: Optional[FileFormat] = None, | |
| schema: Optional[DatasetSchema] = None, | |
| dataset_name: Optional[str] = None, | |
| **read_kwargs, | |
| ) -> pd.DataFrame: | |
| path = Path(path) | |
| file_format = fmt or _infer_format(path) | |
| dataset_name = dataset_name or path.name | |
| if file_format == "csv": | |
| df = pd.read_csv(path, **read_kwargs) | |
| elif file_format == "xlsx": | |
| df = pd.read_excel(path, **read_kwargs) | |
| elif file_format == "json": | |
| df = pd.read_json(path, **read_kwargs) | |
| else: | |
| raise ValueError(f"Unsupported file format: {file_format}") | |
| if schema is not None: | |
| validator = ColumnValidator(schema) | |
| validator.validate(df, dataset_name=dataset_name) | |
| df = validator.align_columns(df) | |
| return df | |
| def write_dataset( | |
| df: pd.DataFrame, | |
| path: str | Path, | |
| *, | |
| fmt: Optional[FileFormat] = None, | |
| schema: Optional[DatasetSchema] = None, | |
| dataset_name: Optional[str] = None, | |
| index: bool = False, | |
| **write_kwargs, | |
| ) -> None: | |
| path = Path(path) | |
| file_format = fmt or _infer_format(path) | |
| dataset_name = dataset_name or path.name | |
| if schema is not None: | |
| validator = ColumnValidator(schema) | |
| validator.validate(df, dataset_name=dataset_name) | |
| df = validator.align_columns(df) | |
| if file_format == "csv": | |
| df.to_csv(path, index=index, **write_kwargs) | |
| elif file_format == "xlsx": | |
| df.to_excel(path, index=index, **write_kwargs) | |
| elif file_format == "json": | |
| df.to_json(path, **write_kwargs) | |
| else: | |
| raise ValueError(f"Unsupported file format: {file_format}") | |