Agile_Lab / data_io.py
jonels1988's picture
Update data_io.py
d6b3672 verified
Raw History Blame Contribute Delete
2.08 kB
from schemas import DatasetSchema
from validator import ColumnValidator
FileFormat = Literal["csv", "xlsx", "json"]
def _infer_format(path: str | Path) -> FileFormat:
ext = Path(path).suffix.lower()
if ext == ".csv":
return "csv"
if ext in {".xlsx", ".xls"}:
return "xlsx"
if ext == ".json":
return "json"
raise ValueError(f"Unsupported file extension: {ext}")
def read_dataset(
path: str | Path,
*,
fmt: Optional[FileFormat] = None,
schema: Optional[DatasetSchema] = None,
dataset_name: Optional[str] = None,
**read_kwargs,
) -> pd.DataFrame:
path = Path(path)
file_format = fmt or _infer_format(path)
dataset_name = dataset_name or path.name
if file_format == "csv":
df = pd.read_csv(path, **read_kwargs)
elif file_format == "xlsx":
df = pd.read_excel(path, **read_kwargs)
elif file_format == "json":
df = pd.read_json(path, **read_kwargs)
else:
raise ValueError(f"Unsupported file format: {file_format}")
if schema is not None:
validator = ColumnValidator(schema)
validator.validate(df, dataset_name=dataset_name)
df = validator.align_columns(df)
return df
def write_dataset(
df: pd.DataFrame,
path: str | Path,
*,
fmt: Optional[FileFormat] = None,
schema: Optional[DatasetSchema] = None,
dataset_name: Optional[str] = None,
index: bool = False,
**write_kwargs,
) -> None:
path = Path(path)
file_format = fmt or _infer_format(path)
dataset_name = dataset_name or path.name
if schema is not None:
validator = ColumnValidator(schema)
validator.validate(df, dataset_name=dataset_name)
df = validator.align_columns(df)
if file_format == "csv":
df.to_csv(path, index=index, **write_kwargs)
elif file_format == "xlsx":
df.to_excel(path, index=index, **write_kwargs)
elif file_format == "json":
df.to_json(path, **write_kwargs)
else:
raise ValueError(f"Unsupported file format: {file_format}")