Spaces:
Sleeping
Sleeping
File size: 2,083 Bytes
98d7b66 e2855b7 d6b3672 e2855b7 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 | from schemas import DatasetSchema
from validator import ColumnValidator
FileFormat = Literal["csv", "xlsx", "json"]
def _infer_format(path: str | Path) -> FileFormat:
ext = Path(path).suffix.lower()
if ext == ".csv":
return "csv"
if ext in {".xlsx", ".xls"}:
return "xlsx"
if ext == ".json":
return "json"
raise ValueError(f"Unsupported file extension: {ext}")
def read_dataset(
path: str | Path,
*,
fmt: Optional[FileFormat] = None,
schema: Optional[DatasetSchema] = None,
dataset_name: Optional[str] = None,
**read_kwargs,
) -> pd.DataFrame:
path = Path(path)
file_format = fmt or _infer_format(path)
dataset_name = dataset_name or path.name
if file_format == "csv":
df = pd.read_csv(path, **read_kwargs)
elif file_format == "xlsx":
df = pd.read_excel(path, **read_kwargs)
elif file_format == "json":
df = pd.read_json(path, **read_kwargs)
else:
raise ValueError(f"Unsupported file format: {file_format}")
if schema is not None:
validator = ColumnValidator(schema)
validator.validate(df, dataset_name=dataset_name)
df = validator.align_columns(df)
return df
def write_dataset(
df: pd.DataFrame,
path: str | Path,
*,
fmt: Optional[FileFormat] = None,
schema: Optional[DatasetSchema] = None,
dataset_name: Optional[str] = None,
index: bool = False,
**write_kwargs,
) -> None:
path = Path(path)
file_format = fmt or _infer_format(path)
dataset_name = dataset_name or path.name
if schema is not None:
validator = ColumnValidator(schema)
validator.validate(df, dataset_name=dataset_name)
df = validator.align_columns(df)
if file_format == "csv":
df.to_csv(path, index=index, **write_kwargs)
elif file_format == "xlsx":
df.to_excel(path, index=index, **write_kwargs)
elif file_format == "json":
df.to_json(path, **write_kwargs)
else:
raise ValueError(f"Unsupported file format: {file_format}")
|