File size: 2,083 Bytes
98d7b66
 
e2855b7
d6b3672
e2855b7
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
from schemas import DatasetSchema
from validator import ColumnValidator


FileFormat = Literal["csv", "xlsx", "json"]


def _infer_format(path: str | Path) -> FileFormat:
    ext = Path(path).suffix.lower()
    if ext == ".csv":
        return "csv"
    if ext in {".xlsx", ".xls"}:
        return "xlsx"
    if ext == ".json":
        return "json"
    raise ValueError(f"Unsupported file extension: {ext}")


def read_dataset(
    path: str | Path,
    *,
    fmt: Optional[FileFormat] = None,
    schema: Optional[DatasetSchema] = None,
    dataset_name: Optional[str] = None,
    **read_kwargs,
) -> pd.DataFrame:
    path = Path(path)
    file_format = fmt or _infer_format(path)
    dataset_name = dataset_name or path.name

    if file_format == "csv":
        df = pd.read_csv(path, **read_kwargs)
    elif file_format == "xlsx":
        df = pd.read_excel(path, **read_kwargs)
    elif file_format == "json":
        df = pd.read_json(path, **read_kwargs)
    else:
        raise ValueError(f"Unsupported file format: {file_format}")

    if schema is not None:
        validator = ColumnValidator(schema)
        validator.validate(df, dataset_name=dataset_name)
        df = validator.align_columns(df)

    return df


def write_dataset(
    df: pd.DataFrame,
    path: str | Path,
    *,
    fmt: Optional[FileFormat] = None,
    schema: Optional[DatasetSchema] = None,
    dataset_name: Optional[str] = None,
    index: bool = False,
    **write_kwargs,
) -> None:
    path = Path(path)
    file_format = fmt or _infer_format(path)
    dataset_name = dataset_name or path.name

    if schema is not None:
        validator = ColumnValidator(schema)
        validator.validate(df, dataset_name=dataset_name)
        df = validator.align_columns(df)

    if file_format == "csv":
        df.to_csv(path, index=index, **write_kwargs)
    elif file_format == "xlsx":
        df.to_excel(path, index=index, **write_kwargs)
    elif file_format == "json":
        df.to_json(path, **write_kwargs)
    else:
        raise ValueError(f"Unsupported file format: {file_format}")