v802-classifier / src /dataset_utils.py
sneo's picture
Add v802 classifier head and source code
59839e3 verified
Raw History Blame Contribute Delete
1.31 kB
from __future__ import annotations
import csv
from dataclasses import dataclass
from pathlib import Path
import numpy as np
@dataclass(frozen=True)
class Dataset:
image_paths: tuple[Path, ...]
file_names: np.ndarray
labels: np.ndarray
classes: tuple[str, ...]
def read_rows(path: Path) -> list[dict[str, str]]:
with path.open(newline="", encoding="utf-8-sig") as handle:
return list(csv.DictReader(handle))
def load_dataset(root: Path) -> Dataset:
rows = read_rows(root / "human-tagged.txt.txt")
classes = tuple(sorted({label for row in rows for label in row["characters"].split("|")}))
class_to_index = {name: index for index, name in enumerate(classes)}
image_paths: list[Path] = []
labels: list[np.ndarray] = []
for row in rows:
path = root / row["file_name"]
if not path.is_file():
raise FileNotFoundError(path)
target = np.zeros(len(classes), dtype=np.uint8)
for name in row["characters"].split("|"):
target[class_to_index[name]] = 1
image_paths.append(path)
labels.append(target)
return Dataset(
image_paths=tuple(image_paths),
file_names=np.asarray([path.name for path in image_paths]),
labels=np.stack(labels),
classes=classes,
)