grassmann-cache / src /services /dataset_service.py
Nicholas Finch
fix: default workers=0 (sequential) for Docker cache — ProcessPoolExecutor hangs on cpu-basic
d7013e6
Raw History Blame Contribute Delete
12.4 kB
"""
Dataset Service — orchestrates data loading, splitting, and DataLoader creation.
Delegates external data access to repositories:
- DatasetRepository: HF Hub fetching, schema resolution, streaming
- CacheRepository: disk cache read/write, HF shard upload/download
This service handles: train/val/test splitting, PyTorch Dataset wrappers,
DataLoader creation, and collation.
"""
from __future__ import annotations
from typing import Self
import torch
import torchaudio.transforms as T
from torch.nn.utils.rnn import pad_sequence
from torch.utils.data import DataLoader, Dataset
from src.models.config import GrassmannConfig
from src.repositories.cache_repository import CacheRepository
from src.repositories.dataset_repository import DatasetRepository
from src.services.tokenizer_service import CharTokenizer
# -- PyTorch Dataset wrappers ---------------------------------------------
class MaterializedAudioDataset(Dataset):
"""Pre-materialized dataset -- samples already in memory as (mel, tokens) pairs."""
def __init__(self, samples: list[tuple[torch.Tensor, torch.Tensor]]):
self.samples = samples
def __len__(self) -> int:
return len(self.samples)
def __getitem__(self, idx: int) -> tuple[torch.Tensor, torch.Tensor]:
return self.samples[idx]
class DiskCachedDataset(Dataset):
"""Dataset backed by individual .pt files on disk."""
def __init__(self, cache: CacheRepository, indices: list[int]):
self._cache = cache
self._indices = indices
def __len__(self) -> int:
return len(self._indices)
def __getitem__(self, idx: int) -> tuple[torch.Tensor, torch.Tensor]:
return self._cache.load(self._indices[idx])
class HFAudioDataset(Dataset):
"""Wraps a (non-streaming) HuggingFace dataset split into a PyTorch Dataset."""
def __init__(self, hf_dataset, repo: DatasetRepository,
audio_col: str, text_col: str):
self.hf_dataset = hf_dataset
self._repo = repo
self.audio_col = audio_col
self.text_col = text_col
def __len__(self) -> int:
return len(self.hf_dataset)
def __getitem__(self, idx: int) -> tuple[torch.Tensor, torch.Tensor]:
sample = self.hf_dataset[idx]
mel, text = self._repo.process_sample(sample, self.audio_col, self.text_col)
tokens = self._repo.tokenizer.encode(text)
return mel, tokens
def collate_fn(
batch: list[tuple[torch.Tensor, torch.Tensor]],
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""Pad mels and tokens to batch-max lengths."""
mels, tokens = zip(*batch)
mel_lengths = torch.tensor([m.shape[-1] for m in mels], dtype=torch.long)
max_mel_len = mel_lengths.max().item()
padded_mels = []
for m in mels:
pad_amount = max_mel_len - m.shape[-1]
padded_mels.append(torch.nn.functional.pad(m, (0, pad_amount)))
mel_batch = torch.stack(padded_mels)
token_lengths = torch.tensor([len(t) for t in tokens], dtype=torch.long)
token_batch = pad_sequence(tokens, batch_first=True, padding_value=0)
return mel_batch, token_batch, mel_lengths, token_lengths
# -- Service ---------------------------------------------------------------
class DatasetService:
"""Orchestrates data loading, splitting, and DataLoader creation.
Usage:
repo = DatasetRepository(config, tokenizer)
cache = CacheRepository("checkpoints/mel_cache")
svc = DatasetService(repo, cache)
# From HF Hub cache:
svc.load_from_hub_cache("tisaconundrum/grassmann-mel-cache")
svc.split(max_samples=28539)
# From recipe:
svc.load_recipe("1750h")
svc.split()
# Direct HF download:
svc.load_dataset("openslr/librispeech_asr", split="train.clean.100")
svc.split()
train_loader = svc.get_train_loader(batch_size=32)
"""
def __init__(self, repo: DatasetRepository, cache: CacheRepository):
self._repo = repo
self._cache = cache
self._hf_train = None
self._hf_val = None
self._hf_test = None
self._materialized: list[tuple[torch.Tensor, torch.Tensor]] | None = None
self._use_cache = False
self._audio_col = "audio"
self._text_col = "text"
self._train_ds: Dataset | None = None
self._val_ds: Dataset | None = None
self._test_ds: Dataset | None = None
def load_from_hub_cache(self, repo_id: str) -> Self:
"""Download pre-cached mel shards from a HF Dataset repo."""
self._cache.download_from_hub(repo_id)
self._use_cache = True
return self
def load_recipe(self, recipe_name: str,
max_samples_per_source: int | None = None,
upload_repo: str | None = None,
samples_per_shard: int = 50000,
workers: int = 0) -> Self:
"""Stream multiple datasets per recipe and cache to disk.
When upload_repo is set, shards are uploaded incrementally to HF Hub
as they fill up — progress survives container restarts.
"""
sources = self._repo.get_recipe(recipe_name)
print(f" Loading recipe '{recipe_name}': {len(sources)} sources")
if upload_repo:
print(f" Incremental upload to: {upload_repo}")
for src in sources:
cfg_str = f" ({src['config']})" if src.get('config') else ""
print(f" - {src['name']} / {src['split']}{cfg_str}")
for src in sources:
src_cap = src.get("max_samples") or max_samples_per_source
stream = self._repo.stream(
src["name"], src["split"],
config_name=src.get("config"),
max_samples=src_cap,
workers=workers,
)
label = f"{src['name']}/{src['split']}"
count = self._cache.save_stream(
stream, source_label=label,
repo_id=upload_repo, samples_per_shard=samples_per_shard,
)
print(f" {label}: {count} samples")
self._use_cache = True
return self
def load_dataset(self, dataset_name: str, split: str | None = None,
config_name: str | None = None,
max_samples: int | None = None) -> Self:
"""Load a single dataset -- streaming or full download."""
self._audio_col, self._text_col = self._repo.resolve_columns(dataset_name)
print(f" Loading dataset: {dataset_name}")
print(f" Columns: audio='{self._audio_col}', text='{self._text_col}'")
if max_samples is not None:
stream = self._repo.stream(
dataset_name, split or "train",
config_name=config_name, max_samples=max_samples,
)
self._materialized = list(stream)
print(f" Materialized {len(self._materialized)} samples into memory.")
return self
# Full download
if split:
self._hf_train = self._repo.fetch_full(dataset_name, split, config_name)
print(f" Loaded split '{split}': {len(self._hf_train)} samples")
else:
try:
self._hf_train = self._repo.fetch_full(dataset_name, "train", config_name)
except Exception:
pass
try:
self._hf_val = self._repo.fetch_full(dataset_name, "validation", config_name)
except Exception:
pass
try:
self._hf_test = self._repo.fetch_full(dataset_name, "test", config_name)
except Exception:
pass
sizes = []
if self._hf_train: sizes.append(f"train={len(self._hf_train)}")
if self._hf_val: sizes.append(f"val={len(self._hf_val)}")
if self._hf_test: sizes.append(f"test={len(self._hf_test)}")
print(f" Loaded splits: {', '.join(sizes)}")
return self
def clean(self) -> Self:
"""Filter out empty/invalid samples from full-download datasets."""
if self._hf_train is not None and hasattr(self._hf_train, "filter"):
original_len = len(self._hf_train)
a, t = self._audio_col, self._text_col
self._hf_train = self._hf_train.filter(
lambda x: len(x[t].strip()) > 0 and len(x[a]["array"]) > 1600
)
print(f" Cleaned: {original_len} -> {len(self._hf_train)} samples")
return self
def preprocess(self) -> Self:
return self
def split(self, train_ratio: float = 0.8, val_ratio: float = 0.1,
max_samples: int | None = None) -> Self:
"""Split loaded data into train/val/test."""
if self._use_cache:
all_indices = self._cache.indices()
if max_samples is not None:
all_indices = all_indices[:max_samples]
n = len(all_indices)
n_train = max(1, int(n * train_ratio))
n_val = max(1, int(n * val_ratio))
self._train_ds = DiskCachedDataset(self._cache, all_indices[:n_train])
self._val_ds = DiskCachedDataset(self._cache, all_indices[n_train:n_train + n_val])
rest = all_indices[n_train + n_val:]
self._test_ds = DiskCachedDataset(self._cache, rest if rest else all_indices[n_train:n_train + n_val])
elif self._materialized is not None:
samples = self._materialized
n = len(samples)
n_train = max(1, int(n * train_ratio))
n_val = max(1, int(n * val_ratio))
self._train_ds = MaterializedAudioDataset(samples[:n_train])
self._val_ds = MaterializedAudioDataset(samples[n_train:n_train + n_val])
rest = samples[n_train + n_val:]
self._test_ds = MaterializedAudioDataset(rest if rest else samples[n_train:n_train + n_val])
elif self._hf_val is not None:
self._train_ds = HFAudioDataset(self._hf_train, self._repo, self._audio_col, self._text_col)
self._val_ds = HFAudioDataset(self._hf_val, self._repo, self._audio_col, self._text_col)
self._test_ds = HFAudioDataset(self._hf_test, self._repo, self._audio_col, self._text_col) if self._hf_test else self._val_ds
elif self._hf_train is not None:
n = len(self._hf_train)
n_train = int(n * train_ratio)
n_val = int(n * val_ratio)
split_ds = self._hf_train.train_test_split(test_size=(n - n_train) / n, seed=42)
train_hf = split_ds["train"]
remaining = split_ds["test"]
if len(remaining) > 1:
val_test = remaining.train_test_split(
test_size=max(1, n - n_train - n_val) / len(remaining), seed=42)
val_hf, test_hf = val_test["train"], val_test["test"]
else:
val_hf = test_hf = remaining
self._train_ds = HFAudioDataset(train_hf, self._repo, self._audio_col, self._text_col)
self._val_ds = HFAudioDataset(val_hf, self._repo, self._audio_col, self._text_col)
self._test_ds = HFAudioDataset(test_hf, self._repo, self._audio_col, self._text_col)
if self._train_ds:
print(f" Splits ready: train={len(self._train_ds)}, "
f"val={len(self._val_ds) if self._val_ds else 0}, "
f"test={len(self._test_ds) if self._test_ds else 0}")
return self
def get_train_loader(self, batch_size: int) -> DataLoader:
assert self._train_ds is not None, "Call load_*() and split() first"
return DataLoader(self._train_ds, batch_size=batch_size, shuffle=True,
collate_fn=collate_fn, num_workers=0)
def get_val_loader(self, batch_size: int) -> DataLoader:
assert self._val_ds is not None, "Call load_*() and split() first"
return DataLoader(self._val_ds, batch_size=batch_size, shuffle=False,
collate_fn=collate_fn, num_workers=0)
def get_test_loader(self, batch_size: int) -> DataLoader:
assert self._test_ds is not None, "Call load_*() and split() first"
return DataLoader(self._test_ds, batch_size=batch_size, shuffle=False,
collate_fn=collate_fn, num_workers=0)