Buckets:

Hooshaai/RL_AEGIS / data /loaders /__init__.py
tahamajs's picture
download
raw
1.12 kB
"""Data loaders registry and exports."""
from typing import Optional
from data.loaders.base import BaseDataLoader, DataSample
from data.loaders.gsm8k import GSM8KLoader
from data.loaders.toxigen import ToxiGenLoader
from data.loaders.hc3 import HC3Loader
LOADER_REGISTRY = {
"gsm8k": GSM8KLoader,
"math": GSM8KLoader,
"toxigen": ToxiGenLoader,
"toxicity": ToxiGenLoader,
"hc3": HC3Loader,
"ai_text": HC3Loader,
}
def get_data_loader(
dataset_name: str,
split: str = "train",
max_samples: Optional[int] = None,
**kwargs,
) -> BaseDataLoader:
"""Factory function returning a BaseDataLoader for the given dataset."""
name_key = dataset_name.lower().strip()
if name_key not in LOADER_REGISTRY:
# Default to GSM8KLoader if unknown or generic
return GSM8KLoader(split=split, max_samples=max_samples, **kwargs)
return LOADER_REGISTRY[name_key](split=split, max_samples=max_samples, **kwargs)
__all__ = [
"BaseDataLoader",
"DataSample",
"GSM8KLoader",
"ToxiGenLoader",
"HC3Loader",
"get_data_loader",
"LOADER_REGISTRY",
]

Xet Storage Details

Size:
1.12 kB
·
Xet hash:
42bc74b73e4d7d4ebdb2ee6083e512c8c0601fd78241330122b6d7296223e986

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.