| """Data loaders registry and exports.""" | |
| from typing import Optional | |
| from data.loaders.base import BaseDataLoader, DataSample | |
| from data.loaders.gsm8k import GSM8KLoader | |
| from data.loaders.toxigen import ToxiGenLoader | |
| from data.loaders.hc3 import HC3Loader | |
| LOADER_REGISTRY = { | |
| "gsm8k": GSM8KLoader, | |
| "math": GSM8KLoader, | |
| "toxigen": ToxiGenLoader, | |
| "toxicity": ToxiGenLoader, | |
| "hc3": HC3Loader, | |
| "ai_text": HC3Loader, | |
| } | |
| def get_data_loader( | |
| dataset_name: str, | |
| split: str = "train", | |
| max_samples: Optional[int] = None, | |
| **kwargs, | |
| ) -> BaseDataLoader: | |
| """Factory function returning a BaseDataLoader for the given dataset.""" | |
| name_key = dataset_name.lower().strip() | |
| if name_key not in LOADER_REGISTRY: | |
| # Default to GSM8KLoader if unknown or generic | |
| return GSM8KLoader(split=split, max_samples=max_samples, **kwargs) | |
| return LOADER_REGISTRY[name_key](split=split, max_samples=max_samples, **kwargs) | |
| __all__ = [ | |
| "BaseDataLoader", | |
| "DataSample", | |
| "GSM8KLoader", | |
| "ToxiGenLoader", | |
| "HC3Loader", | |
| "get_data_loader", | |
| "LOADER_REGISTRY", | |
| ] | |
Xet Storage Details
- Size:
- 1.12 kB
- Xet hash:
- 42bc74b73e4d7d4ebdb2ee6083e512c8c0601fd78241330122b6d7296223e986
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.