| """Script to inspect and pre-process datasets for AEGIS training and evaluation.""" | |
| import argparse | |
| import json | |
| import os | |
| from aegis.data.loaders import get_data_loader | |
| from aegis.utils.logging import get_logger | |
| logger = get_logger("PreprocessData") | |
| def main(): | |
| parser = argparse.ArgumentParser(description="Preprocess and verify datasets for AEGIS") | |
| parser.add_argument("--dataset", type=str, default="gsm8k", help="Dataset name: gsm8k, toxigen, hc3") | |
| parser.add_argument("--split", type=str, default="train", help="Split name") | |
| parser.add_argument("--max_samples", type=int, default=10, help="Number of samples to preview") | |
| parser.add_argument("--save_jsonl", type=str, default=None, help="Optional output JSONL path") | |
| args = parser.parse_args() | |
| logger.info(f"Loading dataset '{args.dataset}' (split: {args.split})...") | |
| loader = get_data_loader(args.dataset, split=args.split, max_samples=args.max_samples) | |
| samples = loader.get_samples() | |
| logger.info(f"Loaded {len(samples)} samples successfully.") | |
| for i, s in enumerate(samples[: min(3, len(samples))]): | |
| logger.info(f"\n--- Sample {i+1} ---") | |
| logger.info(f"Input: {s.input[:150]}...") | |
| logger.info(f"Label: {s.label}") | |
| logger.info(f"Metadata: {s.metadata}") | |
| if args.save_jsonl: | |
| os.makedirs(os.path.dirname(os.path.abspath(args.save_jsonl)), exist_ok=True) | |
| with open(args.save_jsonl, "w", encoding="utf-8") as f: | |
| for s in samples: | |
| f.write(json.dumps({"input": s.input, "label": s.label, "metadata": s.metadata}) + "\n") | |
| logger.info(f"Saved samples to {args.save_jsonl}") | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 1.7 kB
- Xet hash:
- c0e7bda31c9fa08dac5b379100d85573161238177aa9a83976fb253465a0ce28
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.