| """Convert simple source JSONL records into the AEGIS input contract. | |
| Usage: python data/scripts/preprocess.py input.jsonl output.jsonl --domain math | |
| """ | |
| import argparse, json | |
| from pathlib import Path | |
| def main() -> None: | |
| parser = argparse.ArgumentParser() | |
| parser.add_argument('source', type=Path) | |
| parser.add_argument('destination', type=Path) | |
| parser.add_argument('--domain', default='general') | |
| args = parser.parse_args() | |
| args.destination.parent.mkdir(parents=True, exist_ok=True) | |
| with args.source.open(encoding='utf-8') as source, args.destination.open('w', encoding='utf-8') as destination: | |
| for line in source: | |
| if not line.strip(): | |
| continue | |
| item = json.loads(line) | |
| text = item.get('input') or item.get('question') or item.get('text') or item.get('prompt') | |
| if not text: | |
| continue | |
| destination.write(json.dumps({'input': text, 'label': item.get('label') or item.get('answer'), 'domain': args.domain}, ensure_ascii=False) + '\n') | |
| if __name__ == '__main__': | |
| main() | |
Xet Storage Details
- Size:
- 1.1 kB
- Xet hash:
- 594518c4c4b341bd723b03ae8c3d2e3b25c24226fedbfa154ca0de9df271017c
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.