File size: 642 Bytes
7c8c4c9 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 | from datasets import load_dataset, Dataset, concatenate_datasets
from src.data.synthetic import synthetic_finance_data
def load_data():
fiqa = load_dataset("pauri32/fiqa-2018")["train"]
def format_fn(example):
return {
"text": f"""
Instruction: You are a financial analyst.
Input: {example.get('question', '')}
Response: {example.get('answer', '')}
"""
}
fiqa = fiqa.map(format_fn)
# SAFE LIMIT (no crash)
fiqa = fiqa.select(range(min(2000, len(fiqa))))
synthetic = Dataset.from_list(synthetic_finance_data())
dataset = concatenate_datasets([fiqa, synthetic])
return dataset |