File size: 642 Bytes
7c8c4c9
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
from datasets import load_dataset, Dataset, concatenate_datasets
from src.data.synthetic import synthetic_finance_data

def load_data():

    fiqa = load_dataset("pauri32/fiqa-2018")["train"]

    def format_fn(example):
        return {
            "text": f"""
Instruction: You are a financial analyst.
Input: {example.get('question', '')}
Response: {example.get('answer', '')}
"""
        }

    fiqa = fiqa.map(format_fn)

    # SAFE LIMIT (no crash)
    fiqa = fiqa.select(range(min(2000, len(fiqa))))

    synthetic = Dataset.from_list(synthetic_finance_data())

    dataset = concatenate_datasets([fiqa, synthetic])

    return dataset