File size: 641 Bytes
cd6775d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 | import json
import bz2
import os
import csv
import pandas as pd
path = '/home/icml01/multi_rag/RAG/Search-in-the-Chain/data/strategyqa_corpus.csv'
def load_data(path=None):
assert path
corpus = []
count = 0
with open(
'/home/icml01/multi_rag/RAG/Search-in-the-Chain/data/strategyqa.tsv',
'w', encoding='utf-8-sig') as w:
writer = csv.writer(w, delimiter="\t")
df = pd.read_csv(path)
for index, row in df.iterrows():
writer.writerow([count, 'Title: '+row['title']+' Text: '+ row['text']])
count += 1
print(count)
return corpus
load_data(path)
|