File size: 689 Bytes
cd6775d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 | import json
import bz2
import os
import csv
import pandas as pd
import pickle
dataset = 'manyqa_text'
path = f'/data1/liuyaoyang/Papers/icml2025/datasets/{dataset}/corpus.pkl'
with open(path, "rb") as file:
data = pickle.load(file)
def load_data(path=None):
assert path
corpus = []
count = 0
with open(
f'/data1/liuyaoyang/Papers/icml2025/multi_rag/RAG/Search-in-the-Chain/data/{dataset}.tsv',
'w', encoding='utf-8-sig') as w:
writer = csv.writer(w, delimiter="\t")
for item in list(data.values()):
writer.writerow([count, item['text']])
count += 1
print(count)
return corpus
load_data(path)
|