| import pandas as pd | |
| import json | |
| import ollama | |
| import re | |
| from tqdm import tqdm | |
| tqdm.pandas() | |
| from pathlib import Path | |
| data_dir = Path("../datasets") | |
| # faq qa pair | |
| train_faq_df = pd.read_csv(data_dir / "processed" / "bank_faq" / "train_faq.csv") | |
| train_faq_df["QA_pair"] = "Q: " + train_faq_df["Question"] + "\nA: " + train_faq_df["Answer"] | |
| train_faq_df.to_csv(data_dir / "processed" / "bank_faq" / "train_faq.csv", index=False) | |
| # train_faq_df = train_faq_df[["Question", "Answer", "QA_pair", "Class"]].rename(columns={ | |
| # "Question": "question", | |
| # "Answer": "answer", | |
| # "Class": "domain" | |
| # }) | |
| # train_faq_df["source"] = "faq" | |
| # train_faq_df["industry"] = "banking" | |