RAG_Chatbot / data_preparation /chatbot_data_prep.py

added chatbot codes

b22c324 4 months ago

5.18 kB

# data preparation for chatbot
from pathlib import Path
import pandas as pd
import re
from tqdm import tqdm
from nltk.tokenize import word_tokenize
import nltk
from wordfreq import word_frequency, zipf_frequency
try:
    nltk.data.find("tokenizers/punkt")
except LookupError:
    nltk.download("punkt")
from nltk.corpus import words

nltk.download('words', quiet=True)
from lingua import Language
from lingua import LanguageDetectorBuilder
from gibberish_detector import detector
tqdm.pandas()
from sklearn.model_selection import train_test_split

# load datasets
data_dir = Path("../datasets")

faq_df = pd.read_csv(data_dir / "raw" / "BankFAQs.csv")
support_df = pd.read_csv(data_dir / "raw" / "customer_support_data.csv")

print("FAQ dataset")
print(faq_df.head())
print("Customer Support dataset")
print(support_df.head())

eng_words = set(word.lower() for word in words.words())

# load the gibberish detection model
Detector = detector.create_from_model('gibberish-detector.model')

stopwords = {
    "ho", "rahi", "hai", "ke", "mein", "raha", "hoon", "kar"
}

# cleaning customer support dataset
def clean_text(text):
    # handle missing values
    if pd.isna(text):
        return ""

    text = text.replace("â€™", "'").replace("’", "'")
    # text = re.sub(r"[^a-zA-Z0-9'\s-]", ' ', text)
    # text = re.sub(r'\s+', ' ', text).strip()

    # split into phrases by punctuation
    phrases = re.split(r'[.!?,;:/]', text)
    # print(phrases)
    
    clean_phrases = []
    total_removed = 0
    
    for phrase in phrases:
        phrase = phrase.strip()
        if not phrase:
            continue

        tokens = re.findall(r"[a-zA-Z0-9]*[0-9][a-zA-Z][a-zA-Z0-9]*|[a-zA-Z]+(?:'[a-zA-Z]+)*|[0-9]+(?:-[0-9]+)*", phrase)
        
        real_words = []
        removed_words = []
    
        for token in tokens:
            if re.fullmatch(r"[a-zA-Z0-9]*[0-9][a-zA-Z]+", token):
                real_words.append(token)
                continue

            if re.fullmatch(r"[0-9]+(?:-[0-9]+)*", token):
                real_words.append(token)
                continue

            token_lower = token.lower()

            if token_lower in stopwords:
                removed_words.append(token)
                continue

            if Detector.is_gibberish(token_lower):
                # print(token)
                removed_words.append(token)
                continue

            if token_lower in {"a", "i"}:
                real_words.append(token)
                continue
            
            # contractions - keep directly without English check
            if "'" in token_lower:
                real_words.append(token)
                continue

            if token_lower in eng_words or zipf_frequency(token_lower, "en") >= 2.0:
                real_words.append(token)
                continue

            removed_words.append(token)

        # print("Real Words: ", real_words)
        # print(f"  Removed words : {removed_words} ({len(removed_words)} removed)")

        if not real_words:
            continue

        # keep phrases when at least a reasonable portion of tokens are English-like
        # if len(real_words) / len(tokens) < 0.25:
        #     continue

        if len(removed_words) >= len(real_words):
            continue
        
        total_removed += len(removed_words)
        clean_phrases.append(" ".join(real_words))

    # print(f"\nTotal words removed: {total_removed}")
    return " ".join(clean_phrases)

# cleaned_support_df = support_df[support_df["language"] == "en"][["conv_id", "turn_index", "role", "text", "industry", "product", "outcome", "issue_type", "overall_urgency"]].copy()
# cleaned_support_df["text"] = cleaned_support_df["text"].progress_apply(clean_text)

# cleaned_support_df = cleaned_support_df[cleaned_support_df["text"].str.strip() != ""]
# cleaned_support_df = cleaned_support_df.dropna(subset=["text"])
# cleaned_support_df.to_csv(data_dir / "processed" / "customer_support" / "cleaned_support_text.csv", index=False)



# row_text = support_df.loc[support_df["language"] == "en", "text"].iloc[4]

# cleaned_text = clean_text(row_text)
# print(row_text)
# print(cleaned_text)


# cleaning bank faqs
faq_df = faq_df.drop_duplicates(subset=["Question", "Answer"])

def clean_text_faq(text):
    if not isinstance(text, str):
        return text
    text = re.sub(r"[\r\n\t]+", " ", text)             # normalise line breaks
    text = re.sub(r"[^\x00-\x7F]+", "", text)
    text = re.sub(r"<<\s*>>", "", text)  
    text = re.sub(r" {2,}", " ", text)                 # collapse spaces
    return text.strip()

for col in ["Question", "Answer"]:
    faq_df[col] = faq_df[col].progress_apply(clean_text_faq)
faq_df.to_csv(data_dir / "processed" / "bank_faq" / "cleaned_faq.csv", index=False)

# split dataset for train/test
train_faq_df, test_faq_df = train_test_split(
    faq_df,
    test_size=0.2,
    stratify=faq_df["Class"],
    random_state=42
)

print(len(train_faq_df))
print(len(test_faq_df))

train_faq_df.to_csv(data_dir / "processed" / "bank_faq" / "train_faq.csv", index=False)
test_faq_df.to_csv(data_dir / "processed" / "bank_faq" / "test_faq.csv", index=False)

	# data preparation for chatbot
	from pathlib import Path
	import pandas as pd
	import re
	from tqdm import tqdm
	from nltk.tokenize import word_tokenize
	import nltk
	from wordfreq import word_frequency, zipf_frequency
	try:
	nltk.data.find("tokenizers/punkt")
	except LookupError:
	nltk.download("punkt")
	from nltk.corpus import words

	nltk.download('words', quiet=True)
	from lingua import Language
	from lingua import LanguageDetectorBuilder
	from gibberish_detector import detector
	tqdm.pandas()
	from sklearn.model_selection import train_test_split

	# load datasets
	data_dir = Path("../datasets")

	faq_df = pd.read_csv(data_dir / "raw" / "BankFAQs.csv")
	support_df = pd.read_csv(data_dir / "raw" / "customer_support_data.csv")

	print("FAQ dataset")
	print(faq_df.head())
	print("Customer Support dataset")
	print(support_df.head())

	eng_words = set(word.lower() for word in words.words())

	# load the gibberish detection model
	Detector = detector.create_from_model('gibberish-detector.model')

	stopwords = {
	"ho", "rahi", "hai", "ke", "mein", "raha", "hoon", "kar"
	}

	# cleaning customer support dataset
	def clean_text(text):
	# handle missing values
	if pd.isna(text):
	return ""

	text = text.replace("â€™", "'").replace("’", "'")
	# text = re.sub(r"[^a-zA-Z0-9'\s-]", ' ', text)
	# text = re.sub(r'\s+', ' ', text).strip()

	# split into phrases by punctuation
	phrases = re.split(r'[.!?,;:/]', text)
	# print(phrases)

	clean_phrases = []
	total_removed = 0

	for phrase in phrases:
	phrase = phrase.strip()
	if not phrase:
	continue

	tokens = re.findall(r"[a-zA-Z0-9][0-9][a-zA-Z][a-zA-Z0-9]\|[a-zA-Z]+(?:'[a-zA-Z]+)\|[0-9]+(?:-[0-9]+)", phrase)

	real_words = []
	removed_words = []

	for token in tokens:
	if re.fullmatch(r"[a-zA-Z0-9]*[0-9][a-zA-Z]+", token):
	real_words.append(token)
	continue

	if re.fullmatch(r"[0-9]+(?:-[0-9]+)*", token):
	real_words.append(token)
	continue

	token_lower = token.lower()

	if token_lower in stopwords:
	removed_words.append(token)
	continue

	if Detector.is_gibberish(token_lower):
	# print(token)
	removed_words.append(token)
	continue

	if token_lower in {"a", "i"}:
	real_words.append(token)
	continue

	# contractions - keep directly without English check
	if "'" in token_lower:
	real_words.append(token)
	continue

	if token_lower in eng_words or zipf_frequency(token_lower, "en") >= 2.0:
	real_words.append(token)
	continue

	removed_words.append(token)

	# print("Real Words: ", real_words)
	# print(f" Removed words : {removed_words} ({len(removed_words)} removed)")

	if not real_words:
	continue

	# keep phrases when at least a reasonable portion of tokens are English-like
	# if len(real_words) / len(tokens) < 0.25:
	# continue

	if len(removed_words) >= len(real_words):
	continue

	total_removed += len(removed_words)
	clean_phrases.append(" ".join(real_words))

	# print(f"\nTotal words removed: {total_removed}")
	return " ".join(clean_phrases)

	# cleaned_support_df = support_df[support_df["language"] == "en"][["conv_id", "turn_index", "role", "text", "industry", "product", "outcome", "issue_type", "overall_urgency"]].copy()
	# cleaned_support_df["text"] = cleaned_support_df["text"].progress_apply(clean_text)

	# cleaned_support_df = cleaned_support_df[cleaned_support_df["text"].str.strip() != ""]
	# cleaned_support_df = cleaned_support_df.dropna(subset=["text"])
	# cleaned_support_df.to_csv(data_dir / "processed" / "customer_support" / "cleaned_support_text.csv", index=False)



	# row_text = support_df.loc[support_df["language"] == "en", "text"].iloc[4]

	# cleaned_text = clean_text(row_text)
	# print(row_text)
	# print(cleaned_text)


	# cleaning bank faqs
	faq_df = faq_df.drop_duplicates(subset=["Question", "Answer"])

	def clean_text_faq(text):
	if not isinstance(text, str):
	return text
	text = re.sub(r"[\r\n\t]+", " ", text) # normalise line breaks
	text = re.sub(r"[^\x00-\x7F]+", "", text)
	text = re.sub(r"<<\s*>>", "", text)
	text = re.sub(r" {2,}", " ", text) # collapse spaces
	return text.strip()

	for col in ["Question", "Answer"]:
	faq_df[col] = faq_df[col].progress_apply(clean_text_faq)
	faq_df.to_csv(data_dir / "processed" / "bank_faq" / "cleaned_faq.csv", index=False)

	# split dataset for train/test
	train_faq_df, test_faq_df = train_test_split(
	faq_df,
	test_size=0.2,
	stratify=faq_df["Class"],
	random_state=42
	)

	print(len(train_faq_df))
	print(len(test_faq_df))

	train_faq_df.to_csv(data_dir / "processed" / "bank_faq" / "train_faq.csv", index=False)
	test_faq_df.to_csv(data_dir / "processed" / "bank_faq" / "test_faq.csv", index=False)