jiopayragbackend / Data_Preprocessing.py
Shreevats's picture
Upload 5 files
e693532 verified
Raw History Blame Contribute Delete
2.61 kB
import json
import pickle
import os
from langchain.text_splitter import RecursiveCharacterTextSplitter
import pandas as pd
# Cell 4: Data Processing Functions
def process_jio_data(input_file):
with open(input_file, "r", encoding="utf-8") as f: # Specify UTF-8 encoding
data = json.load(f)
documents = []
text_splitter = RecursiveCharacterTextSplitter(
chunk_size=1000,
chunk_overlap=200,
length_function=len
)
for url, content in data.items():
# Extract FAQs
if 'help_center' in content and 'faqs' in content['help_center']:
for section in content['help_center']['faqs']:
for qna in section.get('questions', []):
# Debug: Print to check data format
if not isinstance(qna, dict):
print(f"Skipping invalid FAQ entry: {qna}")
continue
question = qna.get('question', "Unknown Question")
answer = qna.get('answer', "Unknown Answer")
text = f"Q: {question}\nA: {answer}"
chunks = text_splitter.split_text(text)
for chunk in chunks:
documents.append({
"text": chunk,
"metadata": {
"source_url": url,
"section_type": "FAQ",
"section_title": section.get('section', 'Unknown Section')
}
})
# Extract General Content
if 'description' in content:
chunks = text_splitter.split_text(content['description'])
for chunk in chunks:
documents.append({
"text": chunk,
"metadata": {
"source_url": url,
"section_type": "General",
"section_title": "Description"
}
})
return documents
# Cell 5: Run Processing
if os.path.exists("./faq.json"):
documents = process_jio_data("./faq.json")
# Save processed data
with open('processed_docs.pkl', 'wb') as f:
pickle.dump(documents, f)
# Download processed data
# files.download('processed_docs.pkl')
# print("Processing complete! File downloaded as 'processed_docs.pkl'.")
else:
print("Error: File processing failed due to missing input file.")
df = pd.DataFrame(documents)
print(df.head()) # Show first few rows