Spaces:
No application file
No application file
File size: 2,611 Bytes
e693532 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 | import json
import pickle
import os
from langchain.text_splitter import RecursiveCharacterTextSplitter
import pandas as pd
# Cell 4: Data Processing Functions
def process_jio_data(input_file):
with open(input_file, "r", encoding="utf-8") as f: # Specify UTF-8 encoding
data = json.load(f)
documents = []
text_splitter = RecursiveCharacterTextSplitter(
chunk_size=1000,
chunk_overlap=200,
length_function=len
)
for url, content in data.items():
# Extract FAQs
if 'help_center' in content and 'faqs' in content['help_center']:
for section in content['help_center']['faqs']:
for qna in section.get('questions', []):
# Debug: Print to check data format
if not isinstance(qna, dict):
print(f"Skipping invalid FAQ entry: {qna}")
continue
question = qna.get('question', "Unknown Question")
answer = qna.get('answer', "Unknown Answer")
text = f"Q: {question}\nA: {answer}"
chunks = text_splitter.split_text(text)
for chunk in chunks:
documents.append({
"text": chunk,
"metadata": {
"source_url": url,
"section_type": "FAQ",
"section_title": section.get('section', 'Unknown Section')
}
})
# Extract General Content
if 'description' in content:
chunks = text_splitter.split_text(content['description'])
for chunk in chunks:
documents.append({
"text": chunk,
"metadata": {
"source_url": url,
"section_type": "General",
"section_title": "Description"
}
})
return documents
# Cell 5: Run Processing
if os.path.exists("./faq.json"):
documents = process_jio_data("./faq.json")
# Save processed data
with open('processed_docs.pkl', 'wb') as f:
pickle.dump(documents, f)
# Download processed data
# files.download('processed_docs.pkl')
# print("Processing complete! File downloaded as 'processed_docs.pkl'.")
else:
print("Error: File processing failed due to missing input file.")
df = pd.DataFrame(documents)
print(df.head()) # Show first few rows
|