Spaces:
No application file
No application file
Download Data_Preprocessing.py from Shreevats/jiopayragbackend: direct link, hf CLI and curl.
- Browser
- Download file 2.61 kB
-
https://huggingface.co/spaces/Shreevats/jiopayragbackend/resolve/main/Data_Preprocessing.py
- Command line
-
hf download hf://spaces/Shreevats/jiopayragbackend/Data_Preprocessing.py
-
curl -L -o Data_Preprocessing.py https://huggingface.co/spaces/Shreevats/jiopayragbackend/resolve/main/Data_Preprocessing.py
2.61 kB
| import json | |
| import pickle | |
| import os | |
| from langchain.text_splitter import RecursiveCharacterTextSplitter | |
| import pandas as pd | |
| # Cell 4: Data Processing Functions | |
| def process_jio_data(input_file): | |
| with open(input_file, "r", encoding="utf-8") as f: # Specify UTF-8 encoding | |
| data = json.load(f) | |
| documents = [] | |
| text_splitter = RecursiveCharacterTextSplitter( | |
| chunk_size=1000, | |
| chunk_overlap=200, | |
| length_function=len | |
| ) | |
| for url, content in data.items(): | |
| # Extract FAQs | |
| if 'help_center' in content and 'faqs' in content['help_center']: | |
| for section in content['help_center']['faqs']: | |
| for qna in section.get('questions', []): | |
| # Debug: Print to check data format | |
| if not isinstance(qna, dict): | |
| print(f"Skipping invalid FAQ entry: {qna}") | |
| continue | |
| question = qna.get('question', "Unknown Question") | |
| answer = qna.get('answer', "Unknown Answer") | |
| text = f"Q: {question}\nA: {answer}" | |
| chunks = text_splitter.split_text(text) | |
| for chunk in chunks: | |
| documents.append({ | |
| "text": chunk, | |
| "metadata": { | |
| "source_url": url, | |
| "section_type": "FAQ", | |
| "section_title": section.get('section', 'Unknown Section') | |
| } | |
| }) | |
| # Extract General Content | |
| if 'description' in content: | |
| chunks = text_splitter.split_text(content['description']) | |
| for chunk in chunks: | |
| documents.append({ | |
| "text": chunk, | |
| "metadata": { | |
| "source_url": url, | |
| "section_type": "General", | |
| "section_title": "Description" | |
| } | |
| }) | |
| return documents | |
| # Cell 5: Run Processing | |
| if os.path.exists("./faq.json"): | |
| documents = process_jio_data("./faq.json") | |
| # Save processed data | |
| with open('processed_docs.pkl', 'wb') as f: | |
| pickle.dump(documents, f) | |
| # Download processed data | |
| # files.download('processed_docs.pkl') | |
| # print("Processing complete! File downloaded as 'processed_docs.pkl'.") | |
| else: | |
| print("Error: File processing failed due to missing input file.") | |
| df = pd.DataFrame(documents) | |
| print(df.head()) # Show first few rows | |