Download app.py from anomaly05/final: direct link, hf CLI and curl.
- Browser
- Download file 6.53 kB
-
https://huggingface.co/spaces/anomaly05/final/resolve/main/app.py
- Command line
-
hf download hf://spaces/anomaly05/final/app.py
-
curl -L -o app.py https://huggingface.co/spaces/anomaly05/final/resolve/main/app.py
6.53 kB
| import os | |
| import numpy as np | |
| from PyPDF2 import PdfReader | |
| import docx | |
| import faiss | |
| import torch | |
| from sentence_transformers import SentenceTransformer | |
| from transformers import pipeline, AutoTokenizer | |
| import gradio as gr | |
| import re | |
| ########################### | |
| # CONFIGURATION | |
| ########################### | |
| DATA_DIR = "." # Look in the root directory of the Space | |
| MAX_CHUNKS = 10 # Smaller batches = faster loading & inference | |
| MAX_RESPONSE_CHARS = 600 # Hard cutoff to prevent runaway generation | |
| ########################### | |
| # DATA EXTRACTION & CHUNKING | |
| ########################### | |
| def extract_text(file_path): | |
| ext = os.path.splitext(file_path)[1].lower() | |
| if ext == '.pdf': | |
| text = '' | |
| with open(file_path, 'rb') as f: | |
| reader = PdfReader(f) | |
| for page in reader.pages: | |
| t = page.extract_text() | |
| if t: text += t + ' ' | |
| return text | |
| elif ext == '.docx': | |
| docf = docx.Document(file_path) | |
| return ' '.join([p.text for p in docf.paragraphs]) | |
| elif ext == '.txt': | |
| with open(file_path, 'r', encoding='utf-8') as f: | |
| return f.read() | |
| return '' | |
| def chunk_text(text, max_len=300): | |
| sents = text.split('. ') | |
| chunks, chunk = [], '' | |
| for s in sents: | |
| if len(chunk) + len(s) < max_len: | |
| chunk += s + '. ' | |
| else: | |
| if chunk.strip(): | |
| chunks.append(chunk.strip()) | |
| chunk = s + '. ' | |
| if chunk.strip(): | |
| chunks.append(chunk.strip()) | |
| return chunks | |
| all_chunks, all_meta = [], [] | |
| if os.path.exists(DATA_DIR) and len(os.listdir(DATA_DIR)) > 0: | |
| for fname in os.listdir(DATA_DIR): | |
| if not fname.lower().endswith(('.pdf', '.docx', '.txt')): | |
| continue | |
| file_path = os.path.join(DATA_DIR, fname) | |
| subject = 'general' | |
| try: | |
| text = extract_text(file_path) | |
| c = chunk_text(text, max_len=300) | |
| for chunk in c: | |
| if len(chunk) > 40: | |
| all_chunks.append(chunk) | |
| all_meta.append({'subject': subject, 'file': fname}) | |
| if len(all_chunks) >= MAX_CHUNKS: | |
| break | |
| if len(all_chunks) >= MAX_CHUNKS: | |
| break | |
| except Exception as e: | |
| print(f"Failed {file_path}: {e}") | |
| if len(all_chunks) >= MAX_CHUNKS: | |
| break | |
| print('Chunks loaded:', len(all_chunks)) | |
| else: | |
| print(f"DATA_DIR '{DATA_DIR}' not found or empty! Please upload your textbook files.") | |
| ########################### | |
| # EMBEDDINGS & INDEX | |
| ########################### | |
| if len(all_chunks) > 0: | |
| encoder = SentenceTransformer('all-MiniLM-L6-v2') | |
| embeddings = encoder.encode(all_chunks, show_progress_bar=False, batch_size=8) | |
| faiss.normalize_L2(embeddings) | |
| index = faiss.IndexFlatIP(embeddings.shape[1]) | |
| index.add(np.array(embeddings).astype('float32')) | |
| else: | |
| encoder, embeddings, index = None, None, None | |
| ########################### | |
| # CLEAN-UP FUNCTION | |
| ########################### | |
| def clean_and_remove_incomplete_last_sentence(text): | |
| """ | |
| Removes duplicate sentences and deletes the last (possibly incomplete) sentence | |
| if it doesn't end with a period (or ! or ?). | |
| """ | |
| sentences = re.split(r'(?<=[.!?]) +', text) | |
| seen, unique = set(), [] | |
| for s in sentences: | |
| st = s.strip() | |
| s_key = st.lower() | |
| if st and s_key not in seen: | |
| unique.append(st) | |
| seen.add(s_key) | |
| if unique and not unique[-1].endswith(('.', '!', '?')): | |
| unique = unique[:-1] | |
| return ' '.join(unique).strip() | |
| ########################### | |
| # ANSWER FUNCTION | |
| ########################### | |
| def answer_query( | |
| question, | |
| subject=None, | |
| top_k=1, | |
| model_name="Qwen/Qwen2-0.5B" | |
| ): | |
| if encoder is None or index is None: | |
| return "No textbooks found or processed." | |
| q_emb = encoder.encode([question]) | |
| faiss.normalize_L2(q_emb) | |
| D, I = index.search(np.asarray(q_emb, dtype='float32'), top_k) | |
| found = [] | |
| for idx in I[0]: | |
| if subject: | |
| if subject.lower() in all_meta[idx]['subject'].lower(): | |
| found.append(all_chunks[idx]) | |
| else: | |
| found.append(all_chunks[idx]) | |
| if len(found) >= top_k: break | |
| context = "\n".join(found) | |
| prompt = ( | |
| "You are an ICSE board AI tutor. Answer ONLY using the textbook excerpt below. " | |
| "Give a COMPLETE, single, concise answer, ending with proper punctuation. " | |
| "Do NOT repeat yourself or echo the prompt. End your answer with a full stop.\n\n" | |
| f"Textbook excerpt:\n{context}\n\n" | |
| f"Student question: {question}\n" | |
| "Answer:" | |
| ) | |
| tokenizer = AutoTokenizer.from_pretrained(model_name) | |
| pipe = pipeline( | |
| "text-generation", | |
| model=model_name, | |
| tokenizer=tokenizer, | |
| torch_dtype=torch.float16 if torch.cuda.is_available() else torch.float32, | |
| device=0 if torch.cuda.is_available() else -1, | |
| max_new_tokens=64, # or even 80 for slightly longer output | |
| do_sample=False, | |
| pad_token_id=tokenizer.eos_token_id | |
| ) | |
| output = pipe(prompt)[0]['generated_text'] | |
| answer_part = output.split("Answer:")[-1].strip() | |
| for stop_word in ["Textbook", "Student question:", "Context:"]: | |
| if stop_word in answer_part: | |
| answer_part = answer_part.split(stop_word)[0].strip() | |
| clean = clean_and_remove_incomplete_last_sentence(answer_part) | |
| if len(clean) > MAX_RESPONSE_CHARS: | |
| clean = clean[:MAX_RESPONSE_CHARS] + "\n\n[truncated]" | |
| if not clean.strip(): | |
| return "Sorry, no answer found in the textbook." | |
| return clean | |
| ########################### | |
| # GRADIO APP | |
| ########################### | |
| def gradio_chatbot(user_input, subject=""): | |
| subject_input = subject.strip() if subject.strip() else None | |
| return answer_query(user_input, subject=subject_input) | |
| TITLE = "ICSE Textbook Chatbot" | |
| DESC = "Ask any question using ONLY your uploaded ICSE textbooks. The AI answers strictly from your syllabus content." | |
| iface = gr.Interface( | |
| fn=gradio_chatbot, | |
| inputs=[ | |
| gr.Textbox(label="Your question", placeholder="Type your question here...", lines=2), | |
| gr.Textbox(label="Subject (optional)", placeholder="e.g., Physics, Geography", lines=1), | |
| ], | |
| outputs="text", | |
| title=TITLE, | |
| description=DESC, | |
| allow_flagging="never" | |
| ) | |
| if __name__ == "__main__": | |
| iface.launch(share=True) | |