Spaces:
Configuration error
Configuration error
| import os | |
| import json | |
| import re | |
| from typing import List, Tuple | |
| def extract_text_from_file(file_path: str, file_type: str) -> Tuple[str, List[str]]: | |
| text = "" | |
| try: | |
| if file_type == ".txt": | |
| text = _extract_txt(file_path) | |
| elif file_type == ".pdf": | |
| text = _extract_pdf(file_path) | |
| elif file_type == ".docx": | |
| text = _extract_docx(file_path) | |
| elif file_type in (".png", ".jpg", ".jpeg", ".gif"): | |
| text = _extract_image_metadata(file_path, file_type) | |
| elif file_type in (".mp3", ".wav", ".m4a", ".webm", ".mp4"): | |
| text = _extract_audio_metadata(file_path, file_type) | |
| else: | |
| text = f"[File: {os.path.basename(file_path)} - file type {file_type} is stored but text extraction is not supported]" | |
| except Exception as e: | |
| text = f"[Error extracting text from {os.path.basename(file_path)}: {str(e)}]" | |
| chunks = chunk_text(text) if text else [] | |
| return text, chunks | |
| def _extract_txt(file_path: str) -> str: | |
| encodings = ["utf-8", "latin-1", "cp1252"] | |
| for enc in encodings: | |
| try: | |
| with open(file_path, "r", encoding=enc) as f: | |
| return f.read() | |
| except (UnicodeDecodeError, UnicodeError): | |
| continue | |
| return "" | |
| def _extract_pdf(file_path: str) -> str: | |
| try: | |
| from PyPDF2 import PdfReader | |
| reader = PdfReader(file_path) | |
| text_parts = [] | |
| for page in reader.pages: | |
| page_text = page.extract_text() | |
| if page_text: | |
| text_parts.append(page_text) | |
| return "\n\n".join(text_parts) | |
| except Exception as e: | |
| return f"[PDF extraction error: {e}]" | |
| def _extract_docx(file_path: str) -> str: | |
| try: | |
| from docx import Document | |
| doc = Document(file_path) | |
| return "\n\n".join(p.text for p in doc.paragraphs if p.text.strip()) | |
| except Exception as e: | |
| return f"[DOCX extraction error: {e}]" | |
| def _extract_image_metadata(file_path: str, file_type: str) -> str: | |
| size = os.path.getsize(file_path) | |
| return f"[Image file: {os.path.basename(file_path)}, type: {file_type}, size: {size} bytes. Upload text descriptions or notes about this image as separate text files for better memory recall.]" | |
| def _extract_audio_metadata(file_path: str, file_type: str) -> str: | |
| size = os.path.getsize(file_path) | |
| return f"[Audio file: {os.path.basename(file_path)}, type: {file_type}, size: {size} bytes. You can describe the content of this audio in chat to add it to the memory context.]" | |
| def chunk_text(text: str, chunk_size: int = 1000, overlap: int = 200) -> List[str]: | |
| if not text or not text.strip(): | |
| return [] | |
| text = text.strip() | |
| if len(text) <= chunk_size: | |
| return [text] | |
| paragraphs = re.split(r'\n\s*\n', text) | |
| chunks = [] | |
| current_chunk = "" | |
| for para in paragraphs: | |
| para = para.strip() | |
| if not para: | |
| continue | |
| if len(current_chunk) + len(para) + 2 <= chunk_size: | |
| current_chunk = f"{current_chunk}\n\n{para}" if current_chunk else para | |
| else: | |
| if current_chunk: | |
| chunks.append(current_chunk) | |
| if overlap > 0: | |
| words = current_chunk.split() | |
| overlap_words = words[-overlap // 5:] if len(words) > overlap // 5 else [] | |
| current_chunk = " ".join(overlap_words) + "\n\n" + para if overlap_words else para | |
| else: | |
| current_chunk = para | |
| else: | |
| if len(para) > chunk_size: | |
| words = para.split() | |
| temp = "" | |
| for word in words: | |
| if len(temp) + len(word) + 1 <= chunk_size: | |
| temp = f"{temp} {word}" if temp else word | |
| else: | |
| chunks.append(temp) | |
| temp = word | |
| current_chunk = temp | |
| else: | |
| current_chunk = para | |
| if current_chunk: | |
| chunks.append(current_chunk) | |
| return chunks if chunks else [text[:chunk_size]] | |