Download src/utils/loader.py from QuizGenCS/QuizGen: direct link, hf CLI and curl.
- Browser
- Download file 5.51 kB
-
https://huggingface.co/spaces/QuizGenCS/QuizGen/resolve/main/src/utils/loader.py
- Command line
-
hf download hf://spaces/QuizGenCS/QuizGen/src/utils/loader.py
-
curl -L -o loader.py https://huggingface.co/spaces/QuizGenCS/QuizGen/resolve/main/src/utils/loader.py
5.51 kB
| # src/utils/loader.py | |
| from pypdf import PdfReader | |
| from docx import Document | |
| import pandas as pd | |
| import io | |
| import re | |
| import os | |
| # --------------------------------------------- | |
| # 🧩 1. อ่านไฟล์ PDF | |
| # --------------------------------------------- | |
| def extract_text_from_pdf(file_path: str) -> str: | |
| """ | |
| ใช้เมื่อมี path ของไฟล์ PDF จริง (เช่นตอนรันบนเครื่อง) | |
| """ | |
| reader = PdfReader(file_path) | |
| all_text = "" | |
| for i, page in enumerate(reader.pages): | |
| text = page.extract_text() | |
| if text: | |
| all_text += f"\n\n--- Page {i+1} ---\n{text}" | |
| return all_text.strip() | |
| def extract_text_from_pdf_bytes(file_bytes: bytes) -> str: | |
| """ | |
| ใช้เมื่อไฟล์มาจาก Streamlit uploader (เป็น bytes) | |
| """ | |
| reader = PdfReader(io.BytesIO(file_bytes)) | |
| texts = [] | |
| for i, page in enumerate(reader.pages): | |
| text = page.extract_text() | |
| if text: | |
| texts.append(f"--- Page {i+1} ---\n{text}") | |
| return "\n\n".join(texts).strip() | |
| # --------------------------------------------- | |
| # 🧩 2. อ่านไฟล์ DOCX | |
| # --------------------------------------------- | |
| def extract_text_from_docx_bytes(file_bytes: bytes) -> str: | |
| doc = Document(io.BytesIO(file_bytes)) | |
| paras = [p.text for p in doc.paragraphs if p.text.strip()] | |
| return "\n".join(paras).strip() | |
| def extract_text_from_docx_path(file_path: str) -> str: | |
| doc = Document(file_path) | |
| paras = [p.text for p in doc.paragraphs if p.text.strip()] | |
| return "\n".join(paras).strip() | |
| # --------------------------------------------- | |
| # 🧩 3. อ่านไฟล์ TXT / CSV | |
| # --------------------------------------------- | |
| def extract_text_from_txt_bytes(file_bytes: bytes, encoding="utf-8") -> str: | |
| try: | |
| return file_bytes.decode(encoding, errors="ignore") | |
| except Exception: | |
| return file_bytes.decode("utf-8", errors="ignore") | |
| def extract_text_from_txt_path(file_path: str, encoding="utf-8") -> str: | |
| with open(file_path, "r", encoding=encoding, errors="ignore") as f: | |
| return f.read() | |
| def extract_text_from_csv_bytes(file_bytes: bytes) -> str: | |
| df = pd.read_csv(io.BytesIO(file_bytes)) | |
| # รวมทุกคอลัมน์ทุกแถวให้เป็นข้อความเดียว (เพื่อสร้าง quiz ภายหลัง) | |
| lines = [] | |
| for _, row in df.astype(str).iterrows(): | |
| lines.append(" | ".join(row.values)) | |
| return "\n".join(lines) | |
| def extract_text_from_csv_path(file_path: str) -> str: | |
| df = pd.read_csv(file_path) | |
| lines = [] | |
| for _, row in df.astype(str).iterrows(): | |
| lines.append(" | ".join(row.values)) | |
| return "\n".join(lines) | |
| # --------------------------------------------- | |
| # 🧩 4. Clean ข้อความ (ตัดช่องว่าง / จัดรูปแบบ) | |
| # --------------------------------------------- | |
| def clean_text(text: str) -> str: | |
| text = re.sub(r'\s+\n', '\n', text) # เว้นวรรคก่อนขึ้นบรรทัด | |
| text = re.sub(r'\n{3,}', '\n\n', text) # ตัดบรรทัดว่างเกิน 2 บรรทัด | |
| text = re.sub(r'[ \t]{2,}', ' ', text) # ตัดช่องว่างยาว | |
| return text.strip() | |
| # --------------------------------------------- | |
| # 🧩 5. ฟังก์ชันรวมหลัก (Auto-detect) | |
| # --------------------------------------------- | |
| def load_to_text(uploaded_file) -> str: | |
| """ | |
| ใช้ใน Streamlit: รับ uploaded_file แล้วคืนข้อความทั้งหมดเป็น string เดียว | |
| (รองรับ PDF / DOCX / CSV / TXT ทั้งบน local และ Hugging Face) | |
| """ | |
| # ✅ ป้องกันกรณี .read() แล้ว buffer หาย | |
| if hasattr(uploaded_file, "getvalue"): | |
| file_bytes = uploaded_file.getvalue() | |
| else: | |
| file_bytes = uploaded_file.read() | |
| name = uploaded_file.name.lower() | |
| mime = getattr(uploaded_file, "type", "") or "" | |
| if name.endswith(".pdf"): | |
| text = extract_text_from_pdf_bytes(file_bytes) | |
| elif name.endswith(".docx"): | |
| text = extract_text_from_docx_bytes(file_bytes) | |
| elif name.endswith(".csv"): | |
| text = extract_text_from_csv_bytes(file_bytes) | |
| elif name.endswith(".txt") or mime.startswith("text/"): | |
| text = extract_text_from_txt_bytes(file_bytes) | |
| else: | |
| # fallback พยายามอ่านเป็นข้อความ | |
| try: | |
| text = extract_text_from_txt_bytes(file_bytes) | |
| except Exception: | |
| text = "" | |
| return clean_text(text) | |
| def load_from_path(file_path: str) -> str: | |
| """ | |
| ใช้นอก Streamlit (อ่านจาก path ปกติ) | |
| """ | |
| file_path = str(file_path) | |
| if not os.path.exists(file_path): | |
| raise FileNotFoundError(file_path) | |
| if file_path.lower().endswith(".pdf"): | |
| text = extract_text_from_pdf(file_path) | |
| elif file_path.lower().endswith(".docx"): | |
| text = extract_text_from_docx_path(file_path) | |
| elif file_path.lower().endswith(".csv"): | |
| text = extract_text_from_csv_path(file_path) | |
| elif file_path.lower().endswith(".txt"): | |
| text = extract_text_from_txt_path(file_path) | |
| else: | |
| text = "" | |
| return clean_text(text) |