Spaces:
Runtime error
Runtime error
File size: 2,329 Bytes
644f326 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 | from langchain_core.document_loaders.base import BaseLoader
from langchain_core.documents import Document
from typing import Iterator, List
from src.core.exception import MyException
from src.core.logger import logger
import sys
class TextLoader(BaseLoader):
def __init__(self, file_path: str):
self.file_path = file_path
def lazy_load(self) -> Iterator[Document]:
with logger.span("Lazy loading text file: {file_path}", file_path=self.file_path):
f = None
try:
logger.info("Opening text file", file_path=self.file_path)
f = open(file=self.file_path, encoding="utf-8", errors="ignore")
content = f.read()
logger.info("Text file read successfully", file_path=self.file_path, char_count=len(content))
yield Document(page_content=content, metadata={"source": self.file_path, "page": 0})
logger.info("Text lazy load complete", file_path=self.file_path)
except Exception as e:
logger.error("Failed to lazy load text file", file_path=self.file_path, error=str(e))
raise MyException(e, sys)
finally:
if f:
f.close()
def load(self) -> List[Document]:
with logger.span("Loading text file into memory: {file_path}", file_path=self.file_path):
try:
docs = list(self.lazy_load())
logger.info("Text file load complete", file_path=self.file_path, doc_count=len(docs))
return docs
except Exception as e:
logger.error("Failed to load text file", file_path=self.file_path, error=str(e))
raise MyException(e, sys)
async def text_loader(file_path: str):
with logger.span("Async text loader: {file_path}", file_path=file_path):
try:
logger.info("Initiating text file load", file_path=file_path)
loader = TextLoader(file_path=file_path)
docs = await loader.aload()
logger.info("Async text load finished", file_path=file_path, doc_count=len(docs))
return docs
except Exception as e:
logger.error("Async text loader failed", file_path=file_path, error=str(e))
raise MyException(e, sys)
|