Spaces:
Sleeping
Sleeping
File size: 1,817 Bytes
9aa1cd7 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 | """
This module provides classes for file parsing and document representation.
"""
import os
from time import sleep
from streamlit.runtime.uploaded_file_manager import UploadedFile
from llama_index.readers.schema import Document
from llama_index import SimpleDirectoryReader
from llama_index import download_loader
UnstructuredReader = download_loader("UnstructuredReader")
def save_to_temp_file(file_obj: UploadedFile):
"""
Save the contents of an uploaded file to a temporary file.
Args:
file_obj (UploadedFile): The uploaded file object.
Returns:
str: The path of the temporary file where the contents are saved.
"""
file_path = os.path.join(
os.environ["DATA_DIR"], f"{file_obj.file_id}.{file_obj.name}"
)
with open(file_path, "wb") as temp_file:
content = file_obj.read()
temp_file.write(content)
print(f"file saved to {file_path}")
return file_path
class File:
"""
This class defines the interface for parsing files and representing documents.
"""
def __init__(self, file: UploadedFile):
self.file_path = save_to_temp_file(file)
self.documents: list[Document] = self._parse_file()
def __del__(self):
"""
Delete the temporary file.
"""
os.remove(self.file_path)
def _parse_file(self):
"""
Load the documents from the file.
"""
return SimpleDirectoryReader(
input_files=[self.file_path],
# file_extractor={
# ".pdf": UnstructuredReader(),
# ".docx": UnstructuredReader(),
# ".md": UnstructuredReader(),
# ".txt": UnstructuredReader(),
# ".rst": UnstructuredReader(),
# },
).load_data()
|