thomasjacob04 commited on
Commit
63be344
·
verified ·
1 Parent(s): 318ea2d

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +18 -2
app.py CHANGED
@@ -2,11 +2,13 @@ import os
2
  from typing import Iterable
3
  from tempfile import TemporaryDirectory
4
  from dotenv import load_dotenv
5
- from docling import DoclingPDFLoader
 
6
  from langchain.text_splitter import RecursiveCharacterTextSplitter
7
  from langchain_huggingface.embeddings import HuggingFaceEmbeddings
8
  from langchain_milvus import Milvus
9
  from langchain_huggingface import HuggingFaceEndpoint
 
10
  from langchain_core.documents import Document as LCDocument
11
  from langchain_core.output_parsers import StrOutputParser
12
  from langchain_core.prompts import PromptTemplate
@@ -15,9 +17,23 @@ from fastapi import FastAPI, HTTPException
15
  from pydantic import BaseModel
16
  import gradio as gr
17
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
18
  # Load environment variables
19
  load_dotenv()
20
-
21
  # File paths configuration
22
  FILE_PATHS = ["vol1.pdf", "vol2.pdf", "vol3.pdf"]
23
 
 
2
  from typing import Iterable
3
  from tempfile import TemporaryDirectory
4
  from dotenv import load_dotenv
5
+ from typing import Iterator
6
+ from docling.document_converter import DocumentConverter
7
  from langchain.text_splitter import RecursiveCharacterTextSplitter
8
  from langchain_huggingface.embeddings import HuggingFaceEmbeddings
9
  from langchain_milvus import Milvus
10
  from langchain_huggingface import HuggingFaceEndpoint
11
+ from langchain_core.document_loaders import BaseLoader
12
  from langchain_core.documents import Document as LCDocument
13
  from langchain_core.output_parsers import StrOutputParser
14
  from langchain_core.prompts import PromptTemplate
 
17
  from pydantic import BaseModel
18
  import gradio as gr
19
 
20
+ class DoclingPDFLoader(BaseLoader):
21
+ def __init__(self, file_paths: str | list[str]) -> None:
22
+ self._file_paths = file_paths if isinstance(file_paths, list) else [file_paths]
23
+ self._converter = DocumentConverter()
24
+
25
+ def lazy_load(self) -> Iterator[LCDocument]:
26
+ for source in self._file_paths:
27
+ dl_doc = self._converter.convert(source).document
28
+ text = dl_doc.export_to_markdown()
29
+ yield LCDocument(page_content=text)
30
+
31
+ def load(self) -> list[LCDocument]:
32
+ return list(self.lazy_load())
33
+
34
  # Load environment variables
35
  load_dotenv()
36
+
37
  # File paths configuration
38
  FILE_PATHS = ["vol1.pdf", "vol2.pdf", "vol3.pdf"]
39