Update app.py
Browse files
app.py
CHANGED
|
@@ -25,15 +25,15 @@ openai.api_key = os.getenv('OPENAI_API_KEY')
|
|
| 25 |
docs_dir = 'documents/'
|
| 26 |
txts_dir = docs_dir + 'txt/'
|
| 27 |
converted_txts_dir = docs_dir + 'converted_txt/'
|
| 28 |
-
pdfs_dir = docs_dir + 'pdf/'
|
| 29 |
-
txt_documents = [os.path.splitext(doc)[0] for doc in list(os.listdir(converted_txts_dir))]
|
| 30 |
-
pdf_documents = [os.path.splitext(doc)[0] for doc in list(os.listdir(pdfs_dir))]
|
| 31 |
-
for pdf_doc in pdf_documents:
|
| 32 |
-
|
| 33 |
-
|
| 34 |
-
|
| 35 |
-
|
| 36 |
-
|
| 37 |
# intents_df = pd.read_csv(docs_dir + 'intents.csv')
|
| 38 |
# intents_df.sort_values(by=['intent', 'response'], inplace=True)
|
| 39 |
# previous_row = None
|
|
@@ -45,7 +45,7 @@ for pdf_doc in pdf_documents:
|
|
| 45 |
# with open(f'{docs_dir}intents/{row["intent"]}_{intent_index}.txt', 'w') as f:
|
| 46 |
# f.write(row['response'])
|
| 47 |
# previous_row = row
|
| 48 |
-
loader = DirectoryLoader(
|
| 49 |
documents = loader.load()
|
| 50 |
def wrap_text_preserve_newlines(text, width=110):
|
| 51 |
# Split the input text into lines based on newline characters
|
|
|
|
| 25 |
docs_dir = 'documents/'
|
| 26 |
txts_dir = docs_dir + 'txt/'
|
| 27 |
converted_txts_dir = docs_dir + 'converted_txt/'
|
| 28 |
+
# pdfs_dir = docs_dir + 'pdf/'
|
| 29 |
+
# txt_documents = [os.path.splitext(doc)[0] for doc in list(os.listdir(converted_txts_dir))]
|
| 30 |
+
# pdf_documents = [os.path.splitext(doc)[0] for doc in list(os.listdir(pdfs_dir))]
|
| 31 |
+
# for pdf_doc in pdf_documents:
|
| 32 |
+
# if pdf_doc not in txt_documents:
|
| 33 |
+
# print("Converting " + pdf_doc + ' pdf to txt')
|
| 34 |
+
# text = extract_text(pdfs_dir + pdf_doc + '.pdf')
|
| 35 |
+
# with open(converted_txts_dir + pdf_doc + '.txt', 'w') as f:
|
| 36 |
+
# f.write(text)
|
| 37 |
# intents_df = pd.read_csv(docs_dir + 'intents.csv')
|
| 38 |
# intents_df.sort_values(by=['intent', 'response'], inplace=True)
|
| 39 |
# previous_row = None
|
|
|
|
| 45 |
# with open(f'{docs_dir}intents/{row["intent"]}_{intent_index}.txt', 'w') as f:
|
| 46 |
# f.write(row['response'])
|
| 47 |
# previous_row = row
|
| 48 |
+
loader = DirectoryLoader(txts_dir)
|
| 49 |
documents = loader.load()
|
| 50 |
def wrap_text_preserve_newlines(text, width=110):
|
| 51 |
# Split the input text into lines based on newline characters
|