mdevora commited on
Commit
d97f90c
·
1 Parent(s): 9c5a860

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +10 -10
app.py CHANGED
@@ -25,15 +25,15 @@ openai.api_key = os.getenv('OPENAI_API_KEY')
25
  docs_dir = 'documents/'
26
  txts_dir = docs_dir + 'txt/'
27
  converted_txts_dir = docs_dir + 'converted_txt/'
28
- pdfs_dir = docs_dir + 'pdf/'
29
- txt_documents = [os.path.splitext(doc)[0] for doc in list(os.listdir(converted_txts_dir))]
30
- pdf_documents = [os.path.splitext(doc)[0] for doc in list(os.listdir(pdfs_dir))]
31
- for pdf_doc in pdf_documents:
32
- if pdf_doc not in txt_documents:
33
- print("Converting " + pdf_doc + ' pdf to txt')
34
- text = extract_text(pdfs_dir + pdf_doc + '.pdf')
35
- with open(converted_txts_dir + pdf_doc + '.txt', 'w') as f:
36
- f.write(text)
37
  # intents_df = pd.read_csv(docs_dir + 'intents.csv')
38
  # intents_df.sort_values(by=['intent', 'response'], inplace=True)
39
  # previous_row = None
@@ -45,7 +45,7 @@ for pdf_doc in pdf_documents:
45
  # with open(f'{docs_dir}intents/{row["intent"]}_{intent_index}.txt', 'w') as f:
46
  # f.write(row['response'])
47
  # previous_row = row
48
- loader = DirectoryLoader(f'{docs_dir}intents/')
49
  documents = loader.load()
50
  def wrap_text_preserve_newlines(text, width=110):
51
  # Split the input text into lines based on newline characters
 
25
  docs_dir = 'documents/'
26
  txts_dir = docs_dir + 'txt/'
27
  converted_txts_dir = docs_dir + 'converted_txt/'
28
+ # pdfs_dir = docs_dir + 'pdf/'
29
+ # txt_documents = [os.path.splitext(doc)[0] for doc in list(os.listdir(converted_txts_dir))]
30
+ # pdf_documents = [os.path.splitext(doc)[0] for doc in list(os.listdir(pdfs_dir))]
31
+ # for pdf_doc in pdf_documents:
32
+ # if pdf_doc not in txt_documents:
33
+ # print("Converting " + pdf_doc + ' pdf to txt')
34
+ # text = extract_text(pdfs_dir + pdf_doc + '.pdf')
35
+ # with open(converted_txts_dir + pdf_doc + '.txt', 'w') as f:
36
+ # f.write(text)
37
  # intents_df = pd.read_csv(docs_dir + 'intents.csv')
38
  # intents_df.sort_values(by=['intent', 'response'], inplace=True)
39
  # previous_row = None
 
45
  # with open(f'{docs_dir}intents/{row["intent"]}_{intent_index}.txt', 'w') as f:
46
  # f.write(row['response'])
47
  # previous_row = row
48
+ loader = DirectoryLoader(txts_dir)
49
  documents = loader.load()
50
  def wrap_text_preserve_newlines(text, width=110):
51
  # Split the input text into lines based on newline characters