JaySMTA commited on
Commit
eaae9bf
·
verified ·
1 Parent(s): a87c036

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +23 -21
app.py CHANGED
@@ -4,33 +4,34 @@ import gradio as gr
4
  import re
5
  import os
6
  import tempfile
7
- from jina import Flow, Executor, requests as jina_requests, Document
8
  import ujson
9
  import concurrent.futures
10
- import cProfile
11
 
12
- pr = cProfile.Profile()
13
  pr.enable()
14
 
15
- ZYLALABS_API_KEY = os.environ.get("ZYLALABS_API_KEY")
16
- LINKEDIN_API_KEY = os.environ.get("LINKEDIN_API_KEY")
17
- LINKEDIN_API_ENDPOINT = os.environ.get("LINKEDIN_API_ENDPOINT")
18
- DASHBOARD_PASSWORD = os.environ.get("DASHBOARD_PASSWORD", "1984")
 
 
19
 
20
- class GenericWebsiteProcessor(Executor):
21
- @jina_requests
22
- def process(self, docs, **kwargs):
23
- for doc in docs:
24
- doc.tags['processed'] = f"Processed URL: {doc.text}"
25
 
26
- jina_flow = Flow().add(uses=GenericWebsiteProcessor)
 
27
 
28
  def process_with_jina(urls):
29
- with jina_flow:
30
- results = jina_flow.post(on='/process', inputs=[Document(text=url) for url in urls])
31
-
32
- jina_df = pd.DataFrame([doc.tags for doc in results])
33
- return jina_df
34
 
35
  def concurrent_requests(urls, process_func):
36
  with concurrent.futures.ThreadPoolExecutor() as executor:
@@ -100,7 +101,8 @@ def jina_processor_interface(csv_file, password):
100
 
101
  try:
102
  df = pd.read_csv(csv_file.name, names=['URL'], header=None)
103
- output_df = process_with_jina(df['URL'].tolist())
 
104
 
105
  with tempfile.NamedTemporaryFile(mode="w+", suffix=".csv", delete=False) as temp_file:
106
  output_df.to_csv(temp_file.name, index=False)
@@ -157,7 +159,7 @@ jina_interface = gr.Interface(
157
  demo = gr.TabbedInterface([linkedin_interface, facebook_interface, jina_interface], ["LinkedIn Scraper", "Facebook Scraper", "Generic Website Processor"])
158
 
159
  if __name__ == "__main__":
160
- demo.launch()
161
 
162
- pr.disable()
163
  pr.print_stats(sort='cumulative')
 
4
  import re
5
  import os
6
  import tempfile
7
+ from jina import Flow, Document
8
  import ujson
9
  import concurrent.futures
10
+ import cProfile
11
 
12
+ pr = cProfile.Profile()
13
  pr.enable()
14
 
15
+ def get_secret(secret_name):
16
+ docker_secret_file = f"/run/secrets/{secret_name.lower()}"
17
+ if os.path.exists(docker_secret_file):
18
+ with open(docker_secret_file, 'r') as file:
19
+ return file.read().strip()
20
+ return os.environ.get(secret_name)
21
 
22
+ ZYLALABS_API_KEY = get_secret('ZYLALABS_API_KEY')
23
+ LINKEDIN_API_KEY = get_secret('LINKEDIN_API_KEY')
24
+ LINKEDIN_API_ENDPOINT = get_secret('LINKEDIN_API_ENDPOINT')
25
+ DASHBOARD_PASSWORD = get_secret('DASHBOARD_PASSWORD')
 
26
 
27
+ def setup_jina_flow():
28
+ return Flow().add(uses='jinahub://SimpleIndexer')
29
 
30
  def process_with_jina(urls):
31
+ docs = [Document(text=url) for url in urls]
32
+ with setup_jina_flow() as flow:
33
+ results = flow.post(on='/index', inputs=docs)
34
+ return [doc.tags for doc in results]
 
35
 
36
  def concurrent_requests(urls, process_func):
37
  with concurrent.futures.ThreadPoolExecutor() as executor:
 
101
 
102
  try:
103
  df = pd.read_csv(csv_file.name, names=['URL'], header=None)
104
+ results = process_with_jina(df['URL'].tolist())
105
+ output_df = pd.DataFrame(results)
106
 
107
  with tempfile.NamedTemporaryFile(mode="w+", suffix=".csv", delete=False) as temp_file:
108
  output_df.to_csv(temp_file.name, index=False)
 
159
  demo = gr.TabbedInterface([linkedin_interface, facebook_interface, jina_interface], ["LinkedIn Scraper", "Facebook Scraper", "Generic Website Processor"])
160
 
161
  if __name__ == "__main__":
162
+ demo.launch(server_name="0.0.0.0", server_port=int(os.environ.get('PORT', 8080)))
163
 
164
+ pr.disable()
165
  pr.print_stats(sort='cumulative')