Update app.py
Browse files
app.py
CHANGED
|
@@ -4,33 +4,34 @@ import gradio as gr
|
|
| 4 |
import re
|
| 5 |
import os
|
| 6 |
import tempfile
|
| 7 |
-
from jina import Flow,
|
| 8 |
import ujson
|
| 9 |
import concurrent.futures
|
| 10 |
-
import cProfile
|
| 11 |
|
| 12 |
-
pr = cProfile.Profile()
|
| 13 |
pr.enable()
|
| 14 |
|
| 15 |
-
|
| 16 |
-
|
| 17 |
-
|
| 18 |
-
|
|
|
|
|
|
|
| 19 |
|
| 20 |
-
|
| 21 |
-
|
| 22 |
-
|
| 23 |
-
|
| 24 |
-
doc.tags['processed'] = f"Processed URL: {doc.text}"
|
| 25 |
|
| 26 |
-
|
|
|
|
| 27 |
|
| 28 |
def process_with_jina(urls):
|
| 29 |
-
|
| 30 |
-
|
| 31 |
-
|
| 32 |
-
|
| 33 |
-
return jina_df
|
| 34 |
|
| 35 |
def concurrent_requests(urls, process_func):
|
| 36 |
with concurrent.futures.ThreadPoolExecutor() as executor:
|
|
@@ -100,7 +101,8 @@ def jina_processor_interface(csv_file, password):
|
|
| 100 |
|
| 101 |
try:
|
| 102 |
df = pd.read_csv(csv_file.name, names=['URL'], header=None)
|
| 103 |
-
|
|
|
|
| 104 |
|
| 105 |
with tempfile.NamedTemporaryFile(mode="w+", suffix=".csv", delete=False) as temp_file:
|
| 106 |
output_df.to_csv(temp_file.name, index=False)
|
|
@@ -157,7 +159,7 @@ jina_interface = gr.Interface(
|
|
| 157 |
demo = gr.TabbedInterface([linkedin_interface, facebook_interface, jina_interface], ["LinkedIn Scraper", "Facebook Scraper", "Generic Website Processor"])
|
| 158 |
|
| 159 |
if __name__ == "__main__":
|
| 160 |
-
demo.launch()
|
| 161 |
|
| 162 |
-
pr.disable()
|
| 163 |
pr.print_stats(sort='cumulative')
|
|
|
|
| 4 |
import re
|
| 5 |
import os
|
| 6 |
import tempfile
|
| 7 |
+
from jina import Flow, Document
|
| 8 |
import ujson
|
| 9 |
import concurrent.futures
|
| 10 |
+
import cProfile
|
| 11 |
|
| 12 |
+
pr = cProfile.Profile()
|
| 13 |
pr.enable()
|
| 14 |
|
| 15 |
+
def get_secret(secret_name):
|
| 16 |
+
docker_secret_file = f"/run/secrets/{secret_name.lower()}"
|
| 17 |
+
if os.path.exists(docker_secret_file):
|
| 18 |
+
with open(docker_secret_file, 'r') as file:
|
| 19 |
+
return file.read().strip()
|
| 20 |
+
return os.environ.get(secret_name)
|
| 21 |
|
| 22 |
+
ZYLALABS_API_KEY = get_secret('ZYLALABS_API_KEY')
|
| 23 |
+
LINKEDIN_API_KEY = get_secret('LINKEDIN_API_KEY')
|
| 24 |
+
LINKEDIN_API_ENDPOINT = get_secret('LINKEDIN_API_ENDPOINT')
|
| 25 |
+
DASHBOARD_PASSWORD = get_secret('DASHBOARD_PASSWORD')
|
|
|
|
| 26 |
|
| 27 |
+
def setup_jina_flow():
|
| 28 |
+
return Flow().add(uses='jinahub://SimpleIndexer')
|
| 29 |
|
| 30 |
def process_with_jina(urls):
|
| 31 |
+
docs = [Document(text=url) for url in urls]
|
| 32 |
+
with setup_jina_flow() as flow:
|
| 33 |
+
results = flow.post(on='/index', inputs=docs)
|
| 34 |
+
return [doc.tags for doc in results]
|
|
|
|
| 35 |
|
| 36 |
def concurrent_requests(urls, process_func):
|
| 37 |
with concurrent.futures.ThreadPoolExecutor() as executor:
|
|
|
|
| 101 |
|
| 102 |
try:
|
| 103 |
df = pd.read_csv(csv_file.name, names=['URL'], header=None)
|
| 104 |
+
results = process_with_jina(df['URL'].tolist())
|
| 105 |
+
output_df = pd.DataFrame(results)
|
| 106 |
|
| 107 |
with tempfile.NamedTemporaryFile(mode="w+", suffix=".csv", delete=False) as temp_file:
|
| 108 |
output_df.to_csv(temp_file.name, index=False)
|
|
|
|
| 159 |
demo = gr.TabbedInterface([linkedin_interface, facebook_interface, jina_interface], ["LinkedIn Scraper", "Facebook Scraper", "Generic Website Processor"])
|
| 160 |
|
| 161 |
if __name__ == "__main__":
|
| 162 |
+
demo.launch(server_name="0.0.0.0", server_port=int(os.environ.get('PORT', 8080)))
|
| 163 |
|
| 164 |
+
pr.disable()
|
| 165 |
pr.print_stats(sort='cumulative')
|