Spaces:
Sleeping
Sleeping
Ubuntu commited on
Commit ·
0141e03
1
Parent(s): b212436
OK
Browse files- app.py +11 -4
- pages/1_database.py +1 -2
- pages/2_recherche_docs.py +1 -2
app.py
CHANGED
|
@@ -4,6 +4,7 @@ import datetime
|
|
| 4 |
|
| 5 |
import pandas as pd
|
| 6 |
import streamlit as st
|
|
|
|
| 7 |
from streamlit_extras.switch_page_button import switch_page
|
| 8 |
|
| 9 |
os.environ['AWS_ACCESS_KEY_ID'] = st.secrets['AWS_ACCESS_KEY_ID']
|
|
@@ -40,15 +41,20 @@ st.write("# CityScan")
|
|
| 40 |
login_button = False
|
| 41 |
assistant_creation = False
|
| 42 |
|
| 43 |
-
def get_all_past_docs(s3_folder: str, except_init=False):
|
| 44 |
parquet_files = s3.list_objects(path=s3_folder, suffix=".parquet")
|
| 45 |
|
| 46 |
-
if except_init
|
| 47 |
parquet_files = [p for p in parquet_files if '2024-03-04' not in p]
|
| 48 |
|
| 49 |
-
|
|
|
|
| 50 |
|
| 51 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 52 |
df_global = (
|
| 53 |
pd.concat(all_dfs)
|
| 54 |
.sort_values("scan_date", ascending=True)
|
|
@@ -58,6 +64,7 @@ def get_all_past_docs(s3_folder: str, except_init=False):
|
|
| 58 |
df_global = pd.DataFrame(
|
| 59 |
columns=["id", "type", "url", "city_name", "region", "status", "scan_date"]
|
| 60 |
)
|
|
|
|
| 61 |
return df_global
|
| 62 |
|
| 63 |
|
|
|
|
| 4 |
|
| 5 |
import pandas as pd
|
| 6 |
import streamlit as st
|
| 7 |
+
from concurrent.futures import ThreadPoolExecutor
|
| 8 |
from streamlit_extras.switch_page_button import switch_page
|
| 9 |
|
| 10 |
os.environ['AWS_ACCESS_KEY_ID'] = st.secrets['AWS_ACCESS_KEY_ID']
|
|
|
|
| 41 |
login_button = False
|
| 42 |
assistant_creation = False
|
| 43 |
|
| 44 |
+
def get_all_past_docs(s3_folder: str, except_init=False, max_workers=4):
|
| 45 |
parquet_files = s3.list_objects(path=s3_folder, suffix=".parquet")
|
| 46 |
|
| 47 |
+
if except_init:
|
| 48 |
parquet_files = [p for p in parquet_files if '2024-03-04' not in p]
|
| 49 |
|
| 50 |
+
def read_parquet_file(file):
|
| 51 |
+
return s3.read_parquet(file)
|
| 52 |
|
| 53 |
+
# Download in parallel
|
| 54 |
+
with ThreadPoolExecutor(max_workers=max_workers) as executor:
|
| 55 |
+
all_dfs = list(executor.map(read_parquet_file, parquet_files))
|
| 56 |
+
|
| 57 |
+
if all_dfs:
|
| 58 |
df_global = (
|
| 59 |
pd.concat(all_dfs)
|
| 60 |
.sort_values("scan_date", ascending=True)
|
|
|
|
| 64 |
df_global = pd.DataFrame(
|
| 65 |
columns=["id", "type", "url", "city_name", "region", "status", "scan_date"]
|
| 66 |
)
|
| 67 |
+
|
| 68 |
return df_global
|
| 69 |
|
| 70 |
|
pages/1_database.py
CHANGED
|
@@ -1,10 +1,9 @@
|
|
| 1 |
import streamlit as st
|
| 2 |
-
from streamlit_extras.switch_page_button import switch_page
|
| 3 |
|
| 4 |
st.set_page_config(page_title="database", page_icon="👋", layout="wide")
|
| 5 |
|
| 6 |
if not st.session_state.get("user_login_success", False):
|
| 7 |
-
switch_page("
|
| 8 |
|
| 9 |
st.markdown("## Database")
|
| 10 |
|
|
|
|
| 1 |
import streamlit as st
|
|
|
|
| 2 |
|
| 3 |
st.set_page_config(page_title="database", page_icon="👋", layout="wide")
|
| 4 |
|
| 5 |
if not st.session_state.get("user_login_success", False):
|
| 6 |
+
st.switch_page("app.py")
|
| 7 |
|
| 8 |
st.markdown("## Database")
|
| 9 |
|
pages/2_recherche_docs.py
CHANGED
|
@@ -3,14 +3,13 @@ import datetime
|
|
| 3 |
import streamlit as st
|
| 4 |
import pandas as pd
|
| 5 |
import unicodedata
|
| 6 |
-
from streamlit_extras.switch_page_button import switch_page
|
| 7 |
from collections import deque, defaultdict
|
| 8 |
import re
|
| 9 |
|
| 10 |
st.set_page_config(page_title="recherche", page_icon="👋", layout="wide")
|
| 11 |
|
| 12 |
if not st.session_state.get("user_login_success", False):
|
| 13 |
-
switch_page("
|
| 14 |
|
| 15 |
st.markdown("## Recherche de documents par mots clés")
|
| 16 |
|
|
|
|
| 3 |
import streamlit as st
|
| 4 |
import pandas as pd
|
| 5 |
import unicodedata
|
|
|
|
| 6 |
from collections import deque, defaultdict
|
| 7 |
import re
|
| 8 |
|
| 9 |
st.set_page_config(page_title="recherche", page_icon="👋", layout="wide")
|
| 10 |
|
| 11 |
if not st.session_state.get("user_login_success", False):
|
| 12 |
+
st.switch_page("app.py")
|
| 13 |
|
| 14 |
st.markdown("## Recherche de documents par mots clés")
|
| 15 |
|