Ubuntu commited on
Commit
0141e03
·
1 Parent(s): b212436
Files changed (3) hide show
  1. app.py +11 -4
  2. pages/1_database.py +1 -2
  3. pages/2_recherche_docs.py +1 -2
app.py CHANGED
@@ -4,6 +4,7 @@ import datetime
4
 
5
  import pandas as pd
6
  import streamlit as st
 
7
  from streamlit_extras.switch_page_button import switch_page
8
 
9
  os.environ['AWS_ACCESS_KEY_ID'] = st.secrets['AWS_ACCESS_KEY_ID']
@@ -40,15 +41,20 @@ st.write("# CityScan")
40
  login_button = False
41
  assistant_creation = False
42
 
43
- def get_all_past_docs(s3_folder: str, except_init=False):
44
  parquet_files = s3.list_objects(path=s3_folder, suffix=".parquet")
45
 
46
- if except_init is True:
47
  parquet_files = [p for p in parquet_files if '2024-03-04' not in p]
48
 
49
- all_dfs = [s3.read_parquet(item) for item in parquet_files]
 
50
 
51
- if len(all_dfs):
 
 
 
 
52
  df_global = (
53
  pd.concat(all_dfs)
54
  .sort_values("scan_date", ascending=True)
@@ -58,6 +64,7 @@ def get_all_past_docs(s3_folder: str, except_init=False):
58
  df_global = pd.DataFrame(
59
  columns=["id", "type", "url", "city_name", "region", "status", "scan_date"]
60
  )
 
61
  return df_global
62
 
63
 
 
4
 
5
  import pandas as pd
6
  import streamlit as st
7
+ from concurrent.futures import ThreadPoolExecutor
8
  from streamlit_extras.switch_page_button import switch_page
9
 
10
  os.environ['AWS_ACCESS_KEY_ID'] = st.secrets['AWS_ACCESS_KEY_ID']
 
41
  login_button = False
42
  assistant_creation = False
43
 
44
+ def get_all_past_docs(s3_folder: str, except_init=False, max_workers=4):
45
  parquet_files = s3.list_objects(path=s3_folder, suffix=".parquet")
46
 
47
+ if except_init:
48
  parquet_files = [p for p in parquet_files if '2024-03-04' not in p]
49
 
50
+ def read_parquet_file(file):
51
+ return s3.read_parquet(file)
52
 
53
+ # Download in parallel
54
+ with ThreadPoolExecutor(max_workers=max_workers) as executor:
55
+ all_dfs = list(executor.map(read_parquet_file, parquet_files))
56
+
57
+ if all_dfs:
58
  df_global = (
59
  pd.concat(all_dfs)
60
  .sort_values("scan_date", ascending=True)
 
64
  df_global = pd.DataFrame(
65
  columns=["id", "type", "url", "city_name", "region", "status", "scan_date"]
66
  )
67
+
68
  return df_global
69
 
70
 
pages/1_database.py CHANGED
@@ -1,10 +1,9 @@
1
  import streamlit as st
2
- from streamlit_extras.switch_page_button import switch_page
3
 
4
  st.set_page_config(page_title="database", page_icon="👋", layout="wide")
5
 
6
  if not st.session_state.get("user_login_success", False):
7
- switch_page("home")
8
 
9
  st.markdown("## Database")
10
 
 
1
  import streamlit as st
 
2
 
3
  st.set_page_config(page_title="database", page_icon="👋", layout="wide")
4
 
5
  if not st.session_state.get("user_login_success", False):
6
+ st.switch_page("app.py")
7
 
8
  st.markdown("## Database")
9
 
pages/2_recherche_docs.py CHANGED
@@ -3,14 +3,13 @@ import datetime
3
  import streamlit as st
4
  import pandas as pd
5
  import unicodedata
6
- from streamlit_extras.switch_page_button import switch_page
7
  from collections import deque, defaultdict
8
  import re
9
 
10
  st.set_page_config(page_title="recherche", page_icon="👋", layout="wide")
11
 
12
  if not st.session_state.get("user_login_success", False):
13
- switch_page("home")
14
 
15
  st.markdown("## Recherche de documents par mots clés")
16
 
 
3
  import streamlit as st
4
  import pandas as pd
5
  import unicodedata
 
6
  from collections import deque, defaultdict
7
  import re
8
 
9
  st.set_page_config(page_title="recherche", page_icon="👋", layout="wide")
10
 
11
  if not st.session_state.get("user_login_success", False):
12
+ st.switch_page("app.py")
13
 
14
  st.markdown("## Recherche de documents par mots clés")
15