Nicolas Pierson commited on
^^
Browse files- src/pages/1_Historical_Prices.py +31 -109
src/pages/1_Historical_Prices.py
CHANGED
|
@@ -16,19 +16,14 @@ AWS_SECRET_ACCESS_KEY = os.getenv("AWS_SECRET_ACCESS_KEY")
|
|
| 16 |
ROOT_DIR = "./data/"
|
| 17 |
MISSING_VALUE_PLACEHOLDER = -1
|
| 18 |
|
| 19 |
-
|
| 20 |
# --- Asynchronous Data Loading Functions ---
|
| 21 |
async def async_load_file_s3(object_key: str) -> pd.DataFrame:
|
| 22 |
if not AWS_S3_BUCKET or not AWS_ACCESS_KEY_ID or not AWS_SECRET_ACCESS_KEY:
|
| 23 |
-
raise ValueError(
|
| 24 |
-
"AWS credentials or bucket name not set in environment variables."
|
| 25 |
-
)
|
| 26 |
|
| 27 |
session = get_session()
|
| 28 |
async with session.create_client(
|
| 29 |
-
"s3",
|
| 30 |
-
aws_access_key_id=AWS_ACCESS_KEY_ID,
|
| 31 |
-
aws_secret_access_key=AWS_SECRET_ACCESS_KEY,
|
| 32 |
) as s3_client:
|
| 33 |
response = await s3_client.get_object(Bucket=AWS_S3_BUCKET, Key=object_key)
|
| 34 |
status = response.get("ResponseMetadata", {}).get("HTTPStatusCode")
|
|
@@ -39,75 +34,54 @@ async def async_load_file_s3(object_key: str) -> pd.DataFrame:
|
|
| 39 |
return pd.read_csv(io.StringIO(content.decode("utf-8")))
|
| 40 |
raise ValueError(f"Unsuccessful S3 get_object response. Status - {status}")
|
| 41 |
|
| 42 |
-
|
| 43 |
-
@st.cache_resource
|
| 44 |
async def async_load_geojson_from_s3(object_key: str) -> dict:
|
| 45 |
local_file_path = f"{ROOT_DIR}{object_key}"
|
| 46 |
if os.path.exists(local_file_path):
|
| 47 |
with open(local_file_path, "r") as f:
|
| 48 |
return json.load(f)
|
| 49 |
|
| 50 |
-
|
| 51 |
-
|
| 52 |
-
|
| 53 |
-
|
| 54 |
-
|
| 55 |
-
)
|
| 56 |
-
departements_geojson_task = async_load_geojson_from_s3(
|
| 57 |
-
"processed/referentiel/departements.geojson"
|
| 58 |
-
)
|
| 59 |
-
communes_df_task = async_load_file_s3(
|
| 60 |
-
"processed/housing/dataset_housing_prices.csv"
|
| 61 |
-
)
|
| 62 |
-
communes_geojson_task = async_load_geojson_from_s3(
|
| 63 |
-
"processed/referentiel/communes.geojson"
|
| 64 |
-
)
|
| 65 |
|
| 66 |
return await asyncio.gather(
|
| 67 |
departements_df_task,
|
| 68 |
departements_geojson_task,
|
| 69 |
communes_df_task,
|
| 70 |
-
communes_geojson_task
|
| 71 |
)
|
| 72 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 73 |
|
| 74 |
# --- Helper for Color Scale ---
|
| 75 |
def _generate_custom_colorscale(min_val, max_val):
|
| 76 |
custom_colorscale = [[0.0, "lightgrey"]]
|
| 77 |
reversed_rdylgn_colors = pcolors.diverging.RdYlGn[::-1]
|
| 78 |
|
| 79 |
-
normalized_min_actual = (min_val - MISSING_VALUE_PLACEHOLDER) / (
|
| 80 |
-
|
| 81 |
-
)
|
| 82 |
-
normalized_max_actual = (max_val - MISSING_VALUE_PLACEHOLDER) / (
|
| 83 |
-
max_val - MISSING_VALUE_PLACEHOLDER
|
| 84 |
-
)
|
| 85 |
|
| 86 |
num_steps = len(reversed_rdylgn_colors)
|
| 87 |
for i, color in enumerate(reversed_rdylgn_colors):
|
| 88 |
-
normalized_point = normalized_min_actual + (
|
| 89 |
-
normalized_max_actual - normalized_min_actual
|
| 90 |
-
) * (i / (num_steps - 1))
|
| 91 |
if normalized_point > 0.0:
|
| 92 |
custom_colorscale.append([normalized_point, color])
|
| 93 |
return sorted(custom_colorscale, key=lambda x: x[0])
|
| 94 |
|
| 95 |
-
|
| 96 |
# --- Plotly Figure Generation Functions ---
|
| 97 |
@st.cache_data
|
| 98 |
def display_choropleth_map(df: pd.DataFrame, geojson_data: dict):
|
| 99 |
-
df_agg = (
|
| 100 |
-
df[df["prixm2moyen"] > 0]
|
| 101 |
-
.groupby(["code_departement", "annee"])["prixm2moyen"]
|
| 102 |
-
.mean()
|
| 103 |
-
.reset_index()
|
| 104 |
-
)
|
| 105 |
min_actual_prixm2moyen = df_agg["prixm2moyen"].min()
|
| 106 |
max_actual_prixm2moyen = df_agg["prixm2moyen"].max()
|
| 107 |
|
| 108 |
-
custom_colorscale = _generate_custom_colorscale(
|
| 109 |
-
min_actual_prixm2moyen, max_actual_prixm2moyen
|
| 110 |
-
)
|
| 111 |
|
| 112 |
fig = px.choropleth_map(
|
| 113 |
df_agg,
|
|
@@ -139,27 +113,20 @@ def display_choropleth_map(df: pd.DataFrame, geojson_data: dict):
|
|
| 139 |
st.warning("No animation updatemenus found.")
|
| 140 |
return fig
|
| 141 |
|
| 142 |
-
|
| 143 |
@st.cache_data
|
| 144 |
def display_choropleth_map_for_department(
|
| 145 |
-
df: pd.DataFrame,
|
| 146 |
-
|
| 147 |
-
geojson_data: dict,
|
| 148 |
-
min_global_prixm2moyen: float,
|
| 149 |
-
max_global_prixm2moyen: float,
|
| 150 |
):
|
| 151 |
df_filtered = df[df["code_departement"] == department_code].copy()
|
| 152 |
|
| 153 |
filtered_features = [
|
| 154 |
-
feature
|
| 155 |
-
for feature in geojson_data["features"]
|
| 156 |
if feature["properties"]["code"].startswith(department_code)
|
| 157 |
]
|
| 158 |
filtered_geojson = {"type": "FeatureCollection", "features": filtered_features}
|
| 159 |
|
| 160 |
-
custom_colorscale = _generate_custom_colorscale(
|
| 161 |
-
min_global_prixm2moyen, max_global_prixm2moyen
|
| 162 |
-
)
|
| 163 |
|
| 164 |
center_lat, center_lon, zoom_level = 46.603354, 1.888334, 6
|
| 165 |
if filtered_features:
|
|
@@ -170,12 +137,9 @@ def display_choropleth_map_for_department(
|
|
| 170 |
center_lat, center_lon = department_centroid.y, department_centroid.x
|
| 171 |
minx, miny, maxx, maxy = dissolved_department.total_bounds
|
| 172 |
width, height = maxx - minx, maxy - miny
|
| 173 |
-
if width < 0.2 and height < 0.2:
|
| 174 |
-
|
| 175 |
-
|
| 176 |
-
zoom_level = 9
|
| 177 |
-
else:
|
| 178 |
-
zoom_level = 8
|
| 179 |
|
| 180 |
fig = px.choropleth_map(
|
| 181 |
df_filtered,
|
|
@@ -207,29 +171,19 @@ def display_choropleth_map_for_department(
|
|
| 207 |
st.warning("No animation updatemenus found.")
|
| 208 |
return fig
|
| 209 |
|
| 210 |
-
|
| 211 |
# --- Streamlit App Layout ---
|
| 212 |
st.set_page_config(page_title="Oasis", page_icon=":house:", layout="wide")
|
| 213 |
|
| 214 |
st.header("Historical Price - France")
|
| 215 |
st.subheader("An overview of real estate prices in France from 2015 to 2024")
|
| 216 |
|
| 217 |
-
st.write(
|
| 218 |
-
"This map shows the average price per square meter in French departments over the years, with a focus on climatic events."
|
| 219 |
-
)
|
| 220 |
|
| 221 |
with st.spinner("Loading data and preparing maps..."):
|
| 222 |
-
# Call the
|
| 223 |
-
(
|
| 224 |
-
|
| 225 |
-
|
| 226 |
-
dataset_housing_prices,
|
| 227 |
-
communes_geojson,
|
| 228 |
-
) = load_all_data_async()
|
| 229 |
-
|
| 230 |
-
min_global_commune_prixm2moyen = dataset_housing_prices[
|
| 231 |
-
dataset_housing_prices["prixm2moyen"] > 0
|
| 232 |
-
]["prixm2moyen"].min()
|
| 233 |
max_global_commune_prixm2moyen = dataset_housing_prices["prixm2moyen"].max()
|
| 234 |
|
| 235 |
fig_france = display_choropleth_map(
|
|
@@ -238,36 +192,4 @@ fig_france = display_choropleth_map(
|
|
| 238 |
)
|
| 239 |
st.plotly_chart(fig_france, use_container_width=True)
|
| 240 |
st.write("Hover over the map to see detailed information for each department and year.")
|
| 241 |
-
st.write(
|
| 242 |
-
"Missing values are represented in light grey, while actual data is shown in a gradient from red (high prices) to green (low prices)."
|
| 243 |
-
)
|
| 244 |
-
st.write(
|
| 245 |
-
"Note: The color scale is customized to highlight missing values in light grey, while the actual data is represented using a reversed RdYlGn color scale, where red indicates higher prices and green indicates lower prices."
|
| 246 |
-
)
|
| 247 |
-
|
| 248 |
-
st.subheader("Select a Department to View Commune Prices")
|
| 249 |
-
selected_departement = st.selectbox(
|
| 250 |
-
"Select a Department",
|
| 251 |
-
options=dataset_housing_prices["code_departement"].unique(),
|
| 252 |
-
)
|
| 253 |
-
|
| 254 |
-
fig_department = display_choropleth_map_for_department(
|
| 255 |
-
dataset_housing_prices,
|
| 256 |
-
selected_departement,
|
| 257 |
-
communes_geojson,
|
| 258 |
-
min_global_commune_prixm2moyen,
|
| 259 |
-
max_global_commune_prixm2moyen,
|
| 260 |
-
)
|
| 261 |
-
|
| 262 |
-
st.subheader("Average Price per Square Meter in French Communes (2015-2024)")
|
| 263 |
-
st.plotly_chart(fig_department, use_container_width=True)
|
| 264 |
-
st.write("Hover over the map to see detailed information for each commune and year.")
|
| 265 |
-
st.write(
|
| 266 |
-
"Missing values are represented in light grey, while actual data is shown in a gradient from red (high prices) to green (low prices)."
|
| 267 |
-
)
|
| 268 |
-
st.write(
|
| 269 |
-
"Note: The color scale is customized to highlight missing values in light grey, while the actual data is represented using a reversed RdYlGn color scale, where red indicates higher prices and green indicates lower prices."
|
| 270 |
-
)
|
| 271 |
-
st.write(
|
| 272 |
-
"The map is animated by year, allowing you to see how the average price per square meter changes over time."
|
| 273 |
-
)
|
|
|
|
| 16 |
ROOT_DIR = "./data/"
|
| 17 |
MISSING_VALUE_PLACEHOLDER = -1
|
| 18 |
|
|
|
|
| 19 |
# --- Asynchronous Data Loading Functions ---
|
| 20 |
async def async_load_file_s3(object_key: str) -> pd.DataFrame:
|
| 21 |
if not AWS_S3_BUCKET or not AWS_ACCESS_KEY_ID or not AWS_SECRET_ACCESS_KEY:
|
| 22 |
+
raise ValueError("AWS credentials or bucket name not set in environment variables.")
|
|
|
|
|
|
|
| 23 |
|
| 24 |
session = get_session()
|
| 25 |
async with session.create_client(
|
| 26 |
+
"s3", aws_access_key_id=AWS_ACCESS_KEY_ID, aws_secret_access_key=AWS_SECRET_ACCESS_KEY
|
|
|
|
|
|
|
| 27 |
) as s3_client:
|
| 28 |
response = await s3_client.get_object(Bucket=AWS_S3_BUCKET, Key=object_key)
|
| 29 |
status = response.get("ResponseMetadata", {}).get("HTTPStatusCode")
|
|
|
|
| 34 |
return pd.read_csv(io.StringIO(content.decode("utf-8")))
|
| 35 |
raise ValueError(f"Unsuccessful S3 get_object response. Status - {status}")
|
| 36 |
|
|
|
|
|
|
|
| 37 |
async def async_load_geojson_from_s3(object_key: str) -> dict:
|
| 38 |
local_file_path = f"{ROOT_DIR}{object_key}"
|
| 39 |
if os.path.exists(local_file_path):
|
| 40 |
with open(local_file_path, "r") as f:
|
| 41 |
return json.load(f)
|
| 42 |
|
| 43 |
+
# Internal async function to gather all data loading tasks
|
| 44 |
+
async def _load_all_data_async_internal():
|
| 45 |
+
departements_df_task = async_load_file_s3("processed/housing/dataset_departements_housing_prices.csv")
|
| 46 |
+
departements_geojson_task = async_load_geojson_from_s3("processed/referentiel/departements.geojson")
|
| 47 |
+
communes_df_task = async_load_file_s3("processed/housing/dataset_housing_prices.csv")
|
| 48 |
+
communes_geojson_task = async_load_geojson_from_s3("processed/referentiel/communes.geojson")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 49 |
|
| 50 |
return await asyncio.gather(
|
| 51 |
departements_df_task,
|
| 52 |
departements_geojson_task,
|
| 53 |
communes_df_task,
|
| 54 |
+
communes_geojson_task
|
| 55 |
)
|
| 56 |
|
| 57 |
+
@st.cache_resource
|
| 58 |
+
def load_all_data_sync_wrapper():
|
| 59 |
+
# This synchronous wrapper runs the async data loading
|
| 60 |
+
return asyncio.run(_load_all_data_async_internal())
|
| 61 |
|
| 62 |
# --- Helper for Color Scale ---
|
| 63 |
def _generate_custom_colorscale(min_val, max_val):
|
| 64 |
custom_colorscale = [[0.0, "lightgrey"]]
|
| 65 |
reversed_rdylgn_colors = pcolors.diverging.RdYlGn[::-1]
|
| 66 |
|
| 67 |
+
normalized_min_actual = (min_val - MISSING_VALUE_PLACEHOLDER) / (max_val - MISSING_VALUE_PLACEHOLDER)
|
| 68 |
+
normalized_max_actual = (max_val - MISSING_VALUE_PLACEHOLDER) / (max_val - MISSING_VALUE_PLACEHOLDER)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 69 |
|
| 70 |
num_steps = len(reversed_rdylgn_colors)
|
| 71 |
for i, color in enumerate(reversed_rdylgn_colors):
|
| 72 |
+
normalized_point = normalized_min_actual + (normalized_max_actual - normalized_min_actual) * (i / (num_steps - 1))
|
|
|
|
|
|
|
| 73 |
if normalized_point > 0.0:
|
| 74 |
custom_colorscale.append([normalized_point, color])
|
| 75 |
return sorted(custom_colorscale, key=lambda x: x[0])
|
| 76 |
|
|
|
|
| 77 |
# --- Plotly Figure Generation Functions ---
|
| 78 |
@st.cache_data
|
| 79 |
def display_choropleth_map(df: pd.DataFrame, geojson_data: dict):
|
| 80 |
+
df_agg = df[df["prixm2moyen"] > 0].groupby(["code_departement", "annee"])["prixm2moyen"].mean().reset_index()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 81 |
min_actual_prixm2moyen = df_agg["prixm2moyen"].min()
|
| 82 |
max_actual_prixm2moyen = df_agg["prixm2moyen"].max()
|
| 83 |
|
| 84 |
+
custom_colorscale = _generate_custom_colorscale(min_actual_prixm2moyen, max_actual_prixm2moyen)
|
|
|
|
|
|
|
| 85 |
|
| 86 |
fig = px.choropleth_map(
|
| 87 |
df_agg,
|
|
|
|
| 113 |
st.warning("No animation updatemenus found.")
|
| 114 |
return fig
|
| 115 |
|
|
|
|
| 116 |
@st.cache_data
|
| 117 |
def display_choropleth_map_for_department(
|
| 118 |
+
df: pd.DataFrame, department_code: str, geojson_data: dict,
|
| 119 |
+
min_global_prixm2moyen: float, max_global_prixm2moyen: float
|
|
|
|
|
|
|
|
|
|
| 120 |
):
|
| 121 |
df_filtered = df[df["code_departement"] == department_code].copy()
|
| 122 |
|
| 123 |
filtered_features = [
|
| 124 |
+
feature for feature in geojson_data["features"]
|
|
|
|
| 125 |
if feature["properties"]["code"].startswith(department_code)
|
| 126 |
]
|
| 127 |
filtered_geojson = {"type": "FeatureCollection", "features": filtered_features}
|
| 128 |
|
| 129 |
+
custom_colorscale = _generate_custom_colorscale(min_global_prixm2moyen, max_global_prixm2moyen)
|
|
|
|
|
|
|
| 130 |
|
| 131 |
center_lat, center_lon, zoom_level = 46.603354, 1.888334, 6
|
| 132 |
if filtered_features:
|
|
|
|
| 137 |
center_lat, center_lon = department_centroid.y, department_centroid.x
|
| 138 |
minx, miny, maxx, maxy = dissolved_department.total_bounds
|
| 139 |
width, height = maxx - minx, maxy - miny
|
| 140 |
+
if width < 0.2 and height < 0.2: zoom_level = 10
|
| 141 |
+
elif width < 0.5 and height < 0.5: zoom_level = 9
|
| 142 |
+
else: zoom_level = 8
|
|
|
|
|
|
|
|
|
|
| 143 |
|
| 144 |
fig = px.choropleth_map(
|
| 145 |
df_filtered,
|
|
|
|
| 171 |
st.warning("No animation updatemenus found.")
|
| 172 |
return fig
|
| 173 |
|
|
|
|
| 174 |
# --- Streamlit App Layout ---
|
| 175 |
st.set_page_config(page_title="Oasis", page_icon=":house:", layout="wide")
|
| 176 |
|
| 177 |
st.header("Historical Price - France")
|
| 178 |
st.subheader("An overview of real estate prices in France from 2015 to 2024")
|
| 179 |
|
| 180 |
+
st.write("This map shows the average price per square meter in French departments over the years, with a focus on climatic events.")
|
|
|
|
|
|
|
| 181 |
|
| 182 |
with st.spinner("Loading data and preparing maps..."):
|
| 183 |
+
# Call the synchronous wrapper function
|
| 184 |
+
dataset_departements_housing_prices, departements_geojson, dataset_housing_prices, communes_geojson = load_all_data_sync_wrapper()
|
| 185 |
+
|
| 186 |
+
min_global_commune_prixm2moyen = dataset_housing_prices[dataset_housing_prices["prixm2moyen"] > 0]["prixm2moyen"].min()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 187 |
max_global_commune_prixm2moyen = dataset_housing_prices["prixm2moyen"].max()
|
| 188 |
|
| 189 |
fig_france = display_choropleth_map(
|
|
|
|
| 192 |
)
|
| 193 |
st.plotly_chart(fig_france, use_container_width=True)
|
| 194 |
st.write("Hover over the map to see detailed information for each department and year.")
|
| 195 |
+
st.write("Missing values are represented in light grey.")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|