Tabular Regression
Scikit-learn
random-forest
scikit-learn
shap
explainable-ai
public-health
air-quality
Instructions to use oukawa/PRISMA with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Scikit-learn
How to use oukawa/PRISMA with Scikit-learn:
from huggingface_hub import hf_hub_download import joblib model = joblib.load( hf_hub_download("oukawa/PRISMA", "sklearn_model.joblib") ) # only load pickle files from sources you trust # read more about it here https://skops.readthedocs.io/en/stable/persistence.html - Notebooks
- Google Colab
- Kaggle
Upload processing notebook
Browse files
RF_data_processing_temporal_and_spatial.ipynb
ADDED
|
@@ -0,0 +1,448 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"cells": [
|
| 3 |
+
{
|
| 4 |
+
"cell_type": "markdown",
|
| 5 |
+
"metadata": {},
|
| 6 |
+
"source": [
|
| 7 |
+
"# Input data for the temporal and spatial RF models\n",
|
| 8 |
+
"\n",
|
| 9 |
+
"Builds `RF_temporal_data.csv` (weekly) and `RF_spatial_data.csv` (1-km grid), following Sections 2.3.1-2.3.4 of the\n",
|
| 10 |
+
"manuscript and Section S1 of the Supplementary Material. Input files are not included; file and column names are\n",
|
| 11 |
+
"placeholders.\n",
|
| 12 |
+
"\n",
|
| 13 |
+
"Disease groups (ICD-10): LRI = J10-J18 and J21-J22; CRD = J44-J45; Other = J00-J99 excluding LRI and CRD.\n",
|
| 14 |
+
"Age groups: G1 = 0-4 years; G2 = 5-59 years; G3 = 60 years and over."
|
| 15 |
+
]
|
| 16 |
+
},
|
| 17 |
+
{
|
| 18 |
+
"cell_type": "code",
|
| 19 |
+
"execution_count": null,
|
| 20 |
+
"metadata": {},
|
| 21 |
+
"outputs": [],
|
| 22 |
+
"source": [
|
| 23 |
+
"# Libraries\n",
|
| 24 |
+
"import numpy as np\n",
|
| 25 |
+
"import pandas as pd\n",
|
| 26 |
+
"import geopandas as gpd\n",
|
| 27 |
+
"import rasterio\n",
|
| 28 |
+
"from rasterio.mask import mask\n",
|
| 29 |
+
"from shapely.geometry import box, mapping"
|
| 30 |
+
]
|
| 31 |
+
},
|
| 32 |
+
{
|
| 33 |
+
"cell_type": "code",
|
| 34 |
+
"execution_count": null,
|
| 35 |
+
"metadata": {},
|
| 36 |
+
"outputs": [],
|
| 37 |
+
"source": [
|
| 38 |
+
"# Input and output files\n",
|
| 39 |
+
"air_quality_path = '.../path/to/input/air_quality.csv' # datetime, station, so2, no2, nox, co, o3, pm10\n",
|
| 40 |
+
"weather_path = '.../path/to/input/weather.csv' # date, t_mean, t_min, t_max, rh, ws, insolation, evaporation, rainfall\n",
|
| 41 |
+
"admissions_path = '.../path/to/input/admissions.csv' # SIH records\n",
|
| 42 |
+
"population_path = '.../path/to/input/population.csv' # year, population\n",
|
| 43 |
+
"postal_codes_path = '.../path/to/input/pcodes.csv' # cep, x, y\n",
|
| 44 |
+
"\n",
|
| 45 |
+
"boundary_path = '.../path/to/input/municipalities.shp' # Curitiba and Araucária\n",
|
| 46 |
+
"tracts_path = '.../path/to/input/census.shp' # pop, pci, literate, white, black, yellow, brown, indigenous\n",
|
| 47 |
+
"roads_path = '.../path/to/input/roads.shp'\n",
|
| 48 |
+
"gdp_path = '.../path/to/input/gdp.tif'\n",
|
| 49 |
+
"lcz_path = '.../path/to/input/lcz.tif'\n",
|
| 50 |
+
"\n",
|
| 51 |
+
"output_dir = '.../path/to/output/'\n",
|
| 52 |
+
"\n",
|
| 53 |
+
"TARGET_CRS = 'EPSG:31982' # SIRGAS 2000 / UTM zone 22S"
|
| 54 |
+
]
|
| 55 |
+
},
|
| 56 |
+
{
|
| 57 |
+
"cell_type": "code",
|
| 58 |
+
"execution_count": null,
|
| 59 |
+
"metadata": {},
|
| 60 |
+
"outputs": [],
|
| 61 |
+
"source": [
|
| 62 |
+
"# Hospital admissions (DataSUS SIH)\n",
|
| 63 |
+
"# MUNIC_RES: municipality, DT_INTER: admission date, DIAG_PRINC: ICD-10 code,\n",
|
| 64 |
+
"# COD_IDADE: age unit (4 = years), IDADE: age, CEP: postal code\n",
|
| 65 |
+
"adm = pd.read_csv(admissions_path, dtype={'CEP': str, 'DIAG_PRINC': str})\n",
|
| 66 |
+
"\n",
|
| 67 |
+
"# Residents of Curitiba (410690) and Araucária (410180), 2003-2017\n",
|
| 68 |
+
"adm = adm[adm['MUNIC_RES'].isin([410690, 410180])]\n",
|
| 69 |
+
"\n",
|
| 70 |
+
"adm['date'] = pd.to_datetime(adm['DT_INTER'].astype(str), format='%Y%m%d')\n",
|
| 71 |
+
"adm = adm[(adm['date'] >= '2003-01-01') & (adm['date'] <= '2017-12-31')]\n",
|
| 72 |
+
"\n",
|
| 73 |
+
"# Disease group (ICD-10)\n",
|
| 74 |
+
"adm['code'] = adm['DIAG_PRINC'].str[:3]\n",
|
| 75 |
+
"\n",
|
| 76 |
+
"lri_codes = ['J10', 'J11', 'J12', 'J13', 'J14', 'J15', 'J16', 'J17', 'J18', 'J21', 'J22']\n",
|
| 77 |
+
"crd_codes = ['J44', 'J45']\n",
|
| 78 |
+
"\n",
|
| 79 |
+
"adm = adm[adm['code'].str.startswith('J')]\n",
|
| 80 |
+
"\n",
|
| 81 |
+
"adm['disease'] = 'Other'\n",
|
| 82 |
+
"adm.loc[adm['code'].isin(lri_codes), 'disease'] = 'LRI'\n",
|
| 83 |
+
"adm.loc[adm['code'].isin(crd_codes), 'disease'] = 'CRD'\n",
|
| 84 |
+
"\n",
|
| 85 |
+
"# Age in years and age group\n",
|
| 86 |
+
"adm['age'] = np.where(adm['COD_IDADE'] == 4, adm['IDADE'], 0)\n",
|
| 87 |
+
"\n",
|
| 88 |
+
"adm['group'] = 'G2'\n",
|
| 89 |
+
"adm.loc[adm['age'] <= 4, 'group'] = 'G1'\n",
|
| 90 |
+
"adm.loc[adm['age'] >= 60, 'group'] = 'G3'\n",
|
| 91 |
+
"\n",
|
| 92 |
+
"print(adm['disease'].value_counts())\n",
|
| 93 |
+
"print(adm['group'].value_counts())"
|
| 94 |
+
]
|
| 95 |
+
},
|
| 96 |
+
{
|
| 97 |
+
"cell_type": "markdown",
|
| 98 |
+
"metadata": {},
|
| 99 |
+
"source": [
|
| 100 |
+
"## Temporal input matrix"
|
| 101 |
+
]
|
| 102 |
+
},
|
| 103 |
+
{
|
| 104 |
+
"cell_type": "code",
|
| 105 |
+
"execution_count": null,
|
| 106 |
+
"metadata": {},
|
| 107 |
+
"outputs": [],
|
| 108 |
+
"source": [
|
| 109 |
+
"# Air quality: daily mean per station, then mean of the eight stations\n",
|
| 110 |
+
"pollutants = ['so2', 'no2', 'nox', 'co', 'o3', 'pm10']\n",
|
| 111 |
+
"\n",
|
| 112 |
+
"aq = pd.read_csv(air_quality_path, parse_dates=['datetime'])\n",
|
| 113 |
+
"aq['date'] = aq['datetime'].dt.floor('D')\n",
|
| 114 |
+
"\n",
|
| 115 |
+
"aq_station = aq.groupby(['station', 'date'])[pollutants].mean().reset_index()\n",
|
| 116 |
+
"aq_daily = aq_station.groupby('date')[pollutants].mean()\n",
|
| 117 |
+
"\n",
|
| 118 |
+
"print(aq_daily.describe().round(1))"
|
| 119 |
+
]
|
| 120 |
+
},
|
| 121 |
+
{
|
| 122 |
+
"cell_type": "code",
|
| 123 |
+
"execution_count": null,
|
| 124 |
+
"metadata": {},
|
| 125 |
+
"outputs": [],
|
| 126 |
+
"source": [
|
| 127 |
+
"# Weather and daily temperature amplitude\n",
|
| 128 |
+
"wx = pd.read_csv(weather_path, parse_dates=['date']).set_index('date')\n",
|
| 129 |
+
"wx['t_amp'] = wx['t_max'] - wx['t_min']\n",
|
| 130 |
+
"\n",
|
| 131 |
+
"print(wx.describe().round(1))"
|
| 132 |
+
]
|
| 133 |
+
},
|
| 134 |
+
{
|
| 135 |
+
"cell_type": "code",
|
| 136 |
+
"execution_count": null,
|
| 137 |
+
"metadata": {},
|
| 138 |
+
"outputs": [],
|
| 139 |
+
"source": [
|
| 140 |
+
"# Climate indices: consecutive days meeting a condition (reset to zero when not met)\n",
|
| 141 |
+
"# Dry: rainfall < 1 mm; wet: rainfall >= 1 mm; hot: TX > 75th percentile; cold: TN < 25th percentile\n",
|
| 142 |
+
"def consecutive_days(condition):\n",
|
| 143 |
+
" counter = np.zeros(len(condition))\n",
|
| 144 |
+
" for i in range(len(condition)):\n",
|
| 145 |
+
" if condition[i]:\n",
|
| 146 |
+
" if i == 0:\n",
|
| 147 |
+
" counter[i] = 1\n",
|
| 148 |
+
" else:\n",
|
| 149 |
+
" counter[i] = counter[i - 1] + 1\n",
|
| 150 |
+
" else:\n",
|
| 151 |
+
" counter[i] = 0\n",
|
| 152 |
+
" return counter\n",
|
| 153 |
+
"\n",
|
| 154 |
+
"\n",
|
| 155 |
+
"tx_75 = wx['t_max'].quantile(0.75)\n",
|
| 156 |
+
"tn_25 = wx['t_min'].quantile(0.25)\n",
|
| 157 |
+
"\n",
|
| 158 |
+
"wx['cdd'] = consecutive_days((wx['rainfall'] < 1.0).values)\n",
|
| 159 |
+
"wx['cwd'] = consecutive_days((wx['rainfall'] >= 1.0).values)\n",
|
| 160 |
+
"wx['chd'] = consecutive_days((wx['t_max'] > tx_75).values)\n",
|
| 161 |
+
"wx['ccd'] = consecutive_days((wx['t_min'] < tn_25).values)\n",
|
| 162 |
+
"\n",
|
| 163 |
+
"print('TX 75th percentile:', tx_75, '| TN 25th percentile:', tn_25)"
|
| 164 |
+
]
|
| 165 |
+
},
|
| 166 |
+
{
|
| 167 |
+
"cell_type": "code",
|
| 168 |
+
"execution_count": null,
|
| 169 |
+
"metadata": {},
|
| 170 |
+
"outputs": [],
|
| 171 |
+
"source": [
|
| 172 |
+
"# Weekly means (rainfall: weekly total)\n",
|
| 173 |
+
"daily = wx.join(aq_daily, how='outer')\n",
|
| 174 |
+
"\n",
|
| 175 |
+
"weather = ['t_min', 't_max', 't_mean', 't_amp', 'rh', 'ws', 'insolation', 'evaporation']\n",
|
| 176 |
+
"climate = ['cdd', 'cwd', 'chd', 'ccd']\n",
|
| 177 |
+
"\n",
|
| 178 |
+
"weekly = daily[weather + climate + pollutants].resample('7D').mean()\n",
|
| 179 |
+
"weekly['rainfall'] = daily['rainfall'].resample('7D').sum(min_count=1)\n",
|
| 180 |
+
"\n",
|
| 181 |
+
"weekly.index.name = 'date'\n",
|
| 182 |
+
"weekly = weekly.reset_index()"
|
| 183 |
+
]
|
| 184 |
+
},
|
| 185 |
+
{
|
| 186 |
+
"cell_type": "code",
|
| 187 |
+
"execution_count": null,
|
| 188 |
+
"metadata": {},
|
| 189 |
+
"outputs": [],
|
| 190 |
+
"source": [
|
| 191 |
+
"# Trend and season\n",
|
| 192 |
+
"# Unix time (elapsed seconds since 1970-01-01) and Julian day (1-366)\n",
|
| 193 |
+
"weekly['unix_time'] = (weekly['date'] - pd.Timestamp('1970-01-01')).dt.total_seconds()\n",
|
| 194 |
+
"weekly['julian_day'] = weekly['date'].dt.dayofyear\n",
|
| 195 |
+
"\n",
|
| 196 |
+
"# Season (one-hot, summer as reference)\n",
|
| 197 |
+
"season = {12: 'summer', 1: 'summer', 2: 'summer',\n",
|
| 198 |
+
" 3: 'fall', 4: 'fall', 5: 'fall',\n",
|
| 199 |
+
" 6: 'winter', 7: 'winter', 8: 'winter',\n",
|
| 200 |
+
" 9: 'spring', 10: 'spring', 11: 'spring'}\n",
|
| 201 |
+
"\n",
|
| 202 |
+
"weekly['season'] = weekly['date'].dt.month.map(season)\n",
|
| 203 |
+
"\n",
|
| 204 |
+
"weekly['season_fall'] = (weekly['season'] == 'fall').astype(int)\n",
|
| 205 |
+
"weekly['season_spring'] = (weekly['season'] == 'spring').astype(int)\n",
|
| 206 |
+
"weekly['season_winter'] = (weekly['season'] == 'winter').astype(int)\n",
|
| 207 |
+
"\n",
|
| 208 |
+
"weekly = weekly.drop(columns='season')"
|
| 209 |
+
]
|
| 210 |
+
},
|
| 211 |
+
{
|
| 212 |
+
"cell_type": "code",
|
| 213 |
+
"execution_count": null,
|
| 214 |
+
"metadata": {},
|
| 215 |
+
"outputs": [],
|
| 216 |
+
"source": [
|
| 217 |
+
"# Weekly admissions per 1 million inhabitants\n",
|
| 218 |
+
"population = pd.read_csv(population_path).set_index('year')['population']\n",
|
| 219 |
+
"\n",
|
| 220 |
+
"# week of each admission\n",
|
| 221 |
+
"start = weekly['date'].iloc[0]\n",
|
| 222 |
+
"adm['week'] = start + pd.to_timedelta((adm['date'] - start).dt.days // 7 * 7, unit='D')\n",
|
| 223 |
+
"\n",
|
| 224 |
+
"weekly_pop = weekly['date'].dt.year.map(population).values\n",
|
| 225 |
+
"\n",
|
| 226 |
+
"for d in ['LRI', 'CRD', 'Other']:\n",
|
| 227 |
+
" for g in ['General', 'G1', 'G2', 'G3']:\n",
|
| 228 |
+
" if g == 'General':\n",
|
| 229 |
+
" sel = adm[adm['disease'] == d]\n",
|
| 230 |
+
" col_name = d\n",
|
| 231 |
+
" else:\n",
|
| 232 |
+
" sel = adm[(adm['disease'] == d) & (adm['group'] == g)]\n",
|
| 233 |
+
" col_name = d + '_' + g\n",
|
| 234 |
+
"\n",
|
| 235 |
+
" counts = sel.groupby('week').size()\n",
|
| 236 |
+
" weekly[col_name] = counts.reindex(weekly['date']).fillna(0).values / weekly_pop * 1_000_000"
|
| 237 |
+
]
|
| 238 |
+
},
|
| 239 |
+
{
|
| 240 |
+
"cell_type": "code",
|
| 241 |
+
"execution_count": null,
|
| 242 |
+
"metadata": {},
|
| 243 |
+
"outputs": [],
|
| 244 |
+
"source": [
|
| 245 |
+
"# Save\n",
|
| 246 |
+
"weekly.to_csv(output_dir + 'RF_temporal_data.csv', index=False)\n",
|
| 247 |
+
"\n",
|
| 248 |
+
"print(weekly.shape)\n",
|
| 249 |
+
"weekly.head()"
|
| 250 |
+
]
|
| 251 |
+
},
|
| 252 |
+
{
|
| 253 |
+
"cell_type": "markdown",
|
| 254 |
+
"metadata": {},
|
| 255 |
+
"source": [
|
| 256 |
+
"## Spatial input matrix"
|
| 257 |
+
]
|
| 258 |
+
},
|
| 259 |
+
{
|
| 260 |
+
"cell_type": "code",
|
| 261 |
+
"execution_count": null,
|
| 262 |
+
"metadata": {},
|
| 263 |
+
"outputs": [],
|
| 264 |
+
"source": [
|
| 265 |
+
"# 1-km grid\n",
|
| 266 |
+
"boundary = gpd.read_file(boundary_path).to_crs(TARGET_CRS)\n",
|
| 267 |
+
"xmin, ymin, xmax, ymax = boundary.total_bounds\n",
|
| 268 |
+
"\n",
|
| 269 |
+
"cells = []\n",
|
| 270 |
+
"for x in np.arange(xmin, xmax, 1000):\n",
|
| 271 |
+
" for y in np.arange(ymin, ymax, 1000):\n",
|
| 272 |
+
" cells.append(box(x, y, x + 1000, y + 1000))\n",
|
| 273 |
+
"\n",
|
| 274 |
+
"grid = gpd.GeoDataFrame(geometry=cells, crs=TARGET_CRS)\n",
|
| 275 |
+
"grid = grid[grid.intersects(boundary.union_all())].reset_index(drop=True)\n",
|
| 276 |
+
"\n",
|
| 277 |
+
"grid['grid_id'] = range(len(grid))\n",
|
| 278 |
+
"grid['left'] = grid.bounds['minx']\n",
|
| 279 |
+
"grid['bottom'] = grid.bounds['miny']\n",
|
| 280 |
+
"grid['right'] = grid.bounds['maxx']\n",
|
| 281 |
+
"grid['top'] = grid.bounds['maxy']\n",
|
| 282 |
+
"grid['area_km2'] = grid.geometry.area / 1e6\n",
|
| 283 |
+
"\n",
|
| 284 |
+
"print('cells:', len(grid))"
|
| 285 |
+
]
|
| 286 |
+
},
|
| 287 |
+
{
|
| 288 |
+
"cell_type": "code",
|
| 289 |
+
"execution_count": null,
|
| 290 |
+
"metadata": {},
|
| 291 |
+
"outputs": [],
|
| 292 |
+
"source": [
|
| 293 |
+
"# Census data, weighted by the area of each tract inside the cell\n",
|
| 294 |
+
"tracts = gpd.read_file(tracts_path).to_crs(TARGET_CRS)\n",
|
| 295 |
+
"tracts['tract_area'] = tracts.geometry.area\n",
|
| 296 |
+
"\n",
|
| 297 |
+
"pieces = gpd.overlay(tracts, grid[['grid_id', 'geometry']], how='intersection')\n",
|
| 298 |
+
"pieces['piece_area'] = pieces.geometry.area\n",
|
| 299 |
+
"pieces['fraction'] = pieces['piece_area'] / pieces['tract_area']\n",
|
| 300 |
+
"pieces['pop_piece'] = pieces['pop'] * pieces['fraction']\n",
|
| 301 |
+
"\n",
|
| 302 |
+
"census = pieces.groupby('grid_id')['pop_piece'].sum().rename('pop').to_frame()\n",
|
| 303 |
+
"\n",
|
| 304 |
+
"for v in ['pci', 'literate', 'white', 'black', 'yellow', 'brown', 'indigenous']:\n",
|
| 305 |
+
" weighted = (pieces[v] * pieces['piece_area']).groupby(pieces['grid_id']).sum()\n",
|
| 306 |
+
" census[v] = weighted / pieces.groupby('grid_id')['piece_area'].sum()\n",
|
| 307 |
+
"\n",
|
| 308 |
+
"grid = grid.merge(census, left_on='grid_id', right_index=True, how='left')\n",
|
| 309 |
+
"\n",
|
| 310 |
+
"# Population density (inhab. per km2)\n",
|
| 311 |
+
"grid['popd'] = grid['pop'] / grid['area_km2']"
|
| 312 |
+
]
|
| 313 |
+
},
|
| 314 |
+
{
|
| 315 |
+
"cell_type": "code",
|
| 316 |
+
"execution_count": null,
|
| 317 |
+
"metadata": {},
|
| 318 |
+
"outputs": [],
|
| 319 |
+
"source": [
|
| 320 |
+
"# Per capita GDP\n",
|
| 321 |
+
"with rasterio.open(gdp_path) as src:\n",
|
| 322 |
+
" centroids = grid.geometry.centroid.to_crs(src.crs)\n",
|
| 323 |
+
" coords = [(p.x, p.y) for p in centroids]\n",
|
| 324 |
+
" values = np.array([v[0] for v in src.sample(coords)], dtype=float)\n",
|
| 325 |
+
"\n",
|
| 326 |
+
" if src.nodata is not None:\n",
|
| 327 |
+
" values[values == src.nodata] = np.nan\n",
|
| 328 |
+
"\n",
|
| 329 |
+
"grid['gdp'] = values"
|
| 330 |
+
]
|
| 331 |
+
},
|
| 332 |
+
{
|
| 333 |
+
"cell_type": "code",
|
| 334 |
+
"execution_count": null,
|
| 335 |
+
"metadata": {},
|
| 336 |
+
"outputs": [],
|
| 337 |
+
"source": [
|
| 338 |
+
"# Road density (km-1)\n",
|
| 339 |
+
"roads = gpd.read_file(roads_path).to_crs(TARGET_CRS)\n",
|
| 340 |
+
"\n",
|
| 341 |
+
"segments = gpd.overlay(roads, grid[['grid_id', 'geometry']], how='intersection', keep_geom_type=True)\n",
|
| 342 |
+
"segments['length_km'] = segments.geometry.length / 1000\n",
|
| 343 |
+
"\n",
|
| 344 |
+
"road_length = segments.groupby('grid_id')['length_km'].sum()\n",
|
| 345 |
+
"\n",
|
| 346 |
+
"grid['road_length'] = grid['grid_id'].map(road_length).fillna(0)\n",
|
| 347 |
+
"grid['road_dens'] = grid['road_length'] / grid['area_km2']"
|
| 348 |
+
]
|
| 349 |
+
},
|
| 350 |
+
{
|
| 351 |
+
"cell_type": "code",
|
| 352 |
+
"execution_count": null,
|
| 353 |
+
"metadata": {},
|
| 354 |
+
"outputs": [],
|
| 355 |
+
"source": [
|
| 356 |
+
"# Local climate zones: fraction of each class in the cell (classes 11-17 = LCZ A-G)\n",
|
| 357 |
+
"lcz_names = ['1', '2', '3', '4', '5', '6', '7', '8', '9', '10', 'A', 'B', 'C', 'D', 'E', 'F', 'G']\n",
|
| 358 |
+
"\n",
|
| 359 |
+
"for n in lcz_names:\n",
|
| 360 |
+
" grid['LCZ_' + n] = 0.0\n",
|
| 361 |
+
"\n",
|
| 362 |
+
"with rasterio.open(lcz_path) as src:\n",
|
| 363 |
+
" cells_lcz = grid.to_crs(src.crs)\n",
|
| 364 |
+
"\n",
|
| 365 |
+
" for idx, row in cells_lcz.iterrows():\n",
|
| 366 |
+
" out_image, out_transform = mask(src, [mapping(row.geometry)], crop=True)\n",
|
| 367 |
+
" pixels = out_image[0]\n",
|
| 368 |
+
"\n",
|
| 369 |
+
" if src.nodata is not None:\n",
|
| 370 |
+
" pixels = pixels[pixels != src.nodata]\n",
|
| 371 |
+
"\n",
|
| 372 |
+
" if pixels.size == 0:\n",
|
| 373 |
+
" continue\n",
|
| 374 |
+
"\n",
|
| 375 |
+
" values, counts = np.unique(pixels, return_counts=True)\n",
|
| 376 |
+
"\n",
|
| 377 |
+
" for val, count in zip(values, counts):\n",
|
| 378 |
+
" if 1 <= val <= 17:\n",
|
| 379 |
+
" grid.loc[idx, 'LCZ_' + lcz_names[int(val) - 1]] = count / counts.sum()"
|
| 380 |
+
]
|
| 381 |
+
},
|
| 382 |
+
{
|
| 383 |
+
"cell_type": "code",
|
| 384 |
+
"execution_count": null,
|
| 385 |
+
"metadata": {},
|
| 386 |
+
"outputs": [],
|
| 387 |
+
"source": [
|
| 388 |
+
"# Admissions per cell (from postal codes) per 1 million inhabitants\n",
|
| 389 |
+
"postal = pd.read_csv(postal_codes_path, dtype={'cep': str})\n",
|
| 390 |
+
"\n",
|
| 391 |
+
"points = adm.merge(postal, left_on='CEP', right_on='cep', how='inner')\n",
|
| 392 |
+
"points = gpd.GeoDataFrame(points, geometry=gpd.points_from_xy(points['x'], points['y']), crs=TARGET_CRS)\n",
|
| 393 |
+
"\n",
|
| 394 |
+
"points = gpd.sjoin(points, grid[['grid_id', 'geometry']], how='inner', predicate='within')\n",
|
| 395 |
+
"\n",
|
| 396 |
+
"cell_pop = grid['pop'].replace(0, np.nan) # cells without population\n",
|
| 397 |
+
"\n",
|
| 398 |
+
"for d in ['LRI', 'CRD', 'Other']:\n",
|
| 399 |
+
" for g in ['General', 'G1', 'G2', 'G3']:\n",
|
| 400 |
+
" if g == 'General':\n",
|
| 401 |
+
" sel = points[points['disease'] == d]\n",
|
| 402 |
+
" col_name = d\n",
|
| 403 |
+
" else:\n",
|
| 404 |
+
" sel = points[(points['disease'] == d) & (points['group'] == g)]\n",
|
| 405 |
+
" col_name = d + '_' + g\n",
|
| 406 |
+
"\n",
|
| 407 |
+
" counts = sel.groupby('grid_id').size()\n",
|
| 408 |
+
" grid[col_name] = grid['grid_id'].map(counts).fillna(0) / cell_pop * 1_000_000"
|
| 409 |
+
]
|
| 410 |
+
},
|
| 411 |
+
{
|
| 412 |
+
"cell_type": "code",
|
| 413 |
+
"execution_count": null,
|
| 414 |
+
"metadata": {},
|
| 415 |
+
"outputs": [],
|
| 416 |
+
"source": [
|
| 417 |
+
"# Rename and save\n",
|
| 418 |
+
"names = {'white': 'WHT', 'black': 'BLK', 'yellow': 'YEL', 'brown': 'BRN', 'indigenous': 'IND',\n",
|
| 419 |
+
" 'literate': 'LT', 'popd': 'PD', 'pci': 'PCI', 'road_dens': 'RD', 'gdp': 'GDP'}\n",
|
| 420 |
+
"\n",
|
| 421 |
+
"grid = grid.rename(columns=names)\n",
|
| 422 |
+
"\n",
|
| 423 |
+
"lcz_cols = ['LCZ_' + n for n in lcz_names]\n",
|
| 424 |
+
"targets = ['LRI', 'LRI_G1', 'LRI_G2', 'LRI_G3',\n",
|
| 425 |
+
" 'CRD', 'CRD_G1', 'CRD_G2', 'CRD_G3',\n",
|
| 426 |
+
" 'Other', 'Other_G1', 'Other_G2', 'Other_G3']\n",
|
| 427 |
+
"\n",
|
| 428 |
+
"spatial = grid[['grid_id', 'left', 'top', 'right', 'bottom'] + list(names.values()) + lcz_cols + targets]\n",
|
| 429 |
+
"spatial.to_csv(output_dir + 'RF_spatial_data.csv', index=False)\n",
|
| 430 |
+
"\n",
|
| 431 |
+
"print(spatial.shape)\n",
|
| 432 |
+
"spatial.head()"
|
| 433 |
+
]
|
| 434 |
+
}
|
| 435 |
+
],
|
| 436 |
+
"metadata": {
|
| 437 |
+
"kernelspec": {
|
| 438 |
+
"display_name": "Python 3",
|
| 439 |
+
"language": "python",
|
| 440 |
+
"name": "python3"
|
| 441 |
+
},
|
| 442 |
+
"language_info": {
|
| 443 |
+
"name": "python"
|
| 444 |
+
}
|
| 445 |
+
},
|
| 446 |
+
"nbformat": 4,
|
| 447 |
+
"nbformat_minor": 5
|
| 448 |
+
}
|