oukawa commited on
Commit
a4d442f
·
verified ·
1 Parent(s): 1d72c87

Update codes

Browse files
RF_spatial_explainability_all_diseases_and_groups.ipynb CHANGED
@@ -1,138 +1,268 @@
1
- {
2
- "cells": [
3
- {
4
- "cell_type": "code",
5
- "execution_count": null,
6
- "metadata": {
7
- "colab": {
8
- "base_uri": "https://localhost:8080/"
9
- },
10
- "executionInfo": {
11
- "elapsed": 113875,
12
- "status": "ok",
13
- "timestamp": 1757816776531,
14
- "user": {
15
- "displayName": "Gabriel Oukawa",
16
- "userId": "10622206185982477384"
17
- },
18
- "user_tz": 180
19
- },
20
- "id": "SzDbN-vemmfy",
21
- "outputId": "7ef54d4c-7922-45bb-c9c5-2a4a0cad4abe"
22
- },
23
- "outputs": [],
24
- "source": [
25
- "# Libraries\n",
26
- "import pandas as pd\n",
27
- "import numpy as np\n",
28
- "import pickle\n",
29
- "import shap\n",
30
- "import matplotlib.pyplot as plt"
31
- ]
32
- },
33
- {
34
- "cell_type": "code",
35
- "execution_count": null,
36
- "metadata": {},
37
- "outputs": [],
38
- "source": [
39
- "# Load data and models\n",
40
- "q = input('Load which matrix, 'lri', 'crd' or 'other'? ').strip().lower()\n",
41
- "if q not in ['lri', 'crd', 'other']:\n",
42
- " raise ValueError('Invalid input.')\n",
43
- "\n",
44
- "groups = ['general', 'g1', 'g2', 'g3']\n",
45
- "data_path = '.../path/to/input/RF_spatial_data.csv'\n",
46
- "df = pd.read_csv(data_path, delimiter=',', index_col=False)\n",
47
- "\n",
48
- "model_file = f'.../path/to/models/RF_spatial_{q.upper()}_admissions_per_1_million_all_groups.pkl'\n",
49
- "with open(model_file, 'rb') as f:\n",
50
- " all_models = pickle.load(f)\n",
51
- "\n",
52
- "shap_values_list, X_list, titles = [], [], []\n",
53
- "\n",
54
- "for g in groups:\n",
55
- " col_name = f'{q.upper()}' if g == 'general' else f'{q.upper()}_{g.upper()}'\n",
56
- " title = f'{q.upper()} - All Ages' if g == 'general' else f'{q.upper()} - Group {g[-1]}'\n",
57
- " model_key = f'{q.upper()}_General' if g == 'general' else f'{q.upper()}_{g.upper()}'\n",
58
- "\n",
59
- " if col_name not in df.columns or model_key not in all_models:\n",
60
- " continue\n",
61
- "\n",
62
- " model = all_models[model_key]\n",
63
- " X = df.drop(columns=[c for c in df.columns if c.startswith(q.upper())])\n",
64
- " explainer = shap.TreeExplainer(model)\n",
65
- " shap_values = explainer.shap_values(X)\n",
66
- "\n",
67
- " shap_values_list.append(shap_values)\n",
68
- " X_list.append(X)\n",
69
- " titles.append(title)\n",
70
- "\n",
71
- " shap_df = pd.DataFrame(shap_values, columns=X.columns)\n",
72
- " shap_df.to_csv(f\".../path/to/output/SHAP_{model_key}.csv\", index=False)"
73
- ]
74
- },
75
- {
76
- "cell_type": "code",
77
- "execution_count": null,
78
- "metadata": {},
79
- "outputs": [],
80
- "source": [
81
- "# SHAP summary plots\n",
82
- "fig, axes = plt.subplots(2, 2, figsize=(12, 10), dpi=600)\n",
83
- "axes = axes.flatten()\n",
84
- "\n",
85
- "for i in range(len(shap_values_list)):\n",
86
- " plt.sca(axes[i])\n",
87
- " shap.summary_plot(shap_values_list[i], X_list[i], feature_names=X_list[i].columns, plot_type='dot', show=False, max_display=10)\n",
88
- " ax = plt.gca()\n",
89
- " ax.set_title(titles[i], fontsize=12)\n",
90
- " ax.set_xlabel(ax.get_xlabel(), fontsize=12)\n",
91
- " ax.tick_params(axis='both', which='major', labelsize=12)\n",
92
- "\n",
93
- "plt.tight_layout(pad=1.5)\n",
94
- "plt.show()"
95
- ]
96
- },
97
- {
98
- "cell_type": "code",
99
- "execution_count": null,
100
- "metadata": {},
101
- "outputs": [],
102
- "source": [
103
- "# SHAP dependence plots for top 10 features\n",
104
- "mean_abs_shap = np.abs(shap_values.values).mean(axis=0)\n",
105
- "top_features_idx = np.argsort(mean_abs_shap)[-10:][::-1]\n",
106
- "top_features_names = [X.columns[i] for i in top_features_idx]\n",
107
- "\n",
108
- "fig, axes = plt.subplots(4, 2, figsize=(12, 16))\n",
109
- "axes = axes.flatten()\n",
110
- "for i, feature in enumerate(top_features_names):\n",
111
- " shap.plots.scatter(shap_values[:, feature], ax=axes[i], show=False, color=shap_values)\n",
112
- " axes[i].set_title(feature, fontsize=12)\n",
113
- "\n",
114
- "plt.tight_layout()\n",
115
- "plt.show()"
116
- ]
117
- }
118
- ],
119
- "metadata": {
120
- "colab": {
121
- "provenance": [
122
- {
123
- "file_id": "1CTjVKHB6iiDvAQmm4JsyyIgVZShwcp_4",
124
- "timestamp": 1723512979448
125
- }
126
- ]
127
- },
128
- "kernelspec": {
129
- "display_name": "Python 3",
130
- "name": "python3"
131
- },
132
- "language_info": {
133
- "name": "python"
134
- }
135
- },
136
- "nbformat": 4,
137
- "nbformat_minor": 0
138
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "cells": [
3
+ {
4
+ "cell_type": "markdown",
5
+ "metadata": {},
6
+ "source": [
7
+ "# SHAP explainability of the spatial RF models\n",
8
+ "\n",
9
+ "SHAP values of the final models for each age group, importance with the direction of association (Fig. S6),\n",
10
+ "dependence plots (Fig. S7) and colored versions (Fig. S13).\n",
11
+ "\n",
12
+ "Input: `RF_spatial_data.csv` and `RF_spatial_{LRI|CRD|Other}_admissions_per_1_million_all_groups.pkl`."
13
+ ]
14
+ },
15
+ {
16
+ "cell_type": "code",
17
+ "metadata": {},
18
+ "source": [
19
+ "# Libraries\n",
20
+ "import numpy as np\n",
21
+ "import pandas as pd\n",
22
+ "import joblib\n",
23
+ "import shap\n",
24
+ "import matplotlib.pyplot as plt"
25
+ ],
26
+ "execution_count": null,
27
+ "outputs": []
28
+ },
29
+ {
30
+ "cell_type": "code",
31
+ "metadata": {},
32
+ "source": [
33
+ "# Load input data and models\n",
34
+ "q = input(\"Load which disease group: 'LRI', 'CRD' or 'Other'? \").strip()\n",
35
+ "if q not in ['LRI', 'CRD', 'Other']:\n",
36
+ " raise ValueError(\"Invalid option!\")\n",
37
+ "\n",
38
+ "kind = 'spatial'\n",
39
+ "\n",
40
+ "df = pd.read_csv('.../path/to/input/RF_spatial_data.csv')\n",
41
+ "\n",
42
+ "targets = ['LRI', 'LRI_G1', 'LRI_G2', 'LRI_G3',\n",
43
+ " 'CRD', 'CRD_G1', 'CRD_G2', 'CRD_G3',\n",
44
+ " 'Other', 'Other_G1', 'Other_G2', 'Other_G3']\n",
45
+ "\n",
46
+ "df = df.dropna(subset=targets).reset_index(drop=True)\n",
47
+ "\n",
48
+ "models = joblib.load(f'.../path/to/models/RF_spatial_{q}_admissions_per_1_million_all_groups.pkl')\n",
49
+ "\n",
50
+ "X = df[list(models[q + '_General'].feature_names_in_)]\n",
51
+ "print(X.shape)"
52
+ ],
53
+ "execution_count": null,
54
+ "outputs": []
55
+ },
56
+ {
57
+ "cell_type": "code",
58
+ "metadata": {},
59
+ "source": [
60
+ "# SHAP values for each age group\n",
61
+ "groups = ['General', 'G1', 'G2', 'G3']\n",
62
+ "shap_values = {}\n",
63
+ "\n",
64
+ "for g in groups:\n",
65
+ " model = models[q + '_' + g]\n",
66
+ " explainer = shap.TreeExplainer(model)\n",
67
+ "\n",
68
+ " shap_values[g] = pd.DataFrame(explainer.shap_values(X), columns=X.columns)\n",
69
+ " shap_values[g].to_csv(f'SHAP_{kind}_{q}_{g}.csv', index=False)\n",
70
+ "\n",
71
+ " print(g, 'done')"
72
+ ],
73
+ "execution_count": null,
74
+ "outputs": []
75
+ },
76
+ {
77
+ "cell_type": "code",
78
+ "metadata": {},
79
+ "source": [
80
+ "# Mean |SHAP| and direction of association (sign of the Pearson correlation between feature and SHAP values)\n",
81
+ "def importance(S, X):\n",
82
+ " out = pd.DataFrame(index=X.columns)\n",
83
+ " out['mean_abs_shap'] = S.abs().mean()\n",
84
+ " out['positive'] = [X[c].corr(S[c]) > 0 for c in X.columns]\n",
85
+ " return out.sort_values('mean_abs_shap', ascending=False)\n",
86
+ "\n",
87
+ "\n",
88
+ "# Top 10 features (Fig. S6, log scale) and sum of the others\n",
89
+ "n_top = 10\n",
90
+ "\n",
91
+ "fig, axes = plt.subplots(2, 2, figsize=(11, 9))\n",
92
+ "axes = axes.flatten()\n",
93
+ "\n",
94
+ "for i, g in enumerate(groups):\n",
95
+ " imp = importance(shap_values[g], X)\n",
96
+ " top = imp.iloc[:n_top]\n",
97
+ " rest = imp['mean_abs_shap'].iloc[n_top:].sum()\n",
98
+ "\n",
99
+ " ax = axes[i]\n",
100
+ " for k, (name, row) in enumerate(top.iterrows()):\n",
101
+ " if row['positive']:\n",
102
+ " ax.barh(k, row['mean_abs_shap'], color='#ff0055')\n",
103
+ " sign = '(+)'\n",
104
+ " else:\n",
105
+ " ax.barh(k, row['mean_abs_shap'], color='#0088ff', hatch='////', edgecolor='white')\n",
106
+ " sign = '(-)'\n",
107
+ " ax.text(row['mean_abs_shap'], k, f' {row[\"mean_abs_shap\"]:.2f} {sign}', va='center', fontsize=8)\n",
108
+ "\n",
109
+ " ax.barh(n_top, rest, color='0.6')\n",
110
+ " ax.text(rest, n_top, f' {rest:.2f}', va='center', fontsize=8)\n",
111
+ "\n",
112
+ " ax.set_yticks(range(n_top + 1))\n",
113
+ " ax.set_yticklabels(list(top.index) + ['Other features'])\n",
114
+ " ax.invert_yaxis()\n",
115
+ " ax.set_xlabel('mean |SHAP value|')\n",
116
+ " ax.set_xscale('log')\n",
117
+ " ax.set_title(q + ' ' + g)\n",
118
+ "\n",
119
+ "plt.tight_layout()\n",
120
+ "plt.show()"
121
+ ],
122
+ "execution_count": null,
123
+ "outputs": []
124
+ },
125
+ {
126
+ "cell_type": "code",
127
+ "metadata": {},
128
+ "source": [
129
+ "# LOWESS trend (tricube weights) with a 95% bootstrap band\n",
130
+ "def lowess(x, y, frac=0.2):\n",
131
+ " grid = np.linspace(np.percentile(x, 1), np.percentile(x, 99), 150)\n",
132
+ " k = max(int(frac * len(x)), 10)\n",
133
+ " fit = np.zeros(len(grid))\n",
134
+ "\n",
135
+ " for i in range(len(grid)):\n",
136
+ " d = np.abs(x - grid[i])\n",
137
+ " h = max(np.sort(d)[k - 1], 1e-9) # window half-width\n",
138
+ " w = np.clip(1 - (d / h) ** 3, 0, None) ** 3\n",
139
+ "\n",
140
+ " xm = np.sum(w * x) / np.sum(w)\n",
141
+ " ym = np.sum(w * y) / np.sum(w)\n",
142
+ " b = np.sum(w * (x - xm) * (y - ym)) / max(np.sum(w * (x - xm) ** 2), 1e-12)\n",
143
+ " fit[i] = ym + b * (grid[i] - xm)\n",
144
+ "\n",
145
+ " return grid, fit\n",
146
+ "\n",
147
+ "\n",
148
+ "def lowess_band(x, y, n_boot=200):\n",
149
+ " rng = np.random.default_rng(0)\n",
150
+ " grid, fit = lowess(x, y)\n",
151
+ "\n",
152
+ " boots = []\n",
153
+ " for i in range(n_boot):\n",
154
+ " idx = rng.integers(0, len(x), len(x))\n",
155
+ " g, f = lowess(x[idx], y[idx])\n",
156
+ " boots.append(np.interp(grid, g, f))\n",
157
+ "\n",
158
+ " lo, hi = np.percentile(boots, [2.5, 97.5], axis=0)\n",
159
+ " return grid, fit, lo, hi\n",
160
+ "\n",
161
+ "\n",
162
+ "def feature_values(c):\n",
163
+ " x = df[c].values.astype(float)\n",
164
+ " if c == 'unix_time':\n",
165
+ " x = 1970 + x / 31557600 # calendar year\n",
166
+ " return x"
167
+ ],
168
+ "execution_count": null,
169
+ "outputs": []
170
+ },
171
+ {
172
+ "cell_type": "code",
173
+ "metadata": {},
174
+ "source": [
175
+ "# Dependence plots of the 4 most important features (Fig. S7)\n",
176
+ "if q == 'CRD':\n",
177
+ " g = 'G3'\n",
178
+ "else:\n",
179
+ " g = 'G1'\n",
180
+ "\n",
181
+ "S = shap_values[g]\n",
182
+ "features = list(importance(S, X).index[:4])\n",
183
+ "\n",
184
+ "# Fig. S7 shows RD, WHT, PCI and LCZ_6 for LRI G1 (LCZ_6 ranks fifth, after YEL)\n",
185
+ "if q == 'LRI':\n",
186
+ " features = ['RD', 'WHT', 'PCI', 'LCZ_6']\n",
187
+ "\n",
188
+ "fig, axes = plt.subplots(2, 2, figsize=(10, 7))\n",
189
+ "axes = axes.flatten()\n",
190
+ "\n",
191
+ "for i, c in enumerate(features):\n",
192
+ " x = feature_values(c)\n",
193
+ " y = S[c].values\n",
194
+ " ax = axes[i]\n",
195
+ "\n",
196
+ " ax.axhline(0, color='k', lw=0.6, ls='--')\n",
197
+ "\n",
198
+ " if set(np.unique(x)) <= {0, 1}: # binary feature: mean and 95% CI\n",
199
+ " ax.scatter(x, y, s=7, color='#0088ff', alpha=0.45)\n",
200
+ " for v in [0, 1]:\n",
201
+ " yv = y[x == v]\n",
202
+ " ax.errorbar(v, yv.mean(), yerr=1.96 * yv.std() / np.sqrt(len(yv)), fmt='o', color='#ff0055', capsize=4)\n",
203
+ " ax.set_xticks([0, 1])\n",
204
+ " else:\n",
205
+ " ax.scatter(x, y, s=7, color='#0088ff', alpha=0.45)\n",
206
+ " grid, fit, lo, hi = lowess_band(x, y)\n",
207
+ " ax.fill_between(grid, lo, hi, color='#ff0055', alpha=0.25)\n",
208
+ " ax.plot(grid, fit, color='#ff0055', lw=1.6)\n",
209
+ "\n",
210
+ " # feature distribution\n",
211
+ " y0, y1 = ax.get_ylim()\n",
212
+ " counts, edges = np.histogram(x, bins=40)\n",
213
+ " ax.bar(edges[:-1], 0.15 * (y1 - y0) * counts / counts.max(), width=np.diff(edges), bottom=y0,\n",
214
+ " align='edge', color='0.6', alpha=0.5)\n",
215
+ "\n",
216
+ " ax.set_xlabel(c)\n",
217
+ " ax.set_ylabel('SHAP value')\n",
218
+ " ax.set_title(q + ' ' + g)\n",
219
+ "\n",
220
+ "plt.tight_layout()\n",
221
+ "plt.show()"
222
+ ],
223
+ "execution_count": null,
224
+ "outputs": []
225
+ },
226
+ {
227
+ "cell_type": "code",
228
+ "metadata": {},
229
+ "source": [
230
+ "# Dependence plots colored by the most strongly correlated feature (Fig. S13)\n",
231
+ "# (highest absolute Spearman correlation with the SHAP values of the plotted feature)\n",
232
+ "fig, axes = plt.subplots(2, 2, figsize=(11, 7))\n",
233
+ "axes = axes.flatten()\n",
234
+ "\n",
235
+ "for i, c in enumerate(features):\n",
236
+ " r = {}\n",
237
+ " for other in X.columns:\n",
238
+ " if other != c and X[other].std() > 0:\n",
239
+ " r[other] = abs(S[c].corr(X[other], method='spearman'))\n",
240
+ " partner = max(r, key=r.get)\n",
241
+ "\n",
242
+ " sc = axes[i].scatter(feature_values(c), S[c].values, s=7, c=feature_values(partner), cmap='viridis')\n",
243
+ " plt.colorbar(sc, ax=axes[i]).set_label(partner)\n",
244
+ "\n",
245
+ " axes[i].axhline(0, color='k', lw=0.6, ls='--')\n",
246
+ " axes[i].set_xlabel(c)\n",
247
+ " axes[i].set_ylabel('SHAP value')\n",
248
+ "\n",
249
+ "plt.tight_layout()\n",
250
+ "plt.show()"
251
+ ],
252
+ "execution_count": null,
253
+ "outputs": []
254
+ }
255
+ ],
256
+ "metadata": {
257
+ "kernelspec": {
258
+ "display_name": "Python 3",
259
+ "language": "python",
260
+ "name": "python3"
261
+ },
262
+ "language_info": {
263
+ "name": "python"
264
+ }
265
+ },
266
+ "nbformat": 4,
267
+ "nbformat_minor": 5
268
+ }
RF_spatial_training_and_cross-validation_all_diseases_and_groups.ipynb CHANGED
@@ -1,291 +1,298 @@
1
- {
2
- "cells": [
3
- {
4
- "cell_type": "code",
5
- "execution_count": null,
6
- "metadata": {
7
- "colab": {
8
- "base_uri": "https://localhost:8080/"
9
- },
10
- "executionInfo": {
11
- "elapsed": 113875,
12
- "status": "ok",
13
- "timestamp": 1757816776531,
14
- "user": {
15
- "displayName": "Gabriel Oukawa",
16
- "userId": "10622206185982477384"
17
- },
18
- "user_tz": 180
19
- },
20
- "id": "SzDbN-vemmfy",
21
- "outputId": "7ef54d4c-7922-45bb-c9c5-2a4a0cad4abe"
22
- },
23
- "outputs": [],
24
- "source": [
25
- "# Libraries\n",
26
- "import pandas as pd\n",
27
- "import numpy as np\n",
28
- "import joblib\n",
29
- "from pathlib import Path\n",
30
- "from sklearn.ensemble import RandomForestRegressor\n",
31
- "from sklearn.model_selection import train_test_split, GridSearchCV\n",
32
- "from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score"
33
- ]
34
- },
35
- {
36
- "cell_type": "code",
37
- "execution_count": null,
38
- "metadata": {
39
- "colab": {
40
- "base_uri": "https://localhost:8080/"
41
- },
42
- "executionInfo": {
43
- "elapsed": 184,
44
- "status": "ok",
45
- "timestamp": 1757816855352,
46
- "user": {
47
- "displayName": "Gabriel Oukawa",
48
- "userId": "10622206185982477384"
49
- },
50
- "user_tz": 180
51
- },
52
- "id": "39d0a9OJqmxZ",
53
- "outputId": "c100c3b4-66ef-4620-fbd2-25872d749deb"
54
- },
55
- "outputs": [],
56
- "source": [
57
- "# Load input data\n",
58
- "data_path = '.../path/to/input/RF_spatial_data.csv'\n",
59
- "df = pd.read_csv(data_path, delimiter=',', index_col=False)\n",
60
- "\n",
61
- "df = df[(df['date'] >= '2003-01-01') &\n",
62
- " (df['date'] <= '2017-12-31')].reset_index(drop=True)\n",
63
- "\n",
64
- "# print(df.head())\n",
65
- "# print(df.shape)"
66
- ]
67
- },
68
- {
69
- "cell_type": "code",
70
- "execution_count": null,
71
- "metadata": {
72
- "colab": {
73
- "base_uri": "https://localhost:8080/"
74
- },
75
- "executionInfo": {
76
- "elapsed": 6803,
77
- "status": "ok",
78
- "timestamp": 1757816916603,
79
- "user": {
80
- "displayName": "Gabriel Oukawa",
81
- "userId": "10622206185982477384"
82
- },
83
- "user_tz": 180
84
- },
85
- "id": "YnEWfX0s_6QN",
86
- "outputId": "3a892047-134b-4816-886f-7bedc7d0c9b1"
87
- },
88
- "outputs": [],
89
- "source": [
90
- "# Train the RF model, evaluate metrics and save it to a file (single run)\n",
91
- "q = input(\"Load which matrix: 'lri', 'crd' or 'general'?\").strip().lower()\n",
92
- "if q not in ['lri', 'crd', 'general']:\n",
93
- " raise ValueError(\"Invalid option!\")\n",
94
- "\n",
95
- "if q == 'lri':\n",
96
- " name = 'LRI'\n",
97
- "elif q == 'crd':\n",
98
- " name = 'CRD'\n",
99
- "elif q == 'general':\n",
100
- " name = 'Other'\n",
101
- "\n",
102
- "all_models = {}\n",
103
- "\n",
104
- "groups = ['all', 'g1', 'g2', 'g3']\n",
105
- "\n",
106
- "for g in groups:\n",
107
- " if g == 'all':\n",
108
- " col_name = f'{q}'\n",
109
- " model_name = f'{name}_General'\n",
110
- " else:\n",
111
- " col_name = f'{q}_{g}'\n",
112
- " model_name = f'{name}_{g.upper()}'\n",
113
- "\n",
114
- " Y = df[col_name]\n",
115
- " drop_cols = ['date', 'Other', 'Other_G1', 'Other_G2', 'Other_G3',\n",
116
- " 'LRI', 'LRI_G1', 'LRI_G2', 'LRI_G3',\n",
117
- " 'CRD', 'CRD_G1', 'CRD_G2', 'CRD_G3']\n",
118
- " X = df.drop(columns=[c for c in drop_cols if c in df.columns])\n",
119
- "\n",
120
- " X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.2, random_state=42)\n",
121
- "\n",
122
- " model = RandomForestRegressor(\n",
123
- " n_estimators=500,\n",
124
- " oob_score=False,\n",
125
- " bootstrap=True,\n",
126
- " max_depth=20,\n",
127
- " max_features=None,\n",
128
- " min_samples_leaf=1,\n",
129
- " min_samples_split=2,\n",
130
- " n_jobs=-1,\n",
131
- " verbose=1\n",
132
- " )\n",
133
- "\n",
134
- " model.fit(X_train, Y_train)\n",
135
- " Y_pred = model.predict(X_test)\n",
136
- "\n",
137
- " rmse = np.sqrt(mean_squared_error(Y_test, Y_pred))\n",
138
- " mae = mean_absolute_error(Y_test, Y_pred)\n",
139
- " r2 = r2_score(Y_test, Y_pred)\n",
140
- "\n",
141
- " print('RMSE:', rmse)\n",
142
- " print('MAE:', mae)\n",
143
- " print('R2:', r2)\n",
144
- "\n",
145
- " all_models[model_name] = model\n",
146
- "\n",
147
- "output_file = Path(f'RF_spatial_{q.upper()}_all_groups.pkl')\n",
148
- "joblib.dump(all_models, output_file)"
149
- ]
150
- },
151
- {
152
- "cell_type": "code",
153
- "execution_count": null,
154
- "metadata": {},
155
- "outputs": [],
156
- "source": [
157
- "# Train the RF model and display scoring metrics (cross-validation)\n",
158
- "q = input(\"Load which matrix: 'lri', 'crd' or 'general'?\").strip().lower()\n",
159
- "if q not in ['lri', 'crd', 'general']:\n",
160
- " raise ValueError(\"Invalid option!\")\n",
161
- "\n",
162
- "groups = ['all', 'g1', 'g2', 'g3']\n",
163
- "results = []\n",
164
- "\n",
165
- "for g in groups:\n",
166
- " if g == 'all':\n",
167
- " col_name = f'{q}'\n",
168
- " else:\n",
169
- " col_name = f'{q}_{g}'\n",
170
- " \n",
171
- " Y = df[col_name]\n",
172
- "\n",
173
- " X = df.drop(columns=['date',\n",
174
- " 'Other','Other_G1','Other_G2','Other_G3',\n",
175
- " 'LRI','LRI_G1','LRI_G2','LRI_G3',\n",
176
- " 'CRD','CRD_G1','CRD_G2','CRD_G3',\n",
177
- " ])\n",
178
- "\n",
179
- " metrics_df = pd.DataFrame(columns=['R2', 'MAE', 'RMSE'])\n",
180
- "\n",
181
- " for i in range(100):\n",
182
- " X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.2)\n",
183
- "\n",
184
- " model = RandomForestRegressor(\n",
185
- " n_estimators=500,\n",
186
- " oob_score=False,\n",
187
- " bootstrap=True,\n",
188
- " max_depth=20,\n",
189
- " max_features=None,\n",
190
- " min_samples_leaf=1,\n",
191
- " min_samples_split=2,\n",
192
- " n_jobs=-1,\n",
193
- " verbose=1\n",
194
- " )\n",
195
- "\n",
196
- " model.fit(X_train, Y_train)\n",
197
- " Y_pred = model.predict(X_test)\n",
198
- "\n",
199
- " rmse = np.sqrt(mean_squared_error(Y_test, Y_pred))\n",
200
- " mae = mean_absolute_error(Y_test, Y_pred)\n",
201
- " r2 = r2_score(Y_test, Y_pred)\n",
202
- "\n",
203
- " metrics_df.loc[i] = [r2, mae, rmse]\n",
204
- "\n",
205
- " summary = {\n",
206
- " 'group': g,\n",
207
- " 'R2_mean': metrics_df['R2'].mean(),\n",
208
- " 'R2_min': metrics_df['R2'].min(),\n",
209
- " 'R2_max': metrics_df['R2'].max(),\n",
210
- " 'MAE_mean': metrics_df['MAE'].mean(),\n",
211
- " 'MAE_min': metrics_df['MAE'].min(),\n",
212
- " 'MAE_max': metrics_df['MAE'].max(),\n",
213
- " 'RMSE_mean': metrics_df['RMSE'].mean(),\n",
214
- " 'RMSE_min': metrics_df['RMSE'].min(),\n",
215
- " 'RMSE_max': metrics_df['RMSE'].max()\n",
216
- " }\n",
217
- "\n",
218
- " results.append(summary)\n",
219
- "\n",
220
- "summary_metrics = pd.DataFrame(results)\n",
221
- "summary_metrics = summary_metrics.set_index('group')\n",
222
- "\n",
223
- "summary_metrics.head()"
224
- ]
225
- },
226
- {
227
- "cell_type": "code",
228
- "execution_count": null,
229
- "metadata": {},
230
- "outputs": [],
231
- "source": [
232
- "# Test the best combination of hyperparameters\n",
233
- "X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.2, random_state=42) # Keep results reproducible\n",
234
- "\n",
235
- "param_grid = {\n",
236
- " 'n_estimators': [100, 250, 500, 750],\n",
237
- " 'max_depth': [10, 20, 30, None],\n",
238
- " 'min_samples_leaf': [1, 2, 5],\n",
239
- " 'min_samples_split': [2, 5, 10],\n",
240
- " 'max_features': ['sqrt', 'log2', 0.3, None],\n",
241
- " 'bootstrap': [True, False]\n",
242
- "}\n",
243
- "\n",
244
- "rf = RandomForestRegressor(oob_score=False, n_jobs=-1, random_state=42)\n",
245
- "\n",
246
- "grid_search = GridSearchCV(\n",
247
- " estimator=rf,\n",
248
- " param_grid=param_grid,\n",
249
- " scoring='r2',\n",
250
- " cv=5,\n",
251
- " verbose=1,\n",
252
- " n_jobs=-1\n",
253
- ")\n",
254
- "\n",
255
- "grid_search.fit(X_train, Y_train)\n",
256
- "\n",
257
- "best_model = grid_search.best_estimator_\n",
258
- "print(grid_search.best_params_)\n",
259
- "\n",
260
- "Y_pred = best_model.predict(X_test)\n",
261
- "\n",
262
- "r2 = r2_score(Y_test, Y_pred)\n",
263
- "rmse = np.sqrt(mean_squared_error(Y_test, Y_pred))\n",
264
- "mae = mean_absolute_error(Y_test, Y_pred)\n",
265
- "\n",
266
- "print(f'R2: {r2:.4f}')\n",
267
- "print(f'RMSE: {rmse:.4f}')\n",
268
- "print(f'MAE: {mae:.4f}')"
269
- ]
270
- }
271
- ],
272
- "metadata": {
273
- "colab": {
274
- "provenance": [
275
- {
276
- "file_id": "1CTjVKHB6iiDvAQmm4JsyyIgVZShwcp_4",
277
- "timestamp": 1723512979448
278
- }
279
- ]
280
- },
281
- "kernelspec": {
282
- "display_name": "Python 3",
283
- "name": "python3"
284
- },
285
- "language_info": {
286
- "name": "python"
287
- }
288
- },
289
- "nbformat": 4,
290
- "nbformat_minor": 0
291
- }
 
 
 
 
 
 
 
 
1
+ {
2
+ "cells": [
3
+ {
4
+ "cell_type": "markdown",
5
+ "metadata": {},
6
+ "source": [
7
+ "# Spatial RF models (cumulative admissions, 1-km grid)\n",
8
+ "\n",
9
+ "Spatially blocked cross-validation (Table S2), nested models (Table S6), final models and grid search (Table S3), for\n",
10
+ "each disease group (LRI, CRD, Other) and age group (General, G1-G3).\n",
11
+ "\n",
12
+ "Input: `RF_spatial_data.csv`, with the cell bounds (`left`, `top`, `right`, `bottom`), the predictors and the\n",
13
+ "admission rates per 1 million inhabitants over 2003-2017 (`LRI`, `LRI_G1`, ..., `Other_G3`)."
14
+ ]
15
+ },
16
+ {
17
+ "cell_type": "code",
18
+ "metadata": {},
19
+ "source": [
20
+ "# Libraries\n",
21
+ "import pandas as pd\n",
22
+ "import numpy as np\n",
23
+ "import joblib\n",
24
+ "from pathlib import Path\n",
25
+ "from sklearn.ensemble import RandomForestRegressor\n",
26
+ "from sklearn.model_selection import train_test_split, GridSearchCV, GroupKFold\n",
27
+ "from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score"
28
+ ],
29
+ "execution_count": null,
30
+ "outputs": []
31
+ },
32
+ {
33
+ "cell_type": "code",
34
+ "metadata": {},
35
+ "source": [
36
+ "# Load input data\n",
37
+ "data_path = '.../path/to/input/RF_spatial_data.csv'\n",
38
+ "df = pd.read_csv(data_path, delimiter=',', index_col=False)\n",
39
+ "\n",
40
+ "targets = ['LRI', 'LRI_G1', 'LRI_G2', 'LRI_G3',\n",
41
+ " 'CRD', 'CRD_G1', 'CRD_G2', 'CRD_G3',\n",
42
+ " 'Other', 'Other_G1', 'Other_G2', 'Other_G3']\n",
43
+ "\n",
44
+ "df = df.dropna(subset=targets).reset_index(drop=True)\n",
45
+ "\n",
46
+ "print(df.shape)"
47
+ ],
48
+ "execution_count": null,
49
+ "outputs": []
50
+ },
51
+ {
52
+ "cell_type": "code",
53
+ "metadata": {},
54
+ "source": [
55
+ "# Predictors\n",
56
+ "socio = ['WHT', 'BLK', 'YEL', 'BRN', 'IND', 'LT', 'PCI', 'GDP']\n",
57
+ "lcz = ['LCZ_1', 'LCZ_2', 'LCZ_3', 'LCZ_4', 'LCZ_5', 'LCZ_6', 'LCZ_7', 'LCZ_8', 'LCZ_9', 'LCZ_10',\n",
58
+ " 'LCZ_A', 'LCZ_B', 'LCZ_C', 'LCZ_D', 'LCZ_E', 'LCZ_F', 'LCZ_G']\n",
59
+ "\n",
60
+ "full = ['WHT', 'BLK', 'YEL', 'BRN', 'IND', 'LT', 'PD', 'PCI', 'RD', 'GDP'] + lcz\n",
61
+ "\n",
62
+ "# Model variants (Table S6); S2_full is the main model\n",
63
+ "variants = {}\n",
64
+ "variants['S2_full'] = full\n",
65
+ "variants['S0_LCZ_RD'] = lcz + ['RD']\n",
66
+ "variants['S1_plus_PD'] = lcz + ['RD', 'PD']\n",
67
+ "\n",
68
+ "groups = ['General', 'G1', 'G2', 'G3']"
69
+ ],
70
+ "execution_count": null,
71
+ "outputs": []
72
+ },
73
+ {
74
+ "cell_type": "code",
75
+ "metadata": {},
76
+ "source": [
77
+ "# Spatial blocks (5 x 5 km)\n",
78
+ "block_km = 5\n",
79
+ "\n",
80
+ "x_center = (df['left'] + df['right']) / 2\n",
81
+ "y_center = (df['top'] + df['bottom']) / 2\n",
82
+ "\n",
83
+ "block_x = ((x_center - x_center.min()) // (block_km * 1000)).astype(int)\n",
84
+ "block_y = ((y_center - y_center.min()) // (block_km * 1000)).astype(int)\n",
85
+ "\n",
86
+ "blocks = (block_x.astype(str) + '_' + block_y.astype(str)).values\n",
87
+ "\n",
88
+ "print('cells:', len(df), '| blocks:', len(np.unique(blocks)))"
89
+ ],
90
+ "execution_count": null,
91
+ "outputs": []
92
+ },
93
+ {
94
+ "cell_type": "code",
95
+ "metadata": {},
96
+ "source": [
97
+ "def new_model():\n",
98
+ " model = RandomForestRegressor(\n",
99
+ " n_estimators=500,\n",
100
+ " oob_score=False,\n",
101
+ " bootstrap=True,\n",
102
+ " max_depth=20,\n",
103
+ " max_features=None,\n",
104
+ " min_samples_leaf=1,\n",
105
+ " min_samples_split=2,\n",
106
+ " n_jobs=-1,\n",
107
+ " random_state=42\n",
108
+ " )\n",
109
+ " return model\n",
110
+ "\n",
111
+ "def scores(Y, Y_pred):\n",
112
+ " r2 = r2_score(Y, Y_pred)\n",
113
+ " mae = mean_absolute_error(Y, Y_pred)\n",
114
+ " rmse = np.sqrt(mean_squared_error(Y, Y_pred))\n",
115
+ " return r2, mae, rmse\n",
116
+ "\n",
117
+ "\n",
118
+ "def blocked_cv(X, Y):\n",
119
+ " # Out-of-fold predictions\n",
120
+ " Y_pred = np.zeros(len(Y))\n",
121
+ " metrics_df = pd.DataFrame(columns=['R2', 'MAE', 'RMSE'])\n",
122
+ "\n",
123
+ " f = 0\n",
124
+ " for train, test in GroupKFold(n_splits=5).split(X, Y, blocks):\n",
125
+ " model = new_model()\n",
126
+ " model.fit(X.iloc[train], Y.iloc[train])\n",
127
+ " Y_pred[test] = model.predict(X.iloc[test])\n",
128
+ "\n",
129
+ " metrics_df.loc[f] = scores(Y.iloc[test], Y_pred[test])\n",
130
+ " f = f + 1\n",
131
+ "\n",
132
+ " return Y_pred, metrics_df"
133
+ ],
134
+ "execution_count": null,
135
+ "outputs": []
136
+ },
137
+ {
138
+ "cell_type": "code",
139
+ "metadata": {},
140
+ "source": [
141
+ "# Cross-validation of the main model (Table S2)\n",
142
+ "q = input(\"Load which disease group: 'LRI', 'CRD' or 'Other'? \").strip()\n",
143
+ "if q not in ['LRI', 'CRD', 'Other']:\n",
144
+ " raise ValueError(\"Invalid option!\")\n",
145
+ "\n",
146
+ "results = []\n",
147
+ "\n",
148
+ "for g in groups:\n",
149
+ " if g == 'General':\n",
150
+ " col_name = q\n",
151
+ " else:\n",
152
+ " col_name = q + '_' + g\n",
153
+ "\n",
154
+ " X = df[variants['S2_full']]\n",
155
+ " Y = df[col_name]\n",
156
+ "\n",
157
+ " Y_pred, metrics_df = blocked_cv(X, Y)\n",
158
+ " r2, mae, rmse = scores(Y, Y_pred)\n",
159
+ "\n",
160
+ " # pooled scores and range across folds\n",
161
+ " summary = {\n",
162
+ " 'group': g,\n",
163
+ " 'R2': r2,\n",
164
+ " 'R2_min': metrics_df['R2'].min(),\n",
165
+ " 'R2_max': metrics_df['R2'].max(),\n",
166
+ " 'MAE': mae,\n",
167
+ " 'MAE_min': metrics_df['MAE'].min(),\n",
168
+ " 'MAE_max': metrics_df['MAE'].max(),\n",
169
+ " 'RMSE': rmse,\n",
170
+ " 'RMSE_min': metrics_df['RMSE'].min(),\n",
171
+ " 'RMSE_max': metrics_df['RMSE'].max()\n",
172
+ " }\n",
173
+ "\n",
174
+ " results.append(summary)\n",
175
+ " print(g, 'done')\n",
176
+ "\n",
177
+ "summary_metrics = pd.DataFrame(results)\n",
178
+ "summary_metrics = summary_metrics.set_index('group')\n",
179
+ "\n",
180
+ "summary_metrics.round(2)"
181
+ ],
182
+ "execution_count": null,
183
+ "outputs": []
184
+ },
185
+ {
186
+ "cell_type": "code",
187
+ "metadata": {},
188
+ "source": [
189
+ "# R2 of each variant (Table S6)\n",
190
+ "variant_r2 = pd.DataFrame(index=groups, columns=list(variants))\n",
191
+ "\n",
192
+ "for g in groups:\n",
193
+ " if g == 'General':\n",
194
+ " col_name = q\n",
195
+ " else:\n",
196
+ " col_name = q + '_' + g\n",
197
+ "\n",
198
+ " Y = df[col_name]\n",
199
+ "\n",
200
+ " for name in variants:\n",
201
+ " X = df[variants[name]]\n",
202
+ " Y_pred, metrics_df = blocked_cv(X, Y)\n",
203
+ " variant_r2.loc[g, name] = r2_score(Y, Y_pred)\n",
204
+ "\n",
205
+ " print(g, 'done')\n",
206
+ "\n",
207
+ "variant_r2.astype(float).round(2)"
208
+ ],
209
+ "execution_count": null,
210
+ "outputs": []
211
+ },
212
+ {
213
+ "cell_type": "code",
214
+ "metadata": {},
215
+ "source": [
216
+ "# Train the final models on all data and save them (one file per disease group)\n",
217
+ "all_models = {}\n",
218
+ "\n",
219
+ "for g in groups:\n",
220
+ " if g == 'General':\n",
221
+ " col_name = q\n",
222
+ " else:\n",
223
+ " col_name = q + '_' + g\n",
224
+ "\n",
225
+ " X = df[variants['S2_full']]\n",
226
+ " Y = df[col_name]\n",
227
+ "\n",
228
+ " model = new_model()\n",
229
+ " model.fit(X, Y)\n",
230
+ "\n",
231
+ " all_models[q + '_' + g] = model\n",
232
+ " print(g, 'done')\n",
233
+ "\n",
234
+ "output_file = Path(f'RF_spatial_{q}_admissions_per_1_million_all_groups.pkl')\n",
235
+ "joblib.dump(all_models, output_file)"
236
+ ],
237
+ "execution_count": null,
238
+ "outputs": []
239
+ },
240
+ {
241
+ "cell_type": "code",
242
+ "metadata": {},
243
+ "source": [
244
+ "# Hyperparameter grid search (Table S3), LRI G1\n",
245
+ "X = df[variants['S2_full']]\n",
246
+ "Y = df['LRI_G1']\n",
247
+ "\n",
248
+ "X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.2, random_state=42) # Keep results reproducible\n",
249
+ "\n",
250
+ "param_grid = {\n",
251
+ " 'n_estimators': [100, 250, 500, 750],\n",
252
+ " 'max_depth': [10, 20, 30, None],\n",
253
+ " 'min_samples_leaf': [1, 2, 5],\n",
254
+ " 'min_samples_split': [2, 5, 10],\n",
255
+ " 'max_features': ['sqrt', 'log2', 0.3, None],\n",
256
+ " 'bootstrap': [True, False]\n",
257
+ "}\n",
258
+ "\n",
259
+ "rf = RandomForestRegressor(oob_score=False, n_jobs=-1, random_state=42)\n",
260
+ "\n",
261
+ "grid_search = GridSearchCV(\n",
262
+ " estimator=rf,\n",
263
+ " param_grid=param_grid,\n",
264
+ " scoring='r2',\n",
265
+ " cv=5,\n",
266
+ " verbose=1,\n",
267
+ " n_jobs=-1\n",
268
+ ")\n",
269
+ "\n",
270
+ "grid_search.fit(X_train, Y_train)\n",
271
+ "\n",
272
+ "best_model = grid_search.best_estimator_\n",
273
+ "print(grid_search.best_params_)\n",
274
+ "\n",
275
+ "Y_pred = best_model.predict(X_test)\n",
276
+ "r2, mae, rmse = scores(Y_test, Y_pred)\n",
277
+ "\n",
278
+ "print(f'R2: {r2:.4f}')\n",
279
+ "print(f'RMSE: {rmse:.4f}')\n",
280
+ "print(f'MAE: {mae:.4f}')"
281
+ ],
282
+ "execution_count": null,
283
+ "outputs": []
284
+ }
285
+ ],
286
+ "metadata": {
287
+ "kernelspec": {
288
+ "display_name": "Python 3",
289
+ "language": "python",
290
+ "name": "python3"
291
+ },
292
+ "language_info": {
293
+ "name": "python"
294
+ }
295
+ },
296
+ "nbformat": 4,
297
+ "nbformat_minor": 5
298
+ }
RF_temporal_explainability_all_diseases_and_groups.ipynb CHANGED
@@ -1,154 +1,258 @@
1
- {
2
- "cells": [
3
- {
4
- "cell_type": "code",
5
- "execution_count": null,
6
- "metadata": {
7
- "colab": {
8
- "base_uri": "https://localhost:8080/"
9
- },
10
- "executionInfo": {
11
- "elapsed": 71182,
12
- "status": "ok",
13
- "timestamp": 1756577181867,
14
- "user": {
15
- "displayName": "Gabriel Oukawa",
16
- "userId": "10622206185982477384"
17
- },
18
- "user_tz": 180
19
- },
20
- "id": "SzDbN-vemmfy",
21
- "outputId": "593926d4-0a36-4539-9b3f-e297ad04286f"
22
- },
23
- "outputs": [],
24
- "source": [
25
- "# Libraries\n",
26
- "import pandas as pd\n",
27
- "import numpy as np\n",
28
- "import pickle\n",
29
- "import shap\n",
30
- "import matplotlib.pyplot as plt"
31
- ]
32
- },
33
- {
34
- "cell_type": "code",
35
- "execution_count": null,
36
- "metadata": {
37
- "colab": {
38
- "base_uri": "https://localhost:8080/"
39
- },
40
- "executionInfo": {
41
- "elapsed": 2553,
42
- "status": "ok",
43
- "timestamp": 1756577184414,
44
- "user": {
45
- "displayName": "Gabriel Oukawa",
46
- "userId": "10622206185982477384"
47
- },
48
- "user_tz": 180
49
- },
50
- "id": "39d0a9OJqmxZ",
51
- "outputId": "32637b91-7b6e-46cb-ebf5-8252caf03cd0"
52
- },
53
- "outputs": [],
54
- "source": [
55
- "# Load data and models\n",
56
- "q = input('Load which matrix, 'lri', 'crd' or 'other'? ').strip().lower()\n",
57
- "if q not in ['lri', 'crd', 'other']:\n",
58
- " raise ValueError('Invalid input.')\n",
59
- "\n",
60
- "groups = ['general', 'g1', 'g2', 'g3']\n",
61
- "data_path = '.../path/to/input/RF_temporal_data.csv'\n",
62
- "df = pd.read_csv(data_path, delimiter=',', index_col=False)\n",
63
- "\n",
64
- "model_file = f'.../path/to/models/RF_temporal_{q.upper()}_admissions_per_1_million_all_groups.pkl'\n",
65
- "with open(model_file, 'rb') as f:\n",
66
- " all_models = pickle.load(f)\n",
67
- "\n",
68
- "shap_values_list, X_list, titles = [], [], []\n",
69
- "\n",
70
- "for g in groups:\n",
71
- " col_name = f'{q.upper()}' if g == 'general' else f'{q.upper()}_{g.upper()}'\n",
72
- " title = f'{q.upper()} - All Ages' if g == 'general' else f'{q.upper()} - Group {g[-1]}'\n",
73
- " model_key = f'{q.upper()}_General' if g == 'general' else f'{q.upper()}_{g.upper()}'\n",
74
- "\n",
75
- " if col_name not in df.columns or model_key not in all_models:\n",
76
- " continue\n",
77
- "\n",
78
- " model = all_models[model_key]\n",
79
- " X = df.drop(columns=[c for c in df.columns if c.startswith(q.upper())])\n",
80
- " explainer = shap.TreeExplainer(model)\n",
81
- " shap_values = explainer.shap_values(X)\n",
82
- "\n",
83
- " shap_values_list.append(shap_values)\n",
84
- " X_list.append(X)\n",
85
- " titles.append(title)\n",
86
- "\n",
87
- " shap_df = pd.DataFrame(shap_values, columns=X.columns)\n",
88
- " shap_df.to_csv(f\".../path/to/output/SHAP_{model_key}.csv\", index=False)"
89
- ]
90
- },
91
- {
92
- "cell_type": "code",
93
- "execution_count": null,
94
- "metadata": {},
95
- "outputs": [],
96
- "source": [
97
- "# SHAP summary plots\n",
98
- "fig, axes = plt.subplots(2, 2, figsize=(12, 10), dpi=600)\n",
99
- "axes = axes.flatten()\n",
100
- "\n",
101
- "for i in range(len(shap_values_list)):\n",
102
- " plt.sca(axes[i])\n",
103
- " shap.summary_plot(shap_values_list[i], X_list[i], feature_names=X_list[i].columns, plot_type='dot', show=False, max_display=10)\n",
104
- " ax = plt.gca()\n",
105
- " ax.set_title(titles[i], fontsize=12)\n",
106
- " ax.set_xlabel(ax.get_xlabel(), fontsize=12)\n",
107
- " ax.tick_params(axis='both', which='major', labelsize=12)\n",
108
- "\n",
109
- "plt.tight_layout(pad=1.5)\n",
110
- "plt.show()"
111
- ]
112
- },
113
- {
114
- "cell_type": "code",
115
- "execution_count": null,
116
- "metadata": {},
117
- "outputs": [],
118
- "source": [
119
- "# SHAP dependence plots for top 10 features\n",
120
- "mean_abs_shap = np.abs(shap_values.values).mean(axis=0)\n",
121
- "top_features_idx = np.argsort(mean_abs_shap)[-10:][::-1]\n",
122
- "top_features_names = [X.columns[i] for i in top_features_idx]\n",
123
- "\n",
124
- "fig, axes = plt.subplots(4, 2, figsize=(12, 16))\n",
125
- "axes = axes.flatten()\n",
126
- "for i, feature in enumerate(top_features_names):\n",
127
- " shap.plots.scatter(shap_values[:, feature], ax=axes[i], show=False, color=shap_values)\n",
128
- " axes[i].set_title(feature, fontsize=12)\n",
129
- "\n",
130
- "plt.tight_layout()\n",
131
- "plt.show()"
132
- ]
133
- }
134
- ],
135
- "metadata": {
136
- "colab": {
137
- "provenance": [
138
- {
139
- "file_id": "1CTjVKHB6iiDvAQmm4JsyyIgVZShwcp_4",
140
- "timestamp": 1723512979448
141
- }
142
- ]
143
- },
144
- "kernelspec": {
145
- "display_name": "Python 3",
146
- "name": "python3"
147
- },
148
- "language_info": {
149
- "name": "python"
150
- }
151
- },
152
- "nbformat": 4,
153
- "nbformat_minor": 0
154
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "cells": [
3
+ {
4
+ "cell_type": "markdown",
5
+ "metadata": {},
6
+ "source": [
7
+ "# SHAP explainability of the temporal RF models\n",
8
+ "\n",
9
+ "SHAP values of the final models for each age group, importance with the direction of association (Figs. 3 and S5),\n",
10
+ "dependence plots (Figs. 4 and 5) and colored versions (Figs. S11 and S12).\n",
11
+ "\n",
12
+ "Input: `RF_temporal_data.csv` and `RF_temporal_{LRI|CRD|Other}_admissions_per_1_million_all_groups.pkl`."
13
+ ]
14
+ },
15
+ {
16
+ "cell_type": "code",
17
+ "metadata": {},
18
+ "source": [
19
+ "# Libraries\n",
20
+ "import numpy as np\n",
21
+ "import pandas as pd\n",
22
+ "import joblib\n",
23
+ "import shap\n",
24
+ "import matplotlib.pyplot as plt"
25
+ ],
26
+ "execution_count": null,
27
+ "outputs": []
28
+ },
29
+ {
30
+ "cell_type": "code",
31
+ "metadata": {},
32
+ "source": [
33
+ "# Load input data and models\n",
34
+ "q = input(\"Load which disease group: 'LRI', 'CRD' or 'Other'? \").strip()\n",
35
+ "if q not in ['LRI', 'CRD', 'Other']:\n",
36
+ " raise ValueError(\"Invalid option!\")\n",
37
+ "\n",
38
+ "kind = 'temporal'\n",
39
+ "\n",
40
+ "df = pd.read_csv('.../path/to/input/RF_temporal_data.csv', parse_dates=['date'])\n",
41
+ "df = df[(df['date'] >= '2003-01-01') &\n",
42
+ " (df['date'] <= '2017-12-31')].reset_index(drop=True)\n",
43
+ "\n",
44
+ "models = joblib.load(f'.../path/to/models/RF_temporal_{q}_admissions_per_1_million_all_groups.pkl')\n",
45
+ "\n",
46
+ "X = df[list(models[q + '_General'].feature_names_in_)]\n",
47
+ "print(X.shape)"
48
+ ],
49
+ "execution_count": null,
50
+ "outputs": []
51
+ },
52
+ {
53
+ "cell_type": "code",
54
+ "metadata": {},
55
+ "source": [
56
+ "# SHAP values for each age group\n",
57
+ "groups = ['General', 'G1', 'G2', 'G3']\n",
58
+ "shap_values = {}\n",
59
+ "\n",
60
+ "for g in groups:\n",
61
+ " model = models[q + '_' + g]\n",
62
+ " explainer = shap.TreeExplainer(model)\n",
63
+ "\n",
64
+ " shap_values[g] = pd.DataFrame(explainer.shap_values(X), columns=X.columns)\n",
65
+ " shap_values[g].to_csv(f'SHAP_{kind}_{q}_{g}.csv', index=False)\n",
66
+ "\n",
67
+ " print(g, 'done')"
68
+ ],
69
+ "execution_count": null,
70
+ "outputs": []
71
+ },
72
+ {
73
+ "cell_type": "code",
74
+ "metadata": {},
75
+ "source": [
76
+ "# Mean |SHAP| and direction of association (sign of the Pearson correlation between feature and SHAP values)\n",
77
+ "def importance(S, X):\n",
78
+ " out = pd.DataFrame(index=X.columns)\n",
79
+ " out['mean_abs_shap'] = S.abs().mean()\n",
80
+ " out['positive'] = [X[c].corr(S[c]) > 0 for c in X.columns]\n",
81
+ " return out.sort_values('mean_abs_shap', ascending=False)\n",
82
+ "\n",
83
+ "\n",
84
+ "# Top 12 features (Fig. 3) and sum of the others\n",
85
+ "n_top = 12\n",
86
+ "\n",
87
+ "fig, axes = plt.subplots(2, 2, figsize=(11, 9))\n",
88
+ "axes = axes.flatten()\n",
89
+ "\n",
90
+ "for i, g in enumerate(groups):\n",
91
+ " imp = importance(shap_values[g], X)\n",
92
+ " top = imp.iloc[:n_top]\n",
93
+ " rest = imp['mean_abs_shap'].iloc[n_top:].sum()\n",
94
+ "\n",
95
+ " ax = axes[i]\n",
96
+ " for k, (name, row) in enumerate(top.iterrows()):\n",
97
+ " if row['positive']:\n",
98
+ " ax.barh(k, row['mean_abs_shap'], color='#ff0055')\n",
99
+ " sign = '(+)'\n",
100
+ " else:\n",
101
+ " ax.barh(k, row['mean_abs_shap'], color='#0088ff', hatch='////', edgecolor='white')\n",
102
+ " sign = '(-)'\n",
103
+ " ax.text(row['mean_abs_shap'], k, f' {row[\"mean_abs_shap\"]:.2f} {sign}', va='center', fontsize=8)\n",
104
+ "\n",
105
+ " ax.barh(n_top, rest, color='0.6')\n",
106
+ " ax.text(rest, n_top, f' {rest:.2f}', va='center', fontsize=8)\n",
107
+ "\n",
108
+ " ax.set_yticks(range(n_top + 1))\n",
109
+ " ax.set_yticklabels(list(top.index) + ['Other features'])\n",
110
+ " ax.invert_yaxis()\n",
111
+ " ax.set_xlabel('mean |SHAP value|')\n",
112
+ " ax.set_title(q + ' ' + g)\n",
113
+ "\n",
114
+ "plt.tight_layout()\n",
115
+ "plt.show()"
116
+ ],
117
+ "execution_count": null,
118
+ "outputs": []
119
+ },
120
+ {
121
+ "cell_type": "code",
122
+ "metadata": {},
123
+ "source": [
124
+ "# LOWESS trend (tricube weights) with a 95% bootstrap band\n",
125
+ "def lowess(x, y, frac=0.2):\n",
126
+ " grid = np.linspace(np.percentile(x, 1), np.percentile(x, 99), 150)\n",
127
+ " k = max(int(frac * len(x)), 10)\n",
128
+ " fit = np.zeros(len(grid))\n",
129
+ "\n",
130
+ " for i in range(len(grid)):\n",
131
+ " d = np.abs(x - grid[i])\n",
132
+ " h = max(np.sort(d)[k - 1], 1e-9) # window half-width\n",
133
+ " w = np.clip(1 - (d / h) ** 3, 0, None) ** 3\n",
134
+ "\n",
135
+ " xm = np.sum(w * x) / np.sum(w)\n",
136
+ " ym = np.sum(w * y) / np.sum(w)\n",
137
+ " b = np.sum(w * (x - xm) * (y - ym)) / max(np.sum(w * (x - xm) ** 2), 1e-12)\n",
138
+ " fit[i] = ym + b * (grid[i] - xm)\n",
139
+ "\n",
140
+ " return grid, fit\n",
141
+ "\n",
142
+ "\n",
143
+ "def lowess_band(x, y, n_boot=200):\n",
144
+ " rng = np.random.default_rng(0)\n",
145
+ " grid, fit = lowess(x, y)\n",
146
+ "\n",
147
+ " boots = []\n",
148
+ " for i in range(n_boot):\n",
149
+ " idx = rng.integers(0, len(x), len(x))\n",
150
+ " g, f = lowess(x[idx], y[idx])\n",
151
+ " boots.append(np.interp(grid, g, f))\n",
152
+ "\n",
153
+ " lo, hi = np.percentile(boots, [2.5, 97.5], axis=0)\n",
154
+ " return grid, fit, lo, hi\n",
155
+ "\n",
156
+ "\n",
157
+ "def feature_values(c):\n",
158
+ " x = df[c].values.astype(float)\n",
159
+ " if c == 'unix_time':\n",
160
+ " x = 1970 + x / 31557600 # calendar year\n",
161
+ " return x"
162
+ ],
163
+ "execution_count": null,
164
+ "outputs": []
165
+ },
166
+ {
167
+ "cell_type": "code",
168
+ "metadata": {},
169
+ "source": [
170
+ "# Dependence plots of the 6 most important features (Figs. 4 and 5)\n",
171
+ "if q == 'CRD':\n",
172
+ " g = 'G3'\n",
173
+ "else:\n",
174
+ " g = 'G1'\n",
175
+ "\n",
176
+ "S = shap_values[g]\n",
177
+ "features = list(importance(S, X).index[:6])\n",
178
+ "\n",
179
+ "fig, axes = plt.subplots(3, 2, figsize=(10, 10))\n",
180
+ "axes = axes.flatten()\n",
181
+ "\n",
182
+ "for i, c in enumerate(features):\n",
183
+ " x = feature_values(c)\n",
184
+ " y = S[c].values\n",
185
+ " ax = axes[i]\n",
186
+ "\n",
187
+ " ax.axhline(0, color='k', lw=0.6, ls='--')\n",
188
+ "\n",
189
+ " if set(np.unique(x)) <= {0, 1}: # binary feature: mean and 95% CI\n",
190
+ " ax.scatter(x, y, s=7, color='#0088ff', alpha=0.45)\n",
191
+ " for v in [0, 1]:\n",
192
+ " yv = y[x == v]\n",
193
+ " ax.errorbar(v, yv.mean(), yerr=1.96 * yv.std() / np.sqrt(len(yv)), fmt='o', color='#ff0055', capsize=4)\n",
194
+ " ax.set_xticks([0, 1])\n",
195
+ " else:\n",
196
+ " ax.scatter(x, y, s=7, color='#0088ff', alpha=0.45)\n",
197
+ " grid, fit, lo, hi = lowess_band(x, y)\n",
198
+ " ax.fill_between(grid, lo, hi, color='#ff0055', alpha=0.25)\n",
199
+ " ax.plot(grid, fit, color='#ff0055', lw=1.6)\n",
200
+ "\n",
201
+ " # feature distribution\n",
202
+ " y0, y1 = ax.get_ylim()\n",
203
+ " counts, edges = np.histogram(x, bins=40)\n",
204
+ " ax.bar(edges[:-1], 0.15 * (y1 - y0) * counts / counts.max(), width=np.diff(edges), bottom=y0,\n",
205
+ " align='edge', color='0.6', alpha=0.5)\n",
206
+ "\n",
207
+ " ax.set_xlabel(c)\n",
208
+ " ax.set_ylabel('SHAP value')\n",
209
+ " ax.set_title(q + ' ' + g)\n",
210
+ "\n",
211
+ "plt.tight_layout()\n",
212
+ "plt.show()"
213
+ ],
214
+ "execution_count": null,
215
+ "outputs": []
216
+ },
217
+ {
218
+ "cell_type": "code",
219
+ "metadata": {},
220
+ "source": [
221
+ "# Dependence plots colored by the most strongly interacting feature (Figs. S11 and S12)\n",
222
+ "explainer = shap.TreeExplainer(models[q + '_' + g])\n",
223
+ "interactions = explainer.shap_interaction_values(X)\n",
224
+ "mean_interactions = pd.DataFrame(np.abs(interactions).mean(axis=0), index=X.columns, columns=X.columns)\n",
225
+ "\n",
226
+ "fig, axes = plt.subplots(3, 2, figsize=(11, 10))\n",
227
+ "axes = axes.flatten()\n",
228
+ "\n",
229
+ "for i, c in enumerate(features):\n",
230
+ " partner = mean_interactions.loc[c].drop(c).idxmax()\n",
231
+ "\n",
232
+ " sc = axes[i].scatter(feature_values(c), S[c].values, s=7, c=feature_values(partner), cmap='viridis')\n",
233
+ " plt.colorbar(sc, ax=axes[i]).set_label(partner)\n",
234
+ "\n",
235
+ " axes[i].axhline(0, color='k', lw=0.6, ls='--')\n",
236
+ " axes[i].set_xlabel(c)\n",
237
+ " axes[i].set_ylabel('SHAP value')\n",
238
+ "\n",
239
+ "plt.tight_layout()\n",
240
+ "plt.show()"
241
+ ],
242
+ "execution_count": null,
243
+ "outputs": []
244
+ }
245
+ ],
246
+ "metadata": {
247
+ "kernelspec": {
248
+ "display_name": "Python 3",
249
+ "language": "python",
250
+ "name": "python3"
251
+ },
252
+ "language_info": {
253
+ "name": "python"
254
+ }
255
+ },
256
+ "nbformat": 4,
257
+ "nbformat_minor": 5
258
+ }
RF_temporal_training_and_cross-validation_all_diseases_and_groups.ipynb CHANGED
@@ -1,310 +1,347 @@
1
- {
2
- "cells": [
3
- {
4
- "cell_type": "code",
5
- "execution_count": null,
6
- "metadata": {
7
- "colab": {
8
- "base_uri": "https://localhost:8080/"
9
- },
10
- "executionInfo": {
11
- "elapsed": 71182,
12
- "status": "ok",
13
- "timestamp": 1756577181867,
14
- "user": {
15
- "displayName": "Gabriel Oukawa",
16
- "userId": "10622206185982477384"
17
- },
18
- "user_tz": 180
19
- },
20
- "id": "SzDbN-vemmfy",
21
- "outputId": "593926d4-0a36-4539-9b3f-e297ad04286f"
22
- },
23
- "outputs": [],
24
- "source": [
25
- "# Libraries\n",
26
- "import pandas as pd\n",
27
- "import numpy as np\n",
28
- "import joblib\n",
29
- "from pathlib import Path\n",
30
- "from sklearn.ensemble import RandomForestRegressor\n",
31
- "from sklearn.model_selection import train_test_split, GridSearchCV\n",
32
- "from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score"
33
- ]
34
- },
35
- {
36
- "cell_type": "code",
37
- "execution_count": null,
38
- "metadata": {
39
- "colab": {
40
- "base_uri": "https://localhost:8080/"
41
- },
42
- "executionInfo": {
43
- "elapsed": 2553,
44
- "status": "ok",
45
- "timestamp": 1756577184414,
46
- "user": {
47
- "displayName": "Gabriel Oukawa",
48
- "userId": "10622206185982477384"
49
- },
50
- "user_tz": 180
51
- },
52
- "id": "39d0a9OJqmxZ",
53
- "outputId": "32637b91-7b6e-46cb-ebf5-8252caf03cd0"
54
- },
55
- "outputs": [],
56
- "source": [
57
- "# Load input data\n",
58
- "data_path = '.../path/to/input/RF_temporal_data.csv'\n",
59
- "df = pd.read_csv(data_path, delimiter=',', index_col=False)\n",
60
- "\n",
61
- "df = df[(df['date'] >= '2003-01-01') &\n",
62
- " (df['date'] <= '2017-12-31')].reset_index(drop=True)\n",
63
- "\n",
64
- "# print(df.head())\n",
65
- "# print(df.shape)"
66
- ]
67
- },
68
- {
69
- "cell_type": "code",
70
- "execution_count": null,
71
- "metadata": {
72
- "colab": {
73
- "base_uri": "https://localhost:8080/"
74
- },
75
- "executionInfo": {
76
- "elapsed": 8823,
77
- "status": "ok",
78
- "timestamp": 1756494244204,
79
- "user": {
80
- "displayName": "Gabriel Oukawa",
81
- "userId": "10622206185982477384"
82
- },
83
- "user_tz": 180
84
- },
85
- "id": "YnEWfX0s_6QN",
86
- "outputId": "b00bf703-980a-433d-cbf1-950c9ebd978a"
87
- },
88
- "outputs": [],
89
- "source": [
90
- "# Train the RF model, evaluate metrics and save it to a file (single run)\n",
91
- "q = input(\"Load which matrix: 'lri', 'crd' or 'general'?\").strip().lower()\n",
92
- "if q not in ['lri', 'crd', 'general']:\n",
93
- " raise ValueError(\"Invalid option!\")\n",
94
- "\n",
95
- "if q == 'lri':\n",
96
- " name = 'LRI'\n",
97
- "elif q == 'crd':\n",
98
- " name = 'CRD'\n",
99
- "elif q == 'general':\n",
100
- " name = 'Other'\n",
101
- "\n",
102
- "all_models = {}\n",
103
- "\n",
104
- "groups = ['all', 'g1', 'g2', 'g3']\n",
105
- "\n",
106
- "for g in groups:\n",
107
- " if g == 'all':\n",
108
- " col_name = f'{q}'\n",
109
- " model_name = f'{name}_General'\n",
110
- " else:\n",
111
- " col_name = f'{q}_{g}'\n",
112
- " model_name = f'{name}_{g.upper()}'\n",
113
- "\n",
114
- " Y = df[col_name]\n",
115
- " drop_cols = ['date', 'Other', 'Other_G1', 'Other_G2', 'Other_G3',\n",
116
- " 'LRI', 'LRI_G1', 'LRI_G2', 'LRI_G3',\n",
117
- " 'CRD', 'CRD_G1', 'CRD_G2', 'CRD_G3']\n",
118
- " X = df.drop(columns=[c for c in drop_cols if c in df.columns])\n",
119
- "\n",
120
- " X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.2, random_state=42)\n",
121
- "\n",
122
- " model = RandomForestRegressor(\n",
123
- " n_estimators=500,\n",
124
- " oob_score=False,\n",
125
- " bootstrap=True,\n",
126
- " max_depth=20,\n",
127
- " max_features=None,\n",
128
- " min_samples_leaf=1,\n",
129
- " min_samples_split=2,\n",
130
- " n_jobs=-1,\n",
131
- " verbose=1\n",
132
- " )\n",
133
- "\n",
134
- " model.fit(X_train, Y_train)\n",
135
- " Y_pred = model.predict(X_test)\n",
136
- "\n",
137
- " rmse = np.sqrt(mean_squared_error(Y_test, Y_pred))\n",
138
- " mae = mean_absolute_error(Y_test, Y_pred)\n",
139
- " r2 = r2_score(Y_test, Y_pred)\n",
140
- "\n",
141
- " print('RMSE:', rmse)\n",
142
- " print('MAE:', mae)\n",
143
- " print('R2:', r2)\n",
144
- "\n",
145
- " all_models[model_name] = model\n",
146
- "\n",
147
- "output_file = Path(f'RF_temporal_{q.upper()}_all_groups.pkl')\n",
148
- "joblib.dump(all_models, output_file)"
149
- ]
150
- },
151
- {
152
- "cell_type": "code",
153
- "execution_count": null,
154
- "metadata": {
155
- "colab": {
156
- "base_uri": "https://localhost:8080/",
157
- "height": 206
158
- },
159
- "executionInfo": {
160
- "elapsed": 269201,
161
- "status": "ok",
162
- "timestamp": 1756496620139,
163
- "user": {
164
- "displayName": "Gabriel Oukawa",
165
- "userId": "10622206185982477384"
166
- },
167
- "user_tz": 180
168
- },
169
- "id": "deeVkaRIv5II",
170
- "outputId": "a9d76df9-208e-4e67-8cdc-4c70d4b20fcc"
171
- },
172
- "outputs": [],
173
- "source": [
174
- "# Train the RF model and display scoring metrics (cross-validation)\n",
175
- "q = input(\"Load which matrix: 'lri', 'crd' or 'general'?\").strip().lower()\n",
176
- "if q not in ['lri', 'crd', 'general']:\n",
177
- " raise ValueError(\"Invalid option!\")\n",
178
- "\n",
179
- "groups = ['all', 'g1', 'g2', 'g3']\n",
180
- "results = []\n",
181
- "\n",
182
- "for g in groups:\n",
183
- " if g == 'all':\n",
184
- " col_name = f'{q}'\n",
185
- " else:\n",
186
- " col_name = f'{q}_{g}'\n",
187
- " \n",
188
- " Y = df[col_name]\n",
189
- "\n",
190
- " X = df.drop(columns=['date',\n",
191
- " 'Other','Other_G1','Other_G2','Other_G3',\n",
192
- " 'LRI','LRI_G1','LRI_G2','LRI_G3',\n",
193
- " 'CRD','CRD_G1','CRD_G2','CRD_G3',\n",
194
- " ])\n",
195
- "\n",
196
- " metrics_df = pd.DataFrame(columns=['R2', 'MAE', 'RMSE'])\n",
197
- "\n",
198
- " for i in range(100):\n",
199
- " X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.2)\n",
200
- "\n",
201
- " model = RandomForestRegressor(\n",
202
- " n_estimators=500,\n",
203
- " oob_score=False,\n",
204
- " bootstrap=True,\n",
205
- " max_depth=20,\n",
206
- " max_features=None,\n",
207
- " min_samples_leaf=1,\n",
208
- " min_samples_split=2,\n",
209
- " n_jobs=-1,\n",
210
- " verbose=1\n",
211
- " )\n",
212
- "\n",
213
- " model.fit(X_train, Y_train)\n",
214
- " Y_pred = model.predict(X_test)\n",
215
- "\n",
216
- " rmse = np.sqrt(mean_squared_error(Y_test, Y_pred))\n",
217
- " mae = mean_absolute_error(Y_test, Y_pred)\n",
218
- " r2 = r2_score(Y_test, Y_pred)\n",
219
- "\n",
220
- " metrics_df.loc[i] = [r2, mae, rmse]\n",
221
- "\n",
222
- " summary = {\n",
223
- " 'group': g,\n",
224
- " 'R2_mean': metrics_df['R2'].mean(),\n",
225
- " 'R2_min': metrics_df['R2'].min(),\n",
226
- " 'R2_max': metrics_df['R2'].max(),\n",
227
- " 'MAE_mean': metrics_df['MAE'].mean(),\n",
228
- " 'MAE_min': metrics_df['MAE'].min(),\n",
229
- " 'MAE_max': metrics_df['MAE'].max(),\n",
230
- " 'RMSE_mean': metrics_df['RMSE'].mean(),\n",
231
- " 'RMSE_min': metrics_df['RMSE'].min(),\n",
232
- " 'RMSE_max': metrics_df['RMSE'].max()\n",
233
- " }\n",
234
- "\n",
235
- " results.append(summary)\n",
236
- "\n",
237
- "summary_metrics = pd.DataFrame(results)\n",
238
- "summary_metrics = summary_metrics.set_index('group')\n",
239
- "\n",
240
- "summary_metrics.head()"
241
- ]
242
- },
243
- {
244
- "cell_type": "code",
245
- "execution_count": null,
246
- "metadata": {
247
- "id": "FGjVNw-bCUJi"
248
- },
249
- "outputs": [],
250
- "source": [
251
- "# Test the best combination of hyperparameters\n",
252
- "X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.2, random_state=42) # Keep results reproducible\n",
253
- "\n",
254
- "param_grid = {\n",
255
- " 'n_estimators': [100, 250, 500, 750],\n",
256
- " 'max_depth': [10, 20, 30, None],\n",
257
- " 'min_samples_leaf': [1, 2, 5],\n",
258
- " 'min_samples_split': [2, 5, 10],\n",
259
- " 'max_features': ['sqrt', 'log2', 0.3, None],\n",
260
- " 'bootstrap': [True, False]\n",
261
- "}\n",
262
- "\n",
263
- "rf = RandomForestRegressor(oob_score=False, n_jobs=-1, random_state=42)\n",
264
- "\n",
265
- "grid_search = GridSearchCV(\n",
266
- " estimator=rf,\n",
267
- " param_grid=param_grid,\n",
268
- " scoring='r2',\n",
269
- " cv=5,\n",
270
- " verbose=1,\n",
271
- " n_jobs=-1\n",
272
- ")\n",
273
- "\n",
274
- "grid_search.fit(X_train, Y_train)\n",
275
- "\n",
276
- "best_model = grid_search.best_estimator_\n",
277
- "print(grid_search.best_params_)\n",
278
- "\n",
279
- "Y_pred = best_model.predict(X_test)\n",
280
- "\n",
281
- "r2 = r2_score(Y_test, Y_pred)\n",
282
- "rmse = np.sqrt(mean_squared_error(Y_test, Y_pred))\n",
283
- "mae = mean_absolute_error(Y_test, Y_pred)\n",
284
- "\n",
285
- "print(f'R2: {r2:.4f}')\n",
286
- "print(f'RMSE: {rmse:.4f}')\n",
287
- "print(f'MAE: {mae:.4f}')"
288
- ]
289
- }
290
- ],
291
- "metadata": {
292
- "colab": {
293
- "provenance": [
294
- {
295
- "file_id": "1CTjVKHB6iiDvAQmm4JsyyIgVZShwcp_4",
296
- "timestamp": 1723512979448
297
- }
298
- ]
299
- },
300
- "kernelspec": {
301
- "display_name": "Python 3",
302
- "name": "python3"
303
- },
304
- "language_info": {
305
- "name": "python"
306
- }
307
- },
308
- "nbformat": 4,
309
- "nbformat_minor": 0
310
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "cells": [
3
+ {
4
+ "cell_type": "markdown",
5
+ "metadata": {},
6
+ "source": [
7
+ "# Temporal RF models (weekly admissions)\n",
8
+ "\n",
9
+ "Blocked cross-validation (Table S2), buffer test (Table S4), nested models and lags (Tables S5 and S8), final models\n",
10
+ "and grid search (Table S3), for each disease group (LRI, CRD, Other) and age group (General, G1-G3).\n",
11
+ "\n",
12
+ "Input: `RF_temporal_data.csv`, with `date`, the predictors and the admission rates per 1 million inhabitants\n",
13
+ "(`LRI`, `LRI_G1`, ..., `Other_G3`)."
14
+ ]
15
+ },
16
+ {
17
+ "cell_type": "code",
18
+ "metadata": {},
19
+ "source": [
20
+ "# Libraries\n",
21
+ "import pandas as pd\n",
22
+ "import numpy as np\n",
23
+ "import joblib\n",
24
+ "from pathlib import Path\n",
25
+ "from sklearn.ensemble import RandomForestRegressor\n",
26
+ "from sklearn.model_selection import train_test_split, GridSearchCV, GroupKFold\n",
27
+ "from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score"
28
+ ],
29
+ "execution_count": null,
30
+ "outputs": []
31
+ },
32
+ {
33
+ "cell_type": "code",
34
+ "metadata": {},
35
+ "source": [
36
+ "# Load input data (weekly, 2000-2019)\n",
37
+ "data_path = '.../path/to/input/RF_temporal_data.csv'\n",
38
+ "df = pd.read_csv(data_path, delimiter=',', index_col=False, parse_dates=['date'])\n",
39
+ "df = df.sort_values('date').reset_index(drop=True)\n",
40
+ "\n",
41
+ "# Lags (1 and 2 weeks) and moving means (2 and 4 weeks) for Table S8\n",
42
+ "lag_vars = ['t_min', 't_max', 't_mean', 't_amp', 'rh', 'ws',\n",
43
+ " 'so2', 'no2', 'nox', 'co', 'o3', 'pm10']\n",
44
+ "\n",
45
+ "for v in lag_vars:\n",
46
+ " df[v + '_l1'] = df[v].shift(1)\n",
47
+ " df[v + '_l2'] = df[v].shift(2)\n",
48
+ " df[v + '_ma2'] = df[v].rolling(2, min_periods=1).mean()\n",
49
+ " df[v + '_ma4'] = df[v].rolling(4, min_periods=2).mean()\n",
50
+ "\n",
51
+ "# Study period\n",
52
+ "df = df[(df['date'] >= '2003-01-01') &\n",
53
+ " (df['date'] <= '2017-12-31')].reset_index(drop=True)\n",
54
+ "\n",
55
+ "print(df.shape)"
56
+ ],
57
+ "execution_count": null,
58
+ "outputs": []
59
+ },
60
+ {
61
+ "cell_type": "code",
62
+ "metadata": {},
63
+ "source": [
64
+ "# Predictors\n",
65
+ "weather = ['t_min', 't_max', 't_mean', 't_amp', 'rh', 'ws', 'insolation', 'evaporation', 'rainfall']\n",
66
+ "climate = ['cdd', 'cwd', 'chd', 'ccd']\n",
67
+ "trend = ['julian_day', 'unix_time', 'season_fall', 'season_spring', 'season_winter']\n",
68
+ "pollutants = ['so2', 'no2', 'nox', 'co', 'o3', 'pm10']\n",
69
+ "\n",
70
+ "full = weather + climate + trend + pollutants\n",
71
+ "\n",
72
+ "# Model variants (Tables S5 and S8); M2_full is the main model\n",
73
+ "variants = {}\n",
74
+ "variants['M2_full'] = full\n",
75
+ "variants['M0_trend_season'] = trend\n",
76
+ "variants['M1_plus_weather'] = trend + weather + climate\n",
77
+ "variants['E_weather_pollutants'] = weather + climate + pollutants\n",
78
+ "variants['Lag_l1'] = full + [v + '_l1' for v in lag_vars]\n",
79
+ "variants['Lag_l1_l2'] = full + [v + '_l1' for v in lag_vars] + [v + '_l2' for v in lag_vars]\n",
80
+ "variants['Lag_ma2'] = [c for c in full if c not in lag_vars] + [v + '_ma2' for v in lag_vars]\n",
81
+ "variants['Lag_ma4'] = [c for c in full if c not in lag_vars] + [v + '_ma4' for v in lag_vars]\n",
82
+ "\n",
83
+ "groups = ['General', 'G1', 'G2', 'G3']"
84
+ ],
85
+ "execution_count": null,
86
+ "outputs": []
87
+ },
88
+ {
89
+ "cell_type": "code",
90
+ "metadata": {},
91
+ "source": [
92
+ "# Blocked cross-validation folds: one month per season, alternate years, six folds\n",
93
+ "month_position = {12: 2, 1: 0, 2: 1,\n",
94
+ " 3: 0, 4: 1, 5: 2,\n",
95
+ " 6: 0, 7: 1, 8: 2,\n",
96
+ " 9: 0, 10: 1, 11: 2}\n",
97
+ "\n",
98
+ "years = df['date'].dt.year.values\n",
99
+ "months = df['date'].dt.month.values\n",
100
+ "positions = np.array([month_position[m] for m in months])\n",
101
+ "\n",
102
+ "folds = ((positions - years) % 3) * 2 + (years % 2)\n",
103
+ "days = df['date'].values.astype('datetime64[D]').astype(np.int64)\n",
104
+ "\n",
105
+ "print(pd.Series(folds).value_counts().sort_index())"
106
+ ],
107
+ "execution_count": null,
108
+ "outputs": []
109
+ },
110
+ {
111
+ "cell_type": "code",
112
+ "metadata": {},
113
+ "source": [
114
+ "def new_model():\n",
115
+ " model = RandomForestRegressor(\n",
116
+ " n_estimators=500,\n",
117
+ " oob_score=False,\n",
118
+ " bootstrap=True,\n",
119
+ " max_depth=20,\n",
120
+ " max_features=None,\n",
121
+ " min_samples_leaf=1,\n",
122
+ " min_samples_split=2,\n",
123
+ " n_jobs=-1,\n",
124
+ " random_state=42\n",
125
+ " )\n",
126
+ " return model\n",
127
+ "\n",
128
+ "def scores(Y, Y_pred):\n",
129
+ " r2 = r2_score(Y, Y_pred)\n",
130
+ " mae = mean_absolute_error(Y, Y_pred)\n",
131
+ " rmse = np.sqrt(mean_squared_error(Y, Y_pred))\n",
132
+ " return r2, mae, rmse\n",
133
+ "\n",
134
+ "\n",
135
+ "def blocked_cv(X, Y, buffer_weeks=1):\n",
136
+ " # Out-of-fold predictions, without training weeks within the buffer\n",
137
+ " Y_pred = np.zeros(len(Y))\n",
138
+ " metrics_df = pd.DataFrame(columns=['R2', 'MAE', 'RMSE'])\n",
139
+ "\n",
140
+ " for f in range(6):\n",
141
+ " test = folds == f\n",
142
+ " test_days = days[test]\n",
143
+ "\n",
144
+ " # days to the closest test week\n",
145
+ " dist = np.min(np.abs(days[:, None] - test_days[None, :]), axis=1)\n",
146
+ " train = (~test) & (dist > 7 * buffer_weeks)\n",
147
+ "\n",
148
+ " model = new_model()\n",
149
+ " model.fit(X[train], Y[train])\n",
150
+ " Y_pred[test] = model.predict(X[test])\n",
151
+ "\n",
152
+ " metrics_df.loc[f] = scores(Y[test], Y_pred[test])\n",
153
+ "\n",
154
+ " return Y_pred, metrics_df"
155
+ ],
156
+ "execution_count": null,
157
+ "outputs": []
158
+ },
159
+ {
160
+ "cell_type": "code",
161
+ "metadata": {},
162
+ "source": [
163
+ "# Cross-validation of the main model (Table S2)\n",
164
+ "q = input(\"Load which disease group: 'LRI', 'CRD' or 'Other'? \").strip()\n",
165
+ "if q not in ['LRI', 'CRD', 'Other']:\n",
166
+ " raise ValueError(\"Invalid option!\")\n",
167
+ "\n",
168
+ "results = []\n",
169
+ "\n",
170
+ "for g in groups:\n",
171
+ " if g == 'General':\n",
172
+ " col_name = q\n",
173
+ " else:\n",
174
+ " col_name = q + '_' + g\n",
175
+ "\n",
176
+ " X = df[variants['M2_full']]\n",
177
+ " Y = df[col_name]\n",
178
+ "\n",
179
+ " Y_pred, metrics_df = blocked_cv(X, Y)\n",
180
+ " r2, mae, rmse = scores(Y, Y_pred)\n",
181
+ "\n",
182
+ " # pooled scores and range across folds\n",
183
+ " summary = {\n",
184
+ " 'group': g,\n",
185
+ " 'R2': r2,\n",
186
+ " 'R2_min': metrics_df['R2'].min(),\n",
187
+ " 'R2_max': metrics_df['R2'].max(),\n",
188
+ " 'MAE': mae,\n",
189
+ " 'MAE_min': metrics_df['MAE'].min(),\n",
190
+ " 'MAE_max': metrics_df['MAE'].max(),\n",
191
+ " 'RMSE': rmse,\n",
192
+ " 'RMSE_min': metrics_df['RMSE'].min(),\n",
193
+ " 'RMSE_max': metrics_df['RMSE'].max()\n",
194
+ " }\n",
195
+ "\n",
196
+ " results.append(summary)\n",
197
+ " print(g, 'done')\n",
198
+ "\n",
199
+ "summary_metrics = pd.DataFrame(results)\n",
200
+ "summary_metrics = summary_metrics.set_index('group')\n",
201
+ "\n",
202
+ "summary_metrics.round(2)"
203
+ ],
204
+ "execution_count": null,
205
+ "outputs": []
206
+ },
207
+ {
208
+ "cell_type": "code",
209
+ "metadata": {},
210
+ "source": [
211
+ "# Buffer test (Table S4)\n",
212
+ "buffer_r2 = pd.DataFrame(index=groups, columns=[0, 1, 2, 4])\n",
213
+ "\n",
214
+ "for g in groups:\n",
215
+ " if g == 'General':\n",
216
+ " col_name = q\n",
217
+ " else:\n",
218
+ " col_name = q + '_' + g\n",
219
+ "\n",
220
+ " X = df[variants['M2_full']]\n",
221
+ " Y = df[col_name]\n",
222
+ "\n",
223
+ " for b in [0, 1, 2, 4]:\n",
224
+ " Y_pred, metrics_df = blocked_cv(X, Y, buffer_weeks=b)\n",
225
+ " buffer_r2.loc[g, b] = r2_score(Y, Y_pred)\n",
226
+ "\n",
227
+ " print(g, 'done')\n",
228
+ "\n",
229
+ "buffer_r2.astype(float).round(2)"
230
+ ],
231
+ "execution_count": null,
232
+ "outputs": []
233
+ },
234
+ {
235
+ "cell_type": "code",
236
+ "metadata": {},
237
+ "source": [
238
+ "# R2 of each variant (Tables S5 and S8)\n",
239
+ "variant_r2 = pd.DataFrame(index=groups, columns=list(variants))\n",
240
+ "\n",
241
+ "for g in groups:\n",
242
+ " if g == 'General':\n",
243
+ " col_name = q\n",
244
+ " else:\n",
245
+ " col_name = q + '_' + g\n",
246
+ "\n",
247
+ " Y = df[col_name]\n",
248
+ "\n",
249
+ " for name in variants:\n",
250
+ " X = df[variants[name]]\n",
251
+ " Y_pred, metrics_df = blocked_cv(X, Y)\n",
252
+ " variant_r2.loc[g, name] = r2_score(Y, Y_pred)\n",
253
+ "\n",
254
+ " print(g, 'done')\n",
255
+ "\n",
256
+ "variant_r2.astype(float).round(2)"
257
+ ],
258
+ "execution_count": null,
259
+ "outputs": []
260
+ },
261
+ {
262
+ "cell_type": "code",
263
+ "metadata": {},
264
+ "source": [
265
+ "# Train the final models on all data and save them (one file per disease group)\n",
266
+ "all_models = {}\n",
267
+ "\n",
268
+ "for g in groups:\n",
269
+ " if g == 'General':\n",
270
+ " col_name = q\n",
271
+ " else:\n",
272
+ " col_name = q + '_' + g\n",
273
+ "\n",
274
+ " X = df[variants['M2_full']]\n",
275
+ " Y = df[col_name]\n",
276
+ "\n",
277
+ " model = new_model()\n",
278
+ " model.fit(X, Y)\n",
279
+ "\n",
280
+ " all_models[q + '_' + g] = model\n",
281
+ " print(g, 'done')\n",
282
+ "\n",
283
+ "output_file = Path(f'RF_temporal_{q}_admissions_per_1_million_all_groups.pkl')\n",
284
+ "joblib.dump(all_models, output_file)"
285
+ ],
286
+ "execution_count": null,
287
+ "outputs": []
288
+ },
289
+ {
290
+ "cell_type": "code",
291
+ "metadata": {},
292
+ "source": [
293
+ "# Hyperparameter grid search (Table S3), LRI G1\n",
294
+ "X = df[variants['M2_full']]\n",
295
+ "Y = df['LRI_G1']\n",
296
+ "\n",
297
+ "X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.2, random_state=42) # Keep results reproducible\n",
298
+ "\n",
299
+ "param_grid = {\n",
300
+ " 'n_estimators': [100, 250, 500, 750],\n",
301
+ " 'max_depth': [10, 20, 30, None],\n",
302
+ " 'min_samples_leaf': [1, 2, 5],\n",
303
+ " 'min_samples_split': [2, 5, 10],\n",
304
+ " 'max_features': ['sqrt', 'log2', 0.3, None],\n",
305
+ " 'bootstrap': [True, False]\n",
306
+ "}\n",
307
+ "\n",
308
+ "rf = RandomForestRegressor(oob_score=False, n_jobs=-1, random_state=42)\n",
309
+ "\n",
310
+ "grid_search = GridSearchCV(\n",
311
+ " estimator=rf,\n",
312
+ " param_grid=param_grid,\n",
313
+ " scoring='r2',\n",
314
+ " cv=5,\n",
315
+ " verbose=1,\n",
316
+ " n_jobs=-1\n",
317
+ ")\n",
318
+ "\n",
319
+ "grid_search.fit(X_train, Y_train)\n",
320
+ "\n",
321
+ "best_model = grid_search.best_estimator_\n",
322
+ "print(grid_search.best_params_)\n",
323
+ "\n",
324
+ "Y_pred = best_model.predict(X_test)\n",
325
+ "r2, mae, rmse = scores(Y_test, Y_pred)\n",
326
+ "\n",
327
+ "print(f'R2: {r2:.4f}')\n",
328
+ "print(f'RMSE: {rmse:.4f}')\n",
329
+ "print(f'MAE: {mae:.4f}')"
330
+ ],
331
+ "execution_count": null,
332
+ "outputs": []
333
+ }
334
+ ],
335
+ "metadata": {
336
+ "kernelspec": {
337
+ "display_name": "Python 3",
338
+ "language": "python",
339
+ "name": "python3"
340
+ },
341
+ "language_info": {
342
+ "name": "python"
343
+ }
344
+ },
345
+ "nbformat": 4,
346
+ "nbformat_minor": 5
347
+ }