oukawa commited on
Commit
dd30493
·
verified ·
1 Parent(s): 57d6341

Upload codes

Browse files
RF_spatial_explainability_all_diseases_and_groups.ipynb ADDED
@@ -0,0 +1,138 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "cells": [
3
+ {
4
+ "cell_type": "code",
5
+ "execution_count": null,
6
+ "metadata": {
7
+ "colab": {
8
+ "base_uri": "https://localhost:8080/"
9
+ },
10
+ "executionInfo": {
11
+ "elapsed": 113875,
12
+ "status": "ok",
13
+ "timestamp": 1757816776531,
14
+ "user": {
15
+ "displayName": "Gabriel Oukawa",
16
+ "userId": "10622206185982477384"
17
+ },
18
+ "user_tz": 180
19
+ },
20
+ "id": "SzDbN-vemmfy",
21
+ "outputId": "7ef54d4c-7922-45bb-c9c5-2a4a0cad4abe"
22
+ },
23
+ "outputs": [],
24
+ "source": [
25
+ "# Libraries\n",
26
+ "import pandas as pd\n",
27
+ "import numpy as np\n",
28
+ "import pickle\n",
29
+ "import shap\n",
30
+ "import matplotlib.pyplot as plt"
31
+ ]
32
+ },
33
+ {
34
+ "cell_type": "code",
35
+ "execution_count": null,
36
+ "metadata": {},
37
+ "outputs": [],
38
+ "source": [
39
+ "# Load data and models\n",
40
+ "q = input('Load which matrix, 'lri', 'crd' or 'other'? ').strip().lower()\n",
41
+ "if q not in ['lri', 'crd', 'other']:\n",
42
+ " raise ValueError('Invalid input.')\n",
43
+ "\n",
44
+ "groups = ['general', 'g1', 'g2', 'g3']\n",
45
+ "data_path = '.../path/to/input/RF_spatial_data.csv'\n",
46
+ "df = pd.read_csv(data_path, delimiter=',', index_col=False)\n",
47
+ "\n",
48
+ "model_file = f'.../path/to/models/RF_spatial_{q.upper()}_admissions_per_1_million_all_groups.pkl'\n",
49
+ "with open(model_file, 'rb') as f:\n",
50
+ " all_models = pickle.load(f)\n",
51
+ "\n",
52
+ "shap_values_list, X_list, titles = [], [], []\n",
53
+ "\n",
54
+ "for g in groups:\n",
55
+ " col_name = f'{q.upper()}' if g == 'general' else f'{q.upper()}_{g.upper()}'\n",
56
+ " title = f'{q.upper()} - All Ages' if g == 'general' else f'{q.upper()} - Group {g[-1]}'\n",
57
+ " model_key = f'{q.upper()}_General' if g == 'general' else f'{q.upper()}_{g.upper()}'\n",
58
+ "\n",
59
+ " if col_name not in df.columns or model_key not in all_models:\n",
60
+ " continue\n",
61
+ "\n",
62
+ " model = all_models[model_key]\n",
63
+ " X = df.drop(columns=[c for c in df.columns if c.startswith(q.upper())])\n",
64
+ " explainer = shap.TreeExplainer(model)\n",
65
+ " shap_values = explainer.shap_values(X)\n",
66
+ "\n",
67
+ " shap_values_list.append(shap_values)\n",
68
+ " X_list.append(X)\n",
69
+ " titles.append(title)\n",
70
+ "\n",
71
+ " shap_df = pd.DataFrame(shap_values, columns=X.columns)\n",
72
+ " shap_df.to_csv(f\".../path/to/output/SHAP_{model_key}.csv\", index=False)"
73
+ ]
74
+ },
75
+ {
76
+ "cell_type": "code",
77
+ "execution_count": null,
78
+ "metadata": {},
79
+ "outputs": [],
80
+ "source": [
81
+ "# SHAP summary plots\n",
82
+ "fig, axes = plt.subplots(2, 2, figsize=(12, 10), dpi=600)\n",
83
+ "axes = axes.flatten()\n",
84
+ "\n",
85
+ "for i in range(len(shap_values_list)):\n",
86
+ " plt.sca(axes[i])\n",
87
+ " shap.summary_plot(shap_values_list[i], X_list[i], feature_names=X_list[i].columns, plot_type='dot', show=False, max_display=10)\n",
88
+ " ax = plt.gca()\n",
89
+ " ax.set_title(titles[i], fontsize=12)\n",
90
+ " ax.set_xlabel(ax.get_xlabel(), fontsize=12)\n",
91
+ " ax.tick_params(axis='both', which='major', labelsize=12)\n",
92
+ "\n",
93
+ "plt.tight_layout(pad=1.5)\n",
94
+ "plt.show()"
95
+ ]
96
+ },
97
+ {
98
+ "cell_type": "code",
99
+ "execution_count": null,
100
+ "metadata": {},
101
+ "outputs": [],
102
+ "source": [
103
+ "# SHAP dependence plots for top 10 features\n",
104
+ "mean_abs_shap = np.abs(shap_values.values).mean(axis=0)\n",
105
+ "top_features_idx = np.argsort(mean_abs_shap)[-10:][::-1]\n",
106
+ "top_features_names = [X.columns[i] for i in top_features_idx]\n",
107
+ "\n",
108
+ "fig, axes = plt.subplots(4, 2, figsize=(12, 16))\n",
109
+ "axes = axes.flatten()\n",
110
+ "for i, feature in enumerate(top_features_names):\n",
111
+ " shap.plots.scatter(shap_values[:, feature], ax=axes[i], show=False, color=shap_values)\n",
112
+ " axes[i].set_title(feature, fontsize=12)\n",
113
+ "\n",
114
+ "plt.tight_layout()\n",
115
+ "plt.show()"
116
+ ]
117
+ }
118
+ ],
119
+ "metadata": {
120
+ "colab": {
121
+ "provenance": [
122
+ {
123
+ "file_id": "1CTjVKHB6iiDvAQmm4JsyyIgVZShwcp_4",
124
+ "timestamp": 1723512979448
125
+ }
126
+ ]
127
+ },
128
+ "kernelspec": {
129
+ "display_name": "Python 3",
130
+ "name": "python3"
131
+ },
132
+ "language_info": {
133
+ "name": "python"
134
+ }
135
+ },
136
+ "nbformat": 4,
137
+ "nbformat_minor": 0
138
+ }
RF_spatial_training_and_cross-validation_all_diseases_and_groups.ipynb ADDED
@@ -0,0 +1,291 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "cells": [
3
+ {
4
+ "cell_type": "code",
5
+ "execution_count": null,
6
+ "metadata": {
7
+ "colab": {
8
+ "base_uri": "https://localhost:8080/"
9
+ },
10
+ "executionInfo": {
11
+ "elapsed": 113875,
12
+ "status": "ok",
13
+ "timestamp": 1757816776531,
14
+ "user": {
15
+ "displayName": "Gabriel Oukawa",
16
+ "userId": "10622206185982477384"
17
+ },
18
+ "user_tz": 180
19
+ },
20
+ "id": "SzDbN-vemmfy",
21
+ "outputId": "7ef54d4c-7922-45bb-c9c5-2a4a0cad4abe"
22
+ },
23
+ "outputs": [],
24
+ "source": [
25
+ "# Libraries\n",
26
+ "import pandas as pd\n",
27
+ "import numpy as np\n",
28
+ "import joblib\n",
29
+ "from pathlib import Path\n",
30
+ "from sklearn.ensemble import RandomForestRegressor\n",
31
+ "from sklearn.model_selection import train_test_split, GridSearchCV\n",
32
+ "from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score"
33
+ ]
34
+ },
35
+ {
36
+ "cell_type": "code",
37
+ "execution_count": null,
38
+ "metadata": {
39
+ "colab": {
40
+ "base_uri": "https://localhost:8080/"
41
+ },
42
+ "executionInfo": {
43
+ "elapsed": 184,
44
+ "status": "ok",
45
+ "timestamp": 1757816855352,
46
+ "user": {
47
+ "displayName": "Gabriel Oukawa",
48
+ "userId": "10622206185982477384"
49
+ },
50
+ "user_tz": 180
51
+ },
52
+ "id": "39d0a9OJqmxZ",
53
+ "outputId": "c100c3b4-66ef-4620-fbd2-25872d749deb"
54
+ },
55
+ "outputs": [],
56
+ "source": [
57
+ "# Load input data\n",
58
+ "data_path = '.../path/to/input/RF_spatial_data.csv'\n",
59
+ "df = pd.read_csv(data_path, delimiter=',', index_col=False)\n",
60
+ "\n",
61
+ "df = df[(df['date'] >= '2003-01-01') &\n",
62
+ " (df['date'] <= '2017-12-31')].reset_index(drop=True)\n",
63
+ "\n",
64
+ "# print(df.head())\n",
65
+ "# print(df.shape)"
66
+ ]
67
+ },
68
+ {
69
+ "cell_type": "code",
70
+ "execution_count": null,
71
+ "metadata": {
72
+ "colab": {
73
+ "base_uri": "https://localhost:8080/"
74
+ },
75
+ "executionInfo": {
76
+ "elapsed": 6803,
77
+ "status": "ok",
78
+ "timestamp": 1757816916603,
79
+ "user": {
80
+ "displayName": "Gabriel Oukawa",
81
+ "userId": "10622206185982477384"
82
+ },
83
+ "user_tz": 180
84
+ },
85
+ "id": "YnEWfX0s_6QN",
86
+ "outputId": "3a892047-134b-4816-886f-7bedc7d0c9b1"
87
+ },
88
+ "outputs": [],
89
+ "source": [
90
+ "# Train the RF model, evaluate metrics and save it to a file (single run)\n",
91
+ "q = input(\"Load which matrix: 'lri', 'crd' or 'general'?\").strip().lower()\n",
92
+ "if q not in ['lri', 'crd', 'general']:\n",
93
+ " raise ValueError(\"Invalid option!\")\n",
94
+ "\n",
95
+ "if q == 'lri':\n",
96
+ " name = 'LRI'\n",
97
+ "elif q == 'crd':\n",
98
+ " name = 'CRD'\n",
99
+ "elif q == 'general':\n",
100
+ " name = 'Other'\n",
101
+ "\n",
102
+ "all_models = {}\n",
103
+ "\n",
104
+ "groups = ['all', 'g1', 'g2', 'g3']\n",
105
+ "\n",
106
+ "for g in groups:\n",
107
+ " if g == 'all':\n",
108
+ " col_name = f'{q}'\n",
109
+ " model_name = f'{name}_General'\n",
110
+ " else:\n",
111
+ " col_name = f'{q}_{g}'\n",
112
+ " model_name = f'{name}_{g.upper()}'\n",
113
+ "\n",
114
+ " Y = df[col_name]\n",
115
+ " drop_cols = ['date', 'Other', 'Other_G1', 'Other_G2', 'Other_G3',\n",
116
+ " 'LRI', 'LRI_G1', 'LRI_G2', 'LRI_G3',\n",
117
+ " 'CRD', 'CRD_G1', 'CRD_G2', 'CRD_G3']\n",
118
+ " X = df.drop(columns=[c for c in drop_cols if c in df.columns])\n",
119
+ "\n",
120
+ " X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.2, random_state=42)\n",
121
+ "\n",
122
+ " model = RandomForestRegressor(\n",
123
+ " n_estimators=500,\n",
124
+ " oob_score=False,\n",
125
+ " bootstrap=True,\n",
126
+ " max_depth=20,\n",
127
+ " max_features=None,\n",
128
+ " min_samples_leaf=1,\n",
129
+ " min_samples_split=2,\n",
130
+ " n_jobs=-1,\n",
131
+ " verbose=1\n",
132
+ " )\n",
133
+ "\n",
134
+ " model.fit(X_train, Y_train)\n",
135
+ " Y_pred = model.predict(X_test)\n",
136
+ "\n",
137
+ " rmse = np.sqrt(mean_squared_error(Y_test, Y_pred))\n",
138
+ " mae = mean_absolute_error(Y_test, Y_pred)\n",
139
+ " r2 = r2_score(Y_test, Y_pred)\n",
140
+ "\n",
141
+ " print('RMSE:', rmse)\n",
142
+ " print('MAE:', mae)\n",
143
+ " print('R2:', r2)\n",
144
+ "\n",
145
+ " all_models[model_name] = model\n",
146
+ "\n",
147
+ "output_file = Path(f'RF_spatial_{q.upper()}_all_groups.pkl')\n",
148
+ "joblib.dump(all_models, output_file)"
149
+ ]
150
+ },
151
+ {
152
+ "cell_type": "code",
153
+ "execution_count": null,
154
+ "metadata": {},
155
+ "outputs": [],
156
+ "source": [
157
+ "# Train the RF model and display scoring metrics (cross-validation)\n",
158
+ "q = input(\"Load which matrix: 'lri', 'crd' or 'general'?\").strip().lower()\n",
159
+ "if q not in ['lri', 'crd', 'general']:\n",
160
+ " raise ValueError(\"Invalid option!\")\n",
161
+ "\n",
162
+ "groups = ['all', 'g1', 'g2', 'g3']\n",
163
+ "results = []\n",
164
+ "\n",
165
+ "for g in groups:\n",
166
+ " if g == 'all':\n",
167
+ " col_name = f'{q}'\n",
168
+ " else:\n",
169
+ " col_name = f'{q}_{g}'\n",
170
+ " \n",
171
+ " Y = df[col_name]\n",
172
+ "\n",
173
+ " X = df.drop(columns=['date',\n",
174
+ " 'Other','Other_G1','Other_G2','Other_G3',\n",
175
+ " 'LRI','LRI_G1','LRI_G2','LRI_G3',\n",
176
+ " 'CRD','CRD_G1','CRD_G2','CRD_G3',\n",
177
+ " ])\n",
178
+ "\n",
179
+ " metrics_df = pd.DataFrame(columns=['R2', 'MAE', 'RMSE'])\n",
180
+ "\n",
181
+ " for i in range(100):\n",
182
+ " X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.2)\n",
183
+ "\n",
184
+ " model = RandomForestRegressor(\n",
185
+ " n_estimators=500,\n",
186
+ " oob_score=False,\n",
187
+ " bootstrap=True,\n",
188
+ " max_depth=20,\n",
189
+ " max_features=None,\n",
190
+ " min_samples_leaf=1,\n",
191
+ " min_samples_split=2,\n",
192
+ " n_jobs=-1,\n",
193
+ " verbose=1\n",
194
+ " )\n",
195
+ "\n",
196
+ " model.fit(X_train, Y_train)\n",
197
+ " Y_pred = model.predict(X_test)\n",
198
+ "\n",
199
+ " rmse = np.sqrt(mean_squared_error(Y_test, Y_pred))\n",
200
+ " mae = mean_absolute_error(Y_test, Y_pred)\n",
201
+ " r2 = r2_score(Y_test, Y_pred)\n",
202
+ "\n",
203
+ " metrics_df.loc[i] = [r2, mae, rmse]\n",
204
+ "\n",
205
+ " summary = {\n",
206
+ " 'group': g,\n",
207
+ " 'R2_mean': metrics_df['R2'].mean(),\n",
208
+ " 'R2_min': metrics_df['R2'].min(),\n",
209
+ " 'R2_max': metrics_df['R2'].max(),\n",
210
+ " 'MAE_mean': metrics_df['MAE'].mean(),\n",
211
+ " 'MAE_min': metrics_df['MAE'].min(),\n",
212
+ " 'MAE_max': metrics_df['MAE'].max(),\n",
213
+ " 'RMSE_mean': metrics_df['RMSE'].mean(),\n",
214
+ " 'RMSE_min': metrics_df['RMSE'].min(),\n",
215
+ " 'RMSE_max': metrics_df['RMSE'].max()\n",
216
+ " }\n",
217
+ "\n",
218
+ " results.append(summary)\n",
219
+ "\n",
220
+ "summary_metrics = pd.DataFrame(results)\n",
221
+ "summary_metrics = summary_metrics.set_index('group')\n",
222
+ "\n",
223
+ "summary_metrics.head()"
224
+ ]
225
+ },
226
+ {
227
+ "cell_type": "code",
228
+ "execution_count": null,
229
+ "metadata": {},
230
+ "outputs": [],
231
+ "source": [
232
+ "# Test the best combination of hyperparameters\n",
233
+ "X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.2, random_state=42) # Keep results reproducible\n",
234
+ "\n",
235
+ "param_grid = {\n",
236
+ " 'n_estimators': [100, 250, 500, 750],\n",
237
+ " 'max_depth': [10, 20, 30, None],\n",
238
+ " 'min_samples_leaf': [1, 2, 5],\n",
239
+ " 'min_samples_split': [2, 5, 10],\n",
240
+ " 'max_features': ['sqrt', 'log2', 0.3, None],\n",
241
+ " 'bootstrap': [True, False]\n",
242
+ "}\n",
243
+ "\n",
244
+ "rf = RandomForestRegressor(oob_score=False, n_jobs=-1, random_state=42)\n",
245
+ "\n",
246
+ "grid_search = GridSearchCV(\n",
247
+ " estimator=rf,\n",
248
+ " param_grid=param_grid,\n",
249
+ " scoring='r2',\n",
250
+ " cv=5,\n",
251
+ " verbose=1,\n",
252
+ " n_jobs=-1\n",
253
+ ")\n",
254
+ "\n",
255
+ "grid_search.fit(X_train, Y_train)\n",
256
+ "\n",
257
+ "best_model = grid_search.best_estimator_\n",
258
+ "print(grid_search.best_params_)\n",
259
+ "\n",
260
+ "Y_pred = best_model.predict(X_test)\n",
261
+ "\n",
262
+ "r2 = r2_score(Y_test, Y_pred)\n",
263
+ "rmse = np.sqrt(mean_squared_error(Y_test, Y_pred))\n",
264
+ "mae = mean_absolute_error(Y_test, Y_pred)\n",
265
+ "\n",
266
+ "print(f'R2: {r2:.4f}')\n",
267
+ "print(f'RMSE: {rmse:.4f}')\n",
268
+ "print(f'MAE: {mae:.4f}')"
269
+ ]
270
+ }
271
+ ],
272
+ "metadata": {
273
+ "colab": {
274
+ "provenance": [
275
+ {
276
+ "file_id": "1CTjVKHB6iiDvAQmm4JsyyIgVZShwcp_4",
277
+ "timestamp": 1723512979448
278
+ }
279
+ ]
280
+ },
281
+ "kernelspec": {
282
+ "display_name": "Python 3",
283
+ "name": "python3"
284
+ },
285
+ "language_info": {
286
+ "name": "python"
287
+ }
288
+ },
289
+ "nbformat": 4,
290
+ "nbformat_minor": 0
291
+ }
RF_temporal_explainability_all_diseases_and_groups.ipynb ADDED
@@ -0,0 +1,154 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "cells": [
3
+ {
4
+ "cell_type": "code",
5
+ "execution_count": null,
6
+ "metadata": {
7
+ "colab": {
8
+ "base_uri": "https://localhost:8080/"
9
+ },
10
+ "executionInfo": {
11
+ "elapsed": 71182,
12
+ "status": "ok",
13
+ "timestamp": 1756577181867,
14
+ "user": {
15
+ "displayName": "Gabriel Oukawa",
16
+ "userId": "10622206185982477384"
17
+ },
18
+ "user_tz": 180
19
+ },
20
+ "id": "SzDbN-vemmfy",
21
+ "outputId": "593926d4-0a36-4539-9b3f-e297ad04286f"
22
+ },
23
+ "outputs": [],
24
+ "source": [
25
+ "# Libraries\n",
26
+ "import pandas as pd\n",
27
+ "import numpy as np\n",
28
+ "import pickle\n",
29
+ "import shap\n",
30
+ "import matplotlib.pyplot as plt"
31
+ ]
32
+ },
33
+ {
34
+ "cell_type": "code",
35
+ "execution_count": null,
36
+ "metadata": {
37
+ "colab": {
38
+ "base_uri": "https://localhost:8080/"
39
+ },
40
+ "executionInfo": {
41
+ "elapsed": 2553,
42
+ "status": "ok",
43
+ "timestamp": 1756577184414,
44
+ "user": {
45
+ "displayName": "Gabriel Oukawa",
46
+ "userId": "10622206185982477384"
47
+ },
48
+ "user_tz": 180
49
+ },
50
+ "id": "39d0a9OJqmxZ",
51
+ "outputId": "32637b91-7b6e-46cb-ebf5-8252caf03cd0"
52
+ },
53
+ "outputs": [],
54
+ "source": [
55
+ "# Load data and models\n",
56
+ "q = input('Load which matrix, 'lri', 'crd' or 'other'? ').strip().lower()\n",
57
+ "if q not in ['lri', 'crd', 'other']:\n",
58
+ " raise ValueError('Invalid input.')\n",
59
+ "\n",
60
+ "groups = ['general', 'g1', 'g2', 'g3']\n",
61
+ "data_path = '.../path/to/input/RF_temporal_data.csv'\n",
62
+ "df = pd.read_csv(data_path, delimiter=',', index_col=False)\n",
63
+ "\n",
64
+ "model_file = f'.../path/to/models/RF_temporal_{q.upper()}_admissions_per_1_million_all_groups.pkl'\n",
65
+ "with open(model_file, 'rb') as f:\n",
66
+ " all_models = pickle.load(f)\n",
67
+ "\n",
68
+ "shap_values_list, X_list, titles = [], [], []\n",
69
+ "\n",
70
+ "for g in groups:\n",
71
+ " col_name = f'{q.upper()}' if g == 'general' else f'{q.upper()}_{g.upper()}'\n",
72
+ " title = f'{q.upper()} - All Ages' if g == 'general' else f'{q.upper()} - Group {g[-1]}'\n",
73
+ " model_key = f'{q.upper()}_General' if g == 'general' else f'{q.upper()}_{g.upper()}'\n",
74
+ "\n",
75
+ " if col_name not in df.columns or model_key not in all_models:\n",
76
+ " continue\n",
77
+ "\n",
78
+ " model = all_models[model_key]\n",
79
+ " X = df.drop(columns=[c for c in df.columns if c.startswith(q.upper())])\n",
80
+ " explainer = shap.TreeExplainer(model)\n",
81
+ " shap_values = explainer.shap_values(X)\n",
82
+ "\n",
83
+ " shap_values_list.append(shap_values)\n",
84
+ " X_list.append(X)\n",
85
+ " titles.append(title)\n",
86
+ "\n",
87
+ " shap_df = pd.DataFrame(shap_values, columns=X.columns)\n",
88
+ " shap_df.to_csv(f\".../path/to/output/SHAP_{model_key}.csv\", index=False)"
89
+ ]
90
+ },
91
+ {
92
+ "cell_type": "code",
93
+ "execution_count": null,
94
+ "metadata": {},
95
+ "outputs": [],
96
+ "source": [
97
+ "# SHAP summary plots\n",
98
+ "fig, axes = plt.subplots(2, 2, figsize=(12, 10), dpi=600)\n",
99
+ "axes = axes.flatten()\n",
100
+ "\n",
101
+ "for i in range(len(shap_values_list)):\n",
102
+ " plt.sca(axes[i])\n",
103
+ " shap.summary_plot(shap_values_list[i], X_list[i], feature_names=X_list[i].columns, plot_type='dot', show=False, max_display=10)\n",
104
+ " ax = plt.gca()\n",
105
+ " ax.set_title(titles[i], fontsize=12)\n",
106
+ " ax.set_xlabel(ax.get_xlabel(), fontsize=12)\n",
107
+ " ax.tick_params(axis='both', which='major', labelsize=12)\n",
108
+ "\n",
109
+ "plt.tight_layout(pad=1.5)\n",
110
+ "plt.show()"
111
+ ]
112
+ },
113
+ {
114
+ "cell_type": "code",
115
+ "execution_count": null,
116
+ "metadata": {},
117
+ "outputs": [],
118
+ "source": [
119
+ "# SHAP dependence plots for top 10 features\n",
120
+ "mean_abs_shap = np.abs(shap_values.values).mean(axis=0)\n",
121
+ "top_features_idx = np.argsort(mean_abs_shap)[-10:][::-1]\n",
122
+ "top_features_names = [X.columns[i] for i in top_features_idx]\n",
123
+ "\n",
124
+ "fig, axes = plt.subplots(4, 2, figsize=(12, 16))\n",
125
+ "axes = axes.flatten()\n",
126
+ "for i, feature in enumerate(top_features_names):\n",
127
+ " shap.plots.scatter(shap_values[:, feature], ax=axes[i], show=False, color=shap_values)\n",
128
+ " axes[i].set_title(feature, fontsize=12)\n",
129
+ "\n",
130
+ "plt.tight_layout()\n",
131
+ "plt.show()"
132
+ ]
133
+ }
134
+ ],
135
+ "metadata": {
136
+ "colab": {
137
+ "provenance": [
138
+ {
139
+ "file_id": "1CTjVKHB6iiDvAQmm4JsyyIgVZShwcp_4",
140
+ "timestamp": 1723512979448
141
+ }
142
+ ]
143
+ },
144
+ "kernelspec": {
145
+ "display_name": "Python 3",
146
+ "name": "python3"
147
+ },
148
+ "language_info": {
149
+ "name": "python"
150
+ }
151
+ },
152
+ "nbformat": 4,
153
+ "nbformat_minor": 0
154
+ }
RF_temporal_training_and_cross-validation_all_diseases_and_groups.ipynb ADDED
@@ -0,0 +1,310 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "cells": [
3
+ {
4
+ "cell_type": "code",
5
+ "execution_count": null,
6
+ "metadata": {
7
+ "colab": {
8
+ "base_uri": "https://localhost:8080/"
9
+ },
10
+ "executionInfo": {
11
+ "elapsed": 71182,
12
+ "status": "ok",
13
+ "timestamp": 1756577181867,
14
+ "user": {
15
+ "displayName": "Gabriel Oukawa",
16
+ "userId": "10622206185982477384"
17
+ },
18
+ "user_tz": 180
19
+ },
20
+ "id": "SzDbN-vemmfy",
21
+ "outputId": "593926d4-0a36-4539-9b3f-e297ad04286f"
22
+ },
23
+ "outputs": [],
24
+ "source": [
25
+ "# Libraries\n",
26
+ "import pandas as pd\n",
27
+ "import numpy as np\n",
28
+ "import joblib\n",
29
+ "from pathlib import Path\n",
30
+ "from sklearn.ensemble import RandomForestRegressor\n",
31
+ "from sklearn.model_selection import train_test_split, GridSearchCV\n",
32
+ "from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score"
33
+ ]
34
+ },
35
+ {
36
+ "cell_type": "code",
37
+ "execution_count": null,
38
+ "metadata": {
39
+ "colab": {
40
+ "base_uri": "https://localhost:8080/"
41
+ },
42
+ "executionInfo": {
43
+ "elapsed": 2553,
44
+ "status": "ok",
45
+ "timestamp": 1756577184414,
46
+ "user": {
47
+ "displayName": "Gabriel Oukawa",
48
+ "userId": "10622206185982477384"
49
+ },
50
+ "user_tz": 180
51
+ },
52
+ "id": "39d0a9OJqmxZ",
53
+ "outputId": "32637b91-7b6e-46cb-ebf5-8252caf03cd0"
54
+ },
55
+ "outputs": [],
56
+ "source": [
57
+ "# Load input data\n",
58
+ "data_path = '.../path/to/input/RF_temporal_data.csv'\n",
59
+ "df = pd.read_csv(data_path, delimiter=',', index_col=False)\n",
60
+ "\n",
61
+ "df = df[(df['date'] >= '2003-01-01') &\n",
62
+ " (df['date'] <= '2017-12-31')].reset_index(drop=True)\n",
63
+ "\n",
64
+ "# print(df.head())\n",
65
+ "# print(df.shape)"
66
+ ]
67
+ },
68
+ {
69
+ "cell_type": "code",
70
+ "execution_count": null,
71
+ "metadata": {
72
+ "colab": {
73
+ "base_uri": "https://localhost:8080/"
74
+ },
75
+ "executionInfo": {
76
+ "elapsed": 8823,
77
+ "status": "ok",
78
+ "timestamp": 1756494244204,
79
+ "user": {
80
+ "displayName": "Gabriel Oukawa",
81
+ "userId": "10622206185982477384"
82
+ },
83
+ "user_tz": 180
84
+ },
85
+ "id": "YnEWfX0s_6QN",
86
+ "outputId": "b00bf703-980a-433d-cbf1-950c9ebd978a"
87
+ },
88
+ "outputs": [],
89
+ "source": [
90
+ "# Train the RF model, evaluate metrics and save it to a file (single run)\n",
91
+ "q = input(\"Load which matrix: 'lri', 'crd' or 'general'?\").strip().lower()\n",
92
+ "if q not in ['lri', 'crd', 'general']:\n",
93
+ " raise ValueError(\"Invalid option!\")\n",
94
+ "\n",
95
+ "if q == 'lri':\n",
96
+ " name = 'LRI'\n",
97
+ "elif q == 'crd':\n",
98
+ " name = 'CRD'\n",
99
+ "elif q == 'general':\n",
100
+ " name = 'Other'\n",
101
+ "\n",
102
+ "all_models = {}\n",
103
+ "\n",
104
+ "groups = ['all', 'g1', 'g2', 'g3']\n",
105
+ "\n",
106
+ "for g in groups:\n",
107
+ " if g == 'all':\n",
108
+ " col_name = f'{q}'\n",
109
+ " model_name = f'{name}_General'\n",
110
+ " else:\n",
111
+ " col_name = f'{q}_{g}'\n",
112
+ " model_name = f'{name}_{g.upper()}'\n",
113
+ "\n",
114
+ " Y = df[col_name]\n",
115
+ " drop_cols = ['date', 'Other', 'Other_G1', 'Other_G2', 'Other_G3',\n",
116
+ " 'LRI', 'LRI_G1', 'LRI_G2', 'LRI_G3',\n",
117
+ " 'CRD', 'CRD_G1', 'CRD_G2', 'CRD_G3']\n",
118
+ " X = df.drop(columns=[c for c in drop_cols if c in df.columns])\n",
119
+ "\n",
120
+ " X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.2, random_state=42)\n",
121
+ "\n",
122
+ " model = RandomForestRegressor(\n",
123
+ " n_estimators=500,\n",
124
+ " oob_score=False,\n",
125
+ " bootstrap=True,\n",
126
+ " max_depth=20,\n",
127
+ " max_features=None,\n",
128
+ " min_samples_leaf=1,\n",
129
+ " min_samples_split=2,\n",
130
+ " n_jobs=-1,\n",
131
+ " verbose=1\n",
132
+ " )\n",
133
+ "\n",
134
+ " model.fit(X_train, Y_train)\n",
135
+ " Y_pred = model.predict(X_test)\n",
136
+ "\n",
137
+ " rmse = np.sqrt(mean_squared_error(Y_test, Y_pred))\n",
138
+ " mae = mean_absolute_error(Y_test, Y_pred)\n",
139
+ " r2 = r2_score(Y_test, Y_pred)\n",
140
+ "\n",
141
+ " print('RMSE:', rmse)\n",
142
+ " print('MAE:', mae)\n",
143
+ " print('R2:', r2)\n",
144
+ "\n",
145
+ " all_models[model_name] = model\n",
146
+ "\n",
147
+ "output_file = Path(f'RF_temporal_{q.upper()}_all_groups.pkl')\n",
148
+ "joblib.dump(all_models, output_file)"
149
+ ]
150
+ },
151
+ {
152
+ "cell_type": "code",
153
+ "execution_count": null,
154
+ "metadata": {
155
+ "colab": {
156
+ "base_uri": "https://localhost:8080/",
157
+ "height": 206
158
+ },
159
+ "executionInfo": {
160
+ "elapsed": 269201,
161
+ "status": "ok",
162
+ "timestamp": 1756496620139,
163
+ "user": {
164
+ "displayName": "Gabriel Oukawa",
165
+ "userId": "10622206185982477384"
166
+ },
167
+ "user_tz": 180
168
+ },
169
+ "id": "deeVkaRIv5II",
170
+ "outputId": "a9d76df9-208e-4e67-8cdc-4c70d4b20fcc"
171
+ },
172
+ "outputs": [],
173
+ "source": [
174
+ "# Train the RF model and display scoring metrics (cross-validation)\n",
175
+ "q = input(\"Load which matrix: 'lri', 'crd' or 'general'?\").strip().lower()\n",
176
+ "if q not in ['lri', 'crd', 'general']:\n",
177
+ " raise ValueError(\"Invalid option!\")\n",
178
+ "\n",
179
+ "groups = ['all', 'g1', 'g2', 'g3']\n",
180
+ "results = []\n",
181
+ "\n",
182
+ "for g in groups:\n",
183
+ " if g == 'all':\n",
184
+ " col_name = f'{q}'\n",
185
+ " else:\n",
186
+ " col_name = f'{q}_{g}'\n",
187
+ " \n",
188
+ " Y = df[col_name]\n",
189
+ "\n",
190
+ " X = df.drop(columns=['date',\n",
191
+ " 'Other','Other_G1','Other_G2','Other_G3',\n",
192
+ " 'LRI','LRI_G1','LRI_G2','LRI_G3',\n",
193
+ " 'CRD','CRD_G1','CRD_G2','CRD_G3',\n",
194
+ " ])\n",
195
+ "\n",
196
+ " metrics_df = pd.DataFrame(columns=['R2', 'MAE', 'RMSE'])\n",
197
+ "\n",
198
+ " for i in range(100):\n",
199
+ " X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.2)\n",
200
+ "\n",
201
+ " model = RandomForestRegressor(\n",
202
+ " n_estimators=500,\n",
203
+ " oob_score=False,\n",
204
+ " bootstrap=True,\n",
205
+ " max_depth=20,\n",
206
+ " max_features=None,\n",
207
+ " min_samples_leaf=1,\n",
208
+ " min_samples_split=2,\n",
209
+ " n_jobs=-1,\n",
210
+ " verbose=1\n",
211
+ " )\n",
212
+ "\n",
213
+ " model.fit(X_train, Y_train)\n",
214
+ " Y_pred = model.predict(X_test)\n",
215
+ "\n",
216
+ " rmse = np.sqrt(mean_squared_error(Y_test, Y_pred))\n",
217
+ " mae = mean_absolute_error(Y_test, Y_pred)\n",
218
+ " r2 = r2_score(Y_test, Y_pred)\n",
219
+ "\n",
220
+ " metrics_df.loc[i] = [r2, mae, rmse]\n",
221
+ "\n",
222
+ " summary = {\n",
223
+ " 'group': g,\n",
224
+ " 'R2_mean': metrics_df['R2'].mean(),\n",
225
+ " 'R2_min': metrics_df['R2'].min(),\n",
226
+ " 'R2_max': metrics_df['R2'].max(),\n",
227
+ " 'MAE_mean': metrics_df['MAE'].mean(),\n",
228
+ " 'MAE_min': metrics_df['MAE'].min(),\n",
229
+ " 'MAE_max': metrics_df['MAE'].max(),\n",
230
+ " 'RMSE_mean': metrics_df['RMSE'].mean(),\n",
231
+ " 'RMSE_min': metrics_df['RMSE'].min(),\n",
232
+ " 'RMSE_max': metrics_df['RMSE'].max()\n",
233
+ " }\n",
234
+ "\n",
235
+ " results.append(summary)\n",
236
+ "\n",
237
+ "summary_metrics = pd.DataFrame(results)\n",
238
+ "summary_metrics = summary_metrics.set_index('group')\n",
239
+ "\n",
240
+ "summary_metrics.head()"
241
+ ]
242
+ },
243
+ {
244
+ "cell_type": "code",
245
+ "execution_count": null,
246
+ "metadata": {
247
+ "id": "FGjVNw-bCUJi"
248
+ },
249
+ "outputs": [],
250
+ "source": [
251
+ "# Test the best combination of hyperparameters\n",
252
+ "X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.2, random_state=42) # Keep results reproducible\n",
253
+ "\n",
254
+ "param_grid = {\n",
255
+ " 'n_estimators': [100, 250, 500, 750],\n",
256
+ " 'max_depth': [10, 20, 30, None],\n",
257
+ " 'min_samples_leaf': [1, 2, 5],\n",
258
+ " 'min_samples_split': [2, 5, 10],\n",
259
+ " 'max_features': ['sqrt', 'log2', 0.3, None],\n",
260
+ " 'bootstrap': [True, False]\n",
261
+ "}\n",
262
+ "\n",
263
+ "rf = RandomForestRegressor(oob_score=False, n_jobs=-1, random_state=42)\n",
264
+ "\n",
265
+ "grid_search = GridSearchCV(\n",
266
+ " estimator=rf,\n",
267
+ " param_grid=param_grid,\n",
268
+ " scoring='r2',\n",
269
+ " cv=5,\n",
270
+ " verbose=1,\n",
271
+ " n_jobs=-1\n",
272
+ ")\n",
273
+ "\n",
274
+ "grid_search.fit(X_train, Y_train)\n",
275
+ "\n",
276
+ "best_model = grid_search.best_estimator_\n",
277
+ "print(grid_search.best_params_)\n",
278
+ "\n",
279
+ "Y_pred = best_model.predict(X_test)\n",
280
+ "\n",
281
+ "r2 = r2_score(Y_test, Y_pred)\n",
282
+ "rmse = np.sqrt(mean_squared_error(Y_test, Y_pred))\n",
283
+ "mae = mean_absolute_error(Y_test, Y_pred)\n",
284
+ "\n",
285
+ "print(f'R2: {r2:.4f}')\n",
286
+ "print(f'RMSE: {rmse:.4f}')\n",
287
+ "print(f'MAE: {mae:.4f}')"
288
+ ]
289
+ }
290
+ ],
291
+ "metadata": {
292
+ "colab": {
293
+ "provenance": [
294
+ {
295
+ "file_id": "1CTjVKHB6iiDvAQmm4JsyyIgVZShwcp_4",
296
+ "timestamp": 1723512979448
297
+ }
298
+ ]
299
+ },
300
+ "kernelspec": {
301
+ "display_name": "Python 3",
302
+ "name": "python3"
303
+ },
304
+ "language_info": {
305
+ "name": "python"
306
+ }
307
+ },
308
+ "nbformat": 4,
309
+ "nbformat_minor": 0
310
+ }