imrkash commited on
Commit
48831c9
Β·
verified Β·
1 Parent(s): 7a6692e

Upload sequential-algo/jupyter/train.ipynb with huggingface_hub

Browse files
Files changed (1) hide show
  1. sequential-algo/jupyter/train.ipynb +371 -0
sequential-algo/jupyter/train.ipynb ADDED
@@ -0,0 +1,371 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "cells": [
3
+ {
4
+ "cell_type": "markdown",
5
+ "metadata": {},
6
+ "source": [
7
+ "- https://towardsdatascience.com/convolutional-neural-networks-explained-9cc5188c4939\n",
8
+ "- Problem Statement:\n",
9
+ "- Poultry industry faces economic losses due to various diseases. 'Salmonella', 'Newcasttle', 'Coccidiosis' \n",
10
+ "- are major common diseases and it is estimated that the major loss occurred in poultry industry is due to \n",
11
+ "- these diseases.\n"
12
+ ]
13
+ },
14
+ {
15
+ "cell_type": "markdown",
16
+ "metadata": {},
17
+ "source": []
18
+ },
19
+ {
20
+ "cell_type": "code",
21
+ "execution_count": null,
22
+ "metadata": {},
23
+ "outputs": [],
24
+ "source": [
25
+ "import numpy as np\n",
26
+ "import os\n",
27
+ "import matplotlib.pyplot as plt\n",
28
+ "import tensorflow as tf\n",
29
+ "from tensorflow.keras import layers, models\n",
30
+ "from tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping, ReduceLROnPlateau"
31
+ ]
32
+ },
33
+ {
34
+ "cell_type": "code",
35
+ "execution_count": null,
36
+ "metadata": {},
37
+ "outputs": [],
38
+ "source": [
39
+ "# loading data\n",
40
+ "\n",
41
+ "directory='../../../poultry/poultry_diseases'\n"
42
+ ]
43
+ },
44
+ {
45
+ "cell_type": "code",
46
+ "execution_count": null,
47
+ "metadata": {},
48
+ "outputs": [],
49
+ "source": [
50
+ "# variables to use\n",
51
+ "IMAGE_SIZE=320\n",
52
+ "BATCH_SIZE=32\n",
53
+ "CHANNELS=3"
54
+ ]
55
+ },
56
+ {
57
+ "cell_type": "code",
58
+ "execution_count": null,
59
+ "metadata": {},
60
+ "outputs": [],
61
+ "source": [
62
+ "dataset=tf.keras.preprocessing.image_dataset_from_directory(\n",
63
+ " directory,\n",
64
+ " labels=\"inferred\",\n",
65
+ " label_mode=\"int\",\n",
66
+ " class_names=None,\n",
67
+ " color_mode=\"rgb\",\n",
68
+ " batch_size=BATCH_SIZE,\n",
69
+ " image_size=(IMAGE_SIZE, IMAGE_SIZE),\n",
70
+ " shuffle=True,\n",
71
+ " seed=None,\n",
72
+ " validation_split=None,\n",
73
+ " subset=None,\n",
74
+ " interpolation=\"bilinear\",\n",
75
+ " follow_links=False,\n",
76
+ " crop_to_aspect_ratio=False,)\n",
77
+ "\n",
78
+ "# Found 6812 files belonging to 4 classes.\n"
79
+ ]
80
+ },
81
+ {
82
+ "cell_type": "code",
83
+ "execution_count": null,
84
+ "metadata": {},
85
+ "outputs": [],
86
+ "source": [
87
+ "class_names=dataset.class_names\n",
88
+ "# ['cocci', 'healthy', 'ncd', 'salmo']\n"
89
+ ]
90
+ },
91
+ {
92
+ "cell_type": "code",
93
+ "execution_count": null,
94
+ "metadata": {},
95
+ "outputs": [],
96
+ "source": [
97
+ "# Visualizing random images from the dataset\n",
98
+ "def visualize_dataset (dataset):\n",
99
+ " print(len(dataset))\n",
100
+ " plt.figure(figsize=(10,10))\n",
101
+ " for img,label in dataset.take(1):\n",
102
+ " for i in range(9):\n",
103
+ " ax=plt.subplot(3,3,i+1)\n",
104
+ " plt.imshow(img[i].numpy().astype('uint8'))\n",
105
+ " plt.title(class_names[label[i].numpy()])\n",
106
+ " plt.axis(\"off\")\n",
107
+ " \n"
108
+ ]
109
+ },
110
+ {
111
+ "cell_type": "code",
112
+ "execution_count": null,
113
+ "metadata": {},
114
+ "outputs": [],
115
+ "source": [
116
+ "# Splitting the data into train,test and validation data\n",
117
+ "\n",
118
+ "def data_split(data,train_size=0.8,validation_size=0.1,test_size=0.1,shuffle=True):\n",
119
+ " if shuffle:\n",
120
+ " data.shuffle(1000,seed=0)\n",
121
+ " train_set=data.take(int(len(data)*train_size))\n",
122
+ " \n",
123
+ " val_set=data.skip(len(train_set)).take(int(len(data)*validation_size))\n",
124
+ " test_set=data.skip(len(train_set)+len(val_set))\n",
125
+ " return train_set, val_set, test_set"
126
+ ]
127
+ },
128
+ {
129
+ "cell_type": "code",
130
+ "execution_count": null,
131
+ "metadata": {},
132
+ "outputs": [],
133
+ "source": [
134
+ "train_set, val_set, test_set= data_split(dataset)\n",
135
+ "visualize_dataset(train_set)"
136
+ ]
137
+ },
138
+ {
139
+ "cell_type": "code",
140
+ "execution_count": null,
141
+ "metadata": {},
142
+ "outputs": [],
143
+ "source": [
144
+ "# data preprocessing\n",
145
+ "\n",
146
+ "train= train_set.cache().shuffle(100).prefetch(buffer_size=tf.data.AUTOTUNE)\n",
147
+ "valid= val_set.cache().shuffle(100).prefetch(buffer_size=tf.data.AUTOTUNE)\n",
148
+ "test= test_set.cache().shuffle(100).prefetch(buffer_size=tf.data.AUTOTUNE)"
149
+ ]
150
+ },
151
+ {
152
+ "cell_type": "code",
153
+ "execution_count": null,
154
+ "metadata": {},
155
+ "outputs": [],
156
+ "source": [
157
+ "# visualize_dataset(train)"
158
+ ]
159
+ },
160
+ {
161
+ "cell_type": "code",
162
+ "execution_count": null,
163
+ "metadata": {},
164
+ "outputs": [],
165
+ "source": [
166
+ "# from tensorflow.keras import layers # type: ignore\n",
167
+ "img_processor= tf.keras.Sequential([\n",
168
+ " layers.Resizing(IMAGE_SIZE,IMAGE_SIZE),\n",
169
+ " layers.Rescaling(1./255)\n",
170
+ "])\n",
171
+ "\n",
172
+ "data_augmentation=tf.keras.Sequential([\n",
173
+ " layers.RandomFlip(mode=\"horizontal_and_vertical\"),\n",
174
+ " layers.RandomRotation(factor=0.5)\n",
175
+ "])\n",
176
+ "\n"
177
+ ]
178
+ },
179
+ {
180
+ "cell_type": "code",
181
+ "execution_count": null,
182
+ "metadata": {},
183
+ "outputs": [
184
+ {
185
+ "name": "stdout",
186
+ "output_type": "stream",
187
+ "text": [
188
+ "TensorFlow version: 2.17.0\n"
189
+ ]
190
+ }
191
+ ],
192
+ "source": [
193
+ "# Build our model\n",
194
+ "\n",
195
+ "print(\"TensorFlow version:\", tf.__version__)\n",
196
+ "# Define input shape\n",
197
+ "input_shape = (IMAGE_SIZE, IMAGE_SIZE, CHANNELS)\n",
198
+ "\n",
199
+ "CNN = models.Sequential([\n",
200
+ " layers.Input(shape=input_shape),\n",
201
+ " img_processor,\n",
202
+ " data_augmentation,\n",
203
+ " layers.Conv2D(filters=32, kernel_size=(3,3), activation='relu'),\n",
204
+ " layers.MaxPooling2D((2,2)),\n",
205
+ " layers.Conv2D(64, (3,3), activation='relu'),\n",
206
+ " layers.MaxPooling2D((2,2)),\n",
207
+ " layers.Conv2D(128, (3,3), activation='relu'),\n",
208
+ " layers.MaxPooling2D((2,2)),\n",
209
+ " layers.Conv2D(64, (3,3), activation='relu'),\n",
210
+ " layers.MaxPooling2D((2,2)),\n",
211
+ " layers.Conv2D(128, (3,3), activation='relu'),\n",
212
+ " layers.MaxPooling2D((2,2)),\n",
213
+ " layers.Conv2D(64, (3,3), activation='relu'),\n",
214
+ " layers.MaxPooling2D((2,2)),\n",
215
+ " layers.Flatten(),\n",
216
+ " layers.Dense(128, activation='relu'),\n",
217
+ " layers.Dense(len(class_names), activation='softmax')\n",
218
+ "])\n",
219
+ "\n",
220
+ "\n"
221
+ ]
222
+ },
223
+ {
224
+ "cell_type": "code",
225
+ "execution_count": null,
226
+ "metadata": {},
227
+ "outputs": [],
228
+ "source": [
229
+ "# CNN.build(input_shape)\n",
230
+ "CNN.summary()"
231
+ ]
232
+ },
233
+ {
234
+ "cell_type": "code",
235
+ "execution_count": null,
236
+ "metadata": {},
237
+ "outputs": [],
238
+ "source": [
239
+ "\n",
240
+ "# Training our model\n",
241
+ "\n",
242
+ "CNN.compile(optimizer = tf.keras.optimizers.Adam(learning_rate=0.001),\n",
243
+ " loss = 'sparse_categorical_crossentropy',\n",
244
+ " metrics=['accuracy'])\n",
245
+ "\n"
246
+ ]
247
+ },
248
+ {
249
+ "cell_type": "code",
250
+ "execution_count": null,
251
+ "metadata": {},
252
+ "outputs": [],
253
+ "source": [
254
+ "# Define callbacks\n",
255
+ "callbacks = [\n",
256
+ " ModelCheckpoint('best_model.keras', save_best_only=True, monitor='val_accuracy'),\n",
257
+ " EarlyStopping(patience=10, restore_best_weights=True),\n",
258
+ " ReduceLROnPlateau(factor=0.1, patience=3)\n",
259
+ "]"
260
+ ]
261
+ },
262
+ {
263
+ "cell_type": "code",
264
+ "execution_count": null,
265
+ "metadata": {},
266
+ "outputs": [
267
+ {
268
+ "name": "stdout",
269
+ "output_type": "stream",
270
+ "text": [
271
+ "\u001b[1m2/2\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m29s\u001b[0m 25s/step - accuracy: 0.3646 - loss: 1.3761 - val_accuracy: 0.3482 - val_loss: 1.2617 - learning_rate: 0.0010\n"
272
+ ]
273
+ }
274
+ ],
275
+ "source": [
276
+ "# training history\n",
277
+ "\n",
278
+ "# training_his=CNN.fit(train,epochs=10,validation_data=valid,batch_size=BATCH_SIZE)\n",
279
+ "training_his = CNN.fit(\n",
280
+ " train,\n",
281
+ " validation_data=valid,\n",
282
+ " epochs=100,\n",
283
+ " batch_size=BATCH_SIZE,\n",
284
+ " callbacks=callbacks\n",
285
+ ")\n"
286
+ ]
287
+ },
288
+ {
289
+ "cell_type": "code",
290
+ "execution_count": null,
291
+ "metadata": {},
292
+ "outputs": [
293
+ {
294
+ "name": "stdout",
295
+ "output_type": "stream",
296
+ "text": [
297
+ "Model saved to poultry_disease_model.keras\n"
298
+ ]
299
+ }
300
+ ],
301
+ "source": [
302
+ "# Save the trained model\n",
303
+ "from tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping, ReduceLROnPlateau\n",
304
+ "model_save_path = 'poultry_disease_model.keras'\n",
305
+ "tf.keras.models.save_model(CNN, model_save_path)\n",
306
+ "print(f\"Model saved to {model_save_path}\")\n",
307
+ "\n",
308
+ "# If you need to load the model later, you can use:\n",
309
+ "# loaded_model = tf.keras.models.load_model('poultry_disease_model.keras')"
310
+ ]
311
+ },
312
+ {
313
+ "cell_type": "code",
314
+ "execution_count": null,
315
+ "metadata": {},
316
+ "outputs": [
317
+ {
318
+ "name": "stdout",
319
+ "output_type": "stream",
320
+ "text": [
321
+ "\u001b[1m21/21\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m14s\u001b[0m 663ms/step - accuracy: 0.3642 - loss: 1.2607\n"
322
+ ]
323
+ }
324
+ ],
325
+ "source": [
326
+ "scores= CNN.evaluate(valid)\n"
327
+ ]
328
+ },
329
+ {
330
+ "cell_type": "code",
331
+ "execution_count": null,
332
+ "metadata": {},
333
+ "outputs": [],
334
+ "source": [
335
+ "CNN.predict(test)"
336
+ ]
337
+ },
338
+ {
339
+ "cell_type": "code",
340
+ "execution_count": null,
341
+ "metadata": {},
342
+ "outputs": [],
343
+ "source": [
344
+ "# Evaluate the model\n",
345
+ "\n",
346
+ "trained = tf.keras.models.load_model('poultry_disease_model.keras')\n",
347
+ "test_loss, test_accuracy = trained.evaluate(test)\n",
348
+ "print(f\"Test accuracy: {test_accuracy:.4f}\")"
349
+ ]
350
+ },
351
+ {
352
+ "cell_type": "code",
353
+ "execution_count": null,
354
+ "metadata": {},
355
+ "outputs": [],
356
+ "source": [
357
+ "# If you want to save in SavedModel format (for TFLite, TFServing, etc.)\n",
358
+ "savedmodel_path = 'poultry_disease_savedmodel'\n",
359
+ "trained.export(savedmodel_path)\n",
360
+ "print(f\"SavedModel exported to {savedmodel_path}\")"
361
+ ]
362
+ }
363
+ ],
364
+ "metadata": {
365
+ "language_info": {
366
+ "name": "python"
367
+ }
368
+ },
369
+ "nbformat": 4,
370
+ "nbformat_minor": 2
371
+ }