Upload 13 files
Browse files- .gitattributes +2 -0
- README.md +0 -0
- evaluation_results/classification_report_20260211_155639.txt +70 -0
- evaluation_results/confusion_matrix.png +3 -0
- evaluation_results/confusion_matrix_20260211_155639.csv +36 -0
- evaluation_results/confusion_pairs_20260211_155639.csv +3 -0
- evaluation_results/evaluation_metrics_20260211_155639.json +21 -0
- evaluation_results/per_class_metrics_20260211_155639.csv +36 -0
- isl_model_advanced.tflite +3 -0
- landmark_dataset_with_orientation.csv +3 -0
- optimize_sign_assets.py +113 -0
- prepare_local_sign_assets.py +119 -0
- test_full_model.py +586 -0
- train_advanced_model.py +1093 -0
.gitattributes
CHANGED
|
@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
+
evaluation_results/confusion_matrix.png filter=lfs diff=lfs merge=lfs -text
|
| 37 |
+
landmark_dataset_with_orientation.csv filter=lfs diff=lfs merge=lfs -text
|
README.md
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
evaluation_results/classification_report_20260211_155639.txt
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
======================================================================
|
| 2 |
+
KAIROAI MODEL EVALUATION REPORT
|
| 3 |
+
Generated: 2026-02-11 15:56:39
|
| 4 |
+
======================================================================
|
| 5 |
+
|
| 6 |
+
OVERALL METRICS
|
| 7 |
+
----------------------------------------------------------------------
|
| 8 |
+
|
| 9 |
+
TRAINING SET:
|
| 10 |
+
Accuracy: 99.94%
|
| 11 |
+
Precision: 99.94%
|
| 12 |
+
Recall: 99.94%
|
| 13 |
+
F1-Score: 99.94%
|
| 14 |
+
|
| 15 |
+
VALIDATION SET:
|
| 16 |
+
Accuracy: 99.97%
|
| 17 |
+
Precision: 99.97%
|
| 18 |
+
Recall: 99.97%
|
| 19 |
+
F1-Score: 99.97%
|
| 20 |
+
|
| 21 |
+
TEST SET:
|
| 22 |
+
Accuracy: 99.96%
|
| 23 |
+
Precision: 99.96%
|
| 24 |
+
Recall: 99.96%
|
| 25 |
+
F1-Score: 99.96%
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
DETAILED CLASSIFICATION REPORT (Test Set)
|
| 29 |
+
----------------------------------------------------------------------
|
| 30 |
+
precision recall f1-score support
|
| 31 |
+
|
| 32 |
+
A 1.000 1.000 1.000 240
|
| 33 |
+
B 1.000 1.000 1.000 237
|
| 34 |
+
C 1.000 1.000 1.000 266
|
| 35 |
+
D 1.000 1.000 1.000 240
|
| 36 |
+
E 1.000 1.000 1.000 240
|
| 37 |
+
F 1.000 1.000 1.000 240
|
| 38 |
+
G 1.000 1.000 1.000 240
|
| 39 |
+
H 0.992 1.000 0.996 240
|
| 40 |
+
I 1.000 1.000 1.000 276
|
| 41 |
+
J 1.000 1.000 1.000 240
|
| 42 |
+
K 1.000 1.000 1.000 240
|
| 43 |
+
L 1.000 1.000 1.000 240
|
| 44 |
+
M 1.000 0.992 0.996 240
|
| 45 |
+
N 1.000 1.000 1.000 240
|
| 46 |
+
O 1.000 1.000 1.000 262
|
| 47 |
+
P 1.000 1.000 1.000 226
|
| 48 |
+
Q 1.000 1.000 1.000 148
|
| 49 |
+
R 1.000 1.000 1.000 240
|
| 50 |
+
S 1.000 0.996 0.998 234
|
| 51 |
+
T 1.000 1.000 1.000 240
|
| 52 |
+
U 1.000 1.000 1.000 240
|
| 53 |
+
V 1.000 1.000 1.000 257
|
| 54 |
+
W 1.000 1.000 1.000 240
|
| 55 |
+
X 1.000 1.000 1.000 240
|
| 56 |
+
Y 1.000 1.000 1.000 240
|
| 57 |
+
Z 1.000 1.000 1.000 240
|
| 58 |
+
1 1.000 1.000 1.000 240
|
| 59 |
+
2 1.000 1.000 1.000 240
|
| 60 |
+
3 1.000 1.000 1.000 240
|
| 61 |
+
4 1.000 1.000 1.000 240
|
| 62 |
+
5 1.000 1.000 1.000 240
|
| 63 |
+
6 0.996 1.000 0.998 240
|
| 64 |
+
7 1.000 1.000 1.000 240
|
| 65 |
+
8 1.000 1.000 1.000 240
|
| 66 |
+
9 1.000 1.000 1.000 240
|
| 67 |
+
|
| 68 |
+
accuracy 1.000 8386
|
| 69 |
+
macro avg 1.000 1.000 1.000 8386
|
| 70 |
+
weighted avg 1.000 1.000 1.000 8386
|
evaluation_results/confusion_matrix.png
ADDED
|
Git LFS Details
|
evaluation_results/confusion_matrix_20260211_155639.csv
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
,A,B,C,D,E,F,G,H,I,J,K,L,M,N,O,P,Q,R,S,T,U,V,W,X,Y,Z,1,2,3,4,5,6,7,8,9
|
| 2 |
+
A,240,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
|
| 3 |
+
B,0,237,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
|
| 4 |
+
C,0,0,266,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
|
| 5 |
+
D,0,0,0,240,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
|
| 6 |
+
E,0,0,0,0,240,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
|
| 7 |
+
F,0,0,0,0,0,240,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
|
| 8 |
+
G,0,0,0,0,0,0,240,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
|
| 9 |
+
H,0,0,0,0,0,0,0,240,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
|
| 10 |
+
I,0,0,0,0,0,0,0,0,276,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
|
| 11 |
+
J,0,0,0,0,0,0,0,0,0,240,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
|
| 12 |
+
K,0,0,0,0,0,0,0,0,0,0,240,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
|
| 13 |
+
L,0,0,0,0,0,0,0,0,0,0,0,240,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
|
| 14 |
+
M,0,0,0,0,0,0,0,2,0,0,0,0,238,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
|
| 15 |
+
N,0,0,0,0,0,0,0,0,0,0,0,0,0,240,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
|
| 16 |
+
O,0,0,0,0,0,0,0,0,0,0,0,0,0,0,262,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
|
| 17 |
+
P,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,226,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
|
| 18 |
+
Q,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,148,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
|
| 19 |
+
R,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,240,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
|
| 20 |
+
S,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,233,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0
|
| 21 |
+
T,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,240,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
|
| 22 |
+
U,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,240,0,0,0,0,0,0,0,0,0,0,0,0,0,0
|
| 23 |
+
V,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,257,0,0,0,0,0,0,0,0,0,0,0,0,0
|
| 24 |
+
W,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,240,0,0,0,0,0,0,0,0,0,0,0,0
|
| 25 |
+
X,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,240,0,0,0,0,0,0,0,0,0,0,0
|
| 26 |
+
Y,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,240,0,0,0,0,0,0,0,0,0,0
|
| 27 |
+
Z,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,240,0,0,0,0,0,0,0,0,0
|
| 28 |
+
1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,240,0,0,0,0,0,0,0,0
|
| 29 |
+
2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,240,0,0,0,0,0,0,0
|
| 30 |
+
3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,240,0,0,0,0,0,0
|
| 31 |
+
4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,240,0,0,0,0,0
|
| 32 |
+
5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,240,0,0,0,0
|
| 33 |
+
6,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,240,0,0,0
|
| 34 |
+
7,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,240,0,0
|
| 35 |
+
8,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,240,0
|
| 36 |
+
9,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,240
|
evaluation_results/confusion_pairs_20260211_155639.csv
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Pair,Total_Count,Combined_Rate
|
| 2 |
+
H ↔ M,2,0.8333333333333334
|
| 3 |
+
6 ↔ S,1,0.4273504273504274
|
evaluation_results/evaluation_metrics_20260211_155639.json
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"timestamp": "20260211_155639",
|
| 3 |
+
"training": {
|
| 4 |
+
"accuracy": 99.94037637413825,
|
| 5 |
+
"precision": 99.94248078852188,
|
| 6 |
+
"recall": 99.93982979971115,
|
| 7 |
+
"f1_score": 99.94106788684324
|
| 8 |
+
},
|
| 9 |
+
"validation": {
|
| 10 |
+
"accuracy": 99.97018929795797,
|
| 11 |
+
"precision": 99.97165518959099,
|
| 12 |
+
"recall": 99.97170523138831,
|
| 13 |
+
"f1_score": 99.97160969501877
|
| 14 |
+
},
|
| 15 |
+
"test": {
|
| 16 |
+
"accuracy": 99.96422609110422,
|
| 17 |
+
"precision": 99.96453188456206,
|
| 18 |
+
"recall": 99.96398046398046,
|
| 19 |
+
"f1_score": 99.96413197796029
|
| 20 |
+
}
|
| 21 |
+
}
|
evaluation_results/per_class_metrics_20260211_155639.csv
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Class,Precision,Recall,F1-Score,Support
|
| 2 |
+
A,100.0,100.0,100.0,240
|
| 3 |
+
B,100.0,100.0,100.0,237
|
| 4 |
+
C,100.0,100.0,100.0,266
|
| 5 |
+
D,100.0,100.0,100.0,240
|
| 6 |
+
E,100.0,100.0,100.0,240
|
| 7 |
+
F,100.0,100.0,100.0,240
|
| 8 |
+
G,100.0,100.0,100.0,240
|
| 9 |
+
H,99.17355371900827,100.0,99.5850622406639,240
|
| 10 |
+
I,100.0,100.0,100.0,276
|
| 11 |
+
J,100.0,100.0,100.0,240
|
| 12 |
+
K,100.0,100.0,100.0,240
|
| 13 |
+
L,100.0,100.0,100.0,240
|
| 14 |
+
M,100.0,99.16666666666667,99.581589958159,240
|
| 15 |
+
N,100.0,100.0,100.0,240
|
| 16 |
+
O,100.0,100.0,100.0,262
|
| 17 |
+
P,100.0,100.0,100.0,226
|
| 18 |
+
Q,100.0,100.0,100.0,148
|
| 19 |
+
R,100.0,100.0,100.0,240
|
| 20 |
+
S,100.0,99.57264957264957,99.78586723768737,234
|
| 21 |
+
T,100.0,100.0,100.0,240
|
| 22 |
+
U,100.0,100.0,100.0,240
|
| 23 |
+
V,100.0,100.0,100.0,257
|
| 24 |
+
W,100.0,100.0,100.0,240
|
| 25 |
+
X,100.0,100.0,100.0,240
|
| 26 |
+
Y,100.0,100.0,100.0,240
|
| 27 |
+
Z,100.0,100.0,100.0,240
|
| 28 |
+
1,100.0,100.0,100.0,240
|
| 29 |
+
2,100.0,100.0,100.0,240
|
| 30 |
+
3,100.0,100.0,100.0,240
|
| 31 |
+
4,100.0,100.0,100.0,240
|
| 32 |
+
5,100.0,100.0,100.0,240
|
| 33 |
+
6,99.5850622406639,100.0,99.7920997920998,240
|
| 34 |
+
7,100.0,100.0,100.0,240
|
| 35 |
+
8,100.0,100.0,100.0,240
|
| 36 |
+
9,100.0,100.0,100.0,240
|
isl_model_advanced.tflite
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:914c7609950af5e7cf97da31b9542f3b7b98af9b3b8af3c3e8196cf6cd0cf697
|
| 3 |
+
size 494372
|
landmark_dataset_with_orientation.csv
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:12d66a842ea865885f43c78f735f63cff65ca1f4d7178ccecca7b9fc8d48b65c
|
| 3 |
+
size 77254059
|
optimize_sign_assets.py
ADDED
|
@@ -0,0 +1,113 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Optimize bundled sign images in-place to reduce Flutter app size.
|
| 3 |
+
|
| 4 |
+
Default behavior:
|
| 5 |
+
- Scans assets/signs/**/image.*
|
| 6 |
+
- Resizes images to max 960px on the longest side (if larger)
|
| 7 |
+
- Converts to RGB palette PNG (256 colors) and saves optimized
|
| 8 |
+
- Keeps existing file names and paths unchanged
|
| 9 |
+
|
| 10 |
+
Usage:
|
| 11 |
+
e:/KairoAI/.venv/Scripts/python.exe optimize_sign_assets.py
|
| 12 |
+
e:/KairoAI/.venv/Scripts/python.exe optimize_sign_assets.py --max-size 768 --quality 80
|
| 13 |
+
"""
|
| 14 |
+
|
| 15 |
+
from __future__ import annotations
|
| 16 |
+
|
| 17 |
+
import argparse
|
| 18 |
+
from dataclasses import dataclass
|
| 19 |
+
from pathlib import Path
|
| 20 |
+
from typing import Iterable
|
| 21 |
+
|
| 22 |
+
from PIL import Image
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
SUPPORTED = {".png", ".jpg", ".jpeg", ".webp"}
|
| 26 |
+
ASSET_ROOT = Path(__file__).parent / "assets" / "signs"
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
@dataclass
|
| 30 |
+
class FileStat:
|
| 31 |
+
path: Path
|
| 32 |
+
before: int
|
| 33 |
+
after: int
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def iter_image_files(root: Path) -> Iterable[Path]:
|
| 37 |
+
if not root.exists():
|
| 38 |
+
return []
|
| 39 |
+
return sorted(
|
| 40 |
+
[p for p in root.rglob("image.*") if p.is_file() and p.suffix.lower() in SUPPORTED]
|
| 41 |
+
)
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
def optimize_file(path: Path, max_size: int, quality: int) -> FileStat:
|
| 45 |
+
before = path.stat().st_size
|
| 46 |
+
|
| 47 |
+
with Image.open(path) as img:
|
| 48 |
+
img = img.convert("RGB")
|
| 49 |
+
|
| 50 |
+
# Resize only when needed, preserving aspect ratio.
|
| 51 |
+
longest = max(img.width, img.height)
|
| 52 |
+
if longest > max_size:
|
| 53 |
+
scale = max_size / float(longest)
|
| 54 |
+
new_size = (max(1, int(img.width * scale)), max(1, int(img.height * scale)))
|
| 55 |
+
img = img.resize(new_size, Image.Resampling.LANCZOS)
|
| 56 |
+
|
| 57 |
+
# Keep original extension so existing asset paths remain valid.
|
| 58 |
+
ext = path.suffix.lower()
|
| 59 |
+
if ext == ".png":
|
| 60 |
+
pal = img.quantize(colors=256, method=Image.Quantize.MEDIANCUT)
|
| 61 |
+
pal.save(path, format="PNG", optimize=True)
|
| 62 |
+
elif ext in {".jpg", ".jpeg"}:
|
| 63 |
+
img.save(path, format="JPEG", optimize=True, quality=quality)
|
| 64 |
+
elif ext == ".webp":
|
| 65 |
+
img.save(path, format="WEBP", quality=quality, method=6)
|
| 66 |
+
|
| 67 |
+
after = path.stat().st_size
|
| 68 |
+
return FileStat(path=path, before=before, after=after)
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
def fmt_size(num_bytes: int) -> str:
|
| 72 |
+
mb = num_bytes / (1024 * 1024)
|
| 73 |
+
return f"{mb:.2f} MB"
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
def main() -> int:
|
| 77 |
+
parser = argparse.ArgumentParser(description="Optimize local sign assets in-place")
|
| 78 |
+
parser.add_argument("--max-size", type=int, default=960, help="Longest image side after resize")
|
| 79 |
+
parser.add_argument("--quality", type=int, default=82, help="JPEG/WEBP quality")
|
| 80 |
+
args = parser.parse_args()
|
| 81 |
+
|
| 82 |
+
files = list(iter_image_files(ASSET_ROOT))
|
| 83 |
+
if not files:
|
| 84 |
+
print(f"No supported image files found under: {ASSET_ROOT}")
|
| 85 |
+
return 1
|
| 86 |
+
|
| 87 |
+
stats: list[FileStat] = []
|
| 88 |
+
for file_path in files:
|
| 89 |
+
stat = optimize_file(file_path, max_size=args.max_size, quality=args.quality)
|
| 90 |
+
stats.append(stat)
|
| 91 |
+
|
| 92 |
+
total_before = sum(s.before for s in stats)
|
| 93 |
+
total_after = sum(s.after for s in stats)
|
| 94 |
+
saved = total_before - total_after
|
| 95 |
+
pct = (saved / total_before * 100.0) if total_before else 0.0
|
| 96 |
+
|
| 97 |
+
print(f"Processed: {len(stats)} file(s)")
|
| 98 |
+
print(f"Before: {fmt_size(total_before)}")
|
| 99 |
+
print(f"After: {fmt_size(total_after)}")
|
| 100 |
+
print(f"Saved: {fmt_size(saved)} ({pct:.1f}%)")
|
| 101 |
+
|
| 102 |
+
# Print top 5 biggest savings for quick visibility.
|
| 103 |
+
top = sorted(stats, key=lambda s: (s.before - s.after), reverse=True)[:5]
|
| 104 |
+
print("Top savings:")
|
| 105 |
+
for s in top:
|
| 106 |
+
delta = s.before - s.after
|
| 107 |
+
print(f"- {s.path.as_posix()}: {fmt_size(delta)}")
|
| 108 |
+
|
| 109 |
+
return 0
|
| 110 |
+
|
| 111 |
+
|
| 112 |
+
if __name__ == "__main__":
|
| 113 |
+
raise SystemExit(main())
|
prepare_local_sign_assets.py
ADDED
|
@@ -0,0 +1,119 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Copy local sign images into Flutter assets using canonical structure:
|
| 3 |
+
assets/signs/{LABEL}/image.{ext}
|
| 4 |
+
|
| 5 |
+
Default behavior also optimizes copied images in-place to reduce app size.
|
| 6 |
+
|
| 7 |
+
Usage:
|
| 8 |
+
python prepare_local_sign_assets.py "D:\\Downloads\\all the images - 2"
|
| 9 |
+
"""
|
| 10 |
+
|
| 11 |
+
from pathlib import Path
|
| 12 |
+
import shutil
|
| 13 |
+
import sys
|
| 14 |
+
|
| 15 |
+
from PIL import Image
|
| 16 |
+
|
| 17 |
+
SOURCE_DEFAULT = Path(r"D:\Downloads\all the images - 2")
|
| 18 |
+
DEST_ROOT = Path(__file__).parent / "assets" / "signs"
|
| 19 |
+
SUPPORTED = {".png", ".jpg", ".jpeg", ".webp", ".gif"}
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def normalize_label(name: str) -> str:
|
| 23 |
+
return name.strip().upper()
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
def optimize_image(path: Path, max_size: int = 960, quality: int = 82) -> None:
|
| 27 |
+
with Image.open(path) as img:
|
| 28 |
+
img = img.convert("RGB")
|
| 29 |
+
|
| 30 |
+
longest = max(img.width, img.height)
|
| 31 |
+
if longest > max_size:
|
| 32 |
+
scale = max_size / float(longest)
|
| 33 |
+
new_size = (max(1, int(img.width * scale)), max(1, int(img.height * scale)))
|
| 34 |
+
img = img.resize(new_size, Image.Resampling.LANCZOS)
|
| 35 |
+
|
| 36 |
+
ext = path.suffix.lower()
|
| 37 |
+
if ext == ".png":
|
| 38 |
+
pal = img.quantize(colors=256, method=Image.Quantize.MEDIANCUT)
|
| 39 |
+
pal.save(path, format="PNG", optimize=True)
|
| 40 |
+
elif ext in {".jpg", ".jpeg"}:
|
| 41 |
+
img.save(path, format="JPEG", optimize=True, quality=quality)
|
| 42 |
+
elif ext == ".webp":
|
| 43 |
+
img.save(path, format="WEBP", quality=quality, method=6)
|
| 44 |
+
elif ext == ".gif":
|
| 45 |
+
img.save(path, format="GIF", optimize=True)
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
def main() -> int:
|
| 49 |
+
source = Path(sys.argv[1]) if len(sys.argv) > 1 else SOURCE_DEFAULT
|
| 50 |
+
if not source.exists() or not source.is_dir():
|
| 51 |
+
print(f"Source folder not found: {source}")
|
| 52 |
+
return 1
|
| 53 |
+
|
| 54 |
+
copied = 0
|
| 55 |
+
|
| 56 |
+
# Mode 1: nested dataset folders (e.g. E:/ISL_Model_Training/Indian/A/*.jpg)
|
| 57 |
+
subdirs = [p for p in source.iterdir() if p.is_dir()]
|
| 58 |
+
if subdirs:
|
| 59 |
+
for subdir in sorted(subdirs, key=lambda p: p.name.lower()):
|
| 60 |
+
label = normalize_label(subdir.name)
|
| 61 |
+
if not label:
|
| 62 |
+
continue
|
| 63 |
+
|
| 64 |
+
candidates = [
|
| 65 |
+
p for p in subdir.iterdir()
|
| 66 |
+
if p.is_file() and p.suffix.lower() in SUPPORTED
|
| 67 |
+
]
|
| 68 |
+
if not candidates:
|
| 69 |
+
continue
|
| 70 |
+
|
| 71 |
+
# Prefer numerically-small file names such as 0.jpg, 1.jpg.
|
| 72 |
+
def _sort_key(path: Path) -> tuple[int, str]:
|
| 73 |
+
stem = path.stem.strip()
|
| 74 |
+
if stem.isdigit():
|
| 75 |
+
return (0, f"{int(stem):08d}")
|
| 76 |
+
return (1, stem.lower())
|
| 77 |
+
|
| 78 |
+
candidates.sort(key=_sort_key)
|
| 79 |
+
chosen = candidates[0]
|
| 80 |
+
|
| 81 |
+
dest_dir = DEST_ROOT / label
|
| 82 |
+
dest_dir.mkdir(parents=True, exist_ok=True)
|
| 83 |
+
dest_file = dest_dir / f"image{chosen.suffix.lower()}"
|
| 84 |
+
shutil.copy2(chosen, dest_file)
|
| 85 |
+
optimize_image(dest_file)
|
| 86 |
+
copied += 1
|
| 87 |
+
print(f"{subdir.name}/{chosen.name} -> {dest_file}")
|
| 88 |
+
|
| 89 |
+
# Mode 2: flat folder of files (legacy behavior).
|
| 90 |
+
else:
|
| 91 |
+
files = [
|
| 92 |
+
p for p in source.iterdir() if p.is_file() and p.suffix.lower() in SUPPORTED
|
| 93 |
+
]
|
| 94 |
+
files.sort(key=lambda p: p.name.lower())
|
| 95 |
+
|
| 96 |
+
if not files:
|
| 97 |
+
print("No supported image files found.")
|
| 98 |
+
return 1
|
| 99 |
+
|
| 100 |
+
for file_path in files:
|
| 101 |
+
label = normalize_label(file_path.stem)
|
| 102 |
+
dest_dir = DEST_ROOT / label
|
| 103 |
+
dest_dir.mkdir(parents=True, exist_ok=True)
|
| 104 |
+
dest_file = dest_dir / f"image{file_path.suffix.lower()}"
|
| 105 |
+
shutil.copy2(file_path, dest_file)
|
| 106 |
+
optimize_image(dest_file)
|
| 107 |
+
copied += 1
|
| 108 |
+
print(f"{file_path.name} -> {dest_file}")
|
| 109 |
+
|
| 110 |
+
if copied == 0:
|
| 111 |
+
print("No supported image files found.")
|
| 112 |
+
return 1
|
| 113 |
+
|
| 114 |
+
print(f"Copied {copied} file(s) to {DEST_ROOT}")
|
| 115 |
+
return 0
|
| 116 |
+
|
| 117 |
+
|
| 118 |
+
if __name__ == "__main__":
|
| 119 |
+
raise SystemExit(main())
|
test_full_model.py
ADDED
|
@@ -0,0 +1,586 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
================================================================================
|
| 3 |
+
TEST ISL MODEL WITH MEDIAPIPE - WEBCAM REAL-TIME INFERENCE
|
| 4 |
+
================================================================================
|
| 5 |
+
This script tests the trained model (isl_model_full.tflite) using MediaPipe
|
| 6 |
+
for hand landmark detection and orientation calculation.
|
| 7 |
+
|
| 8 |
+
Features:
|
| 9 |
+
- Real-time webcam hand detection
|
| 10 |
+
- Extracts 130 features (126 landmarks + 4 orientation)
|
| 11 |
+
- Runs TFLite inference
|
| 12 |
+
- Displays prediction with confidence
|
| 13 |
+
|
| 14 |
+
Controls:
|
| 15 |
+
- Press 'q' to quit
|
| 16 |
+
- Press 's' to save screenshot
|
| 17 |
+
- Press 'c' to toggle confidence threshold
|
| 18 |
+
|
| 19 |
+
Author: KairoAI
|
| 20 |
+
================================================================================
|
| 21 |
+
"""
|
| 22 |
+
|
| 23 |
+
import cv2
|
| 24 |
+
import numpy as np
|
| 25 |
+
import mediapipe as mp
|
| 26 |
+
import tensorflow as tf
|
| 27 |
+
import json
|
| 28 |
+
import os
|
| 29 |
+
from collections import deque
|
| 30 |
+
|
| 31 |
+
# ============================================================================
|
| 32 |
+
# CONFIGURATION
|
| 33 |
+
# ============================================================================
|
| 34 |
+
|
| 35 |
+
MODEL_PATH = "isl_model_full.tflite"
|
| 36 |
+
LABELS_PATH = "labels_full.json"
|
| 37 |
+
|
| 38 |
+
# Fallback paths if full model not found
|
| 39 |
+
FALLBACK_MODEL = "isl_model.tflite"
|
| 40 |
+
FALLBACK_LABELS = "labels.json"
|
| 41 |
+
|
| 42 |
+
# Display settings
|
| 43 |
+
CONFIDENCE_THRESHOLD = 0.5
|
| 44 |
+
SMOOTHING_WINDOW = 5 # Number of frames to average predictions
|
| 45 |
+
SHOW_LANDMARKS = True
|
| 46 |
+
SHOW_ORIENTATION = True
|
| 47 |
+
|
| 48 |
+
# Colors (BGR)
|
| 49 |
+
COLOR_PALM = (0, 255, 0) # Green for palm
|
| 50 |
+
COLOR_BACK = (0, 165, 255) # Orange for back of hand
|
| 51 |
+
COLOR_TEXT = (255, 255, 255) # White
|
| 52 |
+
COLOR_BOX = (50, 50, 50) # Dark gray
|
| 53 |
+
|
| 54 |
+
# Labels (fallback if json not found)
|
| 55 |
+
DEFAULT_LABELS = [
|
| 56 |
+
'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J',
|
| 57 |
+
'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T',
|
| 58 |
+
'U', 'V', 'W', 'X', 'Y', 'Z', '1', '2', '3', '4',
|
| 59 |
+
'5', '6', '7', '8', '9'
|
| 60 |
+
]
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
# ============================================================================
|
| 64 |
+
# HAND ORIENTATION CALCULATOR
|
| 65 |
+
# ============================================================================
|
| 66 |
+
|
| 67 |
+
class HandOrientationCalculator:
|
| 68 |
+
"""
|
| 69 |
+
Calculate whether palm or back of hand is facing the camera.
|
| 70 |
+
Uses the cross product of vectors on the hand plane.
|
| 71 |
+
"""
|
| 72 |
+
|
| 73 |
+
# Landmark indices
|
| 74 |
+
WRIST = 0
|
| 75 |
+
INDEX_MCP = 5
|
| 76 |
+
PINKY_MCP = 17
|
| 77 |
+
MIDDLE_MCP = 9
|
| 78 |
+
|
| 79 |
+
@staticmethod
|
| 80 |
+
def calculate_orientation(landmarks, handedness):
|
| 81 |
+
"""
|
| 82 |
+
Calculate hand orientation.
|
| 83 |
+
Returns: (is_palm_facing: float, is_left_hand: float)
|
| 84 |
+
- is_palm_facing: 1.0 = palm facing camera, 0.0 = back facing camera
|
| 85 |
+
- is_left_hand: 1.0 = left hand, 0.0 = right hand
|
| 86 |
+
"""
|
| 87 |
+
if landmarks is None:
|
| 88 |
+
return -1.0, -1.0
|
| 89 |
+
|
| 90 |
+
# Get key points
|
| 91 |
+
wrist = np.array([
|
| 92 |
+
landmarks[HandOrientationCalculator.WRIST].x,
|
| 93 |
+
landmarks[HandOrientationCalculator.WRIST].y,
|
| 94 |
+
landmarks[HandOrientationCalculator.WRIST].z
|
| 95 |
+
])
|
| 96 |
+
|
| 97 |
+
index_mcp = np.array([
|
| 98 |
+
landmarks[HandOrientationCalculator.INDEX_MCP].x,
|
| 99 |
+
landmarks[HandOrientationCalculator.INDEX_MCP].y,
|
| 100 |
+
landmarks[HandOrientationCalculator.INDEX_MCP].z
|
| 101 |
+
])
|
| 102 |
+
|
| 103 |
+
pinky_mcp = np.array([
|
| 104 |
+
landmarks[HandOrientationCalculator.PINKY_MCP].x,
|
| 105 |
+
landmarks[HandOrientationCalculator.PINKY_MCP].y,
|
| 106 |
+
landmarks[HandOrientationCalculator.PINKY_MCP].z
|
| 107 |
+
])
|
| 108 |
+
|
| 109 |
+
# Calculate vectors on the palm plane
|
| 110 |
+
v1 = index_mcp - wrist # Wrist to index
|
| 111 |
+
v2 = pinky_mcp - wrist # Wrist to pinky
|
| 112 |
+
|
| 113 |
+
# Cross product gives normal to palm plane
|
| 114 |
+
normal = np.cross(v1, v2)
|
| 115 |
+
|
| 116 |
+
# Z component of normal indicates palm orientation
|
| 117 |
+
# Positive = palm facing camera, Negative = back facing
|
| 118 |
+
z_component = normal[2]
|
| 119 |
+
|
| 120 |
+
# Determine handedness
|
| 121 |
+
is_left = 1.0 if handedness == "Left" else 0.0
|
| 122 |
+
|
| 123 |
+
# For left hand, the normal direction is reversed
|
| 124 |
+
if is_left:
|
| 125 |
+
z_component = -z_component
|
| 126 |
+
|
| 127 |
+
# Convert to 0/1 (back/palm)
|
| 128 |
+
is_palm_facing = 1.0 if z_component > 0 else 0.0
|
| 129 |
+
|
| 130 |
+
return is_palm_facing, is_left
|
| 131 |
+
|
| 132 |
+
|
| 133 |
+
# ============================================================================
|
| 134 |
+
# LANDMARK PROCESSOR
|
| 135 |
+
# ============================================================================
|
| 136 |
+
|
| 137 |
+
class LandmarkProcessor:
|
| 138 |
+
"""Process MediaPipe hand landmarks into model input features."""
|
| 139 |
+
|
| 140 |
+
def __init__(self, input_size=130):
|
| 141 |
+
self.input_size = input_size
|
| 142 |
+
self.has_orientation = input_size == 130
|
| 143 |
+
|
| 144 |
+
def normalize_landmarks(self, landmarks):
|
| 145 |
+
"""
|
| 146 |
+
Normalize landmarks relative to wrist position and hand size.
|
| 147 |
+
Returns 63 values (21 landmarks × 3 coords) for one hand.
|
| 148 |
+
"""
|
| 149 |
+
if landmarks is None:
|
| 150 |
+
return [0.0] * 63
|
| 151 |
+
|
| 152 |
+
coords = []
|
| 153 |
+
for lm in landmarks:
|
| 154 |
+
coords.extend([lm.x, lm.y, lm.z])
|
| 155 |
+
|
| 156 |
+
coords = np.array(coords, dtype=np.float32)
|
| 157 |
+
|
| 158 |
+
# Normalize relative to wrist (first landmark)
|
| 159 |
+
wrist = coords[:3].copy()
|
| 160 |
+
for i in range(21):
|
| 161 |
+
coords[i*3:i*3+3] -= wrist
|
| 162 |
+
|
| 163 |
+
# Scale by hand size (distance from wrist to middle finger MCP)
|
| 164 |
+
middle_mcp = coords[9*3:9*3+3] # Landmark 9
|
| 165 |
+
hand_size = np.linalg.norm(middle_mcp)
|
| 166 |
+
|
| 167 |
+
if hand_size > 0.001:
|
| 168 |
+
coords /= hand_size
|
| 169 |
+
|
| 170 |
+
return coords.tolist()
|
| 171 |
+
|
| 172 |
+
def process_hands(self, results):
|
| 173 |
+
"""
|
| 174 |
+
Process MediaPipe results into model input.
|
| 175 |
+
Returns: (features, hand_info)
|
| 176 |
+
- features: numpy array of shape (input_size,)
|
| 177 |
+
- hand_info: dict with orientation info for display
|
| 178 |
+
"""
|
| 179 |
+
features = [0.0] * self.input_size
|
| 180 |
+
hand_info = {
|
| 181 |
+
'hand1': None,
|
| 182 |
+
'hand2': None,
|
| 183 |
+
'hand1_orientation': None,
|
| 184 |
+
'hand2_orientation': None
|
| 185 |
+
}
|
| 186 |
+
|
| 187 |
+
if results.multi_hand_landmarks is None:
|
| 188 |
+
return np.array(features, dtype=np.float32), hand_info
|
| 189 |
+
|
| 190 |
+
hands_data = []
|
| 191 |
+
|
| 192 |
+
for i, (hand_landmarks, handedness) in enumerate(
|
| 193 |
+
zip(results.multi_hand_landmarks, results.multi_handedness)
|
| 194 |
+
):
|
| 195 |
+
hand_label = handedness.classification[0].label
|
| 196 |
+
landmarks = hand_landmarks.landmark
|
| 197 |
+
|
| 198 |
+
# Normalize landmarks
|
| 199 |
+
normalized = self.normalize_landmarks(landmarks)
|
| 200 |
+
|
| 201 |
+
# Calculate orientation
|
| 202 |
+
is_palm, is_left = HandOrientationCalculator.calculate_orientation(
|
| 203 |
+
landmarks, hand_label
|
| 204 |
+
)
|
| 205 |
+
|
| 206 |
+
hands_data.append({
|
| 207 |
+
'landmarks': normalized,
|
| 208 |
+
'is_palm': is_palm,
|
| 209 |
+
'is_left': is_left,
|
| 210 |
+
'label': hand_label,
|
| 211 |
+
'raw_landmarks': landmarks
|
| 212 |
+
})
|
| 213 |
+
|
| 214 |
+
# Sort by x position (left to right in image)
|
| 215 |
+
if len(hands_data) > 0:
|
| 216 |
+
hands_data.sort(
|
| 217 |
+
key=lambda h: h['raw_landmarks'][0].x
|
| 218 |
+
)
|
| 219 |
+
|
| 220 |
+
# Fill features for hand 1
|
| 221 |
+
if len(hands_data) >= 1:
|
| 222 |
+
h1 = hands_data[0]
|
| 223 |
+
features[0:63] = h1['landmarks']
|
| 224 |
+
hand_info['hand1'] = h1['label']
|
| 225 |
+
hand_info['hand1_orientation'] = 'Palm' if h1['is_palm'] == 1.0 else 'Back'
|
| 226 |
+
|
| 227 |
+
if self.has_orientation:
|
| 228 |
+
features[126] = h1['is_palm']
|
| 229 |
+
features[127] = h1['is_left']
|
| 230 |
+
|
| 231 |
+
# Fill features for hand 2
|
| 232 |
+
if len(hands_data) >= 2:
|
| 233 |
+
h2 = hands_data[1]
|
| 234 |
+
features[63:126] = h2['landmarks']
|
| 235 |
+
hand_info['hand2'] = h2['label']
|
| 236 |
+
hand_info['hand2_orientation'] = 'Palm' if h2['is_palm'] == 1.0 else 'Back'
|
| 237 |
+
|
| 238 |
+
if self.has_orientation:
|
| 239 |
+
features[128] = h2['is_palm']
|
| 240 |
+
features[129] = h2['is_left']
|
| 241 |
+
elif self.has_orientation:
|
| 242 |
+
features[128] = -1.0
|
| 243 |
+
features[129] = -1.0
|
| 244 |
+
|
| 245 |
+
return np.array(features, dtype=np.float32), hand_info
|
| 246 |
+
|
| 247 |
+
|
| 248 |
+
# ============================================================================
|
| 249 |
+
# PREDICTION SMOOTHER
|
| 250 |
+
# ============================================================================
|
| 251 |
+
|
| 252 |
+
class PredictionSmoother:
|
| 253 |
+
"""Smooth predictions over multiple frames to reduce flickering."""
|
| 254 |
+
|
| 255 |
+
def __init__(self, window_size=5, num_classes=35):
|
| 256 |
+
self.window_size = window_size
|
| 257 |
+
self.num_classes = num_classes
|
| 258 |
+
self.predictions = deque(maxlen=window_size)
|
| 259 |
+
|
| 260 |
+
def add_prediction(self, probs):
|
| 261 |
+
"""Add a new prediction probability distribution."""
|
| 262 |
+
self.predictions.append(probs)
|
| 263 |
+
|
| 264 |
+
def get_smoothed_prediction(self):
|
| 265 |
+
"""Get averaged prediction over the window."""
|
| 266 |
+
if len(self.predictions) == 0:
|
| 267 |
+
return None, 0.0
|
| 268 |
+
|
| 269 |
+
avg_probs = np.mean(list(self.predictions), axis=0)
|
| 270 |
+
pred_class = np.argmax(avg_probs)
|
| 271 |
+
confidence = avg_probs[pred_class]
|
| 272 |
+
|
| 273 |
+
return pred_class, confidence
|
| 274 |
+
|
| 275 |
+
def clear(self):
|
| 276 |
+
"""Clear prediction history."""
|
| 277 |
+
self.predictions.clear()
|
| 278 |
+
|
| 279 |
+
|
| 280 |
+
# ============================================================================
|
| 281 |
+
# MAIN TESTER CLASS
|
| 282 |
+
# ============================================================================
|
| 283 |
+
|
| 284 |
+
class ISLModelTester:
|
| 285 |
+
"""Main class for testing the ISL model with webcam."""
|
| 286 |
+
|
| 287 |
+
def __init__(self):
|
| 288 |
+
self.model_path = MODEL_PATH
|
| 289 |
+
self.labels = DEFAULT_LABELS
|
| 290 |
+
self.input_size = 130
|
| 291 |
+
|
| 292 |
+
# Try to load model and labels
|
| 293 |
+
self._load_model()
|
| 294 |
+
self._load_labels()
|
| 295 |
+
|
| 296 |
+
# Initialize components
|
| 297 |
+
self.processor = LandmarkProcessor(self.input_size)
|
| 298 |
+
self.smoother = PredictionSmoother(SMOOTHING_WINDOW, len(self.labels))
|
| 299 |
+
|
| 300 |
+
# Initialize MediaPipe
|
| 301 |
+
self.mp_hands = mp.solutions.hands
|
| 302 |
+
self.mp_draw = mp.solutions.drawing_utils
|
| 303 |
+
self.hands = self.mp_hands.Hands(
|
| 304 |
+
static_image_mode=False,
|
| 305 |
+
max_num_hands=2,
|
| 306 |
+
min_detection_confidence=0.7,
|
| 307 |
+
min_tracking_confidence=0.5
|
| 308 |
+
)
|
| 309 |
+
|
| 310 |
+
# State
|
| 311 |
+
self.confidence_threshold = CONFIDENCE_THRESHOLD
|
| 312 |
+
self.show_landmarks = SHOW_LANDMARKS
|
| 313 |
+
|
| 314 |
+
def _load_model(self):
|
| 315 |
+
"""Load TFLite model."""
|
| 316 |
+
# Try full model first
|
| 317 |
+
if os.path.exists(MODEL_PATH):
|
| 318 |
+
self.model_path = MODEL_PATH
|
| 319 |
+
elif os.path.exists(FALLBACK_MODEL):
|
| 320 |
+
print(f"⚠️ {MODEL_PATH} not found, using {FALLBACK_MODEL}")
|
| 321 |
+
self.model_path = FALLBACK_MODEL
|
| 322 |
+
else:
|
| 323 |
+
raise FileNotFoundError(f"No model found! Train the model first.")
|
| 324 |
+
|
| 325 |
+
print(f"📦 Loading model: {self.model_path}")
|
| 326 |
+
|
| 327 |
+
self.interpreter = tf.lite.Interpreter(model_path=self.model_path)
|
| 328 |
+
self.interpreter.allocate_tensors()
|
| 329 |
+
|
| 330 |
+
self.input_details = self.interpreter.get_input_details()
|
| 331 |
+
self.output_details = self.interpreter.get_output_details()
|
| 332 |
+
|
| 333 |
+
# Get input size from model
|
| 334 |
+
self.input_size = self.input_details[0]['shape'][1]
|
| 335 |
+
print(f" Input size: {self.input_size}")
|
| 336 |
+
|
| 337 |
+
def _load_labels(self):
|
| 338 |
+
"""Load labels from JSON."""
|
| 339 |
+
labels_path = LABELS_PATH if os.path.exists(LABELS_PATH) else FALLBACK_LABELS
|
| 340 |
+
|
| 341 |
+
if os.path.exists(labels_path):
|
| 342 |
+
with open(labels_path, 'r') as f:
|
| 343 |
+
config = json.load(f)
|
| 344 |
+
self.labels = config.get('labels', DEFAULT_LABELS)
|
| 345 |
+
print(f" Labels loaded: {len(self.labels)} classes")
|
| 346 |
+
else:
|
| 347 |
+
print(f" Using default labels: {len(self.labels)} classes")
|
| 348 |
+
|
| 349 |
+
def predict(self, features):
|
| 350 |
+
"""Run inference on features."""
|
| 351 |
+
input_data = features.reshape(1, -1).astype(np.float32)
|
| 352 |
+
self.interpreter.set_tensor(self.input_details[0]['index'], input_data)
|
| 353 |
+
self.interpreter.invoke()
|
| 354 |
+
output = self.interpreter.get_tensor(self.output_details[0]['index'])
|
| 355 |
+
return output[0]
|
| 356 |
+
|
| 357 |
+
def draw_info_box(self, frame, prediction, confidence, hand_info):
|
| 358 |
+
"""Draw prediction info box on frame."""
|
| 359 |
+
h, w = frame.shape[:2]
|
| 360 |
+
|
| 361 |
+
# Draw semi-transparent background
|
| 362 |
+
overlay = frame.copy()
|
| 363 |
+
cv2.rectangle(overlay, (10, 10), (300, 150), COLOR_BOX, -1)
|
| 364 |
+
cv2.addWeighted(overlay, 0.7, frame, 0.3, 0, frame)
|
| 365 |
+
|
| 366 |
+
# Draw prediction
|
| 367 |
+
if prediction is not None and confidence >= self.confidence_threshold:
|
| 368 |
+
label = self.labels[prediction]
|
| 369 |
+
color = (0, 255, 0) if confidence > 0.8 else (0, 255, 255)
|
| 370 |
+
|
| 371 |
+
cv2.putText(frame, f"Sign: {label}", (20, 50),
|
| 372 |
+
cv2.FONT_HERSHEY_SIMPLEX, 1.2, color, 3)
|
| 373 |
+
cv2.putText(frame, f"Confidence: {confidence*100:.1f}%", (20, 85),
|
| 374 |
+
cv2.FONT_HERSHEY_SIMPLEX, 0.7, COLOR_TEXT, 2)
|
| 375 |
+
else:
|
| 376 |
+
cv2.putText(frame, "No sign detected", (20, 50),
|
| 377 |
+
cv2.FONT_HERSHEY_SIMPLEX, 0.8, (100, 100, 100), 2)
|
| 378 |
+
|
| 379 |
+
# Draw hand info
|
| 380 |
+
y_offset = 115
|
| 381 |
+
if hand_info['hand1']:
|
| 382 |
+
orient = hand_info['hand1_orientation'] or "?"
|
| 383 |
+
cv2.putText(frame, f"Hand 1: {hand_info['hand1']} ({orient})",
|
| 384 |
+
(20, y_offset), cv2.FONT_HERSHEY_SIMPLEX, 0.5, COLOR_TEXT, 1)
|
| 385 |
+
y_offset += 20
|
| 386 |
+
|
| 387 |
+
if hand_info['hand2']:
|
| 388 |
+
orient = hand_info['hand2_orientation'] or "?"
|
| 389 |
+
cv2.putText(frame, f"Hand 2: {hand_info['hand2']} ({orient})",
|
| 390 |
+
(20, y_offset), cv2.FONT_HERSHEY_SIMPLEX, 0.5, COLOR_TEXT, 1)
|
| 391 |
+
|
| 392 |
+
# Draw controls hint
|
| 393 |
+
cv2.putText(frame, "Q: Quit | S: Screenshot | L: Toggle landmarks",
|
| 394 |
+
(10, h - 10), cv2.FONT_HERSHEY_SIMPLEX, 0.4, (150, 150, 150), 1)
|
| 395 |
+
|
| 396 |
+
return frame
|
| 397 |
+
|
| 398 |
+
def draw_landmarks(self, frame, results, hand_info):
|
| 399 |
+
"""Draw hand landmarks with orientation-based coloring."""
|
| 400 |
+
if results.multi_hand_landmarks is None:
|
| 401 |
+
return frame
|
| 402 |
+
|
| 403 |
+
for i, hand_landmarks in enumerate(results.multi_hand_landmarks):
|
| 404 |
+
# Choose color based on orientation
|
| 405 |
+
if i == 0 and hand_info['hand1_orientation']:
|
| 406 |
+
color = COLOR_PALM if hand_info['hand1_orientation'] == 'Palm' else COLOR_BACK
|
| 407 |
+
elif i == 1 and hand_info['hand2_orientation']:
|
| 408 |
+
color = COLOR_PALM if hand_info['hand2_orientation'] == 'Palm' else COLOR_BACK
|
| 409 |
+
else:
|
| 410 |
+
color = (200, 200, 200)
|
| 411 |
+
|
| 412 |
+
# Draw connections
|
| 413 |
+
self.mp_draw.draw_landmarks(
|
| 414 |
+
frame, hand_landmarks, self.mp_hands.HAND_CONNECTIONS,
|
| 415 |
+
self.mp_draw.DrawingSpec(color=color, thickness=2, circle_radius=2),
|
| 416 |
+
self.mp_draw.DrawingSpec(color=color, thickness=2)
|
| 417 |
+
)
|
| 418 |
+
|
| 419 |
+
return frame
|
| 420 |
+
|
| 421 |
+
def run(self):
|
| 422 |
+
"""Main loop for webcam testing."""
|
| 423 |
+
print("\n" + "=" * 60)
|
| 424 |
+
print("ISL MODEL TEST WITH MEDIAPIPE")
|
| 425 |
+
print("=" * 60)
|
| 426 |
+
|
| 427 |
+
print("\n📷 Opening webcam...")
|
| 428 |
+
cap = cv2.VideoCapture(0)
|
| 429 |
+
|
| 430 |
+
if not cap.isOpened():
|
| 431 |
+
print("❌ Error: Could not open webcam!")
|
| 432 |
+
return
|
| 433 |
+
|
| 434 |
+
# Set camera properties
|
| 435 |
+
cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640)
|
| 436 |
+
cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 480)
|
| 437 |
+
cap.set(cv2.CAP_PROP_FPS, 30)
|
| 438 |
+
|
| 439 |
+
print("✅ Webcam opened successfully")
|
| 440 |
+
print("\n🎮 Controls:")
|
| 441 |
+
print(" Press 'q' to quit")
|
| 442 |
+
print(" Press 's' to save screenshot")
|
| 443 |
+
print(" Press 'l' to toggle landmarks")
|
| 444 |
+
print(" Press 'c' to cycle confidence threshold")
|
| 445 |
+
print(" Press 'r' to reset smoother")
|
| 446 |
+
print("\n" + "=" * 60 + "\n")
|
| 447 |
+
|
| 448 |
+
screenshot_count = 0
|
| 449 |
+
|
| 450 |
+
while True:
|
| 451 |
+
ret, frame = cap.read()
|
| 452 |
+
if not ret:
|
| 453 |
+
print("❌ Error reading frame")
|
| 454 |
+
break
|
| 455 |
+
|
| 456 |
+
# Flip for mirror effect
|
| 457 |
+
frame = cv2.flip(frame, 1)
|
| 458 |
+
|
| 459 |
+
# Convert to RGB for MediaPipe
|
| 460 |
+
rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
|
| 461 |
+
|
| 462 |
+
# Process with MediaPipe
|
| 463 |
+
results = self.hands.process(rgb_frame)
|
| 464 |
+
|
| 465 |
+
# Extract features
|
| 466 |
+
features, hand_info = self.processor.process_hands(results)
|
| 467 |
+
|
| 468 |
+
# Run inference if hand detected
|
| 469 |
+
prediction = None
|
| 470 |
+
confidence = 0.0
|
| 471 |
+
|
| 472 |
+
if hand_info['hand1'] is not None:
|
| 473 |
+
probs = self.predict(features)
|
| 474 |
+
self.smoother.add_prediction(probs)
|
| 475 |
+
prediction, confidence = self.smoother.get_smoothed_prediction()
|
| 476 |
+
else:
|
| 477 |
+
self.smoother.clear()
|
| 478 |
+
|
| 479 |
+
# Draw landmarks if enabled
|
| 480 |
+
if self.show_landmarks:
|
| 481 |
+
frame = self.draw_landmarks(frame, results, hand_info)
|
| 482 |
+
|
| 483 |
+
# Draw info box
|
| 484 |
+
frame = self.draw_info_box(frame, prediction, confidence, hand_info)
|
| 485 |
+
|
| 486 |
+
# Show frame
|
| 487 |
+
cv2.imshow('ISL Model Test', frame)
|
| 488 |
+
|
| 489 |
+
# Handle key presses
|
| 490 |
+
key = cv2.waitKey(1) & 0xFF
|
| 491 |
+
|
| 492 |
+
if key == ord('q'):
|
| 493 |
+
print("\n👋 Exiting...")
|
| 494 |
+
break
|
| 495 |
+
elif key == ord('s'):
|
| 496 |
+
filename = f"screenshot_{screenshot_count}.png"
|
| 497 |
+
cv2.imwrite(filename, frame)
|
| 498 |
+
print(f"📸 Screenshot saved: {filename}")
|
| 499 |
+
screenshot_count += 1
|
| 500 |
+
elif key == ord('l'):
|
| 501 |
+
self.show_landmarks = not self.show_landmarks
|
| 502 |
+
print(f"🎯 Landmarks: {'ON' if self.show_landmarks else 'OFF'}")
|
| 503 |
+
elif key == ord('c'):
|
| 504 |
+
# Cycle threshold: 0.5 -> 0.7 -> 0.9 -> 0.3 -> 0.5
|
| 505 |
+
thresholds = [0.3, 0.5, 0.7, 0.9]
|
| 506 |
+
idx = thresholds.index(self.confidence_threshold) if self.confidence_threshold in thresholds else 0
|
| 507 |
+
self.confidence_threshold = thresholds[(idx + 1) % len(thresholds)]
|
| 508 |
+
print(f"📊 Confidence threshold: {self.confidence_threshold}")
|
| 509 |
+
elif key == ord('r'):
|
| 510 |
+
self.smoother.clear()
|
| 511 |
+
print("🔄 Smoother reset")
|
| 512 |
+
|
| 513 |
+
cap.release()
|
| 514 |
+
cv2.destroyAllWindows()
|
| 515 |
+
print("\n✅ Test complete!")
|
| 516 |
+
|
| 517 |
+
|
| 518 |
+
# ============================================================================
|
| 519 |
+
# BATCH TEST MODE
|
| 520 |
+
# ============================================================================
|
| 521 |
+
|
| 522 |
+
def test_on_images(image_folder):
|
| 523 |
+
"""Test model on a folder of images."""
|
| 524 |
+
print("\n" + "=" * 60)
|
| 525 |
+
print("BATCH IMAGE TEST")
|
| 526 |
+
print("=" * 60)
|
| 527 |
+
|
| 528 |
+
tester = ISLModelTester()
|
| 529 |
+
|
| 530 |
+
results = []
|
| 531 |
+
|
| 532 |
+
for filename in os.listdir(image_folder):
|
| 533 |
+
if filename.lower().endswith(('.png', '.jpg', '.jpeg')):
|
| 534 |
+
filepath = os.path.join(image_folder, filename)
|
| 535 |
+
|
| 536 |
+
# Load image
|
| 537 |
+
frame = cv2.imread(filepath)
|
| 538 |
+
if frame is None:
|
| 539 |
+
continue
|
| 540 |
+
|
| 541 |
+
# Process
|
| 542 |
+
rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
|
| 543 |
+
mp_results = tester.hands.process(rgb_frame)
|
| 544 |
+
|
| 545 |
+
features, hand_info = tester.processor.process_hands(mp_results)
|
| 546 |
+
|
| 547 |
+
if hand_info['hand1'] is not None:
|
| 548 |
+
probs = tester.predict(features)
|
| 549 |
+
pred_class = np.argmax(probs)
|
| 550 |
+
confidence = probs[pred_class]
|
| 551 |
+
|
| 552 |
+
results.append({
|
| 553 |
+
'file': filename,
|
| 554 |
+
'prediction': tester.labels[pred_class],
|
| 555 |
+
'confidence': confidence
|
| 556 |
+
})
|
| 557 |
+
|
| 558 |
+
print(f" {filename}: {tester.labels[pred_class]} ({confidence*100:.1f}%)")
|
| 559 |
+
else:
|
| 560 |
+
print(f" {filename}: No hand detected")
|
| 561 |
+
|
| 562 |
+
return results
|
| 563 |
+
|
| 564 |
+
|
| 565 |
+
# ============================================================================
|
| 566 |
+
# MAIN
|
| 567 |
+
# ============================================================================
|
| 568 |
+
|
| 569 |
+
def main():
|
| 570 |
+
import sys
|
| 571 |
+
|
| 572 |
+
if len(sys.argv) > 1:
|
| 573 |
+
# Batch mode on folder
|
| 574 |
+
folder = sys.argv[1]
|
| 575 |
+
if os.path.isdir(folder):
|
| 576 |
+
test_on_images(folder)
|
| 577 |
+
else:
|
| 578 |
+
print(f"❌ Folder not found: {folder}")
|
| 579 |
+
else:
|
| 580 |
+
# Webcam mode
|
| 581 |
+
tester = ISLModelTester()
|
| 582 |
+
tester.run()
|
| 583 |
+
|
| 584 |
+
|
| 585 |
+
if __name__ == "__main__":
|
| 586 |
+
main()
|
train_advanced_model.py
ADDED
|
@@ -0,0 +1,1093 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
================================================================================
|
| 3 |
+
ADVANCED ISL MODEL TRAINING - MAXIMUM ACCURACY
|
| 4 |
+
================================================================================
|
| 5 |
+
State-of-the-art techniques for achieving 9-10/10 accuracy:
|
| 6 |
+
|
| 7 |
+
1. Attention Mechanism - Focus on important landmarks
|
| 8 |
+
2. Focal Loss - Better handling of hard examples
|
| 9 |
+
3. Advanced Augmentation - Realistic variations
|
| 10 |
+
4. Residual Connections - Better gradient flow
|
| 11 |
+
5. Class Balancing - Handle imbalanced classes
|
| 12 |
+
6. Hard Example Mining - Focus on confused pairs
|
| 13 |
+
7. Multi-Scale Features - Capture different patterns
|
| 14 |
+
8. Test-Time Augmentation (TTA) - Better inference
|
| 15 |
+
9. Ensemble Ready - Can combine multiple models
|
| 16 |
+
10. Gradual Unfreezing with Warmup
|
| 17 |
+
|
| 18 |
+
Target: 99.5%+ validation accuracy with real-world robustness
|
| 19 |
+
|
| 20 |
+
Author: KairoAI
|
| 21 |
+
================================================================================
|
| 22 |
+
"""
|
| 23 |
+
|
| 24 |
+
import os
|
| 25 |
+
import numpy as np
|
| 26 |
+
import json
|
| 27 |
+
import csv
|
| 28 |
+
import tensorflow as tf
|
| 29 |
+
from tensorflow import keras
|
| 30 |
+
from tensorflow.keras import layers, regularizers, Model
|
| 31 |
+
from tensorflow.keras.callbacks import Callback
|
| 32 |
+
from sklearn.model_selection import StratifiedKFold, train_test_split
|
| 33 |
+
from sklearn.utils.class_weight import compute_class_weight
|
| 34 |
+
from sklearn.metrics import classification_report, confusion_matrix
|
| 35 |
+
import math
|
| 36 |
+
|
| 37 |
+
# Suppress warnings
|
| 38 |
+
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
|
| 39 |
+
|
| 40 |
+
# ============================================================================
|
| 41 |
+
# CONFIGURATION
|
| 42 |
+
# ============================================================================
|
| 43 |
+
|
| 44 |
+
INPUT_CSV = "landmark_dataset_with_orientation.csv"
|
| 45 |
+
OUTPUT_MODEL = "isl_model_advanced.tflite"
|
| 46 |
+
OUTPUT_H5 = "isl_model_advanced.h5"
|
| 47 |
+
OUTPUT_LABELS = "labels_advanced.json"
|
| 48 |
+
|
| 49 |
+
NUM_CLASSES = 35
|
| 50 |
+
|
| 51 |
+
# Training config
|
| 52 |
+
EPOCHS = 200
|
| 53 |
+
BATCH_SIZE = 64
|
| 54 |
+
VALIDATION_SPLIT = 0.15 # Smaller val split = more training data
|
| 55 |
+
|
| 56 |
+
# Advanced config
|
| 57 |
+
USE_FOCAL_LOSS = True
|
| 58 |
+
FOCAL_ALPHA = 0.25
|
| 59 |
+
FOCAL_GAMMA = 2.0
|
| 60 |
+
|
| 61 |
+
USE_CLASS_WEIGHTS = True
|
| 62 |
+
USE_ATTENTION = True
|
| 63 |
+
USE_RESIDUAL = True
|
| 64 |
+
|
| 65 |
+
# Augmentation strength
|
| 66 |
+
AUG_NOISE_STD = 0.03
|
| 67 |
+
AUG_SCALE_RANGE = (0.85, 1.15)
|
| 68 |
+
AUG_ROTATION_RANGE = 20 # degrees
|
| 69 |
+
AUG_SHIFT_RANGE = 0.1
|
| 70 |
+
MIXUP_ALPHA = 0.3
|
| 71 |
+
CUTMIX_ALPHA = 0.3
|
| 72 |
+
|
| 73 |
+
# Learning rate
|
| 74 |
+
INITIAL_LR = 0.002
|
| 75 |
+
MIN_LR = 1e-7
|
| 76 |
+
WARMUP_EPOCHS = 10
|
| 77 |
+
|
| 78 |
+
LABELS = [
|
| 79 |
+
'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J',
|
| 80 |
+
'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T',
|
| 81 |
+
'U', 'V', 'W', 'X', 'Y', 'Z', '1', '2', '3', '4',
|
| 82 |
+
'5', '6', '7', '8', '9'
|
| 83 |
+
]
|
| 84 |
+
|
| 85 |
+
# Commonly confused pairs - will get extra training attention
|
| 86 |
+
CONFUSED_PAIRS = [
|
| 87 |
+
('V', '2'), ('K', 'V'), ('U', 'V'), ('M', 'N'), ('5', 'H'),
|
| 88 |
+
('G', 'H'), ('I', 'J'), ('1', 'D'), ('6', 'W'), ('3', 'W'),
|
| 89 |
+
('B', 'E'), ('A', 'S'), ('O', 'C')
|
| 90 |
+
]
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
# ============================================================================
|
| 94 |
+
# FOCAL LOSS - Better for hard examples
|
| 95 |
+
# ============================================================================
|
| 96 |
+
|
| 97 |
+
class FocalLoss(keras.losses.Loss):
|
| 98 |
+
"""
|
| 99 |
+
Focal Loss focuses training on hard examples.
|
| 100 |
+
Reduces loss contribution from easy examples.
|
| 101 |
+
"""
|
| 102 |
+
def __init__(self, alpha=0.25, gamma=2.0, **kwargs):
|
| 103 |
+
super().__init__(**kwargs)
|
| 104 |
+
self.alpha = alpha
|
| 105 |
+
self.gamma = gamma
|
| 106 |
+
|
| 107 |
+
def call(self, y_true, y_pred):
|
| 108 |
+
# Clip predictions to prevent log(0)
|
| 109 |
+
y_pred = tf.clip_by_value(y_pred, 1e-7, 1 - 1e-7)
|
| 110 |
+
|
| 111 |
+
# Calculate focal loss
|
| 112 |
+
cross_entropy = -y_true * tf.math.log(y_pred)
|
| 113 |
+
weight = self.alpha * y_true * tf.pow(1 - y_pred, self.gamma)
|
| 114 |
+
focal_loss = weight * cross_entropy
|
| 115 |
+
|
| 116 |
+
return tf.reduce_sum(focal_loss, axis=-1)
|
| 117 |
+
|
| 118 |
+
def get_config(self):
|
| 119 |
+
config = super().get_config()
|
| 120 |
+
config.update({'alpha': self.alpha, 'gamma': self.gamma})
|
| 121 |
+
return config
|
| 122 |
+
|
| 123 |
+
|
| 124 |
+
# ============================================================================
|
| 125 |
+
# ATTENTION LAYER - Focus on important features
|
| 126 |
+
# ============================================================================
|
| 127 |
+
|
| 128 |
+
class SelfAttention(layers.Layer):
|
| 129 |
+
"""Self-attention layer to focus on important landmarks."""
|
| 130 |
+
|
| 131 |
+
def __init__(self, units, **kwargs):
|
| 132 |
+
super().__init__(**kwargs)
|
| 133 |
+
self.units = units
|
| 134 |
+
|
| 135 |
+
def build(self, input_shape):
|
| 136 |
+
self.W_q = self.add_weight(
|
| 137 |
+
shape=(input_shape[-1], self.units),
|
| 138 |
+
initializer='glorot_uniform',
|
| 139 |
+
trainable=True,
|
| 140 |
+
name='query_weight'
|
| 141 |
+
)
|
| 142 |
+
self.W_k = self.add_weight(
|
| 143 |
+
shape=(input_shape[-1], self.units),
|
| 144 |
+
initializer='glorot_uniform',
|
| 145 |
+
trainable=True,
|
| 146 |
+
name='key_weight'
|
| 147 |
+
)
|
| 148 |
+
self.W_v = self.add_weight(
|
| 149 |
+
shape=(input_shape[-1], self.units),
|
| 150 |
+
initializer='glorot_uniform',
|
| 151 |
+
trainable=True,
|
| 152 |
+
name='value_weight'
|
| 153 |
+
)
|
| 154 |
+
|
| 155 |
+
def call(self, x):
|
| 156 |
+
q = tf.matmul(x, self.W_q)
|
| 157 |
+
k = tf.matmul(x, self.W_k)
|
| 158 |
+
v = tf.matmul(x, self.W_v)
|
| 159 |
+
|
| 160 |
+
# Scaled dot-product attention
|
| 161 |
+
d_k = tf.cast(tf.shape(k)[-1], tf.float32)
|
| 162 |
+
attention_scores = tf.matmul(q, k, transpose_b=True) / tf.sqrt(d_k)
|
| 163 |
+
attention_weights = tf.nn.softmax(attention_scores, axis=-1)
|
| 164 |
+
|
| 165 |
+
output = tf.matmul(attention_weights, v)
|
| 166 |
+
return output
|
| 167 |
+
|
| 168 |
+
def get_config(self):
|
| 169 |
+
config = super().get_config()
|
| 170 |
+
config.update({'units': self.units})
|
| 171 |
+
return config
|
| 172 |
+
|
| 173 |
+
|
| 174 |
+
class ChannelAttention(layers.Layer):
|
| 175 |
+
"""Channel attention - learns which features are important."""
|
| 176 |
+
|
| 177 |
+
def __init__(self, reduction_ratio=8, **kwargs):
|
| 178 |
+
super().__init__(**kwargs)
|
| 179 |
+
self.reduction_ratio = reduction_ratio
|
| 180 |
+
|
| 181 |
+
def build(self, input_shape):
|
| 182 |
+
channels = input_shape[-1]
|
| 183 |
+
self.dense1 = layers.Dense(channels // self.reduction_ratio, activation='relu')
|
| 184 |
+
self.dense2 = layers.Dense(channels, activation='sigmoid')
|
| 185 |
+
|
| 186 |
+
def call(self, x):
|
| 187 |
+
# Global average pooling
|
| 188 |
+
avg_pool = tf.reduce_mean(x, axis=-1, keepdims=True)
|
| 189 |
+
max_pool = tf.reduce_max(x, axis=-1, keepdims=True)
|
| 190 |
+
|
| 191 |
+
# Shared MLP
|
| 192 |
+
avg_out = self.dense2(self.dense1(avg_pool))
|
| 193 |
+
max_out = self.dense2(self.dense1(max_pool))
|
| 194 |
+
|
| 195 |
+
attention = avg_out + max_out
|
| 196 |
+
return x * attention
|
| 197 |
+
|
| 198 |
+
def get_config(self):
|
| 199 |
+
config = super().get_config()
|
| 200 |
+
config.update({'reduction_ratio': self.reduction_ratio})
|
| 201 |
+
return config
|
| 202 |
+
|
| 203 |
+
|
| 204 |
+
# ============================================================================
|
| 205 |
+
# RESIDUAL BLOCK
|
| 206 |
+
# ============================================================================
|
| 207 |
+
|
| 208 |
+
class ResidualBlock(layers.Layer):
|
| 209 |
+
"""Residual block with skip connection for better gradient flow."""
|
| 210 |
+
|
| 211 |
+
def __init__(self, units, dropout_rate=0.3, l2_reg=0.001, **kwargs):
|
| 212 |
+
super().__init__(**kwargs)
|
| 213 |
+
self.units = units
|
| 214 |
+
self.dropout_rate = dropout_rate
|
| 215 |
+
self.l2_reg = l2_reg
|
| 216 |
+
|
| 217 |
+
def build(self, input_shape):
|
| 218 |
+
self.dense1 = layers.Dense(
|
| 219 |
+
self.units,
|
| 220 |
+
kernel_regularizer=regularizers.l2(self.l2_reg)
|
| 221 |
+
)
|
| 222 |
+
self.bn1 = layers.BatchNormalization()
|
| 223 |
+
self.dropout1 = layers.Dropout(self.dropout_rate)
|
| 224 |
+
|
| 225 |
+
self.dense2 = layers.Dense(
|
| 226 |
+
self.units,
|
| 227 |
+
kernel_regularizer=regularizers.l2(self.l2_reg)
|
| 228 |
+
)
|
| 229 |
+
self.bn2 = layers.BatchNormalization()
|
| 230 |
+
self.dropout2 = layers.Dropout(self.dropout_rate)
|
| 231 |
+
|
| 232 |
+
# Skip connection projection if dimensions don't match
|
| 233 |
+
if input_shape[-1] != self.units:
|
| 234 |
+
self.skip_proj = layers.Dense(self.units, use_bias=False)
|
| 235 |
+
else:
|
| 236 |
+
self.skip_proj = None
|
| 237 |
+
|
| 238 |
+
def call(self, x, training=False):
|
| 239 |
+
# Main path
|
| 240 |
+
h = self.dense1(x)
|
| 241 |
+
h = self.bn1(h, training=training)
|
| 242 |
+
h = tf.nn.gelu(h) # GELU activation (better than ReLU)
|
| 243 |
+
h = self.dropout1(h, training=training)
|
| 244 |
+
|
| 245 |
+
h = self.dense2(h)
|
| 246 |
+
h = self.bn2(h, training=training)
|
| 247 |
+
|
| 248 |
+
# Skip connection
|
| 249 |
+
if self.skip_proj is not None:
|
| 250 |
+
x = self.skip_proj(x)
|
| 251 |
+
|
| 252 |
+
# Add and activate
|
| 253 |
+
out = tf.nn.gelu(h + x)
|
| 254 |
+
out = self.dropout2(out, training=training)
|
| 255 |
+
|
| 256 |
+
return out
|
| 257 |
+
|
| 258 |
+
def get_config(self):
|
| 259 |
+
config = super().get_config()
|
| 260 |
+
config.update({
|
| 261 |
+
'units': self.units,
|
| 262 |
+
'dropout_rate': self.dropout_rate,
|
| 263 |
+
'l2_reg': self.l2_reg
|
| 264 |
+
})
|
| 265 |
+
return config
|
| 266 |
+
|
| 267 |
+
|
| 268 |
+
# ============================================================================
|
| 269 |
+
# ADVANCED DATA AUGMENTATION
|
| 270 |
+
# ============================================================================
|
| 271 |
+
|
| 272 |
+
class AdvancedAugmenter:
|
| 273 |
+
"""Advanced augmentation for landmark data."""
|
| 274 |
+
|
| 275 |
+
def __init__(self, noise_std=0.03, scale_range=(0.85, 1.15),
|
| 276 |
+
rotation_range=20, shift_range=0.1):
|
| 277 |
+
self.noise_std = noise_std
|
| 278 |
+
self.scale_range = scale_range
|
| 279 |
+
self.rotation_range = rotation_range
|
| 280 |
+
self.shift_range = shift_range
|
| 281 |
+
|
| 282 |
+
def add_noise(self, X):
|
| 283 |
+
"""Add Gaussian noise."""
|
| 284 |
+
noise = np.random.normal(0, self.noise_std, X.shape)
|
| 285 |
+
# Don't add noise to orientation features
|
| 286 |
+
if X.shape[1] > 126:
|
| 287 |
+
noise[:, 126:] = 0
|
| 288 |
+
return X + noise
|
| 289 |
+
|
| 290 |
+
def random_scale(self, X):
|
| 291 |
+
"""Random scaling of landmarks."""
|
| 292 |
+
scale = np.random.uniform(*self.scale_range, size=(len(X), 1))
|
| 293 |
+
X_scaled = X.copy()
|
| 294 |
+
X_scaled[:, :126] *= scale
|
| 295 |
+
return X_scaled
|
| 296 |
+
|
| 297 |
+
def random_shift(self, X):
|
| 298 |
+
"""Random translation of landmarks."""
|
| 299 |
+
shift = np.random.uniform(-self.shift_range, self.shift_range,
|
| 300 |
+
size=(len(X), 3))
|
| 301 |
+
X_shifted = X.copy()
|
| 302 |
+
# Apply same shift to all landmarks (x, y, z)
|
| 303 |
+
for i in range(21):
|
| 304 |
+
X_shifted[:, i*3:(i+1)*3] += shift
|
| 305 |
+
# Second hand if present
|
| 306 |
+
for i in range(21, 42):
|
| 307 |
+
X_shifted[:, i*3:(i+1)*3] += shift
|
| 308 |
+
return X_shifted
|
| 309 |
+
|
| 310 |
+
def random_rotation_2d(self, X):
|
| 311 |
+
"""Random 2D rotation around z-axis."""
|
| 312 |
+
angles = np.random.uniform(-self.rotation_range, self.rotation_range,
|
| 313 |
+
size=len(X)) * np.pi / 180
|
| 314 |
+
|
| 315 |
+
X_rotated = X.copy()
|
| 316 |
+
|
| 317 |
+
for i, angle in enumerate(angles):
|
| 318 |
+
cos_a, sin_a = np.cos(angle), np.sin(angle)
|
| 319 |
+
|
| 320 |
+
# Rotate each landmark (x, y only)
|
| 321 |
+
for j in range(42): # 21 landmarks × 2 hands
|
| 322 |
+
if j * 3 + 2 < 126:
|
| 323 |
+
x = X_rotated[i, j*3]
|
| 324 |
+
y = X_rotated[i, j*3 + 1]
|
| 325 |
+
X_rotated[i, j*3] = x * cos_a - y * sin_a
|
| 326 |
+
X_rotated[i, j*3 + 1] = x * sin_a + y * cos_a
|
| 327 |
+
|
| 328 |
+
return X_rotated
|
| 329 |
+
|
| 330 |
+
def random_mirror(self, X):
|
| 331 |
+
"""Randomly mirror hands (swap left/right)."""
|
| 332 |
+
X_mirrored = X.copy()
|
| 333 |
+
mask = np.random.random(len(X)) < 0.3 # 30% chance
|
| 334 |
+
|
| 335 |
+
for i in np.where(mask)[0]:
|
| 336 |
+
# Mirror x coordinates
|
| 337 |
+
for j in range(42):
|
| 338 |
+
if j * 3 < 126:
|
| 339 |
+
X_mirrored[i, j*3] = -X_mirrored[i, j*3]
|
| 340 |
+
|
| 341 |
+
# Swap hand orientation if present
|
| 342 |
+
if X.shape[1] > 126:
|
| 343 |
+
# Swap is_left flags
|
| 344 |
+
X_mirrored[i, 127] = 1.0 - X_mirrored[i, 127] if X_mirrored[i, 127] >= 0 else -1.0
|
| 345 |
+
X_mirrored[i, 129] = 1.0 - X_mirrored[i, 129] if X_mirrored[i, 129] >= 0 else -1.0
|
| 346 |
+
|
| 347 |
+
return X_mirrored
|
| 348 |
+
|
| 349 |
+
def random_finger_jitter(self, X):
|
| 350 |
+
"""Add extra jitter to finger tips (most variable landmarks)."""
|
| 351 |
+
X_jittered = X.copy()
|
| 352 |
+
finger_tips = [4, 8, 12, 16, 20] # Thumb, index, middle, ring, pinky tips
|
| 353 |
+
|
| 354 |
+
for tip in finger_tips:
|
| 355 |
+
jitter = np.random.normal(0, self.noise_std * 2, (len(X), 3))
|
| 356 |
+
X_jittered[:, tip*3:(tip+1)*3] += jitter
|
| 357 |
+
# Second hand
|
| 358 |
+
if (tip + 21) * 3 + 3 <= 126:
|
| 359 |
+
X_jittered[:, (tip+21)*3:(tip+22)*3] += jitter
|
| 360 |
+
|
| 361 |
+
return X_jittered
|
| 362 |
+
|
| 363 |
+
def augment_batch(self, X, p=0.5):
|
| 364 |
+
"""Apply random augmentations to batch."""
|
| 365 |
+
X_aug = X.copy()
|
| 366 |
+
|
| 367 |
+
# Apply each augmentation with probability p
|
| 368 |
+
if np.random.random() < p:
|
| 369 |
+
X_aug = self.add_noise(X_aug)
|
| 370 |
+
if np.random.random() < p * 0.7:
|
| 371 |
+
X_aug = self.random_scale(X_aug)
|
| 372 |
+
if np.random.random() < p * 0.5:
|
| 373 |
+
X_aug = self.random_rotation_2d(X_aug)
|
| 374 |
+
if np.random.random() < p * 0.3:
|
| 375 |
+
X_aug = self.random_shift(X_aug)
|
| 376 |
+
if np.random.random() < p * 0.3:
|
| 377 |
+
X_aug = self.random_finger_jitter(X_aug)
|
| 378 |
+
if np.random.random() < p * 0.2:
|
| 379 |
+
X_aug = self.random_mirror(X_aug)
|
| 380 |
+
|
| 381 |
+
return X_aug
|
| 382 |
+
|
| 383 |
+
|
| 384 |
+
# ============================================================================
|
| 385 |
+
# MIXUP AND CUTMIX
|
| 386 |
+
# ============================================================================
|
| 387 |
+
|
| 388 |
+
def mixup(X, y, alpha=0.3):
|
| 389 |
+
"""Mixup augmentation."""
|
| 390 |
+
if alpha <= 0:
|
| 391 |
+
return X, y
|
| 392 |
+
|
| 393 |
+
batch_size = len(X)
|
| 394 |
+
lam = np.random.beta(alpha, alpha)
|
| 395 |
+
indices = np.random.permutation(batch_size)
|
| 396 |
+
|
| 397 |
+
X_mixed = lam * X + (1 - lam) * X[indices]
|
| 398 |
+
y_mixed = lam * y + (1 - lam) * y[indices]
|
| 399 |
+
|
| 400 |
+
return X_mixed, y_mixed
|
| 401 |
+
|
| 402 |
+
|
| 403 |
+
def cutmix(X, y, alpha=0.3):
|
| 404 |
+
"""CutMix augmentation - mix portions of features."""
|
| 405 |
+
if alpha <= 0:
|
| 406 |
+
return X, y
|
| 407 |
+
|
| 408 |
+
batch_size = len(X)
|
| 409 |
+
lam = np.random.beta(alpha, alpha)
|
| 410 |
+
indices = np.random.permutation(batch_size)
|
| 411 |
+
|
| 412 |
+
# Determine cut size
|
| 413 |
+
feature_size = X.shape[1]
|
| 414 |
+
cut_size = int(feature_size * (1 - lam))
|
| 415 |
+
cut_start = np.random.randint(0, feature_size - cut_size + 1)
|
| 416 |
+
|
| 417 |
+
X_mixed = X.copy()
|
| 418 |
+
X_mixed[:, cut_start:cut_start+cut_size] = X[indices, cut_start:cut_start+cut_size]
|
| 419 |
+
|
| 420 |
+
# Adjust lambda based on actual cut
|
| 421 |
+
lam = 1 - cut_size / feature_size
|
| 422 |
+
y_mixed = lam * y + (1 - lam) * y[indices]
|
| 423 |
+
|
| 424 |
+
return X_mixed, y_mixed
|
| 425 |
+
|
| 426 |
+
|
| 427 |
+
# ============================================================================
|
| 428 |
+
# ADVANCED DATA GENERATOR
|
| 429 |
+
# ============================================================================
|
| 430 |
+
|
| 431 |
+
class AdvancedDataGenerator(keras.utils.Sequence):
|
| 432 |
+
"""Advanced data generator with multiple augmentation techniques."""
|
| 433 |
+
|
| 434 |
+
def __init__(self, X, y, batch_size, augmenter, num_classes,
|
| 435 |
+
use_mixup=True, use_cutmix=True, shuffle=True,
|
| 436 |
+
hard_example_indices=None, hard_example_ratio=0.3):
|
| 437 |
+
self.X = X
|
| 438 |
+
self.y = y
|
| 439 |
+
self.batch_size = batch_size
|
| 440 |
+
self.augmenter = augmenter
|
| 441 |
+
self.num_classes = num_classes
|
| 442 |
+
self.use_mixup = use_mixup
|
| 443 |
+
self.use_cutmix = use_cutmix
|
| 444 |
+
self.shuffle = shuffle
|
| 445 |
+
self.hard_example_indices = hard_example_indices
|
| 446 |
+
self.hard_example_ratio = hard_example_ratio
|
| 447 |
+
self.indices = np.arange(len(X))
|
| 448 |
+
self.on_epoch_end()
|
| 449 |
+
|
| 450 |
+
def __len__(self):
|
| 451 |
+
return int(np.ceil(len(self.X) / self.batch_size))
|
| 452 |
+
|
| 453 |
+
def __getitem__(self, idx):
|
| 454 |
+
batch_indices = self.indices[idx * self.batch_size:(idx + 1) * self.batch_size]
|
| 455 |
+
|
| 456 |
+
# Optionally include hard examples
|
| 457 |
+
if self.hard_example_indices is not None and len(self.hard_example_indices) > 0:
|
| 458 |
+
n_hard = int(len(batch_indices) * self.hard_example_ratio)
|
| 459 |
+
hard_sample = np.random.choice(self.hard_example_indices,
|
| 460 |
+
size=min(n_hard, len(self.hard_example_indices)),
|
| 461 |
+
replace=False)
|
| 462 |
+
batch_indices = np.concatenate([batch_indices[:-n_hard], hard_sample])
|
| 463 |
+
|
| 464 |
+
X_batch = self.X[batch_indices].copy()
|
| 465 |
+
y_batch = self.y[batch_indices].copy()
|
| 466 |
+
|
| 467 |
+
# Apply augmentation
|
| 468 |
+
X_batch = self.augmenter.augment_batch(X_batch, p=0.7)
|
| 469 |
+
|
| 470 |
+
# Convert to one-hot
|
| 471 |
+
y_one_hot = keras.utils.to_categorical(y_batch, self.num_classes)
|
| 472 |
+
|
| 473 |
+
# Apply mixup or cutmix (not both at same time)
|
| 474 |
+
if self.use_mixup and self.use_cutmix:
|
| 475 |
+
if np.random.random() < 0.5:
|
| 476 |
+
X_batch, y_one_hot = mixup(X_batch, y_one_hot, MIXUP_ALPHA)
|
| 477 |
+
else:
|
| 478 |
+
X_batch, y_one_hot = cutmix(X_batch, y_one_hot, CUTMIX_ALPHA)
|
| 479 |
+
elif self.use_mixup:
|
| 480 |
+
X_batch, y_one_hot = mixup(X_batch, y_one_hot, MIXUP_ALPHA)
|
| 481 |
+
elif self.use_cutmix:
|
| 482 |
+
X_batch, y_one_hot = cutmix(X_batch, y_one_hot, CUTMIX_ALPHA)
|
| 483 |
+
|
| 484 |
+
return X_batch.astype(np.float32), y_one_hot.astype(np.float32)
|
| 485 |
+
|
| 486 |
+
def on_epoch_end(self):
|
| 487 |
+
if self.shuffle:
|
| 488 |
+
np.random.shuffle(self.indices)
|
| 489 |
+
|
| 490 |
+
|
| 491 |
+
# ============================================================================
|
| 492 |
+
# CREATE ADVANCED MODEL
|
| 493 |
+
# ============================================================================
|
| 494 |
+
|
| 495 |
+
def create_advanced_model(input_size):
|
| 496 |
+
"""
|
| 497 |
+
Create an advanced model with attention and residual connections.
|
| 498 |
+
"""
|
| 499 |
+
print("\n" + "=" * 60)
|
| 500 |
+
print("CREATING ADVANCED MODEL")
|
| 501 |
+
print("=" * 60)
|
| 502 |
+
|
| 503 |
+
l2_reg = regularizers.l2(0.001)
|
| 504 |
+
|
| 505 |
+
# Input
|
| 506 |
+
inputs = layers.Input(shape=(input_size,), name='input')
|
| 507 |
+
|
| 508 |
+
# Split landmarks and orientation
|
| 509 |
+
landmarks = layers.Lambda(lambda x: x[:, :126], name='landmarks')(inputs)
|
| 510 |
+
|
| 511 |
+
if input_size > 126:
|
| 512 |
+
orientation = layers.Lambda(lambda x: x[:, 126:], name='orientation')(inputs)
|
| 513 |
+
|
| 514 |
+
# ===== LANDMARK PROCESSING PATH =====
|
| 515 |
+
|
| 516 |
+
# Reshape for multi-scale processing (treat as 42 landmarks × 3 coords)
|
| 517 |
+
lm_reshaped = layers.Reshape((42, 3))(landmarks)
|
| 518 |
+
|
| 519 |
+
# Multi-scale feature extraction
|
| 520 |
+
# Scale 1: Per-landmark features
|
| 521 |
+
scale1 = layers.Dense(16, activation='gelu', kernel_regularizer=l2_reg)(lm_reshaped)
|
| 522 |
+
scale1 = layers.Flatten()(scale1)
|
| 523 |
+
|
| 524 |
+
# Scale 2: Landmark group features (fingers)
|
| 525 |
+
scale2 = layers.Conv1D(32, kernel_size=5, activation='gelu', padding='same',
|
| 526 |
+
kernel_regularizer=l2_reg)(lm_reshaped)
|
| 527 |
+
scale2 = layers.GlobalAveragePooling1D()(scale2)
|
| 528 |
+
|
| 529 |
+
# Scale 3: Global hand features
|
| 530 |
+
scale3 = layers.Dense(64, activation='gelu', kernel_regularizer=l2_reg)(landmarks)
|
| 531 |
+
|
| 532 |
+
# Combine scales
|
| 533 |
+
lm_features = layers.Concatenate()([scale1, scale2, scale3])
|
| 534 |
+
lm_features = layers.BatchNormalization()(lm_features)
|
| 535 |
+
lm_features = layers.Dropout(0.3)(lm_features)
|
| 536 |
+
|
| 537 |
+
# Channel attention
|
| 538 |
+
if USE_ATTENTION:
|
| 539 |
+
lm_features = ChannelAttention(reduction_ratio=4)(lm_features)
|
| 540 |
+
|
| 541 |
+
# Residual blocks
|
| 542 |
+
if USE_RESIDUAL:
|
| 543 |
+
x = ResidualBlock(256, dropout_rate=0.4, l2_reg=0.001)(lm_features)
|
| 544 |
+
x = ResidualBlock(128, dropout_rate=0.3, l2_reg=0.001)(x)
|
| 545 |
+
x = ResidualBlock(64, dropout_rate=0.3, l2_reg=0.001)(x)
|
| 546 |
+
else:
|
| 547 |
+
x = layers.Dense(256, activation='gelu', kernel_regularizer=l2_reg)(lm_features)
|
| 548 |
+
x = layers.BatchNormalization()(x)
|
| 549 |
+
x = layers.Dropout(0.4)(x)
|
| 550 |
+
x = layers.Dense(128, activation='gelu', kernel_regularizer=l2_reg)(x)
|
| 551 |
+
x = layers.BatchNormalization()(x)
|
| 552 |
+
x = layers.Dropout(0.3)(x)
|
| 553 |
+
x = layers.Dense(64, activation='gelu', kernel_regularizer=l2_reg)(x)
|
| 554 |
+
x = layers.BatchNormalization()(x)
|
| 555 |
+
x = layers.Dropout(0.3)(x)
|
| 556 |
+
|
| 557 |
+
# ===== ORIENTATION PATH (if available) =====
|
| 558 |
+
if input_size > 126:
|
| 559 |
+
orient_x = layers.Dense(16, activation='gelu', kernel_regularizer=l2_reg)(orientation)
|
| 560 |
+
orient_x = layers.BatchNormalization()(orient_x)
|
| 561 |
+
orient_x = layers.Dense(8, activation='gelu', kernel_regularizer=l2_reg)(orient_x)
|
| 562 |
+
|
| 563 |
+
# Combine with main path
|
| 564 |
+
x = layers.Concatenate()([x, orient_x])
|
| 565 |
+
|
| 566 |
+
# ===== CLASSIFICATION HEAD =====
|
| 567 |
+
x = layers.Dense(64, activation='gelu', kernel_regularizer=l2_reg)(x)
|
| 568 |
+
x = layers.BatchNormalization()(x)
|
| 569 |
+
x = layers.Dropout(0.2)(x)
|
| 570 |
+
|
| 571 |
+
x = layers.Dense(32, activation='gelu', kernel_regularizer=l2_reg)(x)
|
| 572 |
+
x = layers.BatchNormalization()(x)
|
| 573 |
+
x = layers.Dropout(0.2)(x)
|
| 574 |
+
|
| 575 |
+
# Output
|
| 576 |
+
outputs = layers.Dense(NUM_CLASSES, activation='softmax', name='output')(x)
|
| 577 |
+
|
| 578 |
+
model = Model(inputs=inputs, outputs=outputs)
|
| 579 |
+
|
| 580 |
+
# Compile with focal loss or standard loss
|
| 581 |
+
if USE_FOCAL_LOSS:
|
| 582 |
+
loss = FocalLoss(alpha=FOCAL_ALPHA, gamma=FOCAL_GAMMA)
|
| 583 |
+
else:
|
| 584 |
+
loss = keras.losses.CategoricalCrossentropy(label_smoothing=0.1)
|
| 585 |
+
|
| 586 |
+
# Use Adam with weight decay (compatible with TF 2.x)
|
| 587 |
+
try:
|
| 588 |
+
# TensorFlow 2.11+
|
| 589 |
+
optimizer = keras.optimizers.Adam(
|
| 590 |
+
learning_rate=INITIAL_LR,
|
| 591 |
+
)
|
| 592 |
+
except:
|
| 593 |
+
optimizer = keras.optimizers.Adam(learning_rate=INITIAL_LR)
|
| 594 |
+
|
| 595 |
+
model.compile(
|
| 596 |
+
optimizer=optimizer,
|
| 597 |
+
loss=loss,
|
| 598 |
+
metrics=['accuracy']
|
| 599 |
+
)
|
| 600 |
+
|
| 601 |
+
print(f"\n📐 Model Summary:")
|
| 602 |
+
model.summary()
|
| 603 |
+
print(f"\n Total parameters: {model.count_params():,}")
|
| 604 |
+
print(f" Using Focal Loss: {USE_FOCAL_LOSS}")
|
| 605 |
+
print(f" Using Attention: {USE_ATTENTION}")
|
| 606 |
+
print(f" Using Residual: {USE_RESIDUAL}")
|
| 607 |
+
|
| 608 |
+
return model
|
| 609 |
+
|
| 610 |
+
|
| 611 |
+
# ============================================================================
|
| 612 |
+
# LEARNING RATE SCHEDULE
|
| 613 |
+
# ============================================================================
|
| 614 |
+
|
| 615 |
+
class WarmupCosineDecay(Callback):
|
| 616 |
+
"""Warmup + Cosine decay with restarts."""
|
| 617 |
+
|
| 618 |
+
def __init__(self, max_lr, min_lr, warmup_epochs, total_epochs, restarts=2):
|
| 619 |
+
super().__init__()
|
| 620 |
+
self.max_lr = max_lr
|
| 621 |
+
self.min_lr = min_lr
|
| 622 |
+
self.warmup_epochs = warmup_epochs
|
| 623 |
+
self.total_epochs = total_epochs
|
| 624 |
+
self.restarts = restarts
|
| 625 |
+
self.cycle_length = (total_epochs - warmup_epochs) // (restarts + 1)
|
| 626 |
+
|
| 627 |
+
def on_epoch_begin(self, epoch, logs=None):
|
| 628 |
+
if epoch < self.warmup_epochs:
|
| 629 |
+
lr = self.max_lr * (epoch + 1) / self.warmup_epochs
|
| 630 |
+
else:
|
| 631 |
+
epoch_in_cycle = (epoch - self.warmup_epochs) % self.cycle_length
|
| 632 |
+
# Cosine decay within cycle
|
| 633 |
+
lr = self.min_lr + 0.5 * (self.max_lr - self.min_lr) * \
|
| 634 |
+
(1 + math.cos(math.pi * epoch_in_cycle / self.cycle_length))
|
| 635 |
+
|
| 636 |
+
# Reduce max_lr after each restart
|
| 637 |
+
cycle_num = (epoch - self.warmup_epochs) // self.cycle_length
|
| 638 |
+
lr *= (0.8 ** cycle_num)
|
| 639 |
+
|
| 640 |
+
keras.backend.set_value(self.model.optimizer.learning_rate, lr)
|
| 641 |
+
|
| 642 |
+
def on_epoch_end(self, epoch, logs=None):
|
| 643 |
+
logs = logs or {}
|
| 644 |
+
logs['lr'] = float(keras.backend.get_value(self.model.optimizer.learning_rate))
|
| 645 |
+
|
| 646 |
+
|
| 647 |
+
# ============================================================================
|
| 648 |
+
# LOAD DATA
|
| 649 |
+
# ============================================================================
|
| 650 |
+
|
| 651 |
+
def load_data():
|
| 652 |
+
"""Load and prepare data."""
|
| 653 |
+
print("\n" + "=" * 60)
|
| 654 |
+
print("LOADING DATA")
|
| 655 |
+
print("=" * 60)
|
| 656 |
+
|
| 657 |
+
if not os.path.exists(INPUT_CSV):
|
| 658 |
+
print(f"❌ ERROR: {INPUT_CSV} not found!")
|
| 659 |
+
return None, None, None
|
| 660 |
+
|
| 661 |
+
X, y = [], []
|
| 662 |
+
|
| 663 |
+
with open(INPUT_CSV, 'r') as f:
|
| 664 |
+
reader = csv.reader(f)
|
| 665 |
+
next(reader) # Skip header
|
| 666 |
+
|
| 667 |
+
for row in reader:
|
| 668 |
+
features = [float(val) for val in row[:-1]]
|
| 669 |
+
label_str = row[-1]
|
| 670 |
+
|
| 671 |
+
if label_str in LABELS:
|
| 672 |
+
X.append(features)
|
| 673 |
+
y.append(LABELS.index(label_str))
|
| 674 |
+
|
| 675 |
+
X = np.array(X, dtype=np.float32)
|
| 676 |
+
y = np.array(y, dtype=np.int32)
|
| 677 |
+
|
| 678 |
+
print(f"✅ Loaded {len(X)} samples")
|
| 679 |
+
print(f" Feature shape: {X.shape}")
|
| 680 |
+
|
| 681 |
+
# Find hard examples (confused pairs)
|
| 682 |
+
hard_indices = []
|
| 683 |
+
for sign_a, sign_b in CONFUSED_PAIRS:
|
| 684 |
+
if sign_a in LABELS and sign_b in LABELS:
|
| 685 |
+
idx_a, idx_b = LABELS.index(sign_a), LABELS.index(sign_b)
|
| 686 |
+
hard_indices.extend(np.where((y == idx_a) | (y == idx_b))[0].tolist())
|
| 687 |
+
|
| 688 |
+
hard_indices = list(set(hard_indices))
|
| 689 |
+
print(f" Hard examples (confused pairs): {len(hard_indices)}")
|
| 690 |
+
|
| 691 |
+
# Class distribution
|
| 692 |
+
unique, counts = np.unique(y, return_counts=True)
|
| 693 |
+
print(f"\n📊 Class distribution:")
|
| 694 |
+
min_count = min(counts)
|
| 695 |
+
max_count = max(counts)
|
| 696 |
+
print(f" Min samples: {min_count} ({LABELS[unique[np.argmin(counts)]]})")
|
| 697 |
+
print(f" Max samples: {max_count} ({LABELS[unique[np.argmax(counts)]]})")
|
| 698 |
+
print(f" Imbalance ratio: {max_count/min_count:.2f}")
|
| 699 |
+
|
| 700 |
+
return X, y, hard_indices
|
| 701 |
+
|
| 702 |
+
|
| 703 |
+
# ============================================================================
|
| 704 |
+
# TRAINING
|
| 705 |
+
# ============================================================================
|
| 706 |
+
|
| 707 |
+
def train_model(model, X, y, hard_indices):
|
| 708 |
+
"""Train with advanced techniques."""
|
| 709 |
+
print("\n" + "=" * 60)
|
| 710 |
+
print(f"TRAINING FOR {EPOCHS} EPOCHS")
|
| 711 |
+
print("=" * 60)
|
| 712 |
+
|
| 713 |
+
# Split data
|
| 714 |
+
X_train, X_val, y_train, y_val = train_test_split(
|
| 715 |
+
X, y, test_size=VALIDATION_SPLIT, random_state=42, stratify=y
|
| 716 |
+
)
|
| 717 |
+
|
| 718 |
+
print(f"\n📊 Data split:")
|
| 719 |
+
print(f" Training: {len(X_train)} samples")
|
| 720 |
+
print(f" Validation: {len(X_val)} samples")
|
| 721 |
+
|
| 722 |
+
# Compute class weights
|
| 723 |
+
class_weights = None
|
| 724 |
+
if USE_CLASS_WEIGHTS:
|
| 725 |
+
weights = compute_class_weight('balanced', classes=np.unique(y_train), y=y_train)
|
| 726 |
+
class_weights = dict(enumerate(weights))
|
| 727 |
+
print(f" Using class weights: min={min(weights):.2f}, max={max(weights):.2f}")
|
| 728 |
+
|
| 729 |
+
# Find hard examples in training set
|
| 730 |
+
train_hard_indices = []
|
| 731 |
+
if hard_indices is not None:
|
| 732 |
+
train_indices_set = set(range(len(X_train)))
|
| 733 |
+
# Map original indices to training indices (approximation)
|
| 734 |
+
for h_idx in hard_indices:
|
| 735 |
+
if h_idx < len(X_train):
|
| 736 |
+
train_hard_indices.append(h_idx)
|
| 737 |
+
|
| 738 |
+
# Create augmenter and generator
|
| 739 |
+
augmenter = AdvancedAugmenter(
|
| 740 |
+
noise_std=AUG_NOISE_STD,
|
| 741 |
+
scale_range=AUG_SCALE_RANGE,
|
| 742 |
+
rotation_range=AUG_ROTATION_RANGE,
|
| 743 |
+
shift_range=AUG_SHIFT_RANGE
|
| 744 |
+
)
|
| 745 |
+
|
| 746 |
+
train_gen = AdvancedDataGenerator(
|
| 747 |
+
X_train, y_train, BATCH_SIZE, augmenter, NUM_CLASSES,
|
| 748 |
+
use_mixup=True, use_cutmix=True,
|
| 749 |
+
hard_example_indices=train_hard_indices,
|
| 750 |
+
hard_example_ratio=0.2
|
| 751 |
+
)
|
| 752 |
+
|
| 753 |
+
# Validation data
|
| 754 |
+
y_val_one_hot = keras.utils.to_categorical(y_val, NUM_CLASSES)
|
| 755 |
+
|
| 756 |
+
# Callbacks
|
| 757 |
+
callbacks = [
|
| 758 |
+
WarmupCosineDecay(INITIAL_LR, MIN_LR, WARMUP_EPOCHS, EPOCHS, restarts=3),
|
| 759 |
+
|
| 760 |
+
keras.callbacks.ModelCheckpoint(
|
| 761 |
+
OUTPUT_H5,
|
| 762 |
+
monitor='val_accuracy',
|
| 763 |
+
save_best_only=True,
|
| 764 |
+
verbose=1
|
| 765 |
+
),
|
| 766 |
+
|
| 767 |
+
keras.callbacks.ModelCheckpoint(
|
| 768 |
+
'isl_model_advanced_latest.h5',
|
| 769 |
+
save_best_only=False,
|
| 770 |
+
verbose=0
|
| 771 |
+
),
|
| 772 |
+
|
| 773 |
+
# Reduce LR if stuck (backup scheduler)
|
| 774 |
+
keras.callbacks.ReduceLROnPlateau(
|
| 775 |
+
monitor='val_loss',
|
| 776 |
+
factor=0.5,
|
| 777 |
+
patience=15,
|
| 778 |
+
min_lr=MIN_LR,
|
| 779 |
+
verbose=1
|
| 780 |
+
),
|
| 781 |
+
|
| 782 |
+
# Progress logging
|
| 783 |
+
keras.callbacks.LambdaCallback(
|
| 784 |
+
on_epoch_end=lambda epoch, logs: print(
|
| 785 |
+
f"\n 📈 Epoch {epoch+1}/{EPOCHS} - "
|
| 786 |
+
f"acc: {logs.get('accuracy', 0)*100:.2f}% - "
|
| 787 |
+
f"val_acc: {logs.get('val_accuracy', 0)*100:.2f}% - "
|
| 788 |
+
f"lr: {logs.get('lr', 0):.6f}"
|
| 789 |
+
) if (epoch + 1) % 10 == 0 else None
|
| 790 |
+
)
|
| 791 |
+
]
|
| 792 |
+
|
| 793 |
+
# Train
|
| 794 |
+
print(f"\n🚀 Starting training...")
|
| 795 |
+
print(f" Techniques: Focal Loss, Attention, Residual, Mixup, CutMix")
|
| 796 |
+
print(f" Augmentation: Noise, Scale, Rotation, Shift, Mirror\n")
|
| 797 |
+
|
| 798 |
+
history = model.fit(
|
| 799 |
+
train_gen,
|
| 800 |
+
validation_data=(X_val, y_val_one_hot),
|
| 801 |
+
epochs=EPOCHS,
|
| 802 |
+
callbacks=callbacks,
|
| 803 |
+
class_weight=class_weights,
|
| 804 |
+
verbose=1
|
| 805 |
+
)
|
| 806 |
+
|
| 807 |
+
# Load best model
|
| 808 |
+
print("\n📂 Loading best model...")
|
| 809 |
+
model = keras.models.load_model(
|
| 810 |
+
OUTPUT_H5,
|
| 811 |
+
custom_objects={
|
| 812 |
+
'FocalLoss': FocalLoss,
|
| 813 |
+
'ChannelAttention': ChannelAttention,
|
| 814 |
+
'ResidualBlock': ResidualBlock
|
| 815 |
+
}
|
| 816 |
+
)
|
| 817 |
+
|
| 818 |
+
return history, model, X_val, y_val
|
| 819 |
+
|
| 820 |
+
|
| 821 |
+
# ============================================================================
|
| 822 |
+
# TEST-TIME AUGMENTATION
|
| 823 |
+
# ============================================================================
|
| 824 |
+
|
| 825 |
+
def predict_with_tta(model, X, n_augments=5):
|
| 826 |
+
"""Prediction with test-time augmentation."""
|
| 827 |
+
augmenter = AdvancedAugmenter(noise_std=0.01, scale_range=(0.95, 1.05))
|
| 828 |
+
|
| 829 |
+
predictions = []
|
| 830 |
+
|
| 831 |
+
# Original prediction
|
| 832 |
+
predictions.append(model.predict(X, verbose=0))
|
| 833 |
+
|
| 834 |
+
# Augmented predictions
|
| 835 |
+
for _ in range(n_augments - 1):
|
| 836 |
+
X_aug = augmenter.augment_batch(X, p=0.5)
|
| 837 |
+
predictions.append(model.predict(X_aug, verbose=0))
|
| 838 |
+
|
| 839 |
+
# Average predictions
|
| 840 |
+
avg_pred = np.mean(predictions, axis=0)
|
| 841 |
+
return avg_pred
|
| 842 |
+
|
| 843 |
+
|
| 844 |
+
# ============================================================================
|
| 845 |
+
# EVALUATION
|
| 846 |
+
# ============================================================================
|
| 847 |
+
|
| 848 |
+
def evaluate_model(model, X_val, y_val, use_tta=True):
|
| 849 |
+
"""Comprehensive evaluation."""
|
| 850 |
+
print("\n" + "=" * 60)
|
| 851 |
+
print("EVALUATION")
|
| 852 |
+
print("=" * 60)
|
| 853 |
+
|
| 854 |
+
# Standard prediction
|
| 855 |
+
y_val_one_hot = keras.utils.to_categorical(y_val, NUM_CLASSES)
|
| 856 |
+
val_loss, val_acc = model.evaluate(X_val, y_val_one_hot, verbose=0)
|
| 857 |
+
|
| 858 |
+
print(f"\n📈 Standard Evaluation:")
|
| 859 |
+
print(f" Validation Accuracy: {val_acc*100:.2f}%")
|
| 860 |
+
print(f" Validation Loss: {val_loss:.4f}")
|
| 861 |
+
|
| 862 |
+
# TTA prediction
|
| 863 |
+
if use_tta:
|
| 864 |
+
print(f"\n🔄 Test-Time Augmentation (TTA):")
|
| 865 |
+
tta_pred = predict_with_tta(model, X_val, n_augments=5)
|
| 866 |
+
tta_classes = np.argmax(tta_pred, axis=1)
|
| 867 |
+
tta_acc = np.mean(tta_classes == y_val)
|
| 868 |
+
print(f" TTA Accuracy: {tta_acc*100:.2f}%")
|
| 869 |
+
else:
|
| 870 |
+
tta_pred = model.predict(X_val, verbose=0)
|
| 871 |
+
tta_classes = np.argmax(tta_pred, axis=1)
|
| 872 |
+
|
| 873 |
+
# Classification report
|
| 874 |
+
print("\n" + "-" * 60)
|
| 875 |
+
print("CLASSIFICATION REPORT")
|
| 876 |
+
print("-" * 60)
|
| 877 |
+
print(classification_report(y_val, tta_classes, target_names=LABELS, digits=3))
|
| 878 |
+
|
| 879 |
+
# Confusion analysis for hard pairs
|
| 880 |
+
print("\n" + "-" * 60)
|
| 881 |
+
print("CONFUSED PAIRS ANALYSIS")
|
| 882 |
+
print("-" * 60)
|
| 883 |
+
|
| 884 |
+
total_confusion = 0
|
| 885 |
+
for sign_a, sign_b in CONFUSED_PAIRS:
|
| 886 |
+
if sign_a in LABELS and sign_b in LABELS:
|
| 887 |
+
idx_a, idx_b = LABELS.index(sign_a), LABELS.index(sign_b)
|
| 888 |
+
|
| 889 |
+
mask_a = y_val == idx_a
|
| 890 |
+
mask_b = y_val == idx_b
|
| 891 |
+
|
| 892 |
+
a_as_b = np.sum((tta_classes == idx_b) & mask_a)
|
| 893 |
+
b_as_a = np.sum((tta_classes == idx_a) & mask_b)
|
| 894 |
+
|
| 895 |
+
if a_as_b > 0 or b_as_a > 0:
|
| 896 |
+
print(f" ⚠️ {sign_a} ↔ {sign_b}: {sign_a}→{sign_b}={a_as_b}, {sign_b}→{sign_a}={b_as_a}")
|
| 897 |
+
total_confusion += a_as_b + b_as_a
|
| 898 |
+
else:
|
| 899 |
+
print(f" ✅ {sign_a} ↔ {sign_b}: Perfect!")
|
| 900 |
+
|
| 901 |
+
print(f"\n Total confusions in pairs: {total_confusion}")
|
| 902 |
+
|
| 903 |
+
return val_acc
|
| 904 |
+
|
| 905 |
+
|
| 906 |
+
# ============================================================================
|
| 907 |
+
# CONVERT TO TFLITE
|
| 908 |
+
# ============================================================================
|
| 909 |
+
|
| 910 |
+
def convert_to_tflite(model):
|
| 911 |
+
"""Convert to optimized TFLite."""
|
| 912 |
+
print("\n" + "=" * 60)
|
| 913 |
+
print("CONVERTING TO TFLITE")
|
| 914 |
+
print("=" * 60)
|
| 915 |
+
|
| 916 |
+
# Save a version without custom objects for TFLite conversion
|
| 917 |
+
# Re-create model with standard layers
|
| 918 |
+
print(" Creating export-friendly model...")
|
| 919 |
+
|
| 920 |
+
input_size = model.input_shape[1]
|
| 921 |
+
|
| 922 |
+
# Simple export model (without custom layers)
|
| 923 |
+
export_model = keras.Sequential([
|
| 924 |
+
layers.Input(shape=(input_size,)),
|
| 925 |
+
layers.Dense(512, activation='gelu'),
|
| 926 |
+
layers.BatchNormalization(),
|
| 927 |
+
layers.Dropout(0.4),
|
| 928 |
+
layers.Dense(256, activation='gelu'),
|
| 929 |
+
layers.BatchNormalization(),
|
| 930 |
+
layers.Dropout(0.3),
|
| 931 |
+
layers.Dense(128, activation='gelu'),
|
| 932 |
+
layers.BatchNormalization(),
|
| 933 |
+
layers.Dropout(0.3),
|
| 934 |
+
layers.Dense(64, activation='gelu'),
|
| 935 |
+
layers.BatchNormalization(),
|
| 936 |
+
layers.Dropout(0.2),
|
| 937 |
+
layers.Dense(32, activation='gelu'),
|
| 938 |
+
layers.BatchNormalization(),
|
| 939 |
+
layers.Dropout(0.2),
|
| 940 |
+
layers.Dense(NUM_CLASSES, activation='softmax')
|
| 941 |
+
])
|
| 942 |
+
|
| 943 |
+
export_model.compile(
|
| 944 |
+
optimizer='adam',
|
| 945 |
+
loss='categorical_crossentropy',
|
| 946 |
+
metrics=['accuracy']
|
| 947 |
+
)
|
| 948 |
+
|
| 949 |
+
# Knowledge distillation: train export model on advanced model's predictions
|
| 950 |
+
print(" Performing knowledge distillation...")
|
| 951 |
+
|
| 952 |
+
# Load training data for distillation
|
| 953 |
+
X_all, y_all, _ = load_data()
|
| 954 |
+
if X_all is not None:
|
| 955 |
+
# Get soft labels from advanced model
|
| 956 |
+
soft_labels = model.predict(X_all, verbose=0)
|
| 957 |
+
|
| 958 |
+
# Train export model
|
| 959 |
+
export_model.fit(
|
| 960 |
+
X_all, soft_labels,
|
| 961 |
+
epochs=20,
|
| 962 |
+
batch_size=64,
|
| 963 |
+
verbose=1
|
| 964 |
+
)
|
| 965 |
+
|
| 966 |
+
# Convert to TFLite
|
| 967 |
+
print("\n Converting to TFLite...")
|
| 968 |
+
converter = tf.lite.TFLiteConverter.from_keras_model(export_model)
|
| 969 |
+
converter.optimizations = [tf.lite.Optimize.DEFAULT]
|
| 970 |
+
|
| 971 |
+
# Quantization for smaller size
|
| 972 |
+
converter.target_spec.supported_types = [tf.float16]
|
| 973 |
+
|
| 974 |
+
tflite_model = converter.convert()
|
| 975 |
+
|
| 976 |
+
with open(OUTPUT_MODEL, 'wb') as f:
|
| 977 |
+
f.write(tflite_model)
|
| 978 |
+
|
| 979 |
+
print(f"\n✅ TFLite model saved: {OUTPUT_MODEL}")
|
| 980 |
+
print(f" Size: {os.path.getsize(OUTPUT_MODEL) / 1024:.2f} KB")
|
| 981 |
+
|
| 982 |
+
# Verify
|
| 983 |
+
print("\n📋 Verifying TFLite model...")
|
| 984 |
+
interpreter = tf.lite.Interpreter(model_content=tflite_model)
|
| 985 |
+
interpreter.allocate_tensors()
|
| 986 |
+
|
| 987 |
+
input_details = interpreter.get_input_details()
|
| 988 |
+
output_details = interpreter.get_output_details()
|
| 989 |
+
|
| 990 |
+
print(f" Input shape: {input_details[0]['shape']}")
|
| 991 |
+
print(f" Output shape: {output_details[0]['shape']}")
|
| 992 |
+
|
| 993 |
+
return tflite_model
|
| 994 |
+
|
| 995 |
+
|
| 996 |
+
# ============================================================================
|
| 997 |
+
# SAVE CONFIG
|
| 998 |
+
# ============================================================================
|
| 999 |
+
|
| 1000 |
+
def save_config(val_acc):
|
| 1001 |
+
"""Save configuration."""
|
| 1002 |
+
config = {
|
| 1003 |
+
'labels': LABELS,
|
| 1004 |
+
'num_classes': NUM_CLASSES,
|
| 1005 |
+
'input_size': 130,
|
| 1006 |
+
'validation_accuracy': float(val_acc),
|
| 1007 |
+
'training_config': {
|
| 1008 |
+
'epochs': EPOCHS,
|
| 1009 |
+
'batch_size': BATCH_SIZE,
|
| 1010 |
+
'focal_loss': USE_FOCAL_LOSS,
|
| 1011 |
+
'attention': USE_ATTENTION,
|
| 1012 |
+
'residual': USE_RESIDUAL,
|
| 1013 |
+
'augmentation': {
|
| 1014 |
+
'noise_std': AUG_NOISE_STD,
|
| 1015 |
+
'scale_range': list(AUG_SCALE_RANGE),
|
| 1016 |
+
'rotation_range': AUG_ROTATION_RANGE,
|
| 1017 |
+
'mixup_alpha': MIXUP_ALPHA,
|
| 1018 |
+
'cutmix_alpha': CUTMIX_ALPHA
|
| 1019 |
+
}
|
| 1020 |
+
},
|
| 1021 |
+
'model_files': {
|
| 1022 |
+
'h5': OUTPUT_H5,
|
| 1023 |
+
'tflite': OUTPUT_MODEL
|
| 1024 |
+
}
|
| 1025 |
+
}
|
| 1026 |
+
|
| 1027 |
+
with open(OUTPUT_LABELS, 'w') as f:
|
| 1028 |
+
json.dump(config, f, indent=2)
|
| 1029 |
+
|
| 1030 |
+
print(f"✅ Config saved: {OUTPUT_LABELS}")
|
| 1031 |
+
|
| 1032 |
+
|
| 1033 |
+
# ============================================================================
|
| 1034 |
+
# MAIN
|
| 1035 |
+
# ============================================================================
|
| 1036 |
+
|
| 1037 |
+
def main():
|
| 1038 |
+
print("\n" + "=" * 70)
|
| 1039 |
+
print("🚀 ADVANCED ISL MODEL TRAINING - MAXIMUM ACCURACY")
|
| 1040 |
+
print("=" * 70)
|
| 1041 |
+
print("\nTarget: 9-10/10 accuracy with real-world robustness")
|
| 1042 |
+
print("\nAdvanced Techniques:")
|
| 1043 |
+
print(" ✓ Focal Loss - Focus on hard examples")
|
| 1044 |
+
print(" ✓ Attention Mechanism - Important features")
|
| 1045 |
+
print(" ✓ Residual Connections - Better gradients")
|
| 1046 |
+
print(" ✓ Multi-Scale Features - Different patterns")
|
| 1047 |
+
print(" ✓ Advanced Augmentation - Realistic variations")
|
| 1048 |
+
print(" ✓ Mixup + CutMix - Better generalization")
|
| 1049 |
+
print(" ✓ Class Balancing - Handle imbalance")
|
| 1050 |
+
print(" ✓ Hard Example Mining - Confused pairs")
|
| 1051 |
+
print(" ✓ Test-Time Augmentation - Better inference")
|
| 1052 |
+
print(" ✓ Knowledge Distillation - Better TFLite")
|
| 1053 |
+
|
| 1054 |
+
# Load data
|
| 1055 |
+
X, y, hard_indices = load_data()
|
| 1056 |
+
if X is None:
|
| 1057 |
+
return
|
| 1058 |
+
|
| 1059 |
+
input_size = X.shape[1]
|
| 1060 |
+
|
| 1061 |
+
# Create model
|
| 1062 |
+
model = create_advanced_model(input_size)
|
| 1063 |
+
|
| 1064 |
+
# Train
|
| 1065 |
+
history, model, X_val, y_val = train_model(model, X, y, hard_indices)
|
| 1066 |
+
|
| 1067 |
+
# Evaluate
|
| 1068 |
+
val_acc = evaluate_model(model, X_val, y_val, use_tta=True)
|
| 1069 |
+
|
| 1070 |
+
# Convert to TFLite
|
| 1071 |
+
convert_to_tflite(model)
|
| 1072 |
+
|
| 1073 |
+
# Save config
|
| 1074 |
+
save_config(val_acc)
|
| 1075 |
+
|
| 1076 |
+
# Final summary
|
| 1077 |
+
print("\n" + "=" * 70)
|
| 1078 |
+
print("✅ TRAINING COMPLETE!")
|
| 1079 |
+
print("=" * 70)
|
| 1080 |
+
print(f"\n🎯 Final Validation Accuracy: {val_acc*100:.2f}%")
|
| 1081 |
+
print(f"\n📁 Output files:")
|
| 1082 |
+
print(f" 📄 {OUTPUT_H5} - Best Keras model")
|
| 1083 |
+
print(f" 📄 {OUTPUT_MODEL} - TFLite for Android")
|
| 1084 |
+
print(f" 📄 {OUTPUT_LABELS} - Configuration")
|
| 1085 |
+
print("\n💡 Tips for even better results:")
|
| 1086 |
+
print(" 1. Collect more data for underperforming classes")
|
| 1087 |
+
print(" 2. Add more augmented samples for confused pairs")
|
| 1088 |
+
print(" 3. Use ensemble of multiple models")
|
| 1089 |
+
print(" 4. Fine-tune on real webcam data")
|
| 1090 |
+
|
| 1091 |
+
|
| 1092 |
+
if __name__ == "__main__":
|
| 1093 |
+
main()
|