meghmodi commited on
Commit
07d2eba
·
verified ·
1 Parent(s): 9b2b632

Upload 13 files

Browse files
.gitattributes CHANGED
@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ evaluation_results/confusion_matrix.png filter=lfs diff=lfs merge=lfs -text
37
+ landmark_dataset_with_orientation.csv filter=lfs diff=lfs merge=lfs -text
README.md ADDED
The diff for this file is too large to render. See raw diff
 
evaluation_results/classification_report_20260211_155639.txt ADDED
@@ -0,0 +1,70 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ======================================================================
2
+ KAIROAI MODEL EVALUATION REPORT
3
+ Generated: 2026-02-11 15:56:39
4
+ ======================================================================
5
+
6
+ OVERALL METRICS
7
+ ----------------------------------------------------------------------
8
+
9
+ TRAINING SET:
10
+ Accuracy: 99.94%
11
+ Precision: 99.94%
12
+ Recall: 99.94%
13
+ F1-Score: 99.94%
14
+
15
+ VALIDATION SET:
16
+ Accuracy: 99.97%
17
+ Precision: 99.97%
18
+ Recall: 99.97%
19
+ F1-Score: 99.97%
20
+
21
+ TEST SET:
22
+ Accuracy: 99.96%
23
+ Precision: 99.96%
24
+ Recall: 99.96%
25
+ F1-Score: 99.96%
26
+
27
+
28
+ DETAILED CLASSIFICATION REPORT (Test Set)
29
+ ----------------------------------------------------------------------
30
+ precision recall f1-score support
31
+
32
+ A 1.000 1.000 1.000 240
33
+ B 1.000 1.000 1.000 237
34
+ C 1.000 1.000 1.000 266
35
+ D 1.000 1.000 1.000 240
36
+ E 1.000 1.000 1.000 240
37
+ F 1.000 1.000 1.000 240
38
+ G 1.000 1.000 1.000 240
39
+ H 0.992 1.000 0.996 240
40
+ I 1.000 1.000 1.000 276
41
+ J 1.000 1.000 1.000 240
42
+ K 1.000 1.000 1.000 240
43
+ L 1.000 1.000 1.000 240
44
+ M 1.000 0.992 0.996 240
45
+ N 1.000 1.000 1.000 240
46
+ O 1.000 1.000 1.000 262
47
+ P 1.000 1.000 1.000 226
48
+ Q 1.000 1.000 1.000 148
49
+ R 1.000 1.000 1.000 240
50
+ S 1.000 0.996 0.998 234
51
+ T 1.000 1.000 1.000 240
52
+ U 1.000 1.000 1.000 240
53
+ V 1.000 1.000 1.000 257
54
+ W 1.000 1.000 1.000 240
55
+ X 1.000 1.000 1.000 240
56
+ Y 1.000 1.000 1.000 240
57
+ Z 1.000 1.000 1.000 240
58
+ 1 1.000 1.000 1.000 240
59
+ 2 1.000 1.000 1.000 240
60
+ 3 1.000 1.000 1.000 240
61
+ 4 1.000 1.000 1.000 240
62
+ 5 1.000 1.000 1.000 240
63
+ 6 0.996 1.000 0.998 240
64
+ 7 1.000 1.000 1.000 240
65
+ 8 1.000 1.000 1.000 240
66
+ 9 1.000 1.000 1.000 240
67
+
68
+ accuracy 1.000 8386
69
+ macro avg 1.000 1.000 1.000 8386
70
+ weighted avg 1.000 1.000 1.000 8386
evaluation_results/confusion_matrix.png ADDED

Git LFS Details

  • SHA256: 9f9c6559112b6e9f884148f8048e74f6e01ea0e15510fdab91037818fa4f0a31
  • Pointer size: 131 Bytes
  • Size of remote file: 358 kB
evaluation_results/confusion_matrix_20260211_155639.csv ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ,A,B,C,D,E,F,G,H,I,J,K,L,M,N,O,P,Q,R,S,T,U,V,W,X,Y,Z,1,2,3,4,5,6,7,8,9
2
+ A,240,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
3
+ B,0,237,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
4
+ C,0,0,266,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
5
+ D,0,0,0,240,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
6
+ E,0,0,0,0,240,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
7
+ F,0,0,0,0,0,240,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
8
+ G,0,0,0,0,0,0,240,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
9
+ H,0,0,0,0,0,0,0,240,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
10
+ I,0,0,0,0,0,0,0,0,276,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
11
+ J,0,0,0,0,0,0,0,0,0,240,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
12
+ K,0,0,0,0,0,0,0,0,0,0,240,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
13
+ L,0,0,0,0,0,0,0,0,0,0,0,240,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
14
+ M,0,0,0,0,0,0,0,2,0,0,0,0,238,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
15
+ N,0,0,0,0,0,0,0,0,0,0,0,0,0,240,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
16
+ O,0,0,0,0,0,0,0,0,0,0,0,0,0,0,262,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
17
+ P,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,226,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
18
+ Q,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,148,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
19
+ R,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,240,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
20
+ S,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,233,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0
21
+ T,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,240,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
22
+ U,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,240,0,0,0,0,0,0,0,0,0,0,0,0,0,0
23
+ V,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,257,0,0,0,0,0,0,0,0,0,0,0,0,0
24
+ W,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,240,0,0,0,0,0,0,0,0,0,0,0,0
25
+ X,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,240,0,0,0,0,0,0,0,0,0,0,0
26
+ Y,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,240,0,0,0,0,0,0,0,0,0,0
27
+ Z,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,240,0,0,0,0,0,0,0,0,0
28
+ 1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,240,0,0,0,0,0,0,0,0
29
+ 2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,240,0,0,0,0,0,0,0
30
+ 3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,240,0,0,0,0,0,0
31
+ 4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,240,0,0,0,0,0
32
+ 5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,240,0,0,0,0
33
+ 6,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,240,0,0,0
34
+ 7,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,240,0,0
35
+ 8,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,240,0
36
+ 9,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,240
evaluation_results/confusion_pairs_20260211_155639.csv ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ Pair,Total_Count,Combined_Rate
2
+ H ↔ M,2,0.8333333333333334
3
+ 6 ↔ S,1,0.4273504273504274
evaluation_results/evaluation_metrics_20260211_155639.json ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "timestamp": "20260211_155639",
3
+ "training": {
4
+ "accuracy": 99.94037637413825,
5
+ "precision": 99.94248078852188,
6
+ "recall": 99.93982979971115,
7
+ "f1_score": 99.94106788684324
8
+ },
9
+ "validation": {
10
+ "accuracy": 99.97018929795797,
11
+ "precision": 99.97165518959099,
12
+ "recall": 99.97170523138831,
13
+ "f1_score": 99.97160969501877
14
+ },
15
+ "test": {
16
+ "accuracy": 99.96422609110422,
17
+ "precision": 99.96453188456206,
18
+ "recall": 99.96398046398046,
19
+ "f1_score": 99.96413197796029
20
+ }
21
+ }
evaluation_results/per_class_metrics_20260211_155639.csv ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ Class,Precision,Recall,F1-Score,Support
2
+ A,100.0,100.0,100.0,240
3
+ B,100.0,100.0,100.0,237
4
+ C,100.0,100.0,100.0,266
5
+ D,100.0,100.0,100.0,240
6
+ E,100.0,100.0,100.0,240
7
+ F,100.0,100.0,100.0,240
8
+ G,100.0,100.0,100.0,240
9
+ H,99.17355371900827,100.0,99.5850622406639,240
10
+ I,100.0,100.0,100.0,276
11
+ J,100.0,100.0,100.0,240
12
+ K,100.0,100.0,100.0,240
13
+ L,100.0,100.0,100.0,240
14
+ M,100.0,99.16666666666667,99.581589958159,240
15
+ N,100.0,100.0,100.0,240
16
+ O,100.0,100.0,100.0,262
17
+ P,100.0,100.0,100.0,226
18
+ Q,100.0,100.0,100.0,148
19
+ R,100.0,100.0,100.0,240
20
+ S,100.0,99.57264957264957,99.78586723768737,234
21
+ T,100.0,100.0,100.0,240
22
+ U,100.0,100.0,100.0,240
23
+ V,100.0,100.0,100.0,257
24
+ W,100.0,100.0,100.0,240
25
+ X,100.0,100.0,100.0,240
26
+ Y,100.0,100.0,100.0,240
27
+ Z,100.0,100.0,100.0,240
28
+ 1,100.0,100.0,100.0,240
29
+ 2,100.0,100.0,100.0,240
30
+ 3,100.0,100.0,100.0,240
31
+ 4,100.0,100.0,100.0,240
32
+ 5,100.0,100.0,100.0,240
33
+ 6,99.5850622406639,100.0,99.7920997920998,240
34
+ 7,100.0,100.0,100.0,240
35
+ 8,100.0,100.0,100.0,240
36
+ 9,100.0,100.0,100.0,240
isl_model_advanced.tflite ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:914c7609950af5e7cf97da31b9542f3b7b98af9b3b8af3c3e8196cf6cd0cf697
3
+ size 494372
landmark_dataset_with_orientation.csv ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:12d66a842ea865885f43c78f735f63cff65ca1f4d7178ccecca7b9fc8d48b65c
3
+ size 77254059
optimize_sign_assets.py ADDED
@@ -0,0 +1,113 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Optimize bundled sign images in-place to reduce Flutter app size.
3
+
4
+ Default behavior:
5
+ - Scans assets/signs/**/image.*
6
+ - Resizes images to max 960px on the longest side (if larger)
7
+ - Converts to RGB palette PNG (256 colors) and saves optimized
8
+ - Keeps existing file names and paths unchanged
9
+
10
+ Usage:
11
+ e:/KairoAI/.venv/Scripts/python.exe optimize_sign_assets.py
12
+ e:/KairoAI/.venv/Scripts/python.exe optimize_sign_assets.py --max-size 768 --quality 80
13
+ """
14
+
15
+ from __future__ import annotations
16
+
17
+ import argparse
18
+ from dataclasses import dataclass
19
+ from pathlib import Path
20
+ from typing import Iterable
21
+
22
+ from PIL import Image
23
+
24
+
25
+ SUPPORTED = {".png", ".jpg", ".jpeg", ".webp"}
26
+ ASSET_ROOT = Path(__file__).parent / "assets" / "signs"
27
+
28
+
29
+ @dataclass
30
+ class FileStat:
31
+ path: Path
32
+ before: int
33
+ after: int
34
+
35
+
36
+ def iter_image_files(root: Path) -> Iterable[Path]:
37
+ if not root.exists():
38
+ return []
39
+ return sorted(
40
+ [p for p in root.rglob("image.*") if p.is_file() and p.suffix.lower() in SUPPORTED]
41
+ )
42
+
43
+
44
+ def optimize_file(path: Path, max_size: int, quality: int) -> FileStat:
45
+ before = path.stat().st_size
46
+
47
+ with Image.open(path) as img:
48
+ img = img.convert("RGB")
49
+
50
+ # Resize only when needed, preserving aspect ratio.
51
+ longest = max(img.width, img.height)
52
+ if longest > max_size:
53
+ scale = max_size / float(longest)
54
+ new_size = (max(1, int(img.width * scale)), max(1, int(img.height * scale)))
55
+ img = img.resize(new_size, Image.Resampling.LANCZOS)
56
+
57
+ # Keep original extension so existing asset paths remain valid.
58
+ ext = path.suffix.lower()
59
+ if ext == ".png":
60
+ pal = img.quantize(colors=256, method=Image.Quantize.MEDIANCUT)
61
+ pal.save(path, format="PNG", optimize=True)
62
+ elif ext in {".jpg", ".jpeg"}:
63
+ img.save(path, format="JPEG", optimize=True, quality=quality)
64
+ elif ext == ".webp":
65
+ img.save(path, format="WEBP", quality=quality, method=6)
66
+
67
+ after = path.stat().st_size
68
+ return FileStat(path=path, before=before, after=after)
69
+
70
+
71
+ def fmt_size(num_bytes: int) -> str:
72
+ mb = num_bytes / (1024 * 1024)
73
+ return f"{mb:.2f} MB"
74
+
75
+
76
+ def main() -> int:
77
+ parser = argparse.ArgumentParser(description="Optimize local sign assets in-place")
78
+ parser.add_argument("--max-size", type=int, default=960, help="Longest image side after resize")
79
+ parser.add_argument("--quality", type=int, default=82, help="JPEG/WEBP quality")
80
+ args = parser.parse_args()
81
+
82
+ files = list(iter_image_files(ASSET_ROOT))
83
+ if not files:
84
+ print(f"No supported image files found under: {ASSET_ROOT}")
85
+ return 1
86
+
87
+ stats: list[FileStat] = []
88
+ for file_path in files:
89
+ stat = optimize_file(file_path, max_size=args.max_size, quality=args.quality)
90
+ stats.append(stat)
91
+
92
+ total_before = sum(s.before for s in stats)
93
+ total_after = sum(s.after for s in stats)
94
+ saved = total_before - total_after
95
+ pct = (saved / total_before * 100.0) if total_before else 0.0
96
+
97
+ print(f"Processed: {len(stats)} file(s)")
98
+ print(f"Before: {fmt_size(total_before)}")
99
+ print(f"After: {fmt_size(total_after)}")
100
+ print(f"Saved: {fmt_size(saved)} ({pct:.1f}%)")
101
+
102
+ # Print top 5 biggest savings for quick visibility.
103
+ top = sorted(stats, key=lambda s: (s.before - s.after), reverse=True)[:5]
104
+ print("Top savings:")
105
+ for s in top:
106
+ delta = s.before - s.after
107
+ print(f"- {s.path.as_posix()}: {fmt_size(delta)}")
108
+
109
+ return 0
110
+
111
+
112
+ if __name__ == "__main__":
113
+ raise SystemExit(main())
prepare_local_sign_assets.py ADDED
@@ -0,0 +1,119 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Copy local sign images into Flutter assets using canonical structure:
3
+ assets/signs/{LABEL}/image.{ext}
4
+
5
+ Default behavior also optimizes copied images in-place to reduce app size.
6
+
7
+ Usage:
8
+ python prepare_local_sign_assets.py "D:\\Downloads\\all the images - 2"
9
+ """
10
+
11
+ from pathlib import Path
12
+ import shutil
13
+ import sys
14
+
15
+ from PIL import Image
16
+
17
+ SOURCE_DEFAULT = Path(r"D:\Downloads\all the images - 2")
18
+ DEST_ROOT = Path(__file__).parent / "assets" / "signs"
19
+ SUPPORTED = {".png", ".jpg", ".jpeg", ".webp", ".gif"}
20
+
21
+
22
+ def normalize_label(name: str) -> str:
23
+ return name.strip().upper()
24
+
25
+
26
+ def optimize_image(path: Path, max_size: int = 960, quality: int = 82) -> None:
27
+ with Image.open(path) as img:
28
+ img = img.convert("RGB")
29
+
30
+ longest = max(img.width, img.height)
31
+ if longest > max_size:
32
+ scale = max_size / float(longest)
33
+ new_size = (max(1, int(img.width * scale)), max(1, int(img.height * scale)))
34
+ img = img.resize(new_size, Image.Resampling.LANCZOS)
35
+
36
+ ext = path.suffix.lower()
37
+ if ext == ".png":
38
+ pal = img.quantize(colors=256, method=Image.Quantize.MEDIANCUT)
39
+ pal.save(path, format="PNG", optimize=True)
40
+ elif ext in {".jpg", ".jpeg"}:
41
+ img.save(path, format="JPEG", optimize=True, quality=quality)
42
+ elif ext == ".webp":
43
+ img.save(path, format="WEBP", quality=quality, method=6)
44
+ elif ext == ".gif":
45
+ img.save(path, format="GIF", optimize=True)
46
+
47
+
48
+ def main() -> int:
49
+ source = Path(sys.argv[1]) if len(sys.argv) > 1 else SOURCE_DEFAULT
50
+ if not source.exists() or not source.is_dir():
51
+ print(f"Source folder not found: {source}")
52
+ return 1
53
+
54
+ copied = 0
55
+
56
+ # Mode 1: nested dataset folders (e.g. E:/ISL_Model_Training/Indian/A/*.jpg)
57
+ subdirs = [p for p in source.iterdir() if p.is_dir()]
58
+ if subdirs:
59
+ for subdir in sorted(subdirs, key=lambda p: p.name.lower()):
60
+ label = normalize_label(subdir.name)
61
+ if not label:
62
+ continue
63
+
64
+ candidates = [
65
+ p for p in subdir.iterdir()
66
+ if p.is_file() and p.suffix.lower() in SUPPORTED
67
+ ]
68
+ if not candidates:
69
+ continue
70
+
71
+ # Prefer numerically-small file names such as 0.jpg, 1.jpg.
72
+ def _sort_key(path: Path) -> tuple[int, str]:
73
+ stem = path.stem.strip()
74
+ if stem.isdigit():
75
+ return (0, f"{int(stem):08d}")
76
+ return (1, stem.lower())
77
+
78
+ candidates.sort(key=_sort_key)
79
+ chosen = candidates[0]
80
+
81
+ dest_dir = DEST_ROOT / label
82
+ dest_dir.mkdir(parents=True, exist_ok=True)
83
+ dest_file = dest_dir / f"image{chosen.suffix.lower()}"
84
+ shutil.copy2(chosen, dest_file)
85
+ optimize_image(dest_file)
86
+ copied += 1
87
+ print(f"{subdir.name}/{chosen.name} -> {dest_file}")
88
+
89
+ # Mode 2: flat folder of files (legacy behavior).
90
+ else:
91
+ files = [
92
+ p for p in source.iterdir() if p.is_file() and p.suffix.lower() in SUPPORTED
93
+ ]
94
+ files.sort(key=lambda p: p.name.lower())
95
+
96
+ if not files:
97
+ print("No supported image files found.")
98
+ return 1
99
+
100
+ for file_path in files:
101
+ label = normalize_label(file_path.stem)
102
+ dest_dir = DEST_ROOT / label
103
+ dest_dir.mkdir(parents=True, exist_ok=True)
104
+ dest_file = dest_dir / f"image{file_path.suffix.lower()}"
105
+ shutil.copy2(file_path, dest_file)
106
+ optimize_image(dest_file)
107
+ copied += 1
108
+ print(f"{file_path.name} -> {dest_file}")
109
+
110
+ if copied == 0:
111
+ print("No supported image files found.")
112
+ return 1
113
+
114
+ print(f"Copied {copied} file(s) to {DEST_ROOT}")
115
+ return 0
116
+
117
+
118
+ if __name__ == "__main__":
119
+ raise SystemExit(main())
test_full_model.py ADDED
@@ -0,0 +1,586 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ ================================================================================
3
+ TEST ISL MODEL WITH MEDIAPIPE - WEBCAM REAL-TIME INFERENCE
4
+ ================================================================================
5
+ This script tests the trained model (isl_model_full.tflite) using MediaPipe
6
+ for hand landmark detection and orientation calculation.
7
+
8
+ Features:
9
+ - Real-time webcam hand detection
10
+ - Extracts 130 features (126 landmarks + 4 orientation)
11
+ - Runs TFLite inference
12
+ - Displays prediction with confidence
13
+
14
+ Controls:
15
+ - Press 'q' to quit
16
+ - Press 's' to save screenshot
17
+ - Press 'c' to toggle confidence threshold
18
+
19
+ Author: KairoAI
20
+ ================================================================================
21
+ """
22
+
23
+ import cv2
24
+ import numpy as np
25
+ import mediapipe as mp
26
+ import tensorflow as tf
27
+ import json
28
+ import os
29
+ from collections import deque
30
+
31
+ # ============================================================================
32
+ # CONFIGURATION
33
+ # ============================================================================
34
+
35
+ MODEL_PATH = "isl_model_full.tflite"
36
+ LABELS_PATH = "labels_full.json"
37
+
38
+ # Fallback paths if full model not found
39
+ FALLBACK_MODEL = "isl_model.tflite"
40
+ FALLBACK_LABELS = "labels.json"
41
+
42
+ # Display settings
43
+ CONFIDENCE_THRESHOLD = 0.5
44
+ SMOOTHING_WINDOW = 5 # Number of frames to average predictions
45
+ SHOW_LANDMARKS = True
46
+ SHOW_ORIENTATION = True
47
+
48
+ # Colors (BGR)
49
+ COLOR_PALM = (0, 255, 0) # Green for palm
50
+ COLOR_BACK = (0, 165, 255) # Orange for back of hand
51
+ COLOR_TEXT = (255, 255, 255) # White
52
+ COLOR_BOX = (50, 50, 50) # Dark gray
53
+
54
+ # Labels (fallback if json not found)
55
+ DEFAULT_LABELS = [
56
+ 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J',
57
+ 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T',
58
+ 'U', 'V', 'W', 'X', 'Y', 'Z', '1', '2', '3', '4',
59
+ '5', '6', '7', '8', '9'
60
+ ]
61
+
62
+
63
+ # ============================================================================
64
+ # HAND ORIENTATION CALCULATOR
65
+ # ============================================================================
66
+
67
+ class HandOrientationCalculator:
68
+ """
69
+ Calculate whether palm or back of hand is facing the camera.
70
+ Uses the cross product of vectors on the hand plane.
71
+ """
72
+
73
+ # Landmark indices
74
+ WRIST = 0
75
+ INDEX_MCP = 5
76
+ PINKY_MCP = 17
77
+ MIDDLE_MCP = 9
78
+
79
+ @staticmethod
80
+ def calculate_orientation(landmarks, handedness):
81
+ """
82
+ Calculate hand orientation.
83
+ Returns: (is_palm_facing: float, is_left_hand: float)
84
+ - is_palm_facing: 1.0 = palm facing camera, 0.0 = back facing camera
85
+ - is_left_hand: 1.0 = left hand, 0.0 = right hand
86
+ """
87
+ if landmarks is None:
88
+ return -1.0, -1.0
89
+
90
+ # Get key points
91
+ wrist = np.array([
92
+ landmarks[HandOrientationCalculator.WRIST].x,
93
+ landmarks[HandOrientationCalculator.WRIST].y,
94
+ landmarks[HandOrientationCalculator.WRIST].z
95
+ ])
96
+
97
+ index_mcp = np.array([
98
+ landmarks[HandOrientationCalculator.INDEX_MCP].x,
99
+ landmarks[HandOrientationCalculator.INDEX_MCP].y,
100
+ landmarks[HandOrientationCalculator.INDEX_MCP].z
101
+ ])
102
+
103
+ pinky_mcp = np.array([
104
+ landmarks[HandOrientationCalculator.PINKY_MCP].x,
105
+ landmarks[HandOrientationCalculator.PINKY_MCP].y,
106
+ landmarks[HandOrientationCalculator.PINKY_MCP].z
107
+ ])
108
+
109
+ # Calculate vectors on the palm plane
110
+ v1 = index_mcp - wrist # Wrist to index
111
+ v2 = pinky_mcp - wrist # Wrist to pinky
112
+
113
+ # Cross product gives normal to palm plane
114
+ normal = np.cross(v1, v2)
115
+
116
+ # Z component of normal indicates palm orientation
117
+ # Positive = palm facing camera, Negative = back facing
118
+ z_component = normal[2]
119
+
120
+ # Determine handedness
121
+ is_left = 1.0 if handedness == "Left" else 0.0
122
+
123
+ # For left hand, the normal direction is reversed
124
+ if is_left:
125
+ z_component = -z_component
126
+
127
+ # Convert to 0/1 (back/palm)
128
+ is_palm_facing = 1.0 if z_component > 0 else 0.0
129
+
130
+ return is_palm_facing, is_left
131
+
132
+
133
+ # ============================================================================
134
+ # LANDMARK PROCESSOR
135
+ # ============================================================================
136
+
137
+ class LandmarkProcessor:
138
+ """Process MediaPipe hand landmarks into model input features."""
139
+
140
+ def __init__(self, input_size=130):
141
+ self.input_size = input_size
142
+ self.has_orientation = input_size == 130
143
+
144
+ def normalize_landmarks(self, landmarks):
145
+ """
146
+ Normalize landmarks relative to wrist position and hand size.
147
+ Returns 63 values (21 landmarks × 3 coords) for one hand.
148
+ """
149
+ if landmarks is None:
150
+ return [0.0] * 63
151
+
152
+ coords = []
153
+ for lm in landmarks:
154
+ coords.extend([lm.x, lm.y, lm.z])
155
+
156
+ coords = np.array(coords, dtype=np.float32)
157
+
158
+ # Normalize relative to wrist (first landmark)
159
+ wrist = coords[:3].copy()
160
+ for i in range(21):
161
+ coords[i*3:i*3+3] -= wrist
162
+
163
+ # Scale by hand size (distance from wrist to middle finger MCP)
164
+ middle_mcp = coords[9*3:9*3+3] # Landmark 9
165
+ hand_size = np.linalg.norm(middle_mcp)
166
+
167
+ if hand_size > 0.001:
168
+ coords /= hand_size
169
+
170
+ return coords.tolist()
171
+
172
+ def process_hands(self, results):
173
+ """
174
+ Process MediaPipe results into model input.
175
+ Returns: (features, hand_info)
176
+ - features: numpy array of shape (input_size,)
177
+ - hand_info: dict with orientation info for display
178
+ """
179
+ features = [0.0] * self.input_size
180
+ hand_info = {
181
+ 'hand1': None,
182
+ 'hand2': None,
183
+ 'hand1_orientation': None,
184
+ 'hand2_orientation': None
185
+ }
186
+
187
+ if results.multi_hand_landmarks is None:
188
+ return np.array(features, dtype=np.float32), hand_info
189
+
190
+ hands_data = []
191
+
192
+ for i, (hand_landmarks, handedness) in enumerate(
193
+ zip(results.multi_hand_landmarks, results.multi_handedness)
194
+ ):
195
+ hand_label = handedness.classification[0].label
196
+ landmarks = hand_landmarks.landmark
197
+
198
+ # Normalize landmarks
199
+ normalized = self.normalize_landmarks(landmarks)
200
+
201
+ # Calculate orientation
202
+ is_palm, is_left = HandOrientationCalculator.calculate_orientation(
203
+ landmarks, hand_label
204
+ )
205
+
206
+ hands_data.append({
207
+ 'landmarks': normalized,
208
+ 'is_palm': is_palm,
209
+ 'is_left': is_left,
210
+ 'label': hand_label,
211
+ 'raw_landmarks': landmarks
212
+ })
213
+
214
+ # Sort by x position (left to right in image)
215
+ if len(hands_data) > 0:
216
+ hands_data.sort(
217
+ key=lambda h: h['raw_landmarks'][0].x
218
+ )
219
+
220
+ # Fill features for hand 1
221
+ if len(hands_data) >= 1:
222
+ h1 = hands_data[0]
223
+ features[0:63] = h1['landmarks']
224
+ hand_info['hand1'] = h1['label']
225
+ hand_info['hand1_orientation'] = 'Palm' if h1['is_palm'] == 1.0 else 'Back'
226
+
227
+ if self.has_orientation:
228
+ features[126] = h1['is_palm']
229
+ features[127] = h1['is_left']
230
+
231
+ # Fill features for hand 2
232
+ if len(hands_data) >= 2:
233
+ h2 = hands_data[1]
234
+ features[63:126] = h2['landmarks']
235
+ hand_info['hand2'] = h2['label']
236
+ hand_info['hand2_orientation'] = 'Palm' if h2['is_palm'] == 1.0 else 'Back'
237
+
238
+ if self.has_orientation:
239
+ features[128] = h2['is_palm']
240
+ features[129] = h2['is_left']
241
+ elif self.has_orientation:
242
+ features[128] = -1.0
243
+ features[129] = -1.0
244
+
245
+ return np.array(features, dtype=np.float32), hand_info
246
+
247
+
248
+ # ============================================================================
249
+ # PREDICTION SMOOTHER
250
+ # ============================================================================
251
+
252
+ class PredictionSmoother:
253
+ """Smooth predictions over multiple frames to reduce flickering."""
254
+
255
+ def __init__(self, window_size=5, num_classes=35):
256
+ self.window_size = window_size
257
+ self.num_classes = num_classes
258
+ self.predictions = deque(maxlen=window_size)
259
+
260
+ def add_prediction(self, probs):
261
+ """Add a new prediction probability distribution."""
262
+ self.predictions.append(probs)
263
+
264
+ def get_smoothed_prediction(self):
265
+ """Get averaged prediction over the window."""
266
+ if len(self.predictions) == 0:
267
+ return None, 0.0
268
+
269
+ avg_probs = np.mean(list(self.predictions), axis=0)
270
+ pred_class = np.argmax(avg_probs)
271
+ confidence = avg_probs[pred_class]
272
+
273
+ return pred_class, confidence
274
+
275
+ def clear(self):
276
+ """Clear prediction history."""
277
+ self.predictions.clear()
278
+
279
+
280
+ # ============================================================================
281
+ # MAIN TESTER CLASS
282
+ # ============================================================================
283
+
284
+ class ISLModelTester:
285
+ """Main class for testing the ISL model with webcam."""
286
+
287
+ def __init__(self):
288
+ self.model_path = MODEL_PATH
289
+ self.labels = DEFAULT_LABELS
290
+ self.input_size = 130
291
+
292
+ # Try to load model and labels
293
+ self._load_model()
294
+ self._load_labels()
295
+
296
+ # Initialize components
297
+ self.processor = LandmarkProcessor(self.input_size)
298
+ self.smoother = PredictionSmoother(SMOOTHING_WINDOW, len(self.labels))
299
+
300
+ # Initialize MediaPipe
301
+ self.mp_hands = mp.solutions.hands
302
+ self.mp_draw = mp.solutions.drawing_utils
303
+ self.hands = self.mp_hands.Hands(
304
+ static_image_mode=False,
305
+ max_num_hands=2,
306
+ min_detection_confidence=0.7,
307
+ min_tracking_confidence=0.5
308
+ )
309
+
310
+ # State
311
+ self.confidence_threshold = CONFIDENCE_THRESHOLD
312
+ self.show_landmarks = SHOW_LANDMARKS
313
+
314
+ def _load_model(self):
315
+ """Load TFLite model."""
316
+ # Try full model first
317
+ if os.path.exists(MODEL_PATH):
318
+ self.model_path = MODEL_PATH
319
+ elif os.path.exists(FALLBACK_MODEL):
320
+ print(f"⚠️ {MODEL_PATH} not found, using {FALLBACK_MODEL}")
321
+ self.model_path = FALLBACK_MODEL
322
+ else:
323
+ raise FileNotFoundError(f"No model found! Train the model first.")
324
+
325
+ print(f"📦 Loading model: {self.model_path}")
326
+
327
+ self.interpreter = tf.lite.Interpreter(model_path=self.model_path)
328
+ self.interpreter.allocate_tensors()
329
+
330
+ self.input_details = self.interpreter.get_input_details()
331
+ self.output_details = self.interpreter.get_output_details()
332
+
333
+ # Get input size from model
334
+ self.input_size = self.input_details[0]['shape'][1]
335
+ print(f" Input size: {self.input_size}")
336
+
337
+ def _load_labels(self):
338
+ """Load labels from JSON."""
339
+ labels_path = LABELS_PATH if os.path.exists(LABELS_PATH) else FALLBACK_LABELS
340
+
341
+ if os.path.exists(labels_path):
342
+ with open(labels_path, 'r') as f:
343
+ config = json.load(f)
344
+ self.labels = config.get('labels', DEFAULT_LABELS)
345
+ print(f" Labels loaded: {len(self.labels)} classes")
346
+ else:
347
+ print(f" Using default labels: {len(self.labels)} classes")
348
+
349
+ def predict(self, features):
350
+ """Run inference on features."""
351
+ input_data = features.reshape(1, -1).astype(np.float32)
352
+ self.interpreter.set_tensor(self.input_details[0]['index'], input_data)
353
+ self.interpreter.invoke()
354
+ output = self.interpreter.get_tensor(self.output_details[0]['index'])
355
+ return output[0]
356
+
357
+ def draw_info_box(self, frame, prediction, confidence, hand_info):
358
+ """Draw prediction info box on frame."""
359
+ h, w = frame.shape[:2]
360
+
361
+ # Draw semi-transparent background
362
+ overlay = frame.copy()
363
+ cv2.rectangle(overlay, (10, 10), (300, 150), COLOR_BOX, -1)
364
+ cv2.addWeighted(overlay, 0.7, frame, 0.3, 0, frame)
365
+
366
+ # Draw prediction
367
+ if prediction is not None and confidence >= self.confidence_threshold:
368
+ label = self.labels[prediction]
369
+ color = (0, 255, 0) if confidence > 0.8 else (0, 255, 255)
370
+
371
+ cv2.putText(frame, f"Sign: {label}", (20, 50),
372
+ cv2.FONT_HERSHEY_SIMPLEX, 1.2, color, 3)
373
+ cv2.putText(frame, f"Confidence: {confidence*100:.1f}%", (20, 85),
374
+ cv2.FONT_HERSHEY_SIMPLEX, 0.7, COLOR_TEXT, 2)
375
+ else:
376
+ cv2.putText(frame, "No sign detected", (20, 50),
377
+ cv2.FONT_HERSHEY_SIMPLEX, 0.8, (100, 100, 100), 2)
378
+
379
+ # Draw hand info
380
+ y_offset = 115
381
+ if hand_info['hand1']:
382
+ orient = hand_info['hand1_orientation'] or "?"
383
+ cv2.putText(frame, f"Hand 1: {hand_info['hand1']} ({orient})",
384
+ (20, y_offset), cv2.FONT_HERSHEY_SIMPLEX, 0.5, COLOR_TEXT, 1)
385
+ y_offset += 20
386
+
387
+ if hand_info['hand2']:
388
+ orient = hand_info['hand2_orientation'] or "?"
389
+ cv2.putText(frame, f"Hand 2: {hand_info['hand2']} ({orient})",
390
+ (20, y_offset), cv2.FONT_HERSHEY_SIMPLEX, 0.5, COLOR_TEXT, 1)
391
+
392
+ # Draw controls hint
393
+ cv2.putText(frame, "Q: Quit | S: Screenshot | L: Toggle landmarks",
394
+ (10, h - 10), cv2.FONT_HERSHEY_SIMPLEX, 0.4, (150, 150, 150), 1)
395
+
396
+ return frame
397
+
398
+ def draw_landmarks(self, frame, results, hand_info):
399
+ """Draw hand landmarks with orientation-based coloring."""
400
+ if results.multi_hand_landmarks is None:
401
+ return frame
402
+
403
+ for i, hand_landmarks in enumerate(results.multi_hand_landmarks):
404
+ # Choose color based on orientation
405
+ if i == 0 and hand_info['hand1_orientation']:
406
+ color = COLOR_PALM if hand_info['hand1_orientation'] == 'Palm' else COLOR_BACK
407
+ elif i == 1 and hand_info['hand2_orientation']:
408
+ color = COLOR_PALM if hand_info['hand2_orientation'] == 'Palm' else COLOR_BACK
409
+ else:
410
+ color = (200, 200, 200)
411
+
412
+ # Draw connections
413
+ self.mp_draw.draw_landmarks(
414
+ frame, hand_landmarks, self.mp_hands.HAND_CONNECTIONS,
415
+ self.mp_draw.DrawingSpec(color=color, thickness=2, circle_radius=2),
416
+ self.mp_draw.DrawingSpec(color=color, thickness=2)
417
+ )
418
+
419
+ return frame
420
+
421
+ def run(self):
422
+ """Main loop for webcam testing."""
423
+ print("\n" + "=" * 60)
424
+ print("ISL MODEL TEST WITH MEDIAPIPE")
425
+ print("=" * 60)
426
+
427
+ print("\n📷 Opening webcam...")
428
+ cap = cv2.VideoCapture(0)
429
+
430
+ if not cap.isOpened():
431
+ print("❌ Error: Could not open webcam!")
432
+ return
433
+
434
+ # Set camera properties
435
+ cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640)
436
+ cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 480)
437
+ cap.set(cv2.CAP_PROP_FPS, 30)
438
+
439
+ print("✅ Webcam opened successfully")
440
+ print("\n🎮 Controls:")
441
+ print(" Press 'q' to quit")
442
+ print(" Press 's' to save screenshot")
443
+ print(" Press 'l' to toggle landmarks")
444
+ print(" Press 'c' to cycle confidence threshold")
445
+ print(" Press 'r' to reset smoother")
446
+ print("\n" + "=" * 60 + "\n")
447
+
448
+ screenshot_count = 0
449
+
450
+ while True:
451
+ ret, frame = cap.read()
452
+ if not ret:
453
+ print("❌ Error reading frame")
454
+ break
455
+
456
+ # Flip for mirror effect
457
+ frame = cv2.flip(frame, 1)
458
+
459
+ # Convert to RGB for MediaPipe
460
+ rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
461
+
462
+ # Process with MediaPipe
463
+ results = self.hands.process(rgb_frame)
464
+
465
+ # Extract features
466
+ features, hand_info = self.processor.process_hands(results)
467
+
468
+ # Run inference if hand detected
469
+ prediction = None
470
+ confidence = 0.0
471
+
472
+ if hand_info['hand1'] is not None:
473
+ probs = self.predict(features)
474
+ self.smoother.add_prediction(probs)
475
+ prediction, confidence = self.smoother.get_smoothed_prediction()
476
+ else:
477
+ self.smoother.clear()
478
+
479
+ # Draw landmarks if enabled
480
+ if self.show_landmarks:
481
+ frame = self.draw_landmarks(frame, results, hand_info)
482
+
483
+ # Draw info box
484
+ frame = self.draw_info_box(frame, prediction, confidence, hand_info)
485
+
486
+ # Show frame
487
+ cv2.imshow('ISL Model Test', frame)
488
+
489
+ # Handle key presses
490
+ key = cv2.waitKey(1) & 0xFF
491
+
492
+ if key == ord('q'):
493
+ print("\n👋 Exiting...")
494
+ break
495
+ elif key == ord('s'):
496
+ filename = f"screenshot_{screenshot_count}.png"
497
+ cv2.imwrite(filename, frame)
498
+ print(f"📸 Screenshot saved: {filename}")
499
+ screenshot_count += 1
500
+ elif key == ord('l'):
501
+ self.show_landmarks = not self.show_landmarks
502
+ print(f"🎯 Landmarks: {'ON' if self.show_landmarks else 'OFF'}")
503
+ elif key == ord('c'):
504
+ # Cycle threshold: 0.5 -> 0.7 -> 0.9 -> 0.3 -> 0.5
505
+ thresholds = [0.3, 0.5, 0.7, 0.9]
506
+ idx = thresholds.index(self.confidence_threshold) if self.confidence_threshold in thresholds else 0
507
+ self.confidence_threshold = thresholds[(idx + 1) % len(thresholds)]
508
+ print(f"📊 Confidence threshold: {self.confidence_threshold}")
509
+ elif key == ord('r'):
510
+ self.smoother.clear()
511
+ print("🔄 Smoother reset")
512
+
513
+ cap.release()
514
+ cv2.destroyAllWindows()
515
+ print("\n✅ Test complete!")
516
+
517
+
518
+ # ============================================================================
519
+ # BATCH TEST MODE
520
+ # ============================================================================
521
+
522
+ def test_on_images(image_folder):
523
+ """Test model on a folder of images."""
524
+ print("\n" + "=" * 60)
525
+ print("BATCH IMAGE TEST")
526
+ print("=" * 60)
527
+
528
+ tester = ISLModelTester()
529
+
530
+ results = []
531
+
532
+ for filename in os.listdir(image_folder):
533
+ if filename.lower().endswith(('.png', '.jpg', '.jpeg')):
534
+ filepath = os.path.join(image_folder, filename)
535
+
536
+ # Load image
537
+ frame = cv2.imread(filepath)
538
+ if frame is None:
539
+ continue
540
+
541
+ # Process
542
+ rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
543
+ mp_results = tester.hands.process(rgb_frame)
544
+
545
+ features, hand_info = tester.processor.process_hands(mp_results)
546
+
547
+ if hand_info['hand1'] is not None:
548
+ probs = tester.predict(features)
549
+ pred_class = np.argmax(probs)
550
+ confidence = probs[pred_class]
551
+
552
+ results.append({
553
+ 'file': filename,
554
+ 'prediction': tester.labels[pred_class],
555
+ 'confidence': confidence
556
+ })
557
+
558
+ print(f" {filename}: {tester.labels[pred_class]} ({confidence*100:.1f}%)")
559
+ else:
560
+ print(f" {filename}: No hand detected")
561
+
562
+ return results
563
+
564
+
565
+ # ============================================================================
566
+ # MAIN
567
+ # ============================================================================
568
+
569
+ def main():
570
+ import sys
571
+
572
+ if len(sys.argv) > 1:
573
+ # Batch mode on folder
574
+ folder = sys.argv[1]
575
+ if os.path.isdir(folder):
576
+ test_on_images(folder)
577
+ else:
578
+ print(f"❌ Folder not found: {folder}")
579
+ else:
580
+ # Webcam mode
581
+ tester = ISLModelTester()
582
+ tester.run()
583
+
584
+
585
+ if __name__ == "__main__":
586
+ main()
train_advanced_model.py ADDED
@@ -0,0 +1,1093 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ ================================================================================
3
+ ADVANCED ISL MODEL TRAINING - MAXIMUM ACCURACY
4
+ ================================================================================
5
+ State-of-the-art techniques for achieving 9-10/10 accuracy:
6
+
7
+ 1. Attention Mechanism - Focus on important landmarks
8
+ 2. Focal Loss - Better handling of hard examples
9
+ 3. Advanced Augmentation - Realistic variations
10
+ 4. Residual Connections - Better gradient flow
11
+ 5. Class Balancing - Handle imbalanced classes
12
+ 6. Hard Example Mining - Focus on confused pairs
13
+ 7. Multi-Scale Features - Capture different patterns
14
+ 8. Test-Time Augmentation (TTA) - Better inference
15
+ 9. Ensemble Ready - Can combine multiple models
16
+ 10. Gradual Unfreezing with Warmup
17
+
18
+ Target: 99.5%+ validation accuracy with real-world robustness
19
+
20
+ Author: KairoAI
21
+ ================================================================================
22
+ """
23
+
24
+ import os
25
+ import numpy as np
26
+ import json
27
+ import csv
28
+ import tensorflow as tf
29
+ from tensorflow import keras
30
+ from tensorflow.keras import layers, regularizers, Model
31
+ from tensorflow.keras.callbacks import Callback
32
+ from sklearn.model_selection import StratifiedKFold, train_test_split
33
+ from sklearn.utils.class_weight import compute_class_weight
34
+ from sklearn.metrics import classification_report, confusion_matrix
35
+ import math
36
+
37
+ # Suppress warnings
38
+ os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
39
+
40
+ # ============================================================================
41
+ # CONFIGURATION
42
+ # ============================================================================
43
+
44
+ INPUT_CSV = "landmark_dataset_with_orientation.csv"
45
+ OUTPUT_MODEL = "isl_model_advanced.tflite"
46
+ OUTPUT_H5 = "isl_model_advanced.h5"
47
+ OUTPUT_LABELS = "labels_advanced.json"
48
+
49
+ NUM_CLASSES = 35
50
+
51
+ # Training config
52
+ EPOCHS = 200
53
+ BATCH_SIZE = 64
54
+ VALIDATION_SPLIT = 0.15 # Smaller val split = more training data
55
+
56
+ # Advanced config
57
+ USE_FOCAL_LOSS = True
58
+ FOCAL_ALPHA = 0.25
59
+ FOCAL_GAMMA = 2.0
60
+
61
+ USE_CLASS_WEIGHTS = True
62
+ USE_ATTENTION = True
63
+ USE_RESIDUAL = True
64
+
65
+ # Augmentation strength
66
+ AUG_NOISE_STD = 0.03
67
+ AUG_SCALE_RANGE = (0.85, 1.15)
68
+ AUG_ROTATION_RANGE = 20 # degrees
69
+ AUG_SHIFT_RANGE = 0.1
70
+ MIXUP_ALPHA = 0.3
71
+ CUTMIX_ALPHA = 0.3
72
+
73
+ # Learning rate
74
+ INITIAL_LR = 0.002
75
+ MIN_LR = 1e-7
76
+ WARMUP_EPOCHS = 10
77
+
78
+ LABELS = [
79
+ 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J',
80
+ 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T',
81
+ 'U', 'V', 'W', 'X', 'Y', 'Z', '1', '2', '3', '4',
82
+ '5', '6', '7', '8', '9'
83
+ ]
84
+
85
+ # Commonly confused pairs - will get extra training attention
86
+ CONFUSED_PAIRS = [
87
+ ('V', '2'), ('K', 'V'), ('U', 'V'), ('M', 'N'), ('5', 'H'),
88
+ ('G', 'H'), ('I', 'J'), ('1', 'D'), ('6', 'W'), ('3', 'W'),
89
+ ('B', 'E'), ('A', 'S'), ('O', 'C')
90
+ ]
91
+
92
+
93
+ # ============================================================================
94
+ # FOCAL LOSS - Better for hard examples
95
+ # ============================================================================
96
+
97
+ class FocalLoss(keras.losses.Loss):
98
+ """
99
+ Focal Loss focuses training on hard examples.
100
+ Reduces loss contribution from easy examples.
101
+ """
102
+ def __init__(self, alpha=0.25, gamma=2.0, **kwargs):
103
+ super().__init__(**kwargs)
104
+ self.alpha = alpha
105
+ self.gamma = gamma
106
+
107
+ def call(self, y_true, y_pred):
108
+ # Clip predictions to prevent log(0)
109
+ y_pred = tf.clip_by_value(y_pred, 1e-7, 1 - 1e-7)
110
+
111
+ # Calculate focal loss
112
+ cross_entropy = -y_true * tf.math.log(y_pred)
113
+ weight = self.alpha * y_true * tf.pow(1 - y_pred, self.gamma)
114
+ focal_loss = weight * cross_entropy
115
+
116
+ return tf.reduce_sum(focal_loss, axis=-1)
117
+
118
+ def get_config(self):
119
+ config = super().get_config()
120
+ config.update({'alpha': self.alpha, 'gamma': self.gamma})
121
+ return config
122
+
123
+
124
+ # ============================================================================
125
+ # ATTENTION LAYER - Focus on important features
126
+ # ============================================================================
127
+
128
+ class SelfAttention(layers.Layer):
129
+ """Self-attention layer to focus on important landmarks."""
130
+
131
+ def __init__(self, units, **kwargs):
132
+ super().__init__(**kwargs)
133
+ self.units = units
134
+
135
+ def build(self, input_shape):
136
+ self.W_q = self.add_weight(
137
+ shape=(input_shape[-1], self.units),
138
+ initializer='glorot_uniform',
139
+ trainable=True,
140
+ name='query_weight'
141
+ )
142
+ self.W_k = self.add_weight(
143
+ shape=(input_shape[-1], self.units),
144
+ initializer='glorot_uniform',
145
+ trainable=True,
146
+ name='key_weight'
147
+ )
148
+ self.W_v = self.add_weight(
149
+ shape=(input_shape[-1], self.units),
150
+ initializer='glorot_uniform',
151
+ trainable=True,
152
+ name='value_weight'
153
+ )
154
+
155
+ def call(self, x):
156
+ q = tf.matmul(x, self.W_q)
157
+ k = tf.matmul(x, self.W_k)
158
+ v = tf.matmul(x, self.W_v)
159
+
160
+ # Scaled dot-product attention
161
+ d_k = tf.cast(tf.shape(k)[-1], tf.float32)
162
+ attention_scores = tf.matmul(q, k, transpose_b=True) / tf.sqrt(d_k)
163
+ attention_weights = tf.nn.softmax(attention_scores, axis=-1)
164
+
165
+ output = tf.matmul(attention_weights, v)
166
+ return output
167
+
168
+ def get_config(self):
169
+ config = super().get_config()
170
+ config.update({'units': self.units})
171
+ return config
172
+
173
+
174
+ class ChannelAttention(layers.Layer):
175
+ """Channel attention - learns which features are important."""
176
+
177
+ def __init__(self, reduction_ratio=8, **kwargs):
178
+ super().__init__(**kwargs)
179
+ self.reduction_ratio = reduction_ratio
180
+
181
+ def build(self, input_shape):
182
+ channels = input_shape[-1]
183
+ self.dense1 = layers.Dense(channels // self.reduction_ratio, activation='relu')
184
+ self.dense2 = layers.Dense(channels, activation='sigmoid')
185
+
186
+ def call(self, x):
187
+ # Global average pooling
188
+ avg_pool = tf.reduce_mean(x, axis=-1, keepdims=True)
189
+ max_pool = tf.reduce_max(x, axis=-1, keepdims=True)
190
+
191
+ # Shared MLP
192
+ avg_out = self.dense2(self.dense1(avg_pool))
193
+ max_out = self.dense2(self.dense1(max_pool))
194
+
195
+ attention = avg_out + max_out
196
+ return x * attention
197
+
198
+ def get_config(self):
199
+ config = super().get_config()
200
+ config.update({'reduction_ratio': self.reduction_ratio})
201
+ return config
202
+
203
+
204
+ # ============================================================================
205
+ # RESIDUAL BLOCK
206
+ # ============================================================================
207
+
208
+ class ResidualBlock(layers.Layer):
209
+ """Residual block with skip connection for better gradient flow."""
210
+
211
+ def __init__(self, units, dropout_rate=0.3, l2_reg=0.001, **kwargs):
212
+ super().__init__(**kwargs)
213
+ self.units = units
214
+ self.dropout_rate = dropout_rate
215
+ self.l2_reg = l2_reg
216
+
217
+ def build(self, input_shape):
218
+ self.dense1 = layers.Dense(
219
+ self.units,
220
+ kernel_regularizer=regularizers.l2(self.l2_reg)
221
+ )
222
+ self.bn1 = layers.BatchNormalization()
223
+ self.dropout1 = layers.Dropout(self.dropout_rate)
224
+
225
+ self.dense2 = layers.Dense(
226
+ self.units,
227
+ kernel_regularizer=regularizers.l2(self.l2_reg)
228
+ )
229
+ self.bn2 = layers.BatchNormalization()
230
+ self.dropout2 = layers.Dropout(self.dropout_rate)
231
+
232
+ # Skip connection projection if dimensions don't match
233
+ if input_shape[-1] != self.units:
234
+ self.skip_proj = layers.Dense(self.units, use_bias=False)
235
+ else:
236
+ self.skip_proj = None
237
+
238
+ def call(self, x, training=False):
239
+ # Main path
240
+ h = self.dense1(x)
241
+ h = self.bn1(h, training=training)
242
+ h = tf.nn.gelu(h) # GELU activation (better than ReLU)
243
+ h = self.dropout1(h, training=training)
244
+
245
+ h = self.dense2(h)
246
+ h = self.bn2(h, training=training)
247
+
248
+ # Skip connection
249
+ if self.skip_proj is not None:
250
+ x = self.skip_proj(x)
251
+
252
+ # Add and activate
253
+ out = tf.nn.gelu(h + x)
254
+ out = self.dropout2(out, training=training)
255
+
256
+ return out
257
+
258
+ def get_config(self):
259
+ config = super().get_config()
260
+ config.update({
261
+ 'units': self.units,
262
+ 'dropout_rate': self.dropout_rate,
263
+ 'l2_reg': self.l2_reg
264
+ })
265
+ return config
266
+
267
+
268
+ # ============================================================================
269
+ # ADVANCED DATA AUGMENTATION
270
+ # ============================================================================
271
+
272
+ class AdvancedAugmenter:
273
+ """Advanced augmentation for landmark data."""
274
+
275
+ def __init__(self, noise_std=0.03, scale_range=(0.85, 1.15),
276
+ rotation_range=20, shift_range=0.1):
277
+ self.noise_std = noise_std
278
+ self.scale_range = scale_range
279
+ self.rotation_range = rotation_range
280
+ self.shift_range = shift_range
281
+
282
+ def add_noise(self, X):
283
+ """Add Gaussian noise."""
284
+ noise = np.random.normal(0, self.noise_std, X.shape)
285
+ # Don't add noise to orientation features
286
+ if X.shape[1] > 126:
287
+ noise[:, 126:] = 0
288
+ return X + noise
289
+
290
+ def random_scale(self, X):
291
+ """Random scaling of landmarks."""
292
+ scale = np.random.uniform(*self.scale_range, size=(len(X), 1))
293
+ X_scaled = X.copy()
294
+ X_scaled[:, :126] *= scale
295
+ return X_scaled
296
+
297
+ def random_shift(self, X):
298
+ """Random translation of landmarks."""
299
+ shift = np.random.uniform(-self.shift_range, self.shift_range,
300
+ size=(len(X), 3))
301
+ X_shifted = X.copy()
302
+ # Apply same shift to all landmarks (x, y, z)
303
+ for i in range(21):
304
+ X_shifted[:, i*3:(i+1)*3] += shift
305
+ # Second hand if present
306
+ for i in range(21, 42):
307
+ X_shifted[:, i*3:(i+1)*3] += shift
308
+ return X_shifted
309
+
310
+ def random_rotation_2d(self, X):
311
+ """Random 2D rotation around z-axis."""
312
+ angles = np.random.uniform(-self.rotation_range, self.rotation_range,
313
+ size=len(X)) * np.pi / 180
314
+
315
+ X_rotated = X.copy()
316
+
317
+ for i, angle in enumerate(angles):
318
+ cos_a, sin_a = np.cos(angle), np.sin(angle)
319
+
320
+ # Rotate each landmark (x, y only)
321
+ for j in range(42): # 21 landmarks × 2 hands
322
+ if j * 3 + 2 < 126:
323
+ x = X_rotated[i, j*3]
324
+ y = X_rotated[i, j*3 + 1]
325
+ X_rotated[i, j*3] = x * cos_a - y * sin_a
326
+ X_rotated[i, j*3 + 1] = x * sin_a + y * cos_a
327
+
328
+ return X_rotated
329
+
330
+ def random_mirror(self, X):
331
+ """Randomly mirror hands (swap left/right)."""
332
+ X_mirrored = X.copy()
333
+ mask = np.random.random(len(X)) < 0.3 # 30% chance
334
+
335
+ for i in np.where(mask)[0]:
336
+ # Mirror x coordinates
337
+ for j in range(42):
338
+ if j * 3 < 126:
339
+ X_mirrored[i, j*3] = -X_mirrored[i, j*3]
340
+
341
+ # Swap hand orientation if present
342
+ if X.shape[1] > 126:
343
+ # Swap is_left flags
344
+ X_mirrored[i, 127] = 1.0 - X_mirrored[i, 127] if X_mirrored[i, 127] >= 0 else -1.0
345
+ X_mirrored[i, 129] = 1.0 - X_mirrored[i, 129] if X_mirrored[i, 129] >= 0 else -1.0
346
+
347
+ return X_mirrored
348
+
349
+ def random_finger_jitter(self, X):
350
+ """Add extra jitter to finger tips (most variable landmarks)."""
351
+ X_jittered = X.copy()
352
+ finger_tips = [4, 8, 12, 16, 20] # Thumb, index, middle, ring, pinky tips
353
+
354
+ for tip in finger_tips:
355
+ jitter = np.random.normal(0, self.noise_std * 2, (len(X), 3))
356
+ X_jittered[:, tip*3:(tip+1)*3] += jitter
357
+ # Second hand
358
+ if (tip + 21) * 3 + 3 <= 126:
359
+ X_jittered[:, (tip+21)*3:(tip+22)*3] += jitter
360
+
361
+ return X_jittered
362
+
363
+ def augment_batch(self, X, p=0.5):
364
+ """Apply random augmentations to batch."""
365
+ X_aug = X.copy()
366
+
367
+ # Apply each augmentation with probability p
368
+ if np.random.random() < p:
369
+ X_aug = self.add_noise(X_aug)
370
+ if np.random.random() < p * 0.7:
371
+ X_aug = self.random_scale(X_aug)
372
+ if np.random.random() < p * 0.5:
373
+ X_aug = self.random_rotation_2d(X_aug)
374
+ if np.random.random() < p * 0.3:
375
+ X_aug = self.random_shift(X_aug)
376
+ if np.random.random() < p * 0.3:
377
+ X_aug = self.random_finger_jitter(X_aug)
378
+ if np.random.random() < p * 0.2:
379
+ X_aug = self.random_mirror(X_aug)
380
+
381
+ return X_aug
382
+
383
+
384
+ # ============================================================================
385
+ # MIXUP AND CUTMIX
386
+ # ============================================================================
387
+
388
+ def mixup(X, y, alpha=0.3):
389
+ """Mixup augmentation."""
390
+ if alpha <= 0:
391
+ return X, y
392
+
393
+ batch_size = len(X)
394
+ lam = np.random.beta(alpha, alpha)
395
+ indices = np.random.permutation(batch_size)
396
+
397
+ X_mixed = lam * X + (1 - lam) * X[indices]
398
+ y_mixed = lam * y + (1 - lam) * y[indices]
399
+
400
+ return X_mixed, y_mixed
401
+
402
+
403
+ def cutmix(X, y, alpha=0.3):
404
+ """CutMix augmentation - mix portions of features."""
405
+ if alpha <= 0:
406
+ return X, y
407
+
408
+ batch_size = len(X)
409
+ lam = np.random.beta(alpha, alpha)
410
+ indices = np.random.permutation(batch_size)
411
+
412
+ # Determine cut size
413
+ feature_size = X.shape[1]
414
+ cut_size = int(feature_size * (1 - lam))
415
+ cut_start = np.random.randint(0, feature_size - cut_size + 1)
416
+
417
+ X_mixed = X.copy()
418
+ X_mixed[:, cut_start:cut_start+cut_size] = X[indices, cut_start:cut_start+cut_size]
419
+
420
+ # Adjust lambda based on actual cut
421
+ lam = 1 - cut_size / feature_size
422
+ y_mixed = lam * y + (1 - lam) * y[indices]
423
+
424
+ return X_mixed, y_mixed
425
+
426
+
427
+ # ============================================================================
428
+ # ADVANCED DATA GENERATOR
429
+ # ============================================================================
430
+
431
+ class AdvancedDataGenerator(keras.utils.Sequence):
432
+ """Advanced data generator with multiple augmentation techniques."""
433
+
434
+ def __init__(self, X, y, batch_size, augmenter, num_classes,
435
+ use_mixup=True, use_cutmix=True, shuffle=True,
436
+ hard_example_indices=None, hard_example_ratio=0.3):
437
+ self.X = X
438
+ self.y = y
439
+ self.batch_size = batch_size
440
+ self.augmenter = augmenter
441
+ self.num_classes = num_classes
442
+ self.use_mixup = use_mixup
443
+ self.use_cutmix = use_cutmix
444
+ self.shuffle = shuffle
445
+ self.hard_example_indices = hard_example_indices
446
+ self.hard_example_ratio = hard_example_ratio
447
+ self.indices = np.arange(len(X))
448
+ self.on_epoch_end()
449
+
450
+ def __len__(self):
451
+ return int(np.ceil(len(self.X) / self.batch_size))
452
+
453
+ def __getitem__(self, idx):
454
+ batch_indices = self.indices[idx * self.batch_size:(idx + 1) * self.batch_size]
455
+
456
+ # Optionally include hard examples
457
+ if self.hard_example_indices is not None and len(self.hard_example_indices) > 0:
458
+ n_hard = int(len(batch_indices) * self.hard_example_ratio)
459
+ hard_sample = np.random.choice(self.hard_example_indices,
460
+ size=min(n_hard, len(self.hard_example_indices)),
461
+ replace=False)
462
+ batch_indices = np.concatenate([batch_indices[:-n_hard], hard_sample])
463
+
464
+ X_batch = self.X[batch_indices].copy()
465
+ y_batch = self.y[batch_indices].copy()
466
+
467
+ # Apply augmentation
468
+ X_batch = self.augmenter.augment_batch(X_batch, p=0.7)
469
+
470
+ # Convert to one-hot
471
+ y_one_hot = keras.utils.to_categorical(y_batch, self.num_classes)
472
+
473
+ # Apply mixup or cutmix (not both at same time)
474
+ if self.use_mixup and self.use_cutmix:
475
+ if np.random.random() < 0.5:
476
+ X_batch, y_one_hot = mixup(X_batch, y_one_hot, MIXUP_ALPHA)
477
+ else:
478
+ X_batch, y_one_hot = cutmix(X_batch, y_one_hot, CUTMIX_ALPHA)
479
+ elif self.use_mixup:
480
+ X_batch, y_one_hot = mixup(X_batch, y_one_hot, MIXUP_ALPHA)
481
+ elif self.use_cutmix:
482
+ X_batch, y_one_hot = cutmix(X_batch, y_one_hot, CUTMIX_ALPHA)
483
+
484
+ return X_batch.astype(np.float32), y_one_hot.astype(np.float32)
485
+
486
+ def on_epoch_end(self):
487
+ if self.shuffle:
488
+ np.random.shuffle(self.indices)
489
+
490
+
491
+ # ============================================================================
492
+ # CREATE ADVANCED MODEL
493
+ # ============================================================================
494
+
495
+ def create_advanced_model(input_size):
496
+ """
497
+ Create an advanced model with attention and residual connections.
498
+ """
499
+ print("\n" + "=" * 60)
500
+ print("CREATING ADVANCED MODEL")
501
+ print("=" * 60)
502
+
503
+ l2_reg = regularizers.l2(0.001)
504
+
505
+ # Input
506
+ inputs = layers.Input(shape=(input_size,), name='input')
507
+
508
+ # Split landmarks and orientation
509
+ landmarks = layers.Lambda(lambda x: x[:, :126], name='landmarks')(inputs)
510
+
511
+ if input_size > 126:
512
+ orientation = layers.Lambda(lambda x: x[:, 126:], name='orientation')(inputs)
513
+
514
+ # ===== LANDMARK PROCESSING PATH =====
515
+
516
+ # Reshape for multi-scale processing (treat as 42 landmarks × 3 coords)
517
+ lm_reshaped = layers.Reshape((42, 3))(landmarks)
518
+
519
+ # Multi-scale feature extraction
520
+ # Scale 1: Per-landmark features
521
+ scale1 = layers.Dense(16, activation='gelu', kernel_regularizer=l2_reg)(lm_reshaped)
522
+ scale1 = layers.Flatten()(scale1)
523
+
524
+ # Scale 2: Landmark group features (fingers)
525
+ scale2 = layers.Conv1D(32, kernel_size=5, activation='gelu', padding='same',
526
+ kernel_regularizer=l2_reg)(lm_reshaped)
527
+ scale2 = layers.GlobalAveragePooling1D()(scale2)
528
+
529
+ # Scale 3: Global hand features
530
+ scale3 = layers.Dense(64, activation='gelu', kernel_regularizer=l2_reg)(landmarks)
531
+
532
+ # Combine scales
533
+ lm_features = layers.Concatenate()([scale1, scale2, scale3])
534
+ lm_features = layers.BatchNormalization()(lm_features)
535
+ lm_features = layers.Dropout(0.3)(lm_features)
536
+
537
+ # Channel attention
538
+ if USE_ATTENTION:
539
+ lm_features = ChannelAttention(reduction_ratio=4)(lm_features)
540
+
541
+ # Residual blocks
542
+ if USE_RESIDUAL:
543
+ x = ResidualBlock(256, dropout_rate=0.4, l2_reg=0.001)(lm_features)
544
+ x = ResidualBlock(128, dropout_rate=0.3, l2_reg=0.001)(x)
545
+ x = ResidualBlock(64, dropout_rate=0.3, l2_reg=0.001)(x)
546
+ else:
547
+ x = layers.Dense(256, activation='gelu', kernel_regularizer=l2_reg)(lm_features)
548
+ x = layers.BatchNormalization()(x)
549
+ x = layers.Dropout(0.4)(x)
550
+ x = layers.Dense(128, activation='gelu', kernel_regularizer=l2_reg)(x)
551
+ x = layers.BatchNormalization()(x)
552
+ x = layers.Dropout(0.3)(x)
553
+ x = layers.Dense(64, activation='gelu', kernel_regularizer=l2_reg)(x)
554
+ x = layers.BatchNormalization()(x)
555
+ x = layers.Dropout(0.3)(x)
556
+
557
+ # ===== ORIENTATION PATH (if available) =====
558
+ if input_size > 126:
559
+ orient_x = layers.Dense(16, activation='gelu', kernel_regularizer=l2_reg)(orientation)
560
+ orient_x = layers.BatchNormalization()(orient_x)
561
+ orient_x = layers.Dense(8, activation='gelu', kernel_regularizer=l2_reg)(orient_x)
562
+
563
+ # Combine with main path
564
+ x = layers.Concatenate()([x, orient_x])
565
+
566
+ # ===== CLASSIFICATION HEAD =====
567
+ x = layers.Dense(64, activation='gelu', kernel_regularizer=l2_reg)(x)
568
+ x = layers.BatchNormalization()(x)
569
+ x = layers.Dropout(0.2)(x)
570
+
571
+ x = layers.Dense(32, activation='gelu', kernel_regularizer=l2_reg)(x)
572
+ x = layers.BatchNormalization()(x)
573
+ x = layers.Dropout(0.2)(x)
574
+
575
+ # Output
576
+ outputs = layers.Dense(NUM_CLASSES, activation='softmax', name='output')(x)
577
+
578
+ model = Model(inputs=inputs, outputs=outputs)
579
+
580
+ # Compile with focal loss or standard loss
581
+ if USE_FOCAL_LOSS:
582
+ loss = FocalLoss(alpha=FOCAL_ALPHA, gamma=FOCAL_GAMMA)
583
+ else:
584
+ loss = keras.losses.CategoricalCrossentropy(label_smoothing=0.1)
585
+
586
+ # Use Adam with weight decay (compatible with TF 2.x)
587
+ try:
588
+ # TensorFlow 2.11+
589
+ optimizer = keras.optimizers.Adam(
590
+ learning_rate=INITIAL_LR,
591
+ )
592
+ except:
593
+ optimizer = keras.optimizers.Adam(learning_rate=INITIAL_LR)
594
+
595
+ model.compile(
596
+ optimizer=optimizer,
597
+ loss=loss,
598
+ metrics=['accuracy']
599
+ )
600
+
601
+ print(f"\n📐 Model Summary:")
602
+ model.summary()
603
+ print(f"\n Total parameters: {model.count_params():,}")
604
+ print(f" Using Focal Loss: {USE_FOCAL_LOSS}")
605
+ print(f" Using Attention: {USE_ATTENTION}")
606
+ print(f" Using Residual: {USE_RESIDUAL}")
607
+
608
+ return model
609
+
610
+
611
+ # ============================================================================
612
+ # LEARNING RATE SCHEDULE
613
+ # ============================================================================
614
+
615
+ class WarmupCosineDecay(Callback):
616
+ """Warmup + Cosine decay with restarts."""
617
+
618
+ def __init__(self, max_lr, min_lr, warmup_epochs, total_epochs, restarts=2):
619
+ super().__init__()
620
+ self.max_lr = max_lr
621
+ self.min_lr = min_lr
622
+ self.warmup_epochs = warmup_epochs
623
+ self.total_epochs = total_epochs
624
+ self.restarts = restarts
625
+ self.cycle_length = (total_epochs - warmup_epochs) // (restarts + 1)
626
+
627
+ def on_epoch_begin(self, epoch, logs=None):
628
+ if epoch < self.warmup_epochs:
629
+ lr = self.max_lr * (epoch + 1) / self.warmup_epochs
630
+ else:
631
+ epoch_in_cycle = (epoch - self.warmup_epochs) % self.cycle_length
632
+ # Cosine decay within cycle
633
+ lr = self.min_lr + 0.5 * (self.max_lr - self.min_lr) * \
634
+ (1 + math.cos(math.pi * epoch_in_cycle / self.cycle_length))
635
+
636
+ # Reduce max_lr after each restart
637
+ cycle_num = (epoch - self.warmup_epochs) // self.cycle_length
638
+ lr *= (0.8 ** cycle_num)
639
+
640
+ keras.backend.set_value(self.model.optimizer.learning_rate, lr)
641
+
642
+ def on_epoch_end(self, epoch, logs=None):
643
+ logs = logs or {}
644
+ logs['lr'] = float(keras.backend.get_value(self.model.optimizer.learning_rate))
645
+
646
+
647
+ # ============================================================================
648
+ # LOAD DATA
649
+ # ============================================================================
650
+
651
+ def load_data():
652
+ """Load and prepare data."""
653
+ print("\n" + "=" * 60)
654
+ print("LOADING DATA")
655
+ print("=" * 60)
656
+
657
+ if not os.path.exists(INPUT_CSV):
658
+ print(f"❌ ERROR: {INPUT_CSV} not found!")
659
+ return None, None, None
660
+
661
+ X, y = [], []
662
+
663
+ with open(INPUT_CSV, 'r') as f:
664
+ reader = csv.reader(f)
665
+ next(reader) # Skip header
666
+
667
+ for row in reader:
668
+ features = [float(val) for val in row[:-1]]
669
+ label_str = row[-1]
670
+
671
+ if label_str in LABELS:
672
+ X.append(features)
673
+ y.append(LABELS.index(label_str))
674
+
675
+ X = np.array(X, dtype=np.float32)
676
+ y = np.array(y, dtype=np.int32)
677
+
678
+ print(f"✅ Loaded {len(X)} samples")
679
+ print(f" Feature shape: {X.shape}")
680
+
681
+ # Find hard examples (confused pairs)
682
+ hard_indices = []
683
+ for sign_a, sign_b in CONFUSED_PAIRS:
684
+ if sign_a in LABELS and sign_b in LABELS:
685
+ idx_a, idx_b = LABELS.index(sign_a), LABELS.index(sign_b)
686
+ hard_indices.extend(np.where((y == idx_a) | (y == idx_b))[0].tolist())
687
+
688
+ hard_indices = list(set(hard_indices))
689
+ print(f" Hard examples (confused pairs): {len(hard_indices)}")
690
+
691
+ # Class distribution
692
+ unique, counts = np.unique(y, return_counts=True)
693
+ print(f"\n📊 Class distribution:")
694
+ min_count = min(counts)
695
+ max_count = max(counts)
696
+ print(f" Min samples: {min_count} ({LABELS[unique[np.argmin(counts)]]})")
697
+ print(f" Max samples: {max_count} ({LABELS[unique[np.argmax(counts)]]})")
698
+ print(f" Imbalance ratio: {max_count/min_count:.2f}")
699
+
700
+ return X, y, hard_indices
701
+
702
+
703
+ # ============================================================================
704
+ # TRAINING
705
+ # ============================================================================
706
+
707
+ def train_model(model, X, y, hard_indices):
708
+ """Train with advanced techniques."""
709
+ print("\n" + "=" * 60)
710
+ print(f"TRAINING FOR {EPOCHS} EPOCHS")
711
+ print("=" * 60)
712
+
713
+ # Split data
714
+ X_train, X_val, y_train, y_val = train_test_split(
715
+ X, y, test_size=VALIDATION_SPLIT, random_state=42, stratify=y
716
+ )
717
+
718
+ print(f"\n📊 Data split:")
719
+ print(f" Training: {len(X_train)} samples")
720
+ print(f" Validation: {len(X_val)} samples")
721
+
722
+ # Compute class weights
723
+ class_weights = None
724
+ if USE_CLASS_WEIGHTS:
725
+ weights = compute_class_weight('balanced', classes=np.unique(y_train), y=y_train)
726
+ class_weights = dict(enumerate(weights))
727
+ print(f" Using class weights: min={min(weights):.2f}, max={max(weights):.2f}")
728
+
729
+ # Find hard examples in training set
730
+ train_hard_indices = []
731
+ if hard_indices is not None:
732
+ train_indices_set = set(range(len(X_train)))
733
+ # Map original indices to training indices (approximation)
734
+ for h_idx in hard_indices:
735
+ if h_idx < len(X_train):
736
+ train_hard_indices.append(h_idx)
737
+
738
+ # Create augmenter and generator
739
+ augmenter = AdvancedAugmenter(
740
+ noise_std=AUG_NOISE_STD,
741
+ scale_range=AUG_SCALE_RANGE,
742
+ rotation_range=AUG_ROTATION_RANGE,
743
+ shift_range=AUG_SHIFT_RANGE
744
+ )
745
+
746
+ train_gen = AdvancedDataGenerator(
747
+ X_train, y_train, BATCH_SIZE, augmenter, NUM_CLASSES,
748
+ use_mixup=True, use_cutmix=True,
749
+ hard_example_indices=train_hard_indices,
750
+ hard_example_ratio=0.2
751
+ )
752
+
753
+ # Validation data
754
+ y_val_one_hot = keras.utils.to_categorical(y_val, NUM_CLASSES)
755
+
756
+ # Callbacks
757
+ callbacks = [
758
+ WarmupCosineDecay(INITIAL_LR, MIN_LR, WARMUP_EPOCHS, EPOCHS, restarts=3),
759
+
760
+ keras.callbacks.ModelCheckpoint(
761
+ OUTPUT_H5,
762
+ monitor='val_accuracy',
763
+ save_best_only=True,
764
+ verbose=1
765
+ ),
766
+
767
+ keras.callbacks.ModelCheckpoint(
768
+ 'isl_model_advanced_latest.h5',
769
+ save_best_only=False,
770
+ verbose=0
771
+ ),
772
+
773
+ # Reduce LR if stuck (backup scheduler)
774
+ keras.callbacks.ReduceLROnPlateau(
775
+ monitor='val_loss',
776
+ factor=0.5,
777
+ patience=15,
778
+ min_lr=MIN_LR,
779
+ verbose=1
780
+ ),
781
+
782
+ # Progress logging
783
+ keras.callbacks.LambdaCallback(
784
+ on_epoch_end=lambda epoch, logs: print(
785
+ f"\n 📈 Epoch {epoch+1}/{EPOCHS} - "
786
+ f"acc: {logs.get('accuracy', 0)*100:.2f}% - "
787
+ f"val_acc: {logs.get('val_accuracy', 0)*100:.2f}% - "
788
+ f"lr: {logs.get('lr', 0):.6f}"
789
+ ) if (epoch + 1) % 10 == 0 else None
790
+ )
791
+ ]
792
+
793
+ # Train
794
+ print(f"\n🚀 Starting training...")
795
+ print(f" Techniques: Focal Loss, Attention, Residual, Mixup, CutMix")
796
+ print(f" Augmentation: Noise, Scale, Rotation, Shift, Mirror\n")
797
+
798
+ history = model.fit(
799
+ train_gen,
800
+ validation_data=(X_val, y_val_one_hot),
801
+ epochs=EPOCHS,
802
+ callbacks=callbacks,
803
+ class_weight=class_weights,
804
+ verbose=1
805
+ )
806
+
807
+ # Load best model
808
+ print("\n📂 Loading best model...")
809
+ model = keras.models.load_model(
810
+ OUTPUT_H5,
811
+ custom_objects={
812
+ 'FocalLoss': FocalLoss,
813
+ 'ChannelAttention': ChannelAttention,
814
+ 'ResidualBlock': ResidualBlock
815
+ }
816
+ )
817
+
818
+ return history, model, X_val, y_val
819
+
820
+
821
+ # ============================================================================
822
+ # TEST-TIME AUGMENTATION
823
+ # ============================================================================
824
+
825
+ def predict_with_tta(model, X, n_augments=5):
826
+ """Prediction with test-time augmentation."""
827
+ augmenter = AdvancedAugmenter(noise_std=0.01, scale_range=(0.95, 1.05))
828
+
829
+ predictions = []
830
+
831
+ # Original prediction
832
+ predictions.append(model.predict(X, verbose=0))
833
+
834
+ # Augmented predictions
835
+ for _ in range(n_augments - 1):
836
+ X_aug = augmenter.augment_batch(X, p=0.5)
837
+ predictions.append(model.predict(X_aug, verbose=0))
838
+
839
+ # Average predictions
840
+ avg_pred = np.mean(predictions, axis=0)
841
+ return avg_pred
842
+
843
+
844
+ # ============================================================================
845
+ # EVALUATION
846
+ # ============================================================================
847
+
848
+ def evaluate_model(model, X_val, y_val, use_tta=True):
849
+ """Comprehensive evaluation."""
850
+ print("\n" + "=" * 60)
851
+ print("EVALUATION")
852
+ print("=" * 60)
853
+
854
+ # Standard prediction
855
+ y_val_one_hot = keras.utils.to_categorical(y_val, NUM_CLASSES)
856
+ val_loss, val_acc = model.evaluate(X_val, y_val_one_hot, verbose=0)
857
+
858
+ print(f"\n📈 Standard Evaluation:")
859
+ print(f" Validation Accuracy: {val_acc*100:.2f}%")
860
+ print(f" Validation Loss: {val_loss:.4f}")
861
+
862
+ # TTA prediction
863
+ if use_tta:
864
+ print(f"\n🔄 Test-Time Augmentation (TTA):")
865
+ tta_pred = predict_with_tta(model, X_val, n_augments=5)
866
+ tta_classes = np.argmax(tta_pred, axis=1)
867
+ tta_acc = np.mean(tta_classes == y_val)
868
+ print(f" TTA Accuracy: {tta_acc*100:.2f}%")
869
+ else:
870
+ tta_pred = model.predict(X_val, verbose=0)
871
+ tta_classes = np.argmax(tta_pred, axis=1)
872
+
873
+ # Classification report
874
+ print("\n" + "-" * 60)
875
+ print("CLASSIFICATION REPORT")
876
+ print("-" * 60)
877
+ print(classification_report(y_val, tta_classes, target_names=LABELS, digits=3))
878
+
879
+ # Confusion analysis for hard pairs
880
+ print("\n" + "-" * 60)
881
+ print("CONFUSED PAIRS ANALYSIS")
882
+ print("-" * 60)
883
+
884
+ total_confusion = 0
885
+ for sign_a, sign_b in CONFUSED_PAIRS:
886
+ if sign_a in LABELS and sign_b in LABELS:
887
+ idx_a, idx_b = LABELS.index(sign_a), LABELS.index(sign_b)
888
+
889
+ mask_a = y_val == idx_a
890
+ mask_b = y_val == idx_b
891
+
892
+ a_as_b = np.sum((tta_classes == idx_b) & mask_a)
893
+ b_as_a = np.sum((tta_classes == idx_a) & mask_b)
894
+
895
+ if a_as_b > 0 or b_as_a > 0:
896
+ print(f" ⚠️ {sign_a} ↔ {sign_b}: {sign_a}→{sign_b}={a_as_b}, {sign_b}→{sign_a}={b_as_a}")
897
+ total_confusion += a_as_b + b_as_a
898
+ else:
899
+ print(f" ✅ {sign_a} ↔ {sign_b}: Perfect!")
900
+
901
+ print(f"\n Total confusions in pairs: {total_confusion}")
902
+
903
+ return val_acc
904
+
905
+
906
+ # ============================================================================
907
+ # CONVERT TO TFLITE
908
+ # ============================================================================
909
+
910
+ def convert_to_tflite(model):
911
+ """Convert to optimized TFLite."""
912
+ print("\n" + "=" * 60)
913
+ print("CONVERTING TO TFLITE")
914
+ print("=" * 60)
915
+
916
+ # Save a version without custom objects for TFLite conversion
917
+ # Re-create model with standard layers
918
+ print(" Creating export-friendly model...")
919
+
920
+ input_size = model.input_shape[1]
921
+
922
+ # Simple export model (without custom layers)
923
+ export_model = keras.Sequential([
924
+ layers.Input(shape=(input_size,)),
925
+ layers.Dense(512, activation='gelu'),
926
+ layers.BatchNormalization(),
927
+ layers.Dropout(0.4),
928
+ layers.Dense(256, activation='gelu'),
929
+ layers.BatchNormalization(),
930
+ layers.Dropout(0.3),
931
+ layers.Dense(128, activation='gelu'),
932
+ layers.BatchNormalization(),
933
+ layers.Dropout(0.3),
934
+ layers.Dense(64, activation='gelu'),
935
+ layers.BatchNormalization(),
936
+ layers.Dropout(0.2),
937
+ layers.Dense(32, activation='gelu'),
938
+ layers.BatchNormalization(),
939
+ layers.Dropout(0.2),
940
+ layers.Dense(NUM_CLASSES, activation='softmax')
941
+ ])
942
+
943
+ export_model.compile(
944
+ optimizer='adam',
945
+ loss='categorical_crossentropy',
946
+ metrics=['accuracy']
947
+ )
948
+
949
+ # Knowledge distillation: train export model on advanced model's predictions
950
+ print(" Performing knowledge distillation...")
951
+
952
+ # Load training data for distillation
953
+ X_all, y_all, _ = load_data()
954
+ if X_all is not None:
955
+ # Get soft labels from advanced model
956
+ soft_labels = model.predict(X_all, verbose=0)
957
+
958
+ # Train export model
959
+ export_model.fit(
960
+ X_all, soft_labels,
961
+ epochs=20,
962
+ batch_size=64,
963
+ verbose=1
964
+ )
965
+
966
+ # Convert to TFLite
967
+ print("\n Converting to TFLite...")
968
+ converter = tf.lite.TFLiteConverter.from_keras_model(export_model)
969
+ converter.optimizations = [tf.lite.Optimize.DEFAULT]
970
+
971
+ # Quantization for smaller size
972
+ converter.target_spec.supported_types = [tf.float16]
973
+
974
+ tflite_model = converter.convert()
975
+
976
+ with open(OUTPUT_MODEL, 'wb') as f:
977
+ f.write(tflite_model)
978
+
979
+ print(f"\n✅ TFLite model saved: {OUTPUT_MODEL}")
980
+ print(f" Size: {os.path.getsize(OUTPUT_MODEL) / 1024:.2f} KB")
981
+
982
+ # Verify
983
+ print("\n📋 Verifying TFLite model...")
984
+ interpreter = tf.lite.Interpreter(model_content=tflite_model)
985
+ interpreter.allocate_tensors()
986
+
987
+ input_details = interpreter.get_input_details()
988
+ output_details = interpreter.get_output_details()
989
+
990
+ print(f" Input shape: {input_details[0]['shape']}")
991
+ print(f" Output shape: {output_details[0]['shape']}")
992
+
993
+ return tflite_model
994
+
995
+
996
+ # ============================================================================
997
+ # SAVE CONFIG
998
+ # ============================================================================
999
+
1000
+ def save_config(val_acc):
1001
+ """Save configuration."""
1002
+ config = {
1003
+ 'labels': LABELS,
1004
+ 'num_classes': NUM_CLASSES,
1005
+ 'input_size': 130,
1006
+ 'validation_accuracy': float(val_acc),
1007
+ 'training_config': {
1008
+ 'epochs': EPOCHS,
1009
+ 'batch_size': BATCH_SIZE,
1010
+ 'focal_loss': USE_FOCAL_LOSS,
1011
+ 'attention': USE_ATTENTION,
1012
+ 'residual': USE_RESIDUAL,
1013
+ 'augmentation': {
1014
+ 'noise_std': AUG_NOISE_STD,
1015
+ 'scale_range': list(AUG_SCALE_RANGE),
1016
+ 'rotation_range': AUG_ROTATION_RANGE,
1017
+ 'mixup_alpha': MIXUP_ALPHA,
1018
+ 'cutmix_alpha': CUTMIX_ALPHA
1019
+ }
1020
+ },
1021
+ 'model_files': {
1022
+ 'h5': OUTPUT_H5,
1023
+ 'tflite': OUTPUT_MODEL
1024
+ }
1025
+ }
1026
+
1027
+ with open(OUTPUT_LABELS, 'w') as f:
1028
+ json.dump(config, f, indent=2)
1029
+
1030
+ print(f"✅ Config saved: {OUTPUT_LABELS}")
1031
+
1032
+
1033
+ # ============================================================================
1034
+ # MAIN
1035
+ # ============================================================================
1036
+
1037
+ def main():
1038
+ print("\n" + "=" * 70)
1039
+ print("🚀 ADVANCED ISL MODEL TRAINING - MAXIMUM ACCURACY")
1040
+ print("=" * 70)
1041
+ print("\nTarget: 9-10/10 accuracy with real-world robustness")
1042
+ print("\nAdvanced Techniques:")
1043
+ print(" ✓ Focal Loss - Focus on hard examples")
1044
+ print(" ✓ Attention Mechanism - Important features")
1045
+ print(" ✓ Residual Connections - Better gradients")
1046
+ print(" ✓ Multi-Scale Features - Different patterns")
1047
+ print(" ✓ Advanced Augmentation - Realistic variations")
1048
+ print(" ✓ Mixup + CutMix - Better generalization")
1049
+ print(" ✓ Class Balancing - Handle imbalance")
1050
+ print(" ✓ Hard Example Mining - Confused pairs")
1051
+ print(" ✓ Test-Time Augmentation - Better inference")
1052
+ print(" ✓ Knowledge Distillation - Better TFLite")
1053
+
1054
+ # Load data
1055
+ X, y, hard_indices = load_data()
1056
+ if X is None:
1057
+ return
1058
+
1059
+ input_size = X.shape[1]
1060
+
1061
+ # Create model
1062
+ model = create_advanced_model(input_size)
1063
+
1064
+ # Train
1065
+ history, model, X_val, y_val = train_model(model, X, y, hard_indices)
1066
+
1067
+ # Evaluate
1068
+ val_acc = evaluate_model(model, X_val, y_val, use_tta=True)
1069
+
1070
+ # Convert to TFLite
1071
+ convert_to_tflite(model)
1072
+
1073
+ # Save config
1074
+ save_config(val_acc)
1075
+
1076
+ # Final summary
1077
+ print("\n" + "=" * 70)
1078
+ print("✅ TRAINING COMPLETE!")
1079
+ print("=" * 70)
1080
+ print(f"\n🎯 Final Validation Accuracy: {val_acc*100:.2f}%")
1081
+ print(f"\n📁 Output files:")
1082
+ print(f" 📄 {OUTPUT_H5} - Best Keras model")
1083
+ print(f" 📄 {OUTPUT_MODEL} - TFLite for Android")
1084
+ print(f" 📄 {OUTPUT_LABELS} - Configuration")
1085
+ print("\n💡 Tips for even better results:")
1086
+ print(" 1. Collect more data for underperforming classes")
1087
+ print(" 2. Add more augmented samples for confused pairs")
1088
+ print(" 3. Use ensemble of multiple models")
1089
+ print(" 4. Fine-tune on real webcam data")
1090
+
1091
+
1092
+ if __name__ == "__main__":
1093
+ main()