Moaz715 commited on
Commit
a413a9a
·
1 Parent(s): 0aa2ff0
Files changed (1) hide show
  1. SVM/multiclass_svm.py +2 -202
SVM/multiclass_svm.py CHANGED
@@ -1,20 +1,11 @@
1
  import numpy as np
2
- import matplotlib.pyplot as plt
3
- from sklearn.datasets import fetch_openml
4
- from sklearn.model_selection import train_test_split
5
- from sklearn.preprocessing import StandardScaler
6
- from sklearn.metrics import (accuracy_score, precision_score, recall_score,
7
- f1_score, classification_report,
8
- confusion_matrix, ConfusionMatrixDisplay)
9
 
 
10
  try:
11
  from linear_svm import LinearSVMScartch
12
  except ModuleNotFoundError:
13
  from SVM.linear_svm import LinearSVMScartch
14
 
15
- RANDOM_STATE = 42
16
-
17
-
18
  # ---------------------------------------------------------------------------
19
  # One-vs-Rest multiclass wrapper around LinearSVMScartch
20
  # ---------------------------------------------------------------------------
@@ -59,195 +50,4 @@ class MultiClassSVMScratch:
59
  def predict(self, X):
60
  # Pick the class whose binary SVM is most confident (+highest score)
61
  scores = self.decision_scores(X)
62
- return self.classes_[np.argmax(scores, axis=1)]
63
-
64
-
65
- # ---------------------------------------------------------------------------
66
- # Data loading
67
- # ---------------------------------------------------------------------------
68
-
69
- def load_data(n_dev=5000):
70
- print("Loading MNIST...")
71
- mnist = fetch_openml('mnist_784', version=1, as_frame=False,
72
- parser='auto', data_home='d:/sklearn_data')
73
- X, y = mnist.data, mnist.target.astype(int)
74
-
75
- # Normalize pixels to [0, 1]
76
- X = X / 255.0
77
-
78
- # 60 / 20 / 20 stratified split
79
- X_trainval, X_test, y_trainval, y_test = train_test_split(
80
- X, y, test_size=0.20, stratify=y, random_state=RANDOM_STATE)
81
- X_train, X_val, y_train, y_val = train_test_split(
82
- X_trainval, y_trainval, test_size=0.25, stratify=y_trainval,
83
- random_state=RANDOM_STATE)
84
-
85
- print(f"Train: {len(y_train)} Val: {len(y_val)} Test: {len(y_test)}")
86
-
87
- # Standardize — fit on train only to avoid leakage
88
- scaler = StandardScaler()
89
- X_train_s = scaler.fit_transform(X_train)
90
- X_val_s = scaler.transform(X_val)
91
- X_test_s = scaler.transform(X_test)
92
-
93
- # Small dev subset for fast hyperparameter search
94
- rng = np.random.RandomState(RANDOM_STATE)
95
- dev_idx = rng.choice(len(X_train_s), size=n_dev, replace=False)
96
- X_dev, y_dev = X_train_s[dev_idx], y_train[dev_idx]
97
- print(f"Dev subset: {len(y_dev)} samples")
98
-
99
- return X_train_s, X_val_s, X_test_s, y_train, y_val, y_test, X_dev, y_dev, scaler
100
-
101
-
102
- # ---------------------------------------------------------------------------
103
- # Manual grid search
104
- # ---------------------------------------------------------------------------
105
-
106
- def manual_grid_search(X_dev, y_dev, X_val, y_val):
107
- C_values = [0.1, 1.0, 10.0]
108
- lr_values = [0.001, 0.0005, 0.0001]
109
-
110
- results = []
111
- print("\n--- Hyperparameter Grid Search ---")
112
- print(f"{'C':>6} {'LR':>8} {'Val Acc':>9}")
113
- print("-" * 30)
114
-
115
- for C in C_values:
116
- for lr in lr_values:
117
- model = MultiClassSVMScratch(C=C, learning_rate=lr,
118
- n_epochs=30, random_state=RANDOM_STATE)
119
- model.fit(X_dev, y_dev)
120
- acc = accuracy_score(y_val, model.predict(X_val))
121
- results.append((C, lr, acc))
122
- print(f"{C:>6} {lr:>8} {acc:>9.4f}")
123
-
124
- best = max(results, key=lambda x: x[2])
125
- print(f"\nBest: C={best[0]}, lr={best[1]}, Val Acc={best[2]:.4f}")
126
- return best[0], best[1]
127
-
128
-
129
- # ---------------------------------------------------------------------------
130
- # Plotting helpers
131
- # ---------------------------------------------------------------------------
132
-
133
- def plot_loss_curves(model):
134
- fig, ax = plt.subplots(figsize=(10, 5))
135
- for k in model.classes_:
136
- ax.plot(model.classifiers[k].history, label=f"Digit {k}")
137
- ax.set_xlabel("Epoch")
138
- ax.set_ylabel("Loss")
139
- ax.set_title("Training Loss per Binary SVM (OvR)")
140
- ax.legend(ncol=5, fontsize=8)
141
- ax.grid(True)
142
- plt.tight_layout()
143
- plt.show()
144
-
145
-
146
- def plot_confusion_matrix(y_true, y_pred):
147
- cm = confusion_matrix(y_true, y_pred)
148
- disp = ConfusionMatrixDisplay(cm, display_labels=list(range(10)))
149
- fig, ax = plt.subplots(figsize=(9, 9))
150
- disp.plot(ax=ax, cmap="Blues", colorbar=False)
151
- ax.set_title("Confusion Matrix — Multiclass SVM (Scratch)")
152
- plt.tight_layout()
153
- plt.show()
154
-
155
-
156
- def plot_decision_scores(model, X_sample, true_label):
157
- scores = model.decision_scores(X_sample.reshape(1, -1))[0]
158
- colors = ["steelblue"] * 10
159
- colors[true_label] = "tomato"
160
- fig, ax = plt.subplots(figsize=(8, 4))
161
- ax.bar(model.classes_, scores, color=colors)
162
- ax.set_xlabel("Digit class")
163
- ax.set_ylabel("Decision score (w·x + b)")
164
- ax.set_title(f"Decision scores for one sample (true label = {true_label})")
165
- ax.axhline(0, color="black", linewidth=0.8, linestyle="--")
166
- plt.tight_layout()
167
- plt.show()
168
-
169
-
170
- def plot_per_class_metrics(y_true, y_pred):
171
- """Bar chart + table showing accuracy, precision, recall, F1 per digit class."""
172
- classes = np.arange(10)
173
- cm = confusion_matrix(y_true, y_pred, labels=classes)
174
- per_class_acc = cm.diagonal() / cm.sum(axis=1)
175
- precision = precision_score(y_true, y_pred, average=None, labels=classes, zero_division=0)
176
- recall = recall_score(y_true, y_pred, average=None, labels=classes, zero_division=0)
177
- f1 = f1_score(y_true, y_pred, average=None, labels=classes, zero_division=0)
178
-
179
- print("\n=== PER-CLASS METRICS ===")
180
- print(f"{'Digit':>6} {'Accuracy':>9} {'Precision':>10} {'Recall':>8} {'F1':>8}")
181
- print("-" * 50)
182
- for c in classes:
183
- print(f"{c:>6} {per_class_acc[c]:>9.4f} {precision[c]:>10.4f} {recall[c]:>8.4f} {f1[c]:>8.4f}")
184
-
185
- x = np.arange(10)
186
- width = 0.2
187
- fig, ax = plt.subplots(figsize=(12, 5))
188
- ax.bar(x - 1.5*width, per_class_acc, width, label="Accuracy")
189
- ax.bar(x - 0.5*width, precision, width, label="Precision")
190
- ax.bar(x + 0.5*width, recall, width, label="Recall")
191
- ax.bar(x + 1.5*width, f1, width, label="F1")
192
- ax.set_xticks(x)
193
- ax.set_xticklabels([f"Digit {c}" for c in classes], rotation=45)
194
- ax.set_ylabel("Score")
195
- ax.set_title("Per-Class Metrics — Multiclass SVM (Scratch)")
196
- ax.set_ylim(0, 1.05)
197
- ax.legend()
198
- ax.grid(axis="y", linestyle="--", alpha=0.5)
199
- plt.tight_layout()
200
- plt.show()
201
-
202
-
203
- def test_single_digit(model, X_test, y_test, digit=7):
204
- """Find one test image of `digit`, run it through the model, print prediction."""
205
- idx = np.where(y_test == digit)[0][0]
206
- X_sample = X_test[idx]
207
- predicted = model.predict(X_sample.reshape(1, -1))[0]
208
- print(f"Input: {digit} → Predicted: {predicted} ({'CORRECT' if predicted == digit else 'WRONG'})")
209
-
210
-
211
- # ---------------------------------------------------------------------------
212
- # Main
213
- # ---------------------------------------------------------------------------
214
-
215
- if __name__ == "__main__":
216
-
217
- # 1. Load data
218
- X_train, X_val, X_test, y_train, y_val, y_test, X_dev, y_dev, scaler = load_data()
219
-
220
- # 2. Hyperparameter tuning
221
- best_C, best_lr = manual_grid_search(X_dev, y_dev, X_val, y_val)
222
-
223
- # 3. Train final model on full training set
224
- print("\nTraining final model on full training set...")
225
- final_model = MultiClassSVMScratch(C=best_C, learning_rate=best_lr,
226
- n_epochs=50, random_state=RANDOM_STATE)
227
- final_model.fit(X_train, y_train)
228
- print("Done.")
229
-
230
- # 4. Evaluate on test set
231
- y_pred = final_model.predict(X_test)
232
-
233
- print("\n=== TEST SET RESULTS (Scratch SVM) ===")
234
- print(f"Accuracy : {accuracy_score(y_test, y_pred):.4f}")
235
- print(f"Precision (macro) : {precision_score(y_test, y_pred, average='macro'):.4f}")
236
- print(f"Recall (macro) : {recall_score(y_test, y_pred, average='macro'):.4f}")
237
- print(f"F1 (macro) : {f1_score(y_test, y_pred, average='macro'):.4f}")
238
- print(f"F1 (weighted): {f1_score(y_test, y_pred, average='weighted'):.4f}")
239
- print("\nClassification Report:")
240
- print(classification_report(y_test, y_pred))
241
-
242
-
243
- # 6. Plots
244
- plot_loss_curves(final_model)
245
- plot_confusion_matrix(y_test, y_pred)
246
- plot_per_class_metrics(y_test, y_pred)
247
-
248
- # Decision score bar chart for one sample (pick first test sample)
249
- sample_idx = 0
250
- plot_decision_scores(final_model, X_test[sample_idx], y_test[sample_idx])
251
-
252
- # 7. Single image test — change digit= to any digit 0-9
253
- test_single_digit(final_model, X_test, y_test, digit=4)
 
1
  import numpy as np
 
 
 
 
 
 
 
2
 
3
+ # We keep this because your Multiclass model uses the binary model inside it!
4
  try:
5
  from linear_svm import LinearSVMScartch
6
  except ModuleNotFoundError:
7
  from SVM.linear_svm import LinearSVMScartch
8
 
 
 
 
9
  # ---------------------------------------------------------------------------
10
  # One-vs-Rest multiclass wrapper around LinearSVMScartch
11
  # ---------------------------------------------------------------------------
 
50
  def predict(self, X):
51
  # Pick the class whose binary SVM is most confident (+highest score)
52
  scores = self.decision_scores(X)
53
+ return self.classes_[np.argmax(scores, axis=1)]