Moaz715 commited on
Commit
ffbf3ef
·
1 Parent(s): a12ed33
LogisticRegresssion/LogisticRegression.py CHANGED
@@ -151,7 +151,7 @@ class MultiLogReg():
151
  db = np.sum(loss_weights*(probas-y_one_hot), axis=0) / weight_sum
152
 
153
  if self.reg_eqn == 'L2':
154
- dw += (2 * self.reg_param / weight_sum) * self.weights
155
  elif self.reg_eqn == 'L1':
156
  dw += (self.reg_param / weight_sum) * np.sign(self.weights)
157
 
 
151
  db = np.sum(loss_weights*(probas-y_one_hot), axis=0) / weight_sum
152
 
153
  if self.reg_eqn == 'L2':
154
+ dw += (self.reg_param / weight_sum) * self.weights
155
  elif self.reg_eqn == 'L1':
156
  dw += (self.reg_param / weight_sum) * np.sign(self.weights)
157
 
LogisticRegresssion/__pycache__/LogisticRegression.cpython-313.pyc CHANGED
Binary files a/LogisticRegresssion/__pycache__/LogisticRegression.cpython-313.pyc and b/LogisticRegresssion/__pycache__/LogisticRegression.cpython-313.pyc differ
 
RF/__pycache__/grappaBroskiSkibiddiLewandoski.cpython-313.pyc CHANGED
Binary files a/RF/__pycache__/grappaBroskiSkibiddiLewandoski.cpython-313.pyc and b/RF/__pycache__/grappaBroskiSkibiddiLewandoski.cpython-313.pyc differ
 
SVM/__pycache__/multiclass_svm.cpython-313.pyc ADDED
Binary file (12.9 kB). View file
 
SVM/multiclass_svm.py ADDED
@@ -0,0 +1,253 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import numpy as np
2
+ import matplotlib.pyplot as plt
3
+ from sklearn.datasets import fetch_openml
4
+ from sklearn.model_selection import train_test_split
5
+ from sklearn.preprocessing import StandardScaler
6
+ from sklearn.metrics import (accuracy_score, precision_score, recall_score,
7
+ f1_score, classification_report,
8
+ confusion_matrix, ConfusionMatrixDisplay)
9
+
10
+ try:
11
+ from linear_svm import LinearSVMScartch
12
+ except ModuleNotFoundError:
13
+ from SVM.linear_svm import LinearSVMScartch
14
+
15
+ RANDOM_STATE = 42
16
+
17
+
18
+ # ---------------------------------------------------------------------------
19
+ # One-vs-Rest multiclass wrapper around LinearSVMScartch
20
+ # ---------------------------------------------------------------------------
21
+
22
+ class MultiClassSVMScratch:
23
+ def __init__(self, C=1.0, learning_rate=0.001, n_epochs=50,
24
+ batch_size=64, random_state=42):
25
+ self.C = C
26
+ self.learning_rate = learning_rate
27
+ self.n_epochs = n_epochs
28
+ self.batch_size = batch_size
29
+ self.random_state = random_state
30
+ self.classifiers = {} # maps class label -> trained LinearSVMScartch
31
+
32
+ def fit(self, X, y):
33
+ self.classes_ = np.unique(y)
34
+ for k in self.classes_:
35
+ # Relabel: +1 if sample belongs to class k, -1 otherwise
36
+ y_binary = np.where(y == k, 1, -1)
37
+
38
+ # use_class_weights=True handles the ~10% / ~90% OvR imbalance
39
+ clf = LinearSVMScartch(
40
+ C=self.C,
41
+ learning_rate=self.learning_rate,
42
+ n_epochs=self.n_epochs,
43
+ batch_size=self.batch_size,
44
+ use_class_weights=True,
45
+ random_state=self.random_state,
46
+ )
47
+ clf.fit(X, y_binary)
48
+ self.classifiers[k] = clf
49
+ return self
50
+
51
+ def decision_scores(self, X):
52
+ # Returns (n_samples, n_classes) matrix of raw w·x+b scores
53
+ scores = np.column_stack([
54
+ self.classifiers[k].decision_function(X)
55
+ for k in self.classes_
56
+ ])
57
+ return scores
58
+
59
+ def predict(self, X):
60
+ # Pick the class whose binary SVM is most confident (+highest score)
61
+ scores = self.decision_scores(X)
62
+ return self.classes_[np.argmax(scores, axis=1)]
63
+
64
+
65
+ # ---------------------------------------------------------------------------
66
+ # Data loading
67
+ # ---------------------------------------------------------------------------
68
+
69
+ def load_data(n_dev=5000):
70
+ print("Loading MNIST...")
71
+ mnist = fetch_openml('mnist_784', version=1, as_frame=False,
72
+ parser='auto', data_home='d:/sklearn_data')
73
+ X, y = mnist.data, mnist.target.astype(int)
74
+
75
+ # Normalize pixels to [0, 1]
76
+ X = X / 255.0
77
+
78
+ # 60 / 20 / 20 stratified split
79
+ X_trainval, X_test, y_trainval, y_test = train_test_split(
80
+ X, y, test_size=0.20, stratify=y, random_state=RANDOM_STATE)
81
+ X_train, X_val, y_train, y_val = train_test_split(
82
+ X_trainval, y_trainval, test_size=0.25, stratify=y_trainval,
83
+ random_state=RANDOM_STATE)
84
+
85
+ print(f"Train: {len(y_train)} Val: {len(y_val)} Test: {len(y_test)}")
86
+
87
+ # Standardize — fit on train only to avoid leakage
88
+ scaler = StandardScaler()
89
+ X_train_s = scaler.fit_transform(X_train)
90
+ X_val_s = scaler.transform(X_val)
91
+ X_test_s = scaler.transform(X_test)
92
+
93
+ # Small dev subset for fast hyperparameter search
94
+ rng = np.random.RandomState(RANDOM_STATE)
95
+ dev_idx = rng.choice(len(X_train_s), size=n_dev, replace=False)
96
+ X_dev, y_dev = X_train_s[dev_idx], y_train[dev_idx]
97
+ print(f"Dev subset: {len(y_dev)} samples")
98
+
99
+ return X_train_s, X_val_s, X_test_s, y_train, y_val, y_test, X_dev, y_dev, scaler
100
+
101
+
102
+ # ---------------------------------------------------------------------------
103
+ # Manual grid search
104
+ # ---------------------------------------------------------------------------
105
+
106
+ def manual_grid_search(X_dev, y_dev, X_val, y_val):
107
+ C_values = [0.1, 1.0, 10.0]
108
+ lr_values = [0.001, 0.0005, 0.0001]
109
+
110
+ results = []
111
+ print("\n--- Hyperparameter Grid Search ---")
112
+ print(f"{'C':>6} {'LR':>8} {'Val Acc':>9}")
113
+ print("-" * 30)
114
+
115
+ for C in C_values:
116
+ for lr in lr_values:
117
+ model = MultiClassSVMScratch(C=C, learning_rate=lr,
118
+ n_epochs=30, random_state=RANDOM_STATE)
119
+ model.fit(X_dev, y_dev)
120
+ acc = accuracy_score(y_val, model.predict(X_val))
121
+ results.append((C, lr, acc))
122
+ print(f"{C:>6} {lr:>8} {acc:>9.4f}")
123
+
124
+ best = max(results, key=lambda x: x[2])
125
+ print(f"\nBest: C={best[0]}, lr={best[1]}, Val Acc={best[2]:.4f}")
126
+ return best[0], best[1]
127
+
128
+
129
+ # ---------------------------------------------------------------------------
130
+ # Plotting helpers
131
+ # ---------------------------------------------------------------------------
132
+
133
+ def plot_loss_curves(model):
134
+ fig, ax = plt.subplots(figsize=(10, 5))
135
+ for k in model.classes_:
136
+ ax.plot(model.classifiers[k].history, label=f"Digit {k}")
137
+ ax.set_xlabel("Epoch")
138
+ ax.set_ylabel("Loss")
139
+ ax.set_title("Training Loss per Binary SVM (OvR)")
140
+ ax.legend(ncol=5, fontsize=8)
141
+ ax.grid(True)
142
+ plt.tight_layout()
143
+ plt.show()
144
+
145
+
146
+ def plot_confusion_matrix(y_true, y_pred):
147
+ cm = confusion_matrix(y_true, y_pred)
148
+ disp = ConfusionMatrixDisplay(cm, display_labels=list(range(10)))
149
+ fig, ax = plt.subplots(figsize=(9, 9))
150
+ disp.plot(ax=ax, cmap="Blues", colorbar=False)
151
+ ax.set_title("Confusion Matrix — Multiclass SVM (Scratch)")
152
+ plt.tight_layout()
153
+ plt.show()
154
+
155
+
156
+ def plot_decision_scores(model, X_sample, true_label):
157
+ scores = model.decision_scores(X_sample.reshape(1, -1))[0]
158
+ colors = ["steelblue"] * 10
159
+ colors[true_label] = "tomato"
160
+ fig, ax = plt.subplots(figsize=(8, 4))
161
+ ax.bar(model.classes_, scores, color=colors)
162
+ ax.set_xlabel("Digit class")
163
+ ax.set_ylabel("Decision score (w·x + b)")
164
+ ax.set_title(f"Decision scores for one sample (true label = {true_label})")
165
+ ax.axhline(0, color="black", linewidth=0.8, linestyle="--")
166
+ plt.tight_layout()
167
+ plt.show()
168
+
169
+
170
+ def plot_per_class_metrics(y_true, y_pred):
171
+ """Bar chart + table showing accuracy, precision, recall, F1 per digit class."""
172
+ classes = np.arange(10)
173
+ cm = confusion_matrix(y_true, y_pred, labels=classes)
174
+ per_class_acc = cm.diagonal() / cm.sum(axis=1)
175
+ precision = precision_score(y_true, y_pred, average=None, labels=classes, zero_division=0)
176
+ recall = recall_score(y_true, y_pred, average=None, labels=classes, zero_division=0)
177
+ f1 = f1_score(y_true, y_pred, average=None, labels=classes, zero_division=0)
178
+
179
+ print("\n=== PER-CLASS METRICS ===")
180
+ print(f"{'Digit':>6} {'Accuracy':>9} {'Precision':>10} {'Recall':>8} {'F1':>8}")
181
+ print("-" * 50)
182
+ for c in classes:
183
+ print(f"{c:>6} {per_class_acc[c]:>9.4f} {precision[c]:>10.4f} {recall[c]:>8.4f} {f1[c]:>8.4f}")
184
+
185
+ x = np.arange(10)
186
+ width = 0.2
187
+ fig, ax = plt.subplots(figsize=(12, 5))
188
+ ax.bar(x - 1.5*width, per_class_acc, width, label="Accuracy")
189
+ ax.bar(x - 0.5*width, precision, width, label="Precision")
190
+ ax.bar(x + 0.5*width, recall, width, label="Recall")
191
+ ax.bar(x + 1.5*width, f1, width, label="F1")
192
+ ax.set_xticks(x)
193
+ ax.set_xticklabels([f"Digit {c}" for c in classes], rotation=45)
194
+ ax.set_ylabel("Score")
195
+ ax.set_title("Per-Class Metrics — Multiclass SVM (Scratch)")
196
+ ax.set_ylim(0, 1.05)
197
+ ax.legend()
198
+ ax.grid(axis="y", linestyle="--", alpha=0.5)
199
+ plt.tight_layout()
200
+ plt.show()
201
+
202
+
203
+ def test_single_digit(model, X_test, y_test, digit=7):
204
+ """Find one test image of `digit`, run it through the model, print prediction."""
205
+ idx = np.where(y_test == digit)[0][0]
206
+ X_sample = X_test[idx]
207
+ predicted = model.predict(X_sample.reshape(1, -1))[0]
208
+ print(f"Input: {digit} → Predicted: {predicted} ({'CORRECT' if predicted == digit else 'WRONG'})")
209
+
210
+
211
+ # ---------------------------------------------------------------------------
212
+ # Main
213
+ # ---------------------------------------------------------------------------
214
+
215
+ if __name__ == "__main__":
216
+
217
+ # 1. Load data
218
+ X_train, X_val, X_test, y_train, y_val, y_test, X_dev, y_dev, scaler = load_data()
219
+
220
+ # 2. Hyperparameter tuning
221
+ best_C, best_lr = manual_grid_search(X_dev, y_dev, X_val, y_val)
222
+
223
+ # 3. Train final model on full training set
224
+ print("\nTraining final model on full training set...")
225
+ final_model = MultiClassSVMScratch(C=best_C, learning_rate=best_lr,
226
+ n_epochs=50, random_state=RANDOM_STATE)
227
+ final_model.fit(X_train, y_train)
228
+ print("Done.")
229
+
230
+ # 4. Evaluate on test set
231
+ y_pred = final_model.predict(X_test)
232
+
233
+ print("\n=== TEST SET RESULTS (Scratch SVM) ===")
234
+ print(f"Accuracy : {accuracy_score(y_test, y_pred):.4f}")
235
+ print(f"Precision (macro) : {precision_score(y_test, y_pred, average='macro'):.4f}")
236
+ print(f"Recall (macro) : {recall_score(y_test, y_pred, average='macro'):.4f}")
237
+ print(f"F1 (macro) : {f1_score(y_test, y_pred, average='macro'):.4f}")
238
+ print(f"F1 (weighted): {f1_score(y_test, y_pred, average='weighted'):.4f}")
239
+ print("\nClassification Report:")
240
+ print(classification_report(y_test, y_pred))
241
+
242
+
243
+ # 6. Plots
244
+ plot_loss_curves(final_model)
245
+ plot_confusion_matrix(y_test, y_pred)
246
+ plot_per_class_metrics(y_test, y_pred)
247
+
248
+ # Decision score bar chart for one sample (pick first test sample)
249
+ sample_idx = 0
250
+ plot_decision_scores(final_model, X_test[sample_idx], y_test[sample_idx])
251
+
252
+ # 7. Single image test — change digit= to any digit 0-9
253
+ test_single_digit(final_model, X_test, y_test, digit=4)
app.py CHANGED
@@ -81,14 +81,16 @@ if gnb_multimode_model is not None:
81
  dt_multimode_model = load_pickle_file('dt_Multi.pkl')
82
  if dt_multimode_model is not None:
83
  dt_multimode_model = dt_multimode_model['model']
84
- svm_multimode_model = None
 
85
  logreg_multimode_model = load_pickle_file('multiLog2_hog.pkl')
86
 
87
 
88
 
89
  def preprocess_image(image, model_choice, mode_choice):
90
  image = np.array(image)
91
-
 
92
  image_hog = hog(
93
  image,
94
  **hog_params
@@ -107,9 +109,10 @@ def preprocess_image(image, model_choice, mode_choice):
107
  image_hog = image_hog.reshape(1, -1)
108
  image_pca = pca_multi.transform(image_hog)
109
  return image_pca
110
- elif model_choice == "DT" or model_choice == "Logistic Regression":
111
  image_hog = image_hog.reshape(1, -1)
112
- return image_hog
 
113
  elif model_choice == "KNN":
114
  image_hog = image_hog.reshape(1, -1)
115
  image_pca = pca_multi.transform(image_hog)
 
81
  dt_multimode_model = load_pickle_file('dt_Multi.pkl')
82
  if dt_multimode_model is not None:
83
  dt_multimode_model = dt_multimode_model['model']
84
+ svm_multimode_model = load_pickle_file('svm_hog.pkl')
85
+ svm_multimode_scaler = load_pickle_file('scaler_hog.pkl')
86
  logreg_multimode_model = load_pickle_file('multiLog2_hog.pkl')
87
 
88
 
89
 
90
  def preprocess_image(image, model_choice, mode_choice):
91
  image = np.array(image)
92
+
93
+ image = image / 255.0
94
  image_hog = hog(
95
  image,
96
  **hog_params
 
109
  image_hog = image_hog.reshape(1, -1)
110
  image_pca = pca_multi.transform(image_hog)
111
  return image_pca
112
+ elif model_choice == "DT" or model_choice == "Logistic Regression" or model_choice == "SVM":
113
  image_hog = image_hog.reshape(1, -1)
114
+ image_scaled = svm_multimode_scaler.transform(image_hog)
115
+ return image_scaled
116
  elif model_choice == "KNN":
117
  image_hog = image_hog.reshape(1, -1)
118
  image_pca = pca_multi.transform(image_hog)
scaler_hog.pkl ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:04752d7b455621763ada8ab62f1f4ed7d7ab4306ef4cb5bbd57d71be1754c492
3
+ size 31567
svm_hog.pkl ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:96b7c0117732f8c40d0fb3e1bdb612db8e9dab703b75c4111fa43be42541c45b
3
+ size 114846