Spaces:
Sleeping
Sleeping
Update src/data_preprocessing.py
Browse files- src/data_preprocessing.py +43 -33
src/data_preprocessing.py
CHANGED
|
@@ -152,26 +152,38 @@ class SolarDataProcessor:
|
|
| 152 |
X_flat = X_array.reshape(X_array.shape[0], -1)
|
| 153 |
|
| 154 |
# Normaliser les entrées et sorties
|
| 155 |
-
|
| 156 |
-
y_scaled = self.output_scaler.fit_transform(y_transformed)
|
| 157 |
|
| 158 |
#en cas de modèle déjà fini
|
| 159 |
if post_train:
|
| 160 |
# Si post_train est True, ne pas diviser les données
|
| 161 |
-
self.X_user = X_scaled
|
| 162 |
-
self.y_user = y_scaled
|
| 163 |
input_path = os.path.join(self.model_dir, 'input_scaler.pkl')
|
| 164 |
output_path = os.path.join(self.model_dir, 'output_scaler.pkl')
|
| 165 |
self.output_scaler = joblib.load(output_path)
|
| 166 |
self.input_scaler = joblib.load(input_path)
|
| 167 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 168 |
return True
|
| 169 |
else:
|
| 170 |
input_path = os.path.join(self.model_dir, 'input_scaler.pkl')
|
| 171 |
output_path = os.path.join(self.model_dir, 'output_scaler.pkl')
|
| 172 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 173 |
joblib.dump(self.input_scaler, input_path)
|
| 174 |
joblib.dump(self.output_scaler, output_path)
|
|
|
|
|
|
|
| 175 |
# Diviser les données: 70% entraînement, 15% validation, 15% test
|
| 176 |
X_train_val, X_test, y_train_val, y_test = train_test_split(
|
| 177 |
X_scaled, y_scaled, test_size=0.15, random_state=42)
|
|
@@ -192,41 +204,39 @@ class SolarDataProcessor:
|
|
| 192 |
|
| 193 |
return True
|
| 194 |
|
| 195 |
-
def preprocess_input(self, csv_files, path, in_the_dark=
|
| 196 |
X_list = []
|
| 197 |
I_list = []
|
| 198 |
V_list = []
|
| 199 |
-
|
| 200 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 201 |
input_path = os.path.join(self.model_dir, 'input_scaler.pkl')
|
| 202 |
output_path = os.path.join(self.model_dir, 'output_scaler.pkl')
|
| 203 |
-
self.input_scaler = joblib.load(input_path)
|
| 204 |
self.output_scaler = joblib.load(output_path)
|
| 205 |
-
|
| 206 |
-
|
| 207 |
-
|
| 208 |
-
|
| 209 |
-
|
| 210 |
-
|
| 211 |
-
|
| 212 |
-
|
| 213 |
-
|
| 214 |
-
|
| 215 |
-
|
| 216 |
-
X_list.append(X_norm)
|
| 217 |
-
V_list.append(X_norm[:, 0])
|
| 218 |
-
I_list.append(X_norm[:, 1])
|
| 219 |
-
except Exception as e:
|
| 220 |
-
logger.warning(f"Erreur lors du traitement de {file_path}: {e}")
|
| 221 |
-
continue
|
| 222 |
-
|
| 223 |
-
if not X_list:
|
| 224 |
-
raise ValueError("Aucun fichier valide n'a pu être traité.")
|
| 225 |
-
|
| 226 |
-
X_array = np.array(X_list)
|
| 227 |
-
X_flat = X_array.reshape(X_array.shape[0], -1)
|
| 228 |
-
X_scaled = self.input_scaler.transform(X_flat)
|
| 229 |
-
|
| 230 |
return X_scaled, I_list, V_list
|
| 231 |
|
| 232 |
def get_data(self):
|
|
|
|
| 152 |
X_flat = X_array.reshape(X_array.shape[0], -1)
|
| 153 |
|
| 154 |
# Normaliser les entrées et sorties
|
| 155 |
+
|
|
|
|
| 156 |
|
| 157 |
#en cas de modèle déjà fini
|
| 158 |
if post_train:
|
| 159 |
# Si post_train est True, ne pas diviser les données
|
|
|
|
|
|
|
| 160 |
input_path = os.path.join(self.model_dir, 'input_scaler.pkl')
|
| 161 |
output_path = os.path.join(self.model_dir, 'output_scaler.pkl')
|
| 162 |
self.output_scaler = joblib.load(output_path)
|
| 163 |
self.input_scaler = joblib.load(input_path)
|
| 164 |
|
| 165 |
+
X_scaled = self.input_scaler.transform(X_flat)
|
| 166 |
+
print(f'post X_scaled {X_scaled}')
|
| 167 |
+
y_scaled = self.output_scaler.transform(y_transformed)
|
| 168 |
+
self.X_user = X_scaled
|
| 169 |
+
self.y_user = y_scaled
|
| 170 |
+
|
| 171 |
+
|
| 172 |
return True
|
| 173 |
else:
|
| 174 |
input_path = os.path.join(self.model_dir, 'input_scaler.pkl')
|
| 175 |
output_path = os.path.join(self.model_dir, 'output_scaler.pkl')
|
| 176 |
|
| 177 |
+
|
| 178 |
+
|
| 179 |
+
X_scaled = self.input_scaler.fit_transform(X_flat)
|
| 180 |
+
y_scaled = self.output_scaler.fit_transform(y_transformed)
|
| 181 |
+
print("Scaler mean:", self.input_scaler.mean_)
|
| 182 |
+
print("Scaler scale:", self.input_scaler.scale_)
|
| 183 |
joblib.dump(self.input_scaler, input_path)
|
| 184 |
joblib.dump(self.output_scaler, output_path)
|
| 185 |
+
print(f'pré X_scaled {X_scaled}')
|
| 186 |
+
|
| 187 |
# Diviser les données: 70% entraînement, 15% validation, 15% test
|
| 188 |
X_train_val, X_test, y_train_val, y_test = train_test_split(
|
| 189 |
X_scaled, y_scaled, test_size=0.15, random_state=42)
|
|
|
|
| 204 |
|
| 205 |
return True
|
| 206 |
|
| 207 |
+
def preprocess_input(self, csv_files, path, in_the_dark = True):
|
| 208 |
X_list = []
|
| 209 |
I_list = []
|
| 210 |
V_list = []
|
| 211 |
+
for file_path in csv_files:
|
| 212 |
+
df = pd.read_csv(path + file_path)
|
| 213 |
+
if not in_the_dark:
|
| 214 |
+
df["I"] = df["I"]*1e-3
|
| 215 |
+
df["I"] = df["I"]-df["I"].iloc[0] # A/m²
|
| 216 |
+
|
| 217 |
+
X = df[['V', 'I']].values
|
| 218 |
+
|
| 219 |
+
X_norm = self.interpolation(X)
|
| 220 |
+
X_list.append(X_norm)
|
| 221 |
+
V_list.append(X[:, 0])
|
| 222 |
+
I_list.append(X[:, 1])
|
| 223 |
+
|
| 224 |
+
X_array = np.array(X_list)
|
| 225 |
+
X_flat = X_array.reshape(X_array.shape[0], -1)
|
| 226 |
input_path = os.path.join(self.model_dir, 'input_scaler.pkl')
|
| 227 |
output_path = os.path.join(self.model_dir, 'output_scaler.pkl')
|
|
|
|
| 228 |
self.output_scaler = joblib.load(output_path)
|
| 229 |
+
|
| 230 |
+
self.input_scaler = joblib.load(input_path)
|
| 231 |
+
# Vérifier si le scaler est fitted en vérifiant les attributs nécessaires
|
| 232 |
+
if not hasattr(self.input_scaler, 'mean_') or not hasattr(self.input_scaler, 'scale_'):
|
| 233 |
+
logger.warning("Le scaler chargé n'est pas fitted. Utilisation de fit_transform.")
|
| 234 |
+
# Si le scaler n'est pas fitted, on utilise fit_transform
|
| 235 |
+
X_scaled = self.input_scaler.fit_transform(X_flat)
|
| 236 |
+
else:
|
| 237 |
+
# Si le scaler est fitted, on utilise transform
|
| 238 |
+
logger.info("Le scaler est correctement fitted. Utilisation de transform.")
|
| 239 |
+
X_scaled = self.input_scaler.transform(X_flat)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 240 |
return X_scaled, I_list, V_list
|
| 241 |
|
| 242 |
def get_data(self):
|