Spaces:
Sleeping
Sleeping
Download train.py from abdurrehmansaeed/ThreadNode: direct link, hf CLI and curl.
- Browser
- Download file 4.65 kB
-
https://huggingface.co/spaces/abdurrehmansaeed/ThreadNode/resolve/main/train.py
- Command line
-
hf download hf://spaces/abdurrehmansaeed/ThreadNode/train.py
-
curl -L -o train.py https://huggingface.co/spaces/abdurrehmansaeed/ThreadNode/resolve/main/train.py
4.65 kB
| import os | |
| import pickle | |
| import numpy as np | |
| import pandas as pd | |
| import torch | |
| import torch.nn as nn | |
| import torch.optim as optim | |
| from torch.utils.data import Dataset, DataLoader | |
| from torchvision import models, transforms | |
| from sklearn.preprocessing import MultiLabelBinarizer | |
| from PIL import Image | |
| # --- 1. CONFIGURATION --- | |
| DATA_DIR = "./dataset" | |
| CSV_PATH = "./dataset/metadata.csv" | |
| MODEL_SAVE_PATH = "fashion_hierarchy_model_v3.pth" | |
| VOCAB_SAVE_PATH = "mlb_vocab.pkl" | |
| BATCH_SIZE = 32 | |
| EPOCHS = 10 | |
| LEARNING_RATE = 1e-4 | |
| device = torch.device("cuda" if torch.cuda.is_available() else "cpu") | |
| print(f"Initializing training pipeline on: {device}") | |
| # --- 2. DATA PIPELINE --- | |
| class ApparelDataset(Dataset): | |
| def __init__(self, df, img_dir, mlb, transform=None): | |
| self.df = df | |
| self.img_dir = img_dir | |
| self.transform = transform | |
| self.mlb = mlb | |
| # Assuming your CSV has 'image_name' and 'tags' (comma separated) columns | |
| self.images = self.df['image_name'].values | |
| self.labels = self.mlb.transform(self.df['tags'].apply(lambda x: x.split(','))) | |
| def __len__(self): | |
| return len(self.images) | |
| def __getitem__(self, idx): | |
| img_path = os.path.join(self.img_dir, self.images[idx]) | |
| image = Image.open(img_path).convert("RGB") | |
| label = torch.tensor(self.labels[idx], dtype=torch.float32) | |
| if self.transform: | |
| image = self.transform(image) | |
| return image, label | |
| def get_transforms(): | |
| return transforms.Compose([ | |
| transforms.Resize((224, 224)), | |
| transforms.RandomHorizontalFlip(), | |
| transforms.ColorJitter(brightness=0.2, contrast=0.2), | |
| transforms.ToTensor(), | |
| transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) | |
| ]) | |
| # --- 3. ARCHITECTURE --- | |
| def build_model(num_classes): | |
| model = models.efficientnet_b0(weights=models.EfficientNet_B0_Weights.IMAGENET1K_V1) | |
| in_features = model.classifier[1].in_features | |
| model.classifier[1] = nn.Linear(in_features, num_classes) | |
| return model.to(device) | |
| # --- 4. LOSS ENGINEERING (Handling Class Imbalance) --- | |
| def calculate_pos_weights(labels_matrix): | |
| class_counts = labels_matrix.sum(axis=0) | |
| total_samples = labels_matrix.shape[0] | |
| # Add small epsilon to prevent division by zero | |
| pos_weights = (total_samples - class_counts) / (class_counts + 1e-5) | |
| return torch.tensor(pos_weights, dtype=torch.float32).to(device) | |
| # --- 5. EXECUTION --- | |
| def main(): | |
| print("Loading metadata...") | |
| df = pd.read_csv(CSV_PATH) | |
| print("Fitting MultiLabelBinarizer...") | |
| mlb = MultiLabelBinarizer() | |
| mlb.fit(df['tags'].apply(lambda x: x.split(','))) | |
| num_classes = len(mlb.classes_) | |
| # Save the vocabulary matrix for inference | |
| with open(VOCAB_SAVE_PATH, 'wb') as f: | |
| pickle.dump(mlb, f) | |
| print(f"Vocabulary matrix locked and saved to {VOCAB_SAVE_PATH}") | |
| print("Initializing DataLoader...") | |
| dataset = ApparelDataset(df, DATA_DIR, mlb, transform=get_transforms()) | |
| dataloader = DataLoader(dataset, batch_size=BATCH_SIZE, shuffle=True, num_workers=4) | |
| print("Building EfficientNet-B0 Architecture...") | |
| model = build_model(num_classes) | |
| print("Computing dynamic positive weights for minority classes...") | |
| pos_weight = calculate_pos_weights(dataset.labels) | |
| criterion = nn.BCEWithLogitsLoss(pos_weight=pos_weight) | |
| optimizer = optim.Adam(model.parameters(), lr=LEARNING_RATE) | |
| print("Commencing Training Loop...") | |
| for epoch in range(EPOCHS): | |
| model.train() | |
| running_loss = 0.0 | |
| for batch_idx, (images, labels) in enumerate(dataloader): | |
| images, labels = images.to(device), labels.to(device) | |
| optimizer.zero_grad() | |
| outputs = model(images) | |
| loss = criterion(outputs, labels) | |
| loss.backward() | |
| optimizer.step() | |
| running_loss += loss.item() | |
| if batch_idx % 50 == 0: | |
| print(f"Epoch [{epoch+1}/{EPOCHS}] Batch {batch_idx}/{len(dataloader)} Loss: {loss.item():.4f}") | |
| epoch_loss = running_loss / len(dataloader) | |
| print(f"--- Epoch {epoch+1} Completed | Average Loss: {epoch_loss:.4f} ---") | |
| print("Training sequence complete. Saving V3 state dict...") | |
| torch.save(model.state_dict(), MODEL_SAVE_PATH) | |
| print("Engine weights successfully exported.") | |
| if __name__ == "__main__": | |
| main() |