src/models/preprocessing.py

import os
import pickle
import numpy as np
import pandas as pd
import torch
from PIL import Image
from torchvision import transforms

# ==========================================================
# IMAGE
# ==========================================================

def load_image_transform(size=(224, 224)):
    return transforms.Compose([
        transforms.Resize(size),
        transforms.ToTensor(),
        transforms.Normalize(
            mean=[0.485, 0.456, 0.406],
            std=[0.229, 0.224, 0.225]
        )
    ])


def process_image_pil(img_pil, device):
    transform = load_image_transform()
    tensor = transform(img_pil).unsqueeze(0).to(device)
    return tensor


# ==========================================================
# METADATA PAD-UFES-20
# ==========================================================

PAD_COLUMNS = [
    "patient_id","lesion_id","smoke","drink",
    "background_father","background_mother",
    "age","pesticide","gender",
    "skin_cancer_history","cancer_history",
    "has_piped_water","has_sewage_system",
    "fitspatrick","region",
    "diameter_1","diameter_2",
    "diagnostic",
    "itch","grew","hurt",
    "changed","bleed",
    "elevation","img_id","biopsed"
]

NUMERICAL_COLS = ["age","diameter_1","diameter_2"]
DROP_COLS = ["patient_id","lesion_id","img_id","biopsed","diagnostic"]


def clean_metadata(df):
    df = df.fillna("EMPTY")
    df = df.replace(r"^\s*$", "EMPTY", regex=True)
    df = df.replace("BRASIL", "BRAZIL")
    return df


def parse_csv_line_to_cols(text_line):
    parts = text_line.split(",")

    if len(parts) < len(PAD_COLUMNS):
        parts += [""] * (len(PAD_COLUMNS) - len(parts))
    elif len(parts) > len(PAD_COLUMNS):
        parts = parts[:len(PAD_COLUMNS)]

    return pd.DataFrame([parts], columns=PAD_COLUMNS)


def process_metadata_pad20(text_line, encoder_dir, device):

    df = parse_csv_line_to_cols(text_line)
    df = clean_metadata(df)

    features = df.drop(columns=DROP_COLS)

    categorical_cols = [c for c in features.columns
                        if c not in NUMERICAL_COLS]

    features[categorical_cols] = features[categorical_cols].astype(str)

    features[NUMERICAL_COLS] = (
        features[NUMERICAL_COLS]
        .apply(pd.to_numeric, errors="coerce")
        .fillna(-1)
    )

    with open(os.path.join(encoder_dir,"ohe_pad_20.pickle"),"rb") as f:
        ohe = pickle.load(f)

    with open(os.path.join(encoder_dir,"scaler_pad_20.pickle"),"rb") as f:
        scaler = pickle.load(f)

    categorical_data = ohe.transform(features[categorical_cols])
    numerical_data = scaler.transform(features[NUMERICAL_COLS])

    processed = np.hstack([categorical_data, numerical_data])

    return torch.tensor(processed, dtype=torch.float32).to(device)