File size: 2,773 Bytes
961cf0c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 | import os
import pickle
import numpy as np
import pandas as pd
import torch
from PIL import Image
from torchvision import transforms
# ==========================================================
# IMAGE
# ==========================================================
def load_image_transform(size=(224, 224)):
return transforms.Compose([
transforms.Resize(size),
transforms.ToTensor(),
transforms.Normalize(
mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225]
)
])
def process_image_pil(img_pil, device):
transform = load_image_transform()
tensor = transform(img_pil).unsqueeze(0).to(device)
return tensor
# ==========================================================
# METADATA PAD-UFES-20
# ==========================================================
PAD_COLUMNS = [
"patient_id","lesion_id","smoke","drink",
"background_father","background_mother",
"age","pesticide","gender",
"skin_cancer_history","cancer_history",
"has_piped_water","has_sewage_system",
"fitspatrick","region",
"diameter_1","diameter_2",
"diagnostic",
"itch","grew","hurt",
"changed","bleed",
"elevation","img_id","biopsed"
]
NUMERICAL_COLS = ["age","diameter_1","diameter_2"]
DROP_COLS = ["patient_id","lesion_id","img_id","biopsed","diagnostic"]
def clean_metadata(df):
df = df.fillna("EMPTY")
df = df.replace(r"^\s*$", "EMPTY", regex=True)
df = df.replace("BRASIL", "BRAZIL")
return df
def parse_csv_line_to_cols(text_line):
parts = text_line.split(",")
if len(parts) < len(PAD_COLUMNS):
parts += [""] * (len(PAD_COLUMNS) - len(parts))
elif len(parts) > len(PAD_COLUMNS):
parts = parts[:len(PAD_COLUMNS)]
return pd.DataFrame([parts], columns=PAD_COLUMNS)
def process_metadata_pad20(text_line, encoder_dir, device):
df = parse_csv_line_to_cols(text_line)
df = clean_metadata(df)
features = df.drop(columns=DROP_COLS)
categorical_cols = [c for c in features.columns
if c not in NUMERICAL_COLS]
features[categorical_cols] = features[categorical_cols].astype(str)
features[NUMERICAL_COLS] = (
features[NUMERICAL_COLS]
.apply(pd.to_numeric, errors="coerce")
.fillna(-1)
)
with open(os.path.join(encoder_dir,"ohe_pad_20.pickle"),"rb") as f:
ohe = pickle.load(f)
with open(os.path.join(encoder_dir,"scaler_pad_20.pickle"),"rb") as f:
scaler = pickle.load(f)
categorical_data = ohe.transform(features[categorical_cols])
numerical_data = scaler.transform(features[NUMERICAL_COLS])
processed = np.hstack([categorical_data, numerical_data])
return torch.tensor(processed, dtype=torch.float32).to(device) |