testdevelop

model/mpl_pequeno_keras/dataset.py

@@ -0,0 +1,106 @@
+"""
+Dataset concreto do pipeline ``mpl_pequeno_keras`` — dados tabulares + Keras.
+
+A maior parte da lógica está em :class:`base_dataset.BaseDataset`
+(``docker/base_dataset.py``). Aqui só fica o que é específico:
+
+  - :py:meth:`build_features` — seleciona colunas numéricas do DataFrame,
+    descartando colunas de identificação (``id``, ``image_path``, ...).
+  - :py:meth:`get_data_loader` — devolve tuplas de arrays (X_train, y_train, X_val, y_val)
+    compatíveis com Keras/TensorFlow.
+"""
+
+from __future__ import annotations
+
+import os
+import sys
+from typing import Optional, Tuple
+
+import numpy as np
+import pandas as pd
+
+try:
+    import tensorflow as tf
+    _TF_AVAILABLE = True
+except ImportError:  # pragma: no cover
+    _TF_AVAILABLE = False
+
+# Importa BaseDataset de ../base_dataset.py
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
+from base_dataset import BaseDataset  # noqa: E402
+
+
+class TrainingDataset(BaseDataset):
+    """Dataset tabular para o pipeline em ``docker/mpl_pequeno_keras``."""
+
+    # ------------------------------------------------------------------
+    # Hooks da base
+    # ------------------------------------------------------------------
+    def build_features(self) -> None:
+        if self.df is None:
+            raise RuntimeError("self.df está vazio; load_raw_data() não rodou.")
+
+        # Prioridade: lista explícita no metadata.json
+        explicit = self.metadata.get("features")
+        if explicit:
+            missing = [c for c in explicit if c not in self.df.columns]
+            if missing:
+                raise ValueError(
+                    f"Colunas declaradas em metadata['features'] não encontradas no DataFrame: {missing}"
+                )
+            self.feature_columns = list(explicit)
+            self.X = self.df[self.feature_columns].to_numpy(dtype=np.float32)
+            return
+
+        # Fallback: inferência automática de colunas numéricas
+        feats = []
+        for col in self.df.columns:
+            if col == self.target_column:
+                continue
+            if col.lower() in self.NON_FEATURE_COLS:
+                continue
+            if not pd.api.types.is_numeric_dtype(self.df[col]):
+                continue
+            feats.append(col)
+
+        if not feats:
+            raise ValueError(
+                "Nenhuma coluna numérica encontrada para usar como feature."
+            )
+
+        self.feature_columns = feats
+        self.X = self.df[feats].to_numpy(dtype=np.float32)
+
+    def get_data_loader(
+        self,
+        batch_size: int = 32,
+        train_ratio: Optional[float] = None,
+        seed: Optional[int] = None,
+    ) -> Tuple:
+        if not _TF_AVAILABLE:
+            raise RuntimeError(
+                "TensorFlow não está instalado; instale-o ou use get_arrays()."
+            )
+
+        train_ratio, seed = self._resolve_split_params(train_ratio, seed)
+        idx_train, idx_val = self._split_indices(train_ratio, seed)
+
+        assert self.X is not None and self.y is not None  # pra mypy
+
+        X_train = self.X[idx_train].astype(np.float32)
+        y_train = self.y[idx_train].astype(np.int32)
+        X_val = self.X[idx_val].astype(np.float32)
+        y_val = self.y[idx_val].astype(np.int32)
+
+        bs = self._resolve_batch_size(batch_size)
+
+        print(
+            f"  - Train samples: {len(X_train)} | Val samples: {len(X_val)}"
+            f" | Batch size: {bs}"
+        )
+        
+        # Retorna tuplas (X, y) para compatibilidade com base_main.py unpacking
+        # train_loader, val_loader = get_data_loader() desempacota corretamente
+        train_loader = (X_train, y_train)
+        val_loader = (X_val, y_val)
+        return train_loader, val_loader

model/mpl_pequeno_keras/help.md

@@ -0,0 +1,34 @@
+# RNNs / LSTMs for Sequences
+
+## Resume
+Recurrent Neural Networks (RNNs) and Long Short-Term Memory (LSTM) networks are sequential neural architectures designed to map time-series data or sequential text structures. LSTMs introduce explicit gating mechanisms to regulate internal information persistence, successfully resolving the vanishing gradient flaws of traditional RNN blocks.
+
+### Common Use Cases:
+* **SMILES Generation:** Autoregressively printing valid chemical strings token by token.
+* **Biosignal Analysis:** Evaluating chronological streaming signals from patient telemetry or ECG readouts.
+* **Clinical Notes Sequence Modeling:** Tracking medical event timelines over long EHR spans.
+
+## Content
+
+### 1. Core Architecture & Gated Memory
+Standard RNNs maintain a recurring hidden state vector $h_t$ that gets updated with every timestamp token input. However, backpropagating through long sequences causes gradients to rapidly disappear or explode.
+
+LSTMs solve this by introducing an internal **Cell State** ($c_t$) alongside three specialized gating mechanisms:
+* **Forget Gate ($f_t$):** Controls how much historical context from the cell state should be discarded.
+* **Input Gate ($i_t$):** Regulates what new contextual state information should be infused into the current cell vector.
+* **Output Gate ($o_t$):** Determines what subset of internal hidden cell states should be exposed as the final output block.
+
+### 2. Operational Execution Sequence
+As a sequence executes, the architecture reads the token input at step $t$, merges it with the prior step's hidden vector $h_{t-1}$, adjusts cell memories through the gating parameters, and pushes forward the updated state vectors. This linear memory highway allows the architecture to carry context across extended sequence matrices.
+
+### 3. Advantages and Disadvantages
+* **Advantages:**
+  * Natural alignment with arbitrary, variable-length chronological or textual stream structures.
+  * O(L) computational complexity scaling linearly with sequence length.
+* **Disadvantages:**
+  * Strict step-by-step dependency makes parallel sequence acceleration impossible during training.
+  * Susceptible to information decay over extremely long sequences compared to attention structures.
+
+### References
+* Hochreiter, S., & Schmidhuber, J. (1997). *Long short-term memory*. Neural Computation.
+* Segler, M. H., et al. (2018). *Generating focused molecule libraries for drug discovery with recurrent neural networks*. ACS Central Science.

model/mpl_pequeno_keras/hyperparameters.json

@@ -0,0 +1,116 @@
+{
+  "setting": {
+    "name": {
+      "text": "Nome do modelo",
+      "value": "mpl_pequeno_keras",
+      "Help": "Identificação do modelo. Pode ser usado para salvar checkpoints ou registros.",
+      "default": "mpl_pequeno_keras",
+      "type": "string",
+      "required": true
+    },
+    "architecture": {
+      "text": "Arquitetura do modelo",
+      "value": "MLP tabular",
+      "Help": "A arquitetura da rede neural, como ResNet, VGG, ou qualquer modelo personalizado.",
+      "default": "MLP tabular",
+      "type": "string",
+      "required": true
+    },
+    "tipo": {
+      "text": "Tipo do modelo",
+      "value": "classification",
+      "Help": "Define o tipo de tarefa do modelo",
+      "default": "classification",
+      "type": "string",
+      "required": true
+    },
+    "learning_type": {
+      "text": "Tipo de aprendizado",
+      "value": "deep_learning",
+      "Help": "Indica se o modelo é de machine learning clássico ou deep learning.",
+      "default": "deep_learning",
+      "type": "string",
+      "required": true
+    }
+  },
+  "train": {
+    "batch_size": {
+      "text": "Tamanho do batch",
+      "Help": "Tamanho do batch utilizado durante o treinamento. Valores maiores podem acelerar o treinamento, mas exigem mais memória.",
+      "default": 32,
+      "range": {
+        "min": 1,
+        "max": null
+      },
+      "type": "integer",
+      "required": true
+    },
+    "epochs": {
+      "text": "Número de épocas",
+      "Help": "Quantidade de vezes que o conjunto de treinamento será iterado.",
+      "default": 10,
+      "range": {
+        "min": 1,
+        "max": 1000
+      },
+      "type": "integer",
+      "required": true
+    },
+    "learning_rate": {
+      "text": "Taxa de aprendizado",
+      "Help": "Controla o quão rápido o modelo ajusta os pesos durante o treinamento.",
+      "default": 0.001,
+      "range": {
+        "min": 0.000001,
+        "max": 1.0
+      },
+      "type": "float",
+      "required": true
+    },
+    "weight_decay": {
+      "text": "Decaimento de peso",
+      "Help": "Regularização L2 para evitar overfitting.",
+      "default": 0.0001,
+      "range": {
+        "min": 0.0,
+        "max": 0.1
+      },
+      "type": "float",
+      "required": false
+    },
+    "optimizer": {
+      "text": "Otimizador",
+      "Help": "Otimizador usado no treinamento. Campo fixo.",
+      "default": "adam",
+      "type": "string",
+      "required": true,
+      "constant": true
+    }
+  },
+  "test": {
+    "batch_size": {
+      "text": "Tamanho do batch",
+      "Help": "Tamanho do batch para avaliação. Deve ser otimizado para memória disponível.",
+      "default": 64,
+      "range": {
+        "min": 1,
+        "max": null
+      },
+      "type": "integer",
+      "required": true
+    }
+  },
+  "predict": {
+    "batch_size": {
+      "text": "Tamanho do batch",
+      "Help": "Tamanho do batch para a previsão. Ajuste conforme necessário para memória.",
+      "default": 64,
+      "range": {
+        "min": 1,
+        "max": null
+      },
+      "type": "integer",
+      "required": true
+    }
+  }
+}

model/mpl_pequeno_keras/main.py

@@ -0,0 +1,36 @@
+"""
+Entrypoint do pipeline ``mpl_pequeno_keras`` (MLP tabular em Keras/TensorFlow).
+
+Toda a orquestração (modos train/test/predict, leitura de config,
+escrita dos artefatos) vive em ``docker/base_main.py``. Aqui só
+registramos as peças concretas desta arquitetura:
+
+  - ``TrainingDataset`` (de :mod:`dataset`)
+  - ``create_model``    (de :mod:`model`)
+
+Como o input é tabular, o ``predict_input_loader`` default do
+``base_main`` (que lê CSV/XLSX) já serve — não precisa passar nada.
+Arquiteturas futuras (ex.: CNN para imagem) podem passar um loader
+próprio na chamada de ``run_pipeline``.
+"""
+
+from __future__ import annotations
+
+import os
+import sys
+
+# Permite importar base_main.py de ../
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
+
+from base_main import run_pipeline  # noqa: E402
+from dataset import TrainingDataset  # noqa: E402
+from model import create_model  # noqa: E402
+
+
+if __name__ == "__main__":
+    sys.exit(
+        run_pipeline(
+            dataset_cls=TrainingDataset,
+            model_factory=create_model,
+        )
+    )

model/mpl_pequeno_keras/model.py

@@ -0,0 +1,235 @@
+"""
+Implementação concreta (Keras/TensorFlow) do modelo treinado pelo pipeline.
+
+Arquitetura: MLP simples para dados tabulares. A configuração de
+camadas, dropout, otimizador, etc. vem de ``config`` (lido do
+``config.json`` do treino) ou de defaults razoáveis.
+
+A classe ``TabularMLP`` herda de ``BaseModel`` (em ``docker/base_model.py``)
+para manter a API uniforme entre frameworks. Uma função ``create_model``
+fábrica é exposta no fim do arquivo para que ``main.py`` continue
+funcionando sem mudanças.
+"""
+
+from __future__ import annotations
+
+import os
+import sys
+from typing import Any, Dict, List, Optional, Tuple
+
+import numpy as np
+import tensorflow as tf
+from tensorflow import keras
+from tensorflow.keras import layers
+
+# Permite importar base_model.py de ../
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
+from base_model import BaseModel  # noqa: E402
+from utils import emit_progress  # noqa: E402
+
+
+# ---------------------------------------------------------------------------
+# Wrapper que implementa a API de BaseModel para Keras
+# ---------------------------------------------------------------------------
+class TabularMLP(BaseModel):
+    """MLP tabular em Keras/TensorFlow, plugável no pipeline."""
+
+    def build_model(self) -> keras.Model:
+        hp = self.config.get("hyperparameters", {})
+        hidden_dims = self.config.get("hidden_dims") or hp.get("hidden_dims") or [
+            self.config.get("hidden_dim1", 32),
+            self.config.get("hidden_dim2", 16),
+        ]
+        dropout = float(hp.get("dropout", self.config.get("dropout", 0.0)))
+
+        # Constrói o modelo Sequential
+        model = keras.Sequential()
+        model.add(layers.InputLayer(input_shape=(self.input_size,)))
+
+        # Adiciona camadas ocultas
+        for h in hidden_dims:
+            model.add(layers.Dense(h, activation="relu"))
+            if dropout and dropout > 0:
+                model.add(layers.Dropout(dropout))
+
+        # Camada de saída
+        model.add(layers.Dense(self.num_classes, activation="softmax"))
+
+        self._hidden_dims = list(hidden_dims)
+        self._dropout = dropout
+        return model
+
+    # ---------------- treinamento ----------------
+    def train_model(
+        self,
+        train_loader: Any,
+        val_loader: Optional[Any],
+        epochs: int,
+        lr: float,
+        **kwargs: Any,
+    ) -> Dict[str, list]:
+        hp = self.config.get("hyperparameters", {})
+        epochs = int(hp.get("epochs", epochs))
+        lr = float(hp.get("learning_rate", lr))
+        weight_decay = float(hp.get("weight_decay", 0.0))
+        optimizer_name = str(hp.get("optimizer", "adam")).lower()
+
+        # Desempacota tuplas (X, y) do dataset
+        if isinstance(train_loader, tuple) and len(train_loader) == 2:
+            X_train, y_train = train_loader
+        else:
+            raise ValueError(f"train_loader deve ser tupla (X, y), recebeu {type(train_loader)}")
+        
+        if isinstance(val_loader, tuple) and len(val_loader) == 2:
+            X_val, y_val = val_loader
+        else:
+            raise ValueError(f"val_loader deve ser tupla (X, y), recebeu {type(val_loader)}")
+
+        # Compila o modelo
+        optimizer = self._build_optimizer(optimizer_name, lr, weight_decay)
+        self.model.compile(
+            optimizer=optimizer,
+            loss="sparse_categorical_crossentropy",
+            metrics=["accuracy"],
+        )
+
+        history = {
+            "train_loss": [],
+            "train_acc": [],
+            "val_loss": [],
+            "val_acc": [],
+        }
+
+        n_batches = max((len(X_train) + 31) // 32, 1)  # Batches por epoch
+        total_train_steps = max(epochs * n_batches, 1)
+        batch_size = int(hp.get("batch_size", 32))
+        best_val_acc = -float("inf")
+        best_epoch = -1
+
+        for epoch in range(1, epochs + 1):
+            # Treina 1 época
+            hist = self.model.fit(
+                X_train,
+                y_train,
+                batch_size=batch_size,
+                epochs=1,
+                validation_data=(X_val, y_val),
+                verbose=0,
+            )
+
+            # Extrai histórico
+            train_loss = float(hist.history["loss"][0])
+            train_acc = float(hist.history["accuracy"][0])
+            val_loss = float(hist.history["val_loss"][0])
+            val_acc = float(hist.history["val_accuracy"][0])
+
+            history["train_loss"].append(round(train_loss, 6))
+            history["train_acc"].append(round(train_acc, 6))
+            history["val_loss"].append(round(val_loss, 6))
+            history["val_acc"].append(round(val_acc, 6))
+
+            # Emite progresso por batch (simula progresso contínuo por época)
+            for batch_idx in range(1, n_batches + 1):
+                global_step = (epoch - 1) * n_batches + batch_idx
+                inner_pct = int(global_step * 100 / total_train_steps)
+                emit_progress(inner_pct, 1)
+
+            # Rastreia melhor validação
+            if val_acc > best_val_acc:
+                best_val_acc = val_acc
+                best_epoch = epoch
+
+            print(
+                f"Epoch [{epoch:>3}/{epochs}] "
+                f"train_loss={train_loss:.4f} train_acc={train_acc:.4f} "
+                f"val_loss={val_loss:.4f} val_acc={val_acc:.4f}"
+            )
+
+        history["best_epoch"] = best_epoch
+        history["best_val_acc"] = round(best_val_acc, 6) if best_epoch > 0 else None
+        self.history = history
+        return history
+
+    # ---------------- avaliação ----------------
+    def evaluate(self, data_loader: Any) -> Dict[str, float]:
+        if isinstance(data_loader, tuple) and len(data_loader) == 2:
+            X_val, y_val = data_loader
+        else:
+            X_val, y_val = data_loader[2], data_loader[3]
+        
+        loss, accuracy = self.model.evaluate(X_val, y_val, verbose=0)
+        return {
+            "loss": float(loss),
+            "accuracy": float(accuracy),
+            "n": len(X_val),
+        }
+
+    # ---------------- inferência ----------------
+    def predict(self, inputs: Any) -> np.ndarray:
+        if isinstance(inputs, tuple):
+            # Se for tuple de (X_train, y_train, X_val, y_val), usa X_val
+            X = inputs[2].astype(np.float32)
+        elif isinstance(inputs, np.ndarray):
+            X = inputs.astype(np.float32)
+        else:
+            X = np.array(inputs, dtype=np.float32)
+
+        if X.ndim == 1:
+            X = X.reshape(1, -1)
+
+        # Predições como classe
+        logits = self.model.predict(X, verbose=0)
+        return np.argmax(logits, axis=1)
+
+    # ---------------- persistência ----------------
+    def save_model(self, filename: str) -> str:
+        path = os.path.join(self.model_dir, filename)
+        # Remove extensão se for .pt (compatibilidade)
+        if path.endswith(".pt"):
+            path = path[:-3] + ".keras"
+
+        self.model.save(path)
+        print(f"✓ Model saved to {path}")
+        return path
+
+    def load_model(self, filename: str) -> None:
+        path = filename if os.path.isabs(filename) else os.path.join(
+            self.model_dir, filename
+        )
+        # Converte .pt para .keras
+        if path.endswith(".pt"):
+            path = path[:-3] + ".keras"
+
+        self.model = keras.models.load_model(path)
+
+    # ---------------- helpers ----------------
+    def _build_optimizer(
+        self, name: str, lr: float, weight_decay: float
+    ) -> keras.optimizers.Optimizer:
+        if name == "sgd":
+            return keras.optimizers.SGD(learning_rate=lr, weight_decay=weight_decay)
+        if name == "rmsprop":
+            return keras.optimizers.RMSprop(learning_rate=lr, weight_decay=weight_decay)
+        # default: adam
+        return keras.optimizers.Adam(learning_rate=lr, weight_decay=weight_decay)
+
+
+# ---------------------------------------------------------------------------
+# Fábrica usada pelo main.py
+# ---------------------------------------------------------------------------
+def create_model(
+    input_size: int,
+    num_classes: int,
+    project_name: str,
+    base_path: str,
+    config: Optional[Dict[str, Any]] = None,
+) -> TabularMLP:
+    """Cria um ``TabularMLP`` pronto pra uso pelo main.py."""
+    return TabularMLP(
+        input_size=input_size,
+        num_classes=num_classes,
+        project_name=project_name,
+        base_path=base_path,
+        framework="keras",
+        config=config or {},
+    )

model/mpl_pequeno_keras/python_version.txt

@@ -0,0 +1 @@
+3.10

model/mpl_pequeno_keras/requirements.txt

@@ -0,0 +1,6 @@
+pandas
+scikit-learn
+joblib
+tensorflow
+openpyxl
+numpy

project/project.json

@@ -0,0 +1,9 @@
+{
+    "name": "testDevelop",
+    "description": "foo",
+    "models": [
+        "mpl_pequeno_keras"
+    ],
+    "created_at": "2026-06-15T22:46:26.302605",
+    "project_id": "079cc401-e6ba-41e9-84c8-3c743ac104c6"
+}