Upload 9 files

config.json

@@ -3,6 +3,9 @@
     "CustomSequenceClassificationModel"
   ],
   "attention_probs_dropout_prob": 0.1,
+  "auto_map": {
+    "AutoModelForSequenceClassification": "modeling_custom_sequence_classifier.CustomSequenceClassificationModel"
+  },
   "class_weight_log_scaling": true,
   "classifier_dropout": null,
   "classifier_head_activation": "gelu",
@@ -1885,11 +1888,12 @@
   "model_type": "bert",
   "num_attention_heads": 12,
   "num_hidden_layers": 12,
-  "num_train_epochs": 5,
+  "num_train_epochs": 10,
   "num_training_samples": 685763,
   "pad_token_id": 0,
   "position_embedding_type": "absolute",
   "problem_type": "multi_label_classification",
+  "resume_from_checkpoint": "last",
   "torch_dtype": "bfloat16",
   "transformers_version": "4.53.3",
   "type_vocab_size": 2,

label_mappings.json

@@ -1869,5 +1869,6 @@
   "use_cyclic_cosine_scheduler": true,
   "cosine_cycle_length_steps": 50000,
   "cosine_cycle_warmup_steps": 5000,
-  "cosine_min_lr_ratio": 0.05
+  "cosine_min_lr_ratio": 0.05,
+  "resume_from_checkpoint": "last"
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:05e8bafe9394995013a4f45b37df9a82085f02e3f31d7ffa5ada06a35782b9a9
+oid sha256:0f4f3c33443d2fb4663c43dbdd4bfde7a6e4f0b9d1adee666ab652da9ab757f2
 size 222778896

modeling_custom_sequence_classifier.py

@@ -0,0 +1,111 @@
+import torch
+import torch.nn as nn
+
+from transformers import AutoModel, PreTrainedModel
+from transformers.modeling_outputs import SequenceClassifierOutput
+
+
+class CustomSequenceClassificationModel(PreTrainedModel):
+    """Sequence classification model with a configurable GELU + dropout dense head."""
+
+    def __init__(
+        self,
+        config,
+        head_layers: int = 1,
+        head_dropout: float = 0.1,
+    ):
+        super().__init__(config)
+        self.num_labels = config.num_labels
+        self.config = config
+        self.backbone = AutoModel.from_config(config)
+
+        hidden_size = getattr(config, "hidden_size", None)
+        if hidden_size is None:
+            raise ValueError(
+                "Backbone config is missing 'hidden_size'; cannot build custom head."
+            )
+
+        self.classifier = self._build_head(
+            input_dim=hidden_size,
+            output_dim=self.num_labels,
+            head_layers=head_layers,
+            head_dropout=head_dropout,
+        )
+        self.post_init()
+
+    @staticmethod
+    def _build_head(
+        input_dim: int, output_dim: int, head_layers: int, head_dropout: float
+    ) -> nn.Module:
+        if head_layers < 1:
+            raise ValueError("head_layers must be >= 1")
+
+        # head_layers includes the final projection layer.
+        # For head_layers=1 this is equivalent to a single linear classifier.
+        if head_layers == 1:
+            return nn.Linear(input_dim, output_dim)
+
+        layers = []
+        for _ in range(head_layers - 1):
+            layers.extend(
+                [
+                    nn.Linear(input_dim, input_dim),
+                    nn.GELU(),
+                    nn.Dropout(head_dropout),
+                ]
+            )
+        layers.append(nn.Linear(input_dim, output_dim))
+        return nn.Sequential(*layers)
+
+    def _pooled_representation(self, outputs) -> torch.Tensor:
+        if hasattr(outputs, "pooler_output") and outputs.pooler_output is not None:
+            return outputs.pooler_output
+        return outputs.last_hidden_state[:, 0]
+
+    def forward(
+        self,
+        input_ids=None,
+        attention_mask=None,
+        token_type_ids=None,
+        labels=None,
+        **kwargs,
+    ):
+        backbone_inputs = {
+            "input_ids": input_ids,
+            "attention_mask": attention_mask,
+            **kwargs,
+        }
+        if token_type_ids is not None:
+            backbone_inputs["token_type_ids"] = token_type_ids
+
+        backbone_outputs = self.backbone(**backbone_inputs)
+
+        pooled = self._pooled_representation(backbone_outputs)
+        logits = self.classifier(pooled)
+
+        loss = None
+        if labels is not None:
+            if self.config.problem_type is None:
+                if self.num_labels == 1:
+                    self.config.problem_type = "regression"
+                elif torch.is_floating_point(labels):
+                    self.config.problem_type = "multi_label_classification"
+                else:
+                    self.config.problem_type = "single_label_classification"
+
+            if self.config.problem_type == "regression":
+                loss_fct = nn.MSELoss()
+                loss = loss_fct(logits.squeeze(), labels.squeeze())
+            elif self.config.problem_type == "single_label_classification":
+                loss_fct = nn.CrossEntropyLoss()
+                loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))
+            else:
+                loss_fct = nn.BCEWithLogitsLoss()
+                loss = loss_fct(logits, labels)
+
+        return SequenceClassifierOutput(
+            loss=loss,
+            logits=logits,
+            hidden_states=backbone_outputs.hidden_states,
+            attentions=backbone_outputs.attentions,
+        )

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f81694998e5d74a701679b1847e19dad57a91f338de55335d8fad8775dd26070
+oid sha256:6acfabddc9f327e61ac9df17cc9a49aa63eb5d694d7fd8d0f8b94a2163f64b81
 size 5368