NAH4444's picture
Update modeling.py
bd165a0 verified
Raw History Blame Contribute Delete
5.24 kB
import torch
import torch.nn as nn
from transformers import AutoModel, AutoTokenizer
import json
import os
# Segment -> Category hierarchy
SEGMENT_TO_CATEGORIES = {
'Macro & Geopolitics': ['Economic Data', 'Central Bank & Monetary Policy', 'Regulation & Policy', 'War & Conflict', 'Geopolitics & Diplomacy'],
'Crypto': ['Bitcoin & Major Crypto', 'DeFi', 'Stablecoin', 'NFT & Gaming', 'Crypto Regulation', 'Exchange & ETF', 'On-Chain & Tokenomics'],
'Forex': ['Major Pairs', 'Cross & Exotic Pairs'],
'Commodities': ['Precious Metals', 'Energy', 'Agricultural', 'Industrial Metals'],
'Stock & Equity': ['Index', 'Earnings & Fundamentals', 'Sector & Thematic', 'IPO, M&A & Corporate Action'],
'Bond & Fixed Income': ['Treasury & Sovereign', 'Credit & Corporate Bond', 'Money Market & Rates'],
'Others': ['Others'],
}
class ModernBERTMultiTask(nn.Module):
def __init__(self, base_model_name, num_segment_labels, num_category_labels,
num_sentiment_labels, num_importance_labels):
super().__init__()
self.bert = AutoModel.from_pretrained(base_model_name)
hidden_size = self.bert.config.hidden_size
self.segment_head = nn.Sequential(
nn.Dropout(0.1), nn.Linear(hidden_size, hidden_size // 2),
nn.GELU(), nn.Dropout(0.1),
nn.Linear(hidden_size // 2, num_segment_labels))
self.category_head = nn.Sequential(
nn.Dropout(0.1), nn.Linear(hidden_size, hidden_size // 2),
nn.GELU(), nn.Dropout(0.1),
nn.Linear(hidden_size // 2, num_category_labels))
self.sentiment_head = nn.Sequential(
nn.Dropout(0.1), nn.Linear(hidden_size, hidden_size // 2),
nn.GELU(), nn.Dropout(0.1),
nn.Linear(hidden_size // 2, num_sentiment_labels))
self.importance_head = nn.Sequential(
nn.Dropout(0.1), nn.Linear(hidden_size, hidden_size // 2),
nn.GELU(), nn.Dropout(0.1),
nn.Linear(hidden_size // 2, num_importance_labels))
def forward(self, input_ids, attention_mask=None, **kwargs):
outputs = self.bert(input_ids=input_ids, attention_mask=attention_mask)
cls_output = outputs.last_hidden_state[:, 0, :]
return {
"segment_logits": self.segment_head(cls_output),
"category_logits": self.category_head(cls_output),
"sentiment_logits": self.sentiment_head(cls_output),
"importance_logits": self.importance_head(cls_output),
}
def build_segment_category_mask(config):
seg_labels = config["segment_labels"]
cat_labels = config["category_labels"]
mask = torch.zeros(len(seg_labels), len(cat_labels))
for seg_name, cat_names in SEGMENT_TO_CATEGORIES.items():
seg_id = seg_labels[seg_name]
for cat_name in cat_names:
cat_id = cat_labels[cat_name]
mask[seg_id, cat_id] = 1.0
return mask
def load_model(model_dir):
with open(os.path.join(model_dir, "label_config.json")) as f:
config = json.load(f)
model = ModernBERTMultiTask(
config["base_model"],
config["num_segment_labels"],
config["num_category_labels"],
config["num_sentiment_labels"],
config["num_importance_labels"])
state_dict = torch.load(os.path.join(model_dir, "pytorch_model.bin"), map_location="cpu")
model.load_state_dict(state_dict, strict=False)
model.eval()
tokenizer = AutoTokenizer.from_pretrained(model_dir)
return model, tokenizer, config
def predict(text, model, tokenizer, config, device="cpu"):
inputs = tokenizer(text, return_tensors="pt", padding=True,
truncation=True, max_length=config.get("max_length", 2048))
inputs = {k: v.to(device) for k, v in inputs.items()}
seg_cat_mask = build_segment_category_mask(config)
with torch.no_grad():
outputs = model(**inputs)
# Segment prediction
segment_logits = outputs["segment_logits"]
segment_probs = torch.softmax(segment_logits, dim=-1)
segment_id = segment_probs.argmax(dim=-1).item()
# Mask category logits based on predicted segment
category_logits = outputs["category_logits"]
mask = seg_cat_mask[segment_id]
category_logits = category_logits.masked_fill(mask == 0, float('-inf'))
category_probs = torch.softmax(category_logits, dim=-1)
sentiment_probs = torch.softmax(outputs["sentiment_logits"], dim=-1)
importance_probs = torch.softmax(outputs["importance_logits"], dim=-1)
id2seg = config["id2segment"]
id2cat = config["id2category"]
id2sent = config["id2sentiment"]
id2imp = config["id2importance"]
return {
"segment": id2seg[str(segment_id)],
"segment_confidence": round(segment_probs.max().item(), 4),
"category": id2cat[str(category_probs.argmax(dim=-1).item())],
"category_confidence": round(category_probs.max().item(), 4),
"sentiment": id2sent[str(sentiment_probs.argmax(dim=-1).item())],
"sentiment_confidence": round(sentiment_probs.max().item(), 4),
"importance": id2imp[str(importance_probs.argmax(dim=-1).item())],
"importance_confidence": round(importance_probs.max().item(), 4),
}