import torch import torch.nn as nn from transformers import AutoModel, AutoTokenizer import json import os # Segment -> Category hierarchy SEGMENT_TO_CATEGORIES = { 'Macro & Geopolitics': ['Economic Data', 'Central Bank & Monetary Policy', 'Regulation & Policy', 'War & Conflict', 'Geopolitics & Diplomacy'], 'Crypto': ['Bitcoin & Major Crypto', 'DeFi', 'Stablecoin', 'NFT & Gaming', 'Crypto Regulation', 'Exchange & ETF', 'On-Chain & Tokenomics'], 'Forex': ['Major Pairs', 'Cross & Exotic Pairs'], 'Commodities': ['Precious Metals', 'Energy', 'Agricultural', 'Industrial Metals'], 'Stock & Equity': ['Index', 'Earnings & Fundamentals', 'Sector & Thematic', 'IPO, M&A & Corporate Action'], 'Bond & Fixed Income': ['Treasury & Sovereign', 'Credit & Corporate Bond', 'Money Market & Rates'], 'Others': ['Others'], } class ModernBERTMultiTask(nn.Module): def __init__(self, base_model_name, num_segment_labels, num_category_labels, num_sentiment_labels, num_importance_labels): super().__init__() self.bert = AutoModel.from_pretrained(base_model_name) hidden_size = self.bert.config.hidden_size self.segment_head = nn.Sequential( nn.Dropout(0.1), nn.Linear(hidden_size, hidden_size // 2), nn.GELU(), nn.Dropout(0.1), nn.Linear(hidden_size // 2, num_segment_labels)) self.category_head = nn.Sequential( nn.Dropout(0.1), nn.Linear(hidden_size, hidden_size // 2), nn.GELU(), nn.Dropout(0.1), nn.Linear(hidden_size // 2, num_category_labels)) self.sentiment_head = nn.Sequential( nn.Dropout(0.1), nn.Linear(hidden_size, hidden_size // 2), nn.GELU(), nn.Dropout(0.1), nn.Linear(hidden_size // 2, num_sentiment_labels)) self.importance_head = nn.Sequential( nn.Dropout(0.1), nn.Linear(hidden_size, hidden_size // 2), nn.GELU(), nn.Dropout(0.1), nn.Linear(hidden_size // 2, num_importance_labels)) def forward(self, input_ids, attention_mask=None, **kwargs): outputs = self.bert(input_ids=input_ids, attention_mask=attention_mask) cls_output = outputs.last_hidden_state[:, 0, :] return { "segment_logits": self.segment_head(cls_output), "category_logits": self.category_head(cls_output), "sentiment_logits": self.sentiment_head(cls_output), "importance_logits": self.importance_head(cls_output), } def build_segment_category_mask(config): seg_labels = config["segment_labels"] cat_labels = config["category_labels"] mask = torch.zeros(len(seg_labels), len(cat_labels)) for seg_name, cat_names in SEGMENT_TO_CATEGORIES.items(): seg_id = seg_labels[seg_name] for cat_name in cat_names: cat_id = cat_labels[cat_name] mask[seg_id, cat_id] = 1.0 return mask def load_model(model_dir): with open(os.path.join(model_dir, "label_config.json")) as f: config = json.load(f) model = ModernBERTMultiTask( config["base_model"], config["num_segment_labels"], config["num_category_labels"], config["num_sentiment_labels"], config["num_importance_labels"]) state_dict = torch.load(os.path.join(model_dir, "pytorch_model.bin"), map_location="cpu") model.load_state_dict(state_dict, strict=False) model.eval() tokenizer = AutoTokenizer.from_pretrained(model_dir) return model, tokenizer, config def predict(text, model, tokenizer, config, device="cpu"): inputs = tokenizer(text, return_tensors="pt", padding=True, truncation=True, max_length=config.get("max_length", 2048)) inputs = {k: v.to(device) for k, v in inputs.items()} seg_cat_mask = build_segment_category_mask(config) with torch.no_grad(): outputs = model(**inputs) # Segment prediction segment_logits = outputs["segment_logits"] segment_probs = torch.softmax(segment_logits, dim=-1) segment_id = segment_probs.argmax(dim=-1).item() # Mask category logits based on predicted segment category_logits = outputs["category_logits"] mask = seg_cat_mask[segment_id] category_logits = category_logits.masked_fill(mask == 0, float('-inf')) category_probs = torch.softmax(category_logits, dim=-1) sentiment_probs = torch.softmax(outputs["sentiment_logits"], dim=-1) importance_probs = torch.softmax(outputs["importance_logits"], dim=-1) id2seg = config["id2segment"] id2cat = config["id2category"] id2sent = config["id2sentiment"] id2imp = config["id2importance"] return { "segment": id2seg[str(segment_id)], "segment_confidence": round(segment_probs.max().item(), 4), "category": id2cat[str(category_probs.argmax(dim=-1).item())], "category_confidence": round(category_probs.max().item(), 4), "sentiment": id2sent[str(sentiment_probs.argmax(dim=-1).item())], "sentiment_confidence": round(sentiment_probs.max().item(), 4), "importance": id2imp[str(importance_probs.argmax(dim=-1).item())], "importance_confidence": round(importance_probs.max().item(), 4), }