Biswajeet Arukha commited on
Commit
a3e599f
·
1 Parent(s): dccc3ae

Deep Prediction Upgrade: Transformer, MLflow, and MiroFish Multi-Agent Simulation

Browse files
backend/api_server.py CHANGED
@@ -4,16 +4,14 @@ import time
4
  import asyncio
5
  import logging
6
  import redis
 
 
7
  from fastapi import FastAPI, HTTPException, WebSocket, WebSocketDisconnect
8
  from fastapi.middleware.cors import CORSMiddleware
9
  from typing import List, Dict, Optional
 
10
  from backend.ml_engine.hybrid_model import RealTimePredictor
11
-
12
- # Environment Config
13
- REDIS_HOST = os.getenv("REDIS_HOST", "localhost")
14
- REDIS_PORT = int(os.getenv("REDIS_PORT", 6379))
15
- REDIS_DB = int(os.getenv("REDIS_DB", 0))
16
- REDIS_PASSWORD = os.getenv("REDIS_PASSWORD", None)
17
 
18
  # Logging configuration
19
  logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
@@ -34,6 +32,51 @@ app.add_middleware(
34
  allow_headers=["*"],
35
  )
36
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
37
  # Global Predictor Instance
38
  predictor: Optional[RealTimePredictor] = None
39
 
@@ -114,14 +157,43 @@ async def get_prediction(match_id: str):
114
  raise HTTPException(status_code=503, detail="ML Engine not initialized")
115
 
116
  try:
117
- result = predictor.predict_live_match(match_id)
118
- if 'error' in result:
119
- raise HTTPException(status_code=404, detail=result['error'])
120
- return result
 
 
 
 
 
 
121
  except Exception as e:
122
  logger.error(f"Prediction Error for {match_id}: {e}")
123
  raise HTTPException(status_code=500, detail="Internal inference error")
124
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
125
  @app.websocket("/ws/predictions/{match_id}")
126
  async def websocket_prediction(websocket: WebSocket, match_id: str):
127
  """Real-time prediction stream via WebSocket"""
 
4
  import asyncio
5
  import logging
6
  import redis
7
+ import random
8
+ import numpy as np
9
  from fastapi import FastAPI, HTTPException, WebSocket, WebSocketDisconnect
10
  from fastapi.middleware.cors import CORSMiddleware
11
  from typing import List, Dict, Optional
12
+ from pydantic import BaseModel
13
  from backend.ml_engine.hybrid_model import RealTimePredictor
14
+ from backend.ml_engine.agent_sim import MultiAgentSimulator
 
 
 
 
 
15
 
16
  # Logging configuration
17
  logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
 
32
  allow_headers=["*"],
33
  )
34
 
35
+ # Enrichment Layers
36
+ agent_swarm = MultiAgentSimulator(num_agents=10)
37
+
38
+ class SimulationRequest(BaseModel):
39
+ num_simulations: int = 1000
40
+ overs_to_project: int = 5
41
+ captain_policy: str = "aggressive" # aggressive, balanced, defensive
42
+
43
+ class MatchSimulator:
44
+ """Monte Carlo engine for match projection"""
45
+ def project_future(self, current_state: Dict, num_sims: int, overs: int, policy: str):
46
+ results = []
47
+ for _ in range(num_sims):
48
+ temp_runs = 0
49
+ temp_wickets = 0
50
+ for _ in range(overs * 6): # ball by ball
51
+ # Simple probabilistic transition (can be improved with Transformer)
52
+ if policy == "aggressive":
53
+ outcome = random.choices([0,1,2,4,6,'W'], weights=[35,20,10,15,10,10])[0]
54
+ else:
55
+ outcome = random.choices([0,1,2,4,6,'W'], weights=[50,25,10,5,5,5])[0]
56
+
57
+ if outcome == 'W':
58
+ temp_wickets += 1
59
+ if temp_wickets + current_state.get('total_wickets', 0) >= 10:
60
+ break
61
+ else:
62
+ temp_runs += outcome
63
+ results.append((temp_runs, temp_wickets))
64
+
65
+ runs_dist = [r for r, w in results]
66
+ return {
67
+ "projected_runs_avg": float(np.mean(runs_dist)),
68
+ "projected_wickets_avg": float(np.mean([w for r, w in results])),
69
+ "confidence_interval": [float(np.percentile(runs_dist, 5)), float(np.percentile(runs_dist, 95))]
70
+ }
71
+
72
+ match_sim = MatchSimulator()
73
+
74
+ # Environment Config
75
+ REDIS_HOST = os.getenv("REDIS_HOST", "localhost")
76
+ REDIS_PORT = int(os.getenv("REDIS_PORT", 6379))
77
+ REDIS_DB = int(os.getenv("REDIS_DB", 0))
78
+ REDIS_PASSWORD = os.getenv("REDIS_PASSWORD", None)
79
+
80
  # Global Predictor Instance
81
  predictor: Optional[RealTimePredictor] = None
82
 
 
157
  raise HTTPException(status_code=503, detail="ML Engine not initialized")
158
 
159
  try:
160
+ prediction = predictor.predict_live_match(match_id)
161
+
162
+ # Enrich with Multi-Agent Simulation features (MiroFish-style)
163
+ # We use current match data as seed for the swarm
164
+ agent_features = agent_swarm.simulate(prediction if "error" not in prediction else {})
165
+ prediction.update(agent_features)
166
+
167
+ if "error" in prediction:
168
+ raise HTTPException(status_code=404, detail=prediction["error"])
169
+ return prediction
170
  except Exception as e:
171
  logger.error(f"Prediction Error for {match_id}: {e}")
172
  raise HTTPException(status_code=500, detail="Internal inference error")
173
 
174
+ @app.post("/simulate/{match_id}")
175
+ async def run_what_if_simulation(match_id: str, request: SimulationRequest):
176
+ """
177
+ Project future match outcomes using Monte Carlo simulations.
178
+ Allows testing 'What If' tactical scenarios.
179
+ """
180
+ prediction = predictor.predict_live_match(match_id)
181
+ if "error" in prediction:
182
+ raise HTTPException(status_code=404, detail="No live data to simulate from")
183
+
184
+ projection = match_sim.project_future(
185
+ prediction,
186
+ request.num_simulations,
187
+ request.overs_to_project,
188
+ request.captain_policy
189
+ )
190
+
191
+ return {
192
+ "match_id": match_id,
193
+ "current_state": prediction,
194
+ "projection": projection
195
+ }
196
+
197
  @app.websocket("/ws/predictions/{match_id}")
198
  async def websocket_prediction(websocket: WebSocket, match_id: str):
199
  """Real-time prediction stream via WebSocket"""
backend/data_pipeline/sentiment_pipeline.py ADDED
@@ -0,0 +1,74 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import logging
3
+ import numpy as np
4
+ import tweepy
5
+ from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer
6
+ from typing import Dict, List, Optional
7
+
8
+ # Logging configuration
9
+ logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
10
+ logger = logging.getLogger(__name__)
11
+
12
+ class SentimentPipeline:
13
+ """Real-time sentiment analyzer for social media match reports."""
14
+ def __init__(self,
15
+ consumer_key: str = None,
16
+ consumer_secret: str = None,
17
+ access_token: str = None,
18
+ access_token_secret: str = None):
19
+
20
+ self.analyzer = SentimentIntensityAnalyzer()
21
+ self.api = None
22
+
23
+ # Initialize Twitter API if keys are provided
24
+ consumer_key = consumer_key or os.getenv("TWITTER_CONSUMER_KEY")
25
+ consumer_secret = consumer_secret or os.getenv("TWITTER_CONSUMER_SECRET")
26
+ access_token = access_token or os.getenv("TWITTER_ACCESS_TOKEN")
27
+ access_token_secret = access_token_secret or os.getenv("TWITTER_ACCESS_TOKEN_SECRET")
28
+
29
+ if all([consumer_key, consumer_secret, access_token, access_token_secret]):
30
+ try:
31
+ auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
32
+ auth.set_access_token(access_token, access_token_secret)
33
+ self.api = tweepy.API(auth)
34
+ logger.info("✅ Twitter API connection established.")
35
+ except Exception as e:
36
+ logger.error(f"❌ Failed to connect to Twitter API: {e}")
37
+ else:
38
+ logger.warning("⚠️ Twitter API keys missing. Sentiment will use mock random data.")
39
+
40
+ def get_match_sentiment(self, hashtag: str = "#IPL2024") -> Dict[str, float]:
41
+ """Scrape latest 100 tweets and return aggregate sentiment."""
42
+ if not self.api:
43
+ # Mock data for demonstration when offline/unauthenticated
44
+ return {
45
+ 'sentiment_score': float(np.random.uniform(0.3, 0.8)),
46
+ 'sentiment_volatility': float(np.random.uniform(0.05, 0.2))
47
+ }
48
+
49
+ try:
50
+ # Search tweets with the given hashtag
51
+ tweets = self.api.search_tweets(q=hashtag, count=100, lang='en', result_type='recent')
52
+ scores = []
53
+
54
+ for tweet in tweets:
55
+ # VADER: compound score (-1.0 to 1.0)
56
+ sentiment = self.analyzer.polarity_scores(tweet.text)['compound']
57
+ scores.append(sentiment)
58
+
59
+ if not scores:
60
+ return {'sentiment_score': 0.0, 'sentiment_volatility': 0.0}
61
+
62
+ return {
63
+ 'sentiment_score': float(np.mean(scores)),
64
+ 'sentiment_volatility': float(np.std(scores))
65
+ }
66
+ except Exception as e:
67
+ logger.error(f"❌ Sentiment analysis failed: {e}")
68
+ return {'sentiment_score': 0.0, 'sentiment_volatility': 0.0}
69
+
70
+ class MomentumMonitor:
71
+ """Combines sentiment and simulation scores into a unified momentum feature."""
72
+ def __init__(self):
73
+ # We integrate the MiroFish sentiment signal into our RealTimePredictor
74
+ pass
backend/ml_engine/agent_sim.py ADDED
@@ -0,0 +1,88 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import random
3
+ import logging
4
+ import numpy as np
5
+ from openai import OpenAI
6
+ from typing import Dict, List, Optional
7
+
8
+ # Logging configuration
9
+ logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
10
+ logger = logging.getLogger(__name__)
11
+
12
+ class CricketAnalystAgent:
13
+ """An AI agent representing a specific cricket analyst archetype."""
14
+ def __init__(self, bias_type: str, client: Optional[OpenAI] = None):
15
+ self.bias_type = bias_type # e.g., 'pitch_focused', 'form_focused', 'tactical'
16
+ self.client = client
17
+
18
+ def analyze(self, match_state: Dict) -> float:
19
+ """Return a sentiment/confidence score [0-1] based on the agent's bias."""
20
+ if self.client:
21
+ try:
22
+ # LLM-based analysis for high-fidelity qualitative signal
23
+ prompt = self._build_prompt(match_state)
24
+ response = self.client.completions.create(
25
+ model="gpt-3.5-turbo-instruct",
26
+ prompt=prompt,
27
+ max_tokens=60,
28
+ temperature=0.7
29
+ )
30
+ text = response.choices[0].text.strip().lower()
31
+
32
+ # Simple sentiment heuristic
33
+ if any(word in text for word in ["high", "confident", "easy", "dominant"]):
34
+ return 0.85
35
+ elif any(word in text for word in ["low", "worried", "struggle", "collapse"]):
36
+ return 0.25
37
+ else:
38
+ return 0.5
39
+ except Exception as e:
40
+ logger.warning(f"LLM Agent ({self.bias_type}) failed: {e}. Falling back to simulation.")
41
+
42
+ # Fallback deterministic simulation based on match state
43
+ return self._simulate_score(match_state)
44
+
45
+ def _build_prompt(self, match_state: Dict) -> str:
46
+ return (f"As a cricket analyst specializing in {self.bias_type}, analyze this IPL match state: "
47
+ f"{match_state}. Does the batting team look dominant or under pressure? "
48
+ f"Give a short 1-sentence analysis and a confidence score.")
49
+
50
+ def _simulate_score(self, match_state: Dict) -> float:
51
+ """Heuristic-based fallback logic to ensure speed when API is unavailable."""
52
+ if self.bias_type == 'pitch_focused':
53
+ # Pitch focus: Confidence high if runs are being scored
54
+ crr = match_state.get('crr', 6.0)
55
+ return 0.8 if crr > 9.0 else 0.4
56
+ elif self.bias_type == 'form_focused':
57
+ # Form focus: Confidence high if few wickets lost
58
+ wickets = match_state.get('total_wickets', 0)
59
+ return 0.9 if wickets < 2 else 0.4
60
+ elif self.bias_type == 'momentum_analyst':
61
+ # Momentum focus: Recent runs
62
+ recent_runs = match_state.get('runs_last_6', 6)
63
+ return 0.85 if recent_runs > 12 else 0.35
64
+ return 0.5
65
+
66
+
67
+ class MultiAgentSimulator:
68
+ """Manages a swarm of analysts to extract emergent confidence features."""
69
+ def __init__(self, num_agents: int = 10, api_key: Optional[str] = None):
70
+ self.num_agents = num_agents
71
+ self.api_key = api_key or os.getenv("OPENAI_API_KEY")
72
+ self.client = OpenAI(api_key=self.api_key) if self.api_key else None
73
+
74
+ self.biases = ['pitch_focused', 'form_focused', 'momentum_analyst', 'tactical_observer', 'collapse_historian']
75
+ self.agents = [CricketAnalystAgent(random.choice(self.biases), self.client) for _ in range(num_agents)]
76
+
77
+ def simulate(self, match_state: Dict) -> Dict[str, float]:
78
+ """Aggregate analysis scores into aggregate ensemble features."""
79
+ scores = [agent.analyze(match_state) for agent in self.agents]
80
+
81
+ # We extract two key metrics for the XGBoost model
82
+ # 1. Agent Confidence: The average sentiment across the swarm
83
+ # 2. Agent Disagreement (Entropy): The standard deviation (uncertainty)
84
+
85
+ return {
86
+ 'agent_confidence': float(np.mean(scores)),
87
+ 'agent_disagreement': float(np.std(scores))
88
+ }
backend/ml_engine/hybrid_model.py CHANGED
@@ -41,6 +41,13 @@ class ModelConfig:
41
  lstm_num_layers: int = 2
42
  lstm_dropout: float = 0.3
43
  sequence_length: int = 18 # Last 18 balls
 
 
 
 
 
 
 
44
 
45
  # Training params
46
  batch_size: int = 64
@@ -440,6 +447,7 @@ class HybridEnsemble:
440
  # Initialize models
441
  self.xgb_model: Optional[xgb.XGBClassifier] = None
442
  self.lstm_model: Optional[MomentumLSTM] = None
 
443
 
444
  # Scalers
445
  self.static_scaler = StandardScaler()
@@ -465,6 +473,18 @@ class HybridEnsemble:
465
  )
466
  return model.to(self.device)
467
 
 
 
 
 
 
 
 
 
 
 
 
 
468
  def train_xgb(self,
469
  X_train: np.ndarray,
470
  y_train: np.ndarray,
@@ -574,6 +594,60 @@ class HybridEnsemble:
574
  self.lstm_model.load_state_dict(torch.load('best_lstm.pth'))
575
  logger.info("LSTM training complete")
576
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
577
  def _validate_lstm(self,
578
  val_sequences: np.ndarray,
579
  val_labels: np.ndarray,
 
41
  lstm_num_layers: int = 2
42
  lstm_dropout: float = 0.3
43
  sequence_length: int = 18 # Last 18 balls
44
+ # Transformer params (New)
45
+ use_transformer: bool = True
46
+ transformer_d_model: int = 128
47
+ transformer_nhead: int = 8
48
+ transformer_num_layers: int = 3
49
+ transformer_dim_feedforward: int = 256
50
+ transformer_dropout: float = 0.1
51
 
52
  # Training params
53
  batch_size: int = 64
 
447
  # Initialize models
448
  self.xgb_model: Optional[xgb.XGBClassifier] = None
449
  self.lstm_model: Optional[MomentumLSTM] = None
450
+ self.transformer_model: Optional[TransformerModel] = None
451
 
452
  # Scalers
453
  self.static_scaler = StandardScaler()
 
473
  )
474
  return model.to(self.device)
475
 
476
+ def build_transformer_model(self) -> TransformerModel:
477
+ """Build Transformer model for sequence features"""
478
+ model = TransformerModel(
479
+ input_dim=3,
480
+ d_model=self.config.transformer_d_model,
481
+ nhead=self.config.transformer_nhead,
482
+ num_layers=self.config.transformer_num_layers,
483
+ dim_feedforward=self.config.transformer_dim_feedforward,
484
+ dropout=self.config.transformer_dropout
485
+ )
486
+ return model.to(self.device)
487
+
488
  def train_xgb(self,
489
  X_train: np.ndarray,
490
  y_train: np.ndarray,
 
594
  self.lstm_model.load_state_dict(torch.load('best_lstm.pth'))
595
  logger.info("LSTM training complete")
596
 
597
+ def train_transformer(self,
598
+ train_sequences: np.ndarray,
599
+ train_labels: np.ndarray,
600
+ val_sequences: Optional[np.ndarray] = None,
601
+ val_labels: Optional[np.ndarray] = None):
602
+ """Train Transformer model"""
603
+ logger.info("Training Transformer model...")
604
+
605
+ self.transformer_model = self.build_transformer_model()
606
+
607
+ train_dataset = SequenceDataset(train_sequences, train_labels)
608
+ train_loader = DataLoader(train_dataset, batch_size=self.config.batch_size, shuffle=True)
609
+
610
+ criterion = nn.BCEWithLogitsLoss()
611
+ optimizer = torch.optim.Adam(self.transformer_model.parameters(), lr=self.config.learning_rate)
612
+
613
+ best_val_loss = float('inf')
614
+ for epoch in range(self.config.num_epochs):
615
+ self.transformer_model.train()
616
+ for batch_seq, batch_labels in train_loader:
617
+ batch_seq = batch_seq.to(self.device)
618
+ batch_labels = batch_labels.to(self.device)
619
+
620
+ optimizer.zero_grad()
621
+ features = self.transformer_model(batch_seq)
622
+ predictions = features[:, 0]
623
+ loss = criterion(predictions, batch_labels.float())
624
+ loss.backward()
625
+ optimizer.step()
626
+
627
+ if val_sequences is not None:
628
+ val_loss = self._validate_transformer(val_sequences, val_labels, criterion)
629
+ if val_loss < best_val_loss:
630
+ best_val_loss = val_loss
631
+ torch.save(self.transformer_model.state_dict(), 'best_transformer.pth')
632
+
633
+ self.transformer_model.load_state_dict(torch.load('best_transformer.pth'))
634
+ logger.info("Transformer training complete")
635
+
636
+ def _validate_transformer(self,
637
+ val_sequences: np.ndarray,
638
+ val_labels: np.ndarray,
639
+ criterion) -> float:
640
+ self.transformer_model.eval()
641
+ val_loader = DataLoader(SequenceDataset(val_sequences, val_labels), batch_size=self.config.batch_size)
642
+ total_loss = 0.0
643
+ with torch.no_grad():
644
+ for seq, labels in val_loader:
645
+ seq, labels = seq.to(self.device), labels.to(self.device)
646
+ features = self.transformer_model(seq)
647
+ loss = criterion(features[:, 0], labels.float())
648
+ total_loss += loss.item()
649
+ return total_loss / len(val_loader)
650
+
651
  def _validate_lstm(self,
652
  val_sequences: np.ndarray,
653
  val_labels: np.ndarray,
backend/ml_engine/knowledge_graph.py ADDED
@@ -0,0 +1,80 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import networkx as nx
2
+ import logging
3
+ import pickle
4
+ import os
5
+ from typing import Dict, List, Optional, Tuple
6
+
7
+ # Logging configuration
8
+ logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
9
+ logger = logging.getLogger(__name__)
10
+
11
+ class CricketKnowledgeGraph:
12
+ """GraphRAG-style knowledge base for deep matchup historical memory."""
13
+ def __init__(self, storage_path: str = "models/matchup_graph.pkl"):
14
+ self.graph = nx.MultiDiGraph()
15
+ self.storage_path = storage_path
16
+
17
+ # Load existing graph if available
18
+ if os.path.exists(self.storage_path):
19
+ try:
20
+ with open(self.storage_path, 'rb') as f:
21
+ self.graph = pickle.load(f)
22
+ logger.info(f"✅ Loaded Knowledge Graph with {self.graph.number_of_nodes()} nodes.")
23
+ except Exception as e:
24
+ logger.warning(f"❌ Failed to load Knowledge Graph from {self.storage_path}: {e}. Creating new.")
25
+
26
+ def add_matchup_event(self, batsman: str, bowler: str, runs: int, wicket: bool, context: Dict):
27
+ """Adds a single ball outcome between batsman and bowler as an edge."""
28
+ # Clean labels
29
+ batsman, bowler = batsman.strip(), bowler.strip()
30
+
31
+ # Add edge with attributes
32
+ self.graph.add_edge(
33
+ bowler,
34
+ batsman,
35
+ runs=runs,
36
+ wicket=wicket,
37
+ venue=context.get('venue', 'N/A'),
38
+ match_id=context.get('match_id', 'N/A')
39
+ )
40
+
41
+ def query_matchup_stats(self, batsman: str, bowler: str) -> Dict[str, float]:
42
+ """Expressed as [Bowler -> Batsman] edges. Returns statistical summary."""
43
+ batsman, bowler = batsman.strip(), bowler.strip()
44
+
45
+ if not self.graph.has_edge(bowler, batsman):
46
+ return {
47
+ 'matchup_count': 0,
48
+ 'avg_runs_per_ball': 0.0,
49
+ 'wicket_rate': 0.0,
50
+ 'dominance_index': 0.5 # Neutral index
51
+ }
52
+
53
+ # Extract edge data
54
+ edge_data = self.graph.get_edge_data(bowler, batsman)
55
+ balls = len(edge_data)
56
+ runs = sum(e['runs'] for e in edge_data.values())
57
+ wickets = sum(1 for e in edge_data.values() if e['wicket'])
58
+
59
+ # Dominance Heuristic: High if batsman dominates bowler
60
+ # (Average runs per ball vs Wicket frequency)
61
+ avg_runs = runs / balls
62
+ wicket_freq = wickets / balls
63
+
64
+ # Dominance Index: 1.0 (Batsman dominates), 0.0 (Bowler dominates)
65
+ dominance = (avg_runs / 2.0) - (wicket_freq * 10.0)
66
+ dominance = max(0.0, min(1.0, 0.5 + dominance)) # Clamped between 0 and 1
67
+
68
+ return {
69
+ 'matchup_count': balls,
70
+ 'avg_runs_per_ball': avg_runs,
71
+ 'wicket_rate': wicket_freq,
72
+ 'dominance_index': dominance
73
+ }
74
+
75
+ def save(self):
76
+ """Persist the graph to disk."""
77
+ os.makedirs(os.path.dirname(self.storage_path), exist_ok=True)
78
+ with open(self.storage_path, 'wb') as f:
79
+ pickle.dump(self.graph, f)
80
+ logger.info(f"✅ Knowledge Graph saved to {self.storage_path}.")
backend/ml_engine/retrain_service.py ADDED
@@ -0,0 +1,67 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import time
3
+ import logging
4
+ from datetime import datetime
5
+ from pathlib import Path
6
+ from backend.ml_engine.train_efficient import train_efficiently
7
+
8
+ # Logging configuration
9
+ logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
10
+ logger = logging.getLogger(__name__)
11
+
12
+ # Configuration
13
+ DATA_DIR = os.getenv("DATA_DIR", "data")
14
+ CHECK_INTERVAL = int(os.getenv("CHECK_INTERVAL", 3600)) # 1 hour
15
+ LAST_CHECK_FILE = "backend/ml_engine/last_retrain.txt"
16
+
17
+ def get_last_retrain_time():
18
+ """Read the last successfull retrain timestamp"""
19
+ if not os.path.exists(LAST_CHECK_FILE):
20
+ return datetime.min
21
+ try:
22
+ with open(LAST_CHECK_FILE, 'r') as f:
23
+ return datetime.fromisoformat(f.read().strip())
24
+ except Exception as e:
25
+ logger.warning(f"Could not read last_retrain file: {e}")
26
+ return datetime.min
27
+
28
+ def update_last_retrain_time():
29
+ """Save the current timestamp as the last retrain time"""
30
+ with open(LAST_CHECK_FILE, 'w') as f:
31
+ f.write(datetime.now().isoformat())
32
+
33
+ def check_for_new_data(last_time):
34
+ """Scan the data directory for any files modified after last_time"""
35
+ new_files = []
36
+ # Search for JSON match files
37
+ for file in Path(DATA_DIR).glob("**/*.json"):
38
+ mod_time = datetime.fromtimestamp(file.stat().st_mtime)
39
+ if mod_time > last_time:
40
+ new_files.append(file)
41
+
42
+ return new_files
43
+
44
+ def main_loop():
45
+ logger.info(f"🚀 Starting Automated Retraining Service (Watching: {DATA_DIR})")
46
+
47
+ while True:
48
+ last_retrain = get_last_retrain_time()
49
+ new_data = check_for_new_data(last_retrain)
50
+
51
+ if new_data:
52
+ logger.info(f"📈 Found {len(new_data)} new match files! Triggering retraining pipeline...")
53
+ try:
54
+ # Trigger the efficient training pipeline
55
+ train_efficiently(data_dir=DATA_DIR)
56
+
57
+ update_last_retrain_time()
58
+ logger.info("✅ Retraining complete and model updated.")
59
+ except Exception as e:
60
+ logger.error(f"❌ Retraining failed: {e}")
61
+ else:
62
+ logger.info("😴 No new data found. Sleeping...")
63
+
64
+ time.sleep(CHECK_INTERVAL)
65
+
66
+ if __name__ == "__main__":
67
+ main_loop()
backend/ml_engine/train_efficient.py CHANGED
@@ -9,6 +9,9 @@ import torch
9
  from torch.utils.data import IterableDataset, DataLoader
10
  import xgboost as xgb
11
  from sklearn.model_selection import train_test_split
 
 
 
12
  from hybrid_model import HybridEnsemble, ModelConfig, CricsheetNormalizer
13
 
14
  logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
@@ -65,6 +68,20 @@ def train_efficiently(data_dir='data'):
65
  config = ModelConfig()
66
  ensemble = HybridEnsemble(config)
67
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
68
  # 1. Get all JSON files (matches)
69
  all_files = glob.glob(os.path.join(data_dir, '*.json'))
70
  if not all_files:
@@ -124,6 +141,9 @@ def train_efficiently(data_dir='data'):
124
  else:
125
  xgb_model = xgb.train(params, dtrain, num_boost_round=10, xgb_model=xgb_model)
126
 
 
 
 
127
  # Free memory!
128
  del chunk_dfs, combined_df, static_features, X_chunk, y_chunk, dtrain
129
  gc.collect()
@@ -161,13 +181,48 @@ def train_efficiently(data_dir='data'):
161
  batch_count += 1
162
  if batch_count % 100 == 0:
163
  logger.info(f"LSTM Batch {batch_count}: Loss = {loss.item():.4f}")
 
164
  gc.collect() # Periodically clean garbage within epoch
165
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
166
  # Save the models
167
  logger.info("Training complete! Saving models...")
168
  os.makedirs('models', exist_ok=True)
169
  ensemble.save_models('models/hybrid_ensemble')
170
 
 
 
 
 
 
171
  if __name__ == "__main__":
172
  import argparse
173
  parser = argparse.ArgumentParser(description="Memory-Efficient Training Pipeline")
 
9
  from torch.utils.data import IterableDataset, DataLoader
10
  import xgboost as xgb
11
  from sklearn.model_selection import train_test_split
12
+ import mlflow
13
+ import mlflow.pytorch
14
+ import mlflow.xgboost
15
  from hybrid_model import HybridEnsemble, ModelConfig, CricsheetNormalizer
16
 
17
  logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
 
68
  config = ModelConfig()
69
  ensemble = HybridEnsemble(config)
70
 
71
+ # Initialize MLflow
72
+ mlflow.set_experiment("IPL_Prediction_Engine_V2")
73
+
74
+ with mlflow.start_run(run_name=f"Hybrid_Train_{pd.Timestamp.now().strftime('%Y%m%d_%H%M%S')}"):
75
+ # Log Hyperparameters
76
+ mlflow.log_params({
77
+ "xgb_max_depth": config.xgb_params['max_depth'],
78
+ "xgb_lr": config.xgb_params['learning_rate'],
79
+ "lstm_hidden": config.lstm_hidden_size,
80
+ "transformer_layers": config.transformer_num_layers,
81
+ "batch_size": config.batch_size,
82
+ "learning_rate": config.learning_rate
83
+ })
84
+
85
  # 1. Get all JSON files (matches)
86
  all_files = glob.glob(os.path.join(data_dir, '*.json'))
87
  if not all_files:
 
141
  else:
142
  xgb_model = xgb.train(params, dtrain, num_boost_round=10, xgb_model=xgb_model)
143
 
144
+ # Log XGBoost status to MLflow
145
+ mlflow.log_metric("xgb_chunk_processed", i//chunk_size + 1)
146
+
147
  # Free memory!
148
  del chunk_dfs, combined_df, static_features, X_chunk, y_chunk, dtrain
149
  gc.collect()
 
181
  batch_count += 1
182
  if batch_count % 100 == 0:
183
  logger.info(f"LSTM Batch {batch_count}: Loss = {loss.item():.4f}")
184
+ mlflow.log_metric("lstm_loss", loss.item(), step=batch_count)
185
  gc.collect() # Periodically clean garbage within epoch
186
 
187
+ # ---------------------------------------------------------
188
+ # PART C: Streaming Training for PyTorch Transformer
189
+ # ---------------------------------------------------------
190
+ logger.info("Starting Transformer Streaming Training...")
191
+ ensemble.transformer_model = ensemble.build_transformer_model()
192
+
193
+ t_optimizer = torch.optim.Adam(ensemble.transformer_model.parameters(), lr=config.learning_rate)
194
+ ensemble.transformer_model.train()
195
+
196
+ t_batch_count = 0
197
+ t_loader = DataLoader(train_dataset, batch_size=config.batch_size)
198
+
199
+ for seq, label in t_loader:
200
+ seq, label = seq.to(ensemble.device), label.to(ensemble.device)
201
+ t_optimizer.zero_grad()
202
+
203
+ features = ensemble.transformer_model(seq)
204
+ predictions = features[:, 0]
205
+
206
+ loss = criterion(predictions, label.squeeze())
207
+ loss.backward()
208
+ t_optimizer.step()
209
+
210
+ t_batch_count += 1
211
+ if t_batch_count % 100 == 0:
212
+ logger.info(f"Transformer Batch {t_batch_count}: Loss = {loss.item():.4f}")
213
+ mlflow.log_metric("transformer_loss", loss.item(), step=t_batch_count)
214
+ gc.collect()
215
+
216
  # Save the models
217
  logger.info("Training complete! Saving models...")
218
  os.makedirs('models', exist_ok=True)
219
  ensemble.save_models('models/hybrid_ensemble')
220
 
221
+ # Log models to MLflow
222
+ mlflow.xgboost.log_model(ensemble.xgb_model, "xgb_static")
223
+ mlflow.pytorch.log_model(ensemble.lstm_model, "lstm_sequence")
224
+ mlflow.pytorch.log_model(ensemble.transformer_model, "transformer_attention")
225
+
226
  if __name__ == "__main__":
227
  import argparse
228
  parser = argparse.ArgumentParser(description="Memory-Efficient Training Pipeline")