tahamajs's picture
download
raw
6.87 kB
"""
AI Insight Engine — analyzes experiment history and generates insights.
Uses simple statistical methods (no external API needed).
"""
import logging
from collections import Counter
logger = logging.getLogger(__name__)
def analyze_user_experiments(user):
"""
Analyze a user's experiments and return a dict of insights.
"""
from experiment.models import TrainingRun
runs = TrainingRun.objects.filter(user=user, status="completed").order_by(
"-created_at"
)
if not runs.exists():
return []
insights = []
# 1. Trend analysis — are we improving over time?
trend = _detect_improvement_trend(runs)
if trend:
insights.append(trend)
# 2. Winning pattern — what makes the best runs special?
winner = _detect_winning_pattern(runs)
if winner:
insights.append(winner)
# 3. Optimization suggestion
optimization = _suggest_optimization(runs)
if optimization:
insights.append(optimization)
# 4. Anomaly detection
anomaly = _detect_anomaly(runs)
if anomaly:
insights.append(anomaly)
return insights
def _detect_improvement_trend(runs):
"""Check if accuracy is improving over recent runs."""
recent = [
r for r in runs if r.final_metrics and r.final_metrics.get("eval_accuracy")
]
if len(recent) < 3:
return None
recent.sort(key=lambda r: r.created_at)
accuracies = [r.final_metrics["eval_accuracy"] for r in recent]
# Simple linear trend: compare first half vs second half
mid = len(accuracies) // 2
first_half_avg = sum(accuracies[:mid]) / mid
second_half_avg = sum(accuracies[mid:]) / (len(accuracies) - mid)
improvement = (second_half_avg - first_half_avg) / max(first_half_avg, 0.001)
if improvement > 0.05:
return {
"insight_type": "trend",
"title": "📈 Accuracy is improving!",
"description": f"Your recent experiments show a {improvement*100:.1f}% improvement in accuracy compared to earlier runs. Keep going with the current approach.",
"confidence": min(0.9, 0.5 + improvement),
"metadata": {
"first_half_avg": round(first_half_avg, 4),
"second_half_avg": round(second_half_avg, 4),
"improvement_pct": round(improvement * 100, 1),
},
}
elif improvement < -0.03:
return {
"insight_type": "trend",
"title": "📉 Accuracy is declining",
"description": f"Recent experiments are performing worse than earlier ones ({abs(improvement)*100:.1f}% drop). Consider reverting to earlier configurations or trying a different approach.",
"confidence": min(0.8, 0.5 + abs(improvement)),
"metadata": {
"first_half_avg": round(first_half_avg, 4),
"second_half_avg": round(second_half_avg, 4),
"decline_pct": round(abs(improvement) * 100, 1),
},
}
return None
def _detect_winning_pattern(runs):
"""Find what the best performing runs have in common."""
completed = [
r for r in runs if r.final_metrics and r.final_metrics.get("eval_accuracy")
]
if len(completed) < 3:
return None
# Sort by accuracy
completed.sort(key=lambda r: r.final_metrics["eval_accuracy"], reverse=True)
top_runs = completed[: max(1, len(completed) // 3)]
# Analyze common config patterns in top runs
configs = [r.config or {} for r in top_runs]
patterns = {}
for key in ["attention_type", "method", "batch_size", "learning_rate", "epochs"]:
values = [c.get(key) for c in configs if c.get(key) is not None]
if values:
most_common = Counter(values).most_common(1)[0]
if most_common[1] >= len(top_runs) * 0.5:
patterns[key] = most_common[0]
if patterns:
pattern_desc = ", ".join(f"{k}={v}" for k, v in patterns.items())
return {
"insight_type": "winner",
"title": "🏆 Winning pattern detected",
"description": f"Your best experiments share these traits: {pattern_desc}. Consider using these settings as a baseline.",
"confidence": 0.75,
"metadata": {"patterns": patterns},
}
return None
def _suggest_optimization(runs):
"""Suggest an optimization based on what hasn't been tried."""
completed = [r for r in runs if r.final_metrics and r.config]
if len(completed) < 3:
return None
# What attention types have been tried?
tried_attention = set(
r.config.get("attention_type")
for r in completed
if r.config.get("attention_type")
)
tried_methods = set(
r.config.get("method") for r in completed if r.config.get("method")
)
# Suggest untried attention types
all_attention = {
"full",
"galerkin",
"linformer",
"spectraformer",
"padre",
"aurora",
"nova",
}
untried = all_attention - tried_attention
if untried:
suggestion = list(untried)[0]
return {
"insight_type": "optimization",
"title": f"💡 Try a new attention type: {suggestion}",
"description": f"You haven't experimented with the '{suggestion}' attention mechanism yet. Researchers on similar tasks have seen improvements with it.",
"confidence": 0.6,
"metadata": {"suggestion": suggestion, "category": "attention_type"},
}
return None
def _detect_anomaly(runs):
"""Detect runs that are outliers (much worse or better than average)."""
completed = [
r for r in runs if r.final_metrics and r.final_metrics.get("eval_accuracy")
]
if len(completed) < 5:
return None
accuracies = [r.final_metrics["eval_accuracy"] for r in completed]
avg = sum(accuracies) / len(accuracies)
std = (sum((a - avg) ** 2 for a in accuracies) / len(accuracies)) ** 0.5
# Find outliers (> 2 std away)
for r in completed:
acc = r.final_metrics["eval_accuracy"]
if abs(acc - avg) > 2 * std:
direction = "higher" if acc > avg else "lower"
return {
"insight_type": "anomaly",
"title": "🔍 Unusual result detected",
"description": f"One of your experiments achieved an accuracy of {acc:.4f}, which is significantly {direction} than your average of {avg:.4f}. Investigate what made this run different.",
"confidence": 0.7,
"metadata": {
"anomaly_run_id": str(r.id),
"anomaly_value": acc,
"average": round(avg, 4),
"std": round(std, 4),
},
}
return None

Xet Storage Details

Size:
6.87 kB
·
Xet hash:
97455df8c33dd22b319dbb969bf80b6e1a23b1473d1131f0ab7e47df68ee7c0e

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.