Buckets:

tahamajs
/

Sysmem2_in_AI

Files

xet

tahamajs/Sysmem2_in_AI / ComputerAssignments /CA15_test_time_training /src /visualization /plotter.py

tahamajs

5 months ago

download

raw

8.79 kB

import matplotlib.pyplot as plt
import seaborn as sns
import numpy as np
import json
from pathlib import Path
from typing import Dict, Any, List, Optional
import pandas as pd


plt.style.use("seaborn-v0_8")
sns.set_palette("husl")

class TTTVisualizer:
    def __init__(self, results_dir: str = "results"):
        self.results_dir = Path(results_dir)
        self.plots_dir = self.results_dir / "plots"
        self.plots_dir.mkdir(parents=True, exist_ok=True)

    def load_metrics(self) -> Optional[Dict[str, Any]]:
        metrics_file = self.results_dir / "logs" / "metrics.json"
        if metrics_file.exists():
            with open(metrics_file, "r") as f:
                return json.load(f)
        return None

    def create_performance_comparison(self, metrics: Dict[str, Any]) -> None:
        fig, ax = plt.subplots(1, 1, figsize=(10, 6))

        baseline_acc = metrics.get("baseline_accuracy", 0)
        ttt_acc = metrics.get("ttt_accuracy", 0)

        models = ["Baseline Model", "TTT Adapted Model"]
        accuracies = [baseline_acc, ttt_acc]
        colors = ["lightcoral", "lightgreen"]

        bars = ax.bar(
            models, accuracies, color=colors, alpha=0.8, edgecolor="black", linewidth=1
        )
        ax.set_title(
            "Test-Time Training Performance Comparison",
            fontsize=16,
            fontweight="bold",
            pad=20,
        )
        ax.set_ylabel("Accuracy", fontsize=12)
        ax.set_ylim(0, 1)
        ax.grid(True, alpha=0.3, axis="y")


        for bar, acc in zip(bars, accuracies):
            height = bar.get_height()
            ax.text(
                bar.get_x() + bar.get_width() / 2.0,
                height + 0.01,
                f"{acc:.3f}",
                ha="center",
                va="bottom",
                fontweight="bold",
                fontsize=12,
            )


        improvement = ttt_acc - baseline_acc
        if improvement > 0:
            ax.annotate(
                (
                    f"Improvement: +{improvement:.3f}\n({(improvement/baseline_acc)*100:.1f}% relative)"
                    if baseline_acc > 0
                    else "Achieved success from failure"
                ),
                xy=(1, ttt_acc),
                xytext=(0.5, 0.8),
                arrowprops=dict(arrowstyle="->", color="red", lw=2),
                fontsize=10,
                ha="center",
                va="center",
                bbox=dict(boxstyle="round,pad=0.3", facecolor="yellow", alpha=0.7),
            )

        plt.tight_layout()
        plt.savefig(
            self.plots_dir / "performance_comparison.png", dpi=300, bbox_inches="tight"
        )
        plt.close()
        print("✅ Performance comparison plot saved")

    def create_detailed_results_plot(self, metrics: Dict[str, Any]) -> None:
        if "results_detail" not in metrics:
            return

        results_detail = metrics["results_detail"]
        if not results_detail:
            return


        inputs = [r["input"] for r in results_detail]
        baseline_preds = [r.get("baseline_prediction", "") for r in results_detail]
        adapted_preds = [r.get("adapted_prediction", "") for r in results_detail]
        expected = [r.get("expected", "") for r in results_detail]


        fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(12, 10))


        x = np.arange(len(inputs))
        width = 0.25

        baseline_correct = [
            1 if pred == exp else 0 for pred, exp in zip(baseline_preds, expected)
        ]
        adapted_correct = [
            1 if pred == exp else 0 for pred, exp in zip(adapted_preds, expected)
        ]

        ax1.bar(
            x - width,
            baseline_correct,
            width,
            label="Baseline",
            alpha=0.7,
            color="lightcoral",
        )
        ax1.bar(
            x + width,
            adapted_correct,
            width,
            label="TTT Adapted",
            alpha=0.7,
            color="lightgreen",
        )

        ax1.set_xlabel("Test Cases", fontsize=12)
        ax1.set_ylabel("Correct (1) / Incorrect (0)", fontsize=12)
        ax1.set_title(
            "Individual Test Case Performance", fontsize=14, fontweight="bold"
        )
        ax1.set_xticks(x)
        ax1.set_xticklabels(inputs, rotation=45, ha="right")
        ax1.legend()
        ax1.grid(True, alpha=0.3)


        baseline_acc = sum(baseline_correct) / len(baseline_correct)
        adapted_acc = sum(adapted_correct) / len(adapted_correct)

        models = ["Baseline", "TTT Adapted"]
        accuracies = [baseline_acc, adapted_acc]
        colors = ["lightcoral", "lightgreen"]

        bars = ax2.bar(models, accuracies, color=colors, alpha=0.8, edgecolor="black")
        ax2.set_ylabel("Accuracy", fontsize=12)
        ax2.set_title("Overall Accuracy Comparison", fontsize=14, fontweight="bold")
        ax2.set_ylim(0, 1)
        ax2.grid(True, alpha=0.3, axis="y")

        for bar, acc in zip(bars, accuracies):
            height = bar.get_height()
            ax2.text(
                bar.get_x() + bar.get_width() / 2.0,
                height + 0.01,
                f"{acc:.3f}",
                ha="center",
                va="bottom",
                fontweight="bold",
            )

        plt.tight_layout()
        plt.savefig(
            self.plots_dir / "detailed_results.png", dpi=300, bbox_inches="tight"
        )
        plt.close()
        print("✅ Detailed results plot saved")

    def create_summary_dashboard(self, metrics: Dict[str, Any]) -> None:
        fig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(2, 2, figsize=(15, 12))


        baseline_acc = metrics.get("baseline_accuracy", 0)
        ttt_acc = metrics.get("ttt_accuracy", 0)

        models = ["Baseline", "TTT Adapted"]
        accuracies = [baseline_acc, ttt_acc]
        colors = ["lightcoral", "lightgreen"]

        bars = ax1.bar(models, accuracies, color=colors, alpha=0.8)
        ax1.set_title("Model Performance Comparison", fontsize=14, fontweight="bold")
        ax1.set_ylabel("Accuracy")
        ax1.set_ylim(0, 1)
        ax1.grid(True, alpha=0.3)

        for bar, acc in zip(bars, accuracies):
            height = bar.get_height()
            ax1.text(
                bar.get_x() + bar.get_width() / 2.0,
                height + 0.02,
                f"{acc:.3f}",
                ha="center",
                va="bottom",
                fontweight="bold",
            )


        improvement = ttt_acc - baseline_acc
        ax2.barh(["Absolute Improvement"], [improvement], color="gold", alpha=0.7)
        ax2.set_title("TTT Improvement", fontsize=14, fontweight="bold")
        ax2.set_xlabel("Accuracy Gain")
        ax2.axvline(x=0, color="black", linestyle="--", alpha=0.5)

        ax2.text(
            improvement / 2,
            0,
            f"{improvement:.3f}",
            ha="center",
            va="center",
            fontweight="bold",
            fontsize=16,
        )


        ax3.axis("off")
config_text = f
        ax3.text(
            0.1,
            0.5,
            config_text,
            fontsize=12,
            verticalalignment="center",
            bbox=dict(boxstyle="round,pad=0.5", facecolor="lightblue", alpha=0.7),
        )
        ax3.set_title("Configuration", fontsize=14, fontweight="bold")


        ax4.axis("off")
results_text = f
        ax4.text(
            0.1,
            0.5,
            results_text,
            fontsize=12,
            verticalalignment="center",
            bbox=dict(boxstyle="round,pad=0.5", facecolor="lightgreen", alpha=0.7),
        )
        ax4.set_title("Results", fontsize=14, fontweight="bold")

        plt.suptitle(
            "Test-Time Training (TTT) Complete Analysis Dashboard",
            fontsize=16,
            fontweight="bold",
            y=0.98,
        )
        plt.tight_layout()
        plt.savefig(self.plots_dir / "ttt_dashboard.png", dpi=300, bbox_inches="tight")
        plt.close()
        print("✅ Summary dashboard saved")

    def create_all_visualizations(self) -> None:
        print("📊 Creating TTT visualizations...")

        metrics = self.load_metrics()
        if not metrics:
            print("⚠️  No metrics found. Run the TTT pipeline first.")
            return

        try:
            self.create_performance_comparison(metrics)
            self.create_detailed_results_plot(metrics)
            self.create_summary_dashboard(metrics)

            print(f"🎉 All visualizations saved to {self.plots_dir}")

        except Exception as e:
            print(f"❌ Error creating visualizations: {e}")

def main():
    visualizer = TTTVisualizer()
    visualizer.create_all_visualizations()

if __name__ == "__main__":
    main()

Xet Storage Details

Size:: 8.79 kB
Xet hash:: 95dd6981e777f1f94f50829a8b0e4af83ebb829354741062cbda46f1dbd33041

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.

	import matplotlib.pyplot as plt
	import seaborn as sns
	import numpy as np
	import json
	from pathlib import Path
	from typing import Dict, Any, List, Optional
	import pandas as pd


	plt.style.use("seaborn-v0_8")
	sns.set_palette("husl")

	class TTTVisualizer:
	def __init__(self, results_dir: str = "results"):
	self.results_dir = Path(results_dir)
	self.plots_dir = self.results_dir / "plots"
	self.plots_dir.mkdir(parents=True, exist_ok=True)

	def load_metrics(self) -> Optional[Dict[str, Any]]:
	metrics_file = self.results_dir / "logs" / "metrics.json"
	if metrics_file.exists():
	with open(metrics_file, "r") as f:
	return json.load(f)
	return None

	def create_performance_comparison(self, metrics: Dict[str, Any]) -> None:
	fig, ax = plt.subplots(1, 1, figsize=(10, 6))

	baseline_acc = metrics.get("baseline_accuracy", 0)
	ttt_acc = metrics.get("ttt_accuracy", 0)

	models = ["Baseline Model", "TTT Adapted Model"]
	accuracies = [baseline_acc, ttt_acc]
	colors = ["lightcoral", "lightgreen"]

	bars = ax.bar(
	models, accuracies, color=colors, alpha=0.8, edgecolor="black", linewidth=1
	)
	ax.set_title(
	"Test-Time Training Performance Comparison",
	fontsize=16,
	fontweight="bold",
	pad=20,
	)
	ax.set_ylabel("Accuracy", fontsize=12)
	ax.set_ylim(0, 1)
	ax.grid(True, alpha=0.3, axis="y")


	for bar, acc in zip(bars, accuracies):
	height = bar.get_height()
	ax.text(
	bar.get_x() + bar.get_width() / 2.0,
	height + 0.01,
	f"{acc:.3f}",
	ha="center",
	va="bottom",
	fontweight="bold",
	fontsize=12,
	)


	improvement = ttt_acc - baseline_acc
	if improvement > 0:
	ax.annotate(
	(
	f"Improvement: +{improvement:.3f}\n({(improvement/baseline_acc)*100:.1f}% relative)"
	if baseline_acc > 0
	else "Achieved success from failure"
	),
	xy=(1, ttt_acc),
	xytext=(0.5, 0.8),
	arrowprops=dict(arrowstyle="->", color="red", lw=2),
	fontsize=10,
	ha="center",
	va="center",
	bbox=dict(boxstyle="round,pad=0.3", facecolor="yellow", alpha=0.7),
	)

	plt.tight_layout()
	plt.savefig(
	self.plots_dir / "performance_comparison.png", dpi=300, bbox_inches="tight"
	)
	plt.close()
	print("✅ Performance comparison plot saved")

	def create_detailed_results_plot(self, metrics: Dict[str, Any]) -> None:
	if "results_detail" not in metrics:
	return

	results_detail = metrics["results_detail"]
	if not results_detail:
	return


	inputs = [r["input"] for r in results_detail]
	baseline_preds = [r.get("baseline_prediction", "") for r in results_detail]
	adapted_preds = [r.get("adapted_prediction", "") for r in results_detail]
	expected = [r.get("expected", "") for r in results_detail]


	fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(12, 10))


	x = np.arange(len(inputs))
	width = 0.25

	baseline_correct = [
	1 if pred == exp else 0 for pred, exp in zip(baseline_preds, expected)
	]
	adapted_correct = [
	1 if pred == exp else 0 for pred, exp in zip(adapted_preds, expected)
	]

	ax1.bar(
	x - width,
	baseline_correct,
	width,
	label="Baseline",
	alpha=0.7,
	color="lightcoral",
	)
	ax1.bar(
	x + width,
	adapted_correct,
	width,
	label="TTT Adapted",
	alpha=0.7,
	color="lightgreen",
	)

	ax1.set_xlabel("Test Cases", fontsize=12)
	ax1.set_ylabel("Correct (1) / Incorrect (0)", fontsize=12)
	ax1.set_title(
	"Individual Test Case Performance", fontsize=14, fontweight="bold"
	)
	ax1.set_xticks(x)
	ax1.set_xticklabels(inputs, rotation=45, ha="right")
	ax1.legend()
	ax1.grid(True, alpha=0.3)


	baseline_acc = sum(baseline_correct) / len(baseline_correct)
	adapted_acc = sum(adapted_correct) / len(adapted_correct)

	models = ["Baseline", "TTT Adapted"]
	accuracies = [baseline_acc, adapted_acc]
	colors = ["lightcoral", "lightgreen"]

	bars = ax2.bar(models, accuracies, color=colors, alpha=0.8, edgecolor="black")
	ax2.set_ylabel("Accuracy", fontsize=12)
	ax2.set_title("Overall Accuracy Comparison", fontsize=14, fontweight="bold")
	ax2.set_ylim(0, 1)
	ax2.grid(True, alpha=0.3, axis="y")

	for bar, acc in zip(bars, accuracies):
	height = bar.get_height()
	ax2.text(
	bar.get_x() + bar.get_width() / 2.0,
	height + 0.01,
	f"{acc:.3f}",
	ha="center",
	va="bottom",
	fontweight="bold",
	)

	plt.tight_layout()
	plt.savefig(
	self.plots_dir / "detailed_results.png", dpi=300, bbox_inches="tight"
	)
	plt.close()
	print("✅ Detailed results plot saved")

	def create_summary_dashboard(self, metrics: Dict[str, Any]) -> None:
	fig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(2, 2, figsize=(15, 12))


	baseline_acc = metrics.get("baseline_accuracy", 0)
	ttt_acc = metrics.get("ttt_accuracy", 0)

	models = ["Baseline", "TTT Adapted"]
	accuracies = [baseline_acc, ttt_acc]
	colors = ["lightcoral", "lightgreen"]

	bars = ax1.bar(models, accuracies, color=colors, alpha=0.8)
	ax1.set_title("Model Performance Comparison", fontsize=14, fontweight="bold")
	ax1.set_ylabel("Accuracy")
	ax1.set_ylim(0, 1)
	ax1.grid(True, alpha=0.3)

	for bar, acc in zip(bars, accuracies):
	height = bar.get_height()
	ax1.text(
	bar.get_x() + bar.get_width() / 2.0,
	height + 0.02,
	f"{acc:.3f}",
	ha="center",
	va="bottom",
	fontweight="bold",
	)


	improvement = ttt_acc - baseline_acc
	ax2.barh(["Absolute Improvement"], [improvement], color="gold", alpha=0.7)
	ax2.set_title("TTT Improvement", fontsize=14, fontweight="bold")
	ax2.set_xlabel("Accuracy Gain")
	ax2.axvline(x=0, color="black", linestyle="--", alpha=0.5)

	ax2.text(
	improvement / 2,
	0,
	f"{improvement:.3f}",
	ha="center",
	va="center",
	fontweight="bold",
	fontsize=16,
	)


	ax3.axis("off")
	config_text = f
	ax3.text(
	0.1,
	0.5,
	config_text,
	fontsize=12,
	verticalalignment="center",
	bbox=dict(boxstyle="round,pad=0.5", facecolor="lightblue", alpha=0.7),
	)
	ax3.set_title("Configuration", fontsize=14, fontweight="bold")


	ax4.axis("off")
	results_text = f
	ax4.text(
	0.1,
	0.5,
	results_text,
	fontsize=12,
	verticalalignment="center",
	bbox=dict(boxstyle="round,pad=0.5", facecolor="lightgreen", alpha=0.7),
	)
	ax4.set_title("Results", fontsize=14, fontweight="bold")

	plt.suptitle(
	"Test-Time Training (TTT) Complete Analysis Dashboard",
	fontsize=16,
	fontweight="bold",
	y=0.98,
	)
	plt.tight_layout()
	plt.savefig(self.plots_dir / "ttt_dashboard.png", dpi=300, bbox_inches="tight")
	plt.close()
	print("✅ Summary dashboard saved")

	def create_all_visualizations(self) -> None:
	print("📊 Creating TTT visualizations...")

	metrics = self.load_metrics()
	if not metrics:
	print("⚠️ No metrics found. Run the TTT pipeline first.")
	return

	try:
	self.create_performance_comparison(metrics)
	self.create_detailed_results_plot(metrics)
	self.create_summary_dashboard(metrics)

	print(f"🎉 All visualizations saved to {self.plots_dir}")

	except Exception as e:
	print(f"❌ Error creating visualizations: {e}")

	def main():
	visualizer = TTTVisualizer()
	visualizer.create_all_visualizations()

	if __name__ == "__main__":
	main()