File size: 3,094 Bytes
60b21d3 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 | # SPDX-FileCopyrightText: 2025 Stanford University, ETH Zurich, and the project authors (see CONTRIBUTORS.md)
# SPDX-FileCopyrightText: 2025 This source file is part of the OpenTSLM open-source project.
#
# SPDX-License-Identifier: MIT
from typing import Dict, Any, Callable
from common_evaluator import CommonEvaluator
from opentslm.time_series_datasets.TSQADataset import TSQADataset
from opentslm.time_series_datasets.pamap2.PAMAP2AccQADataset import PAMAP2AccQADataset
from opentslm.time_series_datasets.pamap2.PAMAP2CoTQADataset import PAMAP2CoTQADataset
# Import evaluation functions
from evaluate_tsqa import evaluate_tsqa
from evaluate_pamap import evaluate_pamap_acc, evaluate_pamap_cot
def main():
"""Main function to run comprehensive evaluation across all datasets."""
# Define models to evaluate
model_names = [
"meta-llama/Llama-3.2-1B",
# Add more models as needed
# "google/gemma-3n-e2b",
# "google/gemma-3n-e2b-it",
# "microsoft/DialoGPT-medium",
# "gpt2",
]
# Define datasets to evaluate on
dataset_classes = [
TSQADataset,
PAMAP2AccQADataset,
PAMAP2CoTQADataset,
]
# Define evaluation functions
evaluation_functions = {
"TSQADataset": evaluate_tsqa,
"PAMAP2AccQADataset": evaluate_pamap_acc,
"PAMAP2CoTQADataset": evaluate_pamap_cot,
}
# Initialize evaluator
evaluator = CommonEvaluator()
# Run comprehensive evaluation
results_df = evaluator.evaluate_multiple_models(
model_names=model_names,
dataset_classes=dataset_classes,
evaluation_functions=evaluation_functions,
max_samples=50, # Limit for faster testing, set to None for full evaluation
)
print("\n" + "="*80)
print("COMPREHENSIVE EVALUATION RESULTS")
print("="*80)
print(results_df.to_string(index=False))
# Print summary statistics
print("\n" + "="*80)
print("SUMMARY STATISTICS")
print("="*80)
# Group by dataset and show average metrics
for dataset in results_df['dataset'].unique():
dataset_results = results_df[results_df['dataset'] == dataset]
print(f"\n{dataset}:")
print(f" Number of models evaluated: {len(dataset_results)}")
# Show average accuracy
if 'accuracy' in dataset_results.columns:
avg_accuracy = dataset_results['accuracy'].mean()
print(f" Average accuracy: {avg_accuracy:.1f}%")
# Group by model and show average metrics
print(f"\nBy Model:")
for model in results_df['model'].unique():
model_results = results_df[results_df['model'] == model]
print(f"\n{model}:")
print(f" Number of datasets evaluated: {len(model_results)}")
# Show average accuracy
if 'accuracy' in model_results.columns:
avg_accuracy = model_results['accuracy'].mean()
print(f" Average accuracy: {avg_accuracy:.1f}%")
return results_df
if __name__ == "__main__":
main() |