Buckets:
| from typing import Dict, Any, List, Optional | |
| from dataclasses import dataclass | |
| from utils.settings import ModelArgs, DataArgs, EvaluatorArgs | |
| from dataset.tta_dataset import construct_demonstrations, AbstractTaskRule | |
| class BaseEvaluationSuite: | |
| demos: Dict[str, Any] | |
| test_inputs: List[str] | |
| model_args: ModelArgs | |
| data_args: DataArgs | |
| evaluator_args: EvaluatorArgs | |
| def run_all(self, evaluator, logger) -> Dict[str, Any]: | |
| logger.log({"stage": "baseline_evaluation"}) | |
| demos_data = construct_demonstrations( | |
| task_rule=AbstractTaskRule.ReverseAppendLastChar, | |
| extra_inputs=self.data_args.demonstrations, | |
| n_demo_in_loo=3, | |
| ) | |
| results = evaluator.run_all( | |
| demos=demos_data, | |
| test_inputs=self.test_inputs, | |
| model_args=self.model_args, | |
| data_args=self.data_args, | |
| evaluator_args=self.evaluator_args, | |
| ) | |
| logger.log(results) | |
| return EvalResults(results) | |
| class EvalResults: | |
| def __init__(self, data: Dict[str, Any]): | |
| self.data = data | |
| def render_str(self) -> str: | |
| acc_bas = self.data.get("baseline_accuracy", 0.0) | |
| acc_ttt = self.data.get("ttt_accuracy", 0.0) | |
| improvement = acc_ttt - acc_bas | |
| return ( | |
| f"baseline={acc_bas:.3f}, ttt={acc_ttt:.3f}, improvement={improvement:.3f}" | |
| ) | |
Xet Storage Details
- Size:
- 1.42 kB
- Xet hash:
- eca0be4e19a592c673252568f175fa308aca0082c9e67cf8936d5a600cfb74ae
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.