Buckets:
| import os | |
| import sys | |
| if "/Users/tahamajs/Documents/uni/Sysmem2_in_AI/CAs/CA15/src" not in sys.path: | |
| sys.path.insert(0, "/Users/tahamajs/Documents/uni/Sysmem2_in_AI/CAs/CA15/src") | |
| import torch | |
| import numpy as np | |
| from typing import Dict, List, Tuple, Optional, Union | |
| from model.base import BaseModelManager | |
| from model.adapter import LoRATTTAdapter | |
| from dataset.tta_dataset import TTADataset, construct_demonstrations | |
| from evaluation.evaluator import LOOEvaluator, MetricsComputer, WeightsBiasesLogger | |
| from utils.settings import ( | |
| DataArgs, | |
| ModelArgs, | |
| EvaluatorArgs, | |
| ) | |
| from utils.io import ensure_dir, ensure_file, SafeDictAccessReadOnly as SafeReadOnly | |
| from utils.logging import ExperimentLogger | |
| from utils.suite import BaseEvaluationSuite | |
| from dataset.tta_dataset import AbstractTaskRule | |
| logger = ExperimentLogger.setup("logs", "run_example", verbosity=1) | |
| def main(arg_dict=None): | |
| arg_dict = arg_dict or {} | |
| cfg_file = arg_dict.get( | |
| "cfg", "/Users/tahamajs/Documents/uni/Sysmem2_in_AI/CAs/CA15/config.json" | |
| ) | |
| run_id = arg_dict.get("run_id", "run-000") | |
| seed = int(arg_dict.get("seed", "42")) | |
| device = arg_dict.get("device", None) or ( | |
| "cuda" if torch.cuda.is_available() else "cpu" | |
| ) | |
| torch.manual_seed(seed) | |
| np.random.seed(seed) | |
| ensure_dir("results") | |
| ensure_dir("results/model") | |
| ensure_dir("results/logs") | |
| ensure_dir("results/data") | |
| cfg_access = SafeReadOnly(cfg_file) | |
| data_name = cfg_access.string_with_default("dataset.name", "demo_reverse_lastchar") | |
| print("Building Demonstrations") | |
| demos_dict = construct_demonstrations( | |
| task_rule=AbstractTaskRule.ReverseAppendLastChar, | |
| n_demo_in_loo=3, | |
| extra_inputs=["hello", "world", "apple", "python"], | |
| ) | |
| print("Saving demos") | |
| TTADataset.write_dict( | |
| "/Users/tahamajs/Documents/uni/Sysmem2_in_AI/CAs/CA15/results/data/demos.json", | |
| demos_dict, | |
| ) | |
| d_args = DataArgs() | |
| d_args._apply_dict(cfg_access["dataset"]) if cfg_access.get("dataset") else None | |
| m_args = ModelArgs() | |
| m_args._apply_dict(cfg_access["model"]) if cfg_access.get("model") else None | |
| m_args.device = device | |
| ev = LOOEvaluator( | |
| BaseModelManager(clamp_device_to_cpu=True), | |
| LoRATTTAdapter(m_args), | |
| MetricsComputer(), | |
| ( | |
| WeightsBiasesLogger( | |
| project=arg_dict.get("wandb_project", ""), | |
| entity=arg_dict.get("wandb_entity", ""), | |
| run_id=run_id, | |
| ) | |
| if arg_dict.get("wandb_project") | |
| else None | |
| ), | |
| ) | |
| suite = BaseEvaluationSuite( | |
| demos=demos_dict, | |
| test_inputs=["gemini"] | |
| + ( | |
| arg_dict.get("test_inputs", "").split(",") | |
| if arg_dict.get("test_inputs") | |
| else [] | |
| ), | |
| model_args=m_args, | |
| data_args=d_args, | |
| evaluator_args=EvaluatorArgs(), | |
| ) | |
| results = suite.run_all(ev, logger) | |
| print(results.render_str()) | |
| logger.dump( | |
| "/Users/tahamajs/Documents/uni/Sysmem2_in_AI/CAs/CA15/results/logs/metrics.json" | |
| ) | |
| return results | |
| if __name__ == "__main__": | |
| import json | |
| cfg_from_json = {} | |
| if len(sys.argv) > 1: | |
| cfg_from_json["cfg"] = sys.argv[1] | |
| if len(sys.argv) > 2: | |
| cfg_from_json.update(json.loads(sys.argv[2])) | |
| main(cfg_from_json) | |
Xet Storage Details
- Size:
- 3.39 kB
- Xet hash:
- a1831853dad7a0f3d460d0b19d41457e245723fc30d0b4d9564b40c5f04b766e
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.