File size: 4,062 Bytes
0d48417 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 | """Small shared utilities."""
from __future__ import annotations
import json
import os
import random
import subprocess
import time
from pathlib import Path
from typing import Any, Dict, Iterable, Optional
import numpy as np
import yaml
def set_seed(seed: int) -> None:
random.seed(seed)
os.environ["PYTHONHASHSEED"] = str(seed)
np.random.seed(seed)
try:
import torch
except Exception:
return
torch.manual_seed(seed)
if torch.cuda.is_available():
torch.cuda.manual_seed_all(seed)
def get_device(spec: str = "auto"):
import torch
if spec == "auto":
return torch.device("cuda" if torch.cuda.is_available() else "cpu")
device = torch.device(spec)
if device.type == "cuda" and not torch.cuda.is_available():
raise RuntimeError(
"CUDA requested but torch.cuda.is_available() is False. "
"Check the NVIDIA driver and install a CUDA PyTorch wheel "
"(see GPU_SETUP.md), or use --device cpu / --device auto."
)
return device
def ensure_dir(path: str | Path) -> Path:
p = Path(path)
p.mkdir(parents=True, exist_ok=True)
return p
def save_json(obj: Any, path: str | Path) -> None:
p = Path(path)
ensure_dir(p.parent)
with open(p, "w", encoding="utf-8") as f:
json.dump(obj, f, indent=2, ensure_ascii=False, default=str)
def load_json(path: str | Path) -> Any:
with open(path, "r", encoding="utf-8") as f:
return json.load(f)
def save_yaml(obj: Any, path: str | Path) -> None:
p = Path(path)
ensure_dir(p.parent)
with open(p, "w", encoding="utf-8") as f:
yaml.safe_dump(obj, f, sort_keys=False, allow_unicode=True)
def load_yaml(path: str | Path) -> Any:
with open(path, "r", encoding="utf-8") as f:
return yaml.safe_load(f)
def deep_update(base: Dict[str, Any], override: Dict[str, Any]) -> Dict[str, Any]:
out = dict(base)
for k, v in override.items():
if k in out and isinstance(out[k], dict) and isinstance(v, dict):
out[k] = deep_update(out[k], v)
else:
out[k] = v
return out
def get_nested(d: Dict[str, Any], key: str, default: Any = None) -> Any:
cur: Any = d
for part in key.split("."):
if not isinstance(cur, dict) or part not in cur:
return default
cur = cur[part]
return cur
def set_nested(d: Dict[str, Any], key: str, value: Any) -> None:
parts = key.split(".")
cur = d
for part in parts[:-1]:
cur = cur.setdefault(part, {})
cur[parts[-1]] = value
def parse_scalar(value: str) -> Any:
try:
return yaml.safe_load(value)
except Exception:
return value
def apply_overrides(config: Dict[str, Any], overrides: Iterable[str]) -> Dict[str, Any]:
out = deep_update({}, config)
for item in overrides:
if "=" not in item:
raise ValueError(f"Override must be key=value, got: {item}")
key, value = item.split("=", 1)
set_nested(out, key.strip(), parse_scalar(value.strip()))
return out
def now_stamp() -> str:
return time.strftime("%Y%m%d-%H%M%S")
def count_parameters(model) -> int:
return sum(p.numel() for p in model.parameters() if p.requires_grad)
def git_commit() -> Optional[str]:
try:
out = subprocess.check_output(
["git", "rev-parse", "HEAD"], stderr=subprocess.DEVNULL, text=True
).strip()
return out
except Exception:
return None
def format_seconds(seconds: float) -> str:
seconds = int(seconds)
h, rem = divmod(seconds, 3600)
m, s = divmod(rem, 60)
if h:
return f"{h}h {m}m {s}s"
if m:
return f"{m}m {s}s"
return f"{s}s"
def flatten_dict(d: Dict[str, Any], prefix: str = "") -> Dict[str, Any]:
out: Dict[str, Any] = {}
for k, v in d.items():
key = f"{prefix}.{k}" if prefix else k
if isinstance(v, dict):
out.update(flatten_dict(v, key))
else:
out[key] = v
return out
|