fryabcfry's picture
Publish AMP-PD checkpoints, portable loaders and verified metrics (part 4)
0d48417 verified
Raw History Blame Contribute Delete
4.06 kB
"""Small shared utilities."""
from __future__ import annotations
import json
import os
import random
import subprocess
import time
from pathlib import Path
from typing import Any, Dict, Iterable, Optional
import numpy as np
import yaml
def set_seed(seed: int) -> None:
random.seed(seed)
os.environ["PYTHONHASHSEED"] = str(seed)
np.random.seed(seed)
try:
import torch
except Exception:
return
torch.manual_seed(seed)
if torch.cuda.is_available():
torch.cuda.manual_seed_all(seed)
def get_device(spec: str = "auto"):
import torch
if spec == "auto":
return torch.device("cuda" if torch.cuda.is_available() else "cpu")
device = torch.device(spec)
if device.type == "cuda" and not torch.cuda.is_available():
raise RuntimeError(
"CUDA requested but torch.cuda.is_available() is False. "
"Check the NVIDIA driver and install a CUDA PyTorch wheel "
"(see GPU_SETUP.md), or use --device cpu / --device auto."
)
return device
def ensure_dir(path: str | Path) -> Path:
p = Path(path)
p.mkdir(parents=True, exist_ok=True)
return p
def save_json(obj: Any, path: str | Path) -> None:
p = Path(path)
ensure_dir(p.parent)
with open(p, "w", encoding="utf-8") as f:
json.dump(obj, f, indent=2, ensure_ascii=False, default=str)
def load_json(path: str | Path) -> Any:
with open(path, "r", encoding="utf-8") as f:
return json.load(f)
def save_yaml(obj: Any, path: str | Path) -> None:
p = Path(path)
ensure_dir(p.parent)
with open(p, "w", encoding="utf-8") as f:
yaml.safe_dump(obj, f, sort_keys=False, allow_unicode=True)
def load_yaml(path: str | Path) -> Any:
with open(path, "r", encoding="utf-8") as f:
return yaml.safe_load(f)
def deep_update(base: Dict[str, Any], override: Dict[str, Any]) -> Dict[str, Any]:
out = dict(base)
for k, v in override.items():
if k in out and isinstance(out[k], dict) and isinstance(v, dict):
out[k] = deep_update(out[k], v)
else:
out[k] = v
return out
def get_nested(d: Dict[str, Any], key: str, default: Any = None) -> Any:
cur: Any = d
for part in key.split("."):
if not isinstance(cur, dict) or part not in cur:
return default
cur = cur[part]
return cur
def set_nested(d: Dict[str, Any], key: str, value: Any) -> None:
parts = key.split(".")
cur = d
for part in parts[:-1]:
cur = cur.setdefault(part, {})
cur[parts[-1]] = value
def parse_scalar(value: str) -> Any:
try:
return yaml.safe_load(value)
except Exception:
return value
def apply_overrides(config: Dict[str, Any], overrides: Iterable[str]) -> Dict[str, Any]:
out = deep_update({}, config)
for item in overrides:
if "=" not in item:
raise ValueError(f"Override must be key=value, got: {item}")
key, value = item.split("=", 1)
set_nested(out, key.strip(), parse_scalar(value.strip()))
return out
def now_stamp() -> str:
return time.strftime("%Y%m%d-%H%M%S")
def count_parameters(model) -> int:
return sum(p.numel() for p in model.parameters() if p.requires_grad)
def git_commit() -> Optional[str]:
try:
out = subprocess.check_output(
["git", "rev-parse", "HEAD"], stderr=subprocess.DEVNULL, text=True
).strip()
return out
except Exception:
return None
def format_seconds(seconds: float) -> str:
seconds = int(seconds)
h, rem = divmod(seconds, 3600)
m, s = divmod(rem, 60)
if h:
return f"{h}h {m}m {s}s"
if m:
return f"{m}m {s}s"
return f"{s}s"
def flatten_dict(d: Dict[str, Any], prefix: str = "") -> Dict[str, Any]:
out: Dict[str, Any] = {}
for k, v in d.items():
key = f"{prefix}.{k}" if prefix else k
if isinstance(v, dict):
out.update(flatten_dict(v, key))
else:
out[key] = v
return out