Buckets:
| """Complete nested numeric metrics and pre-clipping branch diagnostics.""" | |
| from collections.abc import Mapping | |
| import math | |
| def log_scalars(writer, namespace, metrics, step): | |
| if writer is None: | |
| return | |
| def visit(prefix, value): | |
| if isinstance(value, Mapping): | |
| for key, child in value.items(): | |
| # Individual verdicts belong to the saved sample artifacts. | |
| if key != "verdicts": | |
| visit(f"{prefix}/{key}", child) | |
| elif isinstance(value, (int, float, bool)): | |
| # Keep undefined subgroup values as NaN, with their count metrics, | |
| # rather than silently omitting a tag from TensorBoard. | |
| writer.add_scalar(prefix, value, step) | |
| visit(namespace, metrics) | |
| writer.flush() | |
| def branch_gradient_metrics(parameters, opd_gradients, *, synchronize_gradients=None): | |
| """Compare globally summed g_OPD and weighted g_replay before clipping.""" | |
| import torch | |
| total = [parameter.grad for parameter in parameters] | |
| try: | |
| for parameter, gradient in zip(parameters, opd_gradients, strict=True): | |
| parameter.grad = gradient | |
| if synchronize_gradients is not None: | |
| synchronize_gradients(parameters) | |
| # Sum vectors before taking norms; averaging per-rank norms is wrong. | |
| values = parameters[0].new_zeros(3, dtype=torch.float64) | |
| for parameter, merged in zip(parameters, total, strict=True): | |
| if merged is None and parameter.grad is None: | |
| continue | |
| opd = parameter.grad.float() if parameter.grad is not None else torch.zeros_like(merged, dtype=torch.float32) | |
| replay = (merged.float() if merged is not None else torch.zeros_like(opd)) - opd | |
| values[0] += opd.square().sum(dtype=torch.float64) | |
| values[1] += replay.square().sum(dtype=torch.float64) | |
| values[2] += (opd * replay).sum(dtype=torch.float64) | |
| oo, rr, dot = values.cpu().tolist() | |
| finally: | |
| for parameter, gradient in zip(parameters, total, strict=True): | |
| parameter.grad = gradient | |
| on, rn = math.sqrt(oo), math.sqrt(rr) | |
| return { | |
| "grad/opd_norm": on, | |
| "grad/replay_weighted_norm": rn, | |
| "grad/replay_to_opd_ratio": rn / on if on else float("nan"), | |
| "grad/opd_replay_cosine": max(-1., min(1., dot / (on * rn))) if on and rn else float("nan"), | |
| } | |
Xet Storage Details
- Size:
- 2.43 kB
- Xet hash:
- 43cd6850e1c6261f0e31f7c480d0639d6385e294dd8120af9aec6a51e0ddf6a3
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.