File size: 3,584 Bytes
c8ba9c6 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 | """Optional CUDA graph replay evidence for disposable benchmarks.
Uses private PyTorch 2.14 APIs, imported only when enabled. Counts describe host
capture/replay calls, not GPU durations or kernel coverage. No CUDA work or
synchronization is added. Use only one context at a time in a benchmark process.
"""
from __future__ import annotations
from collections import Counter
from contextlib import contextmanager
from dataclasses import dataclass
from importlib import import_module
from typing import Callable, Iterator
@dataclass
class CudagraphDiagnostics:
_replays: Counter[int]
_read_counters: Callable[[], dict[str, int]]
_initial_counters: dict[str, int]
def snapshot(self) -> dict:
"""Return cumulative counts since context entry; subtract for an interval."""
current = self._read_counters()
return {
**{key: value - self._initial_counters[key] for key, value in current.items()},
"replay_calls": sum(self._replays.values()),
"replay_calls_by_graph_id": {
str(key): value for key, value in sorted(self._replays.items())
},
}
@contextmanager
def collect_cudagraph_diagnostics() -> Iterator[CudagraphDiagnostics]:
"""Count successful, non-kernel-free tree replays and restore the wrapper.
``capture_calls`` counts entries in PyTorch's timed recording metrics.
``cudagraph_skips`` and ``recorded_non_static_inputs`` are compiler counter
deltas. These alone do not establish replay. ``replay_calls`` increments
after a real ``CUDAGraphNode.run_graph`` returns successfully; its initial
constructor replay is outside this method and is deliberately excluded.
Existing benchmark synchronization establishes completion of queued work.
"""
try:
trees = import_module("torch._inductor.cudagraph_trees")
utils = import_module("torch._dynamo.utils")
node_type = trees.CUDAGraphNode
original = node_type.run_graph
if not callable(original):
raise AttributeError("CUDAGraphNode.run_graph is not callable")
# Validate availability before changing any process-wide method.
utils.compilation_time_metrics.get("CUDAGraphNode.record", [])
utils.counters.get("inductor", {})
except (ImportError, AttributeError) as error:
raise RuntimeError(
"CUDA graph diagnostics require the private PyTorch 2.14 "
"CUDAGraphNode and compiler counter APIs"
) from error
def read_counters() -> dict[str, int]:
inductor = utils.counters.get("inductor", {})
return {
"capture_calls": len(utils.compilation_time_metrics.get("CUDAGraphNode.record", [])),
"cudagraph_skips": int(inductor.get("cudagraph_skips", 0)),
"recorded_non_static_inputs": int(inductor.get("cudagraph_recorded_non_static_inputs", 0)),
}
replays: Counter[int] = Counter()
diagnostics = CudagraphDiagnostics(replays, read_counters, read_counters())
def counted_run_graph(node):
try:
kernel_free = node.wrapped_function.kernel_free_cudagraph
graph_id = node.id.id
except AttributeError as error:
raise RuntimeError("CUDA graph diagnostics encountered an unsupported node API") from error
result = original(node)
if not kernel_free:
replays[graph_id] += 1
return result
node_type.run_graph = counted_run_graph
try:
yield diagnostics
finally:
node_type.run_graph = original
|