fix: sync DiT Linear precision diagnostics from PR #28
Browse filesSync the final precision-reporting changes from upstream ca17e1d. Report
installed FP8 Linear counts and configured base precision, preserve lazy
conversion states, and use one post-denoise reporting hook for eager and
CUDA graph execution. Keep the existing ZeroGPU entry point and integration.
Validation outside the repository:
- 10 CPU/runtime checks passed.
- All four image/text FP8 flag combinations passed small-model CUDA
forwards and CUDA graph capture/replay; logging preserved eager output.
- Changed files match upstream byte-for-byte; syntax and diff checks pass.
No test files are included in this commit.
- xvideo/fp8_status.py +66 -0
- xvideo/models/dit/dit.py +12 -0
- xvideo/serving/joyomni_streaming.py +22 -0
xvideo/fp8_status.py
ADDED
|
@@ -0,0 +1,66 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Dependency-free helpers for reporting DiT Linear quantization state."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
from collections.abc import Mapping
|
| 6 |
+
from typing import Any
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
_STREAMS = ("img", "txt")
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
def fp8_precision_status(
|
| 13 |
+
transformer: Any,
|
| 14 |
+
requested_by_stream: Mapping[str, bool],
|
| 15 |
+
*,
|
| 16 |
+
base_precision: str,
|
| 17 |
+
) -> dict[str, dict[str, Any]]:
|
| 18 |
+
"""Return requested and installed FP8 Linear state for each DiT stream.
|
| 19 |
+
|
| 20 |
+
FP8 weights are installed lazily during block forwards, so a requested
|
| 21 |
+
stream with no converted blocks is pending. Unconverted streams use the
|
| 22 |
+
configured base precision; this does not describe every DiT operator.
|
| 23 |
+
"""
|
| 24 |
+
blocks = tuple(getattr(transformer, "double_blocks", ()))
|
| 25 |
+
total_blocks = len(blocks)
|
| 26 |
+
result: dict[str, dict[str, Any]] = {}
|
| 27 |
+
|
| 28 |
+
for stream in _STREAMS:
|
| 29 |
+
requested = bool(requested_by_stream.get(stream, False))
|
| 30 |
+
installed_blocks = sum(
|
| 31 |
+
bool(getattr(block, f"_fp8_{stream}_installed", False))
|
| 32 |
+
for block in blocks
|
| 33 |
+
)
|
| 34 |
+
if total_blocks == 0:
|
| 35 |
+
effective = "unknown"
|
| 36 |
+
elif installed_blocks == total_blocks:
|
| 37 |
+
effective = "fp8"
|
| 38 |
+
elif installed_blocks:
|
| 39 |
+
effective = "mixed"
|
| 40 |
+
elif requested:
|
| 41 |
+
effective = "pending"
|
| 42 |
+
else:
|
| 43 |
+
effective = base_precision
|
| 44 |
+
result[stream] = {
|
| 45 |
+
"requested": requested,
|
| 46 |
+
"effective": effective,
|
| 47 |
+
"installed_blocks": installed_blocks,
|
| 48 |
+
"total_blocks": total_blocks,
|
| 49 |
+
}
|
| 50 |
+
|
| 51 |
+
return result
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def format_fp8_precision_status(status: Mapping[str, Mapping[str, Any]]) -> str:
|
| 55 |
+
"""Format per-stream status for a concise runtime log line."""
|
| 56 |
+
parts = []
|
| 57 |
+
for stream in _STREAMS:
|
| 58 |
+
item = status[stream]
|
| 59 |
+
effective = item["effective"]
|
| 60 |
+
requested = "yes" if item["requested"] else "no"
|
| 61 |
+
installed = item["installed_blocks"]
|
| 62 |
+
total = item["total_blocks"]
|
| 63 |
+
parts.append(
|
| 64 |
+
f"{stream}={effective} (requested={requested}; FP8 blocks={installed}/{total})"
|
| 65 |
+
)
|
| 66 |
+
return "DiT Linear precision: " + ", ".join(parts)
|
xvideo/models/dit/dit.py
CHANGED
|
@@ -9,6 +9,8 @@ import torch.nn as nn
|
|
| 9 |
import torch.nn.functional as F
|
| 10 |
from torch.nn.attention import SDPBackend, sdpa_kernel
|
| 11 |
|
|
|
|
|
|
|
| 12 |
|
| 13 |
from diffusers.models import ModelMixin
|
| 14 |
from diffusers.configuration_utils import ConfigMixin, register_to_config
|
|
@@ -103,6 +105,16 @@ def _fp8_stream_enabled(block, stream: str) -> bool:
|
|
| 103 |
return bool(getattr(block, f"_fp8_{stream}_installed", False))
|
| 104 |
|
| 105 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 106 |
|
| 107 |
from xvideo.models.dit import sgl_fused_ops as _sgl_fused
|
| 108 |
|
|
|
|
| 9 |
import torch.nn.functional as F
|
| 10 |
from torch.nn.attention import SDPBackend, sdpa_kernel
|
| 11 |
|
| 12 |
+
from xvideo.fp8_status import format_fp8_precision_status, fp8_precision_status
|
| 13 |
+
|
| 14 |
|
| 15 |
from diffusers.models import ModelMixin
|
| 16 |
from diffusers.configuration_utils import ConfigMixin, register_to_config
|
|
|
|
| 105 |
return bool(getattr(block, f"_fp8_{stream}_installed", False))
|
| 106 |
|
| 107 |
|
| 108 |
+
def fp8_precision_report(transformer, *, base_precision: str) -> tuple[dict, str]:
|
| 109 |
+
"""Return DiT Linear FP8 state and its runtime summary."""
|
| 110 |
+
status = fp8_precision_status(
|
| 111 |
+
transformer,
|
| 112 |
+
{stream: _fp8_stream_wanted(stream) for stream in ("img", "txt")},
|
| 113 |
+
base_precision=base_precision,
|
| 114 |
+
)
|
| 115 |
+
return status, format_fp8_precision_status(status)
|
| 116 |
+
|
| 117 |
+
|
| 118 |
|
| 119 |
from xvideo.models.dit import sgl_fused_ops as _sgl_fused
|
| 120 |
|
xvideo/serving/joyomni_streaming.py
CHANGED
|
@@ -206,6 +206,25 @@ class JoyOmniRuntime:
|
|
| 206 |
self.lossless_output = False
|
| 207 |
self.graph_runners: dict[tuple, StreamingGraphRunner] = {}
|
| 208 |
self.graph_capture_failures: dict[tuple, int] = {}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 209 |
|
| 210 |
@classmethod
|
| 211 |
def load(
|
|
@@ -358,6 +377,8 @@ class JoyOmniRuntime:
|
|
| 358 |
for (_wh, _ww) in _orientations:
|
| 359 |
runtime.warmup_full_pipeline(height=_wh, width=_ww)
|
| 360 |
|
|
|
|
|
|
|
| 361 |
if device_obj.type == "cuda":
|
| 362 |
_free_b, _total_b = torch.cuda.mem_get_info(device_obj)
|
| 363 |
print(
|
|
@@ -1471,6 +1492,7 @@ class JoyOmniV2VStreamingSession:
|
|
| 1471 |
chunk_idx=encoded.job.chunk_idx,
|
| 1472 |
frozen_anchor_id=encoded.job.frozen_anchor_id,
|
| 1473 |
)
|
|
|
|
| 1474 |
self._evict_after_store(
|
| 1475 |
encoded.job.chunk_idx,
|
| 1476 |
frozen_anchor_id=encoded.job.frozen_anchor_id,
|
|
|
|
| 206 |
self.lossless_output = False
|
| 207 |
self.graph_runners: dict[tuple, StreamingGraphRunner] = {}
|
| 208 |
self.graph_capture_failures: dict[tuple, int] = {}
|
| 209 |
+
self._last_dit_precision_summary: str | None = None
|
| 210 |
+
self._dit_precision_settled = False
|
| 211 |
+
|
| 212 |
+
def _log_dit_precision(self) -> None:
|
| 213 |
+
if self._dit_precision_settled:
|
| 214 |
+
return
|
| 215 |
+
|
| 216 |
+
from xvideo.models.dit.dit import fp8_precision_report
|
| 217 |
+
|
| 218 |
+
status, summary = fp8_precision_report(
|
| 219 |
+
self.pipeline.transformer, base_precision=self.cfg.dit_precision,
|
| 220 |
+
)
|
| 221 |
+
if summary != self._last_dit_precision_summary:
|
| 222 |
+
print(f"#####[STREAM] {summary}", flush=True)
|
| 223 |
+
self._last_dit_precision_summary = summary
|
| 224 |
+
self._dit_precision_settled = all(
|
| 225 |
+
stream_status["effective"] in {"fp8", self.cfg.dit_precision}
|
| 226 |
+
for stream_status in status.values()
|
| 227 |
+
)
|
| 228 |
|
| 229 |
@classmethod
|
| 230 |
def load(
|
|
|
|
| 377 |
for (_wh, _ww) in _orientations:
|
| 378 |
runtime.warmup_full_pipeline(height=_wh, width=_ww)
|
| 379 |
|
| 380 |
+
runtime._log_dit_precision()
|
| 381 |
+
|
| 382 |
if device_obj.type == "cuda":
|
| 383 |
_free_b, _total_b = torch.cuda.mem_get_info(device_obj)
|
| 384 |
print(
|
|
|
|
| 1492 |
chunk_idx=encoded.job.chunk_idx,
|
| 1493 |
frozen_anchor_id=encoded.job.frozen_anchor_id,
|
| 1494 |
)
|
| 1495 |
+
self.runtime._log_dit_precision()
|
| 1496 |
self._evict_after_store(
|
| 1497 |
encoded.job.chunk_idx,
|
| 1498 |
frozen_anchor_id=encoded.job.frozen_anchor_id,
|