kylinfires commited on
Commit
9f083fb
·
verified ·
1 Parent(s): 3c15e2b

Sync main 91da25b: text K/V host cache (repeat sessions 1.4 s) + audit fixes

Browse files
README.md CHANGED
@@ -1,19 +1,3 @@
1
- ---
2
- license: apache-2.0
3
- language:
4
- - en
5
- - zh
6
- library_name: diffusers
7
- pipeline_tag: video-to-video
8
- base_model: jdopensource/JoyAI-Video-Edit
9
- tags:
10
- - video-editing
11
- - streaming
12
- - quantization
13
- - int8
14
- - text-stream-stripping
15
- ---
16
-
17
  # Joy-face_change_optimized
18
 
19
  [JoyAI-Video-Edit](https://github.com/jd-opensource/JoyAI-Video-Edit) 的优化分支(Apache-2.0)。
@@ -73,11 +57,12 @@ PyTorch 回退与 CUDA 版**逐张量验证一致**:99.997% 元素完全相同
73
  |---|---:|---:|---:|
74
  | 基线 | 25.5 GiB(26124 MiB) | 1.8 s | 0.314 s |
75
  | + 文本 K/V + 权重卸载 | 19.5 GiB(19926 MiB) | 6.4 s | 0.280 s |
76
- | + tight(关 graph) | **15.3 GiB**(15668 MiB) | 10.7 s | 0.314 s |
77
  | + tight + `JOYOMNI_KV_DTYPE=int8` | **14.7 GiB** | 10.7 s | 0.316 s |
78
 
79
  > 注意:文本 K/V 缓存**本身不省显存**——启动时 24.8 GiB vs 基线 23.6 GiB(缓存 0.46 + graph 池 0.46),
80
  > 必须和 `JOYOMNI_TEXT_OFFLOAD` 一起用。表中数字为"跑完一个会话"后的 nvidia-smi,括号内为原始 MiB。
 
81
 
82
  - **16 GB 卡可跑**:`JOYOMNI_VRAM_CAP_GB=16` 的硬上限下 17 帧会话跑通(reserved 14.5 GiB,峰值 13.7 GiB)。
83
  - dit 每 chunk 0.219 → 0.185 s(graph 稳态,文本分支不再参与)。
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  # Joy-face_change_optimized
2
 
3
  [JoyAI-Video-Edit](https://github.com/jd-opensource/JoyAI-Video-Edit) 的优化分支(Apache-2.0)。
 
57
  |---|---:|---:|---:|
58
  | 基线 | 25.5 GiB(26124 MiB) | 1.8 s | 0.314 s |
59
  | + 文本 K/V + 权重卸载 | 19.5 GiB(19926 MiB) | 6.4 s | 0.280 s |
60
+ | + tight(关 graph) | **15.3 GiB**(15668 MiB) | 10.7 s(首次 prompt)/ **1.4 s**(缓存命中) | 0.314 s |
61
  | + tight + `JOYOMNI_KV_DTYPE=int8` | **14.7 GiB** | 10.7 s | 0.316 s |
62
 
63
  > 注意:文本 K/V 缓存**本身不省显存**——启动时 24.8 GiB vs 基线 23.6 GiB(缓存 0.46 + graph 池 0.46),
64
  > 必须和 `JOYOMNI_TEXT_OFFLOAD` 一起用。表中数字为"跑完一个会话"后的 nvidia-smi,括号内为原始 MiB。
65
+ > 文本 K/V 抓取的结果按(prompt embeds + 参考图 latent + steps + seed)做主机侧缓存(LRU 2),同一组输入的新会话直接拷回显存:首帧 **11.1 → 1.4 s**,逐帧与首次抓取 **138.13 dB** 完全一致(不同 prompt 各自 8–11 s 首次抓取,LRU 满时淘汰最旧一条)。
66
 
67
  - **16 GB 卡可跑**:`JOYOMNI_VRAM_CAP_GB=16` 的硬上限下 17 帧会话跑通(reserved 14.5 GiB,峰值 13.7 GiB)。
68
  - dit 每 chunk 0.219 → 0.185 s(graph 稳态,文本分支不再参与)。
deploy/xvideo/serving/joyomni_streaming.py CHANGED
@@ -215,6 +215,11 @@ class JoyOmniRuntime:
215
  self.ref_kv_cache: dict[str, dict] = {}
216
  self.ref_kv_order: list[str] = []
217
  self.ref_kv_cache_max = 2
 
 
 
 
 
218
  self.output_quality = 60
219
  self.lossless_output = False
220
  self.graph_runners: dict[tuple, StreamingGraphRunner] = {}
@@ -693,7 +698,7 @@ class JoyOmniV2VStreamingSession:
693
  self._prefill_or_reuse_reference_kv()
694
 
695
  if text_kv_env_enabled():
696
- self._text_kv_by_step = self._capture_text_kv_by_step()
697
 
698
  self._maybe_prepare_graph_runner()
699
  self._maybe_offload_text_stream()
@@ -761,6 +766,46 @@ class JoyOmniV2VStreamingSession:
761
  flush=True,
762
  )
763
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
764
  @torch.no_grad()
765
  def _capture_text_kv_by_step(self) -> list[dict[int, tuple[torch.Tensor, torch.Tensor]]]:
766
  """Prefill the text stream once per denoise step.
 
215
  self.ref_kv_cache: dict[str, dict] = {}
216
  self.ref_kv_order: list[str] = []
217
  self.ref_kv_cache_max = 2
218
+ # Host-side text K/V cache, keyed by everything the capture depends on
219
+ # (prompt embeds, reference latent, step count, seed).
220
+ self.text_kv_cache: dict[str, list] = {}
221
+ self.text_kv_order: list[str] = []
222
+ self.text_kv_cache_max = 2
223
  self.output_quality = 60
224
  self.lossless_output = False
225
  self.graph_runners: dict[tuple, StreamingGraphRunner] = {}
 
698
  self._prefill_or_reuse_reference_kv()
699
 
700
  if text_kv_env_enabled():
701
+ self._text_kv_by_step = self._text_kv_from_cache_or_capture()
702
 
703
  self._maybe_prepare_graph_runner()
704
  self._maybe_offload_text_stream()
 
766
  flush=True,
767
  )
768
 
769
+ def _text_kv_key(self) -> str:
770
+ """Hash the exact inputs of the text K/V capture."""
771
+ digest = hashlib.sha1()
772
+ embeds = self.streaming_cond_embeds.detach().to("cpu", torch.float32).contiguous()
773
+ digest.update(embeds.numpy().tobytes())
774
+ if self.ref_image_latent is None:
775
+ digest.update(b"no-reference")
776
+ else:
777
+ reference = self.ref_image_latent.detach().to("cpu", torch.float32).contiguous()
778
+ digest.update(reference.numpy().tobytes())
779
+ digest.update(f"{int(self.settings.num_inference_steps)}|{int(self.settings.seed)}".encode())
780
+ return digest.hexdigest()[:16]
781
+
782
+ def _text_kv_from_cache_or_capture(self) -> list[dict[int, tuple[torch.Tensor, torch.Tensor]]]:
783
+ key = self._text_kv_key()
784
+ cached = self.runtime.text_kv_cache.get(key)
785
+ if cached is not None:
786
+ caches = [
787
+ {layer: tuple(t.to(self.device) for t in pair) for layer, pair in step.items()}
788
+ for step in cached
789
+ ]
790
+ print(f"#####[TEXT-KV] reused host cache ({key})", flush=True)
791
+ return caches
792
+
793
+ started = time.perf_counter()
794
+ caches = self._capture_text_kv_by_step()
795
+ self.runtime.text_kv_cache[key] = [
796
+ {layer: tuple(t.to("cpu") for t in pair) for layer, pair in step.items()}
797
+ for step in caches
798
+ ]
799
+ self.runtime.text_kv_order.append(key)
800
+ while len(self.runtime.text_kv_order) > self.runtime.text_kv_cache_max:
801
+ self.runtime.text_kv_cache.pop(self.runtime.text_kv_order.pop(0), None)
802
+ print(
803
+ f"#####[TEXT-KV] cached on host ({key}, "
804
+ f"{len(self.runtime.text_kv_order)} entr(ies), {time.perf_counter() - started:.2f}s)",
805
+ flush=True,
806
+ )
807
+ return caches
808
+
809
  @torch.no_grad()
810
  def _capture_text_kv_by_step(self) -> list[dict[int, tuple[torch.Tensor, torch.Tensor]]]:
811
  """Prefill the text stream once per denoise step.